dev.c (9594408763d439287742d5582842db6a476bbd71) dev.c (eeb85a14ee3494febb85ccfbee0772eda0823b13)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * NET3 Protocol independent device support routines.
4 *
5 * Derived from the non IP parts of dev.c 1.0.19
6 * Authors: Ross Biro
7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>

--- 834 unchanged lines hidden (view full) ---

843 return -ENOMEM;
844 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
845 return -EINVAL;
846
847 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
848}
849EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
850
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * NET3 Protocol independent device support routines.
4 *
5 * Derived from the non IP parts of dev.c 1.0.19
6 * Authors: Ross Biro
7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>

--- 834 unchanged lines hidden (view full) ---

843 return -ENOMEM;
844 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
845 return -EINVAL;
846
847 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
848}
849EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
850
851static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
852{
853 int k = stack->num_paths++;
854
855 if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
856 return NULL;
857
858 return &stack->path[k];
859}
860
861int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
862 struct net_device_path_stack *stack)
863{
864 const struct net_device *last_dev;
865 struct net_device_path_ctx ctx = {
866 .dev = dev,
867 .daddr = daddr,
868 };
869 struct net_device_path *path;
870 int ret = 0;
871
872 stack->num_paths = 0;
873 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
874 last_dev = ctx.dev;
875 path = dev_fwd_path(stack);
876 if (!path)
877 return -1;
878
879 memset(path, 0, sizeof(struct net_device_path));
880 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
881 if (ret < 0)
882 return -1;
883
884 if (WARN_ON_ONCE(last_dev == ctx.dev))
885 return -1;
886 }
887 path = dev_fwd_path(stack);
888 if (!path)
889 return -1;
890 path->type = DEV_PATH_ETHERNET;
891 path->dev = ctx.dev;
892
893 return ret;
894}
895EXPORT_SYMBOL_GPL(dev_fill_forward_path);
896
851/**
852 * __dev_get_by_name - find a device by its name
853 * @net: the applicable net namespace
854 * @name: name to find
855 *
856 * Find an interface by name. Must be called under RTNL semaphore
857 * or @dev_base_lock. If the name is found a pointer to the device
858 * is returned. If the name is not found then %NULL is returned. The

--- 1599 unchanged lines hidden (view full) ---

2458 return -1;
2459 }
2460
2461 return 0;
2462}
2463EXPORT_SYMBOL(netdev_txq_to_tc);
2464
2465#ifdef CONFIG_XPS
897/**
898 * __dev_get_by_name - find a device by its name
899 * @net: the applicable net namespace
900 * @name: name to find
901 *
902 * Find an interface by name. Must be called under RTNL semaphore
903 * or @dev_base_lock. If the name is found a pointer to the device
904 * is returned. If the name is not found then %NULL is returned. The

--- 1599 unchanged lines hidden (view full) ---

2504 return -1;
2505 }
2506
2507 return 0;
2508}
2509EXPORT_SYMBOL(netdev_txq_to_tc);
2510
2511#ifdef CONFIG_XPS
2466struct static_key xps_needed __read_mostly;
2467EXPORT_SYMBOL(xps_needed);
2468struct static_key xps_rxqs_needed __read_mostly;
2469EXPORT_SYMBOL(xps_rxqs_needed);
2512static struct static_key xps_needed __read_mostly;
2513static struct static_key xps_rxqs_needed __read_mostly;
2470static DEFINE_MUTEX(xps_map_mutex);
2471#define xmap_dereference(P) \
2472 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2473
2474static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2514static DEFINE_MUTEX(xps_map_mutex);
2515#define xmap_dereference(P) \
2516 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2517
2518static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2475 int tci, u16 index)
2519 struct xps_dev_maps *old_maps, int tci, u16 index)
2476{
2477 struct xps_map *map = NULL;
2478 int pos;
2479
2480 if (dev_maps)
2481 map = xmap_dereference(dev_maps->attr_map[tci]);
2482 if (!map)
2483 return false;
2484
2485 for (pos = map->len; pos--;) {
2486 if (map->queues[pos] != index)
2487 continue;
2488
2489 if (map->len > 1) {
2490 map->queues[pos] = map->queues[--map->len];
2491 break;
2492 }
2493
2520{
2521 struct xps_map *map = NULL;
2522 int pos;
2523
2524 if (dev_maps)
2525 map = xmap_dereference(dev_maps->attr_map[tci]);
2526 if (!map)
2527 return false;
2528
2529 for (pos = map->len; pos--;) {
2530 if (map->queues[pos] != index)
2531 continue;
2532
2533 if (map->len > 1) {
2534 map->queues[pos] = map->queues[--map->len];
2535 break;
2536 }
2537
2538 if (old_maps)
2539 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2494 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2495 kfree_rcu(map, rcu);
2496 return false;
2497 }
2498
2499 return true;
2500}
2501
2502static bool remove_xps_queue_cpu(struct net_device *dev,
2503 struct xps_dev_maps *dev_maps,
2504 int cpu, u16 offset, u16 count)
2505{
2540 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2541 kfree_rcu(map, rcu);
2542 return false;
2543 }
2544
2545 return true;
2546}
2547
2548static bool remove_xps_queue_cpu(struct net_device *dev,
2549 struct xps_dev_maps *dev_maps,
2550 int cpu, u16 offset, u16 count)
2551{
2506 int num_tc = dev->num_tc ? : 1;
2552 int num_tc = dev_maps->num_tc;
2507 bool active = false;
2508 int tci;
2509
2510 for (tci = cpu * num_tc; num_tc--; tci++) {
2511 int i, j;
2512
2513 for (i = count, j = offset; i--; j++) {
2553 bool active = false;
2554 int tci;
2555
2556 for (tci = cpu * num_tc; num_tc--; tci++) {
2557 int i, j;
2558
2559 for (i = count, j = offset; i--; j++) {
2514 if (!remove_xps_queue(dev_maps, tci, j))
2560 if (!remove_xps_queue(dev_maps, NULL, tci, j))
2515 break;
2516 }
2517
2518 active |= i < 0;
2519 }
2520
2521 return active;
2522}
2523
2524static void reset_xps_maps(struct net_device *dev,
2525 struct xps_dev_maps *dev_maps,
2561 break;
2562 }
2563
2564 active |= i < 0;
2565 }
2566
2567 return active;
2568}
2569
2570static void reset_xps_maps(struct net_device *dev,
2571 struct xps_dev_maps *dev_maps,
2526 bool is_rxqs_map)
2572 enum xps_map_type type)
2527{
2573{
2528 if (is_rxqs_map) {
2529 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2530 RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
2531 } else {
2532 RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
2533 }
2534 static_key_slow_dec_cpuslocked(&xps_needed);
2574 static_key_slow_dec_cpuslocked(&xps_needed);
2575 if (type == XPS_RXQS)
2576 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2577
2578 RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2579
2535 kfree_rcu(dev_maps, rcu);
2536}
2537
2580 kfree_rcu(dev_maps, rcu);
2581}
2582
2538static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
2539 struct xps_dev_maps *dev_maps, unsigned int nr_ids,
2540 u16 offset, u16 count, bool is_rxqs_map)
2583static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2584 u16 offset, u16 count)
2541{
2585{
2586 struct xps_dev_maps *dev_maps;
2542 bool active = false;
2543 int i, j;
2544
2587 bool active = false;
2588 int i, j;
2589
2545 for (j = -1; j = netif_attrmask_next(j, mask, nr_ids),
2546 j < nr_ids;)
2547 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset,
2548 count);
2590 dev_maps = xmap_dereference(dev->xps_maps[type]);
2591 if (!dev_maps)
2592 return;
2593
2594 for (j = 0; j < dev_maps->nr_ids; j++)
2595 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2549 if (!active)
2596 if (!active)
2550 reset_xps_maps(dev, dev_maps, is_rxqs_map);
2597 reset_xps_maps(dev, dev_maps, type);
2551
2598
2552 if (!is_rxqs_map) {
2553 for (i = offset + (count - 1); count--; i--) {
2599 if (type == XPS_CPUS) {
2600 for (i = offset + (count - 1); count--; i--)
2554 netdev_queue_numa_node_write(
2601 netdev_queue_numa_node_write(
2555 netdev_get_tx_queue(dev, i),
2556 NUMA_NO_NODE);
2557 }
2602 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2558 }
2559}
2560
2561static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2562 u16 count)
2563{
2603 }
2604}
2605
2606static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2607 u16 count)
2608{
2564 const unsigned long *possible_mask = NULL;
2565 struct xps_dev_maps *dev_maps;
2566 unsigned int nr_ids;
2567
2568 if (!static_key_false(&xps_needed))
2569 return;
2570
2571 cpus_read_lock();
2572 mutex_lock(&xps_map_mutex);
2573
2609 if (!static_key_false(&xps_needed))
2610 return;
2611
2612 cpus_read_lock();
2613 mutex_lock(&xps_map_mutex);
2614
2574 if (static_key_false(&xps_rxqs_needed)) {
2575 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2576 if (dev_maps) {
2577 nr_ids = dev->num_rx_queues;
2578 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids,
2579 offset, count, true);
2580 }
2581 }
2615 if (static_key_false(&xps_rxqs_needed))
2616 clean_xps_maps(dev, XPS_RXQS, offset, count);
2582
2617
2583 dev_maps = xmap_dereference(dev->xps_cpus_map);
2584 if (!dev_maps)
2585 goto out_no_maps;
2618 clean_xps_maps(dev, XPS_CPUS, offset, count);
2586
2619
2587 if (num_possible_cpus() > 1)
2588 possible_mask = cpumask_bits(cpu_possible_mask);
2589 nr_ids = nr_cpu_ids;
2590 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count,
2591 false);
2592
2593out_no_maps:
2594 mutex_unlock(&xps_map_mutex);
2595 cpus_read_unlock();
2596}
2597
2598static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2599{
2600 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2601}

--- 33 unchanged lines hidden (view full) ---

2635 for (i = 0; i < pos; i++)
2636 new_map->queues[i] = map->queues[i];
2637 new_map->alloc_len = alloc_len;
2638 new_map->len = pos;
2639
2640 return new_map;
2641}
2642
2620 mutex_unlock(&xps_map_mutex);
2621 cpus_read_unlock();
2622}
2623
2624static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2625{
2626 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2627}

--- 33 unchanged lines hidden (view full) ---

2661 for (i = 0; i < pos; i++)
2662 new_map->queues[i] = map->queues[i];
2663 new_map->alloc_len = alloc_len;
2664 new_map->len = pos;
2665
2666 return new_map;
2667}
2668
2669/* Copy xps maps at a given index */
2670static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2671 struct xps_dev_maps *new_dev_maps, int index,
2672 int tc, bool skip_tc)
2673{
2674 int i, tci = index * dev_maps->num_tc;
2675 struct xps_map *map;
2676
2677 /* copy maps belonging to foreign traffic classes */
2678 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2679 if (i == tc && skip_tc)
2680 continue;
2681
2682 /* fill in the new device map from the old device map */
2683 map = xmap_dereference(dev_maps->attr_map[tci]);
2684 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2685 }
2686}
2687
2643/* Must be called under cpus_read_lock */
2644int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2688/* Must be called under cpus_read_lock */
2689int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2645 u16 index, bool is_rxqs_map)
2690 u16 index, enum xps_map_type type)
2646{
2691{
2647 const unsigned long *online_mask = NULL, *possible_mask = NULL;
2648 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2692 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2693 const unsigned long *online_mask = NULL;
2694 bool active = false, copy = false;
2649 int i, j, tci, numa_node_id = -2;
2650 int maps_sz, num_tc = 1, tc = 0;
2651 struct xps_map *map, *new_map;
2695 int i, j, tci, numa_node_id = -2;
2696 int maps_sz, num_tc = 1, tc = 0;
2697 struct xps_map *map, *new_map;
2652 bool active = false;
2653 unsigned int nr_ids;
2654
2655 if (dev->num_tc) {
2656 /* Do not allow XPS on subordinate device directly */
2657 num_tc = dev->num_tc;
2658 if (num_tc < 0)
2659 return -EINVAL;
2660
2661 /* If queue belongs to subordinate dev use its map */
2662 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2663
2664 tc = netdev_txq_to_tc(dev, index);
2665 if (tc < 0)
2666 return -EINVAL;
2667 }
2668
2669 mutex_lock(&xps_map_mutex);
2698 unsigned int nr_ids;
2699
2700 if (dev->num_tc) {
2701 /* Do not allow XPS on subordinate device directly */
2702 num_tc = dev->num_tc;
2703 if (num_tc < 0)
2704 return -EINVAL;
2705
2706 /* If queue belongs to subordinate dev use its map */
2707 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2708
2709 tc = netdev_txq_to_tc(dev, index);
2710 if (tc < 0)
2711 return -EINVAL;
2712 }
2713
2714 mutex_lock(&xps_map_mutex);
2670 if (is_rxqs_map) {
2715
2716 dev_maps = xmap_dereference(dev->xps_maps[type]);
2717 if (type == XPS_RXQS) {
2671 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2718 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2672 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2673 nr_ids = dev->num_rx_queues;
2674 } else {
2675 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2719 nr_ids = dev->num_rx_queues;
2720 } else {
2721 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2676 if (num_possible_cpus() > 1) {
2722 if (num_possible_cpus() > 1)
2677 online_mask = cpumask_bits(cpu_online_mask);
2723 online_mask = cpumask_bits(cpu_online_mask);
2678 possible_mask = cpumask_bits(cpu_possible_mask);
2679 }
2680 dev_maps = xmap_dereference(dev->xps_cpus_map);
2681 nr_ids = nr_cpu_ids;
2682 }
2683
2684 if (maps_sz < L1_CACHE_BYTES)
2685 maps_sz = L1_CACHE_BYTES;
2686
2724 nr_ids = nr_cpu_ids;
2725 }
2726
2727 if (maps_sz < L1_CACHE_BYTES)
2728 maps_sz = L1_CACHE_BYTES;
2729
2730 /* The old dev_maps could be larger or smaller than the one we're
2731 * setting up now, as dev->num_tc or nr_ids could have been updated in
2732 * between. We could try to be smart, but let's be safe instead and only
2733 * copy foreign traffic classes if the two map sizes match.
2734 */
2735 if (dev_maps &&
2736 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2737 copy = true;
2738
2687 /* allocate memory for queue storage */
2688 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2689 j < nr_ids;) {
2739 /* allocate memory for queue storage */
2740 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2741 j < nr_ids;) {
2690 if (!new_dev_maps)
2691 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2692 if (!new_dev_maps) {
2742 if (!new_dev_maps) {
2693 mutex_unlock(&xps_map_mutex);
2694 return -ENOMEM;
2743 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2744 if (!new_dev_maps) {
2745 mutex_unlock(&xps_map_mutex);
2746 return -ENOMEM;
2747 }
2748
2749 new_dev_maps->nr_ids = nr_ids;
2750 new_dev_maps->num_tc = num_tc;
2695 }
2696
2697 tci = j * num_tc + tc;
2751 }
2752
2753 tci = j * num_tc + tc;
2698 map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) :
2699 NULL;
2754 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2700
2755
2701 map = expand_xps_map(map, j, index, is_rxqs_map);
2756 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2702 if (!map)
2703 goto error;
2704
2705 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2706 }
2707
2708 if (!new_dev_maps)
2709 goto out_no_new_maps;
2710
2711 if (!dev_maps) {
2712 /* Increment static keys at most once per type */
2713 static_key_slow_inc_cpuslocked(&xps_needed);
2757 if (!map)
2758 goto error;
2759
2760 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2761 }
2762
2763 if (!new_dev_maps)
2764 goto out_no_new_maps;
2765
2766 if (!dev_maps) {
2767 /* Increment static keys at most once per type */
2768 static_key_slow_inc_cpuslocked(&xps_needed);
2714 if (is_rxqs_map)
2769 if (type == XPS_RXQS)
2715 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2716 }
2717
2770 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2771 }
2772
2718 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2719 j < nr_ids;) {
2720 /* copy maps belonging to foreign traffic classes */
2721 for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) {
2722 /* fill in the new device map from the old device map */
2723 map = xmap_dereference(dev_maps->attr_map[tci]);
2724 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2725 }
2773 for (j = 0; j < nr_ids; j++) {
2774 bool skip_tc = false;
2726
2775
2727 /* We need to explicitly update tci as prevous loop
2728 * could break out early if dev_maps is NULL.
2729 */
2730 tci = j * num_tc + tc;
2776 tci = j * num_tc + tc;
2731
2732 if (netif_attr_test_mask(j, mask, nr_ids) &&
2733 netif_attr_test_online(j, online_mask, nr_ids)) {
2734 /* add tx-queue to CPU/rx-queue maps */
2735 int pos = 0;
2736
2777 if (netif_attr_test_mask(j, mask, nr_ids) &&
2778 netif_attr_test_online(j, online_mask, nr_ids)) {
2779 /* add tx-queue to CPU/rx-queue maps */
2780 int pos = 0;
2781
2782 skip_tc = true;
2783
2737 map = xmap_dereference(new_dev_maps->attr_map[tci]);
2738 while ((pos < map->len) && (map->queues[pos] != index))
2739 pos++;
2740
2741 if (pos == map->len)
2742 map->queues[map->len++] = index;
2743#ifdef CONFIG_NUMA
2784 map = xmap_dereference(new_dev_maps->attr_map[tci]);
2785 while ((pos < map->len) && (map->queues[pos] != index))
2786 pos++;
2787
2788 if (pos == map->len)
2789 map->queues[map->len++] = index;
2790#ifdef CONFIG_NUMA
2744 if (!is_rxqs_map) {
2791 if (type == XPS_CPUS) {
2745 if (numa_node_id == -2)
2746 numa_node_id = cpu_to_node(j);
2747 else if (numa_node_id != cpu_to_node(j))
2748 numa_node_id = -1;
2749 }
2750#endif
2792 if (numa_node_id == -2)
2793 numa_node_id = cpu_to_node(j);
2794 else if (numa_node_id != cpu_to_node(j))
2795 numa_node_id = -1;
2796 }
2797#endif
2751 } else if (dev_maps) {
2752 /* fill in the new device map from the old device map */
2753 map = xmap_dereference(dev_maps->attr_map[tci]);
2754 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2755 }
2756
2798 }
2799
2757 /* copy maps belonging to foreign traffic classes */
2758 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2759 /* fill in the new device map from the old device map */
2760 map = xmap_dereference(dev_maps->attr_map[tci]);
2761 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2762 }
2800 if (copy)
2801 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2802 skip_tc);
2763 }
2764
2803 }
2804
2765 if (is_rxqs_map)
2766 rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps);
2767 else
2768 rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps);
2805 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2769
2770 /* Cleanup old maps */
2771 if (!dev_maps)
2772 goto out_no_old_maps;
2773
2806
2807 /* Cleanup old maps */
2808 if (!dev_maps)
2809 goto out_no_old_maps;
2810
2774 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2775 j < nr_ids;) {
2776 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2777 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2811 for (j = 0; j < dev_maps->nr_ids; j++) {
2812 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2778 map = xmap_dereference(dev_maps->attr_map[tci]);
2813 map = xmap_dereference(dev_maps->attr_map[tci]);
2779 if (map && map != new_map)
2780 kfree_rcu(map, rcu);
2814 if (!map)
2815 continue;
2816
2817 if (copy) {
2818 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2819 if (map == new_map)
2820 continue;
2821 }
2822
2823 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2824 kfree_rcu(map, rcu);
2781 }
2782 }
2783
2825 }
2826 }
2827
2784 kfree_rcu(dev_maps, rcu);
2828 old_dev_maps = dev_maps;
2785
2786out_no_old_maps:
2787 dev_maps = new_dev_maps;
2788 active = true;
2789
2790out_no_new_maps:
2829
2830out_no_old_maps:
2831 dev_maps = new_dev_maps;
2832 active = true;
2833
2834out_no_new_maps:
2791 if (!is_rxqs_map) {
2835 if (type == XPS_CPUS)
2792 /* update Tx queue numa node */
2793 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2794 (numa_node_id >= 0) ?
2795 numa_node_id : NUMA_NO_NODE);
2836 /* update Tx queue numa node */
2837 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2838 (numa_node_id >= 0) ?
2839 numa_node_id : NUMA_NO_NODE);
2796 }
2797
2798 if (!dev_maps)
2799 goto out_no_maps;
2800
2801 /* removes tx-queue from unused CPUs/rx-queues */
2840
2841 if (!dev_maps)
2842 goto out_no_maps;
2843
2844 /* removes tx-queue from unused CPUs/rx-queues */
2802 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2803 j < nr_ids;) {
2804 for (i = tc, tci = j * num_tc; i--; tci++)
2805 active |= remove_xps_queue(dev_maps, tci, index);
2806 if (!netif_attr_test_mask(j, mask, nr_ids) ||
2807 !netif_attr_test_online(j, online_mask, nr_ids))
2808 active |= remove_xps_queue(dev_maps, tci, index);
2809 for (i = num_tc - tc, tci++; --i; tci++)
2810 active |= remove_xps_queue(dev_maps, tci, index);
2845 for (j = 0; j < dev_maps->nr_ids; j++) {
2846 tci = j * dev_maps->num_tc;
2847
2848 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2849 if (i == tc &&
2850 netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2851 netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2852 continue;
2853
2854 active |= remove_xps_queue(dev_maps,
2855 copy ? old_dev_maps : NULL,
2856 tci, index);
2857 }
2811 }
2812
2858 }
2859
2860 if (old_dev_maps)
2861 kfree_rcu(old_dev_maps, rcu);
2862
2813 /* free map if not active */
2814 if (!active)
2863 /* free map if not active */
2864 if (!active)
2815 reset_xps_maps(dev, dev_maps, is_rxqs_map);
2865 reset_xps_maps(dev, dev_maps, type);
2816
2817out_no_maps:
2818 mutex_unlock(&xps_map_mutex);
2819
2820 return 0;
2821error:
2822 /* remove any maps that we added */
2866
2867out_no_maps:
2868 mutex_unlock(&xps_map_mutex);
2869
2870 return 0;
2871error:
2872 /* remove any maps that we added */
2823 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2824 j < nr_ids;) {
2873 for (j = 0; j < nr_ids; j++) {
2825 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2826 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2874 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2875 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2827 map = dev_maps ?
2876 map = copy ?
2828 xmap_dereference(dev_maps->attr_map[tci]) :
2829 NULL;
2830 if (new_map && new_map != map)
2831 kfree(new_map);
2832 }
2833 }
2834
2835 mutex_unlock(&xps_map_mutex);

--- 4 unchanged lines hidden (view full) ---

2840EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2841
2842int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2843 u16 index)
2844{
2845 int ret;
2846
2847 cpus_read_lock();
2877 xmap_dereference(dev_maps->attr_map[tci]) :
2878 NULL;
2879 if (new_map && new_map != map)
2880 kfree(new_map);
2881 }
2882 }
2883
2884 mutex_unlock(&xps_map_mutex);

--- 4 unchanged lines hidden (view full) ---

2889EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2890
2891int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2892 u16 index)
2893{
2894 int ret;
2895
2896 cpus_read_lock();
2848 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, false);
2897 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2849 cpus_read_unlock();
2850
2851 return ret;
2852}
2853EXPORT_SYMBOL(netif_set_xps_queue);
2854
2855#endif
2856static void netdev_unbind_all_sb_channels(struct net_device *dev)

--- 1094 unchanged lines hidden (view full) ---

3951 return skb;
3952}
3953#endif /* CONFIG_NET_EGRESS */
3954
3955#ifdef CONFIG_XPS
3956static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
3957 struct xps_dev_maps *dev_maps, unsigned int tci)
3958{
2898 cpus_read_unlock();
2899
2900 return ret;
2901}
2902EXPORT_SYMBOL(netif_set_xps_queue);
2903
2904#endif
2905static void netdev_unbind_all_sb_channels(struct net_device *dev)

--- 1094 unchanged lines hidden (view full) ---

4000 return skb;
4001}
4002#endif /* CONFIG_NET_EGRESS */
4003
4004#ifdef CONFIG_XPS
4005static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4006 struct xps_dev_maps *dev_maps, unsigned int tci)
4007{
4008 int tc = netdev_get_prio_tc_map(dev, skb->priority);
3959 struct xps_map *map;
3960 int queue_index = -1;
3961
4009 struct xps_map *map;
4010 int queue_index = -1;
4011
3962 if (dev->num_tc) {
3963 tci *= dev->num_tc;
3964 tci += netdev_get_prio_tc_map(dev, skb->priority);
3965 }
4012 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4013 return queue_index;
3966
4014
4015 tci *= dev_maps->num_tc;
4016 tci += tc;
4017
3967 map = rcu_dereference(dev_maps->attr_map[tci]);
3968 if (map) {
3969 if (map->len == 1)
3970 queue_index = map->queues[0];
3971 else
3972 queue_index = map->queues[reciprocal_scale(
3973 skb_get_hash(skb), map->len)];
3974 if (unlikely(queue_index >= dev->real_num_tx_queues))

--- 13 unchanged lines hidden (view full) ---

3988
3989 if (!static_key_false(&xps_needed))
3990 return -1;
3991
3992 rcu_read_lock();
3993 if (!static_key_false(&xps_rxqs_needed))
3994 goto get_cpus_map;
3995
4018 map = rcu_dereference(dev_maps->attr_map[tci]);
4019 if (map) {
4020 if (map->len == 1)
4021 queue_index = map->queues[0];
4022 else
4023 queue_index = map->queues[reciprocal_scale(
4024 skb_get_hash(skb), map->len)];
4025 if (unlikely(queue_index >= dev->real_num_tx_queues))

--- 13 unchanged lines hidden (view full) ---

4039
4040 if (!static_key_false(&xps_needed))
4041 return -1;
4042
4043 rcu_read_lock();
4044 if (!static_key_false(&xps_rxqs_needed))
4045 goto get_cpus_map;
4046
3996 dev_maps = rcu_dereference(sb_dev->xps_rxqs_map);
4047 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
3997 if (dev_maps) {
3998 int tci = sk_rx_queue_get(sk);
3999
4048 if (dev_maps) {
4049 int tci = sk_rx_queue_get(sk);
4050
4000 if (tci >= 0 && tci < dev->num_rx_queues)
4051 if (tci >= 0)
4001 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4002 tci);
4003 }
4004
4005get_cpus_map:
4006 if (queue_index < 0) {
4052 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4053 tci);
4054 }
4055
4056get_cpus_map:
4057 if (queue_index < 0) {
4007 dev_maps = rcu_dereference(sb_dev->xps_cpus_map);
4058 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4008 if (dev_maps) {
4009 unsigned int tci = skb->sender_cpu - 1;
4010
4011 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4012 tci);
4013 }
4014 }
4015 rcu_read_unlock();

--- 1263 unchanged lines hidden (view full) ---

5279 switch (rx_handler(&skb)) {
5280 case RX_HANDLER_CONSUMED:
5281 ret = NET_RX_SUCCESS;
5282 goto out;
5283 case RX_HANDLER_ANOTHER:
5284 goto another_round;
5285 case RX_HANDLER_EXACT:
5286 deliver_exact = true;
4059 if (dev_maps) {
4060 unsigned int tci = skb->sender_cpu - 1;
4061
4062 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4063 tci);
4064 }
4065 }
4066 rcu_read_unlock();

--- 1263 unchanged lines hidden (view full) ---

5330 switch (rx_handler(&skb)) {
5331 case RX_HANDLER_CONSUMED:
5332 ret = NET_RX_SUCCESS;
5333 goto out;
5334 case RX_HANDLER_ANOTHER:
5335 goto another_round;
5336 case RX_HANDLER_EXACT:
5337 deliver_exact = true;
5338 break;
5287 case RX_HANDLER_PASS:
5288 break;
5289 default:
5290 BUG();
5291 }
5292 }
5293
5294 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {

--- 576 unchanged lines hidden (view full) ---

5871 while ((i = ffs(bitmask)) != 0) {
5872 bitmask >>= i;
5873 base += i;
5874 __napi_gro_flush_chain(napi, base, flush_old);
5875 }
5876}
5877EXPORT_SYMBOL(napi_gro_flush);
5878
5339 case RX_HANDLER_PASS:
5340 break;
5341 default:
5342 BUG();
5343 }
5344 }
5345
5346 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {

--- 576 unchanged lines hidden (view full) ---

5923 while ((i = ffs(bitmask)) != 0) {
5924 bitmask >>= i;
5925 base += i;
5926 __napi_gro_flush_chain(napi, base, flush_old);
5927 }
5928}
5929EXPORT_SYMBOL(napi_gro_flush);
5930
5879static struct list_head *gro_list_prepare(struct napi_struct *napi,
5880 struct sk_buff *skb)
5931static void gro_list_prepare(const struct list_head *head,
5932 const struct sk_buff *skb)
5881{
5882 unsigned int maclen = skb->dev->hard_header_len;
5883 u32 hash = skb_get_hash_raw(skb);
5933{
5934 unsigned int maclen = skb->dev->hard_header_len;
5935 u32 hash = skb_get_hash_raw(skb);
5884 struct list_head *head;
5885 struct sk_buff *p;
5886
5936 struct sk_buff *p;
5937
5887 head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list;
5888 list_for_each_entry(p, head, list) {
5889 unsigned long diffs;
5890
5891 NAPI_GRO_CB(p)->flush = 0;
5892
5893 if (hash != skb_get_hash_raw(p)) {
5894 NAPI_GRO_CB(p)->same_flow = 0;
5895 continue;

--- 9 unchanged lines hidden (view full) ---

5905 diffs |= compare_ether_header(skb_mac_header(p),
5906 skb_mac_header(skb));
5907 else if (!diffs)
5908 diffs = memcmp(skb_mac_header(p),
5909 skb_mac_header(skb),
5910 maclen);
5911 NAPI_GRO_CB(p)->same_flow = !diffs;
5912 }
5938 list_for_each_entry(p, head, list) {
5939 unsigned long diffs;
5940
5941 NAPI_GRO_CB(p)->flush = 0;
5942
5943 if (hash != skb_get_hash_raw(p)) {
5944 NAPI_GRO_CB(p)->same_flow = 0;
5945 continue;

--- 9 unchanged lines hidden (view full) ---

5955 diffs |= compare_ether_header(skb_mac_header(p),
5956 skb_mac_header(skb));
5957 else if (!diffs)
5958 diffs = memcmp(skb_mac_header(p),
5959 skb_mac_header(skb),
5960 maclen);
5961 NAPI_GRO_CB(p)->same_flow = !diffs;
5962 }
5913
5914 return head;
5915}
5916
5917static void skb_gro_reset_offset(struct sk_buff *skb)
5918{
5919 const struct skb_shared_info *pinfo = skb_shinfo(skb);
5920 const skb_frag_t *frag0 = &pinfo->frags[0];
5921
5922 NAPI_GRO_CB(skb)->data_offset = 0;

--- 46 unchanged lines hidden (view full) ---

5969 * SKB to the chain.
5970 */
5971 skb_list_del_init(oldest);
5972 napi_gro_complete(napi, oldest);
5973}
5974
5975static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
5976{
5963}
5964
5965static void skb_gro_reset_offset(struct sk_buff *skb)
5966{
5967 const struct skb_shared_info *pinfo = skb_shinfo(skb);
5968 const skb_frag_t *frag0 = &pinfo->frags[0];
5969
5970 NAPI_GRO_CB(skb)->data_offset = 0;

--- 46 unchanged lines hidden (view full) ---

6017 * SKB to the chain.
6018 */
6019 skb_list_del_init(oldest);
6020 napi_gro_complete(napi, oldest);
6021}
6022
6023static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
6024{
5977 u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
6025 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
6026 struct gro_list *gro_list = &napi->gro_hash[bucket];
5978 struct list_head *head = &offload_base;
5979 struct packet_offload *ptype;
5980 __be16 type = skb->protocol;
6027 struct list_head *head = &offload_base;
6028 struct packet_offload *ptype;
6029 __be16 type = skb->protocol;
5981 struct list_head *gro_head;
5982 struct sk_buff *pp = NULL;
5983 enum gro_result ret;
5984 int same_flow;
5985 int grow;
5986
5987 if (netif_elide_gro(skb->dev))
5988 goto normal;
5989
6030 struct sk_buff *pp = NULL;
6031 enum gro_result ret;
6032 int same_flow;
6033 int grow;
6034
6035 if (netif_elide_gro(skb->dev))
6036 goto normal;
6037
5990 gro_head = gro_list_prepare(napi, skb);
6038 gro_list_prepare(&gro_list->list, skb);
5991
5992 rcu_read_lock();
5993 list_for_each_entry_rcu(ptype, head, list) {
5994 if (ptype->type != type || !ptype->callbacks.gro_receive)
5995 continue;
5996
5997 skb_set_network_header(skb, skb_gro_offset(skb));
5998 skb_reset_mac_len(skb);

--- 19 unchanged lines hidden (view full) ---

6018 break;
6019 default:
6020 NAPI_GRO_CB(skb)->csum_cnt = 0;
6021 NAPI_GRO_CB(skb)->csum_valid = 0;
6022 }
6023
6024 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
6025 ipv6_gro_receive, inet_gro_receive,
6039
6040 rcu_read_lock();
6041 list_for_each_entry_rcu(ptype, head, list) {
6042 if (ptype->type != type || !ptype->callbacks.gro_receive)
6043 continue;
6044
6045 skb_set_network_header(skb, skb_gro_offset(skb));
6046 skb_reset_mac_len(skb);

--- 19 unchanged lines hidden (view full) ---

6066 break;
6067 default:
6068 NAPI_GRO_CB(skb)->csum_cnt = 0;
6069 NAPI_GRO_CB(skb)->csum_valid = 0;
6070 }
6071
6072 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
6073 ipv6_gro_receive, inet_gro_receive,
6026 gro_head, skb);
6074 &gro_list->list, skb);
6027 break;
6028 }
6029 rcu_read_unlock();
6030
6031 if (&ptype->list == head)
6032 goto normal;
6033
6034 if (PTR_ERR(pp) == -EINPROGRESS) {
6035 ret = GRO_CONSUMED;
6036 goto ok;
6037 }
6038
6039 same_flow = NAPI_GRO_CB(skb)->same_flow;
6040 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
6041
6042 if (pp) {
6043 skb_list_del_init(pp);
6044 napi_gro_complete(napi, pp);
6075 break;
6076 }
6077 rcu_read_unlock();
6078
6079 if (&ptype->list == head)
6080 goto normal;
6081
6082 if (PTR_ERR(pp) == -EINPROGRESS) {
6083 ret = GRO_CONSUMED;
6084 goto ok;
6085 }
6086
6087 same_flow = NAPI_GRO_CB(skb)->same_flow;
6088 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
6089
6090 if (pp) {
6091 skb_list_del_init(pp);
6092 napi_gro_complete(napi, pp);
6045 napi->gro_hash[hash].count--;
6093 gro_list->count--;
6046 }
6047
6048 if (same_flow)
6049 goto ok;
6050
6051 if (NAPI_GRO_CB(skb)->flush)
6052 goto normal;
6053
6094 }
6095
6096 if (same_flow)
6097 goto ok;
6098
6099 if (NAPI_GRO_CB(skb)->flush)
6100 goto normal;
6101
6054 if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
6055 gro_flush_oldest(napi, gro_head);
6056 } else {
6057 napi->gro_hash[hash].count++;
6058 }
6102 if (unlikely(gro_list->count >= MAX_GRO_SKBS))
6103 gro_flush_oldest(napi, &gro_list->list);
6104 else
6105 gro_list->count++;
6106
6059 NAPI_GRO_CB(skb)->count = 1;
6060 NAPI_GRO_CB(skb)->age = jiffies;
6061 NAPI_GRO_CB(skb)->last = skb;
6062 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
6107 NAPI_GRO_CB(skb)->count = 1;
6108 NAPI_GRO_CB(skb)->age = jiffies;
6109 NAPI_GRO_CB(skb)->last = skb;
6110 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
6063 list_add(&skb->list, gro_head);
6111 list_add(&skb->list, &gro_list->list);
6064 ret = GRO_HELD;
6065
6066pull:
6067 grow = skb_gro_offset(skb) - skb_headlen(skb);
6068 if (grow > 0)
6069 gro_pull_from_frag0(skb, grow);
6070ok:
6112 ret = GRO_HELD;
6113
6114pull:
6115 grow = skb_gro_offset(skb) - skb_headlen(skb);
6116 if (grow > 0)
6117 gro_pull_from_frag0(skb, grow);
6118ok:
6071 if (napi->gro_hash[hash].count) {
6072 if (!test_bit(hash, &napi->gro_bitmask))
6073 __set_bit(hash, &napi->gro_bitmask);
6074 } else if (test_bit(hash, &napi->gro_bitmask)) {
6075 __clear_bit(hash, &napi->gro_bitmask);
6119 if (gro_list->count) {
6120 if (!test_bit(bucket, &napi->gro_bitmask))
6121 __set_bit(bucket, &napi->gro_bitmask);
6122 } else if (test_bit(bucket, &napi->gro_bitmask)) {
6123 __clear_bit(bucket, &napi->gro_bitmask);
6076 }
6077
6078 return ret;
6079
6080normal:
6081 ret = GRO_NORMAL;
6082 goto pull;
6083}

--- 700 unchanged lines hidden (view full) ---

6784 if (threaded)
6785 set_bit(NAPI_STATE_THREADED, &napi->state);
6786 else
6787 clear_bit(NAPI_STATE_THREADED, &napi->state);
6788 }
6789
6790 return err;
6791}
6124 }
6125
6126 return ret;
6127
6128normal:
6129 ret = GRO_NORMAL;
6130 goto pull;
6131}

--- 700 unchanged lines hidden (view full) ---

6832 if (threaded)
6833 set_bit(NAPI_STATE_THREADED, &napi->state);
6834 else
6835 clear_bit(NAPI_STATE_THREADED, &napi->state);
6836 }
6837
6838 return err;
6839}
6840EXPORT_SYMBOL(dev_set_threaded);
6792
6793void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
6794 int (*poll)(struct napi_struct *, int), int weight)
6795{
6796 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6797 return;
6798
6799 INIT_LIST_HEAD(&napi->poll_list);

--- 3531 unchanged lines hidden (view full) ---

10331 err = register_netdevice(dev);
10332 rtnl_unlock();
10333 return err;
10334}
10335EXPORT_SYMBOL(register_netdev);
10336
10337int netdev_refcnt_read(const struct net_device *dev)
10338{
6841
6842void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
6843 int (*poll)(struct napi_struct *, int), int weight)
6844{
6845 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6846 return;
6847
6848 INIT_LIST_HEAD(&napi->poll_list);

--- 3531 unchanged lines hidden (view full) ---

10380 err = register_netdevice(dev);
10381 rtnl_unlock();
10382 return err;
10383}
10384EXPORT_SYMBOL(register_netdev);
10385
10386int netdev_refcnt_read(const struct net_device *dev)
10387{
10388#ifdef CONFIG_PCPU_DEV_REFCNT
10339 int i, refcnt = 0;
10340
10341 for_each_possible_cpu(i)
10342 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10343 return refcnt;
10389 int i, refcnt = 0;
10390
10391 for_each_possible_cpu(i)
10392 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10393 return refcnt;
10394#else
10395 return refcount_read(&dev->dev_refcnt);
10396#endif
10344}
10345EXPORT_SYMBOL(netdev_refcnt_read);
10346
10397}
10398EXPORT_SYMBOL(netdev_refcnt_read);
10399
10400int netdev_unregister_timeout_secs __read_mostly = 10;
10401
10347#define WAIT_REFS_MIN_MSECS 1
10348#define WAIT_REFS_MAX_MSECS 250
10349/**
10350 * netdev_wait_allrefs - wait until all references are gone.
10351 * @dev: target net_device
10352 *
10353 * This is called when unregistering network devices.
10354 *

--- 8 unchanged lines hidden (view full) ---

10363 unsigned long rebroadcast_time, warning_time;
10364 int wait = 0, refcnt;
10365
10366 linkwatch_forget_dev(dev);
10367
10368 rebroadcast_time = warning_time = jiffies;
10369 refcnt = netdev_refcnt_read(dev);
10370
10402#define WAIT_REFS_MIN_MSECS 1
10403#define WAIT_REFS_MAX_MSECS 250
10404/**
10405 * netdev_wait_allrefs - wait until all references are gone.
10406 * @dev: target net_device
10407 *
10408 * This is called when unregistering network devices.
10409 *

--- 8 unchanged lines hidden (view full) ---

10418 unsigned long rebroadcast_time, warning_time;
10419 int wait = 0, refcnt;
10420
10421 linkwatch_forget_dev(dev);
10422
10423 rebroadcast_time = warning_time = jiffies;
10424 refcnt = netdev_refcnt_read(dev);
10425
10371 while (refcnt != 0) {
10426 while (refcnt != 1) {
10372 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10373 rtnl_lock();
10374
10375 /* Rebroadcast unregister notification */
10376 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10377
10378 __rtnl_unlock();
10379 rcu_barrier();

--- 20 unchanged lines hidden (view full) ---

10400 wait = WAIT_REFS_MIN_MSECS;
10401 } else {
10402 msleep(wait);
10403 wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10404 }
10405
10406 refcnt = netdev_refcnt_read(dev);
10407
10427 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10428 rtnl_lock();
10429
10430 /* Rebroadcast unregister notification */
10431 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10432
10433 __rtnl_unlock();
10434 rcu_barrier();

--- 20 unchanged lines hidden (view full) ---

10455 wait = WAIT_REFS_MIN_MSECS;
10456 } else {
10457 msleep(wait);
10458 wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10459 }
10460
10461 refcnt = netdev_refcnt_read(dev);
10462
10408 if (refcnt && time_after(jiffies, warning_time + 10 * HZ)) {
10463 if (refcnt != 1 &&
10464 time_after(jiffies, warning_time +
10465 netdev_unregister_timeout_secs * HZ)) {
10409 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10410 dev->name, refcnt);
10411 warning_time = jiffies;
10412 }
10413 }
10414}
10415
10416/* The sequence is:

--- 59 unchanged lines hidden (view full) ---

10476 continue;
10477 }
10478
10479 dev->reg_state = NETREG_UNREGISTERED;
10480
10481 netdev_wait_allrefs(dev);
10482
10483 /* paranoia */
10466 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10467 dev->name, refcnt);
10468 warning_time = jiffies;
10469 }
10470 }
10471}
10472
10473/* The sequence is:

--- 59 unchanged lines hidden (view full) ---

10533 continue;
10534 }
10535
10536 dev->reg_state = NETREG_UNREGISTERED;
10537
10538 netdev_wait_allrefs(dev);
10539
10540 /* paranoia */
10484 BUG_ON(netdev_refcnt_read(dev));
10541 BUG_ON(netdev_refcnt_read(dev) != 1);
10485 BUG_ON(!list_empty(&dev->ptype_all));
10486 BUG_ON(!list_empty(&dev->ptype_specific));
10487 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10488 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10489#if IS_ENABLED(CONFIG_DECNET)
10490 WARN_ON(dev->dn_ptr);
10491#endif
10492 if (dev->priv_destructor)

--- 200 unchanged lines hidden (view full) ---

10693
10694 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
10695 if (!p)
10696 return NULL;
10697
10698 dev = PTR_ALIGN(p, NETDEV_ALIGN);
10699 dev->padded = (char *)dev - (char *)p;
10700
10542 BUG_ON(!list_empty(&dev->ptype_all));
10543 BUG_ON(!list_empty(&dev->ptype_specific));
10544 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10545 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10546#if IS_ENABLED(CONFIG_DECNET)
10547 WARN_ON(dev->dn_ptr);
10548#endif
10549 if (dev->priv_destructor)

--- 200 unchanged lines hidden (view full) ---

10750
10751 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
10752 if (!p)
10753 return NULL;
10754
10755 dev = PTR_ALIGN(p, NETDEV_ALIGN);
10756 dev->padded = (char *)dev - (char *)p;
10757
10758#ifdef CONFIG_PCPU_DEV_REFCNT
10701 dev->pcpu_refcnt = alloc_percpu(int);
10702 if (!dev->pcpu_refcnt)
10703 goto free_dev;
10759 dev->pcpu_refcnt = alloc_percpu(int);
10760 if (!dev->pcpu_refcnt)
10761 goto free_dev;
10762 dev_hold(dev);
10763#else
10764 refcount_set(&dev->dev_refcnt, 1);
10765#endif
10704
10705 if (dev_addr_init(dev))
10706 goto free_pcpu;
10707
10708 dev_mc_init(dev);
10709 dev_uc_init(dev);
10710
10711 dev_net_set(dev, &init_net);

--- 47 unchanged lines hidden (view full) ---

10759
10760 return dev;
10761
10762free_all:
10763 free_netdev(dev);
10764 return NULL;
10765
10766free_pcpu:
10766
10767 if (dev_addr_init(dev))
10768 goto free_pcpu;
10769
10770 dev_mc_init(dev);
10771 dev_uc_init(dev);
10772
10773 dev_net_set(dev, &init_net);

--- 47 unchanged lines hidden (view full) ---

10821
10822 return dev;
10823
10824free_all:
10825 free_netdev(dev);
10826 return NULL;
10827
10828free_pcpu:
10829#ifdef CONFIG_PCPU_DEV_REFCNT
10767 free_percpu(dev->pcpu_refcnt);
10768free_dev:
10830 free_percpu(dev->pcpu_refcnt);
10831free_dev:
10832#endif
10769 netdev_freemem(dev);
10770 return NULL;
10771}
10772EXPORT_SYMBOL(alloc_netdev_mqs);
10773
10774/**
10775 * free_netdev - free network device
10776 * @dev: device

--- 25 unchanged lines hidden (view full) ---

10802 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
10803
10804 /* Flush device addresses */
10805 dev_addr_flush(dev);
10806
10807 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10808 netif_napi_del(p);
10809
10833 netdev_freemem(dev);
10834 return NULL;
10835}
10836EXPORT_SYMBOL(alloc_netdev_mqs);
10837
10838/**
10839 * free_netdev - free network device
10840 * @dev: device

--- 25 unchanged lines hidden (view full) ---

10866 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
10867
10868 /* Flush device addresses */
10869 dev_addr_flush(dev);
10870
10871 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10872 netif_napi_del(p);
10873
10874#ifdef CONFIG_PCPU_DEV_REFCNT
10810 free_percpu(dev->pcpu_refcnt);
10811 dev->pcpu_refcnt = NULL;
10875 free_percpu(dev->pcpu_refcnt);
10876 dev->pcpu_refcnt = NULL;
10877#endif
10812 free_percpu(dev->xdp_bulkq);
10813 dev->xdp_bulkq = NULL;
10814
10815 /* Compatibility with error handling in drivers */
10816 if (dev->reg_state == NETREG_UNINITIALIZED) {
10817 netdev_freemem(dev);
10818 return;
10819 }

--- 176 unchanged lines hidden (view full) ---

10996EXPORT_SYMBOL(unregister_netdev);
10997
10998/**
10999 * dev_change_net_namespace - move device to different nethost namespace
11000 * @dev: device
11001 * @net: network namespace
11002 * @pat: If not NULL name pattern to try if the current device name
11003 * is already taken in the destination network namespace.
10878 free_percpu(dev->xdp_bulkq);
10879 dev->xdp_bulkq = NULL;
10880
10881 /* Compatibility with error handling in drivers */
10882 if (dev->reg_state == NETREG_UNINITIALIZED) {
10883 netdev_freemem(dev);
10884 return;
10885 }

--- 176 unchanged lines hidden (view full) ---

11062EXPORT_SYMBOL(unregister_netdev);
11063
11064/**
11065 * dev_change_net_namespace - move device to different nethost namespace
11066 * @dev: device
11067 * @net: network namespace
11068 * @pat: If not NULL name pattern to try if the current device name
11069 * is already taken in the destination network namespace.
11070 * @new_ifindex: If not zero, specifies device index in the target
11071 * namespace.
11004 *
11005 * This function shuts down a device interface and moves it
11006 * to a new network namespace. On success 0 is returned, on
11007 * a failure a netagive errno code is returned.
11008 *
11009 * Callers must hold the rtnl semaphore.
11010 */
11011
11072 *
11073 * This function shuts down a device interface and moves it
11074 * to a new network namespace. On success 0 is returned, on
11075 * a failure a netagive errno code is returned.
11076 *
11077 * Callers must hold the rtnl semaphore.
11078 */
11079
11012int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
11080int dev_change_net_namespace(struct net_device *dev, struct net *net,
11081 const char *pat, int new_ifindex)
11013{
11014 struct net *net_old = dev_net(dev);
11082{
11083 struct net *net_old = dev_net(dev);
11015 int err, new_nsid, new_ifindex;
11084 int err, new_nsid;
11016
11017 ASSERT_RTNL();
11018
11019 /* Don't allow namespace local devices to be moved. */
11020 err = -EINVAL;
11021 if (dev->features & NETIF_F_NETNS_LOCAL)
11022 goto out;
11023

--- 14 unchanged lines hidden (view full) ---

11038 /* We get here if we can't use the current device name */
11039 if (!pat)
11040 goto out;
11041 err = dev_get_valid_name(net, dev, pat);
11042 if (err < 0)
11043 goto out;
11044 }
11045
11085
11086 ASSERT_RTNL();
11087
11088 /* Don't allow namespace local devices to be moved. */
11089 err = -EINVAL;
11090 if (dev->features & NETIF_F_NETNS_LOCAL)
11091 goto out;
11092

--- 14 unchanged lines hidden (view full) ---

11107 /* We get here if we can't use the current device name */
11108 if (!pat)
11109 goto out;
11110 err = dev_get_valid_name(net, dev, pat);
11111 if (err < 0)
11112 goto out;
11113 }
11114
11115 /* Check that new_ifindex isn't used yet. */
11116 err = -EBUSY;
11117 if (new_ifindex && __dev_get_by_index(net, new_ifindex))
11118 goto out;
11119
11046 /*
11047 * And now a mini version of register_netdevice unregister_netdevice.
11048 */
11049
11050 /* If device is running close it first. */
11051 dev_close(dev);
11052
11053 /* And unlink it from device chain */

--- 11 unchanged lines hidden (view full) ---

11065 * This is wanted because this way 8021q and macvlan know
11066 * the device is just moving and can keep their slaves up.
11067 */
11068 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11069 rcu_barrier();
11070
11071 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11072 /* If there is an ifindex conflict assign a new one */
11120 /*
11121 * And now a mini version of register_netdevice unregister_netdevice.
11122 */
11123
11124 /* If device is running close it first. */
11125 dev_close(dev);
11126
11127 /* And unlink it from device chain */

--- 11 unchanged lines hidden (view full) ---

11139 * This is wanted because this way 8021q and macvlan know
11140 * the device is just moving and can keep their slaves up.
11141 */
11142 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11143 rcu_barrier();
11144
11145 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11146 /* If there is an ifindex conflict assign a new one */
11073 if (__dev_get_by_index(net, dev->ifindex))
11074 new_ifindex = dev_new_index(net);
11075 else
11076 new_ifindex = dev->ifindex;
11147 if (!new_ifindex) {
11148 if (__dev_get_by_index(net, dev->ifindex))
11149 new_ifindex = dev_new_index(net);
11150 else
11151 new_ifindex = dev->ifindex;
11152 }
11077
11078 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11079 new_ifindex);
11080
11081 /*
11082 * Flush the unicast and multicast chains
11083 */
11084 dev_uc_flush(dev);

--- 292 unchanged lines hidden (view full) ---

11377 /* Leave virtual devices for the generic cleanup */
11378 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
11379 continue;
11380
11381 /* Push remaining network devices to init_net */
11382 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
11383 if (__dev_get_by_name(&init_net, fb_name))
11384 snprintf(fb_name, IFNAMSIZ, "dev%%d");
11153
11154 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11155 new_ifindex);
11156
11157 /*
11158 * Flush the unicast and multicast chains
11159 */
11160 dev_uc_flush(dev);

--- 292 unchanged lines hidden (view full) ---

11453 /* Leave virtual devices for the generic cleanup */
11454 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
11455 continue;
11456
11457 /* Push remaining network devices to init_net */
11458 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
11459 if (__dev_get_by_name(&init_net, fb_name))
11460 snprintf(fb_name, IFNAMSIZ, "dev%%d");
11385 err = dev_change_net_namespace(dev, &init_net, fb_name);
11461 err = dev_change_net_namespace(dev, &init_net, fb_name, 0);
11386 if (err) {
11387 pr_emerg("%s: failed to move %s to init_net: %d\n",
11388 __func__, dev->name, err);
11389 BUG();
11390 }
11391 }
11392 rtnl_unlock();
11393}

--- 157 unchanged lines hidden --- 		11462 if (err) {
11463 pr_emerg("%s: failed to move %s to init_net: %d\n",
11464 __func__, dev->name, err);
11465 BUG();
11466 }
11467 }
11468 rtnl_unlock();
11469}

--- 157 unchanged lines hidden ---