11751cc36SAlex Maftei (amaftei) // SPDX-License-Identifier: GPL-2.0-only
21751cc36SAlex Maftei (amaftei) /****************************************************************************
31751cc36SAlex Maftei (amaftei) * Driver for Solarflare network controllers and boards
41751cc36SAlex Maftei (amaftei) * Copyright 2018 Solarflare Communications Inc.
51751cc36SAlex Maftei (amaftei) *
61751cc36SAlex Maftei (amaftei) * This program is free software; you can redistribute it and/or modify it
71751cc36SAlex Maftei (amaftei) * under the terms of the GNU General Public License version 2 as published
81751cc36SAlex Maftei (amaftei) * by the Free Software Foundation, incorporated herein by reference.
91751cc36SAlex Maftei (amaftei) */
101751cc36SAlex Maftei (amaftei)
111751cc36SAlex Maftei (amaftei) #include "net_driver.h"
121751cc36SAlex Maftei (amaftei) #include <linux/module.h>
133d95b884SAlex Maftei (amaftei) #include <linux/iommu.h>
141751cc36SAlex Maftei (amaftei) #include "efx.h"
151751cc36SAlex Maftei (amaftei) #include "nic.h"
161751cc36SAlex Maftei (amaftei) #include "rx_common.h"
171751cc36SAlex Maftei (amaftei)
181751cc36SAlex Maftei (amaftei) /* This is the percentage fill level below which new RX descriptors
191751cc36SAlex Maftei (amaftei) * will be added to the RX descriptor ring.
201751cc36SAlex Maftei (amaftei) */
211751cc36SAlex Maftei (amaftei) static unsigned int rx_refill_threshold;
221751cc36SAlex Maftei (amaftei) module_param(rx_refill_threshold, uint, 0444);
231751cc36SAlex Maftei (amaftei) MODULE_PARM_DESC(rx_refill_threshold,
241751cc36SAlex Maftei (amaftei) "RX descriptor ring refill threshold (%)");
251751cc36SAlex Maftei (amaftei)
261751cc36SAlex Maftei (amaftei) /* RX maximum head room required.
271751cc36SAlex Maftei (amaftei) *
281751cc36SAlex Maftei (amaftei) * This must be at least 1 to prevent overflow, plus one packet-worth
291751cc36SAlex Maftei (amaftei) * to allow pipelined receives.
301751cc36SAlex Maftei (amaftei) */
311751cc36SAlex Maftei (amaftei) #define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS)
321751cc36SAlex Maftei (amaftei)
333d95b884SAlex Maftei (amaftei) /* Check the RX page recycle ring for a page that can be reused. */
efx_reuse_page(struct efx_rx_queue * rx_queue)343d95b884SAlex Maftei (amaftei) static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)
353d95b884SAlex Maftei (amaftei) {
363d95b884SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
373d95b884SAlex Maftei (amaftei) struct efx_rx_page_state *state;
383d95b884SAlex Maftei (amaftei) unsigned int index;
393d95b884SAlex Maftei (amaftei) struct page *page;
403d95b884SAlex Maftei (amaftei)
411d5a4742SMartin Habets if (unlikely(!rx_queue->page_ring))
421d5a4742SMartin Habets return NULL;
433d95b884SAlex Maftei (amaftei) index = rx_queue->page_remove & rx_queue->page_ptr_mask;
443d95b884SAlex Maftei (amaftei) page = rx_queue->page_ring[index];
453d95b884SAlex Maftei (amaftei) if (page == NULL)
463d95b884SAlex Maftei (amaftei) return NULL;
473d95b884SAlex Maftei (amaftei)
483d95b884SAlex Maftei (amaftei) rx_queue->page_ring[index] = NULL;
493d95b884SAlex Maftei (amaftei) /* page_remove cannot exceed page_add. */
503d95b884SAlex Maftei (amaftei) if (rx_queue->page_remove != rx_queue->page_add)
513d95b884SAlex Maftei (amaftei) ++rx_queue->page_remove;
523d95b884SAlex Maftei (amaftei)
533d95b884SAlex Maftei (amaftei) /* If page_count is 1 then we hold the only reference to this page. */
543d95b884SAlex Maftei (amaftei) if (page_count(page) == 1) {
553d95b884SAlex Maftei (amaftei) ++rx_queue->page_recycle_count;
563d95b884SAlex Maftei (amaftei) return page;
573d95b884SAlex Maftei (amaftei) } else {
583d95b884SAlex Maftei (amaftei) state = page_address(page);
593d95b884SAlex Maftei (amaftei) dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
603d95b884SAlex Maftei (amaftei) PAGE_SIZE << efx->rx_buffer_order,
613d95b884SAlex Maftei (amaftei) DMA_FROM_DEVICE);
623d95b884SAlex Maftei (amaftei) put_page(page);
633d95b884SAlex Maftei (amaftei) ++rx_queue->page_recycle_failed;
643d95b884SAlex Maftei (amaftei) }
653d95b884SAlex Maftei (amaftei)
663d95b884SAlex Maftei (amaftei) return NULL;
673d95b884SAlex Maftei (amaftei) }
683d95b884SAlex Maftei (amaftei)
693d95b884SAlex Maftei (amaftei) /* Attempt to recycle the page if there is an RX recycle ring; the page can
703d95b884SAlex Maftei (amaftei) * only be added if this is the final RX buffer, to prevent pages being used in
713d95b884SAlex Maftei (amaftei) * the descriptor ring and appearing in the recycle ring simultaneously.
723d95b884SAlex Maftei (amaftei) */
efx_recycle_rx_page(struct efx_channel * channel,struct efx_rx_buffer * rx_buf)733d95b884SAlex Maftei (amaftei) static void efx_recycle_rx_page(struct efx_channel *channel,
743d95b884SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf)
753d95b884SAlex Maftei (amaftei) {
763d95b884SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
773d95b884SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
783d95b884SAlex Maftei (amaftei) struct page *page = rx_buf->page;
793d95b884SAlex Maftei (amaftei) unsigned int index;
803d95b884SAlex Maftei (amaftei)
813d95b884SAlex Maftei (amaftei) /* Only recycle the page after processing the final buffer. */
823d95b884SAlex Maftei (amaftei) if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE))
833d95b884SAlex Maftei (amaftei) return;
843d95b884SAlex Maftei (amaftei)
853d95b884SAlex Maftei (amaftei) index = rx_queue->page_add & rx_queue->page_ptr_mask;
863d95b884SAlex Maftei (amaftei) if (rx_queue->page_ring[index] == NULL) {
873d95b884SAlex Maftei (amaftei) unsigned int read_index = rx_queue->page_remove &
883d95b884SAlex Maftei (amaftei) rx_queue->page_ptr_mask;
893d95b884SAlex Maftei (amaftei)
903d95b884SAlex Maftei (amaftei) /* The next slot in the recycle ring is available, but
913d95b884SAlex Maftei (amaftei) * increment page_remove if the read pointer currently
923d95b884SAlex Maftei (amaftei) * points here.
933d95b884SAlex Maftei (amaftei) */
943d95b884SAlex Maftei (amaftei) if (read_index == index)
953d95b884SAlex Maftei (amaftei) ++rx_queue->page_remove;
963d95b884SAlex Maftei (amaftei) rx_queue->page_ring[index] = page;
973d95b884SAlex Maftei (amaftei) ++rx_queue->page_add;
983d95b884SAlex Maftei (amaftei) return;
993d95b884SAlex Maftei (amaftei) }
1003d95b884SAlex Maftei (amaftei) ++rx_queue->page_recycle_full;
1013d95b884SAlex Maftei (amaftei) efx_unmap_rx_buffer(efx, rx_buf);
1023d95b884SAlex Maftei (amaftei) put_page(rx_buf->page);
1033d95b884SAlex Maftei (amaftei) }
1043d95b884SAlex Maftei (amaftei)
1053d95b884SAlex Maftei (amaftei) /* Recycle the pages that are used by buffers that have just been received. */
efx_recycle_rx_pages(struct efx_channel * channel,struct efx_rx_buffer * rx_buf,unsigned int n_frags)1063d95b884SAlex Maftei (amaftei) void efx_recycle_rx_pages(struct efx_channel *channel,
1073d95b884SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf,
1083d95b884SAlex Maftei (amaftei) unsigned int n_frags)
1093d95b884SAlex Maftei (amaftei) {
1103d95b884SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
1113d95b884SAlex Maftei (amaftei)
1121d5a4742SMartin Habets if (unlikely(!rx_queue->page_ring))
1131d5a4742SMartin Habets return;
1141d5a4742SMartin Habets
1153d95b884SAlex Maftei (amaftei) do {
1163d95b884SAlex Maftei (amaftei) efx_recycle_rx_page(channel, rx_buf);
1173d95b884SAlex Maftei (amaftei) rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
1183d95b884SAlex Maftei (amaftei) } while (--n_frags);
1193d95b884SAlex Maftei (amaftei) }
1203d95b884SAlex Maftei (amaftei)
efx_discard_rx_packet(struct efx_channel * channel,struct efx_rx_buffer * rx_buf,unsigned int n_frags)1213d95b884SAlex Maftei (amaftei) void efx_discard_rx_packet(struct efx_channel *channel,
1223d95b884SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf,
1233d95b884SAlex Maftei (amaftei) unsigned int n_frags)
1243d95b884SAlex Maftei (amaftei) {
1253d95b884SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
1263d95b884SAlex Maftei (amaftei)
1273d95b884SAlex Maftei (amaftei) efx_recycle_rx_pages(channel, rx_buf, n_frags);
1283d95b884SAlex Maftei (amaftei)
1293d95b884SAlex Maftei (amaftei) efx_free_rx_buffers(rx_queue, rx_buf, n_frags);
1303d95b884SAlex Maftei (amaftei) }
1313d95b884SAlex Maftei (amaftei)
efx_init_rx_recycle_ring(struct efx_rx_queue * rx_queue)1323d95b884SAlex Maftei (amaftei) static void efx_init_rx_recycle_ring(struct efx_rx_queue *rx_queue)
1333d95b884SAlex Maftei (amaftei) {
1343d95b884SAlex Maftei (amaftei) unsigned int bufs_in_recycle_ring, page_ring_size;
1353d95b884SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
1363d95b884SAlex Maftei (amaftei)
137000fe940SMartin Habets bufs_in_recycle_ring = efx_rx_recycle_ring_size(efx);
1383d95b884SAlex Maftei (amaftei) page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring /
1393d95b884SAlex Maftei (amaftei) efx->rx_bufs_per_page);
1403d95b884SAlex Maftei (amaftei) rx_queue->page_ring = kcalloc(page_ring_size,
1413d95b884SAlex Maftei (amaftei) sizeof(*rx_queue->page_ring), GFP_KERNEL);
142bdf1b5c3SJiasheng Jiang if (!rx_queue->page_ring)
143bdf1b5c3SJiasheng Jiang rx_queue->page_ptr_mask = 0;
144bdf1b5c3SJiasheng Jiang else
1453d95b884SAlex Maftei (amaftei) rx_queue->page_ptr_mask = page_ring_size - 1;
1463d95b884SAlex Maftei (amaftei) }
1473d95b884SAlex Maftei (amaftei)
efx_fini_rx_recycle_ring(struct efx_rx_queue * rx_queue)1483d95b884SAlex Maftei (amaftei) static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue)
1493d95b884SAlex Maftei (amaftei) {
1503d95b884SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
1513d95b884SAlex Maftei (amaftei) int i;
1523d95b884SAlex Maftei (amaftei)
153458f5d92SMartin Habets if (unlikely(!rx_queue->page_ring))
154458f5d92SMartin Habets return;
155458f5d92SMartin Habets
1563d95b884SAlex Maftei (amaftei) /* Unmap and release the pages in the recycle ring. Remove the ring. */
1573d95b884SAlex Maftei (amaftei) for (i = 0; i <= rx_queue->page_ptr_mask; i++) {
1583d95b884SAlex Maftei (amaftei) struct page *page = rx_queue->page_ring[i];
1593d95b884SAlex Maftei (amaftei) struct efx_rx_page_state *state;
1603d95b884SAlex Maftei (amaftei)
1613d95b884SAlex Maftei (amaftei) if (page == NULL)
1623d95b884SAlex Maftei (amaftei) continue;
1633d95b884SAlex Maftei (amaftei)
1643d95b884SAlex Maftei (amaftei) state = page_address(page);
1653d95b884SAlex Maftei (amaftei) dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
1663d95b884SAlex Maftei (amaftei) PAGE_SIZE << efx->rx_buffer_order,
1673d95b884SAlex Maftei (amaftei) DMA_FROM_DEVICE);
1683d95b884SAlex Maftei (amaftei) put_page(page);
1693d95b884SAlex Maftei (amaftei) }
1703d95b884SAlex Maftei (amaftei) kfree(rx_queue->page_ring);
1713d95b884SAlex Maftei (amaftei) rx_queue->page_ring = NULL;
1723d95b884SAlex Maftei (amaftei) }
1733d95b884SAlex Maftei (amaftei)
efx_fini_rx_buffer(struct efx_rx_queue * rx_queue,struct efx_rx_buffer * rx_buf)1741751cc36SAlex Maftei (amaftei) static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,
1751751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf)
1761751cc36SAlex Maftei (amaftei) {
1771751cc36SAlex Maftei (amaftei) /* Release the page reference we hold for the buffer. */
1781751cc36SAlex Maftei (amaftei) if (rx_buf->page)
1791751cc36SAlex Maftei (amaftei) put_page(rx_buf->page);
1801751cc36SAlex Maftei (amaftei)
1811751cc36SAlex Maftei (amaftei) /* If this is the last buffer in a page, unmap and free it. */
1821751cc36SAlex Maftei (amaftei) if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) {
1831751cc36SAlex Maftei (amaftei) efx_unmap_rx_buffer(rx_queue->efx, rx_buf);
1841751cc36SAlex Maftei (amaftei) efx_free_rx_buffers(rx_queue, rx_buf, 1);
1851751cc36SAlex Maftei (amaftei) }
1861751cc36SAlex Maftei (amaftei) rx_buf->page = NULL;
1871751cc36SAlex Maftei (amaftei) }
1881751cc36SAlex Maftei (amaftei)
efx_probe_rx_queue(struct efx_rx_queue * rx_queue)1891751cc36SAlex Maftei (amaftei) int efx_probe_rx_queue(struct efx_rx_queue *rx_queue)
1901751cc36SAlex Maftei (amaftei) {
1911751cc36SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
1921751cc36SAlex Maftei (amaftei) unsigned int entries;
1931751cc36SAlex Maftei (amaftei) int rc;
1941751cc36SAlex Maftei (amaftei)
1951751cc36SAlex Maftei (amaftei) /* Create the smallest power-of-two aligned ring */
1961751cc36SAlex Maftei (amaftei) entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE);
1971751cc36SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
1981751cc36SAlex Maftei (amaftei) rx_queue->ptr_mask = entries - 1;
1991751cc36SAlex Maftei (amaftei)
2001751cc36SAlex Maftei (amaftei) netif_dbg(efx, probe, efx->net_dev,
2011751cc36SAlex Maftei (amaftei) "creating RX queue %d size %#x mask %#x\n",
2021751cc36SAlex Maftei (amaftei) efx_rx_queue_index(rx_queue), efx->rxq_entries,
2031751cc36SAlex Maftei (amaftei) rx_queue->ptr_mask);
2041751cc36SAlex Maftei (amaftei)
2051751cc36SAlex Maftei (amaftei) /* Allocate RX buffers */
2061751cc36SAlex Maftei (amaftei) rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),
2071751cc36SAlex Maftei (amaftei) GFP_KERNEL);
2081751cc36SAlex Maftei (amaftei) if (!rx_queue->buffer)
2091751cc36SAlex Maftei (amaftei) return -ENOMEM;
2101751cc36SAlex Maftei (amaftei)
2111751cc36SAlex Maftei (amaftei) rc = efx_nic_probe_rx(rx_queue);
2121751cc36SAlex Maftei (amaftei) if (rc) {
2131751cc36SAlex Maftei (amaftei) kfree(rx_queue->buffer);
2141751cc36SAlex Maftei (amaftei) rx_queue->buffer = NULL;
2151751cc36SAlex Maftei (amaftei) }
2161751cc36SAlex Maftei (amaftei)
2171751cc36SAlex Maftei (amaftei) return rc;
2181751cc36SAlex Maftei (amaftei) }
2191751cc36SAlex Maftei (amaftei)
efx_init_rx_queue(struct efx_rx_queue * rx_queue)2201751cc36SAlex Maftei (amaftei) void efx_init_rx_queue(struct efx_rx_queue *rx_queue)
2211751cc36SAlex Maftei (amaftei) {
2221751cc36SAlex Maftei (amaftei) unsigned int max_fill, trigger, max_trigger;
2231751cc36SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
2241751cc36SAlex Maftei (amaftei) int rc = 0;
2251751cc36SAlex Maftei (amaftei)
2261751cc36SAlex Maftei (amaftei) netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
2271751cc36SAlex Maftei (amaftei) "initialising RX queue %d\n", efx_rx_queue_index(rx_queue));
2281751cc36SAlex Maftei (amaftei)
2291751cc36SAlex Maftei (amaftei) /* Initialise ptr fields */
2301751cc36SAlex Maftei (amaftei) rx_queue->added_count = 0;
2311751cc36SAlex Maftei (amaftei) rx_queue->notified_count = 0;
232e3951539SEdward Cree rx_queue->granted_count = 0;
2331751cc36SAlex Maftei (amaftei) rx_queue->removed_count = 0;
2341751cc36SAlex Maftei (amaftei) rx_queue->min_fill = -1U;
2351751cc36SAlex Maftei (amaftei) efx_init_rx_recycle_ring(rx_queue);
2361751cc36SAlex Maftei (amaftei)
2371751cc36SAlex Maftei (amaftei) rx_queue->page_remove = 0;
2381751cc36SAlex Maftei (amaftei) rx_queue->page_add = rx_queue->page_ptr_mask + 1;
2391751cc36SAlex Maftei (amaftei) rx_queue->page_recycle_count = 0;
2401751cc36SAlex Maftei (amaftei) rx_queue->page_recycle_failed = 0;
2411751cc36SAlex Maftei (amaftei) rx_queue->page_recycle_full = 0;
2421751cc36SAlex Maftei (amaftei)
2431751cc36SAlex Maftei (amaftei) /* Initialise limit fields */
2441751cc36SAlex Maftei (amaftei) max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;
2451751cc36SAlex Maftei (amaftei) max_trigger =
2461751cc36SAlex Maftei (amaftei) max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page;
2471751cc36SAlex Maftei (amaftei) if (rx_refill_threshold != 0) {
2481751cc36SAlex Maftei (amaftei) trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;
2491751cc36SAlex Maftei (amaftei) if (trigger > max_trigger)
2501751cc36SAlex Maftei (amaftei) trigger = max_trigger;
2511751cc36SAlex Maftei (amaftei) } else {
2521751cc36SAlex Maftei (amaftei) trigger = max_trigger;
2531751cc36SAlex Maftei (amaftei) }
2541751cc36SAlex Maftei (amaftei)
2551751cc36SAlex Maftei (amaftei) rx_queue->max_fill = max_fill;
2561751cc36SAlex Maftei (amaftei) rx_queue->fast_fill_trigger = trigger;
2571751cc36SAlex Maftei (amaftei) rx_queue->refill_enabled = true;
2581751cc36SAlex Maftei (amaftei)
2591751cc36SAlex Maftei (amaftei) /* Initialise XDP queue information */
2601751cc36SAlex Maftei (amaftei) rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev,
261b02e5a0eSBjörn Töpel rx_queue->core_index, 0);
2621751cc36SAlex Maftei (amaftei)
2631751cc36SAlex Maftei (amaftei) if (rc) {
2641751cc36SAlex Maftei (amaftei) netif_err(efx, rx_err, efx->net_dev,
2651751cc36SAlex Maftei (amaftei) "Failure to initialise XDP queue information rc=%d\n",
2661751cc36SAlex Maftei (amaftei) rc);
2671751cc36SAlex Maftei (amaftei) efx->xdp_rxq_info_failed = true;
2681751cc36SAlex Maftei (amaftei) } else {
2691751cc36SAlex Maftei (amaftei) rx_queue->xdp_rxq_info_valid = true;
2701751cc36SAlex Maftei (amaftei) }
2711751cc36SAlex Maftei (amaftei)
2721751cc36SAlex Maftei (amaftei) /* Set up RX descriptor ring */
2731751cc36SAlex Maftei (amaftei) efx_nic_init_rx(rx_queue);
2741751cc36SAlex Maftei (amaftei) }
2751751cc36SAlex Maftei (amaftei)
efx_fini_rx_queue(struct efx_rx_queue * rx_queue)2761751cc36SAlex Maftei (amaftei) void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)
2771751cc36SAlex Maftei (amaftei) {
2781751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf;
2791751cc36SAlex Maftei (amaftei) int i;
2801751cc36SAlex Maftei (amaftei)
2811751cc36SAlex Maftei (amaftei) netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
2821751cc36SAlex Maftei (amaftei) "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));
2831751cc36SAlex Maftei (amaftei)
2841751cc36SAlex Maftei (amaftei) del_timer_sync(&rx_queue->slow_fill);
285e3951539SEdward Cree if (rx_queue->grant_credits)
286e3951539SEdward Cree flush_work(&rx_queue->grant_work);
2871751cc36SAlex Maftei (amaftei)
2881751cc36SAlex Maftei (amaftei) /* Release RX buffers from the current read ptr to the write ptr */
2891751cc36SAlex Maftei (amaftei) if (rx_queue->buffer) {
2901751cc36SAlex Maftei (amaftei) for (i = rx_queue->removed_count; i < rx_queue->added_count;
2911751cc36SAlex Maftei (amaftei) i++) {
2921751cc36SAlex Maftei (amaftei) unsigned int index = i & rx_queue->ptr_mask;
2931751cc36SAlex Maftei (amaftei)
2941751cc36SAlex Maftei (amaftei) rx_buf = efx_rx_buffer(rx_queue, index);
2951751cc36SAlex Maftei (amaftei) efx_fini_rx_buffer(rx_queue, rx_buf);
2961751cc36SAlex Maftei (amaftei) }
2971751cc36SAlex Maftei (amaftei) }
2981751cc36SAlex Maftei (amaftei)
2993d95b884SAlex Maftei (amaftei) efx_fini_rx_recycle_ring(rx_queue);
3001751cc36SAlex Maftei (amaftei)
3011751cc36SAlex Maftei (amaftei) if (rx_queue->xdp_rxq_info_valid)
3021751cc36SAlex Maftei (amaftei) xdp_rxq_info_unreg(&rx_queue->xdp_rxq_info);
3031751cc36SAlex Maftei (amaftei)
3041751cc36SAlex Maftei (amaftei) rx_queue->xdp_rxq_info_valid = false;
3051751cc36SAlex Maftei (amaftei) }
3061751cc36SAlex Maftei (amaftei)
efx_remove_rx_queue(struct efx_rx_queue * rx_queue)3071751cc36SAlex Maftei (amaftei) void efx_remove_rx_queue(struct efx_rx_queue *rx_queue)
3081751cc36SAlex Maftei (amaftei) {
3091751cc36SAlex Maftei (amaftei) netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
3101751cc36SAlex Maftei (amaftei) "destroying RX queue %d\n", efx_rx_queue_index(rx_queue));
3111751cc36SAlex Maftei (amaftei)
3121751cc36SAlex Maftei (amaftei) efx_nic_remove_rx(rx_queue);
3131751cc36SAlex Maftei (amaftei)
3141751cc36SAlex Maftei (amaftei) kfree(rx_queue->buffer);
3151751cc36SAlex Maftei (amaftei) rx_queue->buffer = NULL;
3161751cc36SAlex Maftei (amaftei) }
3171751cc36SAlex Maftei (amaftei)
3181751cc36SAlex Maftei (amaftei) /* Unmap a DMA-mapped page. This function is only called for the final RX
3191751cc36SAlex Maftei (amaftei) * buffer in a page.
3201751cc36SAlex Maftei (amaftei) */
efx_unmap_rx_buffer(struct efx_nic * efx,struct efx_rx_buffer * rx_buf)3211751cc36SAlex Maftei (amaftei) void efx_unmap_rx_buffer(struct efx_nic *efx,
3221751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf)
3231751cc36SAlex Maftei (amaftei) {
3241751cc36SAlex Maftei (amaftei) struct page *page = rx_buf->page;
3251751cc36SAlex Maftei (amaftei)
3261751cc36SAlex Maftei (amaftei) if (page) {
3271751cc36SAlex Maftei (amaftei) struct efx_rx_page_state *state = page_address(page);
3281751cc36SAlex Maftei (amaftei)
3291751cc36SAlex Maftei (amaftei) dma_unmap_page(&efx->pci_dev->dev,
3301751cc36SAlex Maftei (amaftei) state->dma_addr,
3311751cc36SAlex Maftei (amaftei) PAGE_SIZE << efx->rx_buffer_order,
3321751cc36SAlex Maftei (amaftei) DMA_FROM_DEVICE);
3331751cc36SAlex Maftei (amaftei) }
3341751cc36SAlex Maftei (amaftei) }
3351751cc36SAlex Maftei (amaftei)
efx_free_rx_buffers(struct efx_rx_queue * rx_queue,struct efx_rx_buffer * rx_buf,unsigned int num_bufs)3361751cc36SAlex Maftei (amaftei) void efx_free_rx_buffers(struct efx_rx_queue *rx_queue,
3371751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf,
3381751cc36SAlex Maftei (amaftei) unsigned int num_bufs)
3391751cc36SAlex Maftei (amaftei) {
3401751cc36SAlex Maftei (amaftei) do {
3411751cc36SAlex Maftei (amaftei) if (rx_buf->page) {
3421751cc36SAlex Maftei (amaftei) put_page(rx_buf->page);
3431751cc36SAlex Maftei (amaftei) rx_buf->page = NULL;
3441751cc36SAlex Maftei (amaftei) }
3451751cc36SAlex Maftei (amaftei) rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
3461751cc36SAlex Maftei (amaftei) } while (--num_bufs);
3471751cc36SAlex Maftei (amaftei) }
3481751cc36SAlex Maftei (amaftei)
efx_rx_slow_fill(struct timer_list * t)3491751cc36SAlex Maftei (amaftei) void efx_rx_slow_fill(struct timer_list *t)
3501751cc36SAlex Maftei (amaftei) {
3511751cc36SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = from_timer(rx_queue, t, slow_fill);
3521751cc36SAlex Maftei (amaftei)
3531751cc36SAlex Maftei (amaftei) /* Post an event to cause NAPI to run and refill the queue */
3541751cc36SAlex Maftei (amaftei) efx_nic_generate_fill_event(rx_queue);
3551751cc36SAlex Maftei (amaftei) ++rx_queue->slow_fill_count;
3561751cc36SAlex Maftei (amaftei) }
3571751cc36SAlex Maftei (amaftei)
efx_schedule_slow_fill(struct efx_rx_queue * rx_queue)3581751cc36SAlex Maftei (amaftei) void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
3591751cc36SAlex Maftei (amaftei) {
3601751cc36SAlex Maftei (amaftei) mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10));
3611751cc36SAlex Maftei (amaftei) }
3621751cc36SAlex Maftei (amaftei)
3631751cc36SAlex Maftei (amaftei) /* efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers
3641751cc36SAlex Maftei (amaftei) *
3651751cc36SAlex Maftei (amaftei) * @rx_queue: Efx RX queue
3661751cc36SAlex Maftei (amaftei) *
3671751cc36SAlex Maftei (amaftei) * This allocates a batch of pages, maps them for DMA, and populates
3681751cc36SAlex Maftei (amaftei) * struct efx_rx_buffers for each one. Return a negative error code or
3691751cc36SAlex Maftei (amaftei) * 0 on success. If a single page can be used for multiple buffers,
3701751cc36SAlex Maftei (amaftei) * then the page will either be inserted fully, or not at all.
3711751cc36SAlex Maftei (amaftei) */
efx_init_rx_buffers(struct efx_rx_queue * rx_queue,bool atomic)3721751cc36SAlex Maftei (amaftei) static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue, bool atomic)
3731751cc36SAlex Maftei (amaftei) {
3741751cc36SAlex Maftei (amaftei) unsigned int page_offset, index, count;
3751751cc36SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
3761751cc36SAlex Maftei (amaftei) struct efx_rx_page_state *state;
3771751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf;
3781751cc36SAlex Maftei (amaftei) dma_addr_t dma_addr;
3791751cc36SAlex Maftei (amaftei) struct page *page;
3801751cc36SAlex Maftei (amaftei)
3811751cc36SAlex Maftei (amaftei) count = 0;
3821751cc36SAlex Maftei (amaftei) do {
3831751cc36SAlex Maftei (amaftei) page = efx_reuse_page(rx_queue);
3841751cc36SAlex Maftei (amaftei) if (page == NULL) {
3851751cc36SAlex Maftei (amaftei) page = alloc_pages(__GFP_COMP |
3861751cc36SAlex Maftei (amaftei) (atomic ? GFP_ATOMIC : GFP_KERNEL),
3871751cc36SAlex Maftei (amaftei) efx->rx_buffer_order);
3881751cc36SAlex Maftei (amaftei) if (unlikely(page == NULL))
3891751cc36SAlex Maftei (amaftei) return -ENOMEM;
3901751cc36SAlex Maftei (amaftei) dma_addr =
3911751cc36SAlex Maftei (amaftei) dma_map_page(&efx->pci_dev->dev, page, 0,
3921751cc36SAlex Maftei (amaftei) PAGE_SIZE << efx->rx_buffer_order,
3931751cc36SAlex Maftei (amaftei) DMA_FROM_DEVICE);
3941751cc36SAlex Maftei (amaftei) if (unlikely(dma_mapping_error(&efx->pci_dev->dev,
3951751cc36SAlex Maftei (amaftei) dma_addr))) {
3961751cc36SAlex Maftei (amaftei) __free_pages(page, efx->rx_buffer_order);
3971751cc36SAlex Maftei (amaftei) return -EIO;
3981751cc36SAlex Maftei (amaftei) }
3991751cc36SAlex Maftei (amaftei) state = page_address(page);
4001751cc36SAlex Maftei (amaftei) state->dma_addr = dma_addr;
4011751cc36SAlex Maftei (amaftei) } else {
4021751cc36SAlex Maftei (amaftei) state = page_address(page);
4031751cc36SAlex Maftei (amaftei) dma_addr = state->dma_addr;
4041751cc36SAlex Maftei (amaftei) }
4051751cc36SAlex Maftei (amaftei)
4061751cc36SAlex Maftei (amaftei) dma_addr += sizeof(struct efx_rx_page_state);
4071751cc36SAlex Maftei (amaftei) page_offset = sizeof(struct efx_rx_page_state);
4081751cc36SAlex Maftei (amaftei)
4091751cc36SAlex Maftei (amaftei) do {
4101751cc36SAlex Maftei (amaftei) index = rx_queue->added_count & rx_queue->ptr_mask;
4111751cc36SAlex Maftei (amaftei) rx_buf = efx_rx_buffer(rx_queue, index);
4121751cc36SAlex Maftei (amaftei) rx_buf->dma_addr = dma_addr + efx->rx_ip_align +
41386e85bf6SJesper Dangaard Brouer EFX_XDP_HEADROOM;
4141751cc36SAlex Maftei (amaftei) rx_buf->page = page;
4151751cc36SAlex Maftei (amaftei) rx_buf->page_offset = page_offset + efx->rx_ip_align +
41686e85bf6SJesper Dangaard Brouer EFX_XDP_HEADROOM;
4171751cc36SAlex Maftei (amaftei) rx_buf->len = efx->rx_dma_len;
4181751cc36SAlex Maftei (amaftei) rx_buf->flags = 0;
4191751cc36SAlex Maftei (amaftei) ++rx_queue->added_count;
4201751cc36SAlex Maftei (amaftei) get_page(page);
4211751cc36SAlex Maftei (amaftei) dma_addr += efx->rx_page_buf_step;
4221751cc36SAlex Maftei (amaftei) page_offset += efx->rx_page_buf_step;
4231751cc36SAlex Maftei (amaftei) } while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE);
4241751cc36SAlex Maftei (amaftei)
4251751cc36SAlex Maftei (amaftei) rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE;
4261751cc36SAlex Maftei (amaftei) } while (++count < efx->rx_pages_per_batch);
4271751cc36SAlex Maftei (amaftei)
4281751cc36SAlex Maftei (amaftei) return 0;
4291751cc36SAlex Maftei (amaftei) }
4301751cc36SAlex Maftei (amaftei)
efx_rx_config_page_split(struct efx_nic * efx)4311751cc36SAlex Maftei (amaftei) void efx_rx_config_page_split(struct efx_nic *efx)
4321751cc36SAlex Maftei (amaftei) {
4331751cc36SAlex Maftei (amaftei) efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + efx->rx_ip_align +
43486e85bf6SJesper Dangaard Brouer EFX_XDP_HEADROOM + EFX_XDP_TAILROOM,
4351751cc36SAlex Maftei (amaftei) EFX_RX_BUF_ALIGNMENT);
4361751cc36SAlex Maftei (amaftei) efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 :
4371751cc36SAlex Maftei (amaftei) ((PAGE_SIZE - sizeof(struct efx_rx_page_state)) /
4381751cc36SAlex Maftei (amaftei) efx->rx_page_buf_step);
4391751cc36SAlex Maftei (amaftei) efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) /
4401751cc36SAlex Maftei (amaftei) efx->rx_bufs_per_page;
4411751cc36SAlex Maftei (amaftei) efx->rx_pages_per_batch = DIV_ROUND_UP(EFX_RX_PREFERRED_BATCH,
4421751cc36SAlex Maftei (amaftei) efx->rx_bufs_per_page);
4431751cc36SAlex Maftei (amaftei) }
4441751cc36SAlex Maftei (amaftei)
4451751cc36SAlex Maftei (amaftei) /* efx_fast_push_rx_descriptors - push new RX descriptors quickly
4461751cc36SAlex Maftei (amaftei) * @rx_queue: RX descriptor queue
4471751cc36SAlex Maftei (amaftei) *
4481751cc36SAlex Maftei (amaftei) * This will aim to fill the RX descriptor queue up to
4491751cc36SAlex Maftei (amaftei) * @rx_queue->@max_fill. If there is insufficient atomic
4501751cc36SAlex Maftei (amaftei) * memory to do so, a slow fill will be scheduled.
4511751cc36SAlex Maftei (amaftei) *
4521751cc36SAlex Maftei (amaftei) * The caller must provide serialisation (none is used here). In practise,
4531751cc36SAlex Maftei (amaftei) * this means this function must run from the NAPI handler, or be called
4541751cc36SAlex Maftei (amaftei) * when NAPI is disabled.
4551751cc36SAlex Maftei (amaftei) */
efx_fast_push_rx_descriptors(struct efx_rx_queue * rx_queue,bool atomic)4561751cc36SAlex Maftei (amaftei) void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue, bool atomic)
4571751cc36SAlex Maftei (amaftei) {
4581751cc36SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
4591751cc36SAlex Maftei (amaftei) unsigned int fill_level, batch_size;
4601751cc36SAlex Maftei (amaftei) int space, rc = 0;
4611751cc36SAlex Maftei (amaftei)
4621751cc36SAlex Maftei (amaftei) if (!rx_queue->refill_enabled)
4631751cc36SAlex Maftei (amaftei) return;
4641751cc36SAlex Maftei (amaftei)
4651751cc36SAlex Maftei (amaftei) /* Calculate current fill level, and exit if we don't need to fill */
4661751cc36SAlex Maftei (amaftei) fill_level = (rx_queue->added_count - rx_queue->removed_count);
4671751cc36SAlex Maftei (amaftei) EFX_WARN_ON_ONCE_PARANOID(fill_level > rx_queue->efx->rxq_entries);
4681751cc36SAlex Maftei (amaftei) if (fill_level >= rx_queue->fast_fill_trigger)
4691751cc36SAlex Maftei (amaftei) goto out;
4701751cc36SAlex Maftei (amaftei)
4711751cc36SAlex Maftei (amaftei) /* Record minimum fill level */
4721751cc36SAlex Maftei (amaftei) if (unlikely(fill_level < rx_queue->min_fill)) {
4731751cc36SAlex Maftei (amaftei) if (fill_level)
4741751cc36SAlex Maftei (amaftei) rx_queue->min_fill = fill_level;
4751751cc36SAlex Maftei (amaftei) }
4761751cc36SAlex Maftei (amaftei)
4771751cc36SAlex Maftei (amaftei) batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page;
4781751cc36SAlex Maftei (amaftei) space = rx_queue->max_fill - fill_level;
4791751cc36SAlex Maftei (amaftei) EFX_WARN_ON_ONCE_PARANOID(space < batch_size);
4801751cc36SAlex Maftei (amaftei)
4811751cc36SAlex Maftei (amaftei) netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
4821751cc36SAlex Maftei (amaftei) "RX queue %d fast-filling descriptor ring from"
4831751cc36SAlex Maftei (amaftei) " level %d to level %d\n",
4841751cc36SAlex Maftei (amaftei) efx_rx_queue_index(rx_queue), fill_level,
4851751cc36SAlex Maftei (amaftei) rx_queue->max_fill);
4861751cc36SAlex Maftei (amaftei)
4871751cc36SAlex Maftei (amaftei) do {
4881751cc36SAlex Maftei (amaftei) rc = efx_init_rx_buffers(rx_queue, atomic);
4891751cc36SAlex Maftei (amaftei) if (unlikely(rc)) {
4901751cc36SAlex Maftei (amaftei) /* Ensure that we don't leave the rx queue empty */
4911751cc36SAlex Maftei (amaftei) efx_schedule_slow_fill(rx_queue);
4921751cc36SAlex Maftei (amaftei) goto out;
4931751cc36SAlex Maftei (amaftei) }
4941751cc36SAlex Maftei (amaftei) } while ((space -= batch_size) >= batch_size);
4951751cc36SAlex Maftei (amaftei)
4961751cc36SAlex Maftei (amaftei) netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
4971751cc36SAlex Maftei (amaftei) "RX queue %d fast-filled descriptor ring "
4981751cc36SAlex Maftei (amaftei) "to level %d\n", efx_rx_queue_index(rx_queue),
4991751cc36SAlex Maftei (amaftei) rx_queue->added_count - rx_queue->removed_count);
5001751cc36SAlex Maftei (amaftei)
5011751cc36SAlex Maftei (amaftei) out:
5021751cc36SAlex Maftei (amaftei) if (rx_queue->notified_count != rx_queue->added_count)
5031751cc36SAlex Maftei (amaftei) efx_nic_notify_rx_desc(rx_queue);
5041751cc36SAlex Maftei (amaftei) }
5053d95b884SAlex Maftei (amaftei)
5063d95b884SAlex Maftei (amaftei) /* Pass a received packet up through GRO. GRO can handle pages
5073d95b884SAlex Maftei (amaftei) * regardless of checksum state and skbs with a good checksum.
5083d95b884SAlex Maftei (amaftei) */
5093d95b884SAlex Maftei (amaftei) void
efx_rx_packet_gro(struct efx_channel * channel,struct efx_rx_buffer * rx_buf,unsigned int n_frags,u8 * eh,__wsum csum)5103d95b884SAlex Maftei (amaftei) efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf,
5114d9c0a2dSEdward Cree unsigned int n_frags, u8 *eh, __wsum csum)
5123d95b884SAlex Maftei (amaftei) {
5133d95b884SAlex Maftei (amaftei) struct napi_struct *napi = &channel->napi_str;
5143d95b884SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx;
5153d95b884SAlex Maftei (amaftei) struct sk_buff *skb;
5163d95b884SAlex Maftei (amaftei)
5173d95b884SAlex Maftei (amaftei) skb = napi_get_frags(napi);
5183d95b884SAlex Maftei (amaftei) if (unlikely(!skb)) {
5193d95b884SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue;
5203d95b884SAlex Maftei (amaftei)
5213d95b884SAlex Maftei (amaftei) rx_queue = efx_channel_get_rx_queue(channel);
5223d95b884SAlex Maftei (amaftei) efx_free_rx_buffers(rx_queue, rx_buf, n_frags);
5233d95b884SAlex Maftei (amaftei) return;
5243d95b884SAlex Maftei (amaftei) }
5253d95b884SAlex Maftei (amaftei)
52606888543SEdward Cree if (efx->net_dev->features & NETIF_F_RXHASH &&
52706888543SEdward Cree efx_rx_buf_hash_valid(efx, eh))
5283d95b884SAlex Maftei (amaftei) skb_set_hash(skb, efx_rx_buf_hash(efx, eh),
5293d95b884SAlex Maftei (amaftei) PKT_HASH_TYPE_L3);
5304d9c0a2dSEdward Cree if (csum) {
5314d9c0a2dSEdward Cree skb->csum = csum;
5324d9c0a2dSEdward Cree skb->ip_summed = CHECKSUM_COMPLETE;
5334d9c0a2dSEdward Cree } else {
5343d95b884SAlex Maftei (amaftei) skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ?
5353d95b884SAlex Maftei (amaftei) CHECKSUM_UNNECESSARY : CHECKSUM_NONE);
5364d9c0a2dSEdward Cree }
5373d95b884SAlex Maftei (amaftei) skb->csum_level = !!(rx_buf->flags & EFX_RX_PKT_CSUM_LEVEL);
5383d95b884SAlex Maftei (amaftei)
5393d95b884SAlex Maftei (amaftei) for (;;) {
5403d95b884SAlex Maftei (amaftei) skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5413d95b884SAlex Maftei (amaftei) rx_buf->page, rx_buf->page_offset,
5423d95b884SAlex Maftei (amaftei) rx_buf->len);
5433d95b884SAlex Maftei (amaftei) rx_buf->page = NULL;
5443d95b884SAlex Maftei (amaftei) skb->len += rx_buf->len;
5453d95b884SAlex Maftei (amaftei) if (skb_shinfo(skb)->nr_frags == n_frags)
5463d95b884SAlex Maftei (amaftei) break;
5473d95b884SAlex Maftei (amaftei)
5483d95b884SAlex Maftei (amaftei) rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
5493d95b884SAlex Maftei (amaftei) }
5503d95b884SAlex Maftei (amaftei)
5513d95b884SAlex Maftei (amaftei) skb->data_len = skb->len;
5523d95b884SAlex Maftei (amaftei) skb->truesize += n_frags * efx->rx_buffer_truesize;
5533d95b884SAlex Maftei (amaftei)
5543d95b884SAlex Maftei (amaftei) skb_record_rx_queue(skb, channel->rx_queue.core_index);
5553d95b884SAlex Maftei (amaftei)
5563d95b884SAlex Maftei (amaftei) napi_gro_frags(napi);
5573d95b884SAlex Maftei (amaftei) }
558960f1627SAlex Maftei (amaftei)
559960f1627SAlex Maftei (amaftei) /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because
560960f1627SAlex Maftei (amaftei) * (a) this is an infrequent control-plane operation and (b) n is small (max 64)
561960f1627SAlex Maftei (amaftei) */
efx_alloc_rss_context_entry(struct efx_nic * efx)562960f1627SAlex Maftei (amaftei) struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx)
563960f1627SAlex Maftei (amaftei) {
564960f1627SAlex Maftei (amaftei) struct list_head *head = &efx->rss_context.list;
565960f1627SAlex Maftei (amaftei) struct efx_rss_context *ctx, *new;
566960f1627SAlex Maftei (amaftei) u32 id = 1; /* Don't use zero, that refers to the master RSS context */
567960f1627SAlex Maftei (amaftei)
568960f1627SAlex Maftei (amaftei) WARN_ON(!mutex_is_locked(&efx->rss_lock));
569960f1627SAlex Maftei (amaftei)
570960f1627SAlex Maftei (amaftei) /* Search for first gap in the numbering */
571960f1627SAlex Maftei (amaftei) list_for_each_entry(ctx, head, list) {
572960f1627SAlex Maftei (amaftei) if (ctx->user_id != id)
573960f1627SAlex Maftei (amaftei) break;
574960f1627SAlex Maftei (amaftei) id++;
575960f1627SAlex Maftei (amaftei) /* Check for wrap. If this happens, we have nearly 2^32
576960f1627SAlex Maftei (amaftei) * allocated RSS contexts, which seems unlikely.
577960f1627SAlex Maftei (amaftei) */
578960f1627SAlex Maftei (amaftei) if (WARN_ON_ONCE(!id))
579960f1627SAlex Maftei (amaftei) return NULL;
580960f1627SAlex Maftei (amaftei) }
581960f1627SAlex Maftei (amaftei)
582960f1627SAlex Maftei (amaftei) /* Create the new entry */
583960f1627SAlex Maftei (amaftei) new = kmalloc(sizeof(*new), GFP_KERNEL);
584960f1627SAlex Maftei (amaftei) if (!new)
585960f1627SAlex Maftei (amaftei) return NULL;
586f7226e0fSAlex Maftei (amaftei) new->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
587960f1627SAlex Maftei (amaftei) new->rx_hash_udp_4tuple = false;
588960f1627SAlex Maftei (amaftei)
589960f1627SAlex Maftei (amaftei) /* Insert the new entry into the gap */
590960f1627SAlex Maftei (amaftei) new->user_id = id;
591960f1627SAlex Maftei (amaftei) list_add_tail(&new->list, &ctx->list);
592960f1627SAlex Maftei (amaftei) return new;
593960f1627SAlex Maftei (amaftei) }
594960f1627SAlex Maftei (amaftei)
efx_find_rss_context_entry(struct efx_nic * efx,u32 id)595960f1627SAlex Maftei (amaftei) struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id)
596960f1627SAlex Maftei (amaftei) {
597960f1627SAlex Maftei (amaftei) struct list_head *head = &efx->rss_context.list;
598960f1627SAlex Maftei (amaftei) struct efx_rss_context *ctx;
599960f1627SAlex Maftei (amaftei)
600960f1627SAlex Maftei (amaftei) WARN_ON(!mutex_is_locked(&efx->rss_lock));
601960f1627SAlex Maftei (amaftei)
602960f1627SAlex Maftei (amaftei) list_for_each_entry(ctx, head, list)
603960f1627SAlex Maftei (amaftei) if (ctx->user_id == id)
604960f1627SAlex Maftei (amaftei) return ctx;
605960f1627SAlex Maftei (amaftei) return NULL;
606960f1627SAlex Maftei (amaftei) }
607960f1627SAlex Maftei (amaftei)
efx_free_rss_context_entry(struct efx_rss_context * ctx)608960f1627SAlex Maftei (amaftei) void efx_free_rss_context_entry(struct efx_rss_context *ctx)
609960f1627SAlex Maftei (amaftei) {
610960f1627SAlex Maftei (amaftei) list_del(&ctx->list);
611960f1627SAlex Maftei (amaftei) kfree(ctx);
612960f1627SAlex Maftei (amaftei) }
613960f1627SAlex Maftei (amaftei)
efx_set_default_rx_indir_table(struct efx_nic * efx,struct efx_rss_context * ctx)614960f1627SAlex Maftei (amaftei) void efx_set_default_rx_indir_table(struct efx_nic *efx,
615960f1627SAlex Maftei (amaftei) struct efx_rss_context *ctx)
616960f1627SAlex Maftei (amaftei) {
617960f1627SAlex Maftei (amaftei) size_t i;
618960f1627SAlex Maftei (amaftei)
619960f1627SAlex Maftei (amaftei) for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
620960f1627SAlex Maftei (amaftei) ctx->rx_indir_table[i] =
621960f1627SAlex Maftei (amaftei) ethtool_rxfh_indir_default(i, efx->rss_spread);
622960f1627SAlex Maftei (amaftei) }
623f7226e0fSAlex Maftei (amaftei)
624f7226e0fSAlex Maftei (amaftei) /**
625f7226e0fSAlex Maftei (amaftei) * efx_filter_is_mc_recipient - test whether spec is a multicast recipient
626f7226e0fSAlex Maftei (amaftei) * @spec: Specification to test
627f7226e0fSAlex Maftei (amaftei) *
628f7226e0fSAlex Maftei (amaftei) * Return: %true if the specification is a non-drop RX filter that
629f7226e0fSAlex Maftei (amaftei) * matches a local MAC address I/G bit value of 1 or matches a local
630f7226e0fSAlex Maftei (amaftei) * IPv4 or IPv6 address value in the respective multicast address
631f7226e0fSAlex Maftei (amaftei) * range. Otherwise %false.
632f7226e0fSAlex Maftei (amaftei) */
efx_filter_is_mc_recipient(const struct efx_filter_spec * spec)633f7226e0fSAlex Maftei (amaftei) bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec)
634f7226e0fSAlex Maftei (amaftei) {
635f7226e0fSAlex Maftei (amaftei) if (!(spec->flags & EFX_FILTER_FLAG_RX) ||
636f7226e0fSAlex Maftei (amaftei) spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP)
637f7226e0fSAlex Maftei (amaftei) return false;
638f7226e0fSAlex Maftei (amaftei)
639f7226e0fSAlex Maftei (amaftei) if (spec->match_flags &
640f7226e0fSAlex Maftei (amaftei) (EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_MAC_IG) &&
641f7226e0fSAlex Maftei (amaftei) is_multicast_ether_addr(spec->loc_mac))
642f7226e0fSAlex Maftei (amaftei) return true;
643f7226e0fSAlex Maftei (amaftei)
644f7226e0fSAlex Maftei (amaftei) if ((spec->match_flags &
645f7226e0fSAlex Maftei (amaftei) (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
646f7226e0fSAlex Maftei (amaftei) (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
647f7226e0fSAlex Maftei (amaftei) if (spec->ether_type == htons(ETH_P_IP) &&
648f7226e0fSAlex Maftei (amaftei) ipv4_is_multicast(spec->loc_host[0]))
649f7226e0fSAlex Maftei (amaftei) return true;
650f7226e0fSAlex Maftei (amaftei) if (spec->ether_type == htons(ETH_P_IPV6) &&
651f7226e0fSAlex Maftei (amaftei) ((const u8 *)spec->loc_host)[0] == 0xff)
652f7226e0fSAlex Maftei (amaftei) return true;
653f7226e0fSAlex Maftei (amaftei) }
654f7226e0fSAlex Maftei (amaftei)
655f7226e0fSAlex Maftei (amaftei) return false;
656f7226e0fSAlex Maftei (amaftei) }
657f7226e0fSAlex Maftei (amaftei)
efx_filter_spec_equal(const struct efx_filter_spec * left,const struct efx_filter_spec * right)658f7226e0fSAlex Maftei (amaftei) bool efx_filter_spec_equal(const struct efx_filter_spec *left,
659f7226e0fSAlex Maftei (amaftei) const struct efx_filter_spec *right)
660f7226e0fSAlex Maftei (amaftei) {
661f7226e0fSAlex Maftei (amaftei) if ((left->match_flags ^ right->match_flags) |
662f7226e0fSAlex Maftei (amaftei) ((left->flags ^ right->flags) &
663f7226e0fSAlex Maftei (amaftei) (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))
664f7226e0fSAlex Maftei (amaftei) return false;
665f7226e0fSAlex Maftei (amaftei)
666c2bf23e4SPieter Jansen van Vuuren return memcmp(&left->vport_id, &right->vport_id,
667f7226e0fSAlex Maftei (amaftei) sizeof(struct efx_filter_spec) -
668c2bf23e4SPieter Jansen van Vuuren offsetof(struct efx_filter_spec, vport_id)) == 0;
669f7226e0fSAlex Maftei (amaftei) }
670f7226e0fSAlex Maftei (amaftei)
efx_filter_spec_hash(const struct efx_filter_spec * spec)671f7226e0fSAlex Maftei (amaftei) u32 efx_filter_spec_hash(const struct efx_filter_spec *spec)
672f7226e0fSAlex Maftei (amaftei) {
673c2bf23e4SPieter Jansen van Vuuren BUILD_BUG_ON(offsetof(struct efx_filter_spec, vport_id) & 3);
674c2bf23e4SPieter Jansen van Vuuren return jhash2((const u32 *)&spec->vport_id,
675f7226e0fSAlex Maftei (amaftei) (sizeof(struct efx_filter_spec) -
676c2bf23e4SPieter Jansen van Vuuren offsetof(struct efx_filter_spec, vport_id)) / 4,
677f7226e0fSAlex Maftei (amaftei) 0);
678f7226e0fSAlex Maftei (amaftei) }
679f7226e0fSAlex Maftei (amaftei)
680f7226e0fSAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL
efx_rps_check_rule(struct efx_arfs_rule * rule,unsigned int filter_idx,bool * force)681f7226e0fSAlex Maftei (amaftei) bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx,
682f7226e0fSAlex Maftei (amaftei) bool *force)
683f7226e0fSAlex Maftei (amaftei) {
684f7226e0fSAlex Maftei (amaftei) if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) {
685f7226e0fSAlex Maftei (amaftei) /* ARFS is currently updating this entry, leave it */
686f7226e0fSAlex Maftei (amaftei) return false;
687f7226e0fSAlex Maftei (amaftei) }
688f7226e0fSAlex Maftei (amaftei) if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) {
689f7226e0fSAlex Maftei (amaftei) /* ARFS tried and failed to update this, so it's probably out
690f7226e0fSAlex Maftei (amaftei) * of date. Remove the filter and the ARFS rule entry.
691f7226e0fSAlex Maftei (amaftei) */
692f7226e0fSAlex Maftei (amaftei) rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
693f7226e0fSAlex Maftei (amaftei) *force = true;
694f7226e0fSAlex Maftei (amaftei) return true;
695f7226e0fSAlex Maftei (amaftei) } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */
696f7226e0fSAlex Maftei (amaftei) /* ARFS has moved on, so old filter is not needed. Since we did
697f7226e0fSAlex Maftei (amaftei) * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will
698f7226e0fSAlex Maftei (amaftei) * not be removed by efx_rps_hash_del() subsequently.
699f7226e0fSAlex Maftei (amaftei) */
700f7226e0fSAlex Maftei (amaftei) *force = true;
701f7226e0fSAlex Maftei (amaftei) return true;
702f7226e0fSAlex Maftei (amaftei) }
703f7226e0fSAlex Maftei (amaftei) /* Remove it iff ARFS wants to. */
704f7226e0fSAlex Maftei (amaftei) return true;
705f7226e0fSAlex Maftei (amaftei) }
706f7226e0fSAlex Maftei (amaftei)
707f7226e0fSAlex Maftei (amaftei) static
efx_rps_hash_bucket(struct efx_nic * efx,const struct efx_filter_spec * spec)708f7226e0fSAlex Maftei (amaftei) struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx,
709f7226e0fSAlex Maftei (amaftei) const struct efx_filter_spec *spec)
710f7226e0fSAlex Maftei (amaftei) {
711f7226e0fSAlex Maftei (amaftei) u32 hash = efx_filter_spec_hash(spec);
712f7226e0fSAlex Maftei (amaftei)
713f7226e0fSAlex Maftei (amaftei) lockdep_assert_held(&efx->rps_hash_lock);
714f7226e0fSAlex Maftei (amaftei) if (!efx->rps_hash_table)
715f7226e0fSAlex Maftei (amaftei) return NULL;
716f7226e0fSAlex Maftei (amaftei) return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE];
717f7226e0fSAlex Maftei (amaftei) }
718f7226e0fSAlex Maftei (amaftei)
efx_rps_hash_find(struct efx_nic * efx,const struct efx_filter_spec * spec)719f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx,
720f7226e0fSAlex Maftei (amaftei) const struct efx_filter_spec *spec)
721f7226e0fSAlex Maftei (amaftei) {
722f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *rule;
723f7226e0fSAlex Maftei (amaftei) struct hlist_head *head;
724f7226e0fSAlex Maftei (amaftei) struct hlist_node *node;
725f7226e0fSAlex Maftei (amaftei)
726f7226e0fSAlex Maftei (amaftei) head = efx_rps_hash_bucket(efx, spec);
727f7226e0fSAlex Maftei (amaftei) if (!head)
728f7226e0fSAlex Maftei (amaftei) return NULL;
729f7226e0fSAlex Maftei (amaftei) hlist_for_each(node, head) {
730f7226e0fSAlex Maftei (amaftei) rule = container_of(node, struct efx_arfs_rule, node);
731f7226e0fSAlex Maftei (amaftei) if (efx_filter_spec_equal(spec, &rule->spec))
732f7226e0fSAlex Maftei (amaftei) return rule;
733f7226e0fSAlex Maftei (amaftei) }
734f7226e0fSAlex Maftei (amaftei) return NULL;
735f7226e0fSAlex Maftei (amaftei) }
736f7226e0fSAlex Maftei (amaftei)
efx_rps_hash_add(struct efx_nic * efx,const struct efx_filter_spec * spec,bool * new)737f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,
738f7226e0fSAlex Maftei (amaftei) const struct efx_filter_spec *spec,
739f7226e0fSAlex Maftei (amaftei) bool *new)
740f7226e0fSAlex Maftei (amaftei) {
741f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *rule;
742f7226e0fSAlex Maftei (amaftei) struct hlist_head *head;
743f7226e0fSAlex Maftei (amaftei) struct hlist_node *node;
744f7226e0fSAlex Maftei (amaftei)
745f7226e0fSAlex Maftei (amaftei) head = efx_rps_hash_bucket(efx, spec);
746f7226e0fSAlex Maftei (amaftei) if (!head)
747f7226e0fSAlex Maftei (amaftei) return NULL;
748f7226e0fSAlex Maftei (amaftei) hlist_for_each(node, head) {
749f7226e0fSAlex Maftei (amaftei) rule = container_of(node, struct efx_arfs_rule, node);
750f7226e0fSAlex Maftei (amaftei) if (efx_filter_spec_equal(spec, &rule->spec)) {
751f7226e0fSAlex Maftei (amaftei) *new = false;
752f7226e0fSAlex Maftei (amaftei) return rule;
753f7226e0fSAlex Maftei (amaftei) }
754f7226e0fSAlex Maftei (amaftei) }
755f7226e0fSAlex Maftei (amaftei) rule = kmalloc(sizeof(*rule), GFP_ATOMIC);
756f7226e0fSAlex Maftei (amaftei) *new = true;
757f7226e0fSAlex Maftei (amaftei) if (rule) {
758f7226e0fSAlex Maftei (amaftei) memcpy(&rule->spec, spec, sizeof(rule->spec));
759f7226e0fSAlex Maftei (amaftei) hlist_add_head(&rule->node, head);
760f7226e0fSAlex Maftei (amaftei) }
761f7226e0fSAlex Maftei (amaftei) return rule;
762f7226e0fSAlex Maftei (amaftei) }
763f7226e0fSAlex Maftei (amaftei)
efx_rps_hash_del(struct efx_nic * efx,const struct efx_filter_spec * spec)764f7226e0fSAlex Maftei (amaftei) void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec)
765f7226e0fSAlex Maftei (amaftei) {
766f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *rule;
767f7226e0fSAlex Maftei (amaftei) struct hlist_head *head;
768f7226e0fSAlex Maftei (amaftei) struct hlist_node *node;
769f7226e0fSAlex Maftei (amaftei)
770f7226e0fSAlex Maftei (amaftei) head = efx_rps_hash_bucket(efx, spec);
771f7226e0fSAlex Maftei (amaftei) if (WARN_ON(!head))
772f7226e0fSAlex Maftei (amaftei) return;
773f7226e0fSAlex Maftei (amaftei) hlist_for_each(node, head) {
774f7226e0fSAlex Maftei (amaftei) rule = container_of(node, struct efx_arfs_rule, node);
775f7226e0fSAlex Maftei (amaftei) if (efx_filter_spec_equal(spec, &rule->spec)) {
776f7226e0fSAlex Maftei (amaftei) /* Someone already reused the entry. We know that if
777f7226e0fSAlex Maftei (amaftei) * this check doesn't fire (i.e. filter_id == REMOVING)
778f7226e0fSAlex Maftei (amaftei) * then the REMOVING mark was put there by our caller,
779f7226e0fSAlex Maftei (amaftei) * because caller is holding a lock on filter table and
780f7226e0fSAlex Maftei (amaftei) * only holders of that lock set REMOVING.
781f7226e0fSAlex Maftei (amaftei) */
782f7226e0fSAlex Maftei (amaftei) if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING)
783f7226e0fSAlex Maftei (amaftei) return;
784f7226e0fSAlex Maftei (amaftei) hlist_del(node);
785f7226e0fSAlex Maftei (amaftei) kfree(rule);
786f7226e0fSAlex Maftei (amaftei) return;
787f7226e0fSAlex Maftei (amaftei) }
788f7226e0fSAlex Maftei (amaftei) }
789f7226e0fSAlex Maftei (amaftei) /* We didn't find it. */
790f7226e0fSAlex Maftei (amaftei) WARN_ON(1);
791f7226e0fSAlex Maftei (amaftei) }
792f7226e0fSAlex Maftei (amaftei) #endif
793f7226e0fSAlex Maftei (amaftei)
efx_probe_filters(struct efx_nic * efx)794f7226e0fSAlex Maftei (amaftei) int efx_probe_filters(struct efx_nic *efx)
795f7226e0fSAlex Maftei (amaftei) {
796f7226e0fSAlex Maftei (amaftei) int rc;
797f7226e0fSAlex Maftei (amaftei)
798f7226e0fSAlex Maftei (amaftei) mutex_lock(&efx->mac_lock);
799f7226e0fSAlex Maftei (amaftei) rc = efx->type->filter_table_probe(efx);
800f7226e0fSAlex Maftei (amaftei) if (rc)
801f7226e0fSAlex Maftei (amaftei) goto out_unlock;
802f7226e0fSAlex Maftei (amaftei)
803f7226e0fSAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL
804f7226e0fSAlex Maftei (amaftei) if (efx->type->offload_features & NETIF_F_NTUPLE) {
805f7226e0fSAlex Maftei (amaftei) struct efx_channel *channel;
806f7226e0fSAlex Maftei (amaftei) int i, success = 1;
807f7226e0fSAlex Maftei (amaftei)
808f7226e0fSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) {
809f7226e0fSAlex Maftei (amaftei) channel->rps_flow_id =
810f7226e0fSAlex Maftei (amaftei) kcalloc(efx->type->max_rx_ip_filters,
811f7226e0fSAlex Maftei (amaftei) sizeof(*channel->rps_flow_id),
812f7226e0fSAlex Maftei (amaftei) GFP_KERNEL);
813f7226e0fSAlex Maftei (amaftei) if (!channel->rps_flow_id)
814f7226e0fSAlex Maftei (amaftei) success = 0;
815f7226e0fSAlex Maftei (amaftei) else
816f7226e0fSAlex Maftei (amaftei) for (i = 0;
817f7226e0fSAlex Maftei (amaftei) i < efx->type->max_rx_ip_filters;
818f7226e0fSAlex Maftei (amaftei) ++i)
819f7226e0fSAlex Maftei (amaftei) channel->rps_flow_id[i] =
820f7226e0fSAlex Maftei (amaftei) RPS_FLOW_ID_INVALID;
821f7226e0fSAlex Maftei (amaftei) channel->rfs_expire_index = 0;
822f7226e0fSAlex Maftei (amaftei) channel->rfs_filter_count = 0;
823f7226e0fSAlex Maftei (amaftei) }
824f7226e0fSAlex Maftei (amaftei)
825f7226e0fSAlex Maftei (amaftei) if (!success) {
826*9f2e244bSZhipeng Lu efx_for_each_channel(channel, efx) {
827f7226e0fSAlex Maftei (amaftei) kfree(channel->rps_flow_id);
828*9f2e244bSZhipeng Lu channel->rps_flow_id = NULL;
829*9f2e244bSZhipeng Lu }
830f7226e0fSAlex Maftei (amaftei) efx->type->filter_table_remove(efx);
831f7226e0fSAlex Maftei (amaftei) rc = -ENOMEM;
832f7226e0fSAlex Maftei (amaftei) goto out_unlock;
833f7226e0fSAlex Maftei (amaftei) }
834f7226e0fSAlex Maftei (amaftei) }
835f7226e0fSAlex Maftei (amaftei) #endif
836f7226e0fSAlex Maftei (amaftei) out_unlock:
837f7226e0fSAlex Maftei (amaftei) mutex_unlock(&efx->mac_lock);
838f7226e0fSAlex Maftei (amaftei) return rc;
839f7226e0fSAlex Maftei (amaftei) }
840f7226e0fSAlex Maftei (amaftei)
efx_remove_filters(struct efx_nic * efx)841f7226e0fSAlex Maftei (amaftei) void efx_remove_filters(struct efx_nic *efx)
842f7226e0fSAlex Maftei (amaftei) {
843f7226e0fSAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL
844f7226e0fSAlex Maftei (amaftei) struct efx_channel *channel;
845f7226e0fSAlex Maftei (amaftei)
846f7226e0fSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) {
847f7226e0fSAlex Maftei (amaftei) cancel_delayed_work_sync(&channel->filter_work);
848f7226e0fSAlex Maftei (amaftei) kfree(channel->rps_flow_id);
849788f920aSEdward Cree channel->rps_flow_id = NULL;
850f7226e0fSAlex Maftei (amaftei) }
851f7226e0fSAlex Maftei (amaftei) #endif
852f7226e0fSAlex Maftei (amaftei) efx->type->filter_table_remove(efx);
853f7226e0fSAlex Maftei (amaftei) }
85428abe825SEdward Cree
85528abe825SEdward Cree #ifdef CONFIG_RFS_ACCEL
85628abe825SEdward Cree
efx_filter_rfs_work(struct work_struct * data)85728abe825SEdward Cree static void efx_filter_rfs_work(struct work_struct *data)
85828abe825SEdward Cree {
85928abe825SEdward Cree struct efx_async_filter_insertion *req = container_of(data, struct efx_async_filter_insertion,
86028abe825SEdward Cree work);
8618cb03f4eSJonathan Cooper struct efx_nic *efx = efx_netdev_priv(req->net_dev);
86228abe825SEdward Cree struct efx_channel *channel = efx_get_channel(efx, req->rxq_index);
86328abe825SEdward Cree int slot_idx = req - efx->rps_slot;
86428abe825SEdward Cree struct efx_arfs_rule *rule;
86528abe825SEdward Cree u16 arfs_id = 0;
86628abe825SEdward Cree int rc;
86728abe825SEdward Cree
86828abe825SEdward Cree rc = efx->type->filter_insert(efx, &req->spec, true);
86928abe825SEdward Cree if (rc >= 0)
87028abe825SEdward Cree /* Discard 'priority' part of EF10+ filter ID (mcdi_filters) */
87128abe825SEdward Cree rc %= efx->type->max_rx_ip_filters;
87228abe825SEdward Cree if (efx->rps_hash_table) {
87328abe825SEdward Cree spin_lock_bh(&efx->rps_hash_lock);
87428abe825SEdward Cree rule = efx_rps_hash_find(efx, &req->spec);
87528abe825SEdward Cree /* The rule might have already gone, if someone else's request
87628abe825SEdward Cree * for the same spec was already worked and then expired before
87728abe825SEdward Cree * we got around to our work. In that case we have nothing
87828abe825SEdward Cree * tying us to an arfs_id, meaning that as soon as the filter
87928abe825SEdward Cree * is considered for expiry it will be removed.
88028abe825SEdward Cree */
88128abe825SEdward Cree if (rule) {
88228abe825SEdward Cree if (rc < 0)
88328abe825SEdward Cree rule->filter_id = EFX_ARFS_FILTER_ID_ERROR;
88428abe825SEdward Cree else
88528abe825SEdward Cree rule->filter_id = rc;
88628abe825SEdward Cree arfs_id = rule->arfs_id;
88728abe825SEdward Cree }
88828abe825SEdward Cree spin_unlock_bh(&efx->rps_hash_lock);
88928abe825SEdward Cree }
89028abe825SEdward Cree if (rc >= 0) {
89128abe825SEdward Cree /* Remember this so we can check whether to expire the filter
89228abe825SEdward Cree * later.
89328abe825SEdward Cree */
89428abe825SEdward Cree mutex_lock(&efx->rps_mutex);
89528abe825SEdward Cree if (channel->rps_flow_id[rc] == RPS_FLOW_ID_INVALID)
89628abe825SEdward Cree channel->rfs_filter_count++;
89728abe825SEdward Cree channel->rps_flow_id[rc] = req->flow_id;
89828abe825SEdward Cree mutex_unlock(&efx->rps_mutex);
89928abe825SEdward Cree
90028abe825SEdward Cree if (req->spec.ether_type == htons(ETH_P_IP))
90128abe825SEdward Cree netif_info(efx, rx_status, efx->net_dev,
90228abe825SEdward Cree "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d id %u]\n",
90328abe825SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
90428abe825SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port),
90528abe825SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port),
90628abe825SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id);
90728abe825SEdward Cree else
90828abe825SEdward Cree netif_info(efx, rx_status, efx->net_dev,
90928abe825SEdward Cree "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d id %u]\n",
91028abe825SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
91128abe825SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port),
91228abe825SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port),
91328abe825SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id);
91428abe825SEdward Cree channel->n_rfs_succeeded++;
91528abe825SEdward Cree } else {
91628abe825SEdward Cree if (req->spec.ether_type == htons(ETH_P_IP))
91728abe825SEdward Cree netif_dbg(efx, rx_status, efx->net_dev,
91828abe825SEdward Cree "failed to steer %s %pI4:%u:%pI4:%u to queue %u [flow %u rc %d id %u]\n",
91928abe825SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
92028abe825SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port),
92128abe825SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port),
92228abe825SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id);
92328abe825SEdward Cree else
92428abe825SEdward Cree netif_dbg(efx, rx_status, efx->net_dev,
92528abe825SEdward Cree "failed to steer %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u rc %d id %u]\n",
92628abe825SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
92728abe825SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port),
92828abe825SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port),
92928abe825SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id);
93028abe825SEdward Cree channel->n_rfs_failed++;
93128abe825SEdward Cree /* We're overloading the NIC's filter tables, so let's do a
93228abe825SEdward Cree * chunk of extra expiry work.
93328abe825SEdward Cree */
93428abe825SEdward Cree __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count,
93528abe825SEdward Cree 100u));
93628abe825SEdward Cree }
93728abe825SEdward Cree
93828abe825SEdward Cree /* Release references */
93928abe825SEdward Cree clear_bit(slot_idx, &efx->rps_slot_map);
94028abe825SEdward Cree dev_put(req->net_dev);
94128abe825SEdward Cree }
94228abe825SEdward Cree
efx_filter_rfs(struct net_device * net_dev,const struct sk_buff * skb,u16 rxq_index,u32 flow_id)94328abe825SEdward Cree int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
94428abe825SEdward Cree u16 rxq_index, u32 flow_id)
94528abe825SEdward Cree {
9468cb03f4eSJonathan Cooper struct efx_nic *efx = efx_netdev_priv(net_dev);
94728abe825SEdward Cree struct efx_async_filter_insertion *req;
94828abe825SEdward Cree struct efx_arfs_rule *rule;
94928abe825SEdward Cree struct flow_keys fk;
95028abe825SEdward Cree int slot_idx;
95128abe825SEdward Cree bool new;
95228abe825SEdward Cree int rc;
95328abe825SEdward Cree
95428abe825SEdward Cree /* find a free slot */
95528abe825SEdward Cree for (slot_idx = 0; slot_idx < EFX_RPS_MAX_IN_FLIGHT; slot_idx++)
95628abe825SEdward Cree if (!test_and_set_bit(slot_idx, &efx->rps_slot_map))
95728abe825SEdward Cree break;
95828abe825SEdward Cree if (slot_idx >= EFX_RPS_MAX_IN_FLIGHT)
95928abe825SEdward Cree return -EBUSY;
96028abe825SEdward Cree
96128abe825SEdward Cree if (flow_id == RPS_FLOW_ID_INVALID) {
96228abe825SEdward Cree rc = -EINVAL;
96328abe825SEdward Cree goto out_clear;
96428abe825SEdward Cree }
96528abe825SEdward Cree
96628abe825SEdward Cree if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) {
96728abe825SEdward Cree rc = -EPROTONOSUPPORT;
96828abe825SEdward Cree goto out_clear;
96928abe825SEdward Cree }
97028abe825SEdward Cree
97128abe825SEdward Cree if (fk.basic.n_proto != htons(ETH_P_IP) && fk.basic.n_proto != htons(ETH_P_IPV6)) {
97228abe825SEdward Cree rc = -EPROTONOSUPPORT;
97328abe825SEdward Cree goto out_clear;
97428abe825SEdward Cree }
97528abe825SEdward Cree if (fk.control.flags & FLOW_DIS_IS_FRAGMENT) {
97628abe825SEdward Cree rc = -EPROTONOSUPPORT;
97728abe825SEdward Cree goto out_clear;
97828abe825SEdward Cree }
97928abe825SEdward Cree
98028abe825SEdward Cree req = efx->rps_slot + slot_idx;
98128abe825SEdward Cree efx_filter_init_rx(&req->spec, EFX_FILTER_PRI_HINT,
98228abe825SEdward Cree efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0,
98328abe825SEdward Cree rxq_index);
98428abe825SEdward Cree req->spec.match_flags =
98528abe825SEdward Cree EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
98628abe825SEdward Cree EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
98728abe825SEdward Cree EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
98828abe825SEdward Cree req->spec.ether_type = fk.basic.n_proto;
98928abe825SEdward Cree req->spec.ip_proto = fk.basic.ip_proto;
99028abe825SEdward Cree
99128abe825SEdward Cree if (fk.basic.n_proto == htons(ETH_P_IP)) {
99228abe825SEdward Cree req->spec.rem_host[0] = fk.addrs.v4addrs.src;
99328abe825SEdward Cree req->spec.loc_host[0] = fk.addrs.v4addrs.dst;
99428abe825SEdward Cree } else {
99528abe825SEdward Cree memcpy(req->spec.rem_host, &fk.addrs.v6addrs.src,
99628abe825SEdward Cree sizeof(struct in6_addr));
99728abe825SEdward Cree memcpy(req->spec.loc_host, &fk.addrs.v6addrs.dst,
99828abe825SEdward Cree sizeof(struct in6_addr));
99928abe825SEdward Cree }
100028abe825SEdward Cree
100128abe825SEdward Cree req->spec.rem_port = fk.ports.src;
100228abe825SEdward Cree req->spec.loc_port = fk.ports.dst;
100328abe825SEdward Cree
100428abe825SEdward Cree if (efx->rps_hash_table) {
100528abe825SEdward Cree /* Add it to ARFS hash table */
100628abe825SEdward Cree spin_lock(&efx->rps_hash_lock);
100728abe825SEdward Cree rule = efx_rps_hash_add(efx, &req->spec, &new);
100828abe825SEdward Cree if (!rule) {
100928abe825SEdward Cree rc = -ENOMEM;
101028abe825SEdward Cree goto out_unlock;
101128abe825SEdward Cree }
101228abe825SEdward Cree if (new)
101328abe825SEdward Cree rule->arfs_id = efx->rps_next_id++ % RPS_NO_FILTER;
101428abe825SEdward Cree rc = rule->arfs_id;
101528abe825SEdward Cree /* Skip if existing or pending filter already does the right thing */
101628abe825SEdward Cree if (!new && rule->rxq_index == rxq_index &&
101728abe825SEdward Cree rule->filter_id >= EFX_ARFS_FILTER_ID_PENDING)
101828abe825SEdward Cree goto out_unlock;
101928abe825SEdward Cree rule->rxq_index = rxq_index;
102028abe825SEdward Cree rule->filter_id = EFX_ARFS_FILTER_ID_PENDING;
102128abe825SEdward Cree spin_unlock(&efx->rps_hash_lock);
102228abe825SEdward Cree } else {
102328abe825SEdward Cree /* Without an ARFS hash table, we just use arfs_id 0 for all
102428abe825SEdward Cree * filters. This means if multiple flows hash to the same
102528abe825SEdward Cree * flow_id, all but the most recently touched will be eligible
102628abe825SEdward Cree * for expiry.
102728abe825SEdward Cree */
102828abe825SEdward Cree rc = 0;
102928abe825SEdward Cree }
103028abe825SEdward Cree
103128abe825SEdward Cree /* Queue the request */
103228abe825SEdward Cree dev_hold(req->net_dev = net_dev);
103328abe825SEdward Cree INIT_WORK(&req->work, efx_filter_rfs_work);
103428abe825SEdward Cree req->rxq_index = rxq_index;
103528abe825SEdward Cree req->flow_id = flow_id;
103628abe825SEdward Cree schedule_work(&req->work);
103728abe825SEdward Cree return rc;
103828abe825SEdward Cree out_unlock:
103928abe825SEdward Cree spin_unlock(&efx->rps_hash_lock);
104028abe825SEdward Cree out_clear:
104128abe825SEdward Cree clear_bit(slot_idx, &efx->rps_slot_map);
104228abe825SEdward Cree return rc;
104328abe825SEdward Cree }
104428abe825SEdward Cree
__efx_filter_rfs_expire(struct efx_channel * channel,unsigned int quota)104528abe825SEdward Cree bool __efx_filter_rfs_expire(struct efx_channel *channel, unsigned int quota)
104628abe825SEdward Cree {
104728abe825SEdward Cree bool (*expire_one)(struct efx_nic *efx, u32 flow_id, unsigned int index);
104828abe825SEdward Cree struct efx_nic *efx = channel->efx;
104928abe825SEdward Cree unsigned int index, size, start;
105028abe825SEdward Cree u32 flow_id;
105128abe825SEdward Cree
105228abe825SEdward Cree if (!mutex_trylock(&efx->rps_mutex))
105328abe825SEdward Cree return false;
105428abe825SEdward Cree expire_one = efx->type->filter_rfs_expire_one;
105528abe825SEdward Cree index = channel->rfs_expire_index;
105628abe825SEdward Cree start = index;
105728abe825SEdward Cree size = efx->type->max_rx_ip_filters;
105828abe825SEdward Cree while (quota) {
105928abe825SEdward Cree flow_id = channel->rps_flow_id[index];
106028abe825SEdward Cree
106128abe825SEdward Cree if (flow_id != RPS_FLOW_ID_INVALID) {
106228abe825SEdward Cree quota--;
106328abe825SEdward Cree if (expire_one(efx, flow_id, index)) {
106428abe825SEdward Cree netif_info(efx, rx_status, efx->net_dev,
106528abe825SEdward Cree "expired filter %d [channel %u flow %u]\n",
106628abe825SEdward Cree index, channel->channel, flow_id);
106728abe825SEdward Cree channel->rps_flow_id[index] = RPS_FLOW_ID_INVALID;
106828abe825SEdward Cree channel->rfs_filter_count--;
106928abe825SEdward Cree }
107028abe825SEdward Cree }
107128abe825SEdward Cree if (++index == size)
107228abe825SEdward Cree index = 0;
107328abe825SEdward Cree /* If we were called with a quota that exceeds the total number
107428abe825SEdward Cree * of filters in the table (which shouldn't happen, but could
107528abe825SEdward Cree * if two callers race), ensure that we don't loop forever -
107628abe825SEdward Cree * stop when we've examined every row of the table.
107728abe825SEdward Cree */
107828abe825SEdward Cree if (index == start)
107928abe825SEdward Cree break;
108028abe825SEdward Cree }
108128abe825SEdward Cree
108228abe825SEdward Cree channel->rfs_expire_index = index;
108328abe825SEdward Cree mutex_unlock(&efx->rps_mutex);
108428abe825SEdward Cree return true;
108528abe825SEdward Cree }
108628abe825SEdward Cree
108728abe825SEdward Cree #endif /* CONFIG_RFS_ACCEL */
1088