136ff6393SMartin Habets // SPDX-License-Identifier: GPL-2.0-only
236ff6393SMartin Habets /****************************************************************************
336ff6393SMartin Habets * Driver for Solarflare network controllers and boards
436ff6393SMartin Habets * Copyright 2005-2006 Fen Systems Ltd.
536ff6393SMartin Habets * Copyright 2006-2013 Solarflare Communications Inc.
636ff6393SMartin Habets */
736ff6393SMartin Habets
836ff6393SMartin Habets #include <linux/bitops.h>
936ff6393SMartin Habets #include <linux/delay.h>
1036ff6393SMartin Habets #include <linux/interrupt.h>
1136ff6393SMartin Habets #include <linux/pci.h>
1236ff6393SMartin Habets #include <linux/module.h>
1336ff6393SMartin Habets #include <linux/seq_file.h>
1436ff6393SMartin Habets #include <linux/crc32.h>
1536ff6393SMartin Habets #include "net_driver.h"
1636ff6393SMartin Habets #include "bitfield.h"
1736ff6393SMartin Habets #include "efx.h"
1836ff6393SMartin Habets #include "rx_common.h"
1936ff6393SMartin Habets #include "tx_common.h"
2036ff6393SMartin Habets #include "nic.h"
2136ff6393SMartin Habets #include "farch_regs.h"
2236ff6393SMartin Habets #include "sriov.h"
2336ff6393SMartin Habets #include "siena_sriov.h"
2436ff6393SMartin Habets #include "io.h"
2536ff6393SMartin Habets #include "workarounds.h"
2636ff6393SMartin Habets
2736ff6393SMartin Habets /* Falcon-architecture (SFC9000-family) support */
2836ff6393SMartin Habets
2936ff6393SMartin Habets /**************************************************************************
3036ff6393SMartin Habets *
3136ff6393SMartin Habets * Configurable values
3236ff6393SMartin Habets *
3336ff6393SMartin Habets **************************************************************************
3436ff6393SMartin Habets */
3536ff6393SMartin Habets
3636ff6393SMartin Habets /* This is set to 16 for a good reason. In summary, if larger than
3736ff6393SMartin Habets * 16, the descriptor cache holds more than a default socket
3836ff6393SMartin Habets * buffer's worth of packets (for UDP we can only have at most one
3936ff6393SMartin Habets * socket buffer's worth outstanding). This combined with the fact
4036ff6393SMartin Habets * that we only get 1 TX event per descriptor cache means the NIC
4136ff6393SMartin Habets * goes idle.
4236ff6393SMartin Habets */
4336ff6393SMartin Habets #define TX_DC_ENTRIES 16
4436ff6393SMartin Habets #define TX_DC_ENTRIES_ORDER 1
4536ff6393SMartin Habets
4636ff6393SMartin Habets #define RX_DC_ENTRIES 64
4736ff6393SMartin Habets #define RX_DC_ENTRIES_ORDER 3
4836ff6393SMartin Habets
4936ff6393SMartin Habets /* If EFX_MAX_INT_ERRORS internal errors occur within
5036ff6393SMartin Habets * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
5136ff6393SMartin Habets * disable it.
5236ff6393SMartin Habets */
5336ff6393SMartin Habets #define EFX_INT_ERROR_EXPIRE 3600
5436ff6393SMartin Habets #define EFX_MAX_INT_ERRORS 5
5536ff6393SMartin Habets
5636ff6393SMartin Habets /* Depth of RX flush request fifo */
5736ff6393SMartin Habets #define EFX_RX_FLUSH_COUNT 4
5836ff6393SMartin Habets
5936ff6393SMartin Habets /* Driver generated events */
6036ff6393SMartin Habets #define _EFX_CHANNEL_MAGIC_TEST 0x000101
6136ff6393SMartin Habets #define _EFX_CHANNEL_MAGIC_FILL 0x000102
6236ff6393SMartin Habets #define _EFX_CHANNEL_MAGIC_RX_DRAIN 0x000103
6336ff6393SMartin Habets #define _EFX_CHANNEL_MAGIC_TX_DRAIN 0x000104
6436ff6393SMartin Habets
6536ff6393SMartin Habets #define _EFX_CHANNEL_MAGIC(_code, _data) ((_code) << 8 | (_data))
6636ff6393SMartin Habets #define _EFX_CHANNEL_MAGIC_CODE(_magic) ((_magic) >> 8)
6736ff6393SMartin Habets
6836ff6393SMartin Habets #define EFX_CHANNEL_MAGIC_TEST(_channel) \
6936ff6393SMartin Habets _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TEST, (_channel)->channel)
7036ff6393SMartin Habets #define EFX_CHANNEL_MAGIC_FILL(_rx_queue) \
7136ff6393SMartin Habets _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_FILL, \
7236ff6393SMartin Habets efx_rx_queue_index(_rx_queue))
7336ff6393SMartin Habets #define EFX_CHANNEL_MAGIC_RX_DRAIN(_rx_queue) \
7436ff6393SMartin Habets _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_RX_DRAIN, \
7536ff6393SMartin Habets efx_rx_queue_index(_rx_queue))
7636ff6393SMartin Habets #define EFX_CHANNEL_MAGIC_TX_DRAIN(_tx_queue) \
7736ff6393SMartin Habets _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TX_DRAIN, \
7836ff6393SMartin Habets (_tx_queue)->queue)
7936ff6393SMartin Habets
8036ff6393SMartin Habets static void efx_farch_magic_event(struct efx_channel *channel, u32 magic);
8136ff6393SMartin Habets
8236ff6393SMartin Habets /**************************************************************************
8336ff6393SMartin Habets *
8436ff6393SMartin Habets * Hardware access
8536ff6393SMartin Habets *
8636ff6393SMartin Habets **************************************************************************/
8736ff6393SMartin Habets
efx_write_buf_tbl(struct efx_nic * efx,efx_qword_t * value,unsigned int index)8836ff6393SMartin Habets static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
8936ff6393SMartin Habets unsigned int index)
9036ff6393SMartin Habets {
9136ff6393SMartin Habets efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
9236ff6393SMartin Habets value, index);
9336ff6393SMartin Habets }
9436ff6393SMartin Habets
efx_masked_compare_oword(const efx_oword_t * a,const efx_oword_t * b,const efx_oword_t * mask)9536ff6393SMartin Habets static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b,
9636ff6393SMartin Habets const efx_oword_t *mask)
9736ff6393SMartin Habets {
9836ff6393SMartin Habets return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) ||
9936ff6393SMartin Habets ((a->u64[1] ^ b->u64[1]) & mask->u64[1]);
10036ff6393SMartin Habets }
10136ff6393SMartin Habets
efx_farch_test_registers(struct efx_nic * efx,const struct efx_farch_register_test * regs,size_t n_regs)10236ff6393SMartin Habets int efx_farch_test_registers(struct efx_nic *efx,
10336ff6393SMartin Habets const struct efx_farch_register_test *regs,
10436ff6393SMartin Habets size_t n_regs)
10536ff6393SMartin Habets {
10636ff6393SMartin Habets unsigned address = 0;
10736ff6393SMartin Habets int i, j;
10836ff6393SMartin Habets efx_oword_t mask, imask, original, reg, buf;
10936ff6393SMartin Habets
11036ff6393SMartin Habets for (i = 0; i < n_regs; ++i) {
11136ff6393SMartin Habets address = regs[i].address;
11236ff6393SMartin Habets mask = imask = regs[i].mask;
11336ff6393SMartin Habets EFX_INVERT_OWORD(imask);
11436ff6393SMartin Habets
11536ff6393SMartin Habets efx_reado(efx, &original, address);
11636ff6393SMartin Habets
11736ff6393SMartin Habets /* bit sweep on and off */
11836ff6393SMartin Habets for (j = 0; j < 128; j++) {
11936ff6393SMartin Habets if (!EFX_EXTRACT_OWORD32(mask, j, j))
12036ff6393SMartin Habets continue;
12136ff6393SMartin Habets
12236ff6393SMartin Habets /* Test this testable bit can be set in isolation */
12336ff6393SMartin Habets EFX_AND_OWORD(reg, original, mask);
12436ff6393SMartin Habets EFX_SET_OWORD32(reg, j, j, 1);
12536ff6393SMartin Habets
12636ff6393SMartin Habets efx_writeo(efx, ®, address);
12736ff6393SMartin Habets efx_reado(efx, &buf, address);
12836ff6393SMartin Habets
12936ff6393SMartin Habets if (efx_masked_compare_oword(®, &buf, &mask))
13036ff6393SMartin Habets goto fail;
13136ff6393SMartin Habets
13236ff6393SMartin Habets /* Test this testable bit can be cleared in isolation */
13336ff6393SMartin Habets EFX_OR_OWORD(reg, original, mask);
13436ff6393SMartin Habets EFX_SET_OWORD32(reg, j, j, 0);
13536ff6393SMartin Habets
13636ff6393SMartin Habets efx_writeo(efx, ®, address);
13736ff6393SMartin Habets efx_reado(efx, &buf, address);
13836ff6393SMartin Habets
13936ff6393SMartin Habets if (efx_masked_compare_oword(®, &buf, &mask))
14036ff6393SMartin Habets goto fail;
14136ff6393SMartin Habets }
14236ff6393SMartin Habets
14336ff6393SMartin Habets efx_writeo(efx, &original, address);
14436ff6393SMartin Habets }
14536ff6393SMartin Habets
14636ff6393SMartin Habets return 0;
14736ff6393SMartin Habets
14836ff6393SMartin Habets fail:
14936ff6393SMartin Habets netif_err(efx, hw, efx->net_dev,
15036ff6393SMartin Habets "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT
15136ff6393SMartin Habets " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg),
15236ff6393SMartin Habets EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask));
15336ff6393SMartin Habets return -EIO;
15436ff6393SMartin Habets }
15536ff6393SMartin Habets
15636ff6393SMartin Habets /**************************************************************************
15736ff6393SMartin Habets *
15836ff6393SMartin Habets * Special buffer handling
15936ff6393SMartin Habets * Special buffers are used for event queues and the TX and RX
16036ff6393SMartin Habets * descriptor rings.
16136ff6393SMartin Habets *
16236ff6393SMartin Habets *************************************************************************/
16336ff6393SMartin Habets
16436ff6393SMartin Habets /*
16536ff6393SMartin Habets * Initialise a special buffer
16636ff6393SMartin Habets *
16736ff6393SMartin Habets * This will define a buffer (previously allocated via
16836ff6393SMartin Habets * efx_alloc_special_buffer()) in the buffer table, allowing
16936ff6393SMartin Habets * it to be used for event queues, descriptor rings etc.
17036ff6393SMartin Habets */
17136ff6393SMartin Habets static void
efx_init_special_buffer(struct efx_nic * efx,struct efx_special_buffer * buffer)17236ff6393SMartin Habets efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
17336ff6393SMartin Habets {
17436ff6393SMartin Habets efx_qword_t buf_desc;
17536ff6393SMartin Habets unsigned int index;
17636ff6393SMartin Habets dma_addr_t dma_addr;
17736ff6393SMartin Habets int i;
17836ff6393SMartin Habets
17936ff6393SMartin Habets EFX_WARN_ON_PARANOID(!buffer->buf.addr);
18036ff6393SMartin Habets
18136ff6393SMartin Habets /* Write buffer descriptors to NIC */
18236ff6393SMartin Habets for (i = 0; i < buffer->entries; i++) {
18336ff6393SMartin Habets index = buffer->index + i;
18436ff6393SMartin Habets dma_addr = buffer->buf.dma_addr + (i * EFX_BUF_SIZE);
18536ff6393SMartin Habets netif_dbg(efx, probe, efx->net_dev,
18636ff6393SMartin Habets "mapping special buffer %d at %llx\n",
18736ff6393SMartin Habets index, (unsigned long long)dma_addr);
18836ff6393SMartin Habets EFX_POPULATE_QWORD_3(buf_desc,
18936ff6393SMartin Habets FRF_AZ_BUF_ADR_REGION, 0,
19036ff6393SMartin Habets FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
19136ff6393SMartin Habets FRF_AZ_BUF_OWNER_ID_FBUF, 0);
19236ff6393SMartin Habets efx_write_buf_tbl(efx, &buf_desc, index);
19336ff6393SMartin Habets }
19436ff6393SMartin Habets }
19536ff6393SMartin Habets
19636ff6393SMartin Habets /* Unmaps a buffer and clears the buffer table entries */
19736ff6393SMartin Habets static void
efx_fini_special_buffer(struct efx_nic * efx,struct efx_special_buffer * buffer)19836ff6393SMartin Habets efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
19936ff6393SMartin Habets {
20036ff6393SMartin Habets efx_oword_t buf_tbl_upd;
20136ff6393SMartin Habets unsigned int start = buffer->index;
20236ff6393SMartin Habets unsigned int end = (buffer->index + buffer->entries - 1);
20336ff6393SMartin Habets
20436ff6393SMartin Habets if (!buffer->entries)
20536ff6393SMartin Habets return;
20636ff6393SMartin Habets
20736ff6393SMartin Habets netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n",
20836ff6393SMartin Habets buffer->index, buffer->index + buffer->entries - 1);
20936ff6393SMartin Habets
21036ff6393SMartin Habets EFX_POPULATE_OWORD_4(buf_tbl_upd,
21136ff6393SMartin Habets FRF_AZ_BUF_UPD_CMD, 0,
21236ff6393SMartin Habets FRF_AZ_BUF_CLR_CMD, 1,
21336ff6393SMartin Habets FRF_AZ_BUF_CLR_END_ID, end,
21436ff6393SMartin Habets FRF_AZ_BUF_CLR_START_ID, start);
21536ff6393SMartin Habets efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
21636ff6393SMartin Habets }
21736ff6393SMartin Habets
21836ff6393SMartin Habets /*
21936ff6393SMartin Habets * Allocate a new special buffer
22036ff6393SMartin Habets *
22136ff6393SMartin Habets * This allocates memory for a new buffer, clears it and allocates a
22236ff6393SMartin Habets * new buffer ID range. It does not write into the buffer table.
22336ff6393SMartin Habets *
22436ff6393SMartin Habets * This call will allocate 4KB buffers, since 8KB buffers can't be
22536ff6393SMartin Habets * used for event queues and descriptor rings.
22636ff6393SMartin Habets */
efx_alloc_special_buffer(struct efx_nic * efx,struct efx_special_buffer * buffer,unsigned int len)22736ff6393SMartin Habets static int efx_alloc_special_buffer(struct efx_nic *efx,
22836ff6393SMartin Habets struct efx_special_buffer *buffer,
22936ff6393SMartin Habets unsigned int len)
23036ff6393SMartin Habets {
231dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
23236ff6393SMartin Habets struct siena_nic_data *nic_data = efx->nic_data;
23336ff6393SMartin Habets #endif
23436ff6393SMartin Habets len = ALIGN(len, EFX_BUF_SIZE);
23536ff6393SMartin Habets
236c8443b69SMartin Habets if (efx_siena_alloc_buffer(efx, &buffer->buf, len, GFP_KERNEL))
23736ff6393SMartin Habets return -ENOMEM;
23836ff6393SMartin Habets buffer->entries = len / EFX_BUF_SIZE;
23936ff6393SMartin Habets BUG_ON(buffer->buf.dma_addr & (EFX_BUF_SIZE - 1));
24036ff6393SMartin Habets
24136ff6393SMartin Habets /* Select new buffer ID */
24236ff6393SMartin Habets buffer->index = efx->next_buffer_table;
24336ff6393SMartin Habets efx->next_buffer_table += buffer->entries;
244dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
24536ff6393SMartin Habets BUG_ON(efx_siena_sriov_enabled(efx) &&
24636ff6393SMartin Habets nic_data->vf_buftbl_base < efx->next_buffer_table);
24736ff6393SMartin Habets #endif
24836ff6393SMartin Habets
24936ff6393SMartin Habets netif_dbg(efx, probe, efx->net_dev,
25036ff6393SMartin Habets "allocating special buffers %d-%d at %llx+%x "
25136ff6393SMartin Habets "(virt %p phys %llx)\n", buffer->index,
25236ff6393SMartin Habets buffer->index + buffer->entries - 1,
25336ff6393SMartin Habets (u64)buffer->buf.dma_addr, len,
25436ff6393SMartin Habets buffer->buf.addr, (u64)virt_to_phys(buffer->buf.addr));
25536ff6393SMartin Habets
25636ff6393SMartin Habets return 0;
25736ff6393SMartin Habets }
25836ff6393SMartin Habets
25936ff6393SMartin Habets static void
efx_free_special_buffer(struct efx_nic * efx,struct efx_special_buffer * buffer)26036ff6393SMartin Habets efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
26136ff6393SMartin Habets {
26236ff6393SMartin Habets if (!buffer->buf.addr)
26336ff6393SMartin Habets return;
26436ff6393SMartin Habets
26536ff6393SMartin Habets netif_dbg(efx, hw, efx->net_dev,
26636ff6393SMartin Habets "deallocating special buffers %d-%d at %llx+%x "
26736ff6393SMartin Habets "(virt %p phys %llx)\n", buffer->index,
26836ff6393SMartin Habets buffer->index + buffer->entries - 1,
26936ff6393SMartin Habets (u64)buffer->buf.dma_addr, buffer->buf.len,
27036ff6393SMartin Habets buffer->buf.addr, (u64)virt_to_phys(buffer->buf.addr));
27136ff6393SMartin Habets
272c8443b69SMartin Habets efx_siena_free_buffer(efx, &buffer->buf);
27336ff6393SMartin Habets buffer->entries = 0;
27436ff6393SMartin Habets }
27536ff6393SMartin Habets
27636ff6393SMartin Habets /**************************************************************************
27736ff6393SMartin Habets *
27836ff6393SMartin Habets * TX path
27936ff6393SMartin Habets *
28036ff6393SMartin Habets **************************************************************************/
28136ff6393SMartin Habets
28236ff6393SMartin Habets /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
efx_farch_notify_tx_desc(struct efx_tx_queue * tx_queue)28336ff6393SMartin Habets static inline void efx_farch_notify_tx_desc(struct efx_tx_queue *tx_queue)
28436ff6393SMartin Habets {
28536ff6393SMartin Habets unsigned write_ptr;
28636ff6393SMartin Habets efx_dword_t reg;
28736ff6393SMartin Habets
28836ff6393SMartin Habets write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
28936ff6393SMartin Habets EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
29036ff6393SMartin Habets efx_writed_page(tx_queue->efx, ®,
29136ff6393SMartin Habets FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
29236ff6393SMartin Habets }
29336ff6393SMartin Habets
29436ff6393SMartin Habets /* Write pointer and first descriptor for TX descriptor ring */
efx_farch_push_tx_desc(struct efx_tx_queue * tx_queue,const efx_qword_t * txd)29536ff6393SMartin Habets static inline void efx_farch_push_tx_desc(struct efx_tx_queue *tx_queue,
29636ff6393SMartin Habets const efx_qword_t *txd)
29736ff6393SMartin Habets {
29836ff6393SMartin Habets unsigned write_ptr;
29936ff6393SMartin Habets efx_oword_t reg;
30036ff6393SMartin Habets
30136ff6393SMartin Habets BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0);
30236ff6393SMartin Habets BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0);
30336ff6393SMartin Habets
30436ff6393SMartin Habets write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
30536ff6393SMartin Habets EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true,
30636ff6393SMartin Habets FRF_AZ_TX_DESC_WPTR, write_ptr);
30736ff6393SMartin Habets reg.qword[0] = *txd;
30836ff6393SMartin Habets efx_writeo_page(tx_queue->efx, ®,
30936ff6393SMartin Habets FR_BZ_TX_DESC_UPD_P0, tx_queue->queue);
31036ff6393SMartin Habets }
31136ff6393SMartin Habets
31236ff6393SMartin Habets
31336ff6393SMartin Habets /* For each entry inserted into the software descriptor ring, create a
31436ff6393SMartin Habets * descriptor in the hardware TX descriptor ring (in host memory), and
31536ff6393SMartin Habets * write a doorbell.
31636ff6393SMartin Habets */
efx_farch_tx_write(struct efx_tx_queue * tx_queue)31736ff6393SMartin Habets void efx_farch_tx_write(struct efx_tx_queue *tx_queue)
31836ff6393SMartin Habets {
31936ff6393SMartin Habets struct efx_tx_buffer *buffer;
32036ff6393SMartin Habets efx_qword_t *txd;
32136ff6393SMartin Habets unsigned write_ptr;
32236ff6393SMartin Habets unsigned old_write_count = tx_queue->write_count;
32336ff6393SMartin Habets
32436ff6393SMartin Habets tx_queue->xmit_pending = false;
32536ff6393SMartin Habets if (unlikely(tx_queue->write_count == tx_queue->insert_count))
32636ff6393SMartin Habets return;
32736ff6393SMartin Habets
32836ff6393SMartin Habets do {
32936ff6393SMartin Habets write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
33036ff6393SMartin Habets buffer = &tx_queue->buffer[write_ptr];
33136ff6393SMartin Habets txd = efx_tx_desc(tx_queue, write_ptr);
33236ff6393SMartin Habets ++tx_queue->write_count;
33336ff6393SMartin Habets
33436ff6393SMartin Habets EFX_WARN_ON_ONCE_PARANOID(buffer->flags & EFX_TX_BUF_OPTION);
33536ff6393SMartin Habets
33636ff6393SMartin Habets /* Create TX descriptor ring entry */
33736ff6393SMartin Habets BUILD_BUG_ON(EFX_TX_BUF_CONT != 1);
33836ff6393SMartin Habets EFX_POPULATE_QWORD_4(*txd,
33936ff6393SMartin Habets FSF_AZ_TX_KER_CONT,
34036ff6393SMartin Habets buffer->flags & EFX_TX_BUF_CONT,
34136ff6393SMartin Habets FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
34236ff6393SMartin Habets FSF_AZ_TX_KER_BUF_REGION, 0,
34336ff6393SMartin Habets FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
34436ff6393SMartin Habets } while (tx_queue->write_count != tx_queue->insert_count);
34536ff6393SMartin Habets
34636ff6393SMartin Habets wmb(); /* Ensure descriptors are written before they are fetched */
34736ff6393SMartin Habets
34836ff6393SMartin Habets if (efx_nic_may_push_tx_desc(tx_queue, old_write_count)) {
34936ff6393SMartin Habets txd = efx_tx_desc(tx_queue,
35036ff6393SMartin Habets old_write_count & tx_queue->ptr_mask);
35136ff6393SMartin Habets efx_farch_push_tx_desc(tx_queue, txd);
35236ff6393SMartin Habets ++tx_queue->pushes;
35336ff6393SMartin Habets } else {
35436ff6393SMartin Habets efx_farch_notify_tx_desc(tx_queue);
35536ff6393SMartin Habets }
35636ff6393SMartin Habets }
35736ff6393SMartin Habets
efx_farch_tx_limit_len(struct efx_tx_queue * tx_queue,dma_addr_t dma_addr,unsigned int len)35836ff6393SMartin Habets unsigned int efx_farch_tx_limit_len(struct efx_tx_queue *tx_queue,
35936ff6393SMartin Habets dma_addr_t dma_addr, unsigned int len)
36036ff6393SMartin Habets {
36136ff6393SMartin Habets /* Don't cross 4K boundaries with descriptors. */
36236ff6393SMartin Habets unsigned int limit = (~dma_addr & (EFX_PAGE_SIZE - 1)) + 1;
36336ff6393SMartin Habets
36436ff6393SMartin Habets len = min(limit, len);
36536ff6393SMartin Habets
36636ff6393SMartin Habets return len;
36736ff6393SMartin Habets }
36836ff6393SMartin Habets
36936ff6393SMartin Habets
37036ff6393SMartin Habets /* Allocate hardware resources for a TX queue */
efx_farch_tx_probe(struct efx_tx_queue * tx_queue)37136ff6393SMartin Habets int efx_farch_tx_probe(struct efx_tx_queue *tx_queue)
37236ff6393SMartin Habets {
37336ff6393SMartin Habets struct efx_nic *efx = tx_queue->efx;
37436ff6393SMartin Habets unsigned entries;
37536ff6393SMartin Habets
37636ff6393SMartin Habets tx_queue->type = ((tx_queue->label & 1) ? EFX_TXQ_TYPE_OUTER_CSUM : 0) |
37736ff6393SMartin Habets ((tx_queue->label & 2) ? EFX_TXQ_TYPE_HIGHPRI : 0);
37836ff6393SMartin Habets entries = tx_queue->ptr_mask + 1;
37936ff6393SMartin Habets return efx_alloc_special_buffer(efx, &tx_queue->txd,
38036ff6393SMartin Habets entries * sizeof(efx_qword_t));
38136ff6393SMartin Habets }
38236ff6393SMartin Habets
efx_farch_tx_init(struct efx_tx_queue * tx_queue)38336ff6393SMartin Habets void efx_farch_tx_init(struct efx_tx_queue *tx_queue)
38436ff6393SMartin Habets {
38536ff6393SMartin Habets int csum = tx_queue->type & EFX_TXQ_TYPE_OUTER_CSUM;
38636ff6393SMartin Habets struct efx_nic *efx = tx_queue->efx;
38736ff6393SMartin Habets efx_oword_t reg;
38836ff6393SMartin Habets
38936ff6393SMartin Habets /* Pin TX descriptor ring */
39036ff6393SMartin Habets efx_init_special_buffer(efx, &tx_queue->txd);
39136ff6393SMartin Habets
39236ff6393SMartin Habets /* Push TX descriptor ring to card */
39336ff6393SMartin Habets EFX_POPULATE_OWORD_10(reg,
39436ff6393SMartin Habets FRF_AZ_TX_DESCQ_EN, 1,
39536ff6393SMartin Habets FRF_AZ_TX_ISCSI_DDIG_EN, 0,
39636ff6393SMartin Habets FRF_AZ_TX_ISCSI_HDIG_EN, 0,
39736ff6393SMartin Habets FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
39836ff6393SMartin Habets FRF_AZ_TX_DESCQ_EVQ_ID,
39936ff6393SMartin Habets tx_queue->channel->channel,
40036ff6393SMartin Habets FRF_AZ_TX_DESCQ_OWNER_ID, 0,
40136ff6393SMartin Habets FRF_AZ_TX_DESCQ_LABEL, tx_queue->label,
40236ff6393SMartin Habets FRF_AZ_TX_DESCQ_SIZE,
40336ff6393SMartin Habets __ffs(tx_queue->txd.entries),
40436ff6393SMartin Habets FRF_AZ_TX_DESCQ_TYPE, 0,
40536ff6393SMartin Habets FRF_BZ_TX_NON_IP_DROP_DIS, 1);
40636ff6393SMartin Habets
40736ff6393SMartin Habets EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
40836ff6393SMartin Habets EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_TCP_CHKSM_DIS, !csum);
40936ff6393SMartin Habets
41036ff6393SMartin Habets efx_writeo_table(efx, ®, efx->type->txd_ptr_tbl_base,
41136ff6393SMartin Habets tx_queue->queue);
41236ff6393SMartin Habets
41336ff6393SMartin Habets EFX_POPULATE_OWORD_1(reg,
41436ff6393SMartin Habets FRF_BZ_TX_PACE,
41536ff6393SMartin Habets (tx_queue->type & EFX_TXQ_TYPE_HIGHPRI) ?
41636ff6393SMartin Habets FFE_BZ_TX_PACE_OFF :
41736ff6393SMartin Habets FFE_BZ_TX_PACE_RESERVED);
41836ff6393SMartin Habets efx_writeo_table(efx, ®, FR_BZ_TX_PACE_TBL, tx_queue->queue);
41936ff6393SMartin Habets
42036ff6393SMartin Habets tx_queue->tso_version = 1;
42136ff6393SMartin Habets }
42236ff6393SMartin Habets
efx_farch_flush_tx_queue(struct efx_tx_queue * tx_queue)42336ff6393SMartin Habets static void efx_farch_flush_tx_queue(struct efx_tx_queue *tx_queue)
42436ff6393SMartin Habets {
42536ff6393SMartin Habets struct efx_nic *efx = tx_queue->efx;
42636ff6393SMartin Habets efx_oword_t tx_flush_descq;
42736ff6393SMartin Habets
42836ff6393SMartin Habets WARN_ON(atomic_read(&tx_queue->flush_outstanding));
42936ff6393SMartin Habets atomic_set(&tx_queue->flush_outstanding, 1);
43036ff6393SMartin Habets
43136ff6393SMartin Habets EFX_POPULATE_OWORD_2(tx_flush_descq,
43236ff6393SMartin Habets FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
43336ff6393SMartin Habets FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
43436ff6393SMartin Habets efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
43536ff6393SMartin Habets }
43636ff6393SMartin Habets
efx_farch_tx_fini(struct efx_tx_queue * tx_queue)43736ff6393SMartin Habets void efx_farch_tx_fini(struct efx_tx_queue *tx_queue)
43836ff6393SMartin Habets {
43936ff6393SMartin Habets struct efx_nic *efx = tx_queue->efx;
44036ff6393SMartin Habets efx_oword_t tx_desc_ptr;
44136ff6393SMartin Habets
44236ff6393SMartin Habets /* Remove TX descriptor ring from card */
44336ff6393SMartin Habets EFX_ZERO_OWORD(tx_desc_ptr);
44436ff6393SMartin Habets efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
44536ff6393SMartin Habets tx_queue->queue);
44636ff6393SMartin Habets
44736ff6393SMartin Habets /* Unpin TX descriptor ring */
44836ff6393SMartin Habets efx_fini_special_buffer(efx, &tx_queue->txd);
44936ff6393SMartin Habets }
45036ff6393SMartin Habets
45136ff6393SMartin Habets /* Free buffers backing TX queue */
efx_farch_tx_remove(struct efx_tx_queue * tx_queue)45236ff6393SMartin Habets void efx_farch_tx_remove(struct efx_tx_queue *tx_queue)
45336ff6393SMartin Habets {
45436ff6393SMartin Habets efx_free_special_buffer(tx_queue->efx, &tx_queue->txd);
45536ff6393SMartin Habets }
45636ff6393SMartin Habets
45736ff6393SMartin Habets /**************************************************************************
45836ff6393SMartin Habets *
45936ff6393SMartin Habets * RX path
46036ff6393SMartin Habets *
46136ff6393SMartin Habets **************************************************************************/
46236ff6393SMartin Habets
46336ff6393SMartin Habets /* This creates an entry in the RX descriptor queue */
46436ff6393SMartin Habets static inline void
efx_farch_build_rx_desc(struct efx_rx_queue * rx_queue,unsigned index)46536ff6393SMartin Habets efx_farch_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index)
46636ff6393SMartin Habets {
46736ff6393SMartin Habets struct efx_rx_buffer *rx_buf;
46836ff6393SMartin Habets efx_qword_t *rxd;
46936ff6393SMartin Habets
47036ff6393SMartin Habets rxd = efx_rx_desc(rx_queue, index);
47136ff6393SMartin Habets rx_buf = efx_rx_buffer(rx_queue, index);
47236ff6393SMartin Habets EFX_POPULATE_QWORD_3(*rxd,
47336ff6393SMartin Habets FSF_AZ_RX_KER_BUF_SIZE,
47436ff6393SMartin Habets rx_buf->len -
47536ff6393SMartin Habets rx_queue->efx->type->rx_buffer_padding,
47636ff6393SMartin Habets FSF_AZ_RX_KER_BUF_REGION, 0,
47736ff6393SMartin Habets FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
47836ff6393SMartin Habets }
47936ff6393SMartin Habets
48036ff6393SMartin Habets /* This writes to the RX_DESC_WPTR register for the specified receive
48136ff6393SMartin Habets * descriptor ring.
48236ff6393SMartin Habets */
efx_farch_rx_write(struct efx_rx_queue * rx_queue)48336ff6393SMartin Habets void efx_farch_rx_write(struct efx_rx_queue *rx_queue)
48436ff6393SMartin Habets {
48536ff6393SMartin Habets struct efx_nic *efx = rx_queue->efx;
48636ff6393SMartin Habets efx_dword_t reg;
48736ff6393SMartin Habets unsigned write_ptr;
48836ff6393SMartin Habets
48936ff6393SMartin Habets while (rx_queue->notified_count != rx_queue->added_count) {
49036ff6393SMartin Habets efx_farch_build_rx_desc(
49136ff6393SMartin Habets rx_queue,
49236ff6393SMartin Habets rx_queue->notified_count & rx_queue->ptr_mask);
49336ff6393SMartin Habets ++rx_queue->notified_count;
49436ff6393SMartin Habets }
49536ff6393SMartin Habets
49636ff6393SMartin Habets wmb();
49736ff6393SMartin Habets write_ptr = rx_queue->added_count & rx_queue->ptr_mask;
49836ff6393SMartin Habets EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
49936ff6393SMartin Habets efx_writed_page(efx, ®, FR_AZ_RX_DESC_UPD_DWORD_P0,
50036ff6393SMartin Habets efx_rx_queue_index(rx_queue));
50136ff6393SMartin Habets }
50236ff6393SMartin Habets
efx_farch_rx_probe(struct efx_rx_queue * rx_queue)50336ff6393SMartin Habets int efx_farch_rx_probe(struct efx_rx_queue *rx_queue)
50436ff6393SMartin Habets {
50536ff6393SMartin Habets struct efx_nic *efx = rx_queue->efx;
50636ff6393SMartin Habets unsigned entries;
50736ff6393SMartin Habets
50836ff6393SMartin Habets entries = rx_queue->ptr_mask + 1;
50936ff6393SMartin Habets return efx_alloc_special_buffer(efx, &rx_queue->rxd,
51036ff6393SMartin Habets entries * sizeof(efx_qword_t));
51136ff6393SMartin Habets }
51236ff6393SMartin Habets
efx_farch_rx_init(struct efx_rx_queue * rx_queue)51336ff6393SMartin Habets void efx_farch_rx_init(struct efx_rx_queue *rx_queue)
51436ff6393SMartin Habets {
51536ff6393SMartin Habets efx_oword_t rx_desc_ptr;
51636ff6393SMartin Habets struct efx_nic *efx = rx_queue->efx;
51736ff6393SMartin Habets bool jumbo_en;
51836ff6393SMartin Habets
51936ff6393SMartin Habets /* For kernel-mode queues in Siena, the JUMBO flag enables scatter. */
52036ff6393SMartin Habets jumbo_en = efx->rx_scatter;
52136ff6393SMartin Habets
52236ff6393SMartin Habets netif_dbg(efx, hw, efx->net_dev,
52336ff6393SMartin Habets "RX queue %d ring in special buffers %d-%d\n",
52436ff6393SMartin Habets efx_rx_queue_index(rx_queue), rx_queue->rxd.index,
52536ff6393SMartin Habets rx_queue->rxd.index + rx_queue->rxd.entries - 1);
52636ff6393SMartin Habets
52736ff6393SMartin Habets rx_queue->scatter_n = 0;
52836ff6393SMartin Habets
52936ff6393SMartin Habets /* Pin RX descriptor ring */
53036ff6393SMartin Habets efx_init_special_buffer(efx, &rx_queue->rxd);
53136ff6393SMartin Habets
53236ff6393SMartin Habets /* Push RX descriptor ring to card */
53336ff6393SMartin Habets EFX_POPULATE_OWORD_10(rx_desc_ptr,
53436ff6393SMartin Habets FRF_AZ_RX_ISCSI_DDIG_EN, true,
53536ff6393SMartin Habets FRF_AZ_RX_ISCSI_HDIG_EN, true,
53636ff6393SMartin Habets FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
53736ff6393SMartin Habets FRF_AZ_RX_DESCQ_EVQ_ID,
53836ff6393SMartin Habets efx_rx_queue_channel(rx_queue)->channel,
53936ff6393SMartin Habets FRF_AZ_RX_DESCQ_OWNER_ID, 0,
54036ff6393SMartin Habets FRF_AZ_RX_DESCQ_LABEL,
54136ff6393SMartin Habets efx_rx_queue_index(rx_queue),
54236ff6393SMartin Habets FRF_AZ_RX_DESCQ_SIZE,
54336ff6393SMartin Habets __ffs(rx_queue->rxd.entries),
54436ff6393SMartin Habets FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
54536ff6393SMartin Habets FRF_AZ_RX_DESCQ_JUMBO, jumbo_en,
54636ff6393SMartin Habets FRF_AZ_RX_DESCQ_EN, 1);
54736ff6393SMartin Habets efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
54836ff6393SMartin Habets efx_rx_queue_index(rx_queue));
54936ff6393SMartin Habets }
55036ff6393SMartin Habets
efx_farch_flush_rx_queue(struct efx_rx_queue * rx_queue)55136ff6393SMartin Habets static void efx_farch_flush_rx_queue(struct efx_rx_queue *rx_queue)
55236ff6393SMartin Habets {
55336ff6393SMartin Habets struct efx_nic *efx = rx_queue->efx;
55436ff6393SMartin Habets efx_oword_t rx_flush_descq;
55536ff6393SMartin Habets
55636ff6393SMartin Habets EFX_POPULATE_OWORD_2(rx_flush_descq,
55736ff6393SMartin Habets FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
55836ff6393SMartin Habets FRF_AZ_RX_FLUSH_DESCQ,
55936ff6393SMartin Habets efx_rx_queue_index(rx_queue));
56036ff6393SMartin Habets efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
56136ff6393SMartin Habets }
56236ff6393SMartin Habets
efx_farch_rx_fini(struct efx_rx_queue * rx_queue)56336ff6393SMartin Habets void efx_farch_rx_fini(struct efx_rx_queue *rx_queue)
56436ff6393SMartin Habets {
56536ff6393SMartin Habets efx_oword_t rx_desc_ptr;
56636ff6393SMartin Habets struct efx_nic *efx = rx_queue->efx;
56736ff6393SMartin Habets
56836ff6393SMartin Habets /* Remove RX descriptor ring from card */
56936ff6393SMartin Habets EFX_ZERO_OWORD(rx_desc_ptr);
57036ff6393SMartin Habets efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
57136ff6393SMartin Habets efx_rx_queue_index(rx_queue));
57236ff6393SMartin Habets
57336ff6393SMartin Habets /* Unpin RX descriptor ring */
57436ff6393SMartin Habets efx_fini_special_buffer(efx, &rx_queue->rxd);
57536ff6393SMartin Habets }
57636ff6393SMartin Habets
57736ff6393SMartin Habets /* Free buffers backing RX queue */
efx_farch_rx_remove(struct efx_rx_queue * rx_queue)57836ff6393SMartin Habets void efx_farch_rx_remove(struct efx_rx_queue *rx_queue)
57936ff6393SMartin Habets {
58036ff6393SMartin Habets efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
58136ff6393SMartin Habets }
58236ff6393SMartin Habets
58336ff6393SMartin Habets /**************************************************************************
58436ff6393SMartin Habets *
58536ff6393SMartin Habets * Flush handling
58636ff6393SMartin Habets *
58736ff6393SMartin Habets **************************************************************************/
58836ff6393SMartin Habets
58936ff6393SMartin Habets /* efx_farch_flush_queues() must be woken up when all flushes are completed,
59036ff6393SMartin Habets * or more RX flushes can be kicked off.
59136ff6393SMartin Habets */
efx_farch_flush_wake(struct efx_nic * efx)59236ff6393SMartin Habets static bool efx_farch_flush_wake(struct efx_nic *efx)
59336ff6393SMartin Habets {
59436ff6393SMartin Habets /* Ensure that all updates are visible to efx_farch_flush_queues() */
59536ff6393SMartin Habets smp_mb();
59636ff6393SMartin Habets
59736ff6393SMartin Habets return (atomic_read(&efx->active_queues) == 0 ||
59836ff6393SMartin Habets (atomic_read(&efx->rxq_flush_outstanding) < EFX_RX_FLUSH_COUNT
59936ff6393SMartin Habets && atomic_read(&efx->rxq_flush_pending) > 0));
60036ff6393SMartin Habets }
60136ff6393SMartin Habets
efx_check_tx_flush_complete(struct efx_nic * efx)60236ff6393SMartin Habets static bool efx_check_tx_flush_complete(struct efx_nic *efx)
60336ff6393SMartin Habets {
60436ff6393SMartin Habets bool i = true;
60536ff6393SMartin Habets efx_oword_t txd_ptr_tbl;
60636ff6393SMartin Habets struct efx_channel *channel;
60736ff6393SMartin Habets struct efx_tx_queue *tx_queue;
60836ff6393SMartin Habets
60936ff6393SMartin Habets efx_for_each_channel(channel, efx) {
61036ff6393SMartin Habets efx_for_each_channel_tx_queue(tx_queue, channel) {
61136ff6393SMartin Habets efx_reado_table(efx, &txd_ptr_tbl,
61236ff6393SMartin Habets FR_BZ_TX_DESC_PTR_TBL, tx_queue->queue);
61336ff6393SMartin Habets if (EFX_OWORD_FIELD(txd_ptr_tbl,
61436ff6393SMartin Habets FRF_AZ_TX_DESCQ_FLUSH) ||
61536ff6393SMartin Habets EFX_OWORD_FIELD(txd_ptr_tbl,
61636ff6393SMartin Habets FRF_AZ_TX_DESCQ_EN)) {
61736ff6393SMartin Habets netif_dbg(efx, hw, efx->net_dev,
61836ff6393SMartin Habets "flush did not complete on TXQ %d\n",
61936ff6393SMartin Habets tx_queue->queue);
62036ff6393SMartin Habets i = false;
62136ff6393SMartin Habets } else if (atomic_cmpxchg(&tx_queue->flush_outstanding,
62236ff6393SMartin Habets 1, 0)) {
62336ff6393SMartin Habets /* The flush is complete, but we didn't
62436ff6393SMartin Habets * receive a flush completion event
62536ff6393SMartin Habets */
62636ff6393SMartin Habets netif_dbg(efx, hw, efx->net_dev,
62736ff6393SMartin Habets "flush complete on TXQ %d, so drain "
62836ff6393SMartin Habets "the queue\n", tx_queue->queue);
62936ff6393SMartin Habets /* Don't need to increment active_queues as it
63036ff6393SMartin Habets * has already been incremented for the queues
63136ff6393SMartin Habets * which did not drain
63236ff6393SMartin Habets */
63336ff6393SMartin Habets efx_farch_magic_event(channel,
63436ff6393SMartin Habets EFX_CHANNEL_MAGIC_TX_DRAIN(
63536ff6393SMartin Habets tx_queue));
63636ff6393SMartin Habets }
63736ff6393SMartin Habets }
63836ff6393SMartin Habets }
63936ff6393SMartin Habets
64036ff6393SMartin Habets return i;
64136ff6393SMartin Habets }
64236ff6393SMartin Habets
64336ff6393SMartin Habets /* Flush all the transmit queues, and continue flushing receive queues until
64436ff6393SMartin Habets * they're all flushed. Wait for the DRAIN events to be received so that there
64536ff6393SMartin Habets * are no more RX and TX events left on any channel. */
efx_farch_do_flush(struct efx_nic * efx)64636ff6393SMartin Habets static int efx_farch_do_flush(struct efx_nic *efx)
64736ff6393SMartin Habets {
64836ff6393SMartin Habets unsigned timeout = msecs_to_jiffies(5000); /* 5s for all flushes and drains */
64936ff6393SMartin Habets struct efx_channel *channel;
65036ff6393SMartin Habets struct efx_rx_queue *rx_queue;
65136ff6393SMartin Habets struct efx_tx_queue *tx_queue;
65236ff6393SMartin Habets int rc = 0;
65336ff6393SMartin Habets
65436ff6393SMartin Habets efx_for_each_channel(channel, efx) {
65536ff6393SMartin Habets efx_for_each_channel_tx_queue(tx_queue, channel) {
65636ff6393SMartin Habets efx_farch_flush_tx_queue(tx_queue);
65736ff6393SMartin Habets }
65836ff6393SMartin Habets efx_for_each_channel_rx_queue(rx_queue, channel) {
65936ff6393SMartin Habets rx_queue->flush_pending = true;
66036ff6393SMartin Habets atomic_inc(&efx->rxq_flush_pending);
66136ff6393SMartin Habets }
66236ff6393SMartin Habets }
66336ff6393SMartin Habets
66436ff6393SMartin Habets while (timeout && atomic_read(&efx->active_queues) > 0) {
66536ff6393SMartin Habets /* If SRIOV is enabled, then offload receive queue flushing to
66636ff6393SMartin Habets * the firmware (though we will still have to poll for
66736ff6393SMartin Habets * completion). If that fails, fall back to the old scheme.
66836ff6393SMartin Habets */
66936ff6393SMartin Habets if (efx_siena_sriov_enabled(efx)) {
6704d49e5cdSMartin Habets rc = efx_siena_mcdi_flush_rxqs(efx);
67136ff6393SMartin Habets if (!rc)
67236ff6393SMartin Habets goto wait;
67336ff6393SMartin Habets }
67436ff6393SMartin Habets
67536ff6393SMartin Habets /* The hardware supports four concurrent rx flushes, each of
67636ff6393SMartin Habets * which may need to be retried if there is an outstanding
67736ff6393SMartin Habets * descriptor fetch
67836ff6393SMartin Habets */
67936ff6393SMartin Habets efx_for_each_channel(channel, efx) {
68036ff6393SMartin Habets efx_for_each_channel_rx_queue(rx_queue, channel) {
68136ff6393SMartin Habets if (atomic_read(&efx->rxq_flush_outstanding) >=
68236ff6393SMartin Habets EFX_RX_FLUSH_COUNT)
68336ff6393SMartin Habets break;
68436ff6393SMartin Habets
68536ff6393SMartin Habets if (rx_queue->flush_pending) {
68636ff6393SMartin Habets rx_queue->flush_pending = false;
68736ff6393SMartin Habets atomic_dec(&efx->rxq_flush_pending);
68836ff6393SMartin Habets atomic_inc(&efx->rxq_flush_outstanding);
68936ff6393SMartin Habets efx_farch_flush_rx_queue(rx_queue);
69036ff6393SMartin Habets }
69136ff6393SMartin Habets }
69236ff6393SMartin Habets }
69336ff6393SMartin Habets
69436ff6393SMartin Habets wait:
69536ff6393SMartin Habets timeout = wait_event_timeout(efx->flush_wq,
69636ff6393SMartin Habets efx_farch_flush_wake(efx),
69736ff6393SMartin Habets timeout);
69836ff6393SMartin Habets }
69936ff6393SMartin Habets
70036ff6393SMartin Habets if (atomic_read(&efx->active_queues) &&
70136ff6393SMartin Habets !efx_check_tx_flush_complete(efx)) {
70236ff6393SMartin Habets netif_err(efx, hw, efx->net_dev, "failed to flush %d queues "
70336ff6393SMartin Habets "(rx %d+%d)\n", atomic_read(&efx->active_queues),
70436ff6393SMartin Habets atomic_read(&efx->rxq_flush_outstanding),
70536ff6393SMartin Habets atomic_read(&efx->rxq_flush_pending));
70636ff6393SMartin Habets rc = -ETIMEDOUT;
70736ff6393SMartin Habets
70836ff6393SMartin Habets atomic_set(&efx->active_queues, 0);
70936ff6393SMartin Habets atomic_set(&efx->rxq_flush_pending, 0);
71036ff6393SMartin Habets atomic_set(&efx->rxq_flush_outstanding, 0);
71136ff6393SMartin Habets }
71236ff6393SMartin Habets
71336ff6393SMartin Habets return rc;
71436ff6393SMartin Habets }
71536ff6393SMartin Habets
efx_farch_fini_dmaq(struct efx_nic * efx)71636ff6393SMartin Habets int efx_farch_fini_dmaq(struct efx_nic *efx)
71736ff6393SMartin Habets {
71836ff6393SMartin Habets struct efx_channel *channel;
71936ff6393SMartin Habets struct efx_tx_queue *tx_queue;
72036ff6393SMartin Habets struct efx_rx_queue *rx_queue;
72136ff6393SMartin Habets int rc = 0;
72236ff6393SMartin Habets
72336ff6393SMartin Habets /* Do not attempt to write to the NIC during EEH recovery */
72436ff6393SMartin Habets if (efx->state != STATE_RECOVERY) {
72536ff6393SMartin Habets /* Only perform flush if DMA is enabled */
72636ff6393SMartin Habets if (efx->pci_dev->is_busmaster) {
72736ff6393SMartin Habets efx->type->prepare_flush(efx);
72836ff6393SMartin Habets rc = efx_farch_do_flush(efx);
72936ff6393SMartin Habets efx->type->finish_flush(efx);
73036ff6393SMartin Habets }
73136ff6393SMartin Habets
73236ff6393SMartin Habets efx_for_each_channel(channel, efx) {
73336ff6393SMartin Habets efx_for_each_channel_rx_queue(rx_queue, channel)
73436ff6393SMartin Habets efx_farch_rx_fini(rx_queue);
73536ff6393SMartin Habets efx_for_each_channel_tx_queue(tx_queue, channel)
73636ff6393SMartin Habets efx_farch_tx_fini(tx_queue);
73736ff6393SMartin Habets }
73836ff6393SMartin Habets }
73936ff6393SMartin Habets
74036ff6393SMartin Habets return rc;
74136ff6393SMartin Habets }
74236ff6393SMartin Habets
74336ff6393SMartin Habets /* Reset queue and flush accounting after FLR
74436ff6393SMartin Habets *
74536ff6393SMartin Habets * One possible cause of FLR recovery is that DMA may be failing (eg. if bus
74636ff6393SMartin Habets * mastering was disabled), in which case we don't receive (RXQ) flush
74736ff6393SMartin Habets * completion events. This means that efx->rxq_flush_outstanding remained at 4
74836ff6393SMartin Habets * after the FLR; also, efx->active_queues was non-zero (as no flush completion
74936ff6393SMartin Habets * events were received, and we didn't go through efx_check_tx_flush_complete())
75071ad88f6SMartin Habets * If we don't fix this up, on the next call to efx_siena_realloc_channels() we
75171ad88f6SMartin Habets * won't flush any RX queues because efx->rxq_flush_outstanding is at the limit
75271ad88f6SMartin Habets * of 4 for batched flush requests; and the efx->active_queues gets messed up
75371ad88f6SMartin Habets * because we keep incrementing for the newly initialised queues, but it never
75471ad88f6SMartin Habets * went to zero previously. Then we get a timeout every time we try to restart
75571ad88f6SMartin Habets * the queues, as it doesn't go back to zero when we should be flushing the
75671ad88f6SMartin Habets * queues.
75736ff6393SMartin Habets */
efx_farch_finish_flr(struct efx_nic * efx)75836ff6393SMartin Habets void efx_farch_finish_flr(struct efx_nic *efx)
75936ff6393SMartin Habets {
76036ff6393SMartin Habets atomic_set(&efx->rxq_flush_pending, 0);
76136ff6393SMartin Habets atomic_set(&efx->rxq_flush_outstanding, 0);
76236ff6393SMartin Habets atomic_set(&efx->active_queues, 0);
76336ff6393SMartin Habets }
76436ff6393SMartin Habets
76536ff6393SMartin Habets
76636ff6393SMartin Habets /**************************************************************************
76736ff6393SMartin Habets *
76836ff6393SMartin Habets * Event queue processing
76936ff6393SMartin Habets * Event queues are processed by per-channel tasklets.
77036ff6393SMartin Habets *
77136ff6393SMartin Habets **************************************************************************/
77236ff6393SMartin Habets
77336ff6393SMartin Habets /* Update a channel's event queue's read pointer (RPTR) register
77436ff6393SMartin Habets *
77536ff6393SMartin Habets * This writes the EVQ_RPTR_REG register for the specified channel's
77636ff6393SMartin Habets * event queue.
77736ff6393SMartin Habets */
efx_farch_ev_read_ack(struct efx_channel * channel)77836ff6393SMartin Habets void efx_farch_ev_read_ack(struct efx_channel *channel)
77936ff6393SMartin Habets {
78036ff6393SMartin Habets efx_dword_t reg;
78136ff6393SMartin Habets struct efx_nic *efx = channel->efx;
78236ff6393SMartin Habets
78336ff6393SMartin Habets EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR,
78436ff6393SMartin Habets channel->eventq_read_ptr & channel->eventq_mask);
78536ff6393SMartin Habets
78636ff6393SMartin Habets /* For Falcon A1, EVQ_RPTR_KER is documented as having a step size
78736ff6393SMartin Habets * of 4 bytes, but it is really 16 bytes just like later revisions.
78836ff6393SMartin Habets */
78936ff6393SMartin Habets efx_writed(efx, ®,
79036ff6393SMartin Habets efx->type->evq_rptr_tbl_base +
79136ff6393SMartin Habets FR_BZ_EVQ_RPTR_STEP * channel->channel);
79236ff6393SMartin Habets }
79336ff6393SMartin Habets
79436ff6393SMartin Habets /* Use HW to insert a SW defined event */
efx_farch_generate_event(struct efx_nic * efx,unsigned int evq,efx_qword_t * event)79536ff6393SMartin Habets void efx_farch_generate_event(struct efx_nic *efx, unsigned int evq,
79636ff6393SMartin Habets efx_qword_t *event)
79736ff6393SMartin Habets {
79836ff6393SMartin Habets efx_oword_t drv_ev_reg;
79936ff6393SMartin Habets
80036ff6393SMartin Habets BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
80136ff6393SMartin Habets FRF_AZ_DRV_EV_DATA_WIDTH != 64);
80236ff6393SMartin Habets drv_ev_reg.u32[0] = event->u32[0];
80336ff6393SMartin Habets drv_ev_reg.u32[1] = event->u32[1];
80436ff6393SMartin Habets drv_ev_reg.u32[2] = 0;
80536ff6393SMartin Habets drv_ev_reg.u32[3] = 0;
80636ff6393SMartin Habets EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, evq);
80736ff6393SMartin Habets efx_writeo(efx, &drv_ev_reg, FR_AZ_DRV_EV);
80836ff6393SMartin Habets }
80936ff6393SMartin Habets
efx_farch_magic_event(struct efx_channel * channel,u32 magic)81036ff6393SMartin Habets static void efx_farch_magic_event(struct efx_channel *channel, u32 magic)
81136ff6393SMartin Habets {
81236ff6393SMartin Habets efx_qword_t event;
81336ff6393SMartin Habets
81436ff6393SMartin Habets EFX_POPULATE_QWORD_2(event, FSF_AZ_EV_CODE,
81536ff6393SMartin Habets FSE_AZ_EV_CODE_DRV_GEN_EV,
81636ff6393SMartin Habets FSF_AZ_DRV_GEN_EV_MAGIC, magic);
81736ff6393SMartin Habets efx_farch_generate_event(channel->efx, channel->channel, &event);
81836ff6393SMartin Habets }
81936ff6393SMartin Habets
82036ff6393SMartin Habets /* Handle a transmit completion event
82136ff6393SMartin Habets *
82236ff6393SMartin Habets * The NIC batches TX completion events; the message we receive is of
82336ff6393SMartin Habets * the form "complete all TX events up to this index".
82436ff6393SMartin Habets */
82536ff6393SMartin Habets static void
efx_farch_handle_tx_event(struct efx_channel * channel,efx_qword_t * event)82636ff6393SMartin Habets efx_farch_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
82736ff6393SMartin Habets {
82836ff6393SMartin Habets unsigned int tx_ev_desc_ptr;
82936ff6393SMartin Habets unsigned int tx_ev_q_label;
83036ff6393SMartin Habets struct efx_tx_queue *tx_queue;
83136ff6393SMartin Habets struct efx_nic *efx = channel->efx;
83236ff6393SMartin Habets
83336ff6393SMartin Habets if (unlikely(READ_ONCE(efx->reset_pending)))
83436ff6393SMartin Habets return;
83536ff6393SMartin Habets
83636ff6393SMartin Habets if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
83736ff6393SMartin Habets /* Transmit completion */
83836ff6393SMartin Habets tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
83936ff6393SMartin Habets tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
84036ff6393SMartin Habets tx_queue = channel->tx_queue +
84136ff6393SMartin Habets (tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL);
84271ad88f6SMartin Habets efx_siena_xmit_done(tx_queue, tx_ev_desc_ptr);
84336ff6393SMartin Habets } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
84436ff6393SMartin Habets /* Rewrite the FIFO write pointer */
84536ff6393SMartin Habets tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
84636ff6393SMartin Habets tx_queue = channel->tx_queue +
84736ff6393SMartin Habets (tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL);
84836ff6393SMartin Habets
84936ff6393SMartin Habets netif_tx_lock(efx->net_dev);
85036ff6393SMartin Habets efx_farch_notify_tx_desc(tx_queue);
85136ff6393SMartin Habets netif_tx_unlock(efx->net_dev);
85236ff6393SMartin Habets } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR)) {
85371ad88f6SMartin Habets efx_siena_schedule_reset(efx, RESET_TYPE_DMA_ERROR);
85436ff6393SMartin Habets } else {
85536ff6393SMartin Habets netif_err(efx, tx_err, efx->net_dev,
85636ff6393SMartin Habets "channel %d unexpected TX event "
85736ff6393SMartin Habets EFX_QWORD_FMT"\n", channel->channel,
85836ff6393SMartin Habets EFX_QWORD_VAL(*event));
85936ff6393SMartin Habets }
86036ff6393SMartin Habets }
86136ff6393SMartin Habets
86236ff6393SMartin Habets /* Detect errors included in the rx_evt_pkt_ok bit. */
efx_farch_handle_rx_not_ok(struct efx_rx_queue * rx_queue,const efx_qword_t * event)86336ff6393SMartin Habets static u16 efx_farch_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
86436ff6393SMartin Habets const efx_qword_t *event)
86536ff6393SMartin Habets {
86636ff6393SMartin Habets struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
86736ff6393SMartin Habets struct efx_nic *efx = rx_queue->efx;
86836ff6393SMartin Habets bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
86936ff6393SMartin Habets bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
87036ff6393SMartin Habets bool rx_ev_frm_trunc, rx_ev_tobe_disc;
87136ff6393SMartin Habets bool rx_ev_other_err, rx_ev_pause_frm;
87236ff6393SMartin Habets
87336ff6393SMartin Habets rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
87436ff6393SMartin Habets rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
87536ff6393SMartin Habets FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
87636ff6393SMartin Habets rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
87736ff6393SMartin Habets FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
87836ff6393SMartin Habets rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
87936ff6393SMartin Habets FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
88036ff6393SMartin Habets rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
88136ff6393SMartin Habets rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
88236ff6393SMartin Habets rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
88336ff6393SMartin Habets
88436ff6393SMartin Habets /* Every error apart from tobe_disc and pause_frm */
88536ff6393SMartin Habets rx_ev_other_err = (rx_ev_tcp_udp_chksum_err |
88636ff6393SMartin Habets rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
88736ff6393SMartin Habets rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);
88836ff6393SMartin Habets
88936ff6393SMartin Habets /* Count errors that are not in MAC stats. Ignore expected
89036ff6393SMartin Habets * checksum errors during self-test. */
89136ff6393SMartin Habets if (rx_ev_frm_trunc)
89236ff6393SMartin Habets ++channel->n_rx_frm_trunc;
89336ff6393SMartin Habets else if (rx_ev_tobe_disc)
89436ff6393SMartin Habets ++channel->n_rx_tobe_disc;
89536ff6393SMartin Habets else if (!efx->loopback_selftest) {
89636ff6393SMartin Habets if (rx_ev_ip_hdr_chksum_err)
89736ff6393SMartin Habets ++channel->n_rx_ip_hdr_chksum_err;
89836ff6393SMartin Habets else if (rx_ev_tcp_udp_chksum_err)
89936ff6393SMartin Habets ++channel->n_rx_tcp_udp_chksum_err;
90036ff6393SMartin Habets }
90136ff6393SMartin Habets
90236ff6393SMartin Habets /* TOBE_DISC is expected on unicast mismatches; don't print out an
90336ff6393SMartin Habets * error message. FRM_TRUNC indicates RXDP dropped the packet due
90436ff6393SMartin Habets * to a FIFO overflow.
90536ff6393SMartin Habets */
90636ff6393SMartin Habets #ifdef DEBUG
90736ff6393SMartin Habets if (rx_ev_other_err && net_ratelimit()) {
90836ff6393SMartin Habets netif_dbg(efx, rx_err, efx->net_dev,
90936ff6393SMartin Habets " RX queue %d unexpected RX event "
91036ff6393SMartin Habets EFX_QWORD_FMT "%s%s%s%s%s%s%s\n",
91136ff6393SMartin Habets efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event),
91236ff6393SMartin Habets rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
91336ff6393SMartin Habets rx_ev_ip_hdr_chksum_err ?
91436ff6393SMartin Habets " [IP_HDR_CHKSUM_ERR]" : "",
91536ff6393SMartin Habets rx_ev_tcp_udp_chksum_err ?
91636ff6393SMartin Habets " [TCP_UDP_CHKSUM_ERR]" : "",
91736ff6393SMartin Habets rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
91836ff6393SMartin Habets rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
91936ff6393SMartin Habets rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
92036ff6393SMartin Habets rx_ev_pause_frm ? " [PAUSE]" : "");
92136ff6393SMartin Habets }
92236ff6393SMartin Habets #else
92336ff6393SMartin Habets (void) rx_ev_other_err;
92436ff6393SMartin Habets #endif
92536ff6393SMartin Habets
92636ff6393SMartin Habets if (efx->net_dev->features & NETIF_F_RXALL)
92736ff6393SMartin Habets /* don't discard frame for CRC error */
92836ff6393SMartin Habets rx_ev_eth_crc_err = false;
92936ff6393SMartin Habets
93036ff6393SMartin Habets /* The frame must be discarded if any of these are true. */
93136ff6393SMartin Habets return (rx_ev_eth_crc_err | rx_ev_frm_trunc |
93236ff6393SMartin Habets rx_ev_tobe_disc | rx_ev_pause_frm) ?
93336ff6393SMartin Habets EFX_RX_PKT_DISCARD : 0;
93436ff6393SMartin Habets }
93536ff6393SMartin Habets
93636ff6393SMartin Habets /* Handle receive events that are not in-order. Return true if this
93736ff6393SMartin Habets * can be handled as a partial packet discard, false if it's more
93836ff6393SMartin Habets * serious.
93936ff6393SMartin Habets */
94036ff6393SMartin Habets static bool
efx_farch_handle_rx_bad_index(struct efx_rx_queue * rx_queue,unsigned index)94136ff6393SMartin Habets efx_farch_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
94236ff6393SMartin Habets {
94336ff6393SMartin Habets struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
94436ff6393SMartin Habets struct efx_nic *efx = rx_queue->efx;
94536ff6393SMartin Habets unsigned expected, dropped;
94636ff6393SMartin Habets
94736ff6393SMartin Habets if (rx_queue->scatter_n &&
94836ff6393SMartin Habets index == ((rx_queue->removed_count + rx_queue->scatter_n - 1) &
94936ff6393SMartin Habets rx_queue->ptr_mask)) {
95036ff6393SMartin Habets ++channel->n_rx_nodesc_trunc;
95136ff6393SMartin Habets return true;
95236ff6393SMartin Habets }
95336ff6393SMartin Habets
95436ff6393SMartin Habets expected = rx_queue->removed_count & rx_queue->ptr_mask;
95536ff6393SMartin Habets dropped = (index - expected) & rx_queue->ptr_mask;
95636ff6393SMartin Habets netif_info(efx, rx_err, efx->net_dev,
95736ff6393SMartin Habets "dropped %d events (index=%d expected=%d)\n",
95836ff6393SMartin Habets dropped, index, expected);
95936ff6393SMartin Habets
96071ad88f6SMartin Habets efx_siena_schedule_reset(efx, RESET_TYPE_DISABLE);
96136ff6393SMartin Habets return false;
96236ff6393SMartin Habets }
96336ff6393SMartin Habets
96436ff6393SMartin Habets /* Handle a packet received event
96536ff6393SMartin Habets *
96636ff6393SMartin Habets * The NIC gives a "discard" flag if it's a unicast packet with the
96736ff6393SMartin Habets * wrong destination address
96836ff6393SMartin Habets * Also "is multicast" and "matches multicast filter" flags can be used to
96936ff6393SMartin Habets * discard non-matching multicast packets.
97036ff6393SMartin Habets */
97136ff6393SMartin Habets static void
efx_farch_handle_rx_event(struct efx_channel * channel,const efx_qword_t * event)97236ff6393SMartin Habets efx_farch_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
97336ff6393SMartin Habets {
97436ff6393SMartin Habets unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
97536ff6393SMartin Habets unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
97636ff6393SMartin Habets unsigned expected_ptr;
97736ff6393SMartin Habets bool rx_ev_pkt_ok, rx_ev_sop, rx_ev_cont;
97836ff6393SMartin Habets u16 flags;
97936ff6393SMartin Habets struct efx_rx_queue *rx_queue;
98036ff6393SMartin Habets struct efx_nic *efx = channel->efx;
98136ff6393SMartin Habets
98236ff6393SMartin Habets if (unlikely(READ_ONCE(efx->reset_pending)))
98336ff6393SMartin Habets return;
98436ff6393SMartin Habets
98536ff6393SMartin Habets rx_ev_cont = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT);
98636ff6393SMartin Habets rx_ev_sop = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP);
98736ff6393SMartin Habets WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
98836ff6393SMartin Habets channel->channel);
98936ff6393SMartin Habets
99036ff6393SMartin Habets rx_queue = efx_channel_get_rx_queue(channel);
99136ff6393SMartin Habets
99236ff6393SMartin Habets rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
99336ff6393SMartin Habets expected_ptr = ((rx_queue->removed_count + rx_queue->scatter_n) &
99436ff6393SMartin Habets rx_queue->ptr_mask);
99536ff6393SMartin Habets
99636ff6393SMartin Habets /* Check for partial drops and other errors */
99736ff6393SMartin Habets if (unlikely(rx_ev_desc_ptr != expected_ptr) ||
99836ff6393SMartin Habets unlikely(rx_ev_sop != (rx_queue->scatter_n == 0))) {
99936ff6393SMartin Habets if (rx_ev_desc_ptr != expected_ptr &&
100036ff6393SMartin Habets !efx_farch_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr))
100136ff6393SMartin Habets return;
100236ff6393SMartin Habets
100336ff6393SMartin Habets /* Discard all pending fragments */
100436ff6393SMartin Habets if (rx_queue->scatter_n) {
100571ad88f6SMartin Habets efx_siena_rx_packet(
100636ff6393SMartin Habets rx_queue,
100736ff6393SMartin Habets rx_queue->removed_count & rx_queue->ptr_mask,
100836ff6393SMartin Habets rx_queue->scatter_n, 0, EFX_RX_PKT_DISCARD);
100936ff6393SMartin Habets rx_queue->removed_count += rx_queue->scatter_n;
101036ff6393SMartin Habets rx_queue->scatter_n = 0;
101136ff6393SMartin Habets }
101236ff6393SMartin Habets
101336ff6393SMartin Habets /* Return if there is no new fragment */
101436ff6393SMartin Habets if (rx_ev_desc_ptr != expected_ptr)
101536ff6393SMartin Habets return;
101636ff6393SMartin Habets
101736ff6393SMartin Habets /* Discard new fragment if not SOP */
101836ff6393SMartin Habets if (!rx_ev_sop) {
101971ad88f6SMartin Habets efx_siena_rx_packet(
102036ff6393SMartin Habets rx_queue,
102136ff6393SMartin Habets rx_queue->removed_count & rx_queue->ptr_mask,
102236ff6393SMartin Habets 1, 0, EFX_RX_PKT_DISCARD);
102336ff6393SMartin Habets ++rx_queue->removed_count;
102436ff6393SMartin Habets return;
102536ff6393SMartin Habets }
102636ff6393SMartin Habets }
102736ff6393SMartin Habets
102836ff6393SMartin Habets ++rx_queue->scatter_n;
102936ff6393SMartin Habets if (rx_ev_cont)
103036ff6393SMartin Habets return;
103136ff6393SMartin Habets
103236ff6393SMartin Habets rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
103336ff6393SMartin Habets rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
103436ff6393SMartin Habets rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
103536ff6393SMartin Habets
103636ff6393SMartin Habets if (likely(rx_ev_pkt_ok)) {
103736ff6393SMartin Habets /* If packet is marked as OK then we can rely on the
103836ff6393SMartin Habets * hardware checksum and classification.
103936ff6393SMartin Habets */
104036ff6393SMartin Habets flags = 0;
104136ff6393SMartin Habets switch (rx_ev_hdr_type) {
104236ff6393SMartin Habets case FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
104336ff6393SMartin Habets flags |= EFX_RX_PKT_TCP;
104436ff6393SMartin Habets fallthrough;
104536ff6393SMartin Habets case FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
104636ff6393SMartin Habets flags |= EFX_RX_PKT_CSUMMED;
104736ff6393SMartin Habets fallthrough;
104836ff6393SMartin Habets case FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
104936ff6393SMartin Habets case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
105036ff6393SMartin Habets break;
105136ff6393SMartin Habets }
105236ff6393SMartin Habets } else {
105336ff6393SMartin Habets flags = efx_farch_handle_rx_not_ok(rx_queue, event);
105436ff6393SMartin Habets }
105536ff6393SMartin Habets
105636ff6393SMartin Habets /* Detect multicast packets that didn't match the filter */
105736ff6393SMartin Habets rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
105836ff6393SMartin Habets if (rx_ev_mcast_pkt) {
105936ff6393SMartin Habets unsigned int rx_ev_mcast_hash_match =
106036ff6393SMartin Habets EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
106136ff6393SMartin Habets
106236ff6393SMartin Habets if (unlikely(!rx_ev_mcast_hash_match)) {
106336ff6393SMartin Habets ++channel->n_rx_mcast_mismatch;
106436ff6393SMartin Habets flags |= EFX_RX_PKT_DISCARD;
106536ff6393SMartin Habets }
106636ff6393SMartin Habets }
106736ff6393SMartin Habets
106836ff6393SMartin Habets channel->irq_mod_score += 2;
106936ff6393SMartin Habets
107036ff6393SMartin Habets /* Handle received packet */
107171ad88f6SMartin Habets efx_siena_rx_packet(rx_queue,
107236ff6393SMartin Habets rx_queue->removed_count & rx_queue->ptr_mask,
107336ff6393SMartin Habets rx_queue->scatter_n, rx_ev_byte_cnt, flags);
107436ff6393SMartin Habets rx_queue->removed_count += rx_queue->scatter_n;
107536ff6393SMartin Habets rx_queue->scatter_n = 0;
107636ff6393SMartin Habets }
107736ff6393SMartin Habets
107836ff6393SMartin Habets /* If this flush done event corresponds to a &struct efx_tx_queue, then
107936ff6393SMartin Habets * send an %EFX_CHANNEL_MAGIC_TX_DRAIN event to drain the event queue
108036ff6393SMartin Habets * of all transmit completions.
108136ff6393SMartin Habets */
108236ff6393SMartin Habets static void
efx_farch_handle_tx_flush_done(struct efx_nic * efx,efx_qword_t * event)108336ff6393SMartin Habets efx_farch_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event)
108436ff6393SMartin Habets {
108536ff6393SMartin Habets struct efx_tx_queue *tx_queue;
108636ff6393SMartin Habets struct efx_channel *channel;
108736ff6393SMartin Habets int qid;
108836ff6393SMartin Habets
108936ff6393SMartin Habets qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
109036ff6393SMartin Habets if (qid < EFX_MAX_TXQ_PER_CHANNEL * (efx->n_tx_channels + efx->n_extra_tx_channels)) {
109136ff6393SMartin Habets channel = efx_get_tx_channel(efx, qid / EFX_MAX_TXQ_PER_CHANNEL);
109236ff6393SMartin Habets tx_queue = channel->tx_queue + (qid % EFX_MAX_TXQ_PER_CHANNEL);
109336ff6393SMartin Habets if (atomic_cmpxchg(&tx_queue->flush_outstanding, 1, 0))
109436ff6393SMartin Habets efx_farch_magic_event(tx_queue->channel,
109536ff6393SMartin Habets EFX_CHANNEL_MAGIC_TX_DRAIN(tx_queue));
109636ff6393SMartin Habets }
109736ff6393SMartin Habets }
109836ff6393SMartin Habets
109936ff6393SMartin Habets /* If this flush done event corresponds to a &struct efx_rx_queue: If the flush
110036ff6393SMartin Habets * was successful then send an %EFX_CHANNEL_MAGIC_RX_DRAIN, otherwise add
110136ff6393SMartin Habets * the RX queue back to the mask of RX queues in need of flushing.
110236ff6393SMartin Habets */
110336ff6393SMartin Habets static void
efx_farch_handle_rx_flush_done(struct efx_nic * efx,efx_qword_t * event)110436ff6393SMartin Habets efx_farch_handle_rx_flush_done(struct efx_nic *efx, efx_qword_t *event)
110536ff6393SMartin Habets {
110636ff6393SMartin Habets struct efx_channel *channel;
110736ff6393SMartin Habets struct efx_rx_queue *rx_queue;
110836ff6393SMartin Habets int qid;
110936ff6393SMartin Habets bool failed;
111036ff6393SMartin Habets
111136ff6393SMartin Habets qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
111236ff6393SMartin Habets failed = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
111336ff6393SMartin Habets if (qid >= efx->n_channels)
111436ff6393SMartin Habets return;
111536ff6393SMartin Habets channel = efx_get_channel(efx, qid);
111636ff6393SMartin Habets if (!efx_channel_has_rx_queue(channel))
111736ff6393SMartin Habets return;
111836ff6393SMartin Habets rx_queue = efx_channel_get_rx_queue(channel);
111936ff6393SMartin Habets
112036ff6393SMartin Habets if (failed) {
112136ff6393SMartin Habets netif_info(efx, hw, efx->net_dev,
112236ff6393SMartin Habets "RXQ %d flush retry\n", qid);
112336ff6393SMartin Habets rx_queue->flush_pending = true;
112436ff6393SMartin Habets atomic_inc(&efx->rxq_flush_pending);
112536ff6393SMartin Habets } else {
112636ff6393SMartin Habets efx_farch_magic_event(efx_rx_queue_channel(rx_queue),
112736ff6393SMartin Habets EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue));
112836ff6393SMartin Habets }
112936ff6393SMartin Habets atomic_dec(&efx->rxq_flush_outstanding);
113036ff6393SMartin Habets if (efx_farch_flush_wake(efx))
113136ff6393SMartin Habets wake_up(&efx->flush_wq);
113236ff6393SMartin Habets }
113336ff6393SMartin Habets
113436ff6393SMartin Habets static void
efx_farch_handle_drain_event(struct efx_channel * channel)113536ff6393SMartin Habets efx_farch_handle_drain_event(struct efx_channel *channel)
113636ff6393SMartin Habets {
113736ff6393SMartin Habets struct efx_nic *efx = channel->efx;
113836ff6393SMartin Habets
113936ff6393SMartin Habets WARN_ON(atomic_read(&efx->active_queues) == 0);
114036ff6393SMartin Habets atomic_dec(&efx->active_queues);
114136ff6393SMartin Habets if (efx_farch_flush_wake(efx))
114236ff6393SMartin Habets wake_up(&efx->flush_wq);
114336ff6393SMartin Habets }
114436ff6393SMartin Habets
efx_farch_handle_generated_event(struct efx_channel * channel,efx_qword_t * event)114536ff6393SMartin Habets static void efx_farch_handle_generated_event(struct efx_channel *channel,
114636ff6393SMartin Habets efx_qword_t *event)
114736ff6393SMartin Habets {
114836ff6393SMartin Habets struct efx_nic *efx = channel->efx;
114936ff6393SMartin Habets struct efx_rx_queue *rx_queue =
115036ff6393SMartin Habets efx_channel_has_rx_queue(channel) ?
115136ff6393SMartin Habets efx_channel_get_rx_queue(channel) : NULL;
115236ff6393SMartin Habets unsigned magic, code;
115336ff6393SMartin Habets
115436ff6393SMartin Habets magic = EFX_QWORD_FIELD(*event, FSF_AZ_DRV_GEN_EV_MAGIC);
115536ff6393SMartin Habets code = _EFX_CHANNEL_MAGIC_CODE(magic);
115636ff6393SMartin Habets
115736ff6393SMartin Habets if (magic == EFX_CHANNEL_MAGIC_TEST(channel)) {
115836ff6393SMartin Habets channel->event_test_cpu = raw_smp_processor_id();
115936ff6393SMartin Habets } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_FILL(rx_queue)) {
116036ff6393SMartin Habets /* The queue must be empty, so we won't receive any rx
116136ff6393SMartin Habets * events, so efx_process_channel() won't refill the
116236ff6393SMartin Habets * queue. Refill it here */
11637f9e4b2aSMartin Habets efx_siena_fast_push_rx_descriptors(rx_queue, true);
116436ff6393SMartin Habets } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue)) {
116536ff6393SMartin Habets efx_farch_handle_drain_event(channel);
116636ff6393SMartin Habets } else if (code == _EFX_CHANNEL_MAGIC_TX_DRAIN) {
116736ff6393SMartin Habets efx_farch_handle_drain_event(channel);
116836ff6393SMartin Habets } else {
116936ff6393SMartin Habets netif_dbg(efx, hw, efx->net_dev, "channel %d received "
117036ff6393SMartin Habets "generated event "EFX_QWORD_FMT"\n",
117136ff6393SMartin Habets channel->channel, EFX_QWORD_VAL(*event));
117236ff6393SMartin Habets }
117336ff6393SMartin Habets }
117436ff6393SMartin Habets
117536ff6393SMartin Habets static void
efx_farch_handle_driver_event(struct efx_channel * channel,efx_qword_t * event)117636ff6393SMartin Habets efx_farch_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
117736ff6393SMartin Habets {
117836ff6393SMartin Habets struct efx_nic *efx = channel->efx;
117936ff6393SMartin Habets unsigned int ev_sub_code;
118036ff6393SMartin Habets unsigned int ev_sub_data;
118136ff6393SMartin Habets
118236ff6393SMartin Habets ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
118336ff6393SMartin Habets ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
118436ff6393SMartin Habets
118536ff6393SMartin Habets switch (ev_sub_code) {
118636ff6393SMartin Habets case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
118736ff6393SMartin Habets netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n",
118836ff6393SMartin Habets channel->channel, ev_sub_data);
118936ff6393SMartin Habets efx_farch_handle_tx_flush_done(efx, event);
1190dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
119136ff6393SMartin Habets efx_siena_sriov_tx_flush_done(efx, event);
119236ff6393SMartin Habets #endif
119336ff6393SMartin Habets break;
119436ff6393SMartin Habets case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
119536ff6393SMartin Habets netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n",
119636ff6393SMartin Habets channel->channel, ev_sub_data);
119736ff6393SMartin Habets efx_farch_handle_rx_flush_done(efx, event);
1198dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
119936ff6393SMartin Habets efx_siena_sriov_rx_flush_done(efx, event);
120036ff6393SMartin Habets #endif
120136ff6393SMartin Habets break;
120236ff6393SMartin Habets case FSE_AZ_EVQ_INIT_DONE_EV:
120336ff6393SMartin Habets netif_dbg(efx, hw, efx->net_dev,
120436ff6393SMartin Habets "channel %d EVQ %d initialised\n",
120536ff6393SMartin Habets channel->channel, ev_sub_data);
120636ff6393SMartin Habets break;
120736ff6393SMartin Habets case FSE_AZ_SRM_UPD_DONE_EV:
120836ff6393SMartin Habets netif_vdbg(efx, hw, efx->net_dev,
120936ff6393SMartin Habets "channel %d SRAM update done\n", channel->channel);
121036ff6393SMartin Habets break;
121136ff6393SMartin Habets case FSE_AZ_WAKE_UP_EV:
121236ff6393SMartin Habets netif_vdbg(efx, hw, efx->net_dev,
121336ff6393SMartin Habets "channel %d RXQ %d wakeup event\n",
121436ff6393SMartin Habets channel->channel, ev_sub_data);
121536ff6393SMartin Habets break;
121636ff6393SMartin Habets case FSE_AZ_TIMER_EV:
121736ff6393SMartin Habets netif_vdbg(efx, hw, efx->net_dev,
121836ff6393SMartin Habets "channel %d RX queue %d timer expired\n",
121936ff6393SMartin Habets channel->channel, ev_sub_data);
122036ff6393SMartin Habets break;
122136ff6393SMartin Habets case FSE_AA_RX_RECOVER_EV:
122236ff6393SMartin Habets netif_err(efx, rx_err, efx->net_dev,
122336ff6393SMartin Habets "channel %d seen DRIVER RX_RESET event. "
122436ff6393SMartin Habets "Resetting.\n", channel->channel);
122536ff6393SMartin Habets atomic_inc(&efx->rx_reset);
122671ad88f6SMartin Habets efx_siena_schedule_reset(efx, RESET_TYPE_DISABLE);
122736ff6393SMartin Habets break;
122836ff6393SMartin Habets case FSE_BZ_RX_DSC_ERROR_EV:
122936ff6393SMartin Habets if (ev_sub_data < EFX_VI_BASE) {
123036ff6393SMartin Habets netif_err(efx, rx_err, efx->net_dev,
123136ff6393SMartin Habets "RX DMA Q %d reports descriptor fetch error."
123236ff6393SMartin Habets " RX Q %d is disabled.\n", ev_sub_data,
123336ff6393SMartin Habets ev_sub_data);
123471ad88f6SMartin Habets efx_siena_schedule_reset(efx, RESET_TYPE_DMA_ERROR);
123536ff6393SMartin Habets }
1236dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
123736ff6393SMartin Habets else
123836ff6393SMartin Habets efx_siena_sriov_desc_fetch_err(efx, ev_sub_data);
123936ff6393SMartin Habets #endif
124036ff6393SMartin Habets break;
124136ff6393SMartin Habets case FSE_BZ_TX_DSC_ERROR_EV:
124236ff6393SMartin Habets if (ev_sub_data < EFX_VI_BASE) {
124336ff6393SMartin Habets netif_err(efx, tx_err, efx->net_dev,
124436ff6393SMartin Habets "TX DMA Q %d reports descriptor fetch error."
124536ff6393SMartin Habets " TX Q %d is disabled.\n", ev_sub_data,
124636ff6393SMartin Habets ev_sub_data);
124771ad88f6SMartin Habets efx_siena_schedule_reset(efx, RESET_TYPE_DMA_ERROR);
124836ff6393SMartin Habets }
1249dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
125036ff6393SMartin Habets else
125136ff6393SMartin Habets efx_siena_sriov_desc_fetch_err(efx, ev_sub_data);
125236ff6393SMartin Habets #endif
125336ff6393SMartin Habets break;
125436ff6393SMartin Habets default:
125536ff6393SMartin Habets netif_vdbg(efx, hw, efx->net_dev,
125636ff6393SMartin Habets "channel %d unknown driver event code %d "
125736ff6393SMartin Habets "data %04x\n", channel->channel, ev_sub_code,
125836ff6393SMartin Habets ev_sub_data);
125936ff6393SMartin Habets break;
126036ff6393SMartin Habets }
126136ff6393SMartin Habets }
126236ff6393SMartin Habets
efx_farch_ev_process(struct efx_channel * channel,int budget)126336ff6393SMartin Habets int efx_farch_ev_process(struct efx_channel *channel, int budget)
126436ff6393SMartin Habets {
126536ff6393SMartin Habets struct efx_nic *efx = channel->efx;
126636ff6393SMartin Habets unsigned int read_ptr;
126736ff6393SMartin Habets efx_qword_t event, *p_event;
126836ff6393SMartin Habets int ev_code;
126936ff6393SMartin Habets int spent = 0;
127036ff6393SMartin Habets
127136ff6393SMartin Habets if (budget <= 0)
127236ff6393SMartin Habets return spent;
127336ff6393SMartin Habets
127436ff6393SMartin Habets read_ptr = channel->eventq_read_ptr;
127536ff6393SMartin Habets
127636ff6393SMartin Habets for (;;) {
127736ff6393SMartin Habets p_event = efx_event(channel, read_ptr);
127836ff6393SMartin Habets event = *p_event;
127936ff6393SMartin Habets
128036ff6393SMartin Habets if (!efx_event_present(&event))
128136ff6393SMartin Habets /* End of events */
128236ff6393SMartin Habets break;
128336ff6393SMartin Habets
128436ff6393SMartin Habets netif_vdbg(channel->efx, intr, channel->efx->net_dev,
128536ff6393SMartin Habets "channel %d event is "EFX_QWORD_FMT"\n",
128636ff6393SMartin Habets channel->channel, EFX_QWORD_VAL(event));
128736ff6393SMartin Habets
128836ff6393SMartin Habets /* Clear this event by marking it all ones */
128936ff6393SMartin Habets EFX_SET_QWORD(*p_event);
129036ff6393SMartin Habets
129136ff6393SMartin Habets ++read_ptr;
129236ff6393SMartin Habets
129336ff6393SMartin Habets ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE);
129436ff6393SMartin Habets
129536ff6393SMartin Habets switch (ev_code) {
129636ff6393SMartin Habets case FSE_AZ_EV_CODE_RX_EV:
129736ff6393SMartin Habets efx_farch_handle_rx_event(channel, &event);
129836ff6393SMartin Habets if (++spent == budget)
129936ff6393SMartin Habets goto out;
130036ff6393SMartin Habets break;
130136ff6393SMartin Habets case FSE_AZ_EV_CODE_TX_EV:
130236ff6393SMartin Habets efx_farch_handle_tx_event(channel, &event);
130336ff6393SMartin Habets break;
130436ff6393SMartin Habets case FSE_AZ_EV_CODE_DRV_GEN_EV:
130536ff6393SMartin Habets efx_farch_handle_generated_event(channel, &event);
130636ff6393SMartin Habets break;
130736ff6393SMartin Habets case FSE_AZ_EV_CODE_DRIVER_EV:
130836ff6393SMartin Habets efx_farch_handle_driver_event(channel, &event);
130936ff6393SMartin Habets break;
1310dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
131136ff6393SMartin Habets case FSE_CZ_EV_CODE_USER_EV:
131236ff6393SMartin Habets efx_siena_sriov_event(channel, &event);
131336ff6393SMartin Habets break;
131436ff6393SMartin Habets #endif
131536ff6393SMartin Habets case FSE_CZ_EV_CODE_MCDI_EV:
13164d49e5cdSMartin Habets efx_siena_mcdi_process_event(channel, &event);
131736ff6393SMartin Habets break;
131836ff6393SMartin Habets case FSE_AZ_EV_CODE_GLOBAL_EV:
131936ff6393SMartin Habets if (efx->type->handle_global_event &&
132036ff6393SMartin Habets efx->type->handle_global_event(channel, &event))
132136ff6393SMartin Habets break;
132236ff6393SMartin Habets fallthrough;
132336ff6393SMartin Habets default:
132436ff6393SMartin Habets netif_err(channel->efx, hw, channel->efx->net_dev,
132536ff6393SMartin Habets "channel %d unknown event type %d (data "
132636ff6393SMartin Habets EFX_QWORD_FMT ")\n", channel->channel,
132736ff6393SMartin Habets ev_code, EFX_QWORD_VAL(event));
132836ff6393SMartin Habets }
132936ff6393SMartin Habets }
133036ff6393SMartin Habets
133136ff6393SMartin Habets out:
133236ff6393SMartin Habets channel->eventq_read_ptr = read_ptr;
133336ff6393SMartin Habets return spent;
133436ff6393SMartin Habets }
133536ff6393SMartin Habets
133636ff6393SMartin Habets /* Allocate buffer table entries for event queue */
efx_farch_ev_probe(struct efx_channel * channel)133736ff6393SMartin Habets int efx_farch_ev_probe(struct efx_channel *channel)
133836ff6393SMartin Habets {
133936ff6393SMartin Habets struct efx_nic *efx = channel->efx;
134036ff6393SMartin Habets unsigned entries;
134136ff6393SMartin Habets
134236ff6393SMartin Habets entries = channel->eventq_mask + 1;
134336ff6393SMartin Habets return efx_alloc_special_buffer(efx, &channel->eventq,
134436ff6393SMartin Habets entries * sizeof(efx_qword_t));
134536ff6393SMartin Habets }
134636ff6393SMartin Habets
efx_farch_ev_init(struct efx_channel * channel)134736ff6393SMartin Habets int efx_farch_ev_init(struct efx_channel *channel)
134836ff6393SMartin Habets {
134936ff6393SMartin Habets efx_oword_t reg;
135036ff6393SMartin Habets struct efx_nic *efx = channel->efx;
135136ff6393SMartin Habets
135236ff6393SMartin Habets netif_dbg(efx, hw, efx->net_dev,
135336ff6393SMartin Habets "channel %d event queue in special buffers %d-%d\n",
135436ff6393SMartin Habets channel->channel, channel->eventq.index,
135536ff6393SMartin Habets channel->eventq.index + channel->eventq.entries - 1);
135636ff6393SMartin Habets
135736ff6393SMartin Habets EFX_POPULATE_OWORD_3(reg,
135836ff6393SMartin Habets FRF_CZ_TIMER_Q_EN, 1,
135936ff6393SMartin Habets FRF_CZ_HOST_NOTIFY_MODE, 0,
136036ff6393SMartin Habets FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
136136ff6393SMartin Habets efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, channel->channel);
136236ff6393SMartin Habets
136336ff6393SMartin Habets /* Pin event queue buffer */
136436ff6393SMartin Habets efx_init_special_buffer(efx, &channel->eventq);
136536ff6393SMartin Habets
136636ff6393SMartin Habets /* Fill event queue with all ones (i.e. empty events) */
136736ff6393SMartin Habets memset(channel->eventq.buf.addr, 0xff, channel->eventq.buf.len);
136836ff6393SMartin Habets
136936ff6393SMartin Habets /* Push event queue to card */
137036ff6393SMartin Habets EFX_POPULATE_OWORD_3(reg,
137136ff6393SMartin Habets FRF_AZ_EVQ_EN, 1,
137236ff6393SMartin Habets FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries),
137336ff6393SMartin Habets FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index);
137436ff6393SMartin Habets efx_writeo_table(efx, ®, efx->type->evq_ptr_tbl_base,
137536ff6393SMartin Habets channel->channel);
137636ff6393SMartin Habets
137736ff6393SMartin Habets return 0;
137836ff6393SMartin Habets }
137936ff6393SMartin Habets
efx_farch_ev_fini(struct efx_channel * channel)138036ff6393SMartin Habets void efx_farch_ev_fini(struct efx_channel *channel)
138136ff6393SMartin Habets {
138236ff6393SMartin Habets efx_oword_t reg;
138336ff6393SMartin Habets struct efx_nic *efx = channel->efx;
138436ff6393SMartin Habets
138536ff6393SMartin Habets /* Remove event queue from card */
138636ff6393SMartin Habets EFX_ZERO_OWORD(reg);
138736ff6393SMartin Habets efx_writeo_table(efx, ®, efx->type->evq_ptr_tbl_base,
138836ff6393SMartin Habets channel->channel);
138936ff6393SMartin Habets efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, channel->channel);
139036ff6393SMartin Habets
139136ff6393SMartin Habets /* Unpin event queue */
139236ff6393SMartin Habets efx_fini_special_buffer(efx, &channel->eventq);
139336ff6393SMartin Habets }
139436ff6393SMartin Habets
139536ff6393SMartin Habets /* Free buffers backing event queue */
efx_farch_ev_remove(struct efx_channel * channel)139636ff6393SMartin Habets void efx_farch_ev_remove(struct efx_channel *channel)
139736ff6393SMartin Habets {
139836ff6393SMartin Habets efx_free_special_buffer(channel->efx, &channel->eventq);
139936ff6393SMartin Habets }
140036ff6393SMartin Habets
140136ff6393SMartin Habets
efx_farch_ev_test_generate(struct efx_channel * channel)140236ff6393SMartin Habets void efx_farch_ev_test_generate(struct efx_channel *channel)
140336ff6393SMartin Habets {
140436ff6393SMartin Habets efx_farch_magic_event(channel, EFX_CHANNEL_MAGIC_TEST(channel));
140536ff6393SMartin Habets }
140636ff6393SMartin Habets
efx_farch_rx_defer_refill(struct efx_rx_queue * rx_queue)140736ff6393SMartin Habets void efx_farch_rx_defer_refill(struct efx_rx_queue *rx_queue)
140836ff6393SMartin Habets {
140936ff6393SMartin Habets efx_farch_magic_event(efx_rx_queue_channel(rx_queue),
141036ff6393SMartin Habets EFX_CHANNEL_MAGIC_FILL(rx_queue));
141136ff6393SMartin Habets }
141236ff6393SMartin Habets
141336ff6393SMartin Habets /**************************************************************************
141436ff6393SMartin Habets *
141536ff6393SMartin Habets * Hardware interrupts
141636ff6393SMartin Habets * The hardware interrupt handler does very little work; all the event
141736ff6393SMartin Habets * queue processing is carried out by per-channel tasklets.
141836ff6393SMartin Habets *
141936ff6393SMartin Habets **************************************************************************/
142036ff6393SMartin Habets
142136ff6393SMartin Habets /* Enable/disable/generate interrupts */
efx_farch_interrupts(struct efx_nic * efx,bool enabled,bool force)142236ff6393SMartin Habets static inline void efx_farch_interrupts(struct efx_nic *efx,
142336ff6393SMartin Habets bool enabled, bool force)
142436ff6393SMartin Habets {
142536ff6393SMartin Habets efx_oword_t int_en_reg_ker;
142636ff6393SMartin Habets
142736ff6393SMartin Habets EFX_POPULATE_OWORD_3(int_en_reg_ker,
142836ff6393SMartin Habets FRF_AZ_KER_INT_LEVE_SEL, efx->irq_level,
142936ff6393SMartin Habets FRF_AZ_KER_INT_KER, force,
143036ff6393SMartin Habets FRF_AZ_DRV_INT_EN_KER, enabled);
143136ff6393SMartin Habets efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);
143236ff6393SMartin Habets }
143336ff6393SMartin Habets
efx_farch_irq_enable_master(struct efx_nic * efx)143436ff6393SMartin Habets void efx_farch_irq_enable_master(struct efx_nic *efx)
143536ff6393SMartin Habets {
143636ff6393SMartin Habets EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));
143736ff6393SMartin Habets wmb(); /* Ensure interrupt vector is clear before interrupts enabled */
143836ff6393SMartin Habets
143936ff6393SMartin Habets efx_farch_interrupts(efx, true, false);
144036ff6393SMartin Habets }
144136ff6393SMartin Habets
efx_farch_irq_disable_master(struct efx_nic * efx)144236ff6393SMartin Habets void efx_farch_irq_disable_master(struct efx_nic *efx)
144336ff6393SMartin Habets {
144436ff6393SMartin Habets /* Disable interrupts */
144536ff6393SMartin Habets efx_farch_interrupts(efx, false, false);
144636ff6393SMartin Habets }
144736ff6393SMartin Habets
144836ff6393SMartin Habets /* Generate a test interrupt
144936ff6393SMartin Habets * Interrupt must already have been enabled, otherwise nasty things
145036ff6393SMartin Habets * may happen.
145136ff6393SMartin Habets */
efx_farch_irq_test_generate(struct efx_nic * efx)145236ff6393SMartin Habets int efx_farch_irq_test_generate(struct efx_nic *efx)
145336ff6393SMartin Habets {
145436ff6393SMartin Habets efx_farch_interrupts(efx, true, true);
145536ff6393SMartin Habets return 0;
145636ff6393SMartin Habets }
145736ff6393SMartin Habets
145836ff6393SMartin Habets /* Process a fatal interrupt
145936ff6393SMartin Habets * Disable bus mastering ASAP and schedule a reset
146036ff6393SMartin Habets */
efx_farch_fatal_interrupt(struct efx_nic * efx)146136ff6393SMartin Habets irqreturn_t efx_farch_fatal_interrupt(struct efx_nic *efx)
146236ff6393SMartin Habets {
146336ff6393SMartin Habets efx_oword_t *int_ker = efx->irq_status.addr;
146436ff6393SMartin Habets efx_oword_t fatal_intr;
146536ff6393SMartin Habets int error, mem_perr;
146636ff6393SMartin Habets
146736ff6393SMartin Habets efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);
146836ff6393SMartin Habets error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR);
146936ff6393SMartin Habets
147036ff6393SMartin Habets netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status "
147136ff6393SMartin Habets EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker),
147236ff6393SMartin Habets EFX_OWORD_VAL(fatal_intr),
147336ff6393SMartin Habets error ? "disabling bus mastering" : "no recognised error");
147436ff6393SMartin Habets
147536ff6393SMartin Habets /* If this is a memory parity error dump which blocks are offending */
147636ff6393SMartin Habets mem_perr = (EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER) ||
147736ff6393SMartin Habets EFX_OWORD_FIELD(fatal_intr, FRF_AZ_SRM_PERR_INT_KER));
147836ff6393SMartin Habets if (mem_perr) {
147936ff6393SMartin Habets efx_oword_t reg;
148036ff6393SMartin Habets efx_reado(efx, ®, FR_AZ_MEM_STAT);
148136ff6393SMartin Habets netif_err(efx, hw, efx->net_dev,
148236ff6393SMartin Habets "SYSTEM ERROR: memory parity error "EFX_OWORD_FMT"\n",
148336ff6393SMartin Habets EFX_OWORD_VAL(reg));
148436ff6393SMartin Habets }
148536ff6393SMartin Habets
148636ff6393SMartin Habets /* Disable both devices */
148736ff6393SMartin Habets pci_clear_master(efx->pci_dev);
148836ff6393SMartin Habets efx_farch_irq_disable_master(efx);
148936ff6393SMartin Habets
149036ff6393SMartin Habets /* Count errors and reset or disable the NIC accordingly */
149136ff6393SMartin Habets if (efx->int_error_count == 0 ||
149236ff6393SMartin Habets time_after(jiffies, efx->int_error_expire)) {
149336ff6393SMartin Habets efx->int_error_count = 0;
149436ff6393SMartin Habets efx->int_error_expire =
149536ff6393SMartin Habets jiffies + EFX_INT_ERROR_EXPIRE * HZ;
149636ff6393SMartin Habets }
149736ff6393SMartin Habets if (++efx->int_error_count < EFX_MAX_INT_ERRORS) {
149836ff6393SMartin Habets netif_err(efx, hw, efx->net_dev,
149936ff6393SMartin Habets "SYSTEM ERROR - reset scheduled\n");
150071ad88f6SMartin Habets efx_siena_schedule_reset(efx, RESET_TYPE_INT_ERROR);
150136ff6393SMartin Habets } else {
150236ff6393SMartin Habets netif_err(efx, hw, efx->net_dev,
150336ff6393SMartin Habets "SYSTEM ERROR - max number of errors seen."
150436ff6393SMartin Habets "NIC will be disabled\n");
150571ad88f6SMartin Habets efx_siena_schedule_reset(efx, RESET_TYPE_DISABLE);
150636ff6393SMartin Habets }
150736ff6393SMartin Habets
150836ff6393SMartin Habets return IRQ_HANDLED;
150936ff6393SMartin Habets }
151036ff6393SMartin Habets
151136ff6393SMartin Habets /* Handle a legacy interrupt
151236ff6393SMartin Habets * Acknowledges the interrupt and schedule event queue processing.
151336ff6393SMartin Habets */
efx_farch_legacy_interrupt(int irq,void * dev_id)151436ff6393SMartin Habets irqreturn_t efx_farch_legacy_interrupt(int irq, void *dev_id)
151536ff6393SMartin Habets {
151636ff6393SMartin Habets struct efx_nic *efx = dev_id;
151736ff6393SMartin Habets bool soft_enabled = READ_ONCE(efx->irq_soft_enabled);
151836ff6393SMartin Habets efx_oword_t *int_ker = efx->irq_status.addr;
151936ff6393SMartin Habets irqreturn_t result = IRQ_NONE;
152036ff6393SMartin Habets struct efx_channel *channel;
152136ff6393SMartin Habets efx_dword_t reg;
152236ff6393SMartin Habets u32 queues;
152336ff6393SMartin Habets int syserr;
152436ff6393SMartin Habets
152536ff6393SMartin Habets /* Read the ISR which also ACKs the interrupts */
152636ff6393SMartin Habets efx_readd(efx, ®, FR_BZ_INT_ISR0);
152736ff6393SMartin Habets queues = EFX_EXTRACT_DWORD(reg, 0, 31);
152836ff6393SMartin Habets
152936ff6393SMartin Habets /* Legacy interrupts are disabled too late by the EEH kernel
153036ff6393SMartin Habets * code. Disable them earlier.
153136ff6393SMartin Habets * If an EEH error occurred, the read will have returned all ones.
153236ff6393SMartin Habets */
153371ad88f6SMartin Habets if (EFX_DWORD_IS_ALL_ONES(reg) && efx_siena_try_recovery(efx) &&
153436ff6393SMartin Habets !efx->eeh_disabled_legacy_irq) {
153536ff6393SMartin Habets disable_irq_nosync(efx->legacy_irq);
153636ff6393SMartin Habets efx->eeh_disabled_legacy_irq = true;
153736ff6393SMartin Habets }
153836ff6393SMartin Habets
153936ff6393SMartin Habets /* Handle non-event-queue sources */
154036ff6393SMartin Habets if (queues & (1U << efx->irq_level) && soft_enabled) {
154136ff6393SMartin Habets syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
154236ff6393SMartin Habets if (unlikely(syserr))
154336ff6393SMartin Habets return efx_farch_fatal_interrupt(efx);
154436ff6393SMartin Habets efx->last_irq_cpu = raw_smp_processor_id();
154536ff6393SMartin Habets }
154636ff6393SMartin Habets
154736ff6393SMartin Habets if (queues != 0) {
154836ff6393SMartin Habets efx->irq_zero_count = 0;
154936ff6393SMartin Habets
155036ff6393SMartin Habets /* Schedule processing of any interrupting queues */
155136ff6393SMartin Habets if (likely(soft_enabled)) {
155236ff6393SMartin Habets efx_for_each_channel(channel, efx) {
155336ff6393SMartin Habets if (queues & 1)
155436ff6393SMartin Habets efx_schedule_channel_irq(channel);
155536ff6393SMartin Habets queues >>= 1;
155636ff6393SMartin Habets }
155736ff6393SMartin Habets }
155836ff6393SMartin Habets result = IRQ_HANDLED;
155936ff6393SMartin Habets
156036ff6393SMartin Habets } else {
156136ff6393SMartin Habets efx_qword_t *event;
156236ff6393SMartin Habets
156336ff6393SMartin Habets /* Legacy ISR read can return zero once (SF bug 15783) */
156436ff6393SMartin Habets
156536ff6393SMartin Habets /* We can't return IRQ_HANDLED more than once on seeing ISR=0
156636ff6393SMartin Habets * because this might be a shared interrupt. */
156736ff6393SMartin Habets if (efx->irq_zero_count++ == 0)
156836ff6393SMartin Habets result = IRQ_HANDLED;
156936ff6393SMartin Habets
157036ff6393SMartin Habets /* Ensure we schedule or rearm all event queues */
157136ff6393SMartin Habets if (likely(soft_enabled)) {
157236ff6393SMartin Habets efx_for_each_channel(channel, efx) {
157336ff6393SMartin Habets event = efx_event(channel,
157436ff6393SMartin Habets channel->eventq_read_ptr);
157536ff6393SMartin Habets if (efx_event_present(event))
157636ff6393SMartin Habets efx_schedule_channel_irq(channel);
157736ff6393SMartin Habets else
157836ff6393SMartin Habets efx_farch_ev_read_ack(channel);
157936ff6393SMartin Habets }
158036ff6393SMartin Habets }
158136ff6393SMartin Habets }
158236ff6393SMartin Habets
158336ff6393SMartin Habets if (result == IRQ_HANDLED)
158436ff6393SMartin Habets netif_vdbg(efx, intr, efx->net_dev,
158536ff6393SMartin Habets "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
158636ff6393SMartin Habets irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
158736ff6393SMartin Habets
158836ff6393SMartin Habets return result;
158936ff6393SMartin Habets }
159036ff6393SMartin Habets
159136ff6393SMartin Habets /* Handle an MSI interrupt
159236ff6393SMartin Habets *
159336ff6393SMartin Habets * Handle an MSI hardware interrupt. This routine schedules event
159436ff6393SMartin Habets * queue processing. No interrupt acknowledgement cycle is necessary.
159536ff6393SMartin Habets * Also, we never need to check that the interrupt is for us, since
159636ff6393SMartin Habets * MSI interrupts cannot be shared.
159736ff6393SMartin Habets */
efx_farch_msi_interrupt(int irq,void * dev_id)159836ff6393SMartin Habets irqreturn_t efx_farch_msi_interrupt(int irq, void *dev_id)
159936ff6393SMartin Habets {
160036ff6393SMartin Habets struct efx_msi_context *context = dev_id;
160136ff6393SMartin Habets struct efx_nic *efx = context->efx;
160236ff6393SMartin Habets efx_oword_t *int_ker = efx->irq_status.addr;
160336ff6393SMartin Habets int syserr;
160436ff6393SMartin Habets
160536ff6393SMartin Habets netif_vdbg(efx, intr, efx->net_dev,
160636ff6393SMartin Habets "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
160736ff6393SMartin Habets irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
160836ff6393SMartin Habets
160936ff6393SMartin Habets if (!likely(READ_ONCE(efx->irq_soft_enabled)))
161036ff6393SMartin Habets return IRQ_HANDLED;
161136ff6393SMartin Habets
161236ff6393SMartin Habets /* Handle non-event-queue sources */
161336ff6393SMartin Habets if (context->index == efx->irq_level) {
161436ff6393SMartin Habets syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
161536ff6393SMartin Habets if (unlikely(syserr))
161636ff6393SMartin Habets return efx_farch_fatal_interrupt(efx);
161736ff6393SMartin Habets efx->last_irq_cpu = raw_smp_processor_id();
161836ff6393SMartin Habets }
161936ff6393SMartin Habets
162036ff6393SMartin Habets /* Schedule processing of the channel */
162136ff6393SMartin Habets efx_schedule_channel_irq(efx->channel[context->index]);
162236ff6393SMartin Habets
162336ff6393SMartin Habets return IRQ_HANDLED;
162436ff6393SMartin Habets }
162536ff6393SMartin Habets
162636ff6393SMartin Habets /* Setup RSS indirection table.
162736ff6393SMartin Habets * This maps from the hash value of the packet to RXQ
162836ff6393SMartin Habets */
efx_farch_rx_push_indir_table(struct efx_nic * efx)162936ff6393SMartin Habets void efx_farch_rx_push_indir_table(struct efx_nic *efx)
163036ff6393SMartin Habets {
163136ff6393SMartin Habets size_t i = 0;
163236ff6393SMartin Habets efx_dword_t dword;
163336ff6393SMartin Habets
163436ff6393SMartin Habets BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=
163536ff6393SMartin Habets FR_BZ_RX_INDIRECTION_TBL_ROWS);
163636ff6393SMartin Habets
163736ff6393SMartin Habets for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) {
163836ff6393SMartin Habets EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE,
163936ff6393SMartin Habets efx->rss_context.rx_indir_table[i]);
164036ff6393SMartin Habets efx_writed(efx, &dword,
164136ff6393SMartin Habets FR_BZ_RX_INDIRECTION_TBL +
164236ff6393SMartin Habets FR_BZ_RX_INDIRECTION_TBL_STEP * i);
164336ff6393SMartin Habets }
164436ff6393SMartin Habets }
164536ff6393SMartin Habets
efx_farch_rx_pull_indir_table(struct efx_nic * efx)164636ff6393SMartin Habets void efx_farch_rx_pull_indir_table(struct efx_nic *efx)
164736ff6393SMartin Habets {
164836ff6393SMartin Habets size_t i = 0;
164936ff6393SMartin Habets efx_dword_t dword;
165036ff6393SMartin Habets
165136ff6393SMartin Habets BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=
165236ff6393SMartin Habets FR_BZ_RX_INDIRECTION_TBL_ROWS);
165336ff6393SMartin Habets
165436ff6393SMartin Habets for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) {
165536ff6393SMartin Habets efx_readd(efx, &dword,
165636ff6393SMartin Habets FR_BZ_RX_INDIRECTION_TBL +
165736ff6393SMartin Habets FR_BZ_RX_INDIRECTION_TBL_STEP * i);
165836ff6393SMartin Habets efx->rss_context.rx_indir_table[i] = EFX_DWORD_FIELD(dword, FRF_BZ_IT_QUEUE);
165936ff6393SMartin Habets }
166036ff6393SMartin Habets }
166136ff6393SMartin Habets
166236ff6393SMartin Habets /* Looks at available SRAM resources and works out how many queues we
166336ff6393SMartin Habets * can support, and where things like descriptor caches should live.
166436ff6393SMartin Habets *
166536ff6393SMartin Habets * SRAM is split up as follows:
166636ff6393SMartin Habets * 0 buftbl entries for channels
166736ff6393SMartin Habets * efx->vf_buftbl_base buftbl entries for SR-IOV
166836ff6393SMartin Habets * efx->rx_dc_base RX descriptor caches
166936ff6393SMartin Habets * efx->tx_dc_base TX descriptor caches
167036ff6393SMartin Habets */
efx_farch_dimension_resources(struct efx_nic * efx,unsigned sram_lim_qw)167136ff6393SMartin Habets void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
167236ff6393SMartin Habets {
167336ff6393SMartin Habets unsigned vi_count, total_tx_channels;
1674dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
167536ff6393SMartin Habets struct siena_nic_data *nic_data;
167636ff6393SMartin Habets unsigned buftbl_min;
167736ff6393SMartin Habets #endif
167836ff6393SMartin Habets
167936ff6393SMartin Habets total_tx_channels = efx->n_tx_channels + efx->n_extra_tx_channels;
168036ff6393SMartin Habets vi_count = max(efx->n_channels, total_tx_channels * EFX_MAX_TXQ_PER_CHANNEL);
168136ff6393SMartin Habets
1682dfb1cfbdSMartin Habets #ifdef CONFIG_SFC_SIENA_SRIOV
168336ff6393SMartin Habets nic_data = efx->nic_data;
168436ff6393SMartin Habets /* Account for the buffer table entries backing the datapath channels
168536ff6393SMartin Habets * and the descriptor caches for those channels.
168636ff6393SMartin Habets */
168736ff6393SMartin Habets buftbl_min = ((efx->n_rx_channels * EFX_MAX_DMAQ_SIZE +
168836ff6393SMartin Habets total_tx_channels * EFX_MAX_TXQ_PER_CHANNEL * EFX_MAX_DMAQ_SIZE +
168936ff6393SMartin Habets efx->n_channels * EFX_MAX_EVQ_SIZE)
169036ff6393SMartin Habets * sizeof(efx_qword_t) / EFX_BUF_SIZE);
169136ff6393SMartin Habets if (efx->type->sriov_wanted) {
169236ff6393SMartin Habets if (efx->type->sriov_wanted(efx)) {
169336ff6393SMartin Habets unsigned vi_dc_entries, buftbl_free;
169436ff6393SMartin Habets unsigned entries_per_vf, vf_limit;
169536ff6393SMartin Habets
169636ff6393SMartin Habets nic_data->vf_buftbl_base = buftbl_min;
169736ff6393SMartin Habets
169836ff6393SMartin Habets vi_dc_entries = RX_DC_ENTRIES + TX_DC_ENTRIES;
169936ff6393SMartin Habets vi_count = max(vi_count, EFX_VI_BASE);
170036ff6393SMartin Habets buftbl_free = (sram_lim_qw - buftbl_min -
170136ff6393SMartin Habets vi_count * vi_dc_entries);
170236ff6393SMartin Habets
170336ff6393SMartin Habets entries_per_vf = ((vi_dc_entries +
170436ff6393SMartin Habets EFX_VF_BUFTBL_PER_VI) *
170536ff6393SMartin Habets efx_vf_size(efx));
170636ff6393SMartin Habets vf_limit = min(buftbl_free / entries_per_vf,
170736ff6393SMartin Habets (1024U - EFX_VI_BASE) >> efx->vi_scale);
170836ff6393SMartin Habets
170936ff6393SMartin Habets if (efx->vf_count > vf_limit) {
171036ff6393SMartin Habets netif_err(efx, probe, efx->net_dev,
171136ff6393SMartin Habets "Reducing VF count from from %d to %d\n",
171236ff6393SMartin Habets efx->vf_count, vf_limit);
171336ff6393SMartin Habets efx->vf_count = vf_limit;
171436ff6393SMartin Habets }
171536ff6393SMartin Habets vi_count += efx->vf_count * efx_vf_size(efx);
171636ff6393SMartin Habets }
171736ff6393SMartin Habets }
171836ff6393SMartin Habets #endif
171936ff6393SMartin Habets
172036ff6393SMartin Habets efx->tx_dc_base = sram_lim_qw - vi_count * TX_DC_ENTRIES;
172136ff6393SMartin Habets efx->rx_dc_base = efx->tx_dc_base - vi_count * RX_DC_ENTRIES;
172236ff6393SMartin Habets }
172336ff6393SMartin Habets
efx_farch_fpga_ver(struct efx_nic * efx)172436ff6393SMartin Habets u32 efx_farch_fpga_ver(struct efx_nic *efx)
172536ff6393SMartin Habets {
172636ff6393SMartin Habets efx_oword_t altera_build;
172736ff6393SMartin Habets efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);
172836ff6393SMartin Habets return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER);
172936ff6393SMartin Habets }
173036ff6393SMartin Habets
efx_farch_init_common(struct efx_nic * efx)173136ff6393SMartin Habets void efx_farch_init_common(struct efx_nic *efx)
173236ff6393SMartin Habets {
173336ff6393SMartin Habets efx_oword_t temp;
173436ff6393SMartin Habets
173536ff6393SMartin Habets /* Set positions of descriptor caches in SRAM. */
173636ff6393SMartin Habets EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, efx->tx_dc_base);
173736ff6393SMartin Habets efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);
173836ff6393SMartin Habets EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, efx->rx_dc_base);
173936ff6393SMartin Habets efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);
174036ff6393SMartin Habets
174136ff6393SMartin Habets /* Set TX descriptor cache size. */
174236ff6393SMartin Habets BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER));
174336ff6393SMartin Habets EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER);
174436ff6393SMartin Habets efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);
174536ff6393SMartin Habets
174636ff6393SMartin Habets /* Set RX descriptor cache size. Set low watermark to size-8, as
174736ff6393SMartin Habets * this allows most efficient prefetching.
174836ff6393SMartin Habets */
174936ff6393SMartin Habets BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER));
175036ff6393SMartin Habets EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER);
175136ff6393SMartin Habets efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);
175236ff6393SMartin Habets EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8);
175336ff6393SMartin Habets efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);
175436ff6393SMartin Habets
175536ff6393SMartin Habets /* Program INT_KER address */
175636ff6393SMartin Habets EFX_POPULATE_OWORD_2(temp,
175736ff6393SMartin Habets FRF_AZ_NORM_INT_VEC_DIS_KER,
175836ff6393SMartin Habets EFX_INT_MODE_USE_MSI(efx),
175936ff6393SMartin Habets FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);
176036ff6393SMartin Habets efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER);
176136ff6393SMartin Habets
176236ff6393SMartin Habets if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx))
176336ff6393SMartin Habets /* Use an interrupt level unused by event queues */
176436ff6393SMartin Habets efx->irq_level = 0x1f;
176536ff6393SMartin Habets else
176636ff6393SMartin Habets /* Use a valid MSI-X vector */
176736ff6393SMartin Habets efx->irq_level = 0;
176836ff6393SMartin Habets
176936ff6393SMartin Habets /* Enable all the genuinely fatal interrupts. (They are still
177036ff6393SMartin Habets * masked by the overall interrupt mask, controlled by
177136ff6393SMartin Habets * falcon_interrupts()).
177236ff6393SMartin Habets *
177336ff6393SMartin Habets * Note: All other fatal interrupts are enabled
177436ff6393SMartin Habets */
177536ff6393SMartin Habets EFX_POPULATE_OWORD_3(temp,
177636ff6393SMartin Habets FRF_AZ_ILL_ADR_INT_KER_EN, 1,
177736ff6393SMartin Habets FRF_AZ_RBUF_OWN_INT_KER_EN, 1,
177836ff6393SMartin Habets FRF_AZ_TBUF_OWN_INT_KER_EN, 1);
177936ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 1);
178036ff6393SMartin Habets EFX_INVERT_OWORD(temp);
178136ff6393SMartin Habets efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);
178236ff6393SMartin Habets
178336ff6393SMartin Habets /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be
178436ff6393SMartin Habets * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q.
178536ff6393SMartin Habets */
178636ff6393SMartin Habets efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
178736ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe);
178836ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1);
178936ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
179036ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1);
179136ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1);
179236ff6393SMartin Habets /* Enable SW_EV to inherit in char driver - assume harmless here */
179336ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1);
179436ff6393SMartin Habets /* Prefetch threshold 2 => fetch when descriptor cache half empty */
179536ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2);
179636ff6393SMartin Habets /* Disable hardware watchdog which can misfire */
179736ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
179836ff6393SMartin Habets /* Squash TX of packets of 16 bytes or less */
179936ff6393SMartin Habets EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
180036ff6393SMartin Habets efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
180136ff6393SMartin Habets
180236ff6393SMartin Habets EFX_POPULATE_OWORD_4(temp,
180336ff6393SMartin Habets /* Default values */
180436ff6393SMartin Habets FRF_BZ_TX_PACE_SB_NOT_AF, 0x15,
180536ff6393SMartin Habets FRF_BZ_TX_PACE_SB_AF, 0xb,
180636ff6393SMartin Habets FRF_BZ_TX_PACE_FB_BASE, 0,
180736ff6393SMartin Habets /* Allow large pace values in the fast bin. */
180836ff6393SMartin Habets FRF_BZ_TX_PACE_BIN_TH,
180936ff6393SMartin Habets FFE_BZ_TX_PACE_RESERVED);
181036ff6393SMartin Habets efx_writeo(efx, &temp, FR_BZ_TX_PACE);
181136ff6393SMartin Habets }
181236ff6393SMartin Habets
181336ff6393SMartin Habets /**************************************************************************
181436ff6393SMartin Habets *
181536ff6393SMartin Habets * Filter tables
181636ff6393SMartin Habets *
181736ff6393SMartin Habets **************************************************************************
181836ff6393SMartin Habets */
181936ff6393SMartin Habets
182036ff6393SMartin Habets /* "Fudge factors" - difference between programmed value and actual depth.
182136ff6393SMartin Habets * Due to pipelined implementation we need to program H/W with a value that
182236ff6393SMartin Habets * is larger than the hop limit we want.
182336ff6393SMartin Habets */
182436ff6393SMartin Habets #define EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD 3
182536ff6393SMartin Habets #define EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL 1
182636ff6393SMartin Habets
182736ff6393SMartin Habets /* Hard maximum search limit. Hardware will time-out beyond 200-something.
182836ff6393SMartin Habets * We also need to avoid infinite loops in efx_farch_filter_search() when the
182936ff6393SMartin Habets * table is full.
183036ff6393SMartin Habets */
183136ff6393SMartin Habets #define EFX_FARCH_FILTER_CTL_SRCH_MAX 200
183236ff6393SMartin Habets
183336ff6393SMartin Habets /* Don't try very hard to find space for performance hints, as this is
183436ff6393SMartin Habets * counter-productive. */
183536ff6393SMartin Habets #define EFX_FARCH_FILTER_CTL_SRCH_HINT_MAX 5
183636ff6393SMartin Habets
183736ff6393SMartin Habets enum efx_farch_filter_type {
183836ff6393SMartin Habets EFX_FARCH_FILTER_TCP_FULL = 0,
183936ff6393SMartin Habets EFX_FARCH_FILTER_TCP_WILD,
184036ff6393SMartin Habets EFX_FARCH_FILTER_UDP_FULL,
184136ff6393SMartin Habets EFX_FARCH_FILTER_UDP_WILD,
184236ff6393SMartin Habets EFX_FARCH_FILTER_MAC_FULL = 4,
184336ff6393SMartin Habets EFX_FARCH_FILTER_MAC_WILD,
184436ff6393SMartin Habets EFX_FARCH_FILTER_UC_DEF = 8,
184536ff6393SMartin Habets EFX_FARCH_FILTER_MC_DEF,
184636ff6393SMartin Habets EFX_FARCH_FILTER_TYPE_COUNT, /* number of specific types */
184736ff6393SMartin Habets };
184836ff6393SMartin Habets
184936ff6393SMartin Habets enum efx_farch_filter_table_id {
185036ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_IP = 0,
185136ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_MAC,
185236ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_DEF,
185336ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_TX_MAC,
185436ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_COUNT,
185536ff6393SMartin Habets };
185636ff6393SMartin Habets
185736ff6393SMartin Habets enum efx_farch_filter_index {
185836ff6393SMartin Habets EFX_FARCH_FILTER_INDEX_UC_DEF,
185936ff6393SMartin Habets EFX_FARCH_FILTER_INDEX_MC_DEF,
186036ff6393SMartin Habets EFX_FARCH_FILTER_SIZE_RX_DEF,
186136ff6393SMartin Habets };
186236ff6393SMartin Habets
186336ff6393SMartin Habets struct efx_farch_filter_spec {
186436ff6393SMartin Habets u8 type:4;
186536ff6393SMartin Habets u8 priority:4;
186636ff6393SMartin Habets u8 flags;
186736ff6393SMartin Habets u16 dmaq_id;
186836ff6393SMartin Habets u32 data[3];
186936ff6393SMartin Habets };
187036ff6393SMartin Habets
187136ff6393SMartin Habets struct efx_farch_filter_table {
187236ff6393SMartin Habets enum efx_farch_filter_table_id id;
187336ff6393SMartin Habets u32 offset; /* address of table relative to BAR */
187436ff6393SMartin Habets unsigned size; /* number of entries */
187536ff6393SMartin Habets unsigned step; /* step between entries */
187636ff6393SMartin Habets unsigned used; /* number currently used */
187736ff6393SMartin Habets unsigned long *used_bitmap;
187836ff6393SMartin Habets struct efx_farch_filter_spec *spec;
187936ff6393SMartin Habets unsigned search_limit[EFX_FARCH_FILTER_TYPE_COUNT];
188036ff6393SMartin Habets };
188136ff6393SMartin Habets
188236ff6393SMartin Habets struct efx_farch_filter_state {
188336ff6393SMartin Habets struct rw_semaphore lock; /* Protects table contents */
188436ff6393SMartin Habets struct efx_farch_filter_table table[EFX_FARCH_FILTER_TABLE_COUNT];
188536ff6393SMartin Habets };
188636ff6393SMartin Habets
188736ff6393SMartin Habets static void
188836ff6393SMartin Habets efx_farch_filter_table_clear_entry(struct efx_nic *efx,
188936ff6393SMartin Habets struct efx_farch_filter_table *table,
189036ff6393SMartin Habets unsigned int filter_idx);
189136ff6393SMartin Habets
189236ff6393SMartin Habets /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
189336ff6393SMartin Habets * key derived from the n-tuple. The initial LFSR state is 0xffff. */
efx_farch_filter_hash(u32 key)189436ff6393SMartin Habets static u16 efx_farch_filter_hash(u32 key)
189536ff6393SMartin Habets {
189636ff6393SMartin Habets u16 tmp;
189736ff6393SMartin Habets
189836ff6393SMartin Habets /* First 16 rounds */
189936ff6393SMartin Habets tmp = 0x1fff ^ key >> 16;
190036ff6393SMartin Habets tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
190136ff6393SMartin Habets tmp = tmp ^ tmp >> 9;
190236ff6393SMartin Habets /* Last 16 rounds */
190336ff6393SMartin Habets tmp = tmp ^ tmp << 13 ^ key;
190436ff6393SMartin Habets tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
190536ff6393SMartin Habets return tmp ^ tmp >> 9;
190636ff6393SMartin Habets }
190736ff6393SMartin Habets
190836ff6393SMartin Habets /* To allow for hash collisions, filter search continues at these
190936ff6393SMartin Habets * increments from the first possible entry selected by the hash. */
efx_farch_filter_increment(u32 key)191036ff6393SMartin Habets static u16 efx_farch_filter_increment(u32 key)
191136ff6393SMartin Habets {
191236ff6393SMartin Habets return key * 2 - 1;
191336ff6393SMartin Habets }
191436ff6393SMartin Habets
191536ff6393SMartin Habets static enum efx_farch_filter_table_id
efx_farch_filter_spec_table_id(const struct efx_farch_filter_spec * spec)191636ff6393SMartin Habets efx_farch_filter_spec_table_id(const struct efx_farch_filter_spec *spec)
191736ff6393SMartin Habets {
191836ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP !=
191936ff6393SMartin Habets (EFX_FARCH_FILTER_TCP_FULL >> 2));
192036ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP !=
192136ff6393SMartin Habets (EFX_FARCH_FILTER_TCP_WILD >> 2));
192236ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP !=
192336ff6393SMartin Habets (EFX_FARCH_FILTER_UDP_FULL >> 2));
192436ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP !=
192536ff6393SMartin Habets (EFX_FARCH_FILTER_UDP_WILD >> 2));
192636ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_MAC !=
192736ff6393SMartin Habets (EFX_FARCH_FILTER_MAC_FULL >> 2));
192836ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_MAC !=
192936ff6393SMartin Habets (EFX_FARCH_FILTER_MAC_WILD >> 2));
193036ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_TX_MAC !=
193136ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_MAC + 2);
193236ff6393SMartin Habets return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0);
193336ff6393SMartin Habets }
193436ff6393SMartin Habets
efx_farch_filter_push_rx_config(struct efx_nic * efx)193536ff6393SMartin Habets static void efx_farch_filter_push_rx_config(struct efx_nic *efx)
193636ff6393SMartin Habets {
193736ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
193836ff6393SMartin Habets struct efx_farch_filter_table *table;
193936ff6393SMartin Habets efx_oword_t filter_ctl;
194036ff6393SMartin Habets
194136ff6393SMartin Habets efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
194236ff6393SMartin Habets
194336ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP];
194436ff6393SMartin Habets EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
194536ff6393SMartin Habets table->search_limit[EFX_FARCH_FILTER_TCP_FULL] +
194636ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL);
194736ff6393SMartin Habets EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
194836ff6393SMartin Habets table->search_limit[EFX_FARCH_FILTER_TCP_WILD] +
194936ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD);
195036ff6393SMartin Habets EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
195136ff6393SMartin Habets table->search_limit[EFX_FARCH_FILTER_UDP_FULL] +
195236ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL);
195336ff6393SMartin Habets EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
195436ff6393SMartin Habets table->search_limit[EFX_FARCH_FILTER_UDP_WILD] +
195536ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD);
195636ff6393SMartin Habets
195736ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_RX_MAC];
195836ff6393SMartin Habets if (table->size) {
195936ff6393SMartin Habets EFX_SET_OWORD_FIELD(
196036ff6393SMartin Habets filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
196136ff6393SMartin Habets table->search_limit[EFX_FARCH_FILTER_MAC_FULL] +
196236ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL);
196336ff6393SMartin Habets EFX_SET_OWORD_FIELD(
196436ff6393SMartin Habets filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
196536ff6393SMartin Habets table->search_limit[EFX_FARCH_FILTER_MAC_WILD] +
196636ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD);
196736ff6393SMartin Habets }
196836ff6393SMartin Habets
196936ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF];
197036ff6393SMartin Habets if (table->size) {
197136ff6393SMartin Habets EFX_SET_OWORD_FIELD(
197236ff6393SMartin Habets filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID,
197336ff6393SMartin Habets table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].dmaq_id);
197436ff6393SMartin Habets EFX_SET_OWORD_FIELD(
197536ff6393SMartin Habets filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED,
197636ff6393SMartin Habets !!(table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].flags &
197736ff6393SMartin Habets EFX_FILTER_FLAG_RX_RSS));
197836ff6393SMartin Habets EFX_SET_OWORD_FIELD(
197936ff6393SMartin Habets filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID,
198036ff6393SMartin Habets table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].dmaq_id);
198136ff6393SMartin Habets EFX_SET_OWORD_FIELD(
198236ff6393SMartin Habets filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED,
198336ff6393SMartin Habets !!(table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].flags &
198436ff6393SMartin Habets EFX_FILTER_FLAG_RX_RSS));
198536ff6393SMartin Habets
198636ff6393SMartin Habets /* There is a single bit to enable RX scatter for all
198736ff6393SMartin Habets * unmatched packets. Only set it if scatter is
198836ff6393SMartin Habets * enabled in both filter specs.
198936ff6393SMartin Habets */
199036ff6393SMartin Habets EFX_SET_OWORD_FIELD(
199136ff6393SMartin Habets filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q,
199236ff6393SMartin Habets !!(table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].flags &
199336ff6393SMartin Habets table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].flags &
199436ff6393SMartin Habets EFX_FILTER_FLAG_RX_SCATTER));
199536ff6393SMartin Habets } else {
199636ff6393SMartin Habets /* We don't expose 'default' filters because unmatched
199736ff6393SMartin Habets * packets always go to the queue number found in the
199836ff6393SMartin Habets * RSS table. But we still need to set the RX scatter
199936ff6393SMartin Habets * bit here.
200036ff6393SMartin Habets */
200136ff6393SMartin Habets EFX_SET_OWORD_FIELD(
200236ff6393SMartin Habets filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q,
200336ff6393SMartin Habets efx->rx_scatter);
200436ff6393SMartin Habets }
200536ff6393SMartin Habets
200636ff6393SMartin Habets efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
200736ff6393SMartin Habets }
200836ff6393SMartin Habets
efx_farch_filter_push_tx_limits(struct efx_nic * efx)200936ff6393SMartin Habets static void efx_farch_filter_push_tx_limits(struct efx_nic *efx)
201036ff6393SMartin Habets {
201136ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
201236ff6393SMartin Habets struct efx_farch_filter_table *table;
201336ff6393SMartin Habets efx_oword_t tx_cfg;
201436ff6393SMartin Habets
201536ff6393SMartin Habets efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG);
201636ff6393SMartin Habets
201736ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_TX_MAC];
201836ff6393SMartin Habets if (table->size) {
201936ff6393SMartin Habets EFX_SET_OWORD_FIELD(
202036ff6393SMartin Habets tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
202136ff6393SMartin Habets table->search_limit[EFX_FARCH_FILTER_MAC_FULL] +
202236ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL);
202336ff6393SMartin Habets EFX_SET_OWORD_FIELD(
202436ff6393SMartin Habets tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
202536ff6393SMartin Habets table->search_limit[EFX_FARCH_FILTER_MAC_WILD] +
202636ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD);
202736ff6393SMartin Habets }
202836ff6393SMartin Habets
202936ff6393SMartin Habets efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG);
203036ff6393SMartin Habets }
203136ff6393SMartin Habets
203236ff6393SMartin Habets static int
efx_farch_filter_from_gen_spec(struct efx_farch_filter_spec * spec,const struct efx_filter_spec * gen_spec)203336ff6393SMartin Habets efx_farch_filter_from_gen_spec(struct efx_farch_filter_spec *spec,
203436ff6393SMartin Habets const struct efx_filter_spec *gen_spec)
203536ff6393SMartin Habets {
203636ff6393SMartin Habets bool is_full = false;
203736ff6393SMartin Habets
203836ff6393SMartin Habets if ((gen_spec->flags & EFX_FILTER_FLAG_RX_RSS) && gen_spec->rss_context)
203936ff6393SMartin Habets return -EINVAL;
204036ff6393SMartin Habets
204136ff6393SMartin Habets spec->priority = gen_spec->priority;
204236ff6393SMartin Habets spec->flags = gen_spec->flags;
204336ff6393SMartin Habets spec->dmaq_id = gen_spec->dmaq_id;
204436ff6393SMartin Habets
204536ff6393SMartin Habets switch (gen_spec->match_flags) {
204636ff6393SMartin Habets case (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
204736ff6393SMartin Habets EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
204836ff6393SMartin Habets EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT):
204936ff6393SMartin Habets is_full = true;
205036ff6393SMartin Habets fallthrough;
205136ff6393SMartin Habets case (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
205236ff6393SMartin Habets EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT): {
205336ff6393SMartin Habets __be32 rhost, host1, host2;
205436ff6393SMartin Habets __be16 rport, port1, port2;
205536ff6393SMartin Habets
205636ff6393SMartin Habets EFX_WARN_ON_PARANOID(!(gen_spec->flags & EFX_FILTER_FLAG_RX));
205736ff6393SMartin Habets
205836ff6393SMartin Habets if (gen_spec->ether_type != htons(ETH_P_IP))
205936ff6393SMartin Habets return -EPROTONOSUPPORT;
206036ff6393SMartin Habets if (gen_spec->loc_port == 0 ||
206136ff6393SMartin Habets (is_full && gen_spec->rem_port == 0))
206236ff6393SMartin Habets return -EADDRNOTAVAIL;
206336ff6393SMartin Habets switch (gen_spec->ip_proto) {
206436ff6393SMartin Habets case IPPROTO_TCP:
206536ff6393SMartin Habets spec->type = (is_full ? EFX_FARCH_FILTER_TCP_FULL :
206636ff6393SMartin Habets EFX_FARCH_FILTER_TCP_WILD);
206736ff6393SMartin Habets break;
206836ff6393SMartin Habets case IPPROTO_UDP:
206936ff6393SMartin Habets spec->type = (is_full ? EFX_FARCH_FILTER_UDP_FULL :
207036ff6393SMartin Habets EFX_FARCH_FILTER_UDP_WILD);
207136ff6393SMartin Habets break;
207236ff6393SMartin Habets default:
207336ff6393SMartin Habets return -EPROTONOSUPPORT;
207436ff6393SMartin Habets }
207536ff6393SMartin Habets
207636ff6393SMartin Habets /* Filter is constructed in terms of source and destination,
207736ff6393SMartin Habets * with the odd wrinkle that the ports are swapped in a UDP
207836ff6393SMartin Habets * wildcard filter. We need to convert from local and remote
207936ff6393SMartin Habets * (= zero for wildcard) addresses.
208036ff6393SMartin Habets */
208136ff6393SMartin Habets rhost = is_full ? gen_spec->rem_host[0] : 0;
208236ff6393SMartin Habets rport = is_full ? gen_spec->rem_port : 0;
208336ff6393SMartin Habets host1 = rhost;
208436ff6393SMartin Habets host2 = gen_spec->loc_host[0];
208536ff6393SMartin Habets if (!is_full && gen_spec->ip_proto == IPPROTO_UDP) {
208636ff6393SMartin Habets port1 = gen_spec->loc_port;
208736ff6393SMartin Habets port2 = rport;
208836ff6393SMartin Habets } else {
208936ff6393SMartin Habets port1 = rport;
209036ff6393SMartin Habets port2 = gen_spec->loc_port;
209136ff6393SMartin Habets }
209236ff6393SMartin Habets spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
209336ff6393SMartin Habets spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
209436ff6393SMartin Habets spec->data[2] = ntohl(host2);
209536ff6393SMartin Habets
209636ff6393SMartin Habets break;
209736ff6393SMartin Habets }
209836ff6393SMartin Habets
209936ff6393SMartin Habets case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
210036ff6393SMartin Habets is_full = true;
210136ff6393SMartin Habets fallthrough;
210236ff6393SMartin Habets case EFX_FILTER_MATCH_LOC_MAC:
210336ff6393SMartin Habets spec->type = (is_full ? EFX_FARCH_FILTER_MAC_FULL :
210436ff6393SMartin Habets EFX_FARCH_FILTER_MAC_WILD);
210536ff6393SMartin Habets spec->data[0] = is_full ? ntohs(gen_spec->outer_vid) : 0;
210636ff6393SMartin Habets spec->data[1] = (gen_spec->loc_mac[2] << 24 |
210736ff6393SMartin Habets gen_spec->loc_mac[3] << 16 |
210836ff6393SMartin Habets gen_spec->loc_mac[4] << 8 |
210936ff6393SMartin Habets gen_spec->loc_mac[5]);
211036ff6393SMartin Habets spec->data[2] = (gen_spec->loc_mac[0] << 8 |
211136ff6393SMartin Habets gen_spec->loc_mac[1]);
211236ff6393SMartin Habets break;
211336ff6393SMartin Habets
211436ff6393SMartin Habets case EFX_FILTER_MATCH_LOC_MAC_IG:
211536ff6393SMartin Habets spec->type = (is_multicast_ether_addr(gen_spec->loc_mac) ?
211636ff6393SMartin Habets EFX_FARCH_FILTER_MC_DEF :
211736ff6393SMartin Habets EFX_FARCH_FILTER_UC_DEF);
211836ff6393SMartin Habets memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
211936ff6393SMartin Habets break;
212036ff6393SMartin Habets
212136ff6393SMartin Habets default:
212236ff6393SMartin Habets return -EPROTONOSUPPORT;
212336ff6393SMartin Habets }
212436ff6393SMartin Habets
212536ff6393SMartin Habets return 0;
212636ff6393SMartin Habets }
212736ff6393SMartin Habets
212836ff6393SMartin Habets static void
efx_farch_filter_to_gen_spec(struct efx_filter_spec * gen_spec,const struct efx_farch_filter_spec * spec)212936ff6393SMartin Habets efx_farch_filter_to_gen_spec(struct efx_filter_spec *gen_spec,
213036ff6393SMartin Habets const struct efx_farch_filter_spec *spec)
213136ff6393SMartin Habets {
213236ff6393SMartin Habets bool is_full = false;
213336ff6393SMartin Habets
213436ff6393SMartin Habets /* *gen_spec should be completely initialised, to be consistent
213536ff6393SMartin Habets * with efx_filter_init_{rx,tx}() and in case we want to copy
213636ff6393SMartin Habets * it back to userland.
213736ff6393SMartin Habets */
213836ff6393SMartin Habets memset(gen_spec, 0, sizeof(*gen_spec));
213936ff6393SMartin Habets
214036ff6393SMartin Habets gen_spec->priority = spec->priority;
214136ff6393SMartin Habets gen_spec->flags = spec->flags;
214236ff6393SMartin Habets gen_spec->dmaq_id = spec->dmaq_id;
214336ff6393SMartin Habets
214436ff6393SMartin Habets switch (spec->type) {
214536ff6393SMartin Habets case EFX_FARCH_FILTER_TCP_FULL:
214636ff6393SMartin Habets case EFX_FARCH_FILTER_UDP_FULL:
214736ff6393SMartin Habets is_full = true;
214836ff6393SMartin Habets fallthrough;
214936ff6393SMartin Habets case EFX_FARCH_FILTER_TCP_WILD:
215036ff6393SMartin Habets case EFX_FARCH_FILTER_UDP_WILD: {
215136ff6393SMartin Habets __be32 host1, host2;
215236ff6393SMartin Habets __be16 port1, port2;
215336ff6393SMartin Habets
215436ff6393SMartin Habets gen_spec->match_flags =
215536ff6393SMartin Habets EFX_FILTER_MATCH_ETHER_TYPE |
215636ff6393SMartin Habets EFX_FILTER_MATCH_IP_PROTO |
215736ff6393SMartin Habets EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
215836ff6393SMartin Habets if (is_full)
215936ff6393SMartin Habets gen_spec->match_flags |= (EFX_FILTER_MATCH_REM_HOST |
216036ff6393SMartin Habets EFX_FILTER_MATCH_REM_PORT);
216136ff6393SMartin Habets gen_spec->ether_type = htons(ETH_P_IP);
216236ff6393SMartin Habets gen_spec->ip_proto =
216336ff6393SMartin Habets (spec->type == EFX_FARCH_FILTER_TCP_FULL ||
216436ff6393SMartin Habets spec->type == EFX_FARCH_FILTER_TCP_WILD) ?
216536ff6393SMartin Habets IPPROTO_TCP : IPPROTO_UDP;
216636ff6393SMartin Habets
216736ff6393SMartin Habets host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16);
216836ff6393SMartin Habets port1 = htons(spec->data[0]);
216936ff6393SMartin Habets host2 = htonl(spec->data[2]);
217036ff6393SMartin Habets port2 = htons(spec->data[1] >> 16);
217136ff6393SMartin Habets if (spec->flags & EFX_FILTER_FLAG_TX) {
217236ff6393SMartin Habets gen_spec->loc_host[0] = host1;
217336ff6393SMartin Habets gen_spec->rem_host[0] = host2;
217436ff6393SMartin Habets } else {
217536ff6393SMartin Habets gen_spec->loc_host[0] = host2;
217636ff6393SMartin Habets gen_spec->rem_host[0] = host1;
217736ff6393SMartin Habets }
217836ff6393SMartin Habets if (!!(gen_spec->flags & EFX_FILTER_FLAG_TX) ^
217936ff6393SMartin Habets (!is_full && gen_spec->ip_proto == IPPROTO_UDP)) {
218036ff6393SMartin Habets gen_spec->loc_port = port1;
218136ff6393SMartin Habets gen_spec->rem_port = port2;
218236ff6393SMartin Habets } else {
218336ff6393SMartin Habets gen_spec->loc_port = port2;
218436ff6393SMartin Habets gen_spec->rem_port = port1;
218536ff6393SMartin Habets }
218636ff6393SMartin Habets
218736ff6393SMartin Habets break;
218836ff6393SMartin Habets }
218936ff6393SMartin Habets
219036ff6393SMartin Habets case EFX_FARCH_FILTER_MAC_FULL:
219136ff6393SMartin Habets is_full = true;
219236ff6393SMartin Habets fallthrough;
219336ff6393SMartin Habets case EFX_FARCH_FILTER_MAC_WILD:
219436ff6393SMartin Habets gen_spec->match_flags = EFX_FILTER_MATCH_LOC_MAC;
219536ff6393SMartin Habets if (is_full)
219636ff6393SMartin Habets gen_spec->match_flags |= EFX_FILTER_MATCH_OUTER_VID;
219736ff6393SMartin Habets gen_spec->loc_mac[0] = spec->data[2] >> 8;
219836ff6393SMartin Habets gen_spec->loc_mac[1] = spec->data[2];
219936ff6393SMartin Habets gen_spec->loc_mac[2] = spec->data[1] >> 24;
220036ff6393SMartin Habets gen_spec->loc_mac[3] = spec->data[1] >> 16;
220136ff6393SMartin Habets gen_spec->loc_mac[4] = spec->data[1] >> 8;
220236ff6393SMartin Habets gen_spec->loc_mac[5] = spec->data[1];
220336ff6393SMartin Habets gen_spec->outer_vid = htons(spec->data[0]);
220436ff6393SMartin Habets break;
220536ff6393SMartin Habets
220636ff6393SMartin Habets case EFX_FARCH_FILTER_UC_DEF:
220736ff6393SMartin Habets case EFX_FARCH_FILTER_MC_DEF:
220836ff6393SMartin Habets gen_spec->match_flags = EFX_FILTER_MATCH_LOC_MAC_IG;
220936ff6393SMartin Habets gen_spec->loc_mac[0] = spec->type == EFX_FARCH_FILTER_MC_DEF;
221036ff6393SMartin Habets break;
221136ff6393SMartin Habets
221236ff6393SMartin Habets default:
221336ff6393SMartin Habets WARN_ON(1);
221436ff6393SMartin Habets break;
221536ff6393SMartin Habets }
221636ff6393SMartin Habets }
221736ff6393SMartin Habets
221836ff6393SMartin Habets static void
efx_farch_filter_init_rx_auto(struct efx_nic * efx,struct efx_farch_filter_spec * spec)221936ff6393SMartin Habets efx_farch_filter_init_rx_auto(struct efx_nic *efx,
222036ff6393SMartin Habets struct efx_farch_filter_spec *spec)
222136ff6393SMartin Habets {
222236ff6393SMartin Habets /* If there's only one channel then disable RSS for non VF
222336ff6393SMartin Habets * traffic, thereby allowing VFs to use RSS when the PF can't.
222436ff6393SMartin Habets */
222536ff6393SMartin Habets spec->priority = EFX_FILTER_PRI_AUTO;
222636ff6393SMartin Habets spec->flags = (EFX_FILTER_FLAG_RX |
222736ff6393SMartin Habets (efx_rss_enabled(efx) ? EFX_FILTER_FLAG_RX_RSS : 0) |
222836ff6393SMartin Habets (efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0));
222936ff6393SMartin Habets spec->dmaq_id = 0;
223036ff6393SMartin Habets }
223136ff6393SMartin Habets
223236ff6393SMartin Habets /* Build a filter entry and return its n-tuple key. */
efx_farch_filter_build(efx_oword_t * filter,struct efx_farch_filter_spec * spec)223336ff6393SMartin Habets static u32 efx_farch_filter_build(efx_oword_t *filter,
223436ff6393SMartin Habets struct efx_farch_filter_spec *spec)
223536ff6393SMartin Habets {
223636ff6393SMartin Habets u32 data3;
223736ff6393SMartin Habets
223836ff6393SMartin Habets switch (efx_farch_filter_spec_table_id(spec)) {
223936ff6393SMartin Habets case EFX_FARCH_FILTER_TABLE_RX_IP: {
224036ff6393SMartin Habets bool is_udp = (spec->type == EFX_FARCH_FILTER_UDP_FULL ||
224136ff6393SMartin Habets spec->type == EFX_FARCH_FILTER_UDP_WILD);
224236ff6393SMartin Habets EFX_POPULATE_OWORD_7(
224336ff6393SMartin Habets *filter,
224436ff6393SMartin Habets FRF_BZ_RSS_EN,
224536ff6393SMartin Habets !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
224636ff6393SMartin Habets FRF_BZ_SCATTER_EN,
224736ff6393SMartin Habets !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
224836ff6393SMartin Habets FRF_BZ_TCP_UDP, is_udp,
224936ff6393SMartin Habets FRF_BZ_RXQ_ID, spec->dmaq_id,
225036ff6393SMartin Habets EFX_DWORD_2, spec->data[2],
225136ff6393SMartin Habets EFX_DWORD_1, spec->data[1],
225236ff6393SMartin Habets EFX_DWORD_0, spec->data[0]);
225336ff6393SMartin Habets data3 = is_udp;
225436ff6393SMartin Habets break;
225536ff6393SMartin Habets }
225636ff6393SMartin Habets
225736ff6393SMartin Habets case EFX_FARCH_FILTER_TABLE_RX_MAC: {
225836ff6393SMartin Habets bool is_wild = spec->type == EFX_FARCH_FILTER_MAC_WILD;
225936ff6393SMartin Habets EFX_POPULATE_OWORD_7(
226036ff6393SMartin Habets *filter,
226136ff6393SMartin Habets FRF_CZ_RMFT_RSS_EN,
226236ff6393SMartin Habets !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
226336ff6393SMartin Habets FRF_CZ_RMFT_SCATTER_EN,
226436ff6393SMartin Habets !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
226536ff6393SMartin Habets FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
226636ff6393SMartin Habets FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
226736ff6393SMartin Habets FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
226836ff6393SMartin Habets FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
226936ff6393SMartin Habets FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
227036ff6393SMartin Habets data3 = is_wild;
227136ff6393SMartin Habets break;
227236ff6393SMartin Habets }
227336ff6393SMartin Habets
227436ff6393SMartin Habets case EFX_FARCH_FILTER_TABLE_TX_MAC: {
227536ff6393SMartin Habets bool is_wild = spec->type == EFX_FARCH_FILTER_MAC_WILD;
227636ff6393SMartin Habets EFX_POPULATE_OWORD_5(*filter,
227736ff6393SMartin Habets FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id,
227836ff6393SMartin Habets FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
227936ff6393SMartin Habets FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2],
228036ff6393SMartin Habets FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1],
228136ff6393SMartin Habets FRF_CZ_TMFT_VLAN_ID, spec->data[0]);
228236ff6393SMartin Habets data3 = is_wild | spec->dmaq_id << 1;
228336ff6393SMartin Habets break;
228436ff6393SMartin Habets }
228536ff6393SMartin Habets
228636ff6393SMartin Habets default:
228736ff6393SMartin Habets BUG();
228836ff6393SMartin Habets }
228936ff6393SMartin Habets
229036ff6393SMartin Habets return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
229136ff6393SMartin Habets }
229236ff6393SMartin Habets
efx_farch_filter_equal(const struct efx_farch_filter_spec * left,const struct efx_farch_filter_spec * right)229336ff6393SMartin Habets static bool efx_farch_filter_equal(const struct efx_farch_filter_spec *left,
229436ff6393SMartin Habets const struct efx_farch_filter_spec *right)
229536ff6393SMartin Habets {
229636ff6393SMartin Habets if (left->type != right->type ||
229736ff6393SMartin Habets memcmp(left->data, right->data, sizeof(left->data)))
229836ff6393SMartin Habets return false;
229936ff6393SMartin Habets
230036ff6393SMartin Habets if (left->flags & EFX_FILTER_FLAG_TX &&
230136ff6393SMartin Habets left->dmaq_id != right->dmaq_id)
230236ff6393SMartin Habets return false;
230336ff6393SMartin Habets
230436ff6393SMartin Habets return true;
230536ff6393SMartin Habets }
230636ff6393SMartin Habets
230736ff6393SMartin Habets /*
230836ff6393SMartin Habets * Construct/deconstruct external filter IDs. At least the RX filter
230936ff6393SMartin Habets * IDs must be ordered by matching priority, for RX NFC semantics.
231036ff6393SMartin Habets *
231136ff6393SMartin Habets * Deconstruction needs to be robust against invalid IDs so that
231236ff6393SMartin Habets * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can
231336ff6393SMartin Habets * accept user-provided IDs.
231436ff6393SMartin Habets */
231536ff6393SMartin Habets
231636ff6393SMartin Habets #define EFX_FARCH_FILTER_MATCH_PRI_COUNT 5
231736ff6393SMartin Habets
231836ff6393SMartin Habets static const u8 efx_farch_filter_type_match_pri[EFX_FARCH_FILTER_TYPE_COUNT] = {
231936ff6393SMartin Habets [EFX_FARCH_FILTER_TCP_FULL] = 0,
232036ff6393SMartin Habets [EFX_FARCH_FILTER_UDP_FULL] = 0,
232136ff6393SMartin Habets [EFX_FARCH_FILTER_TCP_WILD] = 1,
232236ff6393SMartin Habets [EFX_FARCH_FILTER_UDP_WILD] = 1,
232336ff6393SMartin Habets [EFX_FARCH_FILTER_MAC_FULL] = 2,
232436ff6393SMartin Habets [EFX_FARCH_FILTER_MAC_WILD] = 3,
232536ff6393SMartin Habets [EFX_FARCH_FILTER_UC_DEF] = 4,
232636ff6393SMartin Habets [EFX_FARCH_FILTER_MC_DEF] = 4,
232736ff6393SMartin Habets };
232836ff6393SMartin Habets
232936ff6393SMartin Habets static const enum efx_farch_filter_table_id efx_farch_filter_range_table[] = {
233036ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_IP, /* RX match pri 0 */
233136ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_IP,
233236ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_MAC,
233336ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_MAC,
233436ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_RX_DEF, /* RX match pri 4 */
233536ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_TX_MAC, /* TX match pri 0 */
233636ff6393SMartin Habets EFX_FARCH_FILTER_TABLE_TX_MAC, /* TX match pri 1 */
233736ff6393SMartin Habets };
233836ff6393SMartin Habets
233936ff6393SMartin Habets #define EFX_FARCH_FILTER_INDEX_WIDTH 13
234036ff6393SMartin Habets #define EFX_FARCH_FILTER_INDEX_MASK ((1 << EFX_FARCH_FILTER_INDEX_WIDTH) - 1)
234136ff6393SMartin Habets
234236ff6393SMartin Habets static inline u32
efx_farch_filter_make_id(const struct efx_farch_filter_spec * spec,unsigned int index)234336ff6393SMartin Habets efx_farch_filter_make_id(const struct efx_farch_filter_spec *spec,
234436ff6393SMartin Habets unsigned int index)
234536ff6393SMartin Habets {
234636ff6393SMartin Habets unsigned int range;
234736ff6393SMartin Habets
234836ff6393SMartin Habets range = efx_farch_filter_type_match_pri[spec->type];
234936ff6393SMartin Habets if (!(spec->flags & EFX_FILTER_FLAG_RX))
235036ff6393SMartin Habets range += EFX_FARCH_FILTER_MATCH_PRI_COUNT;
235136ff6393SMartin Habets
235236ff6393SMartin Habets return range << EFX_FARCH_FILTER_INDEX_WIDTH | index;
235336ff6393SMartin Habets }
235436ff6393SMartin Habets
235536ff6393SMartin Habets static inline enum efx_farch_filter_table_id
efx_farch_filter_id_table_id(u32 id)235636ff6393SMartin Habets efx_farch_filter_id_table_id(u32 id)
235736ff6393SMartin Habets {
235836ff6393SMartin Habets unsigned int range = id >> EFX_FARCH_FILTER_INDEX_WIDTH;
235936ff6393SMartin Habets
236036ff6393SMartin Habets if (range < ARRAY_SIZE(efx_farch_filter_range_table))
236136ff6393SMartin Habets return efx_farch_filter_range_table[range];
236236ff6393SMartin Habets else
236336ff6393SMartin Habets return EFX_FARCH_FILTER_TABLE_COUNT; /* invalid */
236436ff6393SMartin Habets }
236536ff6393SMartin Habets
efx_farch_filter_id_index(u32 id)236636ff6393SMartin Habets static inline unsigned int efx_farch_filter_id_index(u32 id)
236736ff6393SMartin Habets {
236836ff6393SMartin Habets return id & EFX_FARCH_FILTER_INDEX_MASK;
236936ff6393SMartin Habets }
237036ff6393SMartin Habets
efx_farch_filter_get_rx_id_limit(struct efx_nic * efx)237136ff6393SMartin Habets u32 efx_farch_filter_get_rx_id_limit(struct efx_nic *efx)
237236ff6393SMartin Habets {
237336ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
237436ff6393SMartin Habets unsigned int range = EFX_FARCH_FILTER_MATCH_PRI_COUNT - 1;
237536ff6393SMartin Habets enum efx_farch_filter_table_id table_id;
237636ff6393SMartin Habets
237736ff6393SMartin Habets do {
237836ff6393SMartin Habets table_id = efx_farch_filter_range_table[range];
237936ff6393SMartin Habets if (state->table[table_id].size != 0)
238036ff6393SMartin Habets return range << EFX_FARCH_FILTER_INDEX_WIDTH |
238136ff6393SMartin Habets state->table[table_id].size;
238236ff6393SMartin Habets } while (range--);
238336ff6393SMartin Habets
238436ff6393SMartin Habets return 0;
238536ff6393SMartin Habets }
238636ff6393SMartin Habets
efx_farch_filter_insert(struct efx_nic * efx,struct efx_filter_spec * gen_spec,bool replace_equal)238736ff6393SMartin Habets s32 efx_farch_filter_insert(struct efx_nic *efx,
238836ff6393SMartin Habets struct efx_filter_spec *gen_spec,
238936ff6393SMartin Habets bool replace_equal)
239036ff6393SMartin Habets {
239136ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
239236ff6393SMartin Habets struct efx_farch_filter_table *table;
239336ff6393SMartin Habets struct efx_farch_filter_spec spec;
239436ff6393SMartin Habets efx_oword_t filter;
239536ff6393SMartin Habets int rep_index, ins_index;
239636ff6393SMartin Habets unsigned int depth = 0;
239736ff6393SMartin Habets int rc;
239836ff6393SMartin Habets
239936ff6393SMartin Habets rc = efx_farch_filter_from_gen_spec(&spec, gen_spec);
240036ff6393SMartin Habets if (rc)
240136ff6393SMartin Habets return rc;
240236ff6393SMartin Habets
240336ff6393SMartin Habets down_write(&state->lock);
240436ff6393SMartin Habets
240536ff6393SMartin Habets table = &state->table[efx_farch_filter_spec_table_id(&spec)];
240636ff6393SMartin Habets if (table->size == 0) {
240736ff6393SMartin Habets rc = -EINVAL;
240836ff6393SMartin Habets goto out_unlock;
240936ff6393SMartin Habets }
241036ff6393SMartin Habets
241136ff6393SMartin Habets netif_vdbg(efx, hw, efx->net_dev,
241236ff6393SMartin Habets "%s: type %d search_limit=%d", __func__, spec.type,
241336ff6393SMartin Habets table->search_limit[spec.type]);
241436ff6393SMartin Habets
241536ff6393SMartin Habets if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) {
241636ff6393SMartin Habets /* One filter spec per type */
241736ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_INDEX_UC_DEF != 0);
241836ff6393SMartin Habets BUILD_BUG_ON(EFX_FARCH_FILTER_INDEX_MC_DEF !=
241936ff6393SMartin Habets EFX_FARCH_FILTER_MC_DEF - EFX_FARCH_FILTER_UC_DEF);
242036ff6393SMartin Habets rep_index = spec.type - EFX_FARCH_FILTER_UC_DEF;
242136ff6393SMartin Habets ins_index = rep_index;
242236ff6393SMartin Habets } else {
242336ff6393SMartin Habets /* Search concurrently for
242436ff6393SMartin Habets * (1) a filter to be replaced (rep_index): any filter
242536ff6393SMartin Habets * with the same match values, up to the current
242636ff6393SMartin Habets * search depth for this type, and
242736ff6393SMartin Habets * (2) the insertion point (ins_index): (1) or any
242836ff6393SMartin Habets * free slot before it or up to the maximum search
242936ff6393SMartin Habets * depth for this priority
243036ff6393SMartin Habets * We fail if we cannot find (2).
243136ff6393SMartin Habets *
243236ff6393SMartin Habets * We can stop once either
243336ff6393SMartin Habets * (a) we find (1), in which case we have definitely
243436ff6393SMartin Habets * found (2) as well; or
243536ff6393SMartin Habets * (b) we have searched exhaustively for (1), and have
243636ff6393SMartin Habets * either found (2) or searched exhaustively for it
243736ff6393SMartin Habets */
243836ff6393SMartin Habets u32 key = efx_farch_filter_build(&filter, &spec);
243936ff6393SMartin Habets unsigned int hash = efx_farch_filter_hash(key);
244036ff6393SMartin Habets unsigned int incr = efx_farch_filter_increment(key);
244136ff6393SMartin Habets unsigned int max_rep_depth = table->search_limit[spec.type];
244236ff6393SMartin Habets unsigned int max_ins_depth =
244336ff6393SMartin Habets spec.priority <= EFX_FILTER_PRI_HINT ?
244436ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_HINT_MAX :
244536ff6393SMartin Habets EFX_FARCH_FILTER_CTL_SRCH_MAX;
244636ff6393SMartin Habets unsigned int i = hash & (table->size - 1);
244736ff6393SMartin Habets
244836ff6393SMartin Habets ins_index = -1;
244936ff6393SMartin Habets depth = 1;
245036ff6393SMartin Habets
245136ff6393SMartin Habets for (;;) {
245236ff6393SMartin Habets if (!test_bit(i, table->used_bitmap)) {
245336ff6393SMartin Habets if (ins_index < 0)
245436ff6393SMartin Habets ins_index = i;
245536ff6393SMartin Habets } else if (efx_farch_filter_equal(&spec,
245636ff6393SMartin Habets &table->spec[i])) {
245736ff6393SMartin Habets /* Case (a) */
245836ff6393SMartin Habets if (ins_index < 0)
245936ff6393SMartin Habets ins_index = i;
246036ff6393SMartin Habets rep_index = i;
246136ff6393SMartin Habets break;
246236ff6393SMartin Habets }
246336ff6393SMartin Habets
246436ff6393SMartin Habets if (depth >= max_rep_depth &&
246536ff6393SMartin Habets (ins_index >= 0 || depth >= max_ins_depth)) {
246636ff6393SMartin Habets /* Case (b) */
246736ff6393SMartin Habets if (ins_index < 0) {
246836ff6393SMartin Habets rc = -EBUSY;
246936ff6393SMartin Habets goto out_unlock;
247036ff6393SMartin Habets }
247136ff6393SMartin Habets rep_index = -1;
247236ff6393SMartin Habets break;
247336ff6393SMartin Habets }
247436ff6393SMartin Habets
247536ff6393SMartin Habets i = (i + incr) & (table->size - 1);
247636ff6393SMartin Habets ++depth;
247736ff6393SMartin Habets }
247836ff6393SMartin Habets }
247936ff6393SMartin Habets
248036ff6393SMartin Habets /* If we found a filter to be replaced, check whether we
248136ff6393SMartin Habets * should do so
248236ff6393SMartin Habets */
248336ff6393SMartin Habets if (rep_index >= 0) {
248436ff6393SMartin Habets struct efx_farch_filter_spec *saved_spec =
248536ff6393SMartin Habets &table->spec[rep_index];
248636ff6393SMartin Habets
248736ff6393SMartin Habets if (spec.priority == saved_spec->priority && !replace_equal) {
248836ff6393SMartin Habets rc = -EEXIST;
248936ff6393SMartin Habets goto out_unlock;
249036ff6393SMartin Habets }
249136ff6393SMartin Habets if (spec.priority < saved_spec->priority) {
249236ff6393SMartin Habets rc = -EPERM;
249336ff6393SMartin Habets goto out_unlock;
249436ff6393SMartin Habets }
249536ff6393SMartin Habets if (saved_spec->priority == EFX_FILTER_PRI_AUTO ||
249636ff6393SMartin Habets saved_spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO)
249736ff6393SMartin Habets spec.flags |= EFX_FILTER_FLAG_RX_OVER_AUTO;
249836ff6393SMartin Habets }
249936ff6393SMartin Habets
250036ff6393SMartin Habets /* Insert the filter */
250136ff6393SMartin Habets if (ins_index != rep_index) {
250236ff6393SMartin Habets __set_bit(ins_index, table->used_bitmap);
250336ff6393SMartin Habets ++table->used;
250436ff6393SMartin Habets }
250536ff6393SMartin Habets table->spec[ins_index] = spec;
250636ff6393SMartin Habets
250736ff6393SMartin Habets if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) {
250836ff6393SMartin Habets efx_farch_filter_push_rx_config(efx);
250936ff6393SMartin Habets } else {
251036ff6393SMartin Habets if (table->search_limit[spec.type] < depth) {
251136ff6393SMartin Habets table->search_limit[spec.type] = depth;
251236ff6393SMartin Habets if (spec.flags & EFX_FILTER_FLAG_TX)
251336ff6393SMartin Habets efx_farch_filter_push_tx_limits(efx);
251436ff6393SMartin Habets else
251536ff6393SMartin Habets efx_farch_filter_push_rx_config(efx);
251636ff6393SMartin Habets }
251736ff6393SMartin Habets
251836ff6393SMartin Habets efx_writeo(efx, &filter,
251936ff6393SMartin Habets table->offset + table->step * ins_index);
252036ff6393SMartin Habets
252136ff6393SMartin Habets /* If we were able to replace a filter by inserting
252236ff6393SMartin Habets * at a lower depth, clear the replaced filter
252336ff6393SMartin Habets */
252436ff6393SMartin Habets if (ins_index != rep_index && rep_index >= 0)
252536ff6393SMartin Habets efx_farch_filter_table_clear_entry(efx, table,
252636ff6393SMartin Habets rep_index);
252736ff6393SMartin Habets }
252836ff6393SMartin Habets
252936ff6393SMartin Habets netif_vdbg(efx, hw, efx->net_dev,
253036ff6393SMartin Habets "%s: filter type %d index %d rxq %u set",
253136ff6393SMartin Habets __func__, spec.type, ins_index, spec.dmaq_id);
253236ff6393SMartin Habets rc = efx_farch_filter_make_id(&spec, ins_index);
253336ff6393SMartin Habets
253436ff6393SMartin Habets out_unlock:
253536ff6393SMartin Habets up_write(&state->lock);
253636ff6393SMartin Habets return rc;
253736ff6393SMartin Habets }
253836ff6393SMartin Habets
253936ff6393SMartin Habets static void
efx_farch_filter_table_clear_entry(struct efx_nic * efx,struct efx_farch_filter_table * table,unsigned int filter_idx)254036ff6393SMartin Habets efx_farch_filter_table_clear_entry(struct efx_nic *efx,
254136ff6393SMartin Habets struct efx_farch_filter_table *table,
254236ff6393SMartin Habets unsigned int filter_idx)
254336ff6393SMartin Habets {
254436ff6393SMartin Habets static efx_oword_t filter;
254536ff6393SMartin Habets
254636ff6393SMartin Habets EFX_WARN_ON_PARANOID(!test_bit(filter_idx, table->used_bitmap));
254736ff6393SMartin Habets BUG_ON(table->offset == 0); /* can't clear MAC default filters */
254836ff6393SMartin Habets
254936ff6393SMartin Habets __clear_bit(filter_idx, table->used_bitmap);
255036ff6393SMartin Habets --table->used;
255136ff6393SMartin Habets memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
255236ff6393SMartin Habets
255336ff6393SMartin Habets efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
255436ff6393SMartin Habets
255536ff6393SMartin Habets /* If this filter required a greater search depth than
255636ff6393SMartin Habets * any other, the search limit for its type can now be
255736ff6393SMartin Habets * decreased. However, it is hard to determine that
255836ff6393SMartin Habets * unless the table has become completely empty - in
255936ff6393SMartin Habets * which case, all its search limits can be set to 0.
256036ff6393SMartin Habets */
256136ff6393SMartin Habets if (unlikely(table->used == 0)) {
256236ff6393SMartin Habets memset(table->search_limit, 0, sizeof(table->search_limit));
256336ff6393SMartin Habets if (table->id == EFX_FARCH_FILTER_TABLE_TX_MAC)
256436ff6393SMartin Habets efx_farch_filter_push_tx_limits(efx);
256536ff6393SMartin Habets else
256636ff6393SMartin Habets efx_farch_filter_push_rx_config(efx);
256736ff6393SMartin Habets }
256836ff6393SMartin Habets }
256936ff6393SMartin Habets
efx_farch_filter_remove(struct efx_nic * efx,struct efx_farch_filter_table * table,unsigned int filter_idx,enum efx_filter_priority priority)257036ff6393SMartin Habets static int efx_farch_filter_remove(struct efx_nic *efx,
257136ff6393SMartin Habets struct efx_farch_filter_table *table,
257236ff6393SMartin Habets unsigned int filter_idx,
257336ff6393SMartin Habets enum efx_filter_priority priority)
257436ff6393SMartin Habets {
257536ff6393SMartin Habets struct efx_farch_filter_spec *spec = &table->spec[filter_idx];
257636ff6393SMartin Habets
257736ff6393SMartin Habets if (!test_bit(filter_idx, table->used_bitmap) ||
257836ff6393SMartin Habets spec->priority != priority)
257936ff6393SMartin Habets return -ENOENT;
258036ff6393SMartin Habets
258136ff6393SMartin Habets if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) {
258236ff6393SMartin Habets efx_farch_filter_init_rx_auto(efx, spec);
258336ff6393SMartin Habets efx_farch_filter_push_rx_config(efx);
258436ff6393SMartin Habets } else {
258536ff6393SMartin Habets efx_farch_filter_table_clear_entry(efx, table, filter_idx);
258636ff6393SMartin Habets }
258736ff6393SMartin Habets
258836ff6393SMartin Habets return 0;
258936ff6393SMartin Habets }
259036ff6393SMartin Habets
efx_farch_filter_remove_safe(struct efx_nic * efx,enum efx_filter_priority priority,u32 filter_id)259136ff6393SMartin Habets int efx_farch_filter_remove_safe(struct efx_nic *efx,
259236ff6393SMartin Habets enum efx_filter_priority priority,
259336ff6393SMartin Habets u32 filter_id)
259436ff6393SMartin Habets {
259536ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
259636ff6393SMartin Habets enum efx_farch_filter_table_id table_id;
259736ff6393SMartin Habets struct efx_farch_filter_table *table;
259836ff6393SMartin Habets unsigned int filter_idx;
259936ff6393SMartin Habets int rc;
260036ff6393SMartin Habets
260136ff6393SMartin Habets table_id = efx_farch_filter_id_table_id(filter_id);
260236ff6393SMartin Habets if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT)
260336ff6393SMartin Habets return -ENOENT;
260436ff6393SMartin Habets table = &state->table[table_id];
260536ff6393SMartin Habets
260636ff6393SMartin Habets filter_idx = efx_farch_filter_id_index(filter_id);
260736ff6393SMartin Habets if (filter_idx >= table->size)
260836ff6393SMartin Habets return -ENOENT;
260936ff6393SMartin Habets down_write(&state->lock);
261036ff6393SMartin Habets
261136ff6393SMartin Habets rc = efx_farch_filter_remove(efx, table, filter_idx, priority);
261236ff6393SMartin Habets up_write(&state->lock);
261336ff6393SMartin Habets
261436ff6393SMartin Habets return rc;
261536ff6393SMartin Habets }
261636ff6393SMartin Habets
efx_farch_filter_get_safe(struct efx_nic * efx,enum efx_filter_priority priority,u32 filter_id,struct efx_filter_spec * spec_buf)261736ff6393SMartin Habets int efx_farch_filter_get_safe(struct efx_nic *efx,
261836ff6393SMartin Habets enum efx_filter_priority priority,
261936ff6393SMartin Habets u32 filter_id, struct efx_filter_spec *spec_buf)
262036ff6393SMartin Habets {
262136ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
262236ff6393SMartin Habets enum efx_farch_filter_table_id table_id;
262336ff6393SMartin Habets struct efx_farch_filter_table *table;
262436ff6393SMartin Habets struct efx_farch_filter_spec *spec;
262536ff6393SMartin Habets unsigned int filter_idx;
262636ff6393SMartin Habets int rc = -ENOENT;
262736ff6393SMartin Habets
262836ff6393SMartin Habets down_read(&state->lock);
262936ff6393SMartin Habets
263036ff6393SMartin Habets table_id = efx_farch_filter_id_table_id(filter_id);
263136ff6393SMartin Habets if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT)
263236ff6393SMartin Habets goto out_unlock;
263336ff6393SMartin Habets table = &state->table[table_id];
263436ff6393SMartin Habets
263536ff6393SMartin Habets filter_idx = efx_farch_filter_id_index(filter_id);
263636ff6393SMartin Habets if (filter_idx >= table->size)
263736ff6393SMartin Habets goto out_unlock;
263836ff6393SMartin Habets spec = &table->spec[filter_idx];
263936ff6393SMartin Habets
264036ff6393SMartin Habets if (test_bit(filter_idx, table->used_bitmap) &&
264136ff6393SMartin Habets spec->priority == priority) {
264236ff6393SMartin Habets efx_farch_filter_to_gen_spec(spec_buf, spec);
264336ff6393SMartin Habets rc = 0;
264436ff6393SMartin Habets }
264536ff6393SMartin Habets
264636ff6393SMartin Habets out_unlock:
264736ff6393SMartin Habets up_read(&state->lock);
264836ff6393SMartin Habets return rc;
264936ff6393SMartin Habets }
265036ff6393SMartin Habets
265136ff6393SMartin Habets static void
efx_farch_filter_table_clear(struct efx_nic * efx,enum efx_farch_filter_table_id table_id,enum efx_filter_priority priority)265236ff6393SMartin Habets efx_farch_filter_table_clear(struct efx_nic *efx,
265336ff6393SMartin Habets enum efx_farch_filter_table_id table_id,
265436ff6393SMartin Habets enum efx_filter_priority priority)
265536ff6393SMartin Habets {
265636ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
265736ff6393SMartin Habets struct efx_farch_filter_table *table = &state->table[table_id];
265836ff6393SMartin Habets unsigned int filter_idx;
265936ff6393SMartin Habets
266036ff6393SMartin Habets down_write(&state->lock);
266136ff6393SMartin Habets for (filter_idx = 0; filter_idx < table->size; ++filter_idx) {
266236ff6393SMartin Habets if (table->spec[filter_idx].priority != EFX_FILTER_PRI_AUTO)
266336ff6393SMartin Habets efx_farch_filter_remove(efx, table,
266436ff6393SMartin Habets filter_idx, priority);
266536ff6393SMartin Habets }
266636ff6393SMartin Habets up_write(&state->lock);
266736ff6393SMartin Habets }
266836ff6393SMartin Habets
efx_farch_filter_clear_rx(struct efx_nic * efx,enum efx_filter_priority priority)266936ff6393SMartin Habets int efx_farch_filter_clear_rx(struct efx_nic *efx,
267036ff6393SMartin Habets enum efx_filter_priority priority)
267136ff6393SMartin Habets {
267236ff6393SMartin Habets efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_IP,
267336ff6393SMartin Habets priority);
267436ff6393SMartin Habets efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_MAC,
267536ff6393SMartin Habets priority);
267636ff6393SMartin Habets efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_DEF,
267736ff6393SMartin Habets priority);
267836ff6393SMartin Habets return 0;
267936ff6393SMartin Habets }
268036ff6393SMartin Habets
efx_farch_filter_count_rx_used(struct efx_nic * efx,enum efx_filter_priority priority)268136ff6393SMartin Habets u32 efx_farch_filter_count_rx_used(struct efx_nic *efx,
268236ff6393SMartin Habets enum efx_filter_priority priority)
268336ff6393SMartin Habets {
268436ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
268536ff6393SMartin Habets enum efx_farch_filter_table_id table_id;
268636ff6393SMartin Habets struct efx_farch_filter_table *table;
268736ff6393SMartin Habets unsigned int filter_idx;
268836ff6393SMartin Habets u32 count = 0;
268936ff6393SMartin Habets
269036ff6393SMartin Habets down_read(&state->lock);
269136ff6393SMartin Habets
269236ff6393SMartin Habets for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP;
269336ff6393SMartin Habets table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF;
269436ff6393SMartin Habets table_id++) {
269536ff6393SMartin Habets table = &state->table[table_id];
269636ff6393SMartin Habets for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
269736ff6393SMartin Habets if (test_bit(filter_idx, table->used_bitmap) &&
269836ff6393SMartin Habets table->spec[filter_idx].priority == priority)
269936ff6393SMartin Habets ++count;
270036ff6393SMartin Habets }
270136ff6393SMartin Habets }
270236ff6393SMartin Habets
270336ff6393SMartin Habets up_read(&state->lock);
270436ff6393SMartin Habets
270536ff6393SMartin Habets return count;
270636ff6393SMartin Habets }
270736ff6393SMartin Habets
efx_farch_filter_get_rx_ids(struct efx_nic * efx,enum efx_filter_priority priority,u32 * buf,u32 size)270836ff6393SMartin Habets s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx,
270936ff6393SMartin Habets enum efx_filter_priority priority,
271036ff6393SMartin Habets u32 *buf, u32 size)
271136ff6393SMartin Habets {
271236ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
271336ff6393SMartin Habets enum efx_farch_filter_table_id table_id;
271436ff6393SMartin Habets struct efx_farch_filter_table *table;
271536ff6393SMartin Habets unsigned int filter_idx;
271636ff6393SMartin Habets s32 count = 0;
271736ff6393SMartin Habets
271836ff6393SMartin Habets down_read(&state->lock);
271936ff6393SMartin Habets
272036ff6393SMartin Habets for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP;
272136ff6393SMartin Habets table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF;
272236ff6393SMartin Habets table_id++) {
272336ff6393SMartin Habets table = &state->table[table_id];
272436ff6393SMartin Habets for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
272536ff6393SMartin Habets if (test_bit(filter_idx, table->used_bitmap) &&
272636ff6393SMartin Habets table->spec[filter_idx].priority == priority) {
272736ff6393SMartin Habets if (count == size) {
272836ff6393SMartin Habets count = -EMSGSIZE;
272936ff6393SMartin Habets goto out;
273036ff6393SMartin Habets }
273136ff6393SMartin Habets buf[count++] = efx_farch_filter_make_id(
273236ff6393SMartin Habets &table->spec[filter_idx], filter_idx);
273336ff6393SMartin Habets }
273436ff6393SMartin Habets }
273536ff6393SMartin Habets }
273636ff6393SMartin Habets out:
273736ff6393SMartin Habets up_read(&state->lock);
273836ff6393SMartin Habets
273936ff6393SMartin Habets return count;
274036ff6393SMartin Habets }
274136ff6393SMartin Habets
274236ff6393SMartin Habets /* Restore filter stater after reset */
efx_farch_filter_table_restore(struct efx_nic * efx)274336ff6393SMartin Habets void efx_farch_filter_table_restore(struct efx_nic *efx)
274436ff6393SMartin Habets {
274536ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
274636ff6393SMartin Habets enum efx_farch_filter_table_id table_id;
274736ff6393SMartin Habets struct efx_farch_filter_table *table;
274836ff6393SMartin Habets efx_oword_t filter;
274936ff6393SMartin Habets unsigned int filter_idx;
275036ff6393SMartin Habets
275136ff6393SMartin Habets down_write(&state->lock);
275236ff6393SMartin Habets
275336ff6393SMartin Habets for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) {
275436ff6393SMartin Habets table = &state->table[table_id];
275536ff6393SMartin Habets
275636ff6393SMartin Habets /* Check whether this is a regular register table */
275736ff6393SMartin Habets if (table->step == 0)
275836ff6393SMartin Habets continue;
275936ff6393SMartin Habets
276036ff6393SMartin Habets for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
276136ff6393SMartin Habets if (!test_bit(filter_idx, table->used_bitmap))
276236ff6393SMartin Habets continue;
276336ff6393SMartin Habets efx_farch_filter_build(&filter, &table->spec[filter_idx]);
276436ff6393SMartin Habets efx_writeo(efx, &filter,
276536ff6393SMartin Habets table->offset + table->step * filter_idx);
276636ff6393SMartin Habets }
276736ff6393SMartin Habets }
276836ff6393SMartin Habets
276936ff6393SMartin Habets efx_farch_filter_push_rx_config(efx);
277036ff6393SMartin Habets efx_farch_filter_push_tx_limits(efx);
277136ff6393SMartin Habets
277236ff6393SMartin Habets up_write(&state->lock);
277336ff6393SMartin Habets }
277436ff6393SMartin Habets
efx_farch_filter_table_remove(struct efx_nic * efx)277536ff6393SMartin Habets void efx_farch_filter_table_remove(struct efx_nic *efx)
277636ff6393SMartin Habets {
277736ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
277836ff6393SMartin Habets enum efx_farch_filter_table_id table_id;
277936ff6393SMartin Habets
278036ff6393SMartin Habets for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) {
2781*820aceb5SChristophe JAILLET bitmap_free(state->table[table_id].used_bitmap);
278236ff6393SMartin Habets vfree(state->table[table_id].spec);
278336ff6393SMartin Habets }
278436ff6393SMartin Habets kfree(state);
278536ff6393SMartin Habets }
278636ff6393SMartin Habets
efx_farch_filter_table_probe(struct efx_nic * efx)278736ff6393SMartin Habets int efx_farch_filter_table_probe(struct efx_nic *efx)
278836ff6393SMartin Habets {
278936ff6393SMartin Habets struct efx_farch_filter_state *state;
279036ff6393SMartin Habets struct efx_farch_filter_table *table;
279136ff6393SMartin Habets unsigned table_id;
279236ff6393SMartin Habets
279336ff6393SMartin Habets state = kzalloc(sizeof(struct efx_farch_filter_state), GFP_KERNEL);
279436ff6393SMartin Habets if (!state)
279536ff6393SMartin Habets return -ENOMEM;
279636ff6393SMartin Habets efx->filter_state = state;
279736ff6393SMartin Habets init_rwsem(&state->lock);
279836ff6393SMartin Habets
279936ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP];
280036ff6393SMartin Habets table->id = EFX_FARCH_FILTER_TABLE_RX_IP;
280136ff6393SMartin Habets table->offset = FR_BZ_RX_FILTER_TBL0;
280236ff6393SMartin Habets table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
280336ff6393SMartin Habets table->step = FR_BZ_RX_FILTER_TBL0_STEP;
280436ff6393SMartin Habets
280536ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_RX_MAC];
280636ff6393SMartin Habets table->id = EFX_FARCH_FILTER_TABLE_RX_MAC;
280736ff6393SMartin Habets table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
280836ff6393SMartin Habets table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
280936ff6393SMartin Habets table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
281036ff6393SMartin Habets
281136ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF];
281236ff6393SMartin Habets table->id = EFX_FARCH_FILTER_TABLE_RX_DEF;
281336ff6393SMartin Habets table->size = EFX_FARCH_FILTER_SIZE_RX_DEF;
281436ff6393SMartin Habets
281536ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_TX_MAC];
281636ff6393SMartin Habets table->id = EFX_FARCH_FILTER_TABLE_TX_MAC;
281736ff6393SMartin Habets table->offset = FR_CZ_TX_MAC_FILTER_TBL0;
281836ff6393SMartin Habets table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
281936ff6393SMartin Habets table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP;
282036ff6393SMartin Habets
282136ff6393SMartin Habets for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) {
282236ff6393SMartin Habets table = &state->table[table_id];
282336ff6393SMartin Habets if (table->size == 0)
282436ff6393SMartin Habets continue;
2825*820aceb5SChristophe JAILLET table->used_bitmap = bitmap_zalloc(table->size, GFP_KERNEL);
282636ff6393SMartin Habets if (!table->used_bitmap)
282736ff6393SMartin Habets goto fail;
282836ff6393SMartin Habets table->spec = vzalloc(array_size(sizeof(*table->spec),
282936ff6393SMartin Habets table->size));
283036ff6393SMartin Habets if (!table->spec)
283136ff6393SMartin Habets goto fail;
283236ff6393SMartin Habets }
283336ff6393SMartin Habets
283436ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF];
283536ff6393SMartin Habets if (table->size) {
283636ff6393SMartin Habets /* RX default filters must always exist */
283736ff6393SMartin Habets struct efx_farch_filter_spec *spec;
283836ff6393SMartin Habets unsigned i;
283936ff6393SMartin Habets
284036ff6393SMartin Habets for (i = 0; i < EFX_FARCH_FILTER_SIZE_RX_DEF; i++) {
284136ff6393SMartin Habets spec = &table->spec[i];
284236ff6393SMartin Habets spec->type = EFX_FARCH_FILTER_UC_DEF + i;
284336ff6393SMartin Habets efx_farch_filter_init_rx_auto(efx, spec);
284436ff6393SMartin Habets __set_bit(i, table->used_bitmap);
284536ff6393SMartin Habets }
284636ff6393SMartin Habets }
284736ff6393SMartin Habets
284836ff6393SMartin Habets efx_farch_filter_push_rx_config(efx);
284936ff6393SMartin Habets
285036ff6393SMartin Habets return 0;
285136ff6393SMartin Habets
285236ff6393SMartin Habets fail:
285336ff6393SMartin Habets efx_farch_filter_table_remove(efx);
285436ff6393SMartin Habets return -ENOMEM;
285536ff6393SMartin Habets }
285636ff6393SMartin Habets
285736ff6393SMartin Habets /* Update scatter enable flags for filters pointing to our own RX queues */
efx_farch_filter_update_rx_scatter(struct efx_nic * efx)285836ff6393SMartin Habets void efx_farch_filter_update_rx_scatter(struct efx_nic *efx)
285936ff6393SMartin Habets {
286036ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
286136ff6393SMartin Habets enum efx_farch_filter_table_id table_id;
286236ff6393SMartin Habets struct efx_farch_filter_table *table;
286336ff6393SMartin Habets efx_oword_t filter;
286436ff6393SMartin Habets unsigned int filter_idx;
286536ff6393SMartin Habets
286636ff6393SMartin Habets down_write(&state->lock);
286736ff6393SMartin Habets
286836ff6393SMartin Habets for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP;
286936ff6393SMartin Habets table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF;
287036ff6393SMartin Habets table_id++) {
287136ff6393SMartin Habets table = &state->table[table_id];
287236ff6393SMartin Habets
287336ff6393SMartin Habets for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
287436ff6393SMartin Habets if (!test_bit(filter_idx, table->used_bitmap) ||
287536ff6393SMartin Habets table->spec[filter_idx].dmaq_id >=
287636ff6393SMartin Habets efx->n_rx_channels)
287736ff6393SMartin Habets continue;
287836ff6393SMartin Habets
287936ff6393SMartin Habets if (efx->rx_scatter)
288036ff6393SMartin Habets table->spec[filter_idx].flags |=
288136ff6393SMartin Habets EFX_FILTER_FLAG_RX_SCATTER;
288236ff6393SMartin Habets else
288336ff6393SMartin Habets table->spec[filter_idx].flags &=
288436ff6393SMartin Habets ~EFX_FILTER_FLAG_RX_SCATTER;
288536ff6393SMartin Habets
288636ff6393SMartin Habets if (table_id == EFX_FARCH_FILTER_TABLE_RX_DEF)
288736ff6393SMartin Habets /* Pushed by efx_farch_filter_push_rx_config() */
288836ff6393SMartin Habets continue;
288936ff6393SMartin Habets
289036ff6393SMartin Habets efx_farch_filter_build(&filter, &table->spec[filter_idx]);
289136ff6393SMartin Habets efx_writeo(efx, &filter,
289236ff6393SMartin Habets table->offset + table->step * filter_idx);
289336ff6393SMartin Habets }
289436ff6393SMartin Habets }
289536ff6393SMartin Habets
289636ff6393SMartin Habets efx_farch_filter_push_rx_config(efx);
289736ff6393SMartin Habets
289836ff6393SMartin Habets up_write(&state->lock);
289936ff6393SMartin Habets }
290036ff6393SMartin Habets
290136ff6393SMartin Habets #ifdef CONFIG_RFS_ACCEL
290236ff6393SMartin Habets
efx_farch_filter_rfs_expire_one(struct efx_nic * efx,u32 flow_id,unsigned int index)290336ff6393SMartin Habets bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
290436ff6393SMartin Habets unsigned int index)
290536ff6393SMartin Habets {
290636ff6393SMartin Habets struct efx_farch_filter_state *state = efx->filter_state;
290736ff6393SMartin Habets struct efx_farch_filter_table *table;
290836ff6393SMartin Habets bool ret = false, force = false;
290936ff6393SMartin Habets u16 arfs_id;
291036ff6393SMartin Habets
291136ff6393SMartin Habets down_write(&state->lock);
291236ff6393SMartin Habets spin_lock_bh(&efx->rps_hash_lock);
291336ff6393SMartin Habets table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP];
291436ff6393SMartin Habets if (test_bit(index, table->used_bitmap) &&
291536ff6393SMartin Habets table->spec[index].priority == EFX_FILTER_PRI_HINT) {
291636ff6393SMartin Habets struct efx_arfs_rule *rule = NULL;
291736ff6393SMartin Habets struct efx_filter_spec spec;
291836ff6393SMartin Habets
291936ff6393SMartin Habets efx_farch_filter_to_gen_spec(&spec, &table->spec[index]);
292036ff6393SMartin Habets if (!efx->rps_hash_table) {
292136ff6393SMartin Habets /* In the absence of the table, we always returned 0 to
292236ff6393SMartin Habets * ARFS, so use the same to query it.
292336ff6393SMartin Habets */
292436ff6393SMartin Habets arfs_id = 0;
292536ff6393SMartin Habets } else {
29267f9e4b2aSMartin Habets rule = efx_siena_rps_hash_find(efx, &spec);
292736ff6393SMartin Habets if (!rule) {
292836ff6393SMartin Habets /* ARFS table doesn't know of this filter, remove it */
292936ff6393SMartin Habets force = true;
293036ff6393SMartin Habets } else {
293136ff6393SMartin Habets arfs_id = rule->arfs_id;
29327f9e4b2aSMartin Habets if (!efx_siena_rps_check_rule(rule, index,
29337f9e4b2aSMartin Habets &force))
293436ff6393SMartin Habets goto out_unlock;
293536ff6393SMartin Habets }
293636ff6393SMartin Habets }
293736ff6393SMartin Habets if (force || rps_may_expire_flow(efx->net_dev, spec.dmaq_id,
293836ff6393SMartin Habets flow_id, arfs_id)) {
293936ff6393SMartin Habets if (rule)
294036ff6393SMartin Habets rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
29417f9e4b2aSMartin Habets efx_siena_rps_hash_del(efx, &spec);
294236ff6393SMartin Habets efx_farch_filter_table_clear_entry(efx, table, index);
294336ff6393SMartin Habets ret = true;
294436ff6393SMartin Habets }
294536ff6393SMartin Habets }
294636ff6393SMartin Habets out_unlock:
294736ff6393SMartin Habets spin_unlock_bh(&efx->rps_hash_lock);
294836ff6393SMartin Habets up_write(&state->lock);
294936ff6393SMartin Habets return ret;
295036ff6393SMartin Habets }
295136ff6393SMartin Habets
295236ff6393SMartin Habets #endif /* CONFIG_RFS_ACCEL */
295336ff6393SMartin Habets
efx_farch_filter_sync_rx_mode(struct efx_nic * efx)295436ff6393SMartin Habets void efx_farch_filter_sync_rx_mode(struct efx_nic *efx)
295536ff6393SMartin Habets {
295636ff6393SMartin Habets struct net_device *net_dev = efx->net_dev;
295736ff6393SMartin Habets struct netdev_hw_addr *ha;
295836ff6393SMartin Habets union efx_multicast_hash *mc_hash = &efx->multicast_hash;
295936ff6393SMartin Habets u32 crc;
296036ff6393SMartin Habets int bit;
296136ff6393SMartin Habets
296236ff6393SMartin Habets if (!efx_dev_registered(efx))
296336ff6393SMartin Habets return;
296436ff6393SMartin Habets
296536ff6393SMartin Habets netif_addr_lock_bh(net_dev);
296636ff6393SMartin Habets
296736ff6393SMartin Habets efx->unicast_filter = !(net_dev->flags & IFF_PROMISC);
296836ff6393SMartin Habets
296936ff6393SMartin Habets /* Build multicast hash table */
297036ff6393SMartin Habets if (net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) {
297136ff6393SMartin Habets memset(mc_hash, 0xff, sizeof(*mc_hash));
297236ff6393SMartin Habets } else {
297336ff6393SMartin Habets memset(mc_hash, 0x00, sizeof(*mc_hash));
297436ff6393SMartin Habets netdev_for_each_mc_addr(ha, net_dev) {
297536ff6393SMartin Habets crc = ether_crc_le(ETH_ALEN, ha->addr);
297636ff6393SMartin Habets bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);
297736ff6393SMartin Habets __set_bit_le(bit, mc_hash);
297836ff6393SMartin Habets }
297936ff6393SMartin Habets
298036ff6393SMartin Habets /* Broadcast packets go through the multicast hash filter.
298136ff6393SMartin Habets * ether_crc_le() of the broadcast address is 0xbe2612ff
298236ff6393SMartin Habets * so we always add bit 0xff to the mask.
298336ff6393SMartin Habets */
298436ff6393SMartin Habets __set_bit_le(0xff, mc_hash);
298536ff6393SMartin Habets }
298636ff6393SMartin Habets
298736ff6393SMartin Habets netif_addr_unlock_bh(net_dev);
298836ff6393SMartin Habets }
2989