1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2005-2006 Fen Systems Ltd.
5 * Copyright 2006-2013 Solarflare Communications Inc.
6 */
7
8 #include <linux/bitops.h>
9 #include <linux/delay.h>
10 #include <linux/interrupt.h>
11 #include <linux/pci.h>
12 #include <linux/module.h>
13 #include <linux/seq_file.h>
14 #include <linux/cpu_rmap.h>
15 #include "net_driver.h"
16 #include "bitfield.h"
17 #include "efx.h"
18 #include "nic.h"
19 #include "farch_regs.h"
20 #include "io.h"
21 #include "workarounds.h"
22
23 /**************************************************************************
24 *
25 * Generic buffer handling
26 * These buffers are used for interrupt status, MAC stats, etc.
27 *
28 **************************************************************************/
29
ef4_nic_alloc_buffer(struct ef4_nic * efx,struct ef4_buffer * buffer,unsigned int len,gfp_t gfp_flags)30 int ef4_nic_alloc_buffer(struct ef4_nic *efx, struct ef4_buffer *buffer,
31 unsigned int len, gfp_t gfp_flags)
32 {
33 buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,
34 &buffer->dma_addr, gfp_flags);
35 if (!buffer->addr)
36 return -ENOMEM;
37 buffer->len = len;
38 return 0;
39 }
40
ef4_nic_free_buffer(struct ef4_nic * efx,struct ef4_buffer * buffer)41 void ef4_nic_free_buffer(struct ef4_nic *efx, struct ef4_buffer *buffer)
42 {
43 if (buffer->addr) {
44 dma_free_coherent(&efx->pci_dev->dev, buffer->len,
45 buffer->addr, buffer->dma_addr);
46 buffer->addr = NULL;
47 }
48 }
49
50 /* Check whether an event is present in the eventq at the current
51 * read pointer. Only useful for self-test.
52 */
ef4_nic_event_present(struct ef4_channel * channel)53 bool ef4_nic_event_present(struct ef4_channel *channel)
54 {
55 return ef4_event_present(ef4_event(channel, channel->eventq_read_ptr));
56 }
57
ef4_nic_event_test_start(struct ef4_channel * channel)58 void ef4_nic_event_test_start(struct ef4_channel *channel)
59 {
60 channel->event_test_cpu = -1;
61 smp_wmb();
62 channel->efx->type->ev_test_generate(channel);
63 }
64
ef4_nic_irq_test_start(struct ef4_nic * efx)65 int ef4_nic_irq_test_start(struct ef4_nic *efx)
66 {
67 efx->last_irq_cpu = -1;
68 smp_wmb();
69 return efx->type->irq_test_generate(efx);
70 }
71
72 /* Hook interrupt handler(s)
73 * Try MSI and then legacy interrupts.
74 */
ef4_nic_init_interrupt(struct ef4_nic * efx)75 int ef4_nic_init_interrupt(struct ef4_nic *efx)
76 {
77 struct ef4_channel *channel;
78 unsigned int n_irqs;
79 int rc;
80
81 if (!EF4_INT_MODE_USE_MSI(efx)) {
82 rc = request_irq(efx->legacy_irq,
83 efx->type->irq_handle_legacy, IRQF_SHARED,
84 efx->name, efx);
85 if (rc) {
86 netif_err(efx, drv, efx->net_dev,
87 "failed to hook legacy IRQ %d\n",
88 efx->pci_dev->irq);
89 goto fail1;
90 }
91 return 0;
92 }
93
94 #ifdef CONFIG_RFS_ACCEL
95 if (efx->interrupt_mode == EF4_INT_MODE_MSIX) {
96 efx->net_dev->rx_cpu_rmap =
97 alloc_irq_cpu_rmap(efx->n_rx_channels);
98 if (!efx->net_dev->rx_cpu_rmap) {
99 rc = -ENOMEM;
100 goto fail1;
101 }
102 }
103 #endif
104
105 /* Hook MSI or MSI-X interrupt */
106 n_irqs = 0;
107 ef4_for_each_channel(channel, efx) {
108 rc = request_irq(channel->irq, efx->type->irq_handle_msi,
109 IRQF_PROBE_SHARED, /* Not shared */
110 efx->msi_context[channel->channel].name,
111 &efx->msi_context[channel->channel]);
112 if (rc) {
113 netif_err(efx, drv, efx->net_dev,
114 "failed to hook IRQ %d\n", channel->irq);
115 goto fail2;
116 }
117 ++n_irqs;
118
119 #ifdef CONFIG_RFS_ACCEL
120 if (efx->interrupt_mode == EF4_INT_MODE_MSIX &&
121 channel->channel < efx->n_rx_channels) {
122 rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,
123 channel->irq);
124 if (rc)
125 goto fail2;
126 }
127 #endif
128 }
129
130 return 0;
131
132 fail2:
133 #ifdef CONFIG_RFS_ACCEL
134 free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
135 efx->net_dev->rx_cpu_rmap = NULL;
136 #endif
137 ef4_for_each_channel(channel, efx) {
138 if (n_irqs-- == 0)
139 break;
140 free_irq(channel->irq, &efx->msi_context[channel->channel]);
141 }
142 fail1:
143 return rc;
144 }
145
ef4_nic_fini_interrupt(struct ef4_nic * efx)146 void ef4_nic_fini_interrupt(struct ef4_nic *efx)
147 {
148 struct ef4_channel *channel;
149
150 #ifdef CONFIG_RFS_ACCEL
151 free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
152 efx->net_dev->rx_cpu_rmap = NULL;
153 #endif
154
155 if (EF4_INT_MODE_USE_MSI(efx)) {
156 /* Disable MSI/MSI-X interrupts */
157 ef4_for_each_channel(channel, efx)
158 free_irq(channel->irq,
159 &efx->msi_context[channel->channel]);
160 } else {
161 /* Disable legacy interrupt */
162 free_irq(efx->legacy_irq, efx);
163 }
164 }
165
166 /* Register dump */
167
168 #define REGISTER_REVISION_FA 1
169 #define REGISTER_REVISION_FB 2
170 #define REGISTER_REVISION_FC 3
171 #define REGISTER_REVISION_FZ 3 /* last Falcon arch revision */
172 #define REGISTER_REVISION_ED 4
173 #define REGISTER_REVISION_EZ 4 /* latest EF10 revision */
174
175 struct ef4_nic_reg {
176 u32 offset:24;
177 u32 min_revision:3, max_revision:3;
178 };
179
180 #define REGISTER(name, arch, min_rev, max_rev) { \
181 arch ## R_ ## min_rev ## max_rev ## _ ## name, \
182 REGISTER_REVISION_ ## arch ## min_rev, \
183 REGISTER_REVISION_ ## arch ## max_rev \
184 }
185 #define REGISTER_AA(name) REGISTER(name, F, A, A)
186 #define REGISTER_AB(name) REGISTER(name, F, A, B)
187 #define REGISTER_AZ(name) REGISTER(name, F, A, Z)
188 #define REGISTER_BB(name) REGISTER(name, F, B, B)
189 #define REGISTER_BZ(name) REGISTER(name, F, B, Z)
190 #define REGISTER_CZ(name) REGISTER(name, F, C, Z)
191
192 static const struct ef4_nic_reg ef4_nic_regs[] = {
193 REGISTER_AZ(ADR_REGION),
194 REGISTER_AZ(INT_EN_KER),
195 REGISTER_BZ(INT_EN_CHAR),
196 REGISTER_AZ(INT_ADR_KER),
197 REGISTER_BZ(INT_ADR_CHAR),
198 /* INT_ACK_KER is WO */
199 /* INT_ISR0 is RC */
200 REGISTER_AZ(HW_INIT),
201 REGISTER_CZ(USR_EV_CFG),
202 REGISTER_AB(EE_SPI_HCMD),
203 REGISTER_AB(EE_SPI_HADR),
204 REGISTER_AB(EE_SPI_HDATA),
205 REGISTER_AB(EE_BASE_PAGE),
206 REGISTER_AB(EE_VPD_CFG0),
207 /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */
208 /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */
209 /* PCIE_CORE_INDIRECT is indirect */
210 REGISTER_AB(NIC_STAT),
211 REGISTER_AB(GPIO_CTL),
212 REGISTER_AB(GLB_CTL),
213 /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */
214 REGISTER_BZ(DP_CTRL),
215 REGISTER_AZ(MEM_STAT),
216 REGISTER_AZ(CS_DEBUG),
217 REGISTER_AZ(ALTERA_BUILD),
218 REGISTER_AZ(CSR_SPARE),
219 REGISTER_AB(PCIE_SD_CTL0123),
220 REGISTER_AB(PCIE_SD_CTL45),
221 REGISTER_AB(PCIE_PCS_CTL_STAT),
222 /* DEBUG_DATA_OUT is not used */
223 /* DRV_EV is WO */
224 REGISTER_AZ(EVQ_CTL),
225 REGISTER_AZ(EVQ_CNT1),
226 REGISTER_AZ(EVQ_CNT2),
227 REGISTER_AZ(BUF_TBL_CFG),
228 REGISTER_AZ(SRM_RX_DC_CFG),
229 REGISTER_AZ(SRM_TX_DC_CFG),
230 REGISTER_AZ(SRM_CFG),
231 /* BUF_TBL_UPD is WO */
232 REGISTER_AZ(SRM_UPD_EVQ),
233 REGISTER_AZ(SRAM_PARITY),
234 REGISTER_AZ(RX_CFG),
235 REGISTER_BZ(RX_FILTER_CTL),
236 /* RX_FLUSH_DESCQ is WO */
237 REGISTER_AZ(RX_DC_CFG),
238 REGISTER_AZ(RX_DC_PF_WM),
239 REGISTER_BZ(RX_RSS_TKEY),
240 /* RX_NODESC_DROP is RC */
241 REGISTER_AA(RX_SELF_RST),
242 /* RX_DEBUG, RX_PUSH_DROP are not used */
243 REGISTER_CZ(RX_RSS_IPV6_REG1),
244 REGISTER_CZ(RX_RSS_IPV6_REG2),
245 REGISTER_CZ(RX_RSS_IPV6_REG3),
246 /* TX_FLUSH_DESCQ is WO */
247 REGISTER_AZ(TX_DC_CFG),
248 REGISTER_AA(TX_CHKSM_CFG),
249 REGISTER_AZ(TX_CFG),
250 /* TX_PUSH_DROP is not used */
251 REGISTER_AZ(TX_RESERVED),
252 REGISTER_BZ(TX_PACE),
253 /* TX_PACE_DROP_QID is RC */
254 REGISTER_BB(TX_VLAN),
255 REGISTER_BZ(TX_IPFIL_PORTEN),
256 REGISTER_AB(MD_TXD),
257 REGISTER_AB(MD_RXD),
258 REGISTER_AB(MD_CS),
259 REGISTER_AB(MD_PHY_ADR),
260 REGISTER_AB(MD_ID),
261 /* MD_STAT is RC */
262 REGISTER_AB(MAC_STAT_DMA),
263 REGISTER_AB(MAC_CTRL),
264 REGISTER_BB(GEN_MODE),
265 REGISTER_AB(MAC_MC_HASH_REG0),
266 REGISTER_AB(MAC_MC_HASH_REG1),
267 REGISTER_AB(GM_CFG1),
268 REGISTER_AB(GM_CFG2),
269 /* GM_IPG and GM_HD are not used */
270 REGISTER_AB(GM_MAX_FLEN),
271 /* GM_TEST is not used */
272 REGISTER_AB(GM_ADR1),
273 REGISTER_AB(GM_ADR2),
274 REGISTER_AB(GMF_CFG0),
275 REGISTER_AB(GMF_CFG1),
276 REGISTER_AB(GMF_CFG2),
277 REGISTER_AB(GMF_CFG3),
278 REGISTER_AB(GMF_CFG4),
279 REGISTER_AB(GMF_CFG5),
280 REGISTER_BB(TX_SRC_MAC_CTL),
281 REGISTER_AB(XM_ADR_LO),
282 REGISTER_AB(XM_ADR_HI),
283 REGISTER_AB(XM_GLB_CFG),
284 REGISTER_AB(XM_TX_CFG),
285 REGISTER_AB(XM_RX_CFG),
286 REGISTER_AB(XM_MGT_INT_MASK),
287 REGISTER_AB(XM_FC),
288 REGISTER_AB(XM_PAUSE_TIME),
289 REGISTER_AB(XM_TX_PARAM),
290 REGISTER_AB(XM_RX_PARAM),
291 /* XM_MGT_INT_MSK (note no 'A') is RC */
292 REGISTER_AB(XX_PWR_RST),
293 REGISTER_AB(XX_SD_CTL),
294 REGISTER_AB(XX_TXDRV_CTL),
295 /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */
296 /* XX_CORE_STAT is partly RC */
297 };
298
299 struct ef4_nic_reg_table {
300 u32 offset:24;
301 u32 min_revision:3, max_revision:3;
302 u32 step:6, rows:21;
303 };
304
305 #define REGISTER_TABLE_DIMENSIONS(_, offset, arch, min_rev, max_rev, step, rows) { \
306 offset, \
307 REGISTER_REVISION_ ## arch ## min_rev, \
308 REGISTER_REVISION_ ## arch ## max_rev, \
309 step, rows \
310 }
311 #define REGISTER_TABLE(name, arch, min_rev, max_rev) \
312 REGISTER_TABLE_DIMENSIONS( \
313 name, arch ## R_ ## min_rev ## max_rev ## _ ## name, \
314 arch, min_rev, max_rev, \
315 arch ## R_ ## min_rev ## max_rev ## _ ## name ## _STEP, \
316 arch ## R_ ## min_rev ## max_rev ## _ ## name ## _ROWS)
317 #define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, F, A, A)
318 #define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, F, A, Z)
319 #define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, F, B, B)
320 #define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, F, B, Z)
321 #define REGISTER_TABLE_BB_CZ(name) \
322 REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, B, B, \
323 FR_BZ_ ## name ## _STEP, \
324 FR_BB_ ## name ## _ROWS), \
325 REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, C, Z, \
326 FR_BZ_ ## name ## _STEP, \
327 FR_CZ_ ## name ## _ROWS)
328 #define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, F, C, Z)
329
330 static const struct ef4_nic_reg_table ef4_nic_reg_tables[] = {
331 /* DRIVER is not used */
332 /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */
333 REGISTER_TABLE_BB(TX_IPFIL_TBL),
334 REGISTER_TABLE_BB(TX_SRC_MAC_TBL),
335 REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),
336 REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),
337 REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),
338 REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),
339 REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),
340 REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),
341 /* We can't reasonably read all of the buffer table (up to 8MB!).
342 * However this driver will only use a few entries. Reading
343 * 1K entries allows for some expansion of queue count and
344 * size before we need to change the version. */
345 REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,
346 F, A, A, 8, 1024),
347 REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,
348 F, B, Z, 8, 1024),
349 REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),
350 REGISTER_TABLE_BB_CZ(TIMER_TBL),
351 REGISTER_TABLE_BB_CZ(TX_PACE_TBL),
352 REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),
353 /* TX_FILTER_TBL0 is huge and not used by this driver */
354 REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),
355 REGISTER_TABLE_CZ(MC_TREG_SMEM),
356 /* MSIX_PBA_TABLE is not mapped */
357 /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */
358 REGISTER_TABLE_BZ(RX_FILTER_TBL0),
359 };
360
ef4_nic_get_regs_len(struct ef4_nic * efx)361 size_t ef4_nic_get_regs_len(struct ef4_nic *efx)
362 {
363 const struct ef4_nic_reg *reg;
364 const struct ef4_nic_reg_table *table;
365 size_t len = 0;
366
367 for (reg = ef4_nic_regs;
368 reg < ef4_nic_regs + ARRAY_SIZE(ef4_nic_regs);
369 reg++)
370 if (efx->type->revision >= reg->min_revision &&
371 efx->type->revision <= reg->max_revision)
372 len += sizeof(ef4_oword_t);
373
374 for (table = ef4_nic_reg_tables;
375 table < ef4_nic_reg_tables + ARRAY_SIZE(ef4_nic_reg_tables);
376 table++)
377 if (efx->type->revision >= table->min_revision &&
378 efx->type->revision <= table->max_revision)
379 len += table->rows * min_t(size_t, table->step, 16);
380
381 return len;
382 }
383
ef4_nic_get_regs(struct ef4_nic * efx,void * buf)384 void ef4_nic_get_regs(struct ef4_nic *efx, void *buf)
385 {
386 const struct ef4_nic_reg *reg;
387 const struct ef4_nic_reg_table *table;
388
389 for (reg = ef4_nic_regs;
390 reg < ef4_nic_regs + ARRAY_SIZE(ef4_nic_regs);
391 reg++) {
392 if (efx->type->revision >= reg->min_revision &&
393 efx->type->revision <= reg->max_revision) {
394 ef4_reado(efx, (ef4_oword_t *)buf, reg->offset);
395 buf += sizeof(ef4_oword_t);
396 }
397 }
398
399 for (table = ef4_nic_reg_tables;
400 table < ef4_nic_reg_tables + ARRAY_SIZE(ef4_nic_reg_tables);
401 table++) {
402 size_t size, i;
403
404 if (!(efx->type->revision >= table->min_revision &&
405 efx->type->revision <= table->max_revision))
406 continue;
407
408 size = min_t(size_t, table->step, 16);
409
410 for (i = 0; i < table->rows; i++) {
411 switch (table->step) {
412 case 4: /* 32-bit SRAM */
413 ef4_readd(efx, buf, table->offset + 4 * i);
414 break;
415 case 8: /* 64-bit SRAM */
416 ef4_sram_readq(efx,
417 efx->membase + table->offset,
418 buf, i);
419 break;
420 case 16: /* 128-bit-readable register */
421 ef4_reado_table(efx, buf, table->offset, i);
422 break;
423 case 32: /* 128-bit register, interleaved */
424 ef4_reado_table(efx, buf, table->offset, 2 * i);
425 break;
426 default:
427 WARN_ON(1);
428 return;
429 }
430 buf += size;
431 }
432 }
433 }
434
435 /**
436 * ef4_nic_describe_stats - Describe supported statistics for ethtool
437 * @desc: Array of &struct ef4_hw_stat_desc describing the statistics
438 * @count: Length of the @desc array
439 * @mask: Bitmask of which elements of @desc are enabled
440 * @names: Buffer to copy names to, or %NULL. The names are copied
441 * starting at intervals of %ETH_GSTRING_LEN bytes.
442 *
443 * Returns the number of visible statistics, i.e. the number of set
444 * bits in the first @count bits of @mask for which a name is defined.
445 */
ef4_nic_describe_stats(const struct ef4_hw_stat_desc * desc,size_t count,const unsigned long * mask,u8 * names)446 size_t ef4_nic_describe_stats(const struct ef4_hw_stat_desc *desc, size_t count,
447 const unsigned long *mask, u8 *names)
448 {
449 size_t visible = 0;
450 size_t index;
451
452 for_each_set_bit(index, mask, count) {
453 if (desc[index].name) {
454 if (names) {
455 strscpy(names, desc[index].name,
456 ETH_GSTRING_LEN);
457 names += ETH_GSTRING_LEN;
458 }
459 ++visible;
460 }
461 }
462
463 return visible;
464 }
465
466 /**
467 * ef4_nic_update_stats - Convert statistics DMA buffer to array of u64
468 * @desc: Array of &struct ef4_hw_stat_desc describing the DMA buffer
469 * layout. DMA widths of 0, 16, 32 and 64 are supported; where
470 * the width is specified as 0 the corresponding element of
471 * @stats is not updated.
472 * @count: Length of the @desc array
473 * @mask: Bitmask of which elements of @desc are enabled
474 * @stats: Buffer to update with the converted statistics. The length
475 * of this array must be at least @count.
476 * @dma_buf: DMA buffer containing hardware statistics
477 * @accumulate: If set, the converted values will be added rather than
478 * directly stored to the corresponding elements of @stats
479 */
ef4_nic_update_stats(const struct ef4_hw_stat_desc * desc,size_t count,const unsigned long * mask,u64 * stats,const void * dma_buf,bool accumulate)480 void ef4_nic_update_stats(const struct ef4_hw_stat_desc *desc, size_t count,
481 const unsigned long *mask,
482 u64 *stats, const void *dma_buf, bool accumulate)
483 {
484 size_t index;
485
486 for_each_set_bit(index, mask, count) {
487 if (desc[index].dma_width) {
488 const void *addr = dma_buf + desc[index].offset;
489 u64 val;
490
491 switch (desc[index].dma_width) {
492 case 16:
493 val = le16_to_cpup((__le16 *)addr);
494 break;
495 case 32:
496 val = le32_to_cpup((__le32 *)addr);
497 break;
498 case 64:
499 val = le64_to_cpup((__le64 *)addr);
500 break;
501 default:
502 WARN_ON(1);
503 val = 0;
504 break;
505 }
506
507 if (accumulate)
508 stats[index] += val;
509 else
510 stats[index] = val;
511 }
512 }
513 }
514
ef4_nic_fix_nodesc_drop_stat(struct ef4_nic * efx,u64 * rx_nodesc_drops)515 void ef4_nic_fix_nodesc_drop_stat(struct ef4_nic *efx, u64 *rx_nodesc_drops)
516 {
517 /* if down, or this is the first update after coming up */
518 if (!(efx->net_dev->flags & IFF_UP) || !efx->rx_nodesc_drops_prev_state)
519 efx->rx_nodesc_drops_while_down +=
520 *rx_nodesc_drops - efx->rx_nodesc_drops_total;
521 efx->rx_nodesc_drops_total = *rx_nodesc_drops;
522 efx->rx_nodesc_drops_prev_state = !!(efx->net_dev->flags & IFF_UP);
523 *rx_nodesc_drops -= efx->rx_nodesc_drops_while_down;
524 }
525