Loading...
1/****************************************************************************
2 * Driver for Solarflare network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2006-2013 Solarflare Communications Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/bitops.h>
12#include <linux/delay.h>
13#include <linux/interrupt.h>
14#include <linux/pci.h>
15#include <linux/module.h>
16#include <linux/seq_file.h>
17#include <linux/cpu_rmap.h>
18#include "net_driver.h"
19#include "bitfield.h"
20#include "efx.h"
21#include "nic.h"
22#include "ef10_regs.h"
23#include "farch_regs.h"
24#include "io.h"
25#include "workarounds.h"
26
27/**************************************************************************
28 *
29 * Generic buffer handling
30 * These buffers are used for interrupt status, MAC stats, etc.
31 *
32 **************************************************************************/
33
34int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer,
35 unsigned int len, gfp_t gfp_flags)
36{
37 buffer->addr = dma_zalloc_coherent(&efx->pci_dev->dev, len,
38 &buffer->dma_addr, gfp_flags);
39 if (!buffer->addr)
40 return -ENOMEM;
41 buffer->len = len;
42 return 0;
43}
44
45void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
46{
47 if (buffer->addr) {
48 dma_free_coherent(&efx->pci_dev->dev, buffer->len,
49 buffer->addr, buffer->dma_addr);
50 buffer->addr = NULL;
51 }
52}
53
54/* Check whether an event is present in the eventq at the current
55 * read pointer. Only useful for self-test.
56 */
57bool efx_nic_event_present(struct efx_channel *channel)
58{
59 return efx_event_present(efx_event(channel, channel->eventq_read_ptr));
60}
61
62void efx_nic_event_test_start(struct efx_channel *channel)
63{
64 channel->event_test_cpu = -1;
65 smp_wmb();
66 channel->efx->type->ev_test_generate(channel);
67}
68
69int efx_nic_irq_test_start(struct efx_nic *efx)
70{
71 efx->last_irq_cpu = -1;
72 smp_wmb();
73 return efx->type->irq_test_generate(efx);
74}
75
76/* Hook interrupt handler(s)
77 * Try MSI and then legacy interrupts.
78 */
79int efx_nic_init_interrupt(struct efx_nic *efx)
80{
81 struct efx_channel *channel;
82 unsigned int n_irqs;
83 int rc;
84
85 if (!EFX_INT_MODE_USE_MSI(efx)) {
86 rc = request_irq(efx->legacy_irq,
87 efx->type->irq_handle_legacy, IRQF_SHARED,
88 efx->name, efx);
89 if (rc) {
90 netif_err(efx, drv, efx->net_dev,
91 "failed to hook legacy IRQ %d\n",
92 efx->pci_dev->irq);
93 goto fail1;
94 }
95 return 0;
96 }
97
98#ifdef CONFIG_RFS_ACCEL
99 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
100 efx->net_dev->rx_cpu_rmap =
101 alloc_irq_cpu_rmap(efx->n_rx_channels);
102 if (!efx->net_dev->rx_cpu_rmap) {
103 rc = -ENOMEM;
104 goto fail1;
105 }
106 }
107#endif
108
109 /* Hook MSI or MSI-X interrupt */
110 n_irqs = 0;
111 efx_for_each_channel(channel, efx) {
112 rc = request_irq(channel->irq, efx->type->irq_handle_msi,
113 IRQF_PROBE_SHARED, /* Not shared */
114 efx->msi_context[channel->channel].name,
115 &efx->msi_context[channel->channel]);
116 if (rc) {
117 netif_err(efx, drv, efx->net_dev,
118 "failed to hook IRQ %d\n", channel->irq);
119 goto fail2;
120 }
121 ++n_irqs;
122
123#ifdef CONFIG_RFS_ACCEL
124 if (efx->interrupt_mode == EFX_INT_MODE_MSIX &&
125 channel->channel < efx->n_rx_channels) {
126 rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,
127 channel->irq);
128 if (rc)
129 goto fail2;
130 }
131#endif
132 }
133
134 return 0;
135
136 fail2:
137#ifdef CONFIG_RFS_ACCEL
138 free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
139 efx->net_dev->rx_cpu_rmap = NULL;
140#endif
141 efx_for_each_channel(channel, efx) {
142 if (n_irqs-- == 0)
143 break;
144 free_irq(channel->irq, &efx->msi_context[channel->channel]);
145 }
146 fail1:
147 return rc;
148}
149
150void efx_nic_fini_interrupt(struct efx_nic *efx)
151{
152 struct efx_channel *channel;
153
154#ifdef CONFIG_RFS_ACCEL
155 free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
156 efx->net_dev->rx_cpu_rmap = NULL;
157#endif
158
159 if (EFX_INT_MODE_USE_MSI(efx)) {
160 /* Disable MSI/MSI-X interrupts */
161 efx_for_each_channel(channel, efx)
162 free_irq(channel->irq,
163 &efx->msi_context[channel->channel]);
164 } else {
165 /* Disable legacy interrupt */
166 free_irq(efx->legacy_irq, efx);
167 }
168}
169
170/* Register dump */
171
172#define REGISTER_REVISION_FA 1
173#define REGISTER_REVISION_FB 2
174#define REGISTER_REVISION_FC 3
175#define REGISTER_REVISION_FZ 3 /* last Falcon arch revision */
176#define REGISTER_REVISION_ED 4
177#define REGISTER_REVISION_EZ 4 /* latest EF10 revision */
178
179struct efx_nic_reg {
180 u32 offset:24;
181 u32 min_revision:3, max_revision:3;
182};
183
184#define REGISTER(name, arch, min_rev, max_rev) { \
185 arch ## R_ ## min_rev ## max_rev ## _ ## name, \
186 REGISTER_REVISION_ ## arch ## min_rev, \
187 REGISTER_REVISION_ ## arch ## max_rev \
188}
189#define REGISTER_AA(name) REGISTER(name, F, A, A)
190#define REGISTER_AB(name) REGISTER(name, F, A, B)
191#define REGISTER_AZ(name) REGISTER(name, F, A, Z)
192#define REGISTER_BB(name) REGISTER(name, F, B, B)
193#define REGISTER_BZ(name) REGISTER(name, F, B, Z)
194#define REGISTER_CZ(name) REGISTER(name, F, C, Z)
195#define REGISTER_DZ(name) REGISTER(name, E, D, Z)
196
197static const struct efx_nic_reg efx_nic_regs[] = {
198 REGISTER_AZ(ADR_REGION),
199 REGISTER_AZ(INT_EN_KER),
200 REGISTER_BZ(INT_EN_CHAR),
201 REGISTER_AZ(INT_ADR_KER),
202 REGISTER_BZ(INT_ADR_CHAR),
203 /* INT_ACK_KER is WO */
204 /* INT_ISR0 is RC */
205 REGISTER_AZ(HW_INIT),
206 REGISTER_CZ(USR_EV_CFG),
207 REGISTER_AB(EE_SPI_HCMD),
208 REGISTER_AB(EE_SPI_HADR),
209 REGISTER_AB(EE_SPI_HDATA),
210 REGISTER_AB(EE_BASE_PAGE),
211 REGISTER_AB(EE_VPD_CFG0),
212 /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */
213 /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */
214 /* PCIE_CORE_INDIRECT is indirect */
215 REGISTER_AB(NIC_STAT),
216 REGISTER_AB(GPIO_CTL),
217 REGISTER_AB(GLB_CTL),
218 /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */
219 REGISTER_BZ(DP_CTRL),
220 REGISTER_AZ(MEM_STAT),
221 REGISTER_AZ(CS_DEBUG),
222 REGISTER_AZ(ALTERA_BUILD),
223 REGISTER_AZ(CSR_SPARE),
224 REGISTER_AB(PCIE_SD_CTL0123),
225 REGISTER_AB(PCIE_SD_CTL45),
226 REGISTER_AB(PCIE_PCS_CTL_STAT),
227 /* DEBUG_DATA_OUT is not used */
228 /* DRV_EV is WO */
229 REGISTER_AZ(EVQ_CTL),
230 REGISTER_AZ(EVQ_CNT1),
231 REGISTER_AZ(EVQ_CNT2),
232 REGISTER_AZ(BUF_TBL_CFG),
233 REGISTER_AZ(SRM_RX_DC_CFG),
234 REGISTER_AZ(SRM_TX_DC_CFG),
235 REGISTER_AZ(SRM_CFG),
236 /* BUF_TBL_UPD is WO */
237 REGISTER_AZ(SRM_UPD_EVQ),
238 REGISTER_AZ(SRAM_PARITY),
239 REGISTER_AZ(RX_CFG),
240 REGISTER_BZ(RX_FILTER_CTL),
241 /* RX_FLUSH_DESCQ is WO */
242 REGISTER_AZ(RX_DC_CFG),
243 REGISTER_AZ(RX_DC_PF_WM),
244 REGISTER_BZ(RX_RSS_TKEY),
245 /* RX_NODESC_DROP is RC */
246 REGISTER_AA(RX_SELF_RST),
247 /* RX_DEBUG, RX_PUSH_DROP are not used */
248 REGISTER_CZ(RX_RSS_IPV6_REG1),
249 REGISTER_CZ(RX_RSS_IPV6_REG2),
250 REGISTER_CZ(RX_RSS_IPV6_REG3),
251 /* TX_FLUSH_DESCQ is WO */
252 REGISTER_AZ(TX_DC_CFG),
253 REGISTER_AA(TX_CHKSM_CFG),
254 REGISTER_AZ(TX_CFG),
255 /* TX_PUSH_DROP is not used */
256 REGISTER_AZ(TX_RESERVED),
257 REGISTER_BZ(TX_PACE),
258 /* TX_PACE_DROP_QID is RC */
259 REGISTER_BB(TX_VLAN),
260 REGISTER_BZ(TX_IPFIL_PORTEN),
261 REGISTER_AB(MD_TXD),
262 REGISTER_AB(MD_RXD),
263 REGISTER_AB(MD_CS),
264 REGISTER_AB(MD_PHY_ADR),
265 REGISTER_AB(MD_ID),
266 /* MD_STAT is RC */
267 REGISTER_AB(MAC_STAT_DMA),
268 REGISTER_AB(MAC_CTRL),
269 REGISTER_BB(GEN_MODE),
270 REGISTER_AB(MAC_MC_HASH_REG0),
271 REGISTER_AB(MAC_MC_HASH_REG1),
272 REGISTER_AB(GM_CFG1),
273 REGISTER_AB(GM_CFG2),
274 /* GM_IPG and GM_HD are not used */
275 REGISTER_AB(GM_MAX_FLEN),
276 /* GM_TEST is not used */
277 REGISTER_AB(GM_ADR1),
278 REGISTER_AB(GM_ADR2),
279 REGISTER_AB(GMF_CFG0),
280 REGISTER_AB(GMF_CFG1),
281 REGISTER_AB(GMF_CFG2),
282 REGISTER_AB(GMF_CFG3),
283 REGISTER_AB(GMF_CFG4),
284 REGISTER_AB(GMF_CFG5),
285 REGISTER_BB(TX_SRC_MAC_CTL),
286 REGISTER_AB(XM_ADR_LO),
287 REGISTER_AB(XM_ADR_HI),
288 REGISTER_AB(XM_GLB_CFG),
289 REGISTER_AB(XM_TX_CFG),
290 REGISTER_AB(XM_RX_CFG),
291 REGISTER_AB(XM_MGT_INT_MASK),
292 REGISTER_AB(XM_FC),
293 REGISTER_AB(XM_PAUSE_TIME),
294 REGISTER_AB(XM_TX_PARAM),
295 REGISTER_AB(XM_RX_PARAM),
296 /* XM_MGT_INT_MSK (note no 'A') is RC */
297 REGISTER_AB(XX_PWR_RST),
298 REGISTER_AB(XX_SD_CTL),
299 REGISTER_AB(XX_TXDRV_CTL),
300 /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */
301 /* XX_CORE_STAT is partly RC */
302 REGISTER_DZ(BIU_HW_REV_ID),
303 REGISTER_DZ(MC_DB_LWRD),
304 REGISTER_DZ(MC_DB_HWRD),
305};
306
307struct efx_nic_reg_table {
308 u32 offset:24;
309 u32 min_revision:3, max_revision:3;
310 u32 step:6, rows:21;
311};
312
313#define REGISTER_TABLE_DIMENSIONS(_, offset, arch, min_rev, max_rev, step, rows) { \
314 offset, \
315 REGISTER_REVISION_ ## arch ## min_rev, \
316 REGISTER_REVISION_ ## arch ## max_rev, \
317 step, rows \
318}
319#define REGISTER_TABLE(name, arch, min_rev, max_rev) \
320 REGISTER_TABLE_DIMENSIONS( \
321 name, arch ## R_ ## min_rev ## max_rev ## _ ## name, \
322 arch, min_rev, max_rev, \
323 arch ## R_ ## min_rev ## max_rev ## _ ## name ## _STEP, \
324 arch ## R_ ## min_rev ## max_rev ## _ ## name ## _ROWS)
325#define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, F, A, A)
326#define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, F, A, Z)
327#define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, F, B, B)
328#define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, F, B, Z)
329#define REGISTER_TABLE_BB_CZ(name) \
330 REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, B, B, \
331 FR_BZ_ ## name ## _STEP, \
332 FR_BB_ ## name ## _ROWS), \
333 REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, C, Z, \
334 FR_BZ_ ## name ## _STEP, \
335 FR_CZ_ ## name ## _ROWS)
336#define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, F, C, Z)
337#define REGISTER_TABLE_DZ(name) REGISTER_TABLE(name, E, D, Z)
338
339static const struct efx_nic_reg_table efx_nic_reg_tables[] = {
340 /* DRIVER is not used */
341 /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */
342 REGISTER_TABLE_BB(TX_IPFIL_TBL),
343 REGISTER_TABLE_BB(TX_SRC_MAC_TBL),
344 REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),
345 REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),
346 REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),
347 REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),
348 REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),
349 REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),
350 /* We can't reasonably read all of the buffer table (up to 8MB!).
351 * However this driver will only use a few entries. Reading
352 * 1K entries allows for some expansion of queue count and
353 * size before we need to change the version. */
354 REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,
355 F, A, A, 8, 1024),
356 REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,
357 F, B, Z, 8, 1024),
358 REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),
359 REGISTER_TABLE_BB_CZ(TIMER_TBL),
360 REGISTER_TABLE_BB_CZ(TX_PACE_TBL),
361 REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),
362 /* TX_FILTER_TBL0 is huge and not used by this driver */
363 REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),
364 REGISTER_TABLE_CZ(MC_TREG_SMEM),
365 /* MSIX_PBA_TABLE is not mapped */
366 /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */
367 REGISTER_TABLE_BZ(RX_FILTER_TBL0),
368 REGISTER_TABLE_DZ(BIU_MC_SFT_STATUS),
369};
370
371size_t efx_nic_get_regs_len(struct efx_nic *efx)
372{
373 const struct efx_nic_reg *reg;
374 const struct efx_nic_reg_table *table;
375 size_t len = 0;
376
377 for (reg = efx_nic_regs;
378 reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
379 reg++)
380 if (efx->type->revision >= reg->min_revision &&
381 efx->type->revision <= reg->max_revision)
382 len += sizeof(efx_oword_t);
383
384 for (table = efx_nic_reg_tables;
385 table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
386 table++)
387 if (efx->type->revision >= table->min_revision &&
388 efx->type->revision <= table->max_revision)
389 len += table->rows * min_t(size_t, table->step, 16);
390
391 return len;
392}
393
394void efx_nic_get_regs(struct efx_nic *efx, void *buf)
395{
396 const struct efx_nic_reg *reg;
397 const struct efx_nic_reg_table *table;
398
399 for (reg = efx_nic_regs;
400 reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
401 reg++) {
402 if (efx->type->revision >= reg->min_revision &&
403 efx->type->revision <= reg->max_revision) {
404 efx_reado(efx, (efx_oword_t *)buf, reg->offset);
405 buf += sizeof(efx_oword_t);
406 }
407 }
408
409 for (table = efx_nic_reg_tables;
410 table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
411 table++) {
412 size_t size, i;
413
414 if (!(efx->type->revision >= table->min_revision &&
415 efx->type->revision <= table->max_revision))
416 continue;
417
418 size = min_t(size_t, table->step, 16);
419
420 for (i = 0; i < table->rows; i++) {
421 switch (table->step) {
422 case 4: /* 32-bit SRAM */
423 efx_readd(efx, buf, table->offset + 4 * i);
424 break;
425 case 8: /* 64-bit SRAM */
426 efx_sram_readq(efx,
427 efx->membase + table->offset,
428 buf, i);
429 break;
430 case 16: /* 128-bit-readable register */
431 efx_reado_table(efx, buf, table->offset, i);
432 break;
433 case 32: /* 128-bit register, interleaved */
434 efx_reado_table(efx, buf, table->offset, 2 * i);
435 break;
436 default:
437 WARN_ON(1);
438 return;
439 }
440 buf += size;
441 }
442 }
443}
444
445/**
446 * efx_nic_describe_stats - Describe supported statistics for ethtool
447 * @desc: Array of &struct efx_hw_stat_desc describing the statistics
448 * @count: Length of the @desc array
449 * @mask: Bitmask of which elements of @desc are enabled
450 * @names: Buffer to copy names to, or %NULL. The names are copied
451 * starting at intervals of %ETH_GSTRING_LEN bytes.
452 *
453 * Returns the number of visible statistics, i.e. the number of set
454 * bits in the first @count bits of @mask for which a name is defined.
455 */
456size_t efx_nic_describe_stats(const struct efx_hw_stat_desc *desc, size_t count,
457 const unsigned long *mask, u8 *names)
458{
459 size_t visible = 0;
460 size_t index;
461
462 for_each_set_bit(index, mask, count) {
463 if (desc[index].name) {
464 if (names) {
465 strlcpy(names, desc[index].name,
466 ETH_GSTRING_LEN);
467 names += ETH_GSTRING_LEN;
468 }
469 ++visible;
470 }
471 }
472
473 return visible;
474}
475
476/**
477 * efx_nic_update_stats - Convert statistics DMA buffer to array of u64
478 * @desc: Array of &struct efx_hw_stat_desc describing the DMA buffer
479 * layout. DMA widths of 0, 16, 32 and 64 are supported; where
480 * the width is specified as 0 the corresponding element of
481 * @stats is not updated.
482 * @count: Length of the @desc array
483 * @mask: Bitmask of which elements of @desc are enabled
484 * @stats: Buffer to update with the converted statistics. The length
485 * of this array must be at least @count.
486 * @dma_buf: DMA buffer containing hardware statistics
487 * @accumulate: If set, the converted values will be added rather than
488 * directly stored to the corresponding elements of @stats
489 */
490void efx_nic_update_stats(const struct efx_hw_stat_desc *desc, size_t count,
491 const unsigned long *mask,
492 u64 *stats, const void *dma_buf, bool accumulate)
493{
494 size_t index;
495
496 for_each_set_bit(index, mask, count) {
497 if (desc[index].dma_width) {
498 const void *addr = dma_buf + desc[index].offset;
499 u64 val;
500
501 switch (desc[index].dma_width) {
502 case 16:
503 val = le16_to_cpup((__le16 *)addr);
504 break;
505 case 32:
506 val = le32_to_cpup((__le32 *)addr);
507 break;
508 case 64:
509 val = le64_to_cpup((__le64 *)addr);
510 break;
511 default:
512 WARN_ON(1);
513 val = 0;
514 break;
515 }
516
517 if (accumulate)
518 stats[index] += val;
519 else
520 stats[index] = val;
521 }
522 }
523}
524
525void efx_nic_fix_nodesc_drop_stat(struct efx_nic *efx, u64 *rx_nodesc_drops)
526{
527 /* if down, or this is the first update after coming up */
528 if (!(efx->net_dev->flags & IFF_UP) || !efx->rx_nodesc_drops_prev_state)
529 efx->rx_nodesc_drops_while_down +=
530 *rx_nodesc_drops - efx->rx_nodesc_drops_total;
531 efx->rx_nodesc_drops_total = *rx_nodesc_drops;
532 efx->rx_nodesc_drops_prev_state = !!(efx->net_dev->flags & IFF_UP);
533 *rx_nodesc_drops -= efx->rx_nodesc_drops_while_down;
534}
1/****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2006-2011 Solarflare Communications Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/bitops.h>
12#include <linux/delay.h>
13#include <linux/interrupt.h>
14#include <linux/pci.h>
15#include <linux/module.h>
16#include <linux/seq_file.h>
17#include "net_driver.h"
18#include "bitfield.h"
19#include "efx.h"
20#include "nic.h"
21#include "regs.h"
22#include "io.h"
23#include "workarounds.h"
24
25/**************************************************************************
26 *
27 * Configurable values
28 *
29 **************************************************************************
30 */
31
32/* This is set to 16 for a good reason. In summary, if larger than
33 * 16, the descriptor cache holds more than a default socket
34 * buffer's worth of packets (for UDP we can only have at most one
35 * socket buffer's worth outstanding). This combined with the fact
36 * that we only get 1 TX event per descriptor cache means the NIC
37 * goes idle.
38 */
39#define TX_DC_ENTRIES 16
40#define TX_DC_ENTRIES_ORDER 1
41
42#define RX_DC_ENTRIES 64
43#define RX_DC_ENTRIES_ORDER 3
44
45/* If EFX_MAX_INT_ERRORS internal errors occur within
46 * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
47 * disable it.
48 */
49#define EFX_INT_ERROR_EXPIRE 3600
50#define EFX_MAX_INT_ERRORS 5
51
52/* Depth of RX flush request fifo */
53#define EFX_RX_FLUSH_COUNT 4
54
55/* Driver generated events */
56#define _EFX_CHANNEL_MAGIC_TEST 0x000101
57#define _EFX_CHANNEL_MAGIC_FILL 0x000102
58#define _EFX_CHANNEL_MAGIC_RX_DRAIN 0x000103
59#define _EFX_CHANNEL_MAGIC_TX_DRAIN 0x000104
60
61#define _EFX_CHANNEL_MAGIC(_code, _data) ((_code) << 8 | (_data))
62#define _EFX_CHANNEL_MAGIC_CODE(_magic) ((_magic) >> 8)
63
64#define EFX_CHANNEL_MAGIC_TEST(_channel) \
65 _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TEST, (_channel)->channel)
66#define EFX_CHANNEL_MAGIC_FILL(_rx_queue) \
67 _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_FILL, \
68 efx_rx_queue_index(_rx_queue))
69#define EFX_CHANNEL_MAGIC_RX_DRAIN(_rx_queue) \
70 _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_RX_DRAIN, \
71 efx_rx_queue_index(_rx_queue))
72#define EFX_CHANNEL_MAGIC_TX_DRAIN(_tx_queue) \
73 _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TX_DRAIN, \
74 (_tx_queue)->queue)
75
76/**************************************************************************
77 *
78 * Solarstorm hardware access
79 *
80 **************************************************************************/
81
82static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
83 unsigned int index)
84{
85 efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
86 value, index);
87}
88
89/* Read the current event from the event queue */
90static inline efx_qword_t *efx_event(struct efx_channel *channel,
91 unsigned int index)
92{
93 return ((efx_qword_t *) (channel->eventq.addr)) +
94 (index & channel->eventq_mask);
95}
96
97/* See if an event is present
98 *
99 * We check both the high and low dword of the event for all ones. We
100 * wrote all ones when we cleared the event, and no valid event can
101 * have all ones in either its high or low dwords. This approach is
102 * robust against reordering.
103 *
104 * Note that using a single 64-bit comparison is incorrect; even
105 * though the CPU read will be atomic, the DMA write may not be.
106 */
107static inline int efx_event_present(efx_qword_t *event)
108{
109 return !(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
110 EFX_DWORD_IS_ALL_ONES(event->dword[1]));
111}
112
113static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b,
114 const efx_oword_t *mask)
115{
116 return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) ||
117 ((a->u64[1] ^ b->u64[1]) & mask->u64[1]);
118}
119
120int efx_nic_test_registers(struct efx_nic *efx,
121 const struct efx_nic_register_test *regs,
122 size_t n_regs)
123{
124 unsigned address = 0, i, j;
125 efx_oword_t mask, imask, original, reg, buf;
126
127 /* Falcon should be in loopback to isolate the XMAC from the PHY */
128 WARN_ON(!LOOPBACK_INTERNAL(efx));
129
130 for (i = 0; i < n_regs; ++i) {
131 address = regs[i].address;
132 mask = imask = regs[i].mask;
133 EFX_INVERT_OWORD(imask);
134
135 efx_reado(efx, &original, address);
136
137 /* bit sweep on and off */
138 for (j = 0; j < 128; j++) {
139 if (!EFX_EXTRACT_OWORD32(mask, j, j))
140 continue;
141
142 /* Test this testable bit can be set in isolation */
143 EFX_AND_OWORD(reg, original, mask);
144 EFX_SET_OWORD32(reg, j, j, 1);
145
146 efx_writeo(efx, ®, address);
147 efx_reado(efx, &buf, address);
148
149 if (efx_masked_compare_oword(®, &buf, &mask))
150 goto fail;
151
152 /* Test this testable bit can be cleared in isolation */
153 EFX_OR_OWORD(reg, original, mask);
154 EFX_SET_OWORD32(reg, j, j, 0);
155
156 efx_writeo(efx, ®, address);
157 efx_reado(efx, &buf, address);
158
159 if (efx_masked_compare_oword(®, &buf, &mask))
160 goto fail;
161 }
162
163 efx_writeo(efx, &original, address);
164 }
165
166 return 0;
167
168fail:
169 netif_err(efx, hw, efx->net_dev,
170 "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT
171 " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg),
172 EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask));
173 return -EIO;
174}
175
176/**************************************************************************
177 *
178 * Special buffer handling
179 * Special buffers are used for event queues and the TX and RX
180 * descriptor rings.
181 *
182 *************************************************************************/
183
184/*
185 * Initialise a special buffer
186 *
187 * This will define a buffer (previously allocated via
188 * efx_alloc_special_buffer()) in the buffer table, allowing
189 * it to be used for event queues, descriptor rings etc.
190 */
191static void
192efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
193{
194 efx_qword_t buf_desc;
195 unsigned int index;
196 dma_addr_t dma_addr;
197 int i;
198
199 EFX_BUG_ON_PARANOID(!buffer->addr);
200
201 /* Write buffer descriptors to NIC */
202 for (i = 0; i < buffer->entries; i++) {
203 index = buffer->index + i;
204 dma_addr = buffer->dma_addr + (i * EFX_BUF_SIZE);
205 netif_dbg(efx, probe, efx->net_dev,
206 "mapping special buffer %d at %llx\n",
207 index, (unsigned long long)dma_addr);
208 EFX_POPULATE_QWORD_3(buf_desc,
209 FRF_AZ_BUF_ADR_REGION, 0,
210 FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
211 FRF_AZ_BUF_OWNER_ID_FBUF, 0);
212 efx_write_buf_tbl(efx, &buf_desc, index);
213 }
214}
215
216/* Unmaps a buffer and clears the buffer table entries */
217static void
218efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
219{
220 efx_oword_t buf_tbl_upd;
221 unsigned int start = buffer->index;
222 unsigned int end = (buffer->index + buffer->entries - 1);
223
224 if (!buffer->entries)
225 return;
226
227 netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n",
228 buffer->index, buffer->index + buffer->entries - 1);
229
230 EFX_POPULATE_OWORD_4(buf_tbl_upd,
231 FRF_AZ_BUF_UPD_CMD, 0,
232 FRF_AZ_BUF_CLR_CMD, 1,
233 FRF_AZ_BUF_CLR_END_ID, end,
234 FRF_AZ_BUF_CLR_START_ID, start);
235 efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
236}
237
238/*
239 * Allocate a new special buffer
240 *
241 * This allocates memory for a new buffer, clears it and allocates a
242 * new buffer ID range. It does not write into the buffer table.
243 *
244 * This call will allocate 4KB buffers, since 8KB buffers can't be
245 * used for event queues and descriptor rings.
246 */
247static int efx_alloc_special_buffer(struct efx_nic *efx,
248 struct efx_special_buffer *buffer,
249 unsigned int len)
250{
251 len = ALIGN(len, EFX_BUF_SIZE);
252
253 buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,
254 &buffer->dma_addr, GFP_KERNEL);
255 if (!buffer->addr)
256 return -ENOMEM;
257 buffer->len = len;
258 buffer->entries = len / EFX_BUF_SIZE;
259 BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1));
260
261 /* All zeros is a potentially valid event so memset to 0xff */
262 memset(buffer->addr, 0xff, len);
263
264 /* Select new buffer ID */
265 buffer->index = efx->next_buffer_table;
266 efx->next_buffer_table += buffer->entries;
267#ifdef CONFIG_SFC_SRIOV
268 BUG_ON(efx_sriov_enabled(efx) &&
269 efx->vf_buftbl_base < efx->next_buffer_table);
270#endif
271
272 netif_dbg(efx, probe, efx->net_dev,
273 "allocating special buffers %d-%d at %llx+%x "
274 "(virt %p phys %llx)\n", buffer->index,
275 buffer->index + buffer->entries - 1,
276 (u64)buffer->dma_addr, len,
277 buffer->addr, (u64)virt_to_phys(buffer->addr));
278
279 return 0;
280}
281
282static void
283efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
284{
285 if (!buffer->addr)
286 return;
287
288 netif_dbg(efx, hw, efx->net_dev,
289 "deallocating special buffers %d-%d at %llx+%x "
290 "(virt %p phys %llx)\n", buffer->index,
291 buffer->index + buffer->entries - 1,
292 (u64)buffer->dma_addr, buffer->len,
293 buffer->addr, (u64)virt_to_phys(buffer->addr));
294
295 dma_free_coherent(&efx->pci_dev->dev, buffer->len, buffer->addr,
296 buffer->dma_addr);
297 buffer->addr = NULL;
298 buffer->entries = 0;
299}
300
301/**************************************************************************
302 *
303 * Generic buffer handling
304 * These buffers are used for interrupt status and MAC stats
305 *
306 **************************************************************************/
307
308int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer,
309 unsigned int len)
310{
311 buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
312 &buffer->dma_addr);
313 if (!buffer->addr)
314 return -ENOMEM;
315 buffer->len = len;
316 memset(buffer->addr, 0, len);
317 return 0;
318}
319
320void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
321{
322 if (buffer->addr) {
323 pci_free_consistent(efx->pci_dev, buffer->len,
324 buffer->addr, buffer->dma_addr);
325 buffer->addr = NULL;
326 }
327}
328
329/**************************************************************************
330 *
331 * TX path
332 *
333 **************************************************************************/
334
335/* Returns a pointer to the specified transmit descriptor in the TX
336 * descriptor queue belonging to the specified channel.
337 */
338static inline efx_qword_t *
339efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
340{
341 return ((efx_qword_t *) (tx_queue->txd.addr)) + index;
342}
343
344/* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
345static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue)
346{
347 unsigned write_ptr;
348 efx_dword_t reg;
349
350 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
351 EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
352 efx_writed_page(tx_queue->efx, ®,
353 FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
354}
355
356/* Write pointer and first descriptor for TX descriptor ring */
357static inline void efx_push_tx_desc(struct efx_tx_queue *tx_queue,
358 const efx_qword_t *txd)
359{
360 unsigned write_ptr;
361 efx_oword_t reg;
362
363 BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0);
364 BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0);
365
366 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
367 EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true,
368 FRF_AZ_TX_DESC_WPTR, write_ptr);
369 reg.qword[0] = *txd;
370 efx_writeo_page(tx_queue->efx, ®,
371 FR_BZ_TX_DESC_UPD_P0, tx_queue->queue);
372}
373
374static inline bool
375efx_may_push_tx_desc(struct efx_tx_queue *tx_queue, unsigned int write_count)
376{
377 unsigned empty_read_count = ACCESS_ONCE(tx_queue->empty_read_count);
378
379 if (empty_read_count == 0)
380 return false;
381
382 tx_queue->empty_read_count = 0;
383 return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID) == 0;
384}
385
386/* For each entry inserted into the software descriptor ring, create a
387 * descriptor in the hardware TX descriptor ring (in host memory), and
388 * write a doorbell.
389 */
390void efx_nic_push_buffers(struct efx_tx_queue *tx_queue)
391{
392
393 struct efx_tx_buffer *buffer;
394 efx_qword_t *txd;
395 unsigned write_ptr;
396 unsigned old_write_count = tx_queue->write_count;
397
398 BUG_ON(tx_queue->write_count == tx_queue->insert_count);
399
400 do {
401 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
402 buffer = &tx_queue->buffer[write_ptr];
403 txd = efx_tx_desc(tx_queue, write_ptr);
404 ++tx_queue->write_count;
405
406 /* Create TX descriptor ring entry */
407 EFX_POPULATE_QWORD_4(*txd,
408 FSF_AZ_TX_KER_CONT, buffer->continuation,
409 FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
410 FSF_AZ_TX_KER_BUF_REGION, 0,
411 FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
412 } while (tx_queue->write_count != tx_queue->insert_count);
413
414 wmb(); /* Ensure descriptors are written before they are fetched */
415
416 if (efx_may_push_tx_desc(tx_queue, old_write_count)) {
417 txd = efx_tx_desc(tx_queue,
418 old_write_count & tx_queue->ptr_mask);
419 efx_push_tx_desc(tx_queue, txd);
420 ++tx_queue->pushes;
421 } else {
422 efx_notify_tx_desc(tx_queue);
423 }
424}
425
426/* Allocate hardware resources for a TX queue */
427int efx_nic_probe_tx(struct efx_tx_queue *tx_queue)
428{
429 struct efx_nic *efx = tx_queue->efx;
430 unsigned entries;
431
432 entries = tx_queue->ptr_mask + 1;
433 return efx_alloc_special_buffer(efx, &tx_queue->txd,
434 entries * sizeof(efx_qword_t));
435}
436
437void efx_nic_init_tx(struct efx_tx_queue *tx_queue)
438{
439 struct efx_nic *efx = tx_queue->efx;
440 efx_oword_t reg;
441
442 /* Pin TX descriptor ring */
443 efx_init_special_buffer(efx, &tx_queue->txd);
444
445 /* Push TX descriptor ring to card */
446 EFX_POPULATE_OWORD_10(reg,
447 FRF_AZ_TX_DESCQ_EN, 1,
448 FRF_AZ_TX_ISCSI_DDIG_EN, 0,
449 FRF_AZ_TX_ISCSI_HDIG_EN, 0,
450 FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
451 FRF_AZ_TX_DESCQ_EVQ_ID,
452 tx_queue->channel->channel,
453 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
454 FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue,
455 FRF_AZ_TX_DESCQ_SIZE,
456 __ffs(tx_queue->txd.entries),
457 FRF_AZ_TX_DESCQ_TYPE, 0,
458 FRF_BZ_TX_NON_IP_DROP_DIS, 1);
459
460 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
461 int csum = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD;
462 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
463 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_TCP_CHKSM_DIS,
464 !csum);
465 }
466
467 efx_writeo_table(efx, ®, efx->type->txd_ptr_tbl_base,
468 tx_queue->queue);
469
470 if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) {
471 /* Only 128 bits in this register */
472 BUILD_BUG_ON(EFX_MAX_TX_QUEUES > 128);
473
474 efx_reado(efx, ®, FR_AA_TX_CHKSM_CFG);
475 if (tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD)
476 clear_bit_le(tx_queue->queue, (void *)®);
477 else
478 set_bit_le(tx_queue->queue, (void *)®);
479 efx_writeo(efx, ®, FR_AA_TX_CHKSM_CFG);
480 }
481
482 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
483 EFX_POPULATE_OWORD_1(reg,
484 FRF_BZ_TX_PACE,
485 (tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ?
486 FFE_BZ_TX_PACE_OFF :
487 FFE_BZ_TX_PACE_RESERVED);
488 efx_writeo_table(efx, ®, FR_BZ_TX_PACE_TBL,
489 tx_queue->queue);
490 }
491}
492
493static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue)
494{
495 struct efx_nic *efx = tx_queue->efx;
496 efx_oword_t tx_flush_descq;
497
498 EFX_POPULATE_OWORD_2(tx_flush_descq,
499 FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
500 FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
501 efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
502}
503
504void efx_nic_fini_tx(struct efx_tx_queue *tx_queue)
505{
506 struct efx_nic *efx = tx_queue->efx;
507 efx_oword_t tx_desc_ptr;
508
509 /* Remove TX descriptor ring from card */
510 EFX_ZERO_OWORD(tx_desc_ptr);
511 efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
512 tx_queue->queue);
513
514 /* Unpin TX descriptor ring */
515 efx_fini_special_buffer(efx, &tx_queue->txd);
516}
517
518/* Free buffers backing TX queue */
519void efx_nic_remove_tx(struct efx_tx_queue *tx_queue)
520{
521 efx_free_special_buffer(tx_queue->efx, &tx_queue->txd);
522}
523
524/**************************************************************************
525 *
526 * RX path
527 *
528 **************************************************************************/
529
530/* Returns a pointer to the specified descriptor in the RX descriptor queue */
531static inline efx_qword_t *
532efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index)
533{
534 return ((efx_qword_t *) (rx_queue->rxd.addr)) + index;
535}
536
537/* This creates an entry in the RX descriptor queue */
538static inline void
539efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index)
540{
541 struct efx_rx_buffer *rx_buf;
542 efx_qword_t *rxd;
543
544 rxd = efx_rx_desc(rx_queue, index);
545 rx_buf = efx_rx_buffer(rx_queue, index);
546 EFX_POPULATE_QWORD_3(*rxd,
547 FSF_AZ_RX_KER_BUF_SIZE,
548 rx_buf->len -
549 rx_queue->efx->type->rx_buffer_padding,
550 FSF_AZ_RX_KER_BUF_REGION, 0,
551 FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
552}
553
554/* This writes to the RX_DESC_WPTR register for the specified receive
555 * descriptor ring.
556 */
557void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue)
558{
559 struct efx_nic *efx = rx_queue->efx;
560 efx_dword_t reg;
561 unsigned write_ptr;
562
563 while (rx_queue->notified_count != rx_queue->added_count) {
564 efx_build_rx_desc(
565 rx_queue,
566 rx_queue->notified_count & rx_queue->ptr_mask);
567 ++rx_queue->notified_count;
568 }
569
570 wmb();
571 write_ptr = rx_queue->added_count & rx_queue->ptr_mask;
572 EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
573 efx_writed_page(efx, ®, FR_AZ_RX_DESC_UPD_DWORD_P0,
574 efx_rx_queue_index(rx_queue));
575}
576
577int efx_nic_probe_rx(struct efx_rx_queue *rx_queue)
578{
579 struct efx_nic *efx = rx_queue->efx;
580 unsigned entries;
581
582 entries = rx_queue->ptr_mask + 1;
583 return efx_alloc_special_buffer(efx, &rx_queue->rxd,
584 entries * sizeof(efx_qword_t));
585}
586
587void efx_nic_init_rx(struct efx_rx_queue *rx_queue)
588{
589 efx_oword_t rx_desc_ptr;
590 struct efx_nic *efx = rx_queue->efx;
591 bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0;
592 bool iscsi_digest_en = is_b0;
593
594 netif_dbg(efx, hw, efx->net_dev,
595 "RX queue %d ring in special buffers %d-%d\n",
596 efx_rx_queue_index(rx_queue), rx_queue->rxd.index,
597 rx_queue->rxd.index + rx_queue->rxd.entries - 1);
598
599 /* Pin RX descriptor ring */
600 efx_init_special_buffer(efx, &rx_queue->rxd);
601
602 /* Push RX descriptor ring to card */
603 EFX_POPULATE_OWORD_10(rx_desc_ptr,
604 FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en,
605 FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en,
606 FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
607 FRF_AZ_RX_DESCQ_EVQ_ID,
608 efx_rx_queue_channel(rx_queue)->channel,
609 FRF_AZ_RX_DESCQ_OWNER_ID, 0,
610 FRF_AZ_RX_DESCQ_LABEL,
611 efx_rx_queue_index(rx_queue),
612 FRF_AZ_RX_DESCQ_SIZE,
613 __ffs(rx_queue->rxd.entries),
614 FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
615 /* For >=B0 this is scatter so disable */
616 FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
617 FRF_AZ_RX_DESCQ_EN, 1);
618 efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
619 efx_rx_queue_index(rx_queue));
620}
621
622static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue)
623{
624 struct efx_nic *efx = rx_queue->efx;
625 efx_oword_t rx_flush_descq;
626
627 EFX_POPULATE_OWORD_2(rx_flush_descq,
628 FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
629 FRF_AZ_RX_FLUSH_DESCQ,
630 efx_rx_queue_index(rx_queue));
631 efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
632}
633
634void efx_nic_fini_rx(struct efx_rx_queue *rx_queue)
635{
636 efx_oword_t rx_desc_ptr;
637 struct efx_nic *efx = rx_queue->efx;
638
639 /* Remove RX descriptor ring from card */
640 EFX_ZERO_OWORD(rx_desc_ptr);
641 efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
642 efx_rx_queue_index(rx_queue));
643
644 /* Unpin RX descriptor ring */
645 efx_fini_special_buffer(efx, &rx_queue->rxd);
646}
647
648/* Free buffers backing RX queue */
649void efx_nic_remove_rx(struct efx_rx_queue *rx_queue)
650{
651 efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
652}
653
654/**************************************************************************
655 *
656 * Flush handling
657 *
658 **************************************************************************/
659
660/* efx_nic_flush_queues() must be woken up when all flushes are completed,
661 * or more RX flushes can be kicked off.
662 */
663static bool efx_flush_wake(struct efx_nic *efx)
664{
665 /* Ensure that all updates are visible to efx_nic_flush_queues() */
666 smp_mb();
667
668 return (atomic_read(&efx->drain_pending) == 0 ||
669 (atomic_read(&efx->rxq_flush_outstanding) < EFX_RX_FLUSH_COUNT
670 && atomic_read(&efx->rxq_flush_pending) > 0));
671}
672
673/* Flush all the transmit queues, and continue flushing receive queues until
674 * they're all flushed. Wait for the DRAIN events to be recieved so that there
675 * are no more RX and TX events left on any channel. */
676int efx_nic_flush_queues(struct efx_nic *efx)
677{
678 unsigned timeout = msecs_to_jiffies(5000); /* 5s for all flushes and drains */
679 struct efx_channel *channel;
680 struct efx_rx_queue *rx_queue;
681 struct efx_tx_queue *tx_queue;
682 int rc = 0;
683
684 efx->fc_disable++;
685 efx->type->prepare_flush(efx);
686
687 efx_for_each_channel(channel, efx) {
688 efx_for_each_channel_tx_queue(tx_queue, channel) {
689 atomic_inc(&efx->drain_pending);
690 efx_flush_tx_queue(tx_queue);
691 }
692 efx_for_each_channel_rx_queue(rx_queue, channel) {
693 atomic_inc(&efx->drain_pending);
694 rx_queue->flush_pending = true;
695 atomic_inc(&efx->rxq_flush_pending);
696 }
697 }
698
699 while (timeout && atomic_read(&efx->drain_pending) > 0) {
700 /* If SRIOV is enabled, then offload receive queue flushing to
701 * the firmware (though we will still have to poll for
702 * completion). If that fails, fall back to the old scheme.
703 */
704 if (efx_sriov_enabled(efx)) {
705 rc = efx_mcdi_flush_rxqs(efx);
706 if (!rc)
707 goto wait;
708 }
709
710 /* The hardware supports four concurrent rx flushes, each of
711 * which may need to be retried if there is an outstanding
712 * descriptor fetch
713 */
714 efx_for_each_channel(channel, efx) {
715 efx_for_each_channel_rx_queue(rx_queue, channel) {
716 if (atomic_read(&efx->rxq_flush_outstanding) >=
717 EFX_RX_FLUSH_COUNT)
718 break;
719
720 if (rx_queue->flush_pending) {
721 rx_queue->flush_pending = false;
722 atomic_dec(&efx->rxq_flush_pending);
723 atomic_inc(&efx->rxq_flush_outstanding);
724 efx_flush_rx_queue(rx_queue);
725 }
726 }
727 }
728
729 wait:
730 timeout = wait_event_timeout(efx->flush_wq, efx_flush_wake(efx),
731 timeout);
732 }
733
734 if (atomic_read(&efx->drain_pending)) {
735 netif_err(efx, hw, efx->net_dev, "failed to flush %d queues "
736 "(rx %d+%d)\n", atomic_read(&efx->drain_pending),
737 atomic_read(&efx->rxq_flush_outstanding),
738 atomic_read(&efx->rxq_flush_pending));
739 rc = -ETIMEDOUT;
740
741 atomic_set(&efx->drain_pending, 0);
742 atomic_set(&efx->rxq_flush_pending, 0);
743 atomic_set(&efx->rxq_flush_outstanding, 0);
744 }
745
746 efx->fc_disable--;
747
748 return rc;
749}
750
751/**************************************************************************
752 *
753 * Event queue processing
754 * Event queues are processed by per-channel tasklets.
755 *
756 **************************************************************************/
757
758/* Update a channel's event queue's read pointer (RPTR) register
759 *
760 * This writes the EVQ_RPTR_REG register for the specified channel's
761 * event queue.
762 */
763void efx_nic_eventq_read_ack(struct efx_channel *channel)
764{
765 efx_dword_t reg;
766 struct efx_nic *efx = channel->efx;
767
768 EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR,
769 channel->eventq_read_ptr & channel->eventq_mask);
770 efx_writed_table(efx, ®, efx->type->evq_rptr_tbl_base,
771 channel->channel);
772}
773
774/* Use HW to insert a SW defined event */
775void efx_generate_event(struct efx_nic *efx, unsigned int evq,
776 efx_qword_t *event)
777{
778 efx_oword_t drv_ev_reg;
779
780 BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
781 FRF_AZ_DRV_EV_DATA_WIDTH != 64);
782 drv_ev_reg.u32[0] = event->u32[0];
783 drv_ev_reg.u32[1] = event->u32[1];
784 drv_ev_reg.u32[2] = 0;
785 drv_ev_reg.u32[3] = 0;
786 EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, evq);
787 efx_writeo(efx, &drv_ev_reg, FR_AZ_DRV_EV);
788}
789
790static void efx_magic_event(struct efx_channel *channel, u32 magic)
791{
792 efx_qword_t event;
793
794 EFX_POPULATE_QWORD_2(event, FSF_AZ_EV_CODE,
795 FSE_AZ_EV_CODE_DRV_GEN_EV,
796 FSF_AZ_DRV_GEN_EV_MAGIC, magic);
797 efx_generate_event(channel->efx, channel->channel, &event);
798}
799
800/* Handle a transmit completion event
801 *
802 * The NIC batches TX completion events; the message we receive is of
803 * the form "complete all TX events up to this index".
804 */
805static int
806efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
807{
808 unsigned int tx_ev_desc_ptr;
809 unsigned int tx_ev_q_label;
810 struct efx_tx_queue *tx_queue;
811 struct efx_nic *efx = channel->efx;
812 int tx_packets = 0;
813
814 if (unlikely(ACCESS_ONCE(efx->reset_pending)))
815 return 0;
816
817 if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
818 /* Transmit completion */
819 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
820 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
821 tx_queue = efx_channel_get_tx_queue(
822 channel, tx_ev_q_label % EFX_TXQ_TYPES);
823 tx_packets = ((tx_ev_desc_ptr - tx_queue->read_count) &
824 tx_queue->ptr_mask);
825 efx_xmit_done(tx_queue, tx_ev_desc_ptr);
826 } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
827 /* Rewrite the FIFO write pointer */
828 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
829 tx_queue = efx_channel_get_tx_queue(
830 channel, tx_ev_q_label % EFX_TXQ_TYPES);
831
832 netif_tx_lock(efx->net_dev);
833 efx_notify_tx_desc(tx_queue);
834 netif_tx_unlock(efx->net_dev);
835 } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) &&
836 EFX_WORKAROUND_10727(efx)) {
837 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
838 } else {
839 netif_err(efx, tx_err, efx->net_dev,
840 "channel %d unexpected TX event "
841 EFX_QWORD_FMT"\n", channel->channel,
842 EFX_QWORD_VAL(*event));
843 }
844
845 return tx_packets;
846}
847
848/* Detect errors included in the rx_evt_pkt_ok bit. */
849static u16 efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
850 const efx_qword_t *event)
851{
852 struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
853 struct efx_nic *efx = rx_queue->efx;
854 bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
855 bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
856 bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc;
857 bool rx_ev_other_err, rx_ev_pause_frm;
858 bool rx_ev_hdr_type, rx_ev_mcast_pkt;
859 unsigned rx_ev_pkt_type;
860
861 rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
862 rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
863 rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
864 rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE);
865 rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
866 FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
867 rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
868 FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
869 rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
870 FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
871 rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
872 rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
873 rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ?
874 0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB));
875 rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
876
877 /* Every error apart from tobe_disc and pause_frm */
878 rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err |
879 rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
880 rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);
881
882 /* Count errors that are not in MAC stats. Ignore expected
883 * checksum errors during self-test. */
884 if (rx_ev_frm_trunc)
885 ++channel->n_rx_frm_trunc;
886 else if (rx_ev_tobe_disc)
887 ++channel->n_rx_tobe_disc;
888 else if (!efx->loopback_selftest) {
889 if (rx_ev_ip_hdr_chksum_err)
890 ++channel->n_rx_ip_hdr_chksum_err;
891 else if (rx_ev_tcp_udp_chksum_err)
892 ++channel->n_rx_tcp_udp_chksum_err;
893 }
894
895 /* TOBE_DISC is expected on unicast mismatches; don't print out an
896 * error message. FRM_TRUNC indicates RXDP dropped the packet due
897 * to a FIFO overflow.
898 */
899#ifdef DEBUG
900 if (rx_ev_other_err && net_ratelimit()) {
901 netif_dbg(efx, rx_err, efx->net_dev,
902 " RX queue %d unexpected RX event "
903 EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n",
904 efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event),
905 rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
906 rx_ev_ip_hdr_chksum_err ?
907 " [IP_HDR_CHKSUM_ERR]" : "",
908 rx_ev_tcp_udp_chksum_err ?
909 " [TCP_UDP_CHKSUM_ERR]" : "",
910 rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
911 rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
912 rx_ev_drib_nib ? " [DRIB_NIB]" : "",
913 rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
914 rx_ev_pause_frm ? " [PAUSE]" : "");
915 }
916#endif
917
918 /* The frame must be discarded if any of these are true. */
919 return (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib |
920 rx_ev_tobe_disc | rx_ev_pause_frm) ?
921 EFX_RX_PKT_DISCARD : 0;
922}
923
924/* Handle receive events that are not in-order. */
925static void
926efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
927{
928 struct efx_nic *efx = rx_queue->efx;
929 unsigned expected, dropped;
930
931 expected = rx_queue->removed_count & rx_queue->ptr_mask;
932 dropped = (index - expected) & rx_queue->ptr_mask;
933 netif_info(efx, rx_err, efx->net_dev,
934 "dropped %d events (index=%d expected=%d)\n",
935 dropped, index, expected);
936
937 efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
938 RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
939}
940
941/* Handle a packet received event
942 *
943 * The NIC gives a "discard" flag if it's a unicast packet with the
944 * wrong destination address
945 * Also "is multicast" and "matches multicast filter" flags can be used to
946 * discard non-matching multicast packets.
947 */
948static void
949efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
950{
951 unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
952 unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
953 unsigned expected_ptr;
954 bool rx_ev_pkt_ok;
955 u16 flags;
956 struct efx_rx_queue *rx_queue;
957 struct efx_nic *efx = channel->efx;
958
959 if (unlikely(ACCESS_ONCE(efx->reset_pending)))
960 return;
961
962 /* Basic packet information */
963 rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
964 rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
965 rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
966 WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
967 WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
968 WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
969 channel->channel);
970
971 rx_queue = efx_channel_get_rx_queue(channel);
972
973 rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
974 expected_ptr = rx_queue->removed_count & rx_queue->ptr_mask;
975 if (unlikely(rx_ev_desc_ptr != expected_ptr))
976 efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);
977
978 if (likely(rx_ev_pkt_ok)) {
979 /* If packet is marked as OK and packet type is TCP/IP or
980 * UDP/IP, then we can rely on the hardware checksum.
981 */
982 flags = (rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP ||
983 rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP) ?
984 EFX_RX_PKT_CSUMMED : 0;
985 } else {
986 flags = efx_handle_rx_not_ok(rx_queue, event);
987 }
988
989 /* Detect multicast packets that didn't match the filter */
990 rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
991 if (rx_ev_mcast_pkt) {
992 unsigned int rx_ev_mcast_hash_match =
993 EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
994
995 if (unlikely(!rx_ev_mcast_hash_match)) {
996 ++channel->n_rx_mcast_mismatch;
997 flags |= EFX_RX_PKT_DISCARD;
998 }
999 }
1000
1001 channel->irq_mod_score += 2;
1002
1003 /* Handle received packet */
1004 efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt, flags);
1005}
1006
1007/* If this flush done event corresponds to a &struct efx_tx_queue, then
1008 * send an %EFX_CHANNEL_MAGIC_TX_DRAIN event to drain the event queue
1009 * of all transmit completions.
1010 */
1011static void
1012efx_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event)
1013{
1014 struct efx_tx_queue *tx_queue;
1015 int qid;
1016
1017 qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
1018 if (qid < EFX_TXQ_TYPES * efx->n_tx_channels) {
1019 tx_queue = efx_get_tx_queue(efx, qid / EFX_TXQ_TYPES,
1020 qid % EFX_TXQ_TYPES);
1021
1022 efx_magic_event(tx_queue->channel,
1023 EFX_CHANNEL_MAGIC_TX_DRAIN(tx_queue));
1024 }
1025}
1026
1027/* If this flush done event corresponds to a &struct efx_rx_queue: If the flush
1028 * was succesful then send an %EFX_CHANNEL_MAGIC_RX_DRAIN, otherwise add
1029 * the RX queue back to the mask of RX queues in need of flushing.
1030 */
1031static void
1032efx_handle_rx_flush_done(struct efx_nic *efx, efx_qword_t *event)
1033{
1034 struct efx_channel *channel;
1035 struct efx_rx_queue *rx_queue;
1036 int qid;
1037 bool failed;
1038
1039 qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
1040 failed = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
1041 if (qid >= efx->n_channels)
1042 return;
1043 channel = efx_get_channel(efx, qid);
1044 if (!efx_channel_has_rx_queue(channel))
1045 return;
1046 rx_queue = efx_channel_get_rx_queue(channel);
1047
1048 if (failed) {
1049 netif_info(efx, hw, efx->net_dev,
1050 "RXQ %d flush retry\n", qid);
1051 rx_queue->flush_pending = true;
1052 atomic_inc(&efx->rxq_flush_pending);
1053 } else {
1054 efx_magic_event(efx_rx_queue_channel(rx_queue),
1055 EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue));
1056 }
1057 atomic_dec(&efx->rxq_flush_outstanding);
1058 if (efx_flush_wake(efx))
1059 wake_up(&efx->flush_wq);
1060}
1061
1062static void
1063efx_handle_drain_event(struct efx_channel *channel)
1064{
1065 struct efx_nic *efx = channel->efx;
1066
1067 WARN_ON(atomic_read(&efx->drain_pending) == 0);
1068 atomic_dec(&efx->drain_pending);
1069 if (efx_flush_wake(efx))
1070 wake_up(&efx->flush_wq);
1071}
1072
1073static void
1074efx_handle_generated_event(struct efx_channel *channel, efx_qword_t *event)
1075{
1076 struct efx_nic *efx = channel->efx;
1077 struct efx_rx_queue *rx_queue =
1078 efx_channel_has_rx_queue(channel) ?
1079 efx_channel_get_rx_queue(channel) : NULL;
1080 unsigned magic, code;
1081
1082 magic = EFX_QWORD_FIELD(*event, FSF_AZ_DRV_GEN_EV_MAGIC);
1083 code = _EFX_CHANNEL_MAGIC_CODE(magic);
1084
1085 if (magic == EFX_CHANNEL_MAGIC_TEST(channel)) {
1086 channel->event_test_cpu = raw_smp_processor_id();
1087 } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_FILL(rx_queue)) {
1088 /* The queue must be empty, so we won't receive any rx
1089 * events, so efx_process_channel() won't refill the
1090 * queue. Refill it here */
1091 efx_fast_push_rx_descriptors(rx_queue);
1092 } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue)) {
1093 rx_queue->enabled = false;
1094 efx_handle_drain_event(channel);
1095 } else if (code == _EFX_CHANNEL_MAGIC_TX_DRAIN) {
1096 efx_handle_drain_event(channel);
1097 } else {
1098 netif_dbg(efx, hw, efx->net_dev, "channel %d received "
1099 "generated event "EFX_QWORD_FMT"\n",
1100 channel->channel, EFX_QWORD_VAL(*event));
1101 }
1102}
1103
1104static void
1105efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
1106{
1107 struct efx_nic *efx = channel->efx;
1108 unsigned int ev_sub_code;
1109 unsigned int ev_sub_data;
1110
1111 ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
1112 ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
1113
1114 switch (ev_sub_code) {
1115 case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
1116 netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n",
1117 channel->channel, ev_sub_data);
1118 efx_handle_tx_flush_done(efx, event);
1119 efx_sriov_tx_flush_done(efx, event);
1120 break;
1121 case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
1122 netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n",
1123 channel->channel, ev_sub_data);
1124 efx_handle_rx_flush_done(efx, event);
1125 efx_sriov_rx_flush_done(efx, event);
1126 break;
1127 case FSE_AZ_EVQ_INIT_DONE_EV:
1128 netif_dbg(efx, hw, efx->net_dev,
1129 "channel %d EVQ %d initialised\n",
1130 channel->channel, ev_sub_data);
1131 break;
1132 case FSE_AZ_SRM_UPD_DONE_EV:
1133 netif_vdbg(efx, hw, efx->net_dev,
1134 "channel %d SRAM update done\n", channel->channel);
1135 break;
1136 case FSE_AZ_WAKE_UP_EV:
1137 netif_vdbg(efx, hw, efx->net_dev,
1138 "channel %d RXQ %d wakeup event\n",
1139 channel->channel, ev_sub_data);
1140 break;
1141 case FSE_AZ_TIMER_EV:
1142 netif_vdbg(efx, hw, efx->net_dev,
1143 "channel %d RX queue %d timer expired\n",
1144 channel->channel, ev_sub_data);
1145 break;
1146 case FSE_AA_RX_RECOVER_EV:
1147 netif_err(efx, rx_err, efx->net_dev,
1148 "channel %d seen DRIVER RX_RESET event. "
1149 "Resetting.\n", channel->channel);
1150 atomic_inc(&efx->rx_reset);
1151 efx_schedule_reset(efx,
1152 EFX_WORKAROUND_6555(efx) ?
1153 RESET_TYPE_RX_RECOVERY :
1154 RESET_TYPE_DISABLE);
1155 break;
1156 case FSE_BZ_RX_DSC_ERROR_EV:
1157 if (ev_sub_data < EFX_VI_BASE) {
1158 netif_err(efx, rx_err, efx->net_dev,
1159 "RX DMA Q %d reports descriptor fetch error."
1160 " RX Q %d is disabled.\n", ev_sub_data,
1161 ev_sub_data);
1162 efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH);
1163 } else
1164 efx_sriov_desc_fetch_err(efx, ev_sub_data);
1165 break;
1166 case FSE_BZ_TX_DSC_ERROR_EV:
1167 if (ev_sub_data < EFX_VI_BASE) {
1168 netif_err(efx, tx_err, efx->net_dev,
1169 "TX DMA Q %d reports descriptor fetch error."
1170 " TX Q %d is disabled.\n", ev_sub_data,
1171 ev_sub_data);
1172 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
1173 } else
1174 efx_sriov_desc_fetch_err(efx, ev_sub_data);
1175 break;
1176 default:
1177 netif_vdbg(efx, hw, efx->net_dev,
1178 "channel %d unknown driver event code %d "
1179 "data %04x\n", channel->channel, ev_sub_code,
1180 ev_sub_data);
1181 break;
1182 }
1183}
1184
1185int efx_nic_process_eventq(struct efx_channel *channel, int budget)
1186{
1187 struct efx_nic *efx = channel->efx;
1188 unsigned int read_ptr;
1189 efx_qword_t event, *p_event;
1190 int ev_code;
1191 int tx_packets = 0;
1192 int spent = 0;
1193
1194 read_ptr = channel->eventq_read_ptr;
1195
1196 for (;;) {
1197 p_event = efx_event(channel, read_ptr);
1198 event = *p_event;
1199
1200 if (!efx_event_present(&event))
1201 /* End of events */
1202 break;
1203
1204 netif_vdbg(channel->efx, intr, channel->efx->net_dev,
1205 "channel %d event is "EFX_QWORD_FMT"\n",
1206 channel->channel, EFX_QWORD_VAL(event));
1207
1208 /* Clear this event by marking it all ones */
1209 EFX_SET_QWORD(*p_event);
1210
1211 ++read_ptr;
1212
1213 ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE);
1214
1215 switch (ev_code) {
1216 case FSE_AZ_EV_CODE_RX_EV:
1217 efx_handle_rx_event(channel, &event);
1218 if (++spent == budget)
1219 goto out;
1220 break;
1221 case FSE_AZ_EV_CODE_TX_EV:
1222 tx_packets += efx_handle_tx_event(channel, &event);
1223 if (tx_packets > efx->txq_entries) {
1224 spent = budget;
1225 goto out;
1226 }
1227 break;
1228 case FSE_AZ_EV_CODE_DRV_GEN_EV:
1229 efx_handle_generated_event(channel, &event);
1230 break;
1231 case FSE_AZ_EV_CODE_DRIVER_EV:
1232 efx_handle_driver_event(channel, &event);
1233 break;
1234 case FSE_CZ_EV_CODE_USER_EV:
1235 efx_sriov_event(channel, &event);
1236 break;
1237 case FSE_CZ_EV_CODE_MCDI_EV:
1238 efx_mcdi_process_event(channel, &event);
1239 break;
1240 case FSE_AZ_EV_CODE_GLOBAL_EV:
1241 if (efx->type->handle_global_event &&
1242 efx->type->handle_global_event(channel, &event))
1243 break;
1244 /* else fall through */
1245 default:
1246 netif_err(channel->efx, hw, channel->efx->net_dev,
1247 "channel %d unknown event type %d (data "
1248 EFX_QWORD_FMT ")\n", channel->channel,
1249 ev_code, EFX_QWORD_VAL(event));
1250 }
1251 }
1252
1253out:
1254 channel->eventq_read_ptr = read_ptr;
1255 return spent;
1256}
1257
1258/* Check whether an event is present in the eventq at the current
1259 * read pointer. Only useful for self-test.
1260 */
1261bool efx_nic_event_present(struct efx_channel *channel)
1262{
1263 return efx_event_present(efx_event(channel, channel->eventq_read_ptr));
1264}
1265
1266/* Allocate buffer table entries for event queue */
1267int efx_nic_probe_eventq(struct efx_channel *channel)
1268{
1269 struct efx_nic *efx = channel->efx;
1270 unsigned entries;
1271
1272 entries = channel->eventq_mask + 1;
1273 return efx_alloc_special_buffer(efx, &channel->eventq,
1274 entries * sizeof(efx_qword_t));
1275}
1276
1277void efx_nic_init_eventq(struct efx_channel *channel)
1278{
1279 efx_oword_t reg;
1280 struct efx_nic *efx = channel->efx;
1281
1282 netif_dbg(efx, hw, efx->net_dev,
1283 "channel %d event queue in special buffers %d-%d\n",
1284 channel->channel, channel->eventq.index,
1285 channel->eventq.index + channel->eventq.entries - 1);
1286
1287 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
1288 EFX_POPULATE_OWORD_3(reg,
1289 FRF_CZ_TIMER_Q_EN, 1,
1290 FRF_CZ_HOST_NOTIFY_MODE, 0,
1291 FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
1292 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, channel->channel);
1293 }
1294
1295 /* Pin event queue buffer */
1296 efx_init_special_buffer(efx, &channel->eventq);
1297
1298 /* Fill event queue with all ones (i.e. empty events) */
1299 memset(channel->eventq.addr, 0xff, channel->eventq.len);
1300
1301 /* Push event queue to card */
1302 EFX_POPULATE_OWORD_3(reg,
1303 FRF_AZ_EVQ_EN, 1,
1304 FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries),
1305 FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index);
1306 efx_writeo_table(efx, ®, efx->type->evq_ptr_tbl_base,
1307 channel->channel);
1308
1309 efx->type->push_irq_moderation(channel);
1310}
1311
1312void efx_nic_fini_eventq(struct efx_channel *channel)
1313{
1314 efx_oword_t reg;
1315 struct efx_nic *efx = channel->efx;
1316
1317 /* Remove event queue from card */
1318 EFX_ZERO_OWORD(reg);
1319 efx_writeo_table(efx, ®, efx->type->evq_ptr_tbl_base,
1320 channel->channel);
1321 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
1322 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, channel->channel);
1323
1324 /* Unpin event queue */
1325 efx_fini_special_buffer(efx, &channel->eventq);
1326}
1327
1328/* Free buffers backing event queue */
1329void efx_nic_remove_eventq(struct efx_channel *channel)
1330{
1331 efx_free_special_buffer(channel->efx, &channel->eventq);
1332}
1333
1334
1335void efx_nic_event_test_start(struct efx_channel *channel)
1336{
1337 channel->event_test_cpu = -1;
1338 smp_wmb();
1339 efx_magic_event(channel, EFX_CHANNEL_MAGIC_TEST(channel));
1340}
1341
1342void efx_nic_generate_fill_event(struct efx_rx_queue *rx_queue)
1343{
1344 efx_magic_event(efx_rx_queue_channel(rx_queue),
1345 EFX_CHANNEL_MAGIC_FILL(rx_queue));
1346}
1347
1348/**************************************************************************
1349 *
1350 * Hardware interrupts
1351 * The hardware interrupt handler does very little work; all the event
1352 * queue processing is carried out by per-channel tasklets.
1353 *
1354 **************************************************************************/
1355
1356/* Enable/disable/generate interrupts */
1357static inline void efx_nic_interrupts(struct efx_nic *efx,
1358 bool enabled, bool force)
1359{
1360 efx_oword_t int_en_reg_ker;
1361
1362 EFX_POPULATE_OWORD_3(int_en_reg_ker,
1363 FRF_AZ_KER_INT_LEVE_SEL, efx->irq_level,
1364 FRF_AZ_KER_INT_KER, force,
1365 FRF_AZ_DRV_INT_EN_KER, enabled);
1366 efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);
1367}
1368
1369void efx_nic_enable_interrupts(struct efx_nic *efx)
1370{
1371 EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));
1372 wmb(); /* Ensure interrupt vector is clear before interrupts enabled */
1373
1374 efx_nic_interrupts(efx, true, false);
1375}
1376
1377void efx_nic_disable_interrupts(struct efx_nic *efx)
1378{
1379 /* Disable interrupts */
1380 efx_nic_interrupts(efx, false, false);
1381}
1382
1383/* Generate a test interrupt
1384 * Interrupt must already have been enabled, otherwise nasty things
1385 * may happen.
1386 */
1387void efx_nic_irq_test_start(struct efx_nic *efx)
1388{
1389 efx->last_irq_cpu = -1;
1390 smp_wmb();
1391 efx_nic_interrupts(efx, true, true);
1392}
1393
1394/* Process a fatal interrupt
1395 * Disable bus mastering ASAP and schedule a reset
1396 */
1397irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx)
1398{
1399 struct falcon_nic_data *nic_data = efx->nic_data;
1400 efx_oword_t *int_ker = efx->irq_status.addr;
1401 efx_oword_t fatal_intr;
1402 int error, mem_perr;
1403
1404 efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);
1405 error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR);
1406
1407 netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status "
1408 EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker),
1409 EFX_OWORD_VAL(fatal_intr),
1410 error ? "disabling bus mastering" : "no recognised error");
1411
1412 /* If this is a memory parity error dump which blocks are offending */
1413 mem_perr = (EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER) ||
1414 EFX_OWORD_FIELD(fatal_intr, FRF_AZ_SRM_PERR_INT_KER));
1415 if (mem_perr) {
1416 efx_oword_t reg;
1417 efx_reado(efx, ®, FR_AZ_MEM_STAT);
1418 netif_err(efx, hw, efx->net_dev,
1419 "SYSTEM ERROR: memory parity error "EFX_OWORD_FMT"\n",
1420 EFX_OWORD_VAL(reg));
1421 }
1422
1423 /* Disable both devices */
1424 pci_clear_master(efx->pci_dev);
1425 if (efx_nic_is_dual_func(efx))
1426 pci_clear_master(nic_data->pci_dev2);
1427 efx_nic_disable_interrupts(efx);
1428
1429 /* Count errors and reset or disable the NIC accordingly */
1430 if (efx->int_error_count == 0 ||
1431 time_after(jiffies, efx->int_error_expire)) {
1432 efx->int_error_count = 0;
1433 efx->int_error_expire =
1434 jiffies + EFX_INT_ERROR_EXPIRE * HZ;
1435 }
1436 if (++efx->int_error_count < EFX_MAX_INT_ERRORS) {
1437 netif_err(efx, hw, efx->net_dev,
1438 "SYSTEM ERROR - reset scheduled\n");
1439 efx_schedule_reset(efx, RESET_TYPE_INT_ERROR);
1440 } else {
1441 netif_err(efx, hw, efx->net_dev,
1442 "SYSTEM ERROR - max number of errors seen."
1443 "NIC will be disabled\n");
1444 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
1445 }
1446
1447 return IRQ_HANDLED;
1448}
1449
1450/* Handle a legacy interrupt
1451 * Acknowledges the interrupt and schedule event queue processing.
1452 */
1453static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id)
1454{
1455 struct efx_nic *efx = dev_id;
1456 efx_oword_t *int_ker = efx->irq_status.addr;
1457 irqreturn_t result = IRQ_NONE;
1458 struct efx_channel *channel;
1459 efx_dword_t reg;
1460 u32 queues;
1461 int syserr;
1462
1463 /* Could this be ours? If interrupts are disabled then the
1464 * channel state may not be valid.
1465 */
1466 if (!efx->legacy_irq_enabled)
1467 return result;
1468
1469 /* Read the ISR which also ACKs the interrupts */
1470 efx_readd(efx, ®, FR_BZ_INT_ISR0);
1471 queues = EFX_EXTRACT_DWORD(reg, 0, 31);
1472
1473 /* Handle non-event-queue sources */
1474 if (queues & (1U << efx->irq_level)) {
1475 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1476 if (unlikely(syserr))
1477 return efx_nic_fatal_interrupt(efx);
1478 efx->last_irq_cpu = raw_smp_processor_id();
1479 }
1480
1481 if (queues != 0) {
1482 if (EFX_WORKAROUND_15783(efx))
1483 efx->irq_zero_count = 0;
1484
1485 /* Schedule processing of any interrupting queues */
1486 efx_for_each_channel(channel, efx) {
1487 if (queues & 1)
1488 efx_schedule_channel_irq(channel);
1489 queues >>= 1;
1490 }
1491 result = IRQ_HANDLED;
1492
1493 } else if (EFX_WORKAROUND_15783(efx)) {
1494 efx_qword_t *event;
1495
1496 /* We can't return IRQ_HANDLED more than once on seeing ISR=0
1497 * because this might be a shared interrupt. */
1498 if (efx->irq_zero_count++ == 0)
1499 result = IRQ_HANDLED;
1500
1501 /* Ensure we schedule or rearm all event queues */
1502 efx_for_each_channel(channel, efx) {
1503 event = efx_event(channel, channel->eventq_read_ptr);
1504 if (efx_event_present(event))
1505 efx_schedule_channel_irq(channel);
1506 else
1507 efx_nic_eventq_read_ack(channel);
1508 }
1509 }
1510
1511 if (result == IRQ_HANDLED)
1512 netif_vdbg(efx, intr, efx->net_dev,
1513 "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
1514 irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
1515
1516 return result;
1517}
1518
1519/* Handle an MSI interrupt
1520 *
1521 * Handle an MSI hardware interrupt. This routine schedules event
1522 * queue processing. No interrupt acknowledgement cycle is necessary.
1523 * Also, we never need to check that the interrupt is for us, since
1524 * MSI interrupts cannot be shared.
1525 */
1526static irqreturn_t efx_msi_interrupt(int irq, void *dev_id)
1527{
1528 struct efx_channel *channel = *(struct efx_channel **)dev_id;
1529 struct efx_nic *efx = channel->efx;
1530 efx_oword_t *int_ker = efx->irq_status.addr;
1531 int syserr;
1532
1533 netif_vdbg(efx, intr, efx->net_dev,
1534 "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
1535 irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
1536
1537 /* Handle non-event-queue sources */
1538 if (channel->channel == efx->irq_level) {
1539 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1540 if (unlikely(syserr))
1541 return efx_nic_fatal_interrupt(efx);
1542 efx->last_irq_cpu = raw_smp_processor_id();
1543 }
1544
1545 /* Schedule processing of the channel */
1546 efx_schedule_channel_irq(channel);
1547
1548 return IRQ_HANDLED;
1549}
1550
1551
1552/* Setup RSS indirection table.
1553 * This maps from the hash value of the packet to RXQ
1554 */
1555void efx_nic_push_rx_indir_table(struct efx_nic *efx)
1556{
1557 size_t i = 0;
1558 efx_dword_t dword;
1559
1560 if (efx_nic_rev(efx) < EFX_REV_FALCON_B0)
1561 return;
1562
1563 BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) !=
1564 FR_BZ_RX_INDIRECTION_TBL_ROWS);
1565
1566 for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) {
1567 EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE,
1568 efx->rx_indir_table[i]);
1569 efx_writed_table(efx, &dword, FR_BZ_RX_INDIRECTION_TBL, i);
1570 }
1571}
1572
1573/* Hook interrupt handler(s)
1574 * Try MSI and then legacy interrupts.
1575 */
1576int efx_nic_init_interrupt(struct efx_nic *efx)
1577{
1578 struct efx_channel *channel;
1579 int rc;
1580
1581 if (!EFX_INT_MODE_USE_MSI(efx)) {
1582 irq_handler_t handler;
1583 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1584 handler = efx_legacy_interrupt;
1585 else
1586 handler = falcon_legacy_interrupt_a1;
1587
1588 rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED,
1589 efx->name, efx);
1590 if (rc) {
1591 netif_err(efx, drv, efx->net_dev,
1592 "failed to hook legacy IRQ %d\n",
1593 efx->pci_dev->irq);
1594 goto fail1;
1595 }
1596 return 0;
1597 }
1598
1599 /* Hook MSI or MSI-X interrupt */
1600 efx_for_each_channel(channel, efx) {
1601 rc = request_irq(channel->irq, efx_msi_interrupt,
1602 IRQF_PROBE_SHARED, /* Not shared */
1603 efx->channel_name[channel->channel],
1604 &efx->channel[channel->channel]);
1605 if (rc) {
1606 netif_err(efx, drv, efx->net_dev,
1607 "failed to hook IRQ %d\n", channel->irq);
1608 goto fail2;
1609 }
1610 }
1611
1612 return 0;
1613
1614 fail2:
1615 efx_for_each_channel(channel, efx)
1616 free_irq(channel->irq, &efx->channel[channel->channel]);
1617 fail1:
1618 return rc;
1619}
1620
1621void efx_nic_fini_interrupt(struct efx_nic *efx)
1622{
1623 struct efx_channel *channel;
1624 efx_oword_t reg;
1625
1626 /* Disable MSI/MSI-X interrupts */
1627 efx_for_each_channel(channel, efx) {
1628 if (channel->irq)
1629 free_irq(channel->irq, &efx->channel[channel->channel]);
1630 }
1631
1632 /* ACK legacy interrupt */
1633 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1634 efx_reado(efx, ®, FR_BZ_INT_ISR0);
1635 else
1636 falcon_irq_ack_a1(efx);
1637
1638 /* Disable legacy interrupt */
1639 if (efx->legacy_irq)
1640 free_irq(efx->legacy_irq, efx);
1641}
1642
1643/* Looks at available SRAM resources and works out how many queues we
1644 * can support, and where things like descriptor caches should live.
1645 *
1646 * SRAM is split up as follows:
1647 * 0 buftbl entries for channels
1648 * efx->vf_buftbl_base buftbl entries for SR-IOV
1649 * efx->rx_dc_base RX descriptor caches
1650 * efx->tx_dc_base TX descriptor caches
1651 */
1652void efx_nic_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
1653{
1654 unsigned vi_count, buftbl_min;
1655
1656 /* Account for the buffer table entries backing the datapath channels
1657 * and the descriptor caches for those channels.
1658 */
1659 buftbl_min = ((efx->n_rx_channels * EFX_MAX_DMAQ_SIZE +
1660 efx->n_tx_channels * EFX_TXQ_TYPES * EFX_MAX_DMAQ_SIZE +
1661 efx->n_channels * EFX_MAX_EVQ_SIZE)
1662 * sizeof(efx_qword_t) / EFX_BUF_SIZE);
1663 vi_count = max(efx->n_channels, efx->n_tx_channels * EFX_TXQ_TYPES);
1664
1665#ifdef CONFIG_SFC_SRIOV
1666 if (efx_sriov_wanted(efx)) {
1667 unsigned vi_dc_entries, buftbl_free, entries_per_vf, vf_limit;
1668
1669 efx->vf_buftbl_base = buftbl_min;
1670
1671 vi_dc_entries = RX_DC_ENTRIES + TX_DC_ENTRIES;
1672 vi_count = max(vi_count, EFX_VI_BASE);
1673 buftbl_free = (sram_lim_qw - buftbl_min -
1674 vi_count * vi_dc_entries);
1675
1676 entries_per_vf = ((vi_dc_entries + EFX_VF_BUFTBL_PER_VI) *
1677 efx_vf_size(efx));
1678 vf_limit = min(buftbl_free / entries_per_vf,
1679 (1024U - EFX_VI_BASE) >> efx->vi_scale);
1680
1681 if (efx->vf_count > vf_limit) {
1682 netif_err(efx, probe, efx->net_dev,
1683 "Reducing VF count from from %d to %d\n",
1684 efx->vf_count, vf_limit);
1685 efx->vf_count = vf_limit;
1686 }
1687 vi_count += efx->vf_count * efx_vf_size(efx);
1688 }
1689#endif
1690
1691 efx->tx_dc_base = sram_lim_qw - vi_count * TX_DC_ENTRIES;
1692 efx->rx_dc_base = efx->tx_dc_base - vi_count * RX_DC_ENTRIES;
1693}
1694
1695u32 efx_nic_fpga_ver(struct efx_nic *efx)
1696{
1697 efx_oword_t altera_build;
1698 efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);
1699 return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER);
1700}
1701
1702void efx_nic_init_common(struct efx_nic *efx)
1703{
1704 efx_oword_t temp;
1705
1706 /* Set positions of descriptor caches in SRAM. */
1707 EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, efx->tx_dc_base);
1708 efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);
1709 EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, efx->rx_dc_base);
1710 efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);
1711
1712 /* Set TX descriptor cache size. */
1713 BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER));
1714 EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER);
1715 efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);
1716
1717 /* Set RX descriptor cache size. Set low watermark to size-8, as
1718 * this allows most efficient prefetching.
1719 */
1720 BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER));
1721 EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER);
1722 efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);
1723 EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8);
1724 efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);
1725
1726 /* Program INT_KER address */
1727 EFX_POPULATE_OWORD_2(temp,
1728 FRF_AZ_NORM_INT_VEC_DIS_KER,
1729 EFX_INT_MODE_USE_MSI(efx),
1730 FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);
1731 efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER);
1732
1733 if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx))
1734 /* Use an interrupt level unused by event queues */
1735 efx->irq_level = 0x1f;
1736 else
1737 /* Use a valid MSI-X vector */
1738 efx->irq_level = 0;
1739
1740 /* Enable all the genuinely fatal interrupts. (They are still
1741 * masked by the overall interrupt mask, controlled by
1742 * falcon_interrupts()).
1743 *
1744 * Note: All other fatal interrupts are enabled
1745 */
1746 EFX_POPULATE_OWORD_3(temp,
1747 FRF_AZ_ILL_ADR_INT_KER_EN, 1,
1748 FRF_AZ_RBUF_OWN_INT_KER_EN, 1,
1749 FRF_AZ_TBUF_OWN_INT_KER_EN, 1);
1750 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
1751 EFX_SET_OWORD_FIELD(temp, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 1);
1752 EFX_INVERT_OWORD(temp);
1753 efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);
1754
1755 efx_nic_push_rx_indir_table(efx);
1756
1757 /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be
1758 * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q.
1759 */
1760 efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
1761 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe);
1762 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1);
1763 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
1764 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1);
1765 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1);
1766 /* Enable SW_EV to inherit in char driver - assume harmless here */
1767 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1);
1768 /* Prefetch threshold 2 => fetch when descriptor cache half empty */
1769 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2);
1770 /* Disable hardware watchdog which can misfire */
1771 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
1772 /* Squash TX of packets of 16 bytes or less */
1773 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1774 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
1775 efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
1776
1777 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
1778 EFX_POPULATE_OWORD_4(temp,
1779 /* Default values */
1780 FRF_BZ_TX_PACE_SB_NOT_AF, 0x15,
1781 FRF_BZ_TX_PACE_SB_AF, 0xb,
1782 FRF_BZ_TX_PACE_FB_BASE, 0,
1783 /* Allow large pace values in the
1784 * fast bin. */
1785 FRF_BZ_TX_PACE_BIN_TH,
1786 FFE_BZ_TX_PACE_RESERVED);
1787 efx_writeo(efx, &temp, FR_BZ_TX_PACE);
1788 }
1789}
1790
1791/* Register dump */
1792
1793#define REGISTER_REVISION_A 1
1794#define REGISTER_REVISION_B 2
1795#define REGISTER_REVISION_C 3
1796#define REGISTER_REVISION_Z 3 /* latest revision */
1797
1798struct efx_nic_reg {
1799 u32 offset:24;
1800 u32 min_revision:2, max_revision:2;
1801};
1802
1803#define REGISTER(name, min_rev, max_rev) { \
1804 FR_ ## min_rev ## max_rev ## _ ## name, \
1805 REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev \
1806}
1807#define REGISTER_AA(name) REGISTER(name, A, A)
1808#define REGISTER_AB(name) REGISTER(name, A, B)
1809#define REGISTER_AZ(name) REGISTER(name, A, Z)
1810#define REGISTER_BB(name) REGISTER(name, B, B)
1811#define REGISTER_BZ(name) REGISTER(name, B, Z)
1812#define REGISTER_CZ(name) REGISTER(name, C, Z)
1813
1814static const struct efx_nic_reg efx_nic_regs[] = {
1815 REGISTER_AZ(ADR_REGION),
1816 REGISTER_AZ(INT_EN_KER),
1817 REGISTER_BZ(INT_EN_CHAR),
1818 REGISTER_AZ(INT_ADR_KER),
1819 REGISTER_BZ(INT_ADR_CHAR),
1820 /* INT_ACK_KER is WO */
1821 /* INT_ISR0 is RC */
1822 REGISTER_AZ(HW_INIT),
1823 REGISTER_CZ(USR_EV_CFG),
1824 REGISTER_AB(EE_SPI_HCMD),
1825 REGISTER_AB(EE_SPI_HADR),
1826 REGISTER_AB(EE_SPI_HDATA),
1827 REGISTER_AB(EE_BASE_PAGE),
1828 REGISTER_AB(EE_VPD_CFG0),
1829 /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */
1830 /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */
1831 /* PCIE_CORE_INDIRECT is indirect */
1832 REGISTER_AB(NIC_STAT),
1833 REGISTER_AB(GPIO_CTL),
1834 REGISTER_AB(GLB_CTL),
1835 /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */
1836 REGISTER_BZ(DP_CTRL),
1837 REGISTER_AZ(MEM_STAT),
1838 REGISTER_AZ(CS_DEBUG),
1839 REGISTER_AZ(ALTERA_BUILD),
1840 REGISTER_AZ(CSR_SPARE),
1841 REGISTER_AB(PCIE_SD_CTL0123),
1842 REGISTER_AB(PCIE_SD_CTL45),
1843 REGISTER_AB(PCIE_PCS_CTL_STAT),
1844 /* DEBUG_DATA_OUT is not used */
1845 /* DRV_EV is WO */
1846 REGISTER_AZ(EVQ_CTL),
1847 REGISTER_AZ(EVQ_CNT1),
1848 REGISTER_AZ(EVQ_CNT2),
1849 REGISTER_AZ(BUF_TBL_CFG),
1850 REGISTER_AZ(SRM_RX_DC_CFG),
1851 REGISTER_AZ(SRM_TX_DC_CFG),
1852 REGISTER_AZ(SRM_CFG),
1853 /* BUF_TBL_UPD is WO */
1854 REGISTER_AZ(SRM_UPD_EVQ),
1855 REGISTER_AZ(SRAM_PARITY),
1856 REGISTER_AZ(RX_CFG),
1857 REGISTER_BZ(RX_FILTER_CTL),
1858 /* RX_FLUSH_DESCQ is WO */
1859 REGISTER_AZ(RX_DC_CFG),
1860 REGISTER_AZ(RX_DC_PF_WM),
1861 REGISTER_BZ(RX_RSS_TKEY),
1862 /* RX_NODESC_DROP is RC */
1863 REGISTER_AA(RX_SELF_RST),
1864 /* RX_DEBUG, RX_PUSH_DROP are not used */
1865 REGISTER_CZ(RX_RSS_IPV6_REG1),
1866 REGISTER_CZ(RX_RSS_IPV6_REG2),
1867 REGISTER_CZ(RX_RSS_IPV6_REG3),
1868 /* TX_FLUSH_DESCQ is WO */
1869 REGISTER_AZ(TX_DC_CFG),
1870 REGISTER_AA(TX_CHKSM_CFG),
1871 REGISTER_AZ(TX_CFG),
1872 /* TX_PUSH_DROP is not used */
1873 REGISTER_AZ(TX_RESERVED),
1874 REGISTER_BZ(TX_PACE),
1875 /* TX_PACE_DROP_QID is RC */
1876 REGISTER_BB(TX_VLAN),
1877 REGISTER_BZ(TX_IPFIL_PORTEN),
1878 REGISTER_AB(MD_TXD),
1879 REGISTER_AB(MD_RXD),
1880 REGISTER_AB(MD_CS),
1881 REGISTER_AB(MD_PHY_ADR),
1882 REGISTER_AB(MD_ID),
1883 /* MD_STAT is RC */
1884 REGISTER_AB(MAC_STAT_DMA),
1885 REGISTER_AB(MAC_CTRL),
1886 REGISTER_BB(GEN_MODE),
1887 REGISTER_AB(MAC_MC_HASH_REG0),
1888 REGISTER_AB(MAC_MC_HASH_REG1),
1889 REGISTER_AB(GM_CFG1),
1890 REGISTER_AB(GM_CFG2),
1891 /* GM_IPG and GM_HD are not used */
1892 REGISTER_AB(GM_MAX_FLEN),
1893 /* GM_TEST is not used */
1894 REGISTER_AB(GM_ADR1),
1895 REGISTER_AB(GM_ADR2),
1896 REGISTER_AB(GMF_CFG0),
1897 REGISTER_AB(GMF_CFG1),
1898 REGISTER_AB(GMF_CFG2),
1899 REGISTER_AB(GMF_CFG3),
1900 REGISTER_AB(GMF_CFG4),
1901 REGISTER_AB(GMF_CFG5),
1902 REGISTER_BB(TX_SRC_MAC_CTL),
1903 REGISTER_AB(XM_ADR_LO),
1904 REGISTER_AB(XM_ADR_HI),
1905 REGISTER_AB(XM_GLB_CFG),
1906 REGISTER_AB(XM_TX_CFG),
1907 REGISTER_AB(XM_RX_CFG),
1908 REGISTER_AB(XM_MGT_INT_MASK),
1909 REGISTER_AB(XM_FC),
1910 REGISTER_AB(XM_PAUSE_TIME),
1911 REGISTER_AB(XM_TX_PARAM),
1912 REGISTER_AB(XM_RX_PARAM),
1913 /* XM_MGT_INT_MSK (note no 'A') is RC */
1914 REGISTER_AB(XX_PWR_RST),
1915 REGISTER_AB(XX_SD_CTL),
1916 REGISTER_AB(XX_TXDRV_CTL),
1917 /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */
1918 /* XX_CORE_STAT is partly RC */
1919};
1920
1921struct efx_nic_reg_table {
1922 u32 offset:24;
1923 u32 min_revision:2, max_revision:2;
1924 u32 step:6, rows:21;
1925};
1926
1927#define REGISTER_TABLE_DIMENSIONS(_, offset, min_rev, max_rev, step, rows) { \
1928 offset, \
1929 REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev, \
1930 step, rows \
1931}
1932#define REGISTER_TABLE(name, min_rev, max_rev) \
1933 REGISTER_TABLE_DIMENSIONS( \
1934 name, FR_ ## min_rev ## max_rev ## _ ## name, \
1935 min_rev, max_rev, \
1936 FR_ ## min_rev ## max_rev ## _ ## name ## _STEP, \
1937 FR_ ## min_rev ## max_rev ## _ ## name ## _ROWS)
1938#define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, A, A)
1939#define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, A, Z)
1940#define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, B, B)
1941#define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, B, Z)
1942#define REGISTER_TABLE_BB_CZ(name) \
1943 REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, B, B, \
1944 FR_BZ_ ## name ## _STEP, \
1945 FR_BB_ ## name ## _ROWS), \
1946 REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, C, Z, \
1947 FR_BZ_ ## name ## _STEP, \
1948 FR_CZ_ ## name ## _ROWS)
1949#define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, C, Z)
1950
1951static const struct efx_nic_reg_table efx_nic_reg_tables[] = {
1952 /* DRIVER is not used */
1953 /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */
1954 REGISTER_TABLE_BB(TX_IPFIL_TBL),
1955 REGISTER_TABLE_BB(TX_SRC_MAC_TBL),
1956 REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),
1957 REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),
1958 REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),
1959 REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),
1960 REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),
1961 REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),
1962 /* We can't reasonably read all of the buffer table (up to 8MB!).
1963 * However this driver will only use a few entries. Reading
1964 * 1K entries allows for some expansion of queue count and
1965 * size before we need to change the version. */
1966 REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,
1967 A, A, 8, 1024),
1968 REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,
1969 B, Z, 8, 1024),
1970 REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),
1971 REGISTER_TABLE_BB_CZ(TIMER_TBL),
1972 REGISTER_TABLE_BB_CZ(TX_PACE_TBL),
1973 REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),
1974 /* TX_FILTER_TBL0 is huge and not used by this driver */
1975 REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),
1976 REGISTER_TABLE_CZ(MC_TREG_SMEM),
1977 /* MSIX_PBA_TABLE is not mapped */
1978 /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */
1979 REGISTER_TABLE_BZ(RX_FILTER_TBL0),
1980};
1981
1982size_t efx_nic_get_regs_len(struct efx_nic *efx)
1983{
1984 const struct efx_nic_reg *reg;
1985 const struct efx_nic_reg_table *table;
1986 size_t len = 0;
1987
1988 for (reg = efx_nic_regs;
1989 reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
1990 reg++)
1991 if (efx->type->revision >= reg->min_revision &&
1992 efx->type->revision <= reg->max_revision)
1993 len += sizeof(efx_oword_t);
1994
1995 for (table = efx_nic_reg_tables;
1996 table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
1997 table++)
1998 if (efx->type->revision >= table->min_revision &&
1999 efx->type->revision <= table->max_revision)
2000 len += table->rows * min_t(size_t, table->step, 16);
2001
2002 return len;
2003}
2004
2005void efx_nic_get_regs(struct efx_nic *efx, void *buf)
2006{
2007 const struct efx_nic_reg *reg;
2008 const struct efx_nic_reg_table *table;
2009
2010 for (reg = efx_nic_regs;
2011 reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
2012 reg++) {
2013 if (efx->type->revision >= reg->min_revision &&
2014 efx->type->revision <= reg->max_revision) {
2015 efx_reado(efx, (efx_oword_t *)buf, reg->offset);
2016 buf += sizeof(efx_oword_t);
2017 }
2018 }
2019
2020 for (table = efx_nic_reg_tables;
2021 table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
2022 table++) {
2023 size_t size, i;
2024
2025 if (!(efx->type->revision >= table->min_revision &&
2026 efx->type->revision <= table->max_revision))
2027 continue;
2028
2029 size = min_t(size_t, table->step, 16);
2030
2031 for (i = 0; i < table->rows; i++) {
2032 switch (table->step) {
2033 case 4: /* 32-bit register or SRAM */
2034 efx_readd_table(efx, buf, table->offset, i);
2035 break;
2036 case 8: /* 64-bit SRAM */
2037 efx_sram_readq(efx,
2038 efx->membase + table->offset,
2039 buf, i);
2040 break;
2041 case 16: /* 128-bit register */
2042 efx_reado_table(efx, buf, table->offset, i);
2043 break;
2044 case 32: /* 128-bit register, interleaved */
2045 efx_reado_table(efx, buf, table->offset, 2 * i);
2046 break;
2047 default:
2048 WARN_ON(1);
2049 return;
2050 }
2051 buf += size;
2052 }
2053 }
2054}