Loading...
Note: File does not exist in v3.15.
1/* Broadcom NetXtreme-C/E network driver.
2 *
3 * Copyright (c) 2014-2016 Broadcom Corporation
4 * Copyright (c) 2016-2019 Broadcom Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 */
10
11#include <linux/module.h>
12
13#include <linux/stringify.h>
14#include <linux/kernel.h>
15#include <linux/timer.h>
16#include <linux/errno.h>
17#include <linux/ioport.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
20#include <linux/interrupt.h>
21#include <linux/pci.h>
22#include <linux/netdevice.h>
23#include <linux/etherdevice.h>
24#include <linux/skbuff.h>
25#include <linux/dma-mapping.h>
26#include <linux/bitops.h>
27#include <linux/io.h>
28#include <linux/irq.h>
29#include <linux/delay.h>
30#include <asm/byteorder.h>
31#include <asm/page.h>
32#include <linux/time.h>
33#include <linux/mii.h>
34#include <linux/mdio.h>
35#include <linux/if.h>
36#include <linux/if_vlan.h>
37#include <linux/if_bridge.h>
38#include <linux/rtc.h>
39#include <linux/bpf.h>
40#include <net/gro.h>
41#include <net/ip.h>
42#include <net/tcp.h>
43#include <net/udp.h>
44#include <net/checksum.h>
45#include <net/ip6_checksum.h>
46#include <net/udp_tunnel.h>
47#include <linux/workqueue.h>
48#include <linux/prefetch.h>
49#include <linux/cache.h>
50#include <linux/log2.h>
51#include <linux/bitmap.h>
52#include <linux/cpu_rmap.h>
53#include <linux/cpumask.h>
54#include <net/pkt_cls.h>
55#include <net/page_pool/helpers.h>
56#include <linux/align.h>
57#include <net/netdev_queues.h>
58
59#include "bnxt_hsi.h"
60#include "bnxt.h"
61#include "bnxt_hwrm.h"
62#include "bnxt_ulp.h"
63#include "bnxt_sriov.h"
64#include "bnxt_ethtool.h"
65#include "bnxt_dcb.h"
66#include "bnxt_xdp.h"
67#include "bnxt_ptp.h"
68#include "bnxt_vfr.h"
69#include "bnxt_tc.h"
70#include "bnxt_devlink.h"
71#include "bnxt_debugfs.h"
72#include "bnxt_hwmon.h"
73
74#define BNXT_TX_TIMEOUT (5 * HZ)
75#define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \
76 NETIF_MSG_TX_ERR)
77
78MODULE_LICENSE("GPL");
79MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
80
81#define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
82#define BNXT_RX_DMA_OFFSET NET_SKB_PAD
83#define BNXT_RX_COPY_THRESH 256
84
85#define BNXT_TX_PUSH_THRESH 164
86
87/* indexed by enum board_idx */
88static const struct {
89 char *name;
90} board_info[] = {
91 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
92 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
93 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
94 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
95 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
96 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
97 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
98 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
99 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
100 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
101 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
102 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
103 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
104 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
105 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
106 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
107 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
108 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
109 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
110 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
111 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
112 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
113 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
114 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
115 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
116 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
117 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
118 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
119 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
120 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
121 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
122 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
123 [BCM57608] = { "Broadcom BCM57608 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
124 [BCM57604] = { "Broadcom BCM57604 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
125 [BCM57602] = { "Broadcom BCM57602 NetXtreme-E 10Gb/25Gb/50Gb/100Gb Ethernet" },
126 [BCM57601] = { "Broadcom BCM57601 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
127 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
128 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
129 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
130 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
131 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
132 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
133 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
134 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
135 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
136 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
137 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
138 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
139 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
140};
141
142static const struct pci_device_id bnxt_pci_tbl[] = {
143 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
144 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
145 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
146 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
147 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
148 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
149 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
150 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
151 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
152 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
153 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
154 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
155 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
156 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
157 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
158 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
159 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
160 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
161 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
162 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
163 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
164 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
165 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
166 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
167 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
168 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
169 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
170 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
171 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
172 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
173 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
174 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
175 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
176 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
177 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
178 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
179 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
180 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
181 { PCI_VDEVICE(BROADCOM, 0x1760), .driver_data = BCM57608 },
182 { PCI_VDEVICE(BROADCOM, 0x1761), .driver_data = BCM57604 },
183 { PCI_VDEVICE(BROADCOM, 0x1762), .driver_data = BCM57602 },
184 { PCI_VDEVICE(BROADCOM, 0x1763), .driver_data = BCM57601 },
185 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
186 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
187 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
188 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
189 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
190 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
191 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
192 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
193#ifdef CONFIG_BNXT_SRIOV
194 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
195 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
196 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
197 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
198 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
199 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
200 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
201 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
202 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
203 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
204 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
205 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
206 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
207 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
208 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
209 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
210 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
211 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
212 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
213 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
214 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
215#endif
216 { 0 }
217};
218
219MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
220
221static const u16 bnxt_vf_req_snif[] = {
222 HWRM_FUNC_CFG,
223 HWRM_FUNC_VF_CFG,
224 HWRM_PORT_PHY_QCFG,
225 HWRM_CFA_L2_FILTER_ALLOC,
226};
227
228static const u16 bnxt_async_events_arr[] = {
229 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
230 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
231 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
232 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
233 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
234 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
235 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
236 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
237 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
238 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
239 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
240 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
241 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
242 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
243 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
244 ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
245};
246
247static struct workqueue_struct *bnxt_pf_wq;
248
249static bool bnxt_vf_pciid(enum board_idx idx)
250{
251 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
252 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
253 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
254 idx == NETXTREME_E_P5_VF_HV);
255}
256
257#define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
258#define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
259#define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
260
261#define BNXT_CP_DB_IRQ_DIS(db) \
262 writel(DB_CP_IRQ_DIS_FLAGS, db)
263
264#define BNXT_DB_CQ(db, idx) \
265 writel(DB_CP_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
266
267#define BNXT_DB_NQ_P5(db, idx) \
268 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | DB_RING_IDX(db, idx),\
269 (db)->doorbell)
270
271#define BNXT_DB_NQ_P7(db, idx) \
272 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_MASK | \
273 DB_RING_IDX(db, idx), (db)->doorbell)
274
275#define BNXT_DB_CQ_ARM(db, idx) \
276 writel(DB_CP_REARM_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
277
278#define BNXT_DB_NQ_ARM_P5(db, idx) \
279 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | \
280 DB_RING_IDX(db, idx), (db)->doorbell)
281
282static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
283{
284 if (bp->flags & BNXT_FLAG_CHIP_P7)
285 BNXT_DB_NQ_P7(db, idx);
286 else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
287 BNXT_DB_NQ_P5(db, idx);
288 else
289 BNXT_DB_CQ(db, idx);
290}
291
292static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
293{
294 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
295 BNXT_DB_NQ_ARM_P5(db, idx);
296 else
297 BNXT_DB_CQ_ARM(db, idx);
298}
299
300static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
301{
302 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
303 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
304 DB_RING_IDX(db, idx), db->doorbell);
305 else
306 BNXT_DB_CQ(db, idx);
307}
308
309static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
310{
311 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
312 return;
313
314 if (BNXT_PF(bp))
315 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
316 else
317 schedule_delayed_work(&bp->fw_reset_task, delay);
318}
319
320static void __bnxt_queue_sp_work(struct bnxt *bp)
321{
322 if (BNXT_PF(bp))
323 queue_work(bnxt_pf_wq, &bp->sp_task);
324 else
325 schedule_work(&bp->sp_task);
326}
327
328static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event)
329{
330 set_bit(event, &bp->sp_event);
331 __bnxt_queue_sp_work(bp);
332}
333
334static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
335{
336 if (!rxr->bnapi->in_reset) {
337 rxr->bnapi->in_reset = true;
338 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
339 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
340 else
341 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
342 __bnxt_queue_sp_work(bp);
343 }
344 rxr->rx_next_cons = 0xffff;
345}
346
347void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
348 u16 curr)
349{
350 struct bnxt_napi *bnapi = txr->bnapi;
351
352 if (bnapi->tx_fault)
353 return;
354
355 netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)",
356 txr->txq_index, txr->tx_hw_cons,
357 txr->tx_cons, txr->tx_prod, curr);
358 WARN_ON_ONCE(1);
359 bnapi->tx_fault = 1;
360 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
361}
362
363const u16 bnxt_lhint_arr[] = {
364 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
365 TX_BD_FLAGS_LHINT_512_TO_1023,
366 TX_BD_FLAGS_LHINT_1024_TO_2047,
367 TX_BD_FLAGS_LHINT_1024_TO_2047,
368 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
369 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
370 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
371 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
372 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
373 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
374 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
375 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
376 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
377 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
378 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
379 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
380 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
381 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
382 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
383};
384
385static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
386{
387 struct metadata_dst *md_dst = skb_metadata_dst(skb);
388
389 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
390 return 0;
391
392 return md_dst->u.port_info.port_id;
393}
394
395static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
396 u16 prod)
397{
398 /* Sync BD data before updating doorbell */
399 wmb();
400 bnxt_db_write(bp, &txr->tx_db, prod);
401 txr->kick_pending = 0;
402}
403
404static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
405{
406 struct bnxt *bp = netdev_priv(dev);
407 struct tx_bd *txbd, *txbd0;
408 struct tx_bd_ext *txbd1;
409 struct netdev_queue *txq;
410 int i;
411 dma_addr_t mapping;
412 unsigned int length, pad = 0;
413 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
414 u16 prod, last_frag;
415 struct pci_dev *pdev = bp->pdev;
416 struct bnxt_tx_ring_info *txr;
417 struct bnxt_sw_tx_bd *tx_buf;
418 __le32 lflags = 0;
419
420 i = skb_get_queue_mapping(skb);
421 if (unlikely(i >= bp->tx_nr_rings)) {
422 dev_kfree_skb_any(skb);
423 dev_core_stats_tx_dropped_inc(dev);
424 return NETDEV_TX_OK;
425 }
426
427 txq = netdev_get_tx_queue(dev, i);
428 txr = &bp->tx_ring[bp->tx_ring_map[i]];
429 prod = txr->tx_prod;
430
431 free_size = bnxt_tx_avail(bp, txr);
432 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
433 /* We must have raced with NAPI cleanup */
434 if (net_ratelimit() && txr->kick_pending)
435 netif_warn(bp, tx_err, dev,
436 "bnxt: ring busy w/ flush pending!\n");
437 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
438 bp->tx_wake_thresh))
439 return NETDEV_TX_BUSY;
440 }
441
442 if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
443 goto tx_free;
444
445 length = skb->len;
446 len = skb_headlen(skb);
447 last_frag = skb_shinfo(skb)->nr_frags;
448
449 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
450
451 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
452 tx_buf->skb = skb;
453 tx_buf->nr_frags = last_frag;
454
455 vlan_tag_flags = 0;
456 cfa_action = bnxt_xmit_get_cfa_action(skb);
457 if (skb_vlan_tag_present(skb)) {
458 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
459 skb_vlan_tag_get(skb);
460 /* Currently supports 8021Q, 8021AD vlan offloads
461 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
462 */
463 if (skb->vlan_proto == htons(ETH_P_8021Q))
464 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
465 }
466
467 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
468 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
469
470 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
471 atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
472 if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
473 &ptp->tx_hdr_off)) {
474 if (vlan_tag_flags)
475 ptp->tx_hdr_off += VLAN_HLEN;
476 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
477 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
478 } else {
479 atomic_inc(&bp->ptp_cfg->tx_avail);
480 }
481 }
482 }
483
484 if (unlikely(skb->no_fcs))
485 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
486
487 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
488 !lflags) {
489 struct tx_push_buffer *tx_push_buf = txr->tx_push;
490 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
491 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
492 void __iomem *db = txr->tx_db.doorbell;
493 void *pdata = tx_push_buf->data;
494 u64 *end;
495 int j, push_len;
496
497 /* Set COAL_NOW to be ready quickly for the next push */
498 tx_push->tx_bd_len_flags_type =
499 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
500 TX_BD_TYPE_LONG_TX_BD |
501 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
502 TX_BD_FLAGS_COAL_NOW |
503 TX_BD_FLAGS_PACKET_END |
504 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
505
506 if (skb->ip_summed == CHECKSUM_PARTIAL)
507 tx_push1->tx_bd_hsize_lflags =
508 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
509 else
510 tx_push1->tx_bd_hsize_lflags = 0;
511
512 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
513 tx_push1->tx_bd_cfa_action =
514 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
515
516 end = pdata + length;
517 end = PTR_ALIGN(end, 8) - 1;
518 *end = 0;
519
520 skb_copy_from_linear_data(skb, pdata, len);
521 pdata += len;
522 for (j = 0; j < last_frag; j++) {
523 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
524 void *fptr;
525
526 fptr = skb_frag_address_safe(frag);
527 if (!fptr)
528 goto normal_tx;
529
530 memcpy(pdata, fptr, skb_frag_size(frag));
531 pdata += skb_frag_size(frag);
532 }
533
534 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
535 txbd->tx_bd_haddr = txr->data_mapping;
536 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2);
537 prod = NEXT_TX(prod);
538 tx_push->tx_bd_opaque = txbd->tx_bd_opaque;
539 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
540 memcpy(txbd, tx_push1, sizeof(*txbd));
541 prod = NEXT_TX(prod);
542 tx_push->doorbell =
543 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH |
544 DB_RING_IDX(&txr->tx_db, prod));
545 WRITE_ONCE(txr->tx_prod, prod);
546
547 tx_buf->is_push = 1;
548 netdev_tx_sent_queue(txq, skb->len);
549 wmb(); /* Sync is_push and byte queue before pushing data */
550
551 push_len = (length + sizeof(*tx_push) + 7) / 8;
552 if (push_len > 16) {
553 __iowrite64_copy(db, tx_push_buf, 16);
554 __iowrite32_copy(db + 4, tx_push_buf + 1,
555 (push_len - 16) << 1);
556 } else {
557 __iowrite64_copy(db, tx_push_buf, push_len);
558 }
559
560 goto tx_done;
561 }
562
563normal_tx:
564 if (length < BNXT_MIN_PKT_SIZE) {
565 pad = BNXT_MIN_PKT_SIZE - length;
566 if (skb_pad(skb, pad))
567 /* SKB already freed. */
568 goto tx_kick_pending;
569 length = BNXT_MIN_PKT_SIZE;
570 }
571
572 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
573
574 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
575 goto tx_free;
576
577 dma_unmap_addr_set(tx_buf, mapping, mapping);
578 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
579 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
580
581 txbd->tx_bd_haddr = cpu_to_le64(mapping);
582 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag);
583
584 prod = NEXT_TX(prod);
585 txbd1 = (struct tx_bd_ext *)
586 &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
587
588 txbd1->tx_bd_hsize_lflags = lflags;
589 if (skb_is_gso(skb)) {
590 bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4);
591 u32 hdr_len;
592
593 if (skb->encapsulation) {
594 if (udp_gso)
595 hdr_len = skb_inner_transport_offset(skb) +
596 sizeof(struct udphdr);
597 else
598 hdr_len = skb_inner_tcp_all_headers(skb);
599 } else if (udp_gso) {
600 hdr_len = skb_transport_offset(skb) +
601 sizeof(struct udphdr);
602 } else {
603 hdr_len = skb_tcp_all_headers(skb);
604 }
605
606 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
607 TX_BD_FLAGS_T_IPID |
608 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
609 length = skb_shinfo(skb)->gso_size;
610 txbd1->tx_bd_mss = cpu_to_le32(length);
611 length += hdr_len;
612 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
613 txbd1->tx_bd_hsize_lflags |=
614 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
615 txbd1->tx_bd_mss = 0;
616 }
617
618 length >>= 9;
619 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
620 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
621 skb->len);
622 i = 0;
623 goto tx_dma_error;
624 }
625 flags |= bnxt_lhint_arr[length];
626 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
627
628 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
629 txbd1->tx_bd_cfa_action =
630 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
631 txbd0 = txbd;
632 for (i = 0; i < last_frag; i++) {
633 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
634
635 prod = NEXT_TX(prod);
636 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
637
638 len = skb_frag_size(frag);
639 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
640 DMA_TO_DEVICE);
641
642 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
643 goto tx_dma_error;
644
645 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
646 dma_unmap_addr_set(tx_buf, mapping, mapping);
647
648 txbd->tx_bd_haddr = cpu_to_le64(mapping);
649
650 flags = len << TX_BD_LEN_SHIFT;
651 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
652 }
653
654 flags &= ~TX_BD_LEN;
655 txbd->tx_bd_len_flags_type =
656 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
657 TX_BD_FLAGS_PACKET_END);
658
659 netdev_tx_sent_queue(txq, skb->len);
660
661 skb_tx_timestamp(skb);
662
663 prod = NEXT_TX(prod);
664 WRITE_ONCE(txr->tx_prod, prod);
665
666 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) {
667 bnxt_txr_db_kick(bp, txr, prod);
668 } else {
669 if (free_size >= bp->tx_wake_thresh)
670 txbd0->tx_bd_len_flags_type |=
671 cpu_to_le32(TX_BD_FLAGS_NO_CMPL);
672 txr->kick_pending = 1;
673 }
674
675tx_done:
676
677 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
678 if (netdev_xmit_more() && !tx_buf->is_push) {
679 txbd0->tx_bd_len_flags_type &=
680 cpu_to_le32(~TX_BD_FLAGS_NO_CMPL);
681 bnxt_txr_db_kick(bp, txr, prod);
682 }
683
684 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
685 bp->tx_wake_thresh);
686 }
687 return NETDEV_TX_OK;
688
689tx_dma_error:
690 if (BNXT_TX_PTP_IS_SET(lflags))
691 atomic_inc(&bp->ptp_cfg->tx_avail);
692
693 last_frag = i;
694
695 /* start back at beginning and unmap skb */
696 prod = txr->tx_prod;
697 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
698 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
699 skb_headlen(skb), DMA_TO_DEVICE);
700 prod = NEXT_TX(prod);
701
702 /* unmap remaining mapped pages */
703 for (i = 0; i < last_frag; i++) {
704 prod = NEXT_TX(prod);
705 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
706 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
707 skb_frag_size(&skb_shinfo(skb)->frags[i]),
708 DMA_TO_DEVICE);
709 }
710
711tx_free:
712 dev_kfree_skb_any(skb);
713tx_kick_pending:
714 if (txr->kick_pending)
715 bnxt_txr_db_kick(bp, txr, txr->tx_prod);
716 txr->tx_buf_ring[txr->tx_prod].skb = NULL;
717 dev_core_stats_tx_dropped_inc(dev);
718 return NETDEV_TX_OK;
719}
720
721static void __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
722 int budget)
723{
724 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
725 struct pci_dev *pdev = bp->pdev;
726 u16 hw_cons = txr->tx_hw_cons;
727 unsigned int tx_bytes = 0;
728 u16 cons = txr->tx_cons;
729 int tx_pkts = 0;
730
731 while (RING_TX(bp, cons) != hw_cons) {
732 struct bnxt_sw_tx_bd *tx_buf;
733 struct sk_buff *skb;
734 int j, last;
735
736 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
737 cons = NEXT_TX(cons);
738 skb = tx_buf->skb;
739 tx_buf->skb = NULL;
740
741 if (unlikely(!skb)) {
742 bnxt_sched_reset_txr(bp, txr, cons);
743 return;
744 }
745
746 tx_pkts++;
747 tx_bytes += skb->len;
748
749 if (tx_buf->is_push) {
750 tx_buf->is_push = 0;
751 goto next_tx_int;
752 }
753
754 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
755 skb_headlen(skb), DMA_TO_DEVICE);
756 last = tx_buf->nr_frags;
757
758 for (j = 0; j < last; j++) {
759 cons = NEXT_TX(cons);
760 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
761 dma_unmap_page(
762 &pdev->dev,
763 dma_unmap_addr(tx_buf, mapping),
764 skb_frag_size(&skb_shinfo(skb)->frags[j]),
765 DMA_TO_DEVICE);
766 }
767 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
768 if (BNXT_CHIP_P5(bp)) {
769 /* PTP worker takes ownership of the skb */
770 if (!bnxt_get_tx_ts_p5(bp, skb))
771 skb = NULL;
772 else
773 atomic_inc(&bp->ptp_cfg->tx_avail);
774 }
775 }
776
777next_tx_int:
778 cons = NEXT_TX(cons);
779
780 dev_consume_skb_any(skb);
781 }
782
783 WRITE_ONCE(txr->tx_cons, cons);
784
785 __netif_txq_completed_wake(txq, tx_pkts, tx_bytes,
786 bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
787 READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
788}
789
790static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
791{
792 struct bnxt_tx_ring_info *txr;
793 int i;
794
795 bnxt_for_each_napi_tx(i, bnapi, txr) {
796 if (txr->tx_hw_cons != RING_TX(bp, txr->tx_cons))
797 __bnxt_tx_int(bp, txr, budget);
798 }
799 bnapi->events &= ~BNXT_TX_CMP_EVENT;
800}
801
802static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
803 struct bnxt_rx_ring_info *rxr,
804 unsigned int *offset,
805 gfp_t gfp)
806{
807 struct page *page;
808
809 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
810 page = page_pool_dev_alloc_frag(rxr->page_pool, offset,
811 BNXT_RX_PAGE_SIZE);
812 } else {
813 page = page_pool_dev_alloc_pages(rxr->page_pool);
814 *offset = 0;
815 }
816 if (!page)
817 return NULL;
818
819 *mapping = page_pool_get_dma_addr(page) + *offset;
820 return page;
821}
822
823static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
824 gfp_t gfp)
825{
826 u8 *data;
827 struct pci_dev *pdev = bp->pdev;
828
829 if (gfp == GFP_ATOMIC)
830 data = napi_alloc_frag(bp->rx_buf_size);
831 else
832 data = netdev_alloc_frag(bp->rx_buf_size);
833 if (!data)
834 return NULL;
835
836 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
837 bp->rx_buf_use_size, bp->rx_dir,
838 DMA_ATTR_WEAK_ORDERING);
839
840 if (dma_mapping_error(&pdev->dev, *mapping)) {
841 skb_free_frag(data);
842 data = NULL;
843 }
844 return data;
845}
846
847int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
848 u16 prod, gfp_t gfp)
849{
850 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
851 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
852 dma_addr_t mapping;
853
854 if (BNXT_RX_PAGE_MODE(bp)) {
855 unsigned int offset;
856 struct page *page =
857 __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
858
859 if (!page)
860 return -ENOMEM;
861
862 mapping += bp->rx_dma_offset;
863 rx_buf->data = page;
864 rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset;
865 } else {
866 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
867
868 if (!data)
869 return -ENOMEM;
870
871 rx_buf->data = data;
872 rx_buf->data_ptr = data + bp->rx_offset;
873 }
874 rx_buf->mapping = mapping;
875
876 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
877 return 0;
878}
879
880void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
881{
882 u16 prod = rxr->rx_prod;
883 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
884 struct bnxt *bp = rxr->bnapi->bp;
885 struct rx_bd *cons_bd, *prod_bd;
886
887 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
888 cons_rx_buf = &rxr->rx_buf_ring[cons];
889
890 prod_rx_buf->data = data;
891 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
892
893 prod_rx_buf->mapping = cons_rx_buf->mapping;
894
895 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
896 cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)];
897
898 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
899}
900
901static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
902{
903 u16 next, max = rxr->rx_agg_bmap_size;
904
905 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
906 if (next >= max)
907 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
908 return next;
909}
910
911static inline int bnxt_alloc_rx_page(struct bnxt *bp,
912 struct bnxt_rx_ring_info *rxr,
913 u16 prod, gfp_t gfp)
914{
915 struct rx_bd *rxbd =
916 &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
917 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
918 struct page *page;
919 dma_addr_t mapping;
920 u16 sw_prod = rxr->rx_sw_agg_prod;
921 unsigned int offset = 0;
922
923 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
924
925 if (!page)
926 return -ENOMEM;
927
928 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
929 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
930
931 __set_bit(sw_prod, rxr->rx_agg_bmap);
932 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
933 rxr->rx_sw_agg_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
934
935 rx_agg_buf->page = page;
936 rx_agg_buf->offset = offset;
937 rx_agg_buf->mapping = mapping;
938 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
939 rxbd->rx_bd_opaque = sw_prod;
940 return 0;
941}
942
943static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
944 struct bnxt_cp_ring_info *cpr,
945 u16 cp_cons, u16 curr)
946{
947 struct rx_agg_cmp *agg;
948
949 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
950 agg = (struct rx_agg_cmp *)
951 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
952 return agg;
953}
954
955static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
956 struct bnxt_rx_ring_info *rxr,
957 u16 agg_id, u16 curr)
958{
959 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
960
961 return &tpa_info->agg_arr[curr];
962}
963
964static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
965 u16 start, u32 agg_bufs, bool tpa)
966{
967 struct bnxt_napi *bnapi = cpr->bnapi;
968 struct bnxt *bp = bnapi->bp;
969 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
970 u16 prod = rxr->rx_agg_prod;
971 u16 sw_prod = rxr->rx_sw_agg_prod;
972 bool p5_tpa = false;
973 u32 i;
974
975 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
976 p5_tpa = true;
977
978 for (i = 0; i < agg_bufs; i++) {
979 u16 cons;
980 struct rx_agg_cmp *agg;
981 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
982 struct rx_bd *prod_bd;
983 struct page *page;
984
985 if (p5_tpa)
986 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
987 else
988 agg = bnxt_get_agg(bp, cpr, idx, start + i);
989 cons = agg->rx_agg_cmp_opaque;
990 __clear_bit(cons, rxr->rx_agg_bmap);
991
992 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
993 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
994
995 __set_bit(sw_prod, rxr->rx_agg_bmap);
996 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
997 cons_rx_buf = &rxr->rx_agg_ring[cons];
998
999 /* It is possible for sw_prod to be equal to cons, so
1000 * set cons_rx_buf->page to NULL first.
1001 */
1002 page = cons_rx_buf->page;
1003 cons_rx_buf->page = NULL;
1004 prod_rx_buf->page = page;
1005 prod_rx_buf->offset = cons_rx_buf->offset;
1006
1007 prod_rx_buf->mapping = cons_rx_buf->mapping;
1008
1009 prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
1010
1011 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
1012 prod_bd->rx_bd_opaque = sw_prod;
1013
1014 prod = NEXT_RX_AGG(prod);
1015 sw_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
1016 }
1017 rxr->rx_agg_prod = prod;
1018 rxr->rx_sw_agg_prod = sw_prod;
1019}
1020
1021static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
1022 struct bnxt_rx_ring_info *rxr,
1023 u16 cons, void *data, u8 *data_ptr,
1024 dma_addr_t dma_addr,
1025 unsigned int offset_and_len)
1026{
1027 unsigned int len = offset_and_len & 0xffff;
1028 struct page *page = data;
1029 u16 prod = rxr->rx_prod;
1030 struct sk_buff *skb;
1031 int err;
1032
1033 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1034 if (unlikely(err)) {
1035 bnxt_reuse_rx_data(rxr, cons, data);
1036 return NULL;
1037 }
1038 dma_addr -= bp->rx_dma_offset;
1039 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1040 bp->rx_dir);
1041 skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE);
1042 if (!skb) {
1043 page_pool_recycle_direct(rxr->page_pool, page);
1044 return NULL;
1045 }
1046 skb_mark_for_recycle(skb);
1047 skb_reserve(skb, bp->rx_offset);
1048 __skb_put(skb, len);
1049
1050 return skb;
1051}
1052
1053static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1054 struct bnxt_rx_ring_info *rxr,
1055 u16 cons, void *data, u8 *data_ptr,
1056 dma_addr_t dma_addr,
1057 unsigned int offset_and_len)
1058{
1059 unsigned int payload = offset_and_len >> 16;
1060 unsigned int len = offset_and_len & 0xffff;
1061 skb_frag_t *frag;
1062 struct page *page = data;
1063 u16 prod = rxr->rx_prod;
1064 struct sk_buff *skb;
1065 int off, err;
1066
1067 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1068 if (unlikely(err)) {
1069 bnxt_reuse_rx_data(rxr, cons, data);
1070 return NULL;
1071 }
1072 dma_addr -= bp->rx_dma_offset;
1073 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1074 bp->rx_dir);
1075
1076 if (unlikely(!payload))
1077 payload = eth_get_headlen(bp->dev, data_ptr, len);
1078
1079 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1080 if (!skb) {
1081 page_pool_recycle_direct(rxr->page_pool, page);
1082 return NULL;
1083 }
1084
1085 skb_mark_for_recycle(skb);
1086 off = (void *)data_ptr - page_address(page);
1087 skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE);
1088 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1089 payload + NET_IP_ALIGN);
1090
1091 frag = &skb_shinfo(skb)->frags[0];
1092 skb_frag_size_sub(frag, payload);
1093 skb_frag_off_add(frag, payload);
1094 skb->data_len -= payload;
1095 skb->tail += payload;
1096
1097 return skb;
1098}
1099
1100static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1101 struct bnxt_rx_ring_info *rxr, u16 cons,
1102 void *data, u8 *data_ptr,
1103 dma_addr_t dma_addr,
1104 unsigned int offset_and_len)
1105{
1106 u16 prod = rxr->rx_prod;
1107 struct sk_buff *skb;
1108 int err;
1109
1110 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1111 if (unlikely(err)) {
1112 bnxt_reuse_rx_data(rxr, cons, data);
1113 return NULL;
1114 }
1115
1116 skb = napi_build_skb(data, bp->rx_buf_size);
1117 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1118 bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1119 if (!skb) {
1120 skb_free_frag(data);
1121 return NULL;
1122 }
1123
1124 skb_reserve(skb, bp->rx_offset);
1125 skb_put(skb, offset_and_len & 0xffff);
1126 return skb;
1127}
1128
1129static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1130 struct bnxt_cp_ring_info *cpr,
1131 struct skb_shared_info *shinfo,
1132 u16 idx, u32 agg_bufs, bool tpa,
1133 struct xdp_buff *xdp)
1134{
1135 struct bnxt_napi *bnapi = cpr->bnapi;
1136 struct pci_dev *pdev = bp->pdev;
1137 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1138 u16 prod = rxr->rx_agg_prod;
1139 u32 i, total_frag_len = 0;
1140 bool p5_tpa = false;
1141
1142 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
1143 p5_tpa = true;
1144
1145 for (i = 0; i < agg_bufs; i++) {
1146 skb_frag_t *frag = &shinfo->frags[i];
1147 u16 cons, frag_len;
1148 struct rx_agg_cmp *agg;
1149 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1150 struct page *page;
1151 dma_addr_t mapping;
1152
1153 if (p5_tpa)
1154 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1155 else
1156 agg = bnxt_get_agg(bp, cpr, idx, i);
1157 cons = agg->rx_agg_cmp_opaque;
1158 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1159 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1160
1161 cons_rx_buf = &rxr->rx_agg_ring[cons];
1162 skb_frag_fill_page_desc(frag, cons_rx_buf->page,
1163 cons_rx_buf->offset, frag_len);
1164 shinfo->nr_frags = i + 1;
1165 __clear_bit(cons, rxr->rx_agg_bmap);
1166
1167 /* It is possible for bnxt_alloc_rx_page() to allocate
1168 * a sw_prod index that equals the cons index, so we
1169 * need to clear the cons entry now.
1170 */
1171 mapping = cons_rx_buf->mapping;
1172 page = cons_rx_buf->page;
1173 cons_rx_buf->page = NULL;
1174
1175 if (xdp && page_is_pfmemalloc(page))
1176 xdp_buff_set_frag_pfmemalloc(xdp);
1177
1178 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1179 --shinfo->nr_frags;
1180 cons_rx_buf->page = page;
1181
1182 /* Update prod since possibly some pages have been
1183 * allocated already.
1184 */
1185 rxr->rx_agg_prod = prod;
1186 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1187 return 0;
1188 }
1189
1190 dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1191 bp->rx_dir);
1192
1193 total_frag_len += frag_len;
1194 prod = NEXT_RX_AGG(prod);
1195 }
1196 rxr->rx_agg_prod = prod;
1197 return total_frag_len;
1198}
1199
1200static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1201 struct bnxt_cp_ring_info *cpr,
1202 struct sk_buff *skb, u16 idx,
1203 u32 agg_bufs, bool tpa)
1204{
1205 struct skb_shared_info *shinfo = skb_shinfo(skb);
1206 u32 total_frag_len = 0;
1207
1208 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1209 agg_bufs, tpa, NULL);
1210 if (!total_frag_len) {
1211 skb_mark_for_recycle(skb);
1212 dev_kfree_skb(skb);
1213 return NULL;
1214 }
1215
1216 skb->data_len += total_frag_len;
1217 skb->len += total_frag_len;
1218 skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs;
1219 return skb;
1220}
1221
1222static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1223 struct bnxt_cp_ring_info *cpr,
1224 struct xdp_buff *xdp, u16 idx,
1225 u32 agg_bufs, bool tpa)
1226{
1227 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1228 u32 total_frag_len = 0;
1229
1230 if (!xdp_buff_has_frags(xdp))
1231 shinfo->nr_frags = 0;
1232
1233 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1234 idx, agg_bufs, tpa, xdp);
1235 if (total_frag_len) {
1236 xdp_buff_set_frags_flag(xdp);
1237 shinfo->nr_frags = agg_bufs;
1238 shinfo->xdp_frags_size = total_frag_len;
1239 }
1240 return total_frag_len;
1241}
1242
1243static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1244 u8 agg_bufs, u32 *raw_cons)
1245{
1246 u16 last;
1247 struct rx_agg_cmp *agg;
1248
1249 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1250 last = RING_CMP(*raw_cons);
1251 agg = (struct rx_agg_cmp *)
1252 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1253 return RX_AGG_CMP_VALID(agg, *raw_cons);
1254}
1255
1256static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1257 unsigned int len,
1258 dma_addr_t mapping)
1259{
1260 struct bnxt *bp = bnapi->bp;
1261 struct pci_dev *pdev = bp->pdev;
1262 struct sk_buff *skb;
1263
1264 skb = napi_alloc_skb(&bnapi->napi, len);
1265 if (!skb)
1266 return NULL;
1267
1268 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1269 bp->rx_dir);
1270
1271 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1272 len + NET_IP_ALIGN);
1273
1274 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1275 bp->rx_dir);
1276
1277 skb_put(skb, len);
1278 return skb;
1279}
1280
1281static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1282 u32 *raw_cons, void *cmp)
1283{
1284 struct rx_cmp *rxcmp = cmp;
1285 u32 tmp_raw_cons = *raw_cons;
1286 u8 cmp_type, agg_bufs = 0;
1287
1288 cmp_type = RX_CMP_TYPE(rxcmp);
1289
1290 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1291 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1292 RX_CMP_AGG_BUFS) >>
1293 RX_CMP_AGG_BUFS_SHIFT;
1294 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1295 struct rx_tpa_end_cmp *tpa_end = cmp;
1296
1297 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1298 return 0;
1299
1300 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1301 }
1302
1303 if (agg_bufs) {
1304 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1305 return -EBUSY;
1306 }
1307 *raw_cons = tmp_raw_cons;
1308 return 0;
1309}
1310
1311static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1312{
1313 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1314 u16 idx = agg_id & MAX_TPA_P5_MASK;
1315
1316 if (test_bit(idx, map->agg_idx_bmap))
1317 idx = find_first_zero_bit(map->agg_idx_bmap,
1318 BNXT_AGG_IDX_BMAP_SIZE);
1319 __set_bit(idx, map->agg_idx_bmap);
1320 map->agg_id_tbl[agg_id] = idx;
1321 return idx;
1322}
1323
1324static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1325{
1326 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1327
1328 __clear_bit(idx, map->agg_idx_bmap);
1329}
1330
1331static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1332{
1333 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1334
1335 return map->agg_id_tbl[agg_id];
1336}
1337
1338static void bnxt_tpa_metadata(struct bnxt_tpa_info *tpa_info,
1339 struct rx_tpa_start_cmp *tpa_start,
1340 struct rx_tpa_start_cmp_ext *tpa_start1)
1341{
1342 tpa_info->cfa_code_valid = 1;
1343 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1344 tpa_info->vlan_valid = 0;
1345 if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
1346 tpa_info->vlan_valid = 1;
1347 tpa_info->metadata =
1348 le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1349 }
1350}
1351
1352static void bnxt_tpa_metadata_v2(struct bnxt_tpa_info *tpa_info,
1353 struct rx_tpa_start_cmp *tpa_start,
1354 struct rx_tpa_start_cmp_ext *tpa_start1)
1355{
1356 tpa_info->vlan_valid = 0;
1357 if (TPA_START_VLAN_VALID(tpa_start)) {
1358 u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start);
1359 u32 vlan_proto = ETH_P_8021Q;
1360
1361 tpa_info->vlan_valid = 1;
1362 if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD)
1363 vlan_proto = ETH_P_8021AD;
1364 tpa_info->metadata = vlan_proto << 16 |
1365 TPA_START_METADATA0_TCI(tpa_start1);
1366 }
1367}
1368
1369static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1370 u8 cmp_type, struct rx_tpa_start_cmp *tpa_start,
1371 struct rx_tpa_start_cmp_ext *tpa_start1)
1372{
1373 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1374 struct bnxt_tpa_info *tpa_info;
1375 u16 cons, prod, agg_id;
1376 struct rx_bd *prod_bd;
1377 dma_addr_t mapping;
1378
1379 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1380 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1381 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1382 } else {
1383 agg_id = TPA_START_AGG_ID(tpa_start);
1384 }
1385 cons = tpa_start->rx_tpa_start_cmp_opaque;
1386 prod = rxr->rx_prod;
1387 cons_rx_buf = &rxr->rx_buf_ring[cons];
1388 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
1389 tpa_info = &rxr->rx_tpa[agg_id];
1390
1391 if (unlikely(cons != rxr->rx_next_cons ||
1392 TPA_START_ERROR(tpa_start))) {
1393 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1394 cons, rxr->rx_next_cons,
1395 TPA_START_ERROR_CODE(tpa_start1));
1396 bnxt_sched_reset_rxr(bp, rxr);
1397 return;
1398 }
1399 prod_rx_buf->data = tpa_info->data;
1400 prod_rx_buf->data_ptr = tpa_info->data_ptr;
1401
1402 mapping = tpa_info->mapping;
1403 prod_rx_buf->mapping = mapping;
1404
1405 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
1406
1407 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1408
1409 tpa_info->data = cons_rx_buf->data;
1410 tpa_info->data_ptr = cons_rx_buf->data_ptr;
1411 cons_rx_buf->data = NULL;
1412 tpa_info->mapping = cons_rx_buf->mapping;
1413
1414 tpa_info->len =
1415 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1416 RX_TPA_START_CMP_LEN_SHIFT;
1417 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1418 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1419 tpa_info->gso_type = SKB_GSO_TCPV4;
1420 if (TPA_START_IS_IPV6(tpa_start1))
1421 tpa_info->gso_type = SKB_GSO_TCPV6;
1422 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1423 else if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP &&
1424 TPA_START_HASH_TYPE(tpa_start) == 3)
1425 tpa_info->gso_type = SKB_GSO_TCPV6;
1426 tpa_info->rss_hash =
1427 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1428 } else {
1429 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1430 tpa_info->gso_type = 0;
1431 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1432 }
1433 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1434 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1435 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP)
1436 bnxt_tpa_metadata(tpa_info, tpa_start, tpa_start1);
1437 else
1438 bnxt_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1);
1439 tpa_info->agg_count = 0;
1440
1441 rxr->rx_prod = NEXT_RX(prod);
1442 cons = RING_RX(bp, NEXT_RX(cons));
1443 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
1444 cons_rx_buf = &rxr->rx_buf_ring[cons];
1445
1446 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1447 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1448 cons_rx_buf->data = NULL;
1449}
1450
1451static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1452{
1453 if (agg_bufs)
1454 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1455}
1456
1457#ifdef CONFIG_INET
1458static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1459{
1460 struct udphdr *uh = NULL;
1461
1462 if (ip_proto == htons(ETH_P_IP)) {
1463 struct iphdr *iph = (struct iphdr *)skb->data;
1464
1465 if (iph->protocol == IPPROTO_UDP)
1466 uh = (struct udphdr *)(iph + 1);
1467 } else {
1468 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1469
1470 if (iph->nexthdr == IPPROTO_UDP)
1471 uh = (struct udphdr *)(iph + 1);
1472 }
1473 if (uh) {
1474 if (uh->check)
1475 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1476 else
1477 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1478 }
1479}
1480#endif
1481
1482static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1483 int payload_off, int tcp_ts,
1484 struct sk_buff *skb)
1485{
1486#ifdef CONFIG_INET
1487 struct tcphdr *th;
1488 int len, nw_off;
1489 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1490 u32 hdr_info = tpa_info->hdr_info;
1491 bool loopback = false;
1492
1493 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1494 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1495 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1496
1497 /* If the packet is an internal loopback packet, the offsets will
1498 * have an extra 4 bytes.
1499 */
1500 if (inner_mac_off == 4) {
1501 loopback = true;
1502 } else if (inner_mac_off > 4) {
1503 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1504 ETH_HLEN - 2));
1505
1506 /* We only support inner iPv4/ipv6. If we don't see the
1507 * correct protocol ID, it must be a loopback packet where
1508 * the offsets are off by 4.
1509 */
1510 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1511 loopback = true;
1512 }
1513 if (loopback) {
1514 /* internal loopback packet, subtract all offsets by 4 */
1515 inner_ip_off -= 4;
1516 inner_mac_off -= 4;
1517 outer_ip_off -= 4;
1518 }
1519
1520 nw_off = inner_ip_off - ETH_HLEN;
1521 skb_set_network_header(skb, nw_off);
1522 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1523 struct ipv6hdr *iph = ipv6_hdr(skb);
1524
1525 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1526 len = skb->len - skb_transport_offset(skb);
1527 th = tcp_hdr(skb);
1528 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1529 } else {
1530 struct iphdr *iph = ip_hdr(skb);
1531
1532 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1533 len = skb->len - skb_transport_offset(skb);
1534 th = tcp_hdr(skb);
1535 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1536 }
1537
1538 if (inner_mac_off) { /* tunnel */
1539 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1540 ETH_HLEN - 2));
1541
1542 bnxt_gro_tunnel(skb, proto);
1543 }
1544#endif
1545 return skb;
1546}
1547
1548static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1549 int payload_off, int tcp_ts,
1550 struct sk_buff *skb)
1551{
1552#ifdef CONFIG_INET
1553 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1554 u32 hdr_info = tpa_info->hdr_info;
1555 int iphdr_len, nw_off;
1556
1557 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1558 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1559 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1560
1561 nw_off = inner_ip_off - ETH_HLEN;
1562 skb_set_network_header(skb, nw_off);
1563 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1564 sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1565 skb_set_transport_header(skb, nw_off + iphdr_len);
1566
1567 if (inner_mac_off) { /* tunnel */
1568 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1569 ETH_HLEN - 2));
1570
1571 bnxt_gro_tunnel(skb, proto);
1572 }
1573#endif
1574 return skb;
1575}
1576
1577#define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1578#define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1579
1580static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1581 int payload_off, int tcp_ts,
1582 struct sk_buff *skb)
1583{
1584#ifdef CONFIG_INET
1585 struct tcphdr *th;
1586 int len, nw_off, tcp_opt_len = 0;
1587
1588 if (tcp_ts)
1589 tcp_opt_len = 12;
1590
1591 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1592 struct iphdr *iph;
1593
1594 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1595 ETH_HLEN;
1596 skb_set_network_header(skb, nw_off);
1597 iph = ip_hdr(skb);
1598 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1599 len = skb->len - skb_transport_offset(skb);
1600 th = tcp_hdr(skb);
1601 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1602 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1603 struct ipv6hdr *iph;
1604
1605 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1606 ETH_HLEN;
1607 skb_set_network_header(skb, nw_off);
1608 iph = ipv6_hdr(skb);
1609 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1610 len = skb->len - skb_transport_offset(skb);
1611 th = tcp_hdr(skb);
1612 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1613 } else {
1614 dev_kfree_skb_any(skb);
1615 return NULL;
1616 }
1617
1618 if (nw_off) /* tunnel */
1619 bnxt_gro_tunnel(skb, skb->protocol);
1620#endif
1621 return skb;
1622}
1623
1624static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1625 struct bnxt_tpa_info *tpa_info,
1626 struct rx_tpa_end_cmp *tpa_end,
1627 struct rx_tpa_end_cmp_ext *tpa_end1,
1628 struct sk_buff *skb)
1629{
1630#ifdef CONFIG_INET
1631 int payload_off;
1632 u16 segs;
1633
1634 segs = TPA_END_TPA_SEGS(tpa_end);
1635 if (segs == 1)
1636 return skb;
1637
1638 NAPI_GRO_CB(skb)->count = segs;
1639 skb_shinfo(skb)->gso_size =
1640 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1641 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1642 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1643 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1644 else
1645 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1646 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1647 if (likely(skb))
1648 tcp_gro_complete(skb);
1649#endif
1650 return skb;
1651}
1652
1653/* Given the cfa_code of a received packet determine which
1654 * netdev (vf-rep or PF) the packet is destined to.
1655 */
1656static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1657{
1658 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1659
1660 /* if vf-rep dev is NULL, the must belongs to the PF */
1661 return dev ? dev : bp->dev;
1662}
1663
1664static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1665 struct bnxt_cp_ring_info *cpr,
1666 u32 *raw_cons,
1667 struct rx_tpa_end_cmp *tpa_end,
1668 struct rx_tpa_end_cmp_ext *tpa_end1,
1669 u8 *event)
1670{
1671 struct bnxt_napi *bnapi = cpr->bnapi;
1672 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1673 struct net_device *dev = bp->dev;
1674 u8 *data_ptr, agg_bufs;
1675 unsigned int len;
1676 struct bnxt_tpa_info *tpa_info;
1677 dma_addr_t mapping;
1678 struct sk_buff *skb;
1679 u16 idx = 0, agg_id;
1680 void *data;
1681 bool gro;
1682
1683 if (unlikely(bnapi->in_reset)) {
1684 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1685
1686 if (rc < 0)
1687 return ERR_PTR(-EBUSY);
1688 return NULL;
1689 }
1690
1691 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1692 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1693 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1694 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1695 tpa_info = &rxr->rx_tpa[agg_id];
1696 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1697 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1698 agg_bufs, tpa_info->agg_count);
1699 agg_bufs = tpa_info->agg_count;
1700 }
1701 tpa_info->agg_count = 0;
1702 *event |= BNXT_AGG_EVENT;
1703 bnxt_free_agg_idx(rxr, agg_id);
1704 idx = agg_id;
1705 gro = !!(bp->flags & BNXT_FLAG_GRO);
1706 } else {
1707 agg_id = TPA_END_AGG_ID(tpa_end);
1708 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1709 tpa_info = &rxr->rx_tpa[agg_id];
1710 idx = RING_CMP(*raw_cons);
1711 if (agg_bufs) {
1712 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1713 return ERR_PTR(-EBUSY);
1714
1715 *event |= BNXT_AGG_EVENT;
1716 idx = NEXT_CMP(idx);
1717 }
1718 gro = !!TPA_END_GRO(tpa_end);
1719 }
1720 data = tpa_info->data;
1721 data_ptr = tpa_info->data_ptr;
1722 prefetch(data_ptr);
1723 len = tpa_info->len;
1724 mapping = tpa_info->mapping;
1725
1726 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1727 bnxt_abort_tpa(cpr, idx, agg_bufs);
1728 if (agg_bufs > MAX_SKB_FRAGS)
1729 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1730 agg_bufs, (int)MAX_SKB_FRAGS);
1731 return NULL;
1732 }
1733
1734 if (len <= bp->rx_copy_thresh) {
1735 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1736 if (!skb) {
1737 bnxt_abort_tpa(cpr, idx, agg_bufs);
1738 cpr->sw_stats.rx.rx_oom_discards += 1;
1739 return NULL;
1740 }
1741 } else {
1742 u8 *new_data;
1743 dma_addr_t new_mapping;
1744
1745 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1746 if (!new_data) {
1747 bnxt_abort_tpa(cpr, idx, agg_bufs);
1748 cpr->sw_stats.rx.rx_oom_discards += 1;
1749 return NULL;
1750 }
1751
1752 tpa_info->data = new_data;
1753 tpa_info->data_ptr = new_data + bp->rx_offset;
1754 tpa_info->mapping = new_mapping;
1755
1756 skb = napi_build_skb(data, bp->rx_buf_size);
1757 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1758 bp->rx_buf_use_size, bp->rx_dir,
1759 DMA_ATTR_WEAK_ORDERING);
1760
1761 if (!skb) {
1762 skb_free_frag(data);
1763 bnxt_abort_tpa(cpr, idx, agg_bufs);
1764 cpr->sw_stats.rx.rx_oom_discards += 1;
1765 return NULL;
1766 }
1767 skb_reserve(skb, bp->rx_offset);
1768 skb_put(skb, len);
1769 }
1770
1771 if (agg_bufs) {
1772 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1773 if (!skb) {
1774 /* Page reuse already handled by bnxt_rx_pages(). */
1775 cpr->sw_stats.rx.rx_oom_discards += 1;
1776 return NULL;
1777 }
1778 }
1779
1780 if (tpa_info->cfa_code_valid)
1781 dev = bnxt_get_pkt_dev(bp, tpa_info->cfa_code);
1782 skb->protocol = eth_type_trans(skb, dev);
1783
1784 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1785 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1786
1787 if (tpa_info->vlan_valid &&
1788 (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1789 __be16 vlan_proto = htons(tpa_info->metadata >>
1790 RX_CMP_FLAGS2_METADATA_TPID_SFT);
1791 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1792
1793 if (eth_type_vlan(vlan_proto)) {
1794 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1795 } else {
1796 dev_kfree_skb(skb);
1797 return NULL;
1798 }
1799 }
1800
1801 skb_checksum_none_assert(skb);
1802 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1803 skb->ip_summed = CHECKSUM_UNNECESSARY;
1804 skb->csum_level =
1805 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1806 }
1807
1808 if (gro)
1809 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1810
1811 return skb;
1812}
1813
1814static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1815 struct rx_agg_cmp *rx_agg)
1816{
1817 u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1818 struct bnxt_tpa_info *tpa_info;
1819
1820 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1821 tpa_info = &rxr->rx_tpa[agg_id];
1822 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1823 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1824}
1825
1826static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1827 struct sk_buff *skb)
1828{
1829 skb_mark_for_recycle(skb);
1830
1831 if (skb->dev != bp->dev) {
1832 /* this packet belongs to a vf-rep */
1833 bnxt_vf_rep_rx(bp, skb);
1834 return;
1835 }
1836 skb_record_rx_queue(skb, bnapi->index);
1837 napi_gro_receive(&bnapi->napi, skb);
1838}
1839
1840static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags,
1841 struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts)
1842{
1843 u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
1844
1845 if (BNXT_PTP_RX_TS_VALID(flags))
1846 goto ts_valid;
1847 if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags))
1848 return false;
1849
1850ts_valid:
1851 *cmpl_ts = ts;
1852 return true;
1853}
1854
1855static struct sk_buff *bnxt_rx_vlan(struct sk_buff *skb, u8 cmp_type,
1856 struct rx_cmp *rxcmp,
1857 struct rx_cmp_ext *rxcmp1)
1858{
1859 __be16 vlan_proto;
1860 u16 vtag;
1861
1862 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1863 __le32 flags2 = rxcmp1->rx_cmp_flags2;
1864 u32 meta_data;
1865
1866 if (!(flags2 & cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)))
1867 return skb;
1868
1869 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1870 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1871 vlan_proto = htons(meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT);
1872 if (eth_type_vlan(vlan_proto))
1873 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1874 else
1875 goto vlan_err;
1876 } else if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
1877 if (RX_CMP_VLAN_VALID(rxcmp)) {
1878 u32 tpid_sel = RX_CMP_VLAN_TPID_SEL(rxcmp);
1879
1880 if (tpid_sel == RX_CMP_METADATA1_TPID_8021Q)
1881 vlan_proto = htons(ETH_P_8021Q);
1882 else if (tpid_sel == RX_CMP_METADATA1_TPID_8021AD)
1883 vlan_proto = htons(ETH_P_8021AD);
1884 else
1885 goto vlan_err;
1886 vtag = RX_CMP_METADATA0_TCI(rxcmp1);
1887 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1888 }
1889 }
1890 return skb;
1891vlan_err:
1892 dev_kfree_skb(skb);
1893 return NULL;
1894}
1895
1896static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp,
1897 struct rx_cmp *rxcmp)
1898{
1899 u8 ext_op;
1900
1901 ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp);
1902 switch (ext_op) {
1903 case EXT_OP_INNER_4:
1904 case EXT_OP_OUTER_4:
1905 case EXT_OP_INNFL_3:
1906 case EXT_OP_OUTFL_3:
1907 return PKT_HASH_TYPE_L4;
1908 default:
1909 return PKT_HASH_TYPE_L3;
1910 }
1911}
1912
1913/* returns the following:
1914 * 1 - 1 packet successfully received
1915 * 0 - successful TPA_START, packet not completed yet
1916 * -EBUSY - completion ring does not have all the agg buffers yet
1917 * -ENOMEM - packet aborted due to out of memory
1918 * -EIO - packet aborted due to hw error indicated in BD
1919 */
1920static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1921 u32 *raw_cons, u8 *event)
1922{
1923 struct bnxt_napi *bnapi = cpr->bnapi;
1924 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1925 struct net_device *dev = bp->dev;
1926 struct rx_cmp *rxcmp;
1927 struct rx_cmp_ext *rxcmp1;
1928 u32 tmp_raw_cons = *raw_cons;
1929 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1930 struct bnxt_sw_rx_bd *rx_buf;
1931 unsigned int len;
1932 u8 *data_ptr, agg_bufs, cmp_type;
1933 bool xdp_active = false;
1934 dma_addr_t dma_addr;
1935 struct sk_buff *skb;
1936 struct xdp_buff xdp;
1937 u32 flags, misc;
1938 u32 cmpl_ts;
1939 void *data;
1940 int rc = 0;
1941
1942 rxcmp = (struct rx_cmp *)
1943 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1944
1945 cmp_type = RX_CMP_TYPE(rxcmp);
1946
1947 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1948 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1949 goto next_rx_no_prod_no_len;
1950 }
1951
1952 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1953 cp_cons = RING_CMP(tmp_raw_cons);
1954 rxcmp1 = (struct rx_cmp_ext *)
1955 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1956
1957 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1958 return -EBUSY;
1959
1960 /* The valid test of the entry must be done first before
1961 * reading any further.
1962 */
1963 dma_rmb();
1964 prod = rxr->rx_prod;
1965
1966 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP ||
1967 cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
1968 bnxt_tpa_start(bp, rxr, cmp_type,
1969 (struct rx_tpa_start_cmp *)rxcmp,
1970 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1971
1972 *event |= BNXT_RX_EVENT;
1973 goto next_rx_no_prod_no_len;
1974
1975 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1976 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1977 (struct rx_tpa_end_cmp *)rxcmp,
1978 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1979
1980 if (IS_ERR(skb))
1981 return -EBUSY;
1982
1983 rc = -ENOMEM;
1984 if (likely(skb)) {
1985 bnxt_deliver_skb(bp, bnapi, skb);
1986 rc = 1;
1987 }
1988 *event |= BNXT_RX_EVENT;
1989 goto next_rx_no_prod_no_len;
1990 }
1991
1992 cons = rxcmp->rx_cmp_opaque;
1993 if (unlikely(cons != rxr->rx_next_cons)) {
1994 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1995
1996 /* 0xffff is forced error, don't print it */
1997 if (rxr->rx_next_cons != 0xffff)
1998 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1999 cons, rxr->rx_next_cons);
2000 bnxt_sched_reset_rxr(bp, rxr);
2001 if (rc1)
2002 return rc1;
2003 goto next_rx_no_prod_no_len;
2004 }
2005 rx_buf = &rxr->rx_buf_ring[cons];
2006 data = rx_buf->data;
2007 data_ptr = rx_buf->data_ptr;
2008 prefetch(data_ptr);
2009
2010 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
2011 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
2012
2013 if (agg_bufs) {
2014 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
2015 return -EBUSY;
2016
2017 cp_cons = NEXT_CMP(cp_cons);
2018 *event |= BNXT_AGG_EVENT;
2019 }
2020 *event |= BNXT_RX_EVENT;
2021
2022 rx_buf->data = NULL;
2023 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
2024 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
2025
2026 bnxt_reuse_rx_data(rxr, cons, data);
2027 if (agg_bufs)
2028 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
2029 false);
2030
2031 rc = -EIO;
2032 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
2033 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
2034 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
2035 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
2036 netdev_warn_once(bp->dev, "RX buffer error %x\n",
2037 rx_err);
2038 bnxt_sched_reset_rxr(bp, rxr);
2039 }
2040 }
2041 goto next_rx_no_len;
2042 }
2043
2044 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
2045 len = flags >> RX_CMP_LEN_SHIFT;
2046 dma_addr = rx_buf->mapping;
2047
2048 if (bnxt_xdp_attached(bp, rxr)) {
2049 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
2050 if (agg_bufs) {
2051 u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
2052 cp_cons, agg_bufs,
2053 false);
2054 if (!frag_len) {
2055 cpr->sw_stats.rx.rx_oom_discards += 1;
2056 rc = -ENOMEM;
2057 goto next_rx;
2058 }
2059 }
2060 xdp_active = true;
2061 }
2062
2063 if (xdp_active) {
2064 if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) {
2065 rc = 1;
2066 goto next_rx;
2067 }
2068 }
2069
2070 if (len <= bp->rx_copy_thresh) {
2071 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
2072 bnxt_reuse_rx_data(rxr, cons, data);
2073 if (!skb) {
2074 if (agg_bufs) {
2075 if (!xdp_active)
2076 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
2077 agg_bufs, false);
2078 else
2079 bnxt_xdp_buff_frags_free(rxr, &xdp);
2080 }
2081 cpr->sw_stats.rx.rx_oom_discards += 1;
2082 rc = -ENOMEM;
2083 goto next_rx;
2084 }
2085 } else {
2086 u32 payload;
2087
2088 if (rx_buf->data_ptr == data_ptr)
2089 payload = misc & RX_CMP_PAYLOAD_OFFSET;
2090 else
2091 payload = 0;
2092 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
2093 payload | len);
2094 if (!skb) {
2095 cpr->sw_stats.rx.rx_oom_discards += 1;
2096 rc = -ENOMEM;
2097 goto next_rx;
2098 }
2099 }
2100
2101 if (agg_bufs) {
2102 if (!xdp_active) {
2103 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
2104 if (!skb) {
2105 cpr->sw_stats.rx.rx_oom_discards += 1;
2106 rc = -ENOMEM;
2107 goto next_rx;
2108 }
2109 } else {
2110 skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
2111 if (!skb) {
2112 /* we should be able to free the old skb here */
2113 bnxt_xdp_buff_frags_free(rxr, &xdp);
2114 cpr->sw_stats.rx.rx_oom_discards += 1;
2115 rc = -ENOMEM;
2116 goto next_rx;
2117 }
2118 }
2119 }
2120
2121 if (RX_CMP_HASH_VALID(rxcmp)) {
2122 enum pkt_hash_types type;
2123
2124 if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2125 type = bnxt_rss_ext_op(bp, rxcmp);
2126 } else {
2127 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
2128
2129 /* RSS profiles 1 and 3 with extract code 0 for inner
2130 * 4-tuple
2131 */
2132 if (hash_type != 1 && hash_type != 3)
2133 type = PKT_HASH_TYPE_L3;
2134 else
2135 type = PKT_HASH_TYPE_L4;
2136 }
2137 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
2138 }
2139
2140 if (cmp_type == CMP_TYPE_RX_L2_CMP)
2141 dev = bnxt_get_pkt_dev(bp, RX_CMP_CFA_CODE(rxcmp1));
2142 skb->protocol = eth_type_trans(skb, dev);
2143
2144 if (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) {
2145 skb = bnxt_rx_vlan(skb, cmp_type, rxcmp, rxcmp1);
2146 if (!skb)
2147 goto next_rx;
2148 }
2149
2150 skb_checksum_none_assert(skb);
2151 if (RX_CMP_L4_CS_OK(rxcmp1)) {
2152 if (dev->features & NETIF_F_RXCSUM) {
2153 skb->ip_summed = CHECKSUM_UNNECESSARY;
2154 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2155 }
2156 } else {
2157 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2158 if (dev->features & NETIF_F_RXCSUM)
2159 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
2160 }
2161 }
2162
2163 if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) {
2164 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
2165 u64 ns, ts;
2166
2167 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2168 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2169
2170 spin_lock_bh(&ptp->ptp_lock);
2171 ns = timecounter_cyc2time(&ptp->tc, ts);
2172 spin_unlock_bh(&ptp->ptp_lock);
2173 memset(skb_hwtstamps(skb), 0,
2174 sizeof(*skb_hwtstamps(skb)));
2175 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2176 }
2177 }
2178 }
2179 bnxt_deliver_skb(bp, bnapi, skb);
2180 rc = 1;
2181
2182next_rx:
2183 cpr->rx_packets += 1;
2184 cpr->rx_bytes += len;
2185
2186next_rx_no_len:
2187 rxr->rx_prod = NEXT_RX(prod);
2188 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
2189
2190next_rx_no_prod_no_len:
2191 *raw_cons = tmp_raw_cons;
2192
2193 return rc;
2194}
2195
2196/* In netpoll mode, if we are using a combined completion ring, we need to
2197 * discard the rx packets and recycle the buffers.
2198 */
2199static int bnxt_force_rx_discard(struct bnxt *bp,
2200 struct bnxt_cp_ring_info *cpr,
2201 u32 *raw_cons, u8 *event)
2202{
2203 u32 tmp_raw_cons = *raw_cons;
2204 struct rx_cmp_ext *rxcmp1;
2205 struct rx_cmp *rxcmp;
2206 u16 cp_cons;
2207 u8 cmp_type;
2208 int rc;
2209
2210 cp_cons = RING_CMP(tmp_raw_cons);
2211 rxcmp = (struct rx_cmp *)
2212 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2213
2214 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2215 cp_cons = RING_CMP(tmp_raw_cons);
2216 rxcmp1 = (struct rx_cmp_ext *)
2217 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2218
2219 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2220 return -EBUSY;
2221
2222 /* The valid test of the entry must be done first before
2223 * reading any further.
2224 */
2225 dma_rmb();
2226 cmp_type = RX_CMP_TYPE(rxcmp);
2227 if (cmp_type == CMP_TYPE_RX_L2_CMP ||
2228 cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2229 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2230 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2231 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2232 struct rx_tpa_end_cmp_ext *tpa_end1;
2233
2234 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2235 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2236 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2237 }
2238 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2239 if (rc && rc != -EBUSY)
2240 cpr->sw_stats.rx.rx_netpoll_discards += 1;
2241 return rc;
2242}
2243
2244u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2245{
2246 struct bnxt_fw_health *fw_health = bp->fw_health;
2247 u32 reg = fw_health->regs[reg_idx];
2248 u32 reg_type, reg_off, val = 0;
2249
2250 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2251 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2252 switch (reg_type) {
2253 case BNXT_FW_HEALTH_REG_TYPE_CFG:
2254 pci_read_config_dword(bp->pdev, reg_off, &val);
2255 break;
2256 case BNXT_FW_HEALTH_REG_TYPE_GRC:
2257 reg_off = fw_health->mapped_regs[reg_idx];
2258 fallthrough;
2259 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2260 val = readl(bp->bar0 + reg_off);
2261 break;
2262 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2263 val = readl(bp->bar1 + reg_off);
2264 break;
2265 }
2266 if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2267 val &= fw_health->fw_reset_inprog_reg_mask;
2268 return val;
2269}
2270
2271static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2272{
2273 int i;
2274
2275 for (i = 0; i < bp->rx_nr_rings; i++) {
2276 u16 grp_idx = bp->rx_ring[i].bnapi->index;
2277 struct bnxt_ring_grp_info *grp_info;
2278
2279 grp_info = &bp->grp_info[grp_idx];
2280 if (grp_info->agg_fw_ring_id == ring_id)
2281 return grp_idx;
2282 }
2283 return INVALID_HW_RING_ID;
2284}
2285
2286static u16 bnxt_get_force_speed(struct bnxt_link_info *link_info)
2287{
2288 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2289
2290 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
2291 return link_info->force_link_speed2;
2292 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4)
2293 return link_info->force_pam4_link_speed;
2294 return link_info->force_link_speed;
2295}
2296
2297static void bnxt_set_force_speed(struct bnxt_link_info *link_info)
2298{
2299 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2300
2301 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2302 link_info->req_link_speed = link_info->force_link_speed2;
2303 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2304 switch (link_info->req_link_speed) {
2305 case BNXT_LINK_SPEED_50GB_PAM4:
2306 case BNXT_LINK_SPEED_100GB_PAM4:
2307 case BNXT_LINK_SPEED_200GB_PAM4:
2308 case BNXT_LINK_SPEED_400GB_PAM4:
2309 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2310 break;
2311 case BNXT_LINK_SPEED_100GB_PAM4_112:
2312 case BNXT_LINK_SPEED_200GB_PAM4_112:
2313 case BNXT_LINK_SPEED_400GB_PAM4_112:
2314 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4_112;
2315 break;
2316 default:
2317 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2318 }
2319 return;
2320 }
2321 link_info->req_link_speed = link_info->force_link_speed;
2322 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2323 if (link_info->force_pam4_link_speed) {
2324 link_info->req_link_speed = link_info->force_pam4_link_speed;
2325 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2326 }
2327}
2328
2329static void bnxt_set_auto_speed(struct bnxt_link_info *link_info)
2330{
2331 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2332
2333 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2334 link_info->advertising = link_info->auto_link_speeds2;
2335 return;
2336 }
2337 link_info->advertising = link_info->auto_link_speeds;
2338 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
2339}
2340
2341static bool bnxt_force_speed_updated(struct bnxt_link_info *link_info)
2342{
2343 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2344
2345 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2346 if (link_info->req_link_speed != link_info->force_link_speed2)
2347 return true;
2348 return false;
2349 }
2350 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
2351 link_info->req_link_speed != link_info->force_link_speed)
2352 return true;
2353 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
2354 link_info->req_link_speed != link_info->force_pam4_link_speed)
2355 return true;
2356 return false;
2357}
2358
2359static bool bnxt_auto_speed_updated(struct bnxt_link_info *link_info)
2360{
2361 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2362
2363 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2364 if (link_info->advertising != link_info->auto_link_speeds2)
2365 return true;
2366 return false;
2367 }
2368 if (link_info->advertising != link_info->auto_link_speeds ||
2369 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
2370 return true;
2371 return false;
2372}
2373
2374#define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2) \
2375 ((data2) & \
2376 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK)
2377
2378#define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2) \
2379 (((data2) & \
2380 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >>\
2381 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT)
2382
2383#define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1) \
2384 ((data1) & \
2385 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK)
2386
2387#define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1) \
2388 (((data1) & \
2389 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) ==\
2390 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING)
2391
2392/* Return true if the workqueue has to be scheduled */
2393static bool bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2394{
2395 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2396
2397 switch (err_type) {
2398 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2399 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2400 BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2401 break;
2402 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2403 netdev_warn(bp->dev, "Pause Storm detected!\n");
2404 break;
2405 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2406 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2407 break;
2408 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD: {
2409 u32 type = EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1);
2410 char *threshold_type;
2411 bool notify = false;
2412 char *dir_str;
2413
2414 switch (type) {
2415 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN:
2416 threshold_type = "warning";
2417 break;
2418 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL:
2419 threshold_type = "critical";
2420 break;
2421 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL:
2422 threshold_type = "fatal";
2423 break;
2424 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN:
2425 threshold_type = "shutdown";
2426 break;
2427 default:
2428 netdev_err(bp->dev, "Unknown Thermal threshold type event\n");
2429 return false;
2430 }
2431 if (EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)) {
2432 dir_str = "above";
2433 notify = true;
2434 } else {
2435 dir_str = "below";
2436 }
2437 netdev_warn(bp->dev, "Chip temperature has gone %s the %s thermal threshold!\n",
2438 dir_str, threshold_type);
2439 netdev_warn(bp->dev, "Temperature (In Celsius), Current: %lu, threshold: %lu\n",
2440 BNXT_EVENT_THERMAL_CURRENT_TEMP(data2),
2441 BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2));
2442 if (notify) {
2443 bp->thermal_threshold_type = type;
2444 set_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event);
2445 return true;
2446 }
2447 return false;
2448 }
2449 default:
2450 netdev_err(bp->dev, "FW reported unknown error type %u\n",
2451 err_type);
2452 break;
2453 }
2454 return false;
2455}
2456
2457#define BNXT_GET_EVENT_PORT(data) \
2458 ((data) & \
2459 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2460
2461#define BNXT_EVENT_RING_TYPE(data2) \
2462 ((data2) & \
2463 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2464
2465#define BNXT_EVENT_RING_TYPE_RX(data2) \
2466 (BNXT_EVENT_RING_TYPE(data2) == \
2467 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2468
2469#define BNXT_EVENT_PHC_EVENT_TYPE(data1) \
2470 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2471 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2472
2473#define BNXT_EVENT_PHC_RTC_UPDATE(data1) \
2474 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2475 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2476
2477#define BNXT_PHC_BITS 48
2478
2479static int bnxt_async_event_process(struct bnxt *bp,
2480 struct hwrm_async_event_cmpl *cmpl)
2481{
2482 u16 event_id = le16_to_cpu(cmpl->event_id);
2483 u32 data1 = le32_to_cpu(cmpl->event_data1);
2484 u32 data2 = le32_to_cpu(cmpl->event_data2);
2485
2486 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2487 event_id, data1, data2);
2488
2489 /* TODO CHIMP_FW: Define event id's for link change, error etc */
2490 switch (event_id) {
2491 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2492 struct bnxt_link_info *link_info = &bp->link_info;
2493
2494 if (BNXT_VF(bp))
2495 goto async_event_process_exit;
2496
2497 /* print unsupported speed warning in forced speed mode only */
2498 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2499 (data1 & 0x20000)) {
2500 u16 fw_speed = bnxt_get_force_speed(link_info);
2501 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2502
2503 if (speed != SPEED_UNKNOWN)
2504 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2505 speed);
2506 }
2507 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2508 }
2509 fallthrough;
2510 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2511 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2512 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2513 fallthrough;
2514 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2515 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2516 break;
2517 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2518 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2519 break;
2520 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2521 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2522
2523 if (BNXT_VF(bp))
2524 break;
2525
2526 if (bp->pf.port_id != port_id)
2527 break;
2528
2529 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2530 break;
2531 }
2532 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2533 if (BNXT_PF(bp))
2534 goto async_event_process_exit;
2535 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2536 break;
2537 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2538 char *type_str = "Solicited";
2539
2540 if (!bp->fw_health)
2541 goto async_event_process_exit;
2542
2543 bp->fw_reset_timestamp = jiffies;
2544 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2545 if (!bp->fw_reset_min_dsecs)
2546 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2547 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2548 if (!bp->fw_reset_max_dsecs)
2549 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2550 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2551 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2552 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2553 type_str = "Fatal";
2554 bp->fw_health->fatalities++;
2555 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2556 } else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2557 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2558 type_str = "Non-fatal";
2559 bp->fw_health->survivals++;
2560 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2561 }
2562 netif_warn(bp, hw, bp->dev,
2563 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2564 type_str, data1, data2,
2565 bp->fw_reset_min_dsecs * 100,
2566 bp->fw_reset_max_dsecs * 100);
2567 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2568 break;
2569 }
2570 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2571 struct bnxt_fw_health *fw_health = bp->fw_health;
2572 char *status_desc = "healthy";
2573 u32 status;
2574
2575 if (!fw_health)
2576 goto async_event_process_exit;
2577
2578 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2579 fw_health->enabled = false;
2580 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2581 break;
2582 }
2583 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2584 fw_health->tmr_multiplier =
2585 DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2586 bp->current_interval * 10);
2587 fw_health->tmr_counter = fw_health->tmr_multiplier;
2588 if (!fw_health->enabled)
2589 fw_health->last_fw_heartbeat =
2590 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2591 fw_health->last_fw_reset_cnt =
2592 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2593 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2594 if (status != BNXT_FW_STATUS_HEALTHY)
2595 status_desc = "unhealthy";
2596 netif_info(bp, drv, bp->dev,
2597 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2598 fw_health->primary ? "primary" : "backup", status,
2599 status_desc, fw_health->last_fw_reset_cnt);
2600 if (!fw_health->enabled) {
2601 /* Make sure tmr_counter is set and visible to
2602 * bnxt_health_check() before setting enabled to true.
2603 */
2604 smp_wmb();
2605 fw_health->enabled = true;
2606 }
2607 goto async_event_process_exit;
2608 }
2609 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2610 netif_notice(bp, hw, bp->dev,
2611 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2612 data1, data2);
2613 goto async_event_process_exit;
2614 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2615 struct bnxt_rx_ring_info *rxr;
2616 u16 grp_idx;
2617
2618 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
2619 goto async_event_process_exit;
2620
2621 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2622 BNXT_EVENT_RING_TYPE(data2), data1);
2623 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2624 goto async_event_process_exit;
2625
2626 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2627 if (grp_idx == INVALID_HW_RING_ID) {
2628 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2629 data1);
2630 goto async_event_process_exit;
2631 }
2632 rxr = bp->bnapi[grp_idx]->rx_ring;
2633 bnxt_sched_reset_rxr(bp, rxr);
2634 goto async_event_process_exit;
2635 }
2636 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2637 struct bnxt_fw_health *fw_health = bp->fw_health;
2638
2639 netif_notice(bp, hw, bp->dev,
2640 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2641 data1, data2);
2642 if (fw_health) {
2643 fw_health->echo_req_data1 = data1;
2644 fw_health->echo_req_data2 = data2;
2645 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2646 break;
2647 }
2648 goto async_event_process_exit;
2649 }
2650 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2651 bnxt_ptp_pps_event(bp, data1, data2);
2652 goto async_event_process_exit;
2653 }
2654 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2655 if (bnxt_event_error_report(bp, data1, data2))
2656 break;
2657 goto async_event_process_exit;
2658 }
2659 case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2660 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2661 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2662 if (BNXT_PTP_USE_RTC(bp)) {
2663 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2664 u64 ns;
2665
2666 if (!ptp)
2667 goto async_event_process_exit;
2668
2669 spin_lock_bh(&ptp->ptp_lock);
2670 bnxt_ptp_update_current_time(bp);
2671 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2672 BNXT_PHC_BITS) | ptp->current_time);
2673 bnxt_ptp_rtc_timecounter_init(ptp, ns);
2674 spin_unlock_bh(&ptp->ptp_lock);
2675 }
2676 break;
2677 }
2678 goto async_event_process_exit;
2679 }
2680 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2681 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2682
2683 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2684 goto async_event_process_exit;
2685 }
2686 default:
2687 goto async_event_process_exit;
2688 }
2689 __bnxt_queue_sp_work(bp);
2690async_event_process_exit:
2691 return 0;
2692}
2693
2694static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2695{
2696 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2697 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2698 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2699 (struct hwrm_fwd_req_cmpl *)txcmp;
2700
2701 switch (cmpl_type) {
2702 case CMPL_BASE_TYPE_HWRM_DONE:
2703 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2704 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2705 break;
2706
2707 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2708 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2709
2710 if ((vf_id < bp->pf.first_vf_id) ||
2711 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2712 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2713 vf_id);
2714 return -EINVAL;
2715 }
2716
2717 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2718 bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT);
2719 break;
2720
2721 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2722 bnxt_async_event_process(bp,
2723 (struct hwrm_async_event_cmpl *)txcmp);
2724 break;
2725
2726 default:
2727 break;
2728 }
2729
2730 return 0;
2731}
2732
2733static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2734{
2735 struct bnxt_napi *bnapi = dev_instance;
2736 struct bnxt *bp = bnapi->bp;
2737 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2738 u32 cons = RING_CMP(cpr->cp_raw_cons);
2739
2740 cpr->event_ctr++;
2741 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2742 napi_schedule(&bnapi->napi);
2743 return IRQ_HANDLED;
2744}
2745
2746static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2747{
2748 u32 raw_cons = cpr->cp_raw_cons;
2749 u16 cons = RING_CMP(raw_cons);
2750 struct tx_cmp *txcmp;
2751
2752 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2753
2754 return TX_CMP_VALID(txcmp, raw_cons);
2755}
2756
2757static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2758{
2759 struct bnxt_napi *bnapi = dev_instance;
2760 struct bnxt *bp = bnapi->bp;
2761 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2762 u32 cons = RING_CMP(cpr->cp_raw_cons);
2763 u32 int_status;
2764
2765 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2766
2767 if (!bnxt_has_work(bp, cpr)) {
2768 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2769 /* return if erroneous interrupt */
2770 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2771 return IRQ_NONE;
2772 }
2773
2774 /* disable ring IRQ */
2775 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2776
2777 /* Return here if interrupt is shared and is disabled. */
2778 if (unlikely(atomic_read(&bp->intr_sem) != 0))
2779 return IRQ_HANDLED;
2780
2781 napi_schedule(&bnapi->napi);
2782 return IRQ_HANDLED;
2783}
2784
2785static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2786 int budget)
2787{
2788 struct bnxt_napi *bnapi = cpr->bnapi;
2789 u32 raw_cons = cpr->cp_raw_cons;
2790 u32 cons;
2791 int rx_pkts = 0;
2792 u8 event = 0;
2793 struct tx_cmp *txcmp;
2794
2795 cpr->has_more_work = 0;
2796 cpr->had_work_done = 1;
2797 while (1) {
2798 u8 cmp_type;
2799 int rc;
2800
2801 cons = RING_CMP(raw_cons);
2802 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2803
2804 if (!TX_CMP_VALID(txcmp, raw_cons))
2805 break;
2806
2807 /* The valid test of the entry must be done first before
2808 * reading any further.
2809 */
2810 dma_rmb();
2811 cmp_type = TX_CMP_TYPE(txcmp);
2812 if (cmp_type == CMP_TYPE_TX_L2_CMP ||
2813 cmp_type == CMP_TYPE_TX_L2_COAL_CMP) {
2814 u32 opaque = txcmp->tx_cmp_opaque;
2815 struct bnxt_tx_ring_info *txr;
2816 u16 tx_freed;
2817
2818 txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)];
2819 event |= BNXT_TX_CMP_EVENT;
2820 if (cmp_type == CMP_TYPE_TX_L2_COAL_CMP)
2821 txr->tx_hw_cons = TX_CMP_SQ_CONS_IDX(txcmp);
2822 else
2823 txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque);
2824 tx_freed = (txr->tx_hw_cons - txr->tx_cons) &
2825 bp->tx_ring_mask;
2826 /* return full budget so NAPI will complete. */
2827 if (unlikely(tx_freed >= bp->tx_wake_thresh)) {
2828 rx_pkts = budget;
2829 raw_cons = NEXT_RAW_CMP(raw_cons);
2830 if (budget)
2831 cpr->has_more_work = 1;
2832 break;
2833 }
2834 } else if (cmp_type >= CMP_TYPE_RX_L2_CMP &&
2835 cmp_type <= CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
2836 if (likely(budget))
2837 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2838 else
2839 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2840 &event);
2841 if (likely(rc >= 0))
2842 rx_pkts += rc;
2843 /* Increment rx_pkts when rc is -ENOMEM to count towards
2844 * the NAPI budget. Otherwise, we may potentially loop
2845 * here forever if we consistently cannot allocate
2846 * buffers.
2847 */
2848 else if (rc == -ENOMEM && budget)
2849 rx_pkts++;
2850 else if (rc == -EBUSY) /* partial completion */
2851 break;
2852 } else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE ||
2853 cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ ||
2854 cmp_type == CMPL_BASE_TYPE_HWRM_ASYNC_EVENT)) {
2855 bnxt_hwrm_handler(bp, txcmp);
2856 }
2857 raw_cons = NEXT_RAW_CMP(raw_cons);
2858
2859 if (rx_pkts && rx_pkts == budget) {
2860 cpr->has_more_work = 1;
2861 break;
2862 }
2863 }
2864
2865 if (event & BNXT_REDIRECT_EVENT)
2866 xdp_do_flush();
2867
2868 if (event & BNXT_TX_EVENT) {
2869 struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0];
2870 u16 prod = txr->tx_prod;
2871
2872 /* Sync BD data before updating doorbell */
2873 wmb();
2874
2875 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2876 }
2877
2878 cpr->cp_raw_cons = raw_cons;
2879 bnapi->events |= event;
2880 return rx_pkts;
2881}
2882
2883static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2884 int budget)
2885{
2886 if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault)
2887 bnapi->tx_int(bp, bnapi, budget);
2888
2889 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2890 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2891
2892 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2893 }
2894 if (bnapi->events & BNXT_AGG_EVENT) {
2895 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2896
2897 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2898 }
2899 bnapi->events &= BNXT_TX_CMP_EVENT;
2900}
2901
2902static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2903 int budget)
2904{
2905 struct bnxt_napi *bnapi = cpr->bnapi;
2906 int rx_pkts;
2907
2908 rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2909
2910 /* ACK completion ring before freeing tx ring and producing new
2911 * buffers in rx/agg rings to prevent overflowing the completion
2912 * ring.
2913 */
2914 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2915
2916 __bnxt_poll_work_done(bp, bnapi, budget);
2917 return rx_pkts;
2918}
2919
2920static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2921{
2922 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2923 struct bnxt *bp = bnapi->bp;
2924 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2925 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2926 struct tx_cmp *txcmp;
2927 struct rx_cmp_ext *rxcmp1;
2928 u32 cp_cons, tmp_raw_cons;
2929 u32 raw_cons = cpr->cp_raw_cons;
2930 bool flush_xdp = false;
2931 u32 rx_pkts = 0;
2932 u8 event = 0;
2933
2934 while (1) {
2935 int rc;
2936
2937 cp_cons = RING_CMP(raw_cons);
2938 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2939
2940 if (!TX_CMP_VALID(txcmp, raw_cons))
2941 break;
2942
2943 /* The valid test of the entry must be done first before
2944 * reading any further.
2945 */
2946 dma_rmb();
2947 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2948 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2949 cp_cons = RING_CMP(tmp_raw_cons);
2950 rxcmp1 = (struct rx_cmp_ext *)
2951 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2952
2953 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2954 break;
2955
2956 /* force an error to recycle the buffer */
2957 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2958 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2959
2960 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2961 if (likely(rc == -EIO) && budget)
2962 rx_pkts++;
2963 else if (rc == -EBUSY) /* partial completion */
2964 break;
2965 if (event & BNXT_REDIRECT_EVENT)
2966 flush_xdp = true;
2967 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2968 CMPL_BASE_TYPE_HWRM_DONE)) {
2969 bnxt_hwrm_handler(bp, txcmp);
2970 } else {
2971 netdev_err(bp->dev,
2972 "Invalid completion received on special ring\n");
2973 }
2974 raw_cons = NEXT_RAW_CMP(raw_cons);
2975
2976 if (rx_pkts == budget)
2977 break;
2978 }
2979
2980 cpr->cp_raw_cons = raw_cons;
2981 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2982 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2983
2984 if (event & BNXT_AGG_EVENT)
2985 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2986 if (flush_xdp)
2987 xdp_do_flush();
2988
2989 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2990 napi_complete_done(napi, rx_pkts);
2991 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2992 }
2993 return rx_pkts;
2994}
2995
2996static int bnxt_poll(struct napi_struct *napi, int budget)
2997{
2998 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2999 struct bnxt *bp = bnapi->bp;
3000 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3001 int work_done = 0;
3002
3003 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3004 napi_complete(napi);
3005 return 0;
3006 }
3007 while (1) {
3008 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
3009
3010 if (work_done >= budget) {
3011 if (!budget)
3012 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3013 break;
3014 }
3015
3016 if (!bnxt_has_work(bp, cpr)) {
3017 if (napi_complete_done(napi, work_done))
3018 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3019 break;
3020 }
3021 }
3022 if (bp->flags & BNXT_FLAG_DIM) {
3023 struct dim_sample dim_sample = {};
3024
3025 dim_update_sample(cpr->event_ctr,
3026 cpr->rx_packets,
3027 cpr->rx_bytes,
3028 &dim_sample);
3029 net_dim(&cpr->dim, dim_sample);
3030 }
3031 return work_done;
3032}
3033
3034static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
3035{
3036 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3037 int i, work_done = 0;
3038
3039 for (i = 0; i < cpr->cp_ring_count; i++) {
3040 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3041
3042 if (cpr2->had_nqe_notify) {
3043 work_done += __bnxt_poll_work(bp, cpr2,
3044 budget - work_done);
3045 cpr->has_more_work |= cpr2->has_more_work;
3046 }
3047 }
3048 return work_done;
3049}
3050
3051static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
3052 u64 dbr_type, int budget)
3053{
3054 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3055 int i;
3056
3057 for (i = 0; i < cpr->cp_ring_count; i++) {
3058 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3059 struct bnxt_db_info *db;
3060
3061 if (cpr2->had_work_done) {
3062 u32 tgl = 0;
3063
3064 if (dbr_type == DBR_TYPE_CQ_ARMALL) {
3065 cpr2->had_nqe_notify = 0;
3066 tgl = cpr2->toggle;
3067 }
3068 db = &cpr2->cp_db;
3069 bnxt_writeq(bp,
3070 db->db_key64 | dbr_type | DB_TOGGLE(tgl) |
3071 DB_RING_IDX(db, cpr2->cp_raw_cons),
3072 db->doorbell);
3073 cpr2->had_work_done = 0;
3074 }
3075 }
3076 __bnxt_poll_work_done(bp, bnapi, budget);
3077}
3078
3079static int bnxt_poll_p5(struct napi_struct *napi, int budget)
3080{
3081 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3082 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3083 struct bnxt_cp_ring_info *cpr_rx;
3084 u32 raw_cons = cpr->cp_raw_cons;
3085 struct bnxt *bp = bnapi->bp;
3086 struct nqe_cn *nqcmp;
3087 int work_done = 0;
3088 u32 cons;
3089
3090 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3091 napi_complete(napi);
3092 return 0;
3093 }
3094 if (cpr->has_more_work) {
3095 cpr->has_more_work = 0;
3096 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
3097 }
3098 while (1) {
3099 u16 type;
3100
3101 cons = RING_CMP(raw_cons);
3102 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
3103
3104 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
3105 if (cpr->has_more_work)
3106 break;
3107
3108 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
3109 budget);
3110 cpr->cp_raw_cons = raw_cons;
3111 if (napi_complete_done(napi, work_done))
3112 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
3113 cpr->cp_raw_cons);
3114 goto poll_done;
3115 }
3116
3117 /* The valid test of the entry must be done first before
3118 * reading any further.
3119 */
3120 dma_rmb();
3121
3122 type = le16_to_cpu(nqcmp->type);
3123 if (NQE_CN_TYPE(type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
3124 u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
3125 u32 cq_type = BNXT_NQ_HDL_TYPE(idx);
3126 struct bnxt_cp_ring_info *cpr2;
3127
3128 /* No more budget for RX work */
3129 if (budget && work_done >= budget &&
3130 cq_type == BNXT_NQ_HDL_TYPE_RX)
3131 break;
3132
3133 idx = BNXT_NQ_HDL_IDX(idx);
3134 cpr2 = &cpr->cp_ring_arr[idx];
3135 cpr2->had_nqe_notify = 1;
3136 cpr2->toggle = NQE_CN_TOGGLE(type);
3137 work_done += __bnxt_poll_work(bp, cpr2,
3138 budget - work_done);
3139 cpr->has_more_work |= cpr2->has_more_work;
3140 } else {
3141 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
3142 }
3143 raw_cons = NEXT_RAW_CMP(raw_cons);
3144 }
3145 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget);
3146 if (raw_cons != cpr->cp_raw_cons) {
3147 cpr->cp_raw_cons = raw_cons;
3148 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
3149 }
3150poll_done:
3151 cpr_rx = &cpr->cp_ring_arr[0];
3152 if (cpr_rx->cp_ring_type == BNXT_NQ_HDL_TYPE_RX &&
3153 (bp->flags & BNXT_FLAG_DIM)) {
3154 struct dim_sample dim_sample = {};
3155
3156 dim_update_sample(cpr->event_ctr,
3157 cpr_rx->rx_packets,
3158 cpr_rx->rx_bytes,
3159 &dim_sample);
3160 net_dim(&cpr->dim, dim_sample);
3161 }
3162 return work_done;
3163}
3164
3165static void bnxt_free_tx_skbs(struct bnxt *bp)
3166{
3167 int i, max_idx;
3168 struct pci_dev *pdev = bp->pdev;
3169
3170 if (!bp->tx_ring)
3171 return;
3172
3173 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
3174 for (i = 0; i < bp->tx_nr_rings; i++) {
3175 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3176 int j;
3177
3178 if (!txr->tx_buf_ring)
3179 continue;
3180
3181 for (j = 0; j < max_idx;) {
3182 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
3183 struct sk_buff *skb;
3184 int k, last;
3185
3186 if (i < bp->tx_nr_rings_xdp &&
3187 tx_buf->action == XDP_REDIRECT) {
3188 dma_unmap_single(&pdev->dev,
3189 dma_unmap_addr(tx_buf, mapping),
3190 dma_unmap_len(tx_buf, len),
3191 DMA_TO_DEVICE);
3192 xdp_return_frame(tx_buf->xdpf);
3193 tx_buf->action = 0;
3194 tx_buf->xdpf = NULL;
3195 j++;
3196 continue;
3197 }
3198
3199 skb = tx_buf->skb;
3200 if (!skb) {
3201 j++;
3202 continue;
3203 }
3204
3205 tx_buf->skb = NULL;
3206
3207 if (tx_buf->is_push) {
3208 dev_kfree_skb(skb);
3209 j += 2;
3210 continue;
3211 }
3212
3213 dma_unmap_single(&pdev->dev,
3214 dma_unmap_addr(tx_buf, mapping),
3215 skb_headlen(skb),
3216 DMA_TO_DEVICE);
3217
3218 last = tx_buf->nr_frags;
3219 j += 2;
3220 for (k = 0; k < last; k++, j++) {
3221 int ring_idx = j & bp->tx_ring_mask;
3222 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
3223
3224 tx_buf = &txr->tx_buf_ring[ring_idx];
3225 dma_unmap_page(
3226 &pdev->dev,
3227 dma_unmap_addr(tx_buf, mapping),
3228 skb_frag_size(frag), DMA_TO_DEVICE);
3229 }
3230 dev_kfree_skb(skb);
3231 }
3232 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
3233 }
3234}
3235
3236static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
3237{
3238 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3239 struct pci_dev *pdev = bp->pdev;
3240 struct bnxt_tpa_idx_map *map;
3241 int i, max_idx, max_agg_idx;
3242
3243 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
3244 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
3245 if (!rxr->rx_tpa)
3246 goto skip_rx_tpa_free;
3247
3248 for (i = 0; i < bp->max_tpa; i++) {
3249 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
3250 u8 *data = tpa_info->data;
3251
3252 if (!data)
3253 continue;
3254
3255 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
3256 bp->rx_buf_use_size, bp->rx_dir,
3257 DMA_ATTR_WEAK_ORDERING);
3258
3259 tpa_info->data = NULL;
3260
3261 skb_free_frag(data);
3262 }
3263
3264skip_rx_tpa_free:
3265 if (!rxr->rx_buf_ring)
3266 goto skip_rx_buf_free;
3267
3268 for (i = 0; i < max_idx; i++) {
3269 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
3270 dma_addr_t mapping = rx_buf->mapping;
3271 void *data = rx_buf->data;
3272
3273 if (!data)
3274 continue;
3275
3276 rx_buf->data = NULL;
3277 if (BNXT_RX_PAGE_MODE(bp)) {
3278 page_pool_recycle_direct(rxr->page_pool, data);
3279 } else {
3280 dma_unmap_single_attrs(&pdev->dev, mapping,
3281 bp->rx_buf_use_size, bp->rx_dir,
3282 DMA_ATTR_WEAK_ORDERING);
3283 skb_free_frag(data);
3284 }
3285 }
3286
3287skip_rx_buf_free:
3288 if (!rxr->rx_agg_ring)
3289 goto skip_rx_agg_free;
3290
3291 for (i = 0; i < max_agg_idx; i++) {
3292 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
3293 struct page *page = rx_agg_buf->page;
3294
3295 if (!page)
3296 continue;
3297
3298 rx_agg_buf->page = NULL;
3299 __clear_bit(i, rxr->rx_agg_bmap);
3300
3301 page_pool_recycle_direct(rxr->page_pool, page);
3302 }
3303
3304skip_rx_agg_free:
3305 map = rxr->rx_tpa_idx_map;
3306 if (map)
3307 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
3308}
3309
3310static void bnxt_free_rx_skbs(struct bnxt *bp)
3311{
3312 int i;
3313
3314 if (!bp->rx_ring)
3315 return;
3316
3317 for (i = 0; i < bp->rx_nr_rings; i++)
3318 bnxt_free_one_rx_ring_skbs(bp, i);
3319}
3320
3321static void bnxt_free_skbs(struct bnxt *bp)
3322{
3323 bnxt_free_tx_skbs(bp);
3324 bnxt_free_rx_skbs(bp);
3325}
3326
3327static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len)
3328{
3329 u8 init_val = ctxm->init_value;
3330 u16 offset = ctxm->init_offset;
3331 u8 *p2 = p;
3332 int i;
3333
3334 if (!init_val)
3335 return;
3336 if (offset == BNXT_CTX_INIT_INVALID_OFFSET) {
3337 memset(p, init_val, len);
3338 return;
3339 }
3340 for (i = 0; i < len; i += ctxm->entry_size)
3341 *(p2 + i + offset) = init_val;
3342}
3343
3344static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3345{
3346 struct pci_dev *pdev = bp->pdev;
3347 int i;
3348
3349 if (!rmem->pg_arr)
3350 goto skip_pages;
3351
3352 for (i = 0; i < rmem->nr_pages; i++) {
3353 if (!rmem->pg_arr[i])
3354 continue;
3355
3356 dma_free_coherent(&pdev->dev, rmem->page_size,
3357 rmem->pg_arr[i], rmem->dma_arr[i]);
3358
3359 rmem->pg_arr[i] = NULL;
3360 }
3361skip_pages:
3362 if (rmem->pg_tbl) {
3363 size_t pg_tbl_size = rmem->nr_pages * 8;
3364
3365 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3366 pg_tbl_size = rmem->page_size;
3367 dma_free_coherent(&pdev->dev, pg_tbl_size,
3368 rmem->pg_tbl, rmem->pg_tbl_map);
3369 rmem->pg_tbl = NULL;
3370 }
3371 if (rmem->vmem_size && *rmem->vmem) {
3372 vfree(*rmem->vmem);
3373 *rmem->vmem = NULL;
3374 }
3375}
3376
3377static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3378{
3379 struct pci_dev *pdev = bp->pdev;
3380 u64 valid_bit = 0;
3381 int i;
3382
3383 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3384 valid_bit = PTU_PTE_VALID;
3385 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3386 size_t pg_tbl_size = rmem->nr_pages * 8;
3387
3388 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3389 pg_tbl_size = rmem->page_size;
3390 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3391 &rmem->pg_tbl_map,
3392 GFP_KERNEL);
3393 if (!rmem->pg_tbl)
3394 return -ENOMEM;
3395 }
3396
3397 for (i = 0; i < rmem->nr_pages; i++) {
3398 u64 extra_bits = valid_bit;
3399
3400 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3401 rmem->page_size,
3402 &rmem->dma_arr[i],
3403 GFP_KERNEL);
3404 if (!rmem->pg_arr[i])
3405 return -ENOMEM;
3406
3407 if (rmem->ctx_mem)
3408 bnxt_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i],
3409 rmem->page_size);
3410 if (rmem->nr_pages > 1 || rmem->depth > 0) {
3411 if (i == rmem->nr_pages - 2 &&
3412 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3413 extra_bits |= PTU_PTE_NEXT_TO_LAST;
3414 else if (i == rmem->nr_pages - 1 &&
3415 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3416 extra_bits |= PTU_PTE_LAST;
3417 rmem->pg_tbl[i] =
3418 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3419 }
3420 }
3421
3422 if (rmem->vmem_size) {
3423 *rmem->vmem = vzalloc(rmem->vmem_size);
3424 if (!(*rmem->vmem))
3425 return -ENOMEM;
3426 }
3427 return 0;
3428}
3429
3430static void bnxt_free_tpa_info(struct bnxt *bp)
3431{
3432 int i, j;
3433
3434 for (i = 0; i < bp->rx_nr_rings; i++) {
3435 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3436
3437 kfree(rxr->rx_tpa_idx_map);
3438 rxr->rx_tpa_idx_map = NULL;
3439 if (rxr->rx_tpa) {
3440 for (j = 0; j < bp->max_tpa; j++) {
3441 kfree(rxr->rx_tpa[j].agg_arr);
3442 rxr->rx_tpa[j].agg_arr = NULL;
3443 }
3444 }
3445 kfree(rxr->rx_tpa);
3446 rxr->rx_tpa = NULL;
3447 }
3448}
3449
3450static int bnxt_alloc_tpa_info(struct bnxt *bp)
3451{
3452 int i, j;
3453
3454 bp->max_tpa = MAX_TPA;
3455 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
3456 if (!bp->max_tpa_v2)
3457 return 0;
3458 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3459 }
3460
3461 for (i = 0; i < bp->rx_nr_rings; i++) {
3462 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3463 struct rx_agg_cmp *agg;
3464
3465 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3466 GFP_KERNEL);
3467 if (!rxr->rx_tpa)
3468 return -ENOMEM;
3469
3470 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3471 continue;
3472 for (j = 0; j < bp->max_tpa; j++) {
3473 agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3474 if (!agg)
3475 return -ENOMEM;
3476 rxr->rx_tpa[j].agg_arr = agg;
3477 }
3478 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3479 GFP_KERNEL);
3480 if (!rxr->rx_tpa_idx_map)
3481 return -ENOMEM;
3482 }
3483 return 0;
3484}
3485
3486static void bnxt_free_rx_rings(struct bnxt *bp)
3487{
3488 int i;
3489
3490 if (!bp->rx_ring)
3491 return;
3492
3493 bnxt_free_tpa_info(bp);
3494 for (i = 0; i < bp->rx_nr_rings; i++) {
3495 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3496 struct bnxt_ring_struct *ring;
3497
3498 if (rxr->xdp_prog)
3499 bpf_prog_put(rxr->xdp_prog);
3500
3501 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3502 xdp_rxq_info_unreg(&rxr->xdp_rxq);
3503
3504 page_pool_destroy(rxr->page_pool);
3505 rxr->page_pool = NULL;
3506
3507 kfree(rxr->rx_agg_bmap);
3508 rxr->rx_agg_bmap = NULL;
3509
3510 ring = &rxr->rx_ring_struct;
3511 bnxt_free_ring(bp, &ring->ring_mem);
3512
3513 ring = &rxr->rx_agg_ring_struct;
3514 bnxt_free_ring(bp, &ring->ring_mem);
3515 }
3516}
3517
3518static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3519 struct bnxt_rx_ring_info *rxr)
3520{
3521 struct page_pool_params pp = { 0 };
3522
3523 pp.pool_size = bp->rx_agg_ring_size;
3524 if (BNXT_RX_PAGE_MODE(bp))
3525 pp.pool_size += bp->rx_ring_size;
3526 pp.nid = dev_to_node(&bp->pdev->dev);
3527 pp.napi = &rxr->bnapi->napi;
3528 pp.netdev = bp->dev;
3529 pp.dev = &bp->pdev->dev;
3530 pp.dma_dir = bp->rx_dir;
3531 pp.max_len = PAGE_SIZE;
3532 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
3533
3534 rxr->page_pool = page_pool_create(&pp);
3535 if (IS_ERR(rxr->page_pool)) {
3536 int err = PTR_ERR(rxr->page_pool);
3537
3538 rxr->page_pool = NULL;
3539 return err;
3540 }
3541 return 0;
3542}
3543
3544static int bnxt_alloc_rx_rings(struct bnxt *bp)
3545{
3546 int i, rc = 0, agg_rings = 0;
3547
3548 if (!bp->rx_ring)
3549 return -ENOMEM;
3550
3551 if (bp->flags & BNXT_FLAG_AGG_RINGS)
3552 agg_rings = 1;
3553
3554 for (i = 0; i < bp->rx_nr_rings; i++) {
3555 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3556 struct bnxt_ring_struct *ring;
3557
3558 ring = &rxr->rx_ring_struct;
3559
3560 rc = bnxt_alloc_rx_page_pool(bp, rxr);
3561 if (rc)
3562 return rc;
3563
3564 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3565 if (rc < 0)
3566 return rc;
3567
3568 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3569 MEM_TYPE_PAGE_POOL,
3570 rxr->page_pool);
3571 if (rc) {
3572 xdp_rxq_info_unreg(&rxr->xdp_rxq);
3573 return rc;
3574 }
3575
3576 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3577 if (rc)
3578 return rc;
3579
3580 ring->grp_idx = i;
3581 if (agg_rings) {
3582 u16 mem_size;
3583
3584 ring = &rxr->rx_agg_ring_struct;
3585 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3586 if (rc)
3587 return rc;
3588
3589 ring->grp_idx = i;
3590 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3591 mem_size = rxr->rx_agg_bmap_size / 8;
3592 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3593 if (!rxr->rx_agg_bmap)
3594 return -ENOMEM;
3595 }
3596 }
3597 if (bp->flags & BNXT_FLAG_TPA)
3598 rc = bnxt_alloc_tpa_info(bp);
3599 return rc;
3600}
3601
3602static void bnxt_free_tx_rings(struct bnxt *bp)
3603{
3604 int i;
3605 struct pci_dev *pdev = bp->pdev;
3606
3607 if (!bp->tx_ring)
3608 return;
3609
3610 for (i = 0; i < bp->tx_nr_rings; i++) {
3611 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3612 struct bnxt_ring_struct *ring;
3613
3614 if (txr->tx_push) {
3615 dma_free_coherent(&pdev->dev, bp->tx_push_size,
3616 txr->tx_push, txr->tx_push_mapping);
3617 txr->tx_push = NULL;
3618 }
3619
3620 ring = &txr->tx_ring_struct;
3621
3622 bnxt_free_ring(bp, &ring->ring_mem);
3623 }
3624}
3625
3626#define BNXT_TC_TO_RING_BASE(bp, tc) \
3627 ((tc) * (bp)->tx_nr_rings_per_tc)
3628
3629#define BNXT_RING_TO_TC_OFF(bp, tx) \
3630 ((tx) % (bp)->tx_nr_rings_per_tc)
3631
3632#define BNXT_RING_TO_TC(bp, tx) \
3633 ((tx) / (bp)->tx_nr_rings_per_tc)
3634
3635static int bnxt_alloc_tx_rings(struct bnxt *bp)
3636{
3637 int i, j, rc;
3638 struct pci_dev *pdev = bp->pdev;
3639
3640 bp->tx_push_size = 0;
3641 if (bp->tx_push_thresh) {
3642 int push_size;
3643
3644 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3645 bp->tx_push_thresh);
3646
3647 if (push_size > 256) {
3648 push_size = 0;
3649 bp->tx_push_thresh = 0;
3650 }
3651
3652 bp->tx_push_size = push_size;
3653 }
3654
3655 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3656 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3657 struct bnxt_ring_struct *ring;
3658 u8 qidx;
3659
3660 ring = &txr->tx_ring_struct;
3661
3662 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3663 if (rc)
3664 return rc;
3665
3666 ring->grp_idx = txr->bnapi->index;
3667 if (bp->tx_push_size) {
3668 dma_addr_t mapping;
3669
3670 /* One pre-allocated DMA buffer to backup
3671 * TX push operation
3672 */
3673 txr->tx_push = dma_alloc_coherent(&pdev->dev,
3674 bp->tx_push_size,
3675 &txr->tx_push_mapping,
3676 GFP_KERNEL);
3677
3678 if (!txr->tx_push)
3679 return -ENOMEM;
3680
3681 mapping = txr->tx_push_mapping +
3682 sizeof(struct tx_push_bd);
3683 txr->data_mapping = cpu_to_le64(mapping);
3684 }
3685 qidx = bp->tc_to_qidx[j];
3686 ring->queue_id = bp->q_info[qidx].queue_id;
3687 spin_lock_init(&txr->xdp_tx_lock);
3688 if (i < bp->tx_nr_rings_xdp)
3689 continue;
3690 if (BNXT_RING_TO_TC_OFF(bp, i) == (bp->tx_nr_rings_per_tc - 1))
3691 j++;
3692 }
3693 return 0;
3694}
3695
3696static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3697{
3698 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3699
3700 kfree(cpr->cp_desc_ring);
3701 cpr->cp_desc_ring = NULL;
3702 ring->ring_mem.pg_arr = NULL;
3703 kfree(cpr->cp_desc_mapping);
3704 cpr->cp_desc_mapping = NULL;
3705 ring->ring_mem.dma_arr = NULL;
3706}
3707
3708static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3709{
3710 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3711 if (!cpr->cp_desc_ring)
3712 return -ENOMEM;
3713 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3714 GFP_KERNEL);
3715 if (!cpr->cp_desc_mapping)
3716 return -ENOMEM;
3717 return 0;
3718}
3719
3720static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3721{
3722 int i;
3723
3724 if (!bp->bnapi)
3725 return;
3726 for (i = 0; i < bp->cp_nr_rings; i++) {
3727 struct bnxt_napi *bnapi = bp->bnapi[i];
3728
3729 if (!bnapi)
3730 continue;
3731 bnxt_free_cp_arrays(&bnapi->cp_ring);
3732 }
3733}
3734
3735static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3736{
3737 int i, n = bp->cp_nr_pages;
3738
3739 for (i = 0; i < bp->cp_nr_rings; i++) {
3740 struct bnxt_napi *bnapi = bp->bnapi[i];
3741 int rc;
3742
3743 if (!bnapi)
3744 continue;
3745 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3746 if (rc)
3747 return rc;
3748 }
3749 return 0;
3750}
3751
3752static void bnxt_free_cp_rings(struct bnxt *bp)
3753{
3754 int i;
3755
3756 if (!bp->bnapi)
3757 return;
3758
3759 for (i = 0; i < bp->cp_nr_rings; i++) {
3760 struct bnxt_napi *bnapi = bp->bnapi[i];
3761 struct bnxt_cp_ring_info *cpr;
3762 struct bnxt_ring_struct *ring;
3763 int j;
3764
3765 if (!bnapi)
3766 continue;
3767
3768 cpr = &bnapi->cp_ring;
3769 ring = &cpr->cp_ring_struct;
3770
3771 bnxt_free_ring(bp, &ring->ring_mem);
3772
3773 if (!cpr->cp_ring_arr)
3774 continue;
3775
3776 for (j = 0; j < cpr->cp_ring_count; j++) {
3777 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
3778
3779 ring = &cpr2->cp_ring_struct;
3780 bnxt_free_ring(bp, &ring->ring_mem);
3781 bnxt_free_cp_arrays(cpr2);
3782 }
3783 kfree(cpr->cp_ring_arr);
3784 cpr->cp_ring_arr = NULL;
3785 cpr->cp_ring_count = 0;
3786 }
3787}
3788
3789static int bnxt_alloc_cp_sub_ring(struct bnxt *bp,
3790 struct bnxt_cp_ring_info *cpr)
3791{
3792 struct bnxt_ring_mem_info *rmem;
3793 struct bnxt_ring_struct *ring;
3794 int rc;
3795
3796 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3797 if (rc) {
3798 bnxt_free_cp_arrays(cpr);
3799 return -ENOMEM;
3800 }
3801 ring = &cpr->cp_ring_struct;
3802 rmem = &ring->ring_mem;
3803 rmem->nr_pages = bp->cp_nr_pages;
3804 rmem->page_size = HW_CMPD_RING_SIZE;
3805 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3806 rmem->dma_arr = cpr->cp_desc_mapping;
3807 rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3808 rc = bnxt_alloc_ring(bp, rmem);
3809 if (rc) {
3810 bnxt_free_ring(bp, rmem);
3811 bnxt_free_cp_arrays(cpr);
3812 }
3813 return rc;
3814}
3815
3816static int bnxt_alloc_cp_rings(struct bnxt *bp)
3817{
3818 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3819 int i, j, rc, ulp_base_vec, ulp_msix;
3820 int tcs = bp->num_tc;
3821
3822 if (!tcs)
3823 tcs = 1;
3824 ulp_msix = bnxt_get_ulp_msix_num(bp);
3825 ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3826 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
3827 struct bnxt_napi *bnapi = bp->bnapi[i];
3828 struct bnxt_cp_ring_info *cpr, *cpr2;
3829 struct bnxt_ring_struct *ring;
3830 int cp_count = 0, k;
3831 int rx = 0, tx = 0;
3832
3833 if (!bnapi)
3834 continue;
3835
3836 cpr = &bnapi->cp_ring;
3837 cpr->bnapi = bnapi;
3838 ring = &cpr->cp_ring_struct;
3839
3840 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3841 if (rc)
3842 return rc;
3843
3844 if (ulp_msix && i >= ulp_base_vec)
3845 ring->map_idx = i + ulp_msix;
3846 else
3847 ring->map_idx = i;
3848
3849 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3850 continue;
3851
3852 if (i < bp->rx_nr_rings) {
3853 cp_count++;
3854 rx = 1;
3855 }
3856 if (i < bp->tx_nr_rings_xdp) {
3857 cp_count++;
3858 tx = 1;
3859 } else if ((sh && i < bp->tx_nr_rings) ||
3860 (!sh && i >= bp->rx_nr_rings)) {
3861 cp_count += tcs;
3862 tx = 1;
3863 }
3864
3865 cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr),
3866 GFP_KERNEL);
3867 if (!cpr->cp_ring_arr)
3868 return -ENOMEM;
3869 cpr->cp_ring_count = cp_count;
3870
3871 for (k = 0; k < cp_count; k++) {
3872 cpr2 = &cpr->cp_ring_arr[k];
3873 rc = bnxt_alloc_cp_sub_ring(bp, cpr2);
3874 if (rc)
3875 return rc;
3876 cpr2->bnapi = bnapi;
3877 cpr2->cp_idx = k;
3878 if (!k && rx) {
3879 bp->rx_ring[i].rx_cpr = cpr2;
3880 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_RX;
3881 } else {
3882 int n, tc = k - rx;
3883
3884 n = BNXT_TC_TO_RING_BASE(bp, tc) + j;
3885 bp->tx_ring[n].tx_cpr = cpr2;
3886 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_TX;
3887 }
3888 }
3889 if (tx)
3890 j++;
3891 }
3892 return 0;
3893}
3894
3895static void bnxt_init_ring_struct(struct bnxt *bp)
3896{
3897 int i, j;
3898
3899 for (i = 0; i < bp->cp_nr_rings; i++) {
3900 struct bnxt_napi *bnapi = bp->bnapi[i];
3901 struct bnxt_ring_mem_info *rmem;
3902 struct bnxt_cp_ring_info *cpr;
3903 struct bnxt_rx_ring_info *rxr;
3904 struct bnxt_tx_ring_info *txr;
3905 struct bnxt_ring_struct *ring;
3906
3907 if (!bnapi)
3908 continue;
3909
3910 cpr = &bnapi->cp_ring;
3911 ring = &cpr->cp_ring_struct;
3912 rmem = &ring->ring_mem;
3913 rmem->nr_pages = bp->cp_nr_pages;
3914 rmem->page_size = HW_CMPD_RING_SIZE;
3915 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3916 rmem->dma_arr = cpr->cp_desc_mapping;
3917 rmem->vmem_size = 0;
3918
3919 rxr = bnapi->rx_ring;
3920 if (!rxr)
3921 goto skip_rx;
3922
3923 ring = &rxr->rx_ring_struct;
3924 rmem = &ring->ring_mem;
3925 rmem->nr_pages = bp->rx_nr_pages;
3926 rmem->page_size = HW_RXBD_RING_SIZE;
3927 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3928 rmem->dma_arr = rxr->rx_desc_mapping;
3929 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3930 rmem->vmem = (void **)&rxr->rx_buf_ring;
3931
3932 ring = &rxr->rx_agg_ring_struct;
3933 rmem = &ring->ring_mem;
3934 rmem->nr_pages = bp->rx_agg_nr_pages;
3935 rmem->page_size = HW_RXBD_RING_SIZE;
3936 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3937 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3938 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3939 rmem->vmem = (void **)&rxr->rx_agg_ring;
3940
3941skip_rx:
3942 bnxt_for_each_napi_tx(j, bnapi, txr) {
3943 ring = &txr->tx_ring_struct;
3944 rmem = &ring->ring_mem;
3945 rmem->nr_pages = bp->tx_nr_pages;
3946 rmem->page_size = HW_TXBD_RING_SIZE;
3947 rmem->pg_arr = (void **)txr->tx_desc_ring;
3948 rmem->dma_arr = txr->tx_desc_mapping;
3949 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3950 rmem->vmem = (void **)&txr->tx_buf_ring;
3951 }
3952 }
3953}
3954
3955static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3956{
3957 int i;
3958 u32 prod;
3959 struct rx_bd **rx_buf_ring;
3960
3961 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3962 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3963 int j;
3964 struct rx_bd *rxbd;
3965
3966 rxbd = rx_buf_ring[i];
3967 if (!rxbd)
3968 continue;
3969
3970 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3971 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3972 rxbd->rx_bd_opaque = prod;
3973 }
3974 }
3975}
3976
3977static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3978{
3979 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3980 struct net_device *dev = bp->dev;
3981 u32 prod;
3982 int i;
3983
3984 prod = rxr->rx_prod;
3985 for (i = 0; i < bp->rx_ring_size; i++) {
3986 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3987 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3988 ring_nr, i, bp->rx_ring_size);
3989 break;
3990 }
3991 prod = NEXT_RX(prod);
3992 }
3993 rxr->rx_prod = prod;
3994
3995 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3996 return 0;
3997
3998 prod = rxr->rx_agg_prod;
3999 for (i = 0; i < bp->rx_agg_ring_size; i++) {
4000 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
4001 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
4002 ring_nr, i, bp->rx_ring_size);
4003 break;
4004 }
4005 prod = NEXT_RX_AGG(prod);
4006 }
4007 rxr->rx_agg_prod = prod;
4008
4009 if (rxr->rx_tpa) {
4010 dma_addr_t mapping;
4011 u8 *data;
4012
4013 for (i = 0; i < bp->max_tpa; i++) {
4014 data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
4015 if (!data)
4016 return -ENOMEM;
4017
4018 rxr->rx_tpa[i].data = data;
4019 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
4020 rxr->rx_tpa[i].mapping = mapping;
4021 }
4022 }
4023 return 0;
4024}
4025
4026static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
4027{
4028 struct bnxt_rx_ring_info *rxr;
4029 struct bnxt_ring_struct *ring;
4030 u32 type;
4031
4032 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
4033 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
4034
4035 if (NET_IP_ALIGN == 2)
4036 type |= RX_BD_FLAGS_SOP;
4037
4038 rxr = &bp->rx_ring[ring_nr];
4039 ring = &rxr->rx_ring_struct;
4040 bnxt_init_rxbd_pages(ring, type);
4041
4042 netif_queue_set_napi(bp->dev, ring_nr, NETDEV_QUEUE_TYPE_RX,
4043 &rxr->bnapi->napi);
4044
4045 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
4046 bpf_prog_add(bp->xdp_prog, 1);
4047 rxr->xdp_prog = bp->xdp_prog;
4048 }
4049 ring->fw_ring_id = INVALID_HW_RING_ID;
4050
4051 ring = &rxr->rx_agg_ring_struct;
4052 ring->fw_ring_id = INVALID_HW_RING_ID;
4053
4054 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
4055 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
4056 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
4057
4058 bnxt_init_rxbd_pages(ring, type);
4059 }
4060
4061 return bnxt_alloc_one_rx_ring(bp, ring_nr);
4062}
4063
4064static void bnxt_init_cp_rings(struct bnxt *bp)
4065{
4066 int i, j;
4067
4068 for (i = 0; i < bp->cp_nr_rings; i++) {
4069 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
4070 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4071
4072 ring->fw_ring_id = INVALID_HW_RING_ID;
4073 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4074 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4075 if (!cpr->cp_ring_arr)
4076 continue;
4077 for (j = 0; j < cpr->cp_ring_count; j++) {
4078 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
4079
4080 ring = &cpr2->cp_ring_struct;
4081 ring->fw_ring_id = INVALID_HW_RING_ID;
4082 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4083 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4084 }
4085 }
4086}
4087
4088static int bnxt_init_rx_rings(struct bnxt *bp)
4089{
4090 int i, rc = 0;
4091
4092 if (BNXT_RX_PAGE_MODE(bp)) {
4093 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
4094 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
4095 } else {
4096 bp->rx_offset = BNXT_RX_OFFSET;
4097 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
4098 }
4099
4100 for (i = 0; i < bp->rx_nr_rings; i++) {
4101 rc = bnxt_init_one_rx_ring(bp, i);
4102 if (rc)
4103 break;
4104 }
4105
4106 return rc;
4107}
4108
4109static int bnxt_init_tx_rings(struct bnxt *bp)
4110{
4111 u16 i;
4112
4113 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
4114 BNXT_MIN_TX_DESC_CNT);
4115
4116 for (i = 0; i < bp->tx_nr_rings; i++) {
4117 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4118 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4119
4120 ring->fw_ring_id = INVALID_HW_RING_ID;
4121
4122 if (i >= bp->tx_nr_rings_xdp)
4123 netif_queue_set_napi(bp->dev, i - bp->tx_nr_rings_xdp,
4124 NETDEV_QUEUE_TYPE_TX,
4125 &txr->bnapi->napi);
4126 }
4127
4128 return 0;
4129}
4130
4131static void bnxt_free_ring_grps(struct bnxt *bp)
4132{
4133 kfree(bp->grp_info);
4134 bp->grp_info = NULL;
4135}
4136
4137static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
4138{
4139 int i;
4140
4141 if (irq_re_init) {
4142 bp->grp_info = kcalloc(bp->cp_nr_rings,
4143 sizeof(struct bnxt_ring_grp_info),
4144 GFP_KERNEL);
4145 if (!bp->grp_info)
4146 return -ENOMEM;
4147 }
4148 for (i = 0; i < bp->cp_nr_rings; i++) {
4149 if (irq_re_init)
4150 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
4151 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4152 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
4153 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
4154 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
4155 }
4156 return 0;
4157}
4158
4159static void bnxt_free_vnics(struct bnxt *bp)
4160{
4161 kfree(bp->vnic_info);
4162 bp->vnic_info = NULL;
4163 bp->nr_vnics = 0;
4164}
4165
4166static int bnxt_alloc_vnics(struct bnxt *bp)
4167{
4168 int num_vnics = 1;
4169
4170#ifdef CONFIG_RFS_ACCEL
4171 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5_PLUS)) == BNXT_FLAG_RFS)
4172 num_vnics += bp->rx_nr_rings;
4173#endif
4174
4175 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4176 num_vnics++;
4177
4178 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
4179 GFP_KERNEL);
4180 if (!bp->vnic_info)
4181 return -ENOMEM;
4182
4183 bp->nr_vnics = num_vnics;
4184 return 0;
4185}
4186
4187static void bnxt_init_vnics(struct bnxt *bp)
4188{
4189 int i;
4190
4191 for (i = 0; i < bp->nr_vnics; i++) {
4192 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4193 int j;
4194
4195 vnic->fw_vnic_id = INVALID_HW_RING_ID;
4196 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
4197 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
4198
4199 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
4200
4201 if (bp->vnic_info[i].rss_hash_key) {
4202 if (!i) {
4203 u8 *key = (void *)vnic->rss_hash_key;
4204 int k;
4205
4206 bp->toeplitz_prefix = 0;
4207 get_random_bytes(vnic->rss_hash_key,
4208 HW_HASH_KEY_SIZE);
4209 for (k = 0; k < 8; k++) {
4210 bp->toeplitz_prefix <<= 8;
4211 bp->toeplitz_prefix |= key[k];
4212 }
4213 } else {
4214 memcpy(vnic->rss_hash_key,
4215 bp->vnic_info[0].rss_hash_key,
4216 HW_HASH_KEY_SIZE);
4217 }
4218 }
4219 }
4220}
4221
4222static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
4223{
4224 int pages;
4225
4226 pages = ring_size / desc_per_pg;
4227
4228 if (!pages)
4229 return 1;
4230
4231 pages++;
4232
4233 while (pages & (pages - 1))
4234 pages++;
4235
4236 return pages;
4237}
4238
4239void bnxt_set_tpa_flags(struct bnxt *bp)
4240{
4241 bp->flags &= ~BNXT_FLAG_TPA;
4242 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
4243 return;
4244 if (bp->dev->features & NETIF_F_LRO)
4245 bp->flags |= BNXT_FLAG_LRO;
4246 else if (bp->dev->features & NETIF_F_GRO_HW)
4247 bp->flags |= BNXT_FLAG_GRO;
4248}
4249
4250/* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
4251 * be set on entry.
4252 */
4253void bnxt_set_ring_params(struct bnxt *bp)
4254{
4255 u32 ring_size, rx_size, rx_space, max_rx_cmpl;
4256 u32 agg_factor = 0, agg_ring_size = 0;
4257
4258 /* 8 for CRC and VLAN */
4259 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
4260
4261 rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
4262 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4263
4264 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
4265 ring_size = bp->rx_ring_size;
4266 bp->rx_agg_ring_size = 0;
4267 bp->rx_agg_nr_pages = 0;
4268
4269 if (bp->flags & BNXT_FLAG_TPA)
4270 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
4271
4272 bp->flags &= ~BNXT_FLAG_JUMBO;
4273 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
4274 u32 jumbo_factor;
4275
4276 bp->flags |= BNXT_FLAG_JUMBO;
4277 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
4278 if (jumbo_factor > agg_factor)
4279 agg_factor = jumbo_factor;
4280 }
4281 if (agg_factor) {
4282 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
4283 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
4284 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
4285 bp->rx_ring_size, ring_size);
4286 bp->rx_ring_size = ring_size;
4287 }
4288 agg_ring_size = ring_size * agg_factor;
4289
4290 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
4291 RX_DESC_CNT);
4292 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
4293 u32 tmp = agg_ring_size;
4294
4295 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
4296 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
4297 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
4298 tmp, agg_ring_size);
4299 }
4300 bp->rx_agg_ring_size = agg_ring_size;
4301 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
4302
4303 if (BNXT_RX_PAGE_MODE(bp)) {
4304 rx_space = PAGE_SIZE;
4305 rx_size = PAGE_SIZE -
4306 ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
4307 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4308 } else {
4309 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
4310 rx_space = rx_size + NET_SKB_PAD +
4311 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4312 }
4313 }
4314
4315 bp->rx_buf_use_size = rx_size;
4316 bp->rx_buf_size = rx_space;
4317
4318 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
4319 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
4320
4321 ring_size = bp->tx_ring_size;
4322 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
4323 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
4324
4325 max_rx_cmpl = bp->rx_ring_size;
4326 /* MAX TPA needs to be added because TPA_START completions are
4327 * immediately recycled, so the TPA completions are not bound by
4328 * the RX ring size.
4329 */
4330 if (bp->flags & BNXT_FLAG_TPA)
4331 max_rx_cmpl += bp->max_tpa;
4332 /* RX and TPA completions are 32-byte, all others are 16-byte */
4333 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
4334 bp->cp_ring_size = ring_size;
4335
4336 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
4337 if (bp->cp_nr_pages > MAX_CP_PAGES) {
4338 bp->cp_nr_pages = MAX_CP_PAGES;
4339 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
4340 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
4341 ring_size, bp->cp_ring_size);
4342 }
4343 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
4344 bp->cp_ring_mask = bp->cp_bit - 1;
4345}
4346
4347/* Changing allocation mode of RX rings.
4348 * TODO: Update when extending xdp_rxq_info to support allocation modes.
4349 */
4350int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
4351{
4352 struct net_device *dev = bp->dev;
4353
4354 if (page_mode) {
4355 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
4356 bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
4357
4358 if (bp->xdp_prog->aux->xdp_has_frags)
4359 dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
4360 else
4361 dev->max_mtu =
4362 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
4363 if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
4364 bp->flags |= BNXT_FLAG_JUMBO;
4365 bp->rx_skb_func = bnxt_rx_multi_page_skb;
4366 } else {
4367 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4368 bp->rx_skb_func = bnxt_rx_page_skb;
4369 }
4370 bp->rx_dir = DMA_BIDIRECTIONAL;
4371 /* Disable LRO or GRO_HW */
4372 netdev_update_features(dev);
4373 } else {
4374 dev->max_mtu = bp->max_mtu;
4375 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4376 bp->rx_dir = DMA_FROM_DEVICE;
4377 bp->rx_skb_func = bnxt_rx_skb;
4378 }
4379 return 0;
4380}
4381
4382static void bnxt_free_vnic_attributes(struct bnxt *bp)
4383{
4384 int i;
4385 struct bnxt_vnic_info *vnic;
4386 struct pci_dev *pdev = bp->pdev;
4387
4388 if (!bp->vnic_info)
4389 return;
4390
4391 for (i = 0; i < bp->nr_vnics; i++) {
4392 vnic = &bp->vnic_info[i];
4393
4394 kfree(vnic->fw_grp_ids);
4395 vnic->fw_grp_ids = NULL;
4396
4397 kfree(vnic->uc_list);
4398 vnic->uc_list = NULL;
4399
4400 if (vnic->mc_list) {
4401 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4402 vnic->mc_list, vnic->mc_list_mapping);
4403 vnic->mc_list = NULL;
4404 }
4405
4406 if (vnic->rss_table) {
4407 dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4408 vnic->rss_table,
4409 vnic->rss_table_dma_addr);
4410 vnic->rss_table = NULL;
4411 }
4412
4413 vnic->rss_hash_key = NULL;
4414 vnic->flags = 0;
4415 }
4416}
4417
4418static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4419{
4420 int i, rc = 0, size;
4421 struct bnxt_vnic_info *vnic;
4422 struct pci_dev *pdev = bp->pdev;
4423 int max_rings;
4424
4425 for (i = 0; i < bp->nr_vnics; i++) {
4426 vnic = &bp->vnic_info[i];
4427
4428 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4429 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4430
4431 if (mem_size > 0) {
4432 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4433 if (!vnic->uc_list) {
4434 rc = -ENOMEM;
4435 goto out;
4436 }
4437 }
4438 }
4439
4440 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4441 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4442 vnic->mc_list =
4443 dma_alloc_coherent(&pdev->dev,
4444 vnic->mc_list_size,
4445 &vnic->mc_list_mapping,
4446 GFP_KERNEL);
4447 if (!vnic->mc_list) {
4448 rc = -ENOMEM;
4449 goto out;
4450 }
4451 }
4452
4453 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4454 goto vnic_skip_grps;
4455
4456 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4457 max_rings = bp->rx_nr_rings;
4458 else
4459 max_rings = 1;
4460
4461 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4462 if (!vnic->fw_grp_ids) {
4463 rc = -ENOMEM;
4464 goto out;
4465 }
4466vnic_skip_grps:
4467 if ((bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) &&
4468 !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4469 continue;
4470
4471 /* Allocate rss table and hash key */
4472 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4473 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4474 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4475
4476 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4477 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4478 vnic->rss_table_size,
4479 &vnic->rss_table_dma_addr,
4480 GFP_KERNEL);
4481 if (!vnic->rss_table) {
4482 rc = -ENOMEM;
4483 goto out;
4484 }
4485
4486 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4487 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4488 }
4489 return 0;
4490
4491out:
4492 return rc;
4493}
4494
4495static void bnxt_free_hwrm_resources(struct bnxt *bp)
4496{
4497 struct bnxt_hwrm_wait_token *token;
4498
4499 dma_pool_destroy(bp->hwrm_dma_pool);
4500 bp->hwrm_dma_pool = NULL;
4501
4502 rcu_read_lock();
4503 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4504 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4505 rcu_read_unlock();
4506}
4507
4508static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4509{
4510 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4511 BNXT_HWRM_DMA_SIZE,
4512 BNXT_HWRM_DMA_ALIGN, 0);
4513 if (!bp->hwrm_dma_pool)
4514 return -ENOMEM;
4515
4516 INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4517
4518 return 0;
4519}
4520
4521static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4522{
4523 kfree(stats->hw_masks);
4524 stats->hw_masks = NULL;
4525 kfree(stats->sw_stats);
4526 stats->sw_stats = NULL;
4527 if (stats->hw_stats) {
4528 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4529 stats->hw_stats_map);
4530 stats->hw_stats = NULL;
4531 }
4532}
4533
4534static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4535 bool alloc_masks)
4536{
4537 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4538 &stats->hw_stats_map, GFP_KERNEL);
4539 if (!stats->hw_stats)
4540 return -ENOMEM;
4541
4542 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4543 if (!stats->sw_stats)
4544 goto stats_mem_err;
4545
4546 if (alloc_masks) {
4547 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4548 if (!stats->hw_masks)
4549 goto stats_mem_err;
4550 }
4551 return 0;
4552
4553stats_mem_err:
4554 bnxt_free_stats_mem(bp, stats);
4555 return -ENOMEM;
4556}
4557
4558static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4559{
4560 int i;
4561
4562 for (i = 0; i < count; i++)
4563 mask_arr[i] = mask;
4564}
4565
4566static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4567{
4568 int i;
4569
4570 for (i = 0; i < count; i++)
4571 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4572}
4573
4574static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4575 struct bnxt_stats_mem *stats)
4576{
4577 struct hwrm_func_qstats_ext_output *resp;
4578 struct hwrm_func_qstats_ext_input *req;
4579 __le64 *hw_masks;
4580 int rc;
4581
4582 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4583 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4584 return -EOPNOTSUPP;
4585
4586 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4587 if (rc)
4588 return rc;
4589
4590 req->fid = cpu_to_le16(0xffff);
4591 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4592
4593 resp = hwrm_req_hold(bp, req);
4594 rc = hwrm_req_send(bp, req);
4595 if (!rc) {
4596 hw_masks = &resp->rx_ucast_pkts;
4597 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4598 }
4599 hwrm_req_drop(bp, req);
4600 return rc;
4601}
4602
4603static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4604static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4605
4606static void bnxt_init_stats(struct bnxt *bp)
4607{
4608 struct bnxt_napi *bnapi = bp->bnapi[0];
4609 struct bnxt_cp_ring_info *cpr;
4610 struct bnxt_stats_mem *stats;
4611 __le64 *rx_stats, *tx_stats;
4612 int rc, rx_count, tx_count;
4613 u64 *rx_masks, *tx_masks;
4614 u64 mask;
4615 u8 flags;
4616
4617 cpr = &bnapi->cp_ring;
4618 stats = &cpr->stats;
4619 rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4620 if (rc) {
4621 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4622 mask = (1ULL << 48) - 1;
4623 else
4624 mask = -1ULL;
4625 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4626 }
4627 if (bp->flags & BNXT_FLAG_PORT_STATS) {
4628 stats = &bp->port_stats;
4629 rx_stats = stats->hw_stats;
4630 rx_masks = stats->hw_masks;
4631 rx_count = sizeof(struct rx_port_stats) / 8;
4632 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4633 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4634 tx_count = sizeof(struct tx_port_stats) / 8;
4635
4636 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4637 rc = bnxt_hwrm_port_qstats(bp, flags);
4638 if (rc) {
4639 mask = (1ULL << 40) - 1;
4640
4641 bnxt_fill_masks(rx_masks, mask, rx_count);
4642 bnxt_fill_masks(tx_masks, mask, tx_count);
4643 } else {
4644 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4645 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4646 bnxt_hwrm_port_qstats(bp, 0);
4647 }
4648 }
4649 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4650 stats = &bp->rx_port_stats_ext;
4651 rx_stats = stats->hw_stats;
4652 rx_masks = stats->hw_masks;
4653 rx_count = sizeof(struct rx_port_stats_ext) / 8;
4654 stats = &bp->tx_port_stats_ext;
4655 tx_stats = stats->hw_stats;
4656 tx_masks = stats->hw_masks;
4657 tx_count = sizeof(struct tx_port_stats_ext) / 8;
4658
4659 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4660 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4661 if (rc) {
4662 mask = (1ULL << 40) - 1;
4663
4664 bnxt_fill_masks(rx_masks, mask, rx_count);
4665 if (tx_stats)
4666 bnxt_fill_masks(tx_masks, mask, tx_count);
4667 } else {
4668 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4669 if (tx_stats)
4670 bnxt_copy_hw_masks(tx_masks, tx_stats,
4671 tx_count);
4672 bnxt_hwrm_port_qstats_ext(bp, 0);
4673 }
4674 }
4675}
4676
4677static void bnxt_free_port_stats(struct bnxt *bp)
4678{
4679 bp->flags &= ~BNXT_FLAG_PORT_STATS;
4680 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4681
4682 bnxt_free_stats_mem(bp, &bp->port_stats);
4683 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4684 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4685}
4686
4687static void bnxt_free_ring_stats(struct bnxt *bp)
4688{
4689 int i;
4690
4691 if (!bp->bnapi)
4692 return;
4693
4694 for (i = 0; i < bp->cp_nr_rings; i++) {
4695 struct bnxt_napi *bnapi = bp->bnapi[i];
4696 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4697
4698 bnxt_free_stats_mem(bp, &cpr->stats);
4699 }
4700}
4701
4702static int bnxt_alloc_stats(struct bnxt *bp)
4703{
4704 u32 size, i;
4705 int rc;
4706
4707 size = bp->hw_ring_stats_size;
4708
4709 for (i = 0; i < bp->cp_nr_rings; i++) {
4710 struct bnxt_napi *bnapi = bp->bnapi[i];
4711 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4712
4713 cpr->stats.len = size;
4714 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4715 if (rc)
4716 return rc;
4717
4718 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4719 }
4720
4721 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4722 return 0;
4723
4724 if (bp->port_stats.hw_stats)
4725 goto alloc_ext_stats;
4726
4727 bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4728 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4729 if (rc)
4730 return rc;
4731
4732 bp->flags |= BNXT_FLAG_PORT_STATS;
4733
4734alloc_ext_stats:
4735 /* Display extended statistics only if FW supports it */
4736 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4737 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4738 return 0;
4739
4740 if (bp->rx_port_stats_ext.hw_stats)
4741 goto alloc_tx_ext_stats;
4742
4743 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4744 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4745 /* Extended stats are optional */
4746 if (rc)
4747 return 0;
4748
4749alloc_tx_ext_stats:
4750 if (bp->tx_port_stats_ext.hw_stats)
4751 return 0;
4752
4753 if (bp->hwrm_spec_code >= 0x10902 ||
4754 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4755 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4756 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4757 /* Extended stats are optional */
4758 if (rc)
4759 return 0;
4760 }
4761 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4762 return 0;
4763}
4764
4765static void bnxt_clear_ring_indices(struct bnxt *bp)
4766{
4767 int i, j;
4768
4769 if (!bp->bnapi)
4770 return;
4771
4772 for (i = 0; i < bp->cp_nr_rings; i++) {
4773 struct bnxt_napi *bnapi = bp->bnapi[i];
4774 struct bnxt_cp_ring_info *cpr;
4775 struct bnxt_rx_ring_info *rxr;
4776 struct bnxt_tx_ring_info *txr;
4777
4778 if (!bnapi)
4779 continue;
4780
4781 cpr = &bnapi->cp_ring;
4782 cpr->cp_raw_cons = 0;
4783
4784 bnxt_for_each_napi_tx(j, bnapi, txr) {
4785 txr->tx_prod = 0;
4786 txr->tx_cons = 0;
4787 txr->tx_hw_cons = 0;
4788 }
4789
4790 rxr = bnapi->rx_ring;
4791 if (rxr) {
4792 rxr->rx_prod = 0;
4793 rxr->rx_agg_prod = 0;
4794 rxr->rx_sw_agg_prod = 0;
4795 rxr->rx_next_cons = 0;
4796 }
4797 bnapi->events = 0;
4798 }
4799}
4800
4801static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool all)
4802{
4803 int i;
4804
4805 /* Under rtnl_lock and all our NAPIs have been disabled. It's
4806 * safe to delete the hash table.
4807 */
4808 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4809 struct hlist_head *head;
4810 struct hlist_node *tmp;
4811 struct bnxt_ntuple_filter *fltr;
4812
4813 head = &bp->ntp_fltr_hash_tbl[i];
4814 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
4815 bnxt_del_l2_filter(bp, fltr->l2_fltr);
4816 if (!all && (fltr->base.flags & BNXT_ACT_FUNC_DST))
4817 continue;
4818 hlist_del(&fltr->base.hash);
4819 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
4820 bp->ntp_fltr_count--;
4821 kfree(fltr);
4822 }
4823 }
4824 if (!all)
4825 return;
4826
4827 bitmap_free(bp->ntp_fltr_bmap);
4828 bp->ntp_fltr_bmap = NULL;
4829 bp->ntp_fltr_count = 0;
4830}
4831
4832static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4833{
4834 int i, rc = 0;
4835
4836 if (!(bp->flags & BNXT_FLAG_RFS) || bp->ntp_fltr_bmap)
4837 return 0;
4838
4839 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4840 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4841
4842 bp->ntp_fltr_count = 0;
4843 bp->ntp_fltr_bmap = bitmap_zalloc(BNXT_MAX_FLTR, GFP_KERNEL);
4844
4845 if (!bp->ntp_fltr_bmap)
4846 rc = -ENOMEM;
4847
4848 return rc;
4849}
4850
4851static void bnxt_free_l2_filters(struct bnxt *bp, bool all)
4852{
4853 int i;
4854
4855 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) {
4856 struct hlist_head *head;
4857 struct hlist_node *tmp;
4858 struct bnxt_l2_filter *fltr;
4859
4860 head = &bp->l2_fltr_hash_tbl[i];
4861 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
4862 if (!all && (fltr->base.flags & BNXT_ACT_FUNC_DST))
4863 continue;
4864 hlist_del(&fltr->base.hash);
4865 if (fltr->base.flags) {
4866 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
4867 bp->ntp_fltr_count--;
4868 }
4869 kfree(fltr);
4870 }
4871 }
4872}
4873
4874static void bnxt_init_l2_fltr_tbl(struct bnxt *bp)
4875{
4876 int i;
4877
4878 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++)
4879 INIT_HLIST_HEAD(&bp->l2_fltr_hash_tbl[i]);
4880 get_random_bytes(&bp->hash_seed, sizeof(bp->hash_seed));
4881}
4882
4883static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4884{
4885 bnxt_free_vnic_attributes(bp);
4886 bnxt_free_tx_rings(bp);
4887 bnxt_free_rx_rings(bp);
4888 bnxt_free_cp_rings(bp);
4889 bnxt_free_all_cp_arrays(bp);
4890 bnxt_free_ntp_fltrs(bp, false);
4891 bnxt_free_l2_filters(bp, false);
4892 if (irq_re_init) {
4893 bnxt_free_ring_stats(bp);
4894 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
4895 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4896 bnxt_free_port_stats(bp);
4897 bnxt_free_ring_grps(bp);
4898 bnxt_free_vnics(bp);
4899 kfree(bp->tx_ring_map);
4900 bp->tx_ring_map = NULL;
4901 kfree(bp->tx_ring);
4902 bp->tx_ring = NULL;
4903 kfree(bp->rx_ring);
4904 bp->rx_ring = NULL;
4905 kfree(bp->bnapi);
4906 bp->bnapi = NULL;
4907 } else {
4908 bnxt_clear_ring_indices(bp);
4909 }
4910}
4911
4912static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4913{
4914 int i, j, rc, size, arr_size;
4915 void *bnapi;
4916
4917 if (irq_re_init) {
4918 /* Allocate bnapi mem pointer array and mem block for
4919 * all queues
4920 */
4921 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4922 bp->cp_nr_rings);
4923 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4924 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4925 if (!bnapi)
4926 return -ENOMEM;
4927
4928 bp->bnapi = bnapi;
4929 bnapi += arr_size;
4930 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4931 bp->bnapi[i] = bnapi;
4932 bp->bnapi[i]->index = i;
4933 bp->bnapi[i]->bp = bp;
4934 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
4935 struct bnxt_cp_ring_info *cpr =
4936 &bp->bnapi[i]->cp_ring;
4937
4938 cpr->cp_ring_struct.ring_mem.flags =
4939 BNXT_RMEM_RING_PTE_FLAG;
4940 }
4941 }
4942
4943 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4944 sizeof(struct bnxt_rx_ring_info),
4945 GFP_KERNEL);
4946 if (!bp->rx_ring)
4947 return -ENOMEM;
4948
4949 for (i = 0; i < bp->rx_nr_rings; i++) {
4950 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4951
4952 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
4953 rxr->rx_ring_struct.ring_mem.flags =
4954 BNXT_RMEM_RING_PTE_FLAG;
4955 rxr->rx_agg_ring_struct.ring_mem.flags =
4956 BNXT_RMEM_RING_PTE_FLAG;
4957 } else {
4958 rxr->rx_cpr = &bp->bnapi[i]->cp_ring;
4959 }
4960 rxr->bnapi = bp->bnapi[i];
4961 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4962 }
4963
4964 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4965 sizeof(struct bnxt_tx_ring_info),
4966 GFP_KERNEL);
4967 if (!bp->tx_ring)
4968 return -ENOMEM;
4969
4970 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4971 GFP_KERNEL);
4972
4973 if (!bp->tx_ring_map)
4974 return -ENOMEM;
4975
4976 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4977 j = 0;
4978 else
4979 j = bp->rx_nr_rings;
4980
4981 for (i = 0; i < bp->tx_nr_rings; i++) {
4982 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4983 struct bnxt_napi *bnapi2;
4984
4985 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4986 txr->tx_ring_struct.ring_mem.flags =
4987 BNXT_RMEM_RING_PTE_FLAG;
4988 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4989 if (i >= bp->tx_nr_rings_xdp) {
4990 int k = j + BNXT_RING_TO_TC_OFF(bp, i);
4991
4992 bnapi2 = bp->bnapi[k];
4993 txr->txq_index = i - bp->tx_nr_rings_xdp;
4994 txr->tx_napi_idx =
4995 BNXT_RING_TO_TC(bp, txr->txq_index);
4996 bnapi2->tx_ring[txr->tx_napi_idx] = txr;
4997 bnapi2->tx_int = bnxt_tx_int;
4998 } else {
4999 bnapi2 = bp->bnapi[j];
5000 bnapi2->flags |= BNXT_NAPI_FLAG_XDP;
5001 bnapi2->tx_ring[0] = txr;
5002 bnapi2->tx_int = bnxt_tx_int_xdp;
5003 j++;
5004 }
5005 txr->bnapi = bnapi2;
5006 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
5007 txr->tx_cpr = &bnapi2->cp_ring;
5008 }
5009
5010 rc = bnxt_alloc_stats(bp);
5011 if (rc)
5012 goto alloc_mem_err;
5013 bnxt_init_stats(bp);
5014
5015 rc = bnxt_alloc_ntp_fltrs(bp);
5016 if (rc)
5017 goto alloc_mem_err;
5018
5019 rc = bnxt_alloc_vnics(bp);
5020 if (rc)
5021 goto alloc_mem_err;
5022 }
5023
5024 rc = bnxt_alloc_all_cp_arrays(bp);
5025 if (rc)
5026 goto alloc_mem_err;
5027
5028 bnxt_init_ring_struct(bp);
5029
5030 rc = bnxt_alloc_rx_rings(bp);
5031 if (rc)
5032 goto alloc_mem_err;
5033
5034 rc = bnxt_alloc_tx_rings(bp);
5035 if (rc)
5036 goto alloc_mem_err;
5037
5038 rc = bnxt_alloc_cp_rings(bp);
5039 if (rc)
5040 goto alloc_mem_err;
5041
5042 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
5043 BNXT_VNIC_UCAST_FLAG;
5044 rc = bnxt_alloc_vnic_attributes(bp);
5045 if (rc)
5046 goto alloc_mem_err;
5047 return 0;
5048
5049alloc_mem_err:
5050 bnxt_free_mem(bp, true);
5051 return rc;
5052}
5053
5054static void bnxt_disable_int(struct bnxt *bp)
5055{
5056 int i;
5057
5058 if (!bp->bnapi)
5059 return;
5060
5061 for (i = 0; i < bp->cp_nr_rings; i++) {
5062 struct bnxt_napi *bnapi = bp->bnapi[i];
5063 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5064 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5065
5066 if (ring->fw_ring_id != INVALID_HW_RING_ID)
5067 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5068 }
5069}
5070
5071static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
5072{
5073 struct bnxt_napi *bnapi = bp->bnapi[n];
5074 struct bnxt_cp_ring_info *cpr;
5075
5076 cpr = &bnapi->cp_ring;
5077 return cpr->cp_ring_struct.map_idx;
5078}
5079
5080static void bnxt_disable_int_sync(struct bnxt *bp)
5081{
5082 int i;
5083
5084 if (!bp->irq_tbl)
5085 return;
5086
5087 atomic_inc(&bp->intr_sem);
5088
5089 bnxt_disable_int(bp);
5090 for (i = 0; i < bp->cp_nr_rings; i++) {
5091 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
5092
5093 synchronize_irq(bp->irq_tbl[map_idx].vector);
5094 }
5095}
5096
5097static void bnxt_enable_int(struct bnxt *bp)
5098{
5099 int i;
5100
5101 atomic_set(&bp->intr_sem, 0);
5102 for (i = 0; i < bp->cp_nr_rings; i++) {
5103 struct bnxt_napi *bnapi = bp->bnapi[i];
5104 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5105
5106 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
5107 }
5108}
5109
5110int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
5111 bool async_only)
5112{
5113 DECLARE_BITMAP(async_events_bmap, 256);
5114 u32 *events = (u32 *)async_events_bmap;
5115 struct hwrm_func_drv_rgtr_output *resp;
5116 struct hwrm_func_drv_rgtr_input *req;
5117 u32 flags;
5118 int rc, i;
5119
5120 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
5121 if (rc)
5122 return rc;
5123
5124 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
5125 FUNC_DRV_RGTR_REQ_ENABLES_VER |
5126 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5127
5128 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
5129 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
5130 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
5131 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
5132 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
5133 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
5134 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
5135 req->flags = cpu_to_le32(flags);
5136 req->ver_maj_8b = DRV_VER_MAJ;
5137 req->ver_min_8b = DRV_VER_MIN;
5138 req->ver_upd_8b = DRV_VER_UPD;
5139 req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
5140 req->ver_min = cpu_to_le16(DRV_VER_MIN);
5141 req->ver_upd = cpu_to_le16(DRV_VER_UPD);
5142
5143 if (BNXT_PF(bp)) {
5144 u32 data[8];
5145 int i;
5146
5147 memset(data, 0, sizeof(data));
5148 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
5149 u16 cmd = bnxt_vf_req_snif[i];
5150 unsigned int bit, idx;
5151
5152 idx = cmd / 32;
5153 bit = cmd % 32;
5154 data[idx] |= 1 << bit;
5155 }
5156
5157 for (i = 0; i < 8; i++)
5158 req->vf_req_fwd[i] = cpu_to_le32(data[i]);
5159
5160 req->enables |=
5161 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
5162 }
5163
5164 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
5165 req->flags |= cpu_to_le32(
5166 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
5167
5168 memset(async_events_bmap, 0, sizeof(async_events_bmap));
5169 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
5170 u16 event_id = bnxt_async_events_arr[i];
5171
5172 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
5173 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
5174 continue;
5175 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE &&
5176 !bp->ptp_cfg)
5177 continue;
5178 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
5179 }
5180 if (bmap && bmap_size) {
5181 for (i = 0; i < bmap_size; i++) {
5182 if (test_bit(i, bmap))
5183 __set_bit(i, async_events_bmap);
5184 }
5185 }
5186 for (i = 0; i < 8; i++)
5187 req->async_event_fwd[i] |= cpu_to_le32(events[i]);
5188
5189 if (async_only)
5190 req->enables =
5191 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5192
5193 resp = hwrm_req_hold(bp, req);
5194 rc = hwrm_req_send(bp, req);
5195 if (!rc) {
5196 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
5197 if (resp->flags &
5198 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
5199 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
5200 }
5201 hwrm_req_drop(bp, req);
5202 return rc;
5203}
5204
5205int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
5206{
5207 struct hwrm_func_drv_unrgtr_input *req;
5208 int rc;
5209
5210 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
5211 return 0;
5212
5213 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
5214 if (rc)
5215 return rc;
5216 return hwrm_req_send(bp, req);
5217}
5218
5219static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa);
5220
5221static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
5222{
5223 struct hwrm_tunnel_dst_port_free_input *req;
5224 int rc;
5225
5226 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
5227 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
5228 return 0;
5229 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
5230 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
5231 return 0;
5232
5233 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
5234 if (rc)
5235 return rc;
5236
5237 req->tunnel_type = tunnel_type;
5238
5239 switch (tunnel_type) {
5240 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
5241 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
5242 bp->vxlan_port = 0;
5243 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
5244 break;
5245 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
5246 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
5247 bp->nge_port = 0;
5248 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
5249 break;
5250 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE:
5251 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_gpe_fw_dst_port_id);
5252 bp->vxlan_gpe_port = 0;
5253 bp->vxlan_gpe_fw_dst_port_id = INVALID_HW_RING_ID;
5254 break;
5255 default:
5256 break;
5257 }
5258
5259 rc = hwrm_req_send(bp, req);
5260 if (rc)
5261 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
5262 rc);
5263 if (bp->flags & BNXT_FLAG_TPA)
5264 bnxt_set_tpa(bp, true);
5265 return rc;
5266}
5267
5268static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
5269 u8 tunnel_type)
5270{
5271 struct hwrm_tunnel_dst_port_alloc_output *resp;
5272 struct hwrm_tunnel_dst_port_alloc_input *req;
5273 int rc;
5274
5275 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
5276 if (rc)
5277 return rc;
5278
5279 req->tunnel_type = tunnel_type;
5280 req->tunnel_dst_port_val = port;
5281
5282 resp = hwrm_req_hold(bp, req);
5283 rc = hwrm_req_send(bp, req);
5284 if (rc) {
5285 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
5286 rc);
5287 goto err_out;
5288 }
5289
5290 switch (tunnel_type) {
5291 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
5292 bp->vxlan_port = port;
5293 bp->vxlan_fw_dst_port_id =
5294 le16_to_cpu(resp->tunnel_dst_port_id);
5295 break;
5296 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
5297 bp->nge_port = port;
5298 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
5299 break;
5300 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE:
5301 bp->vxlan_gpe_port = port;
5302 bp->vxlan_gpe_fw_dst_port_id =
5303 le16_to_cpu(resp->tunnel_dst_port_id);
5304 break;
5305 default:
5306 break;
5307 }
5308 if (bp->flags & BNXT_FLAG_TPA)
5309 bnxt_set_tpa(bp, true);
5310
5311err_out:
5312 hwrm_req_drop(bp, req);
5313 return rc;
5314}
5315
5316static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
5317{
5318 struct hwrm_cfa_l2_set_rx_mask_input *req;
5319 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5320 int rc;
5321
5322 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
5323 if (rc)
5324 return rc;
5325
5326 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5327 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
5328 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
5329 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
5330 }
5331 req->mask = cpu_to_le32(vnic->rx_mask);
5332 return hwrm_req_send_silent(bp, req);
5333}
5334
5335void bnxt_del_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5336{
5337 if (!atomic_dec_and_test(&fltr->refcnt))
5338 return;
5339 spin_lock_bh(&bp->ntp_fltr_lock);
5340 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
5341 spin_unlock_bh(&bp->ntp_fltr_lock);
5342 return;
5343 }
5344 hlist_del_rcu(&fltr->base.hash);
5345 if (fltr->base.flags) {
5346 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
5347 bp->ntp_fltr_count--;
5348 }
5349 spin_unlock_bh(&bp->ntp_fltr_lock);
5350 kfree_rcu(fltr, base.rcu);
5351}
5352
5353static struct bnxt_l2_filter *__bnxt_lookup_l2_filter(struct bnxt *bp,
5354 struct bnxt_l2_key *key,
5355 u32 idx)
5356{
5357 struct hlist_head *head = &bp->l2_fltr_hash_tbl[idx];
5358 struct bnxt_l2_filter *fltr;
5359
5360 hlist_for_each_entry_rcu(fltr, head, base.hash) {
5361 struct bnxt_l2_key *l2_key = &fltr->l2_key;
5362
5363 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) &&
5364 l2_key->vlan == key->vlan)
5365 return fltr;
5366 }
5367 return NULL;
5368}
5369
5370static struct bnxt_l2_filter *bnxt_lookup_l2_filter(struct bnxt *bp,
5371 struct bnxt_l2_key *key,
5372 u32 idx)
5373{
5374 struct bnxt_l2_filter *fltr = NULL;
5375
5376 rcu_read_lock();
5377 fltr = __bnxt_lookup_l2_filter(bp, key, idx);
5378 if (fltr)
5379 atomic_inc(&fltr->refcnt);
5380 rcu_read_unlock();
5381 return fltr;
5382}
5383
5384#define BNXT_IPV4_4TUPLE(bp, fkeys) \
5385 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \
5386 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4) || \
5387 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \
5388 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4))
5389
5390#define BNXT_IPV6_4TUPLE(bp, fkeys) \
5391 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \
5392 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6) || \
5393 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \
5394 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6))
5395
5396static u32 bnxt_get_rss_flow_tuple_len(struct bnxt *bp, struct flow_keys *fkeys)
5397{
5398 if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5399 if (BNXT_IPV4_4TUPLE(bp, fkeys))
5400 return sizeof(fkeys->addrs.v4addrs) +
5401 sizeof(fkeys->ports);
5402
5403 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4)
5404 return sizeof(fkeys->addrs.v4addrs);
5405 }
5406
5407 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) {
5408 if (BNXT_IPV6_4TUPLE(bp, fkeys))
5409 return sizeof(fkeys->addrs.v6addrs) +
5410 sizeof(fkeys->ports);
5411
5412 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6)
5413 return sizeof(fkeys->addrs.v6addrs);
5414 }
5415
5416 return 0;
5417}
5418
5419static u32 bnxt_toeplitz(struct bnxt *bp, struct flow_keys *fkeys,
5420 const unsigned char *key)
5421{
5422 u64 prefix = bp->toeplitz_prefix, hash = 0;
5423 struct bnxt_ipv4_tuple tuple4;
5424 struct bnxt_ipv6_tuple tuple6;
5425 int i, j, len = 0;
5426 u8 *four_tuple;
5427
5428 len = bnxt_get_rss_flow_tuple_len(bp, fkeys);
5429 if (!len)
5430 return 0;
5431
5432 if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5433 tuple4.v4addrs = fkeys->addrs.v4addrs;
5434 tuple4.ports = fkeys->ports;
5435 four_tuple = (unsigned char *)&tuple4;
5436 } else {
5437 tuple6.v6addrs = fkeys->addrs.v6addrs;
5438 tuple6.ports = fkeys->ports;
5439 four_tuple = (unsigned char *)&tuple6;
5440 }
5441
5442 for (i = 0, j = 8; i < len; i++, j++) {
5443 u8 byte = four_tuple[i];
5444 int bit;
5445
5446 for (bit = 0; bit < 8; bit++, prefix <<= 1, byte <<= 1) {
5447 if (byte & 0x80)
5448 hash ^= prefix;
5449 }
5450 prefix |= (j < HW_HASH_KEY_SIZE) ? key[j] : 0;
5451 }
5452
5453 /* The valid part of the hash is in the upper 32 bits. */
5454 return (hash >> 32) & BNXT_NTP_FLTR_HASH_MASK;
5455}
5456
5457#ifdef CONFIG_RFS_ACCEL
5458static struct bnxt_l2_filter *
5459bnxt_lookup_l2_filter_from_key(struct bnxt *bp, struct bnxt_l2_key *key)
5460{
5461 struct bnxt_l2_filter *fltr;
5462 u32 idx;
5463
5464 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5465 BNXT_L2_FLTR_HASH_MASK;
5466 fltr = bnxt_lookup_l2_filter(bp, key, idx);
5467 return fltr;
5468}
5469#endif
5470
5471static int bnxt_init_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr,
5472 struct bnxt_l2_key *key, u32 idx)
5473{
5474 struct hlist_head *head;
5475
5476 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr);
5477 fltr->l2_key.vlan = key->vlan;
5478 fltr->base.type = BNXT_FLTR_TYPE_L2;
5479 if (fltr->base.flags) {
5480 int bit_id;
5481
5482 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
5483 BNXT_MAX_FLTR, 0);
5484 if (bit_id < 0)
5485 return -ENOMEM;
5486 fltr->base.sw_id = (u16)bit_id;
5487 }
5488 head = &bp->l2_fltr_hash_tbl[idx];
5489 hlist_add_head_rcu(&fltr->base.hash, head);
5490 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
5491 atomic_set(&fltr->refcnt, 1);
5492 return 0;
5493}
5494
5495static struct bnxt_l2_filter *bnxt_alloc_l2_filter(struct bnxt *bp,
5496 struct bnxt_l2_key *key,
5497 gfp_t gfp)
5498{
5499 struct bnxt_l2_filter *fltr;
5500 u32 idx;
5501 int rc;
5502
5503 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5504 BNXT_L2_FLTR_HASH_MASK;
5505 fltr = bnxt_lookup_l2_filter(bp, key, idx);
5506 if (fltr)
5507 return fltr;
5508
5509 fltr = kzalloc(sizeof(*fltr), gfp);
5510 if (!fltr)
5511 return ERR_PTR(-ENOMEM);
5512 spin_lock_bh(&bp->ntp_fltr_lock);
5513 rc = bnxt_init_l2_filter(bp, fltr, key, idx);
5514 spin_unlock_bh(&bp->ntp_fltr_lock);
5515 if (rc) {
5516 bnxt_del_l2_filter(bp, fltr);
5517 fltr = ERR_PTR(rc);
5518 }
5519 return fltr;
5520}
5521
5522static u16 bnxt_vf_target_id(struct bnxt_pf_info *pf, u16 vf_idx)
5523{
5524#ifdef CONFIG_BNXT_SRIOV
5525 struct bnxt_vf_info *vf = &pf->vf[vf_idx];
5526
5527 return vf->fw_fid;
5528#else
5529 return INVALID_HW_RING_ID;
5530#endif
5531}
5532
5533int bnxt_hwrm_l2_filter_free(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5534{
5535 struct hwrm_cfa_l2_filter_free_input *req;
5536 u16 target_id = 0xffff;
5537 int rc;
5538
5539 if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5540 struct bnxt_pf_info *pf = &bp->pf;
5541
5542 if (fltr->base.vf_idx >= pf->active_vfs)
5543 return -EINVAL;
5544
5545 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5546 if (target_id == INVALID_HW_RING_ID)
5547 return -EINVAL;
5548 }
5549
5550 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5551 if (rc)
5552 return rc;
5553
5554 req->target_id = cpu_to_le16(target_id);
5555 req->l2_filter_id = fltr->base.filter_id;
5556 return hwrm_req_send(bp, req);
5557}
5558
5559int bnxt_hwrm_l2_filter_alloc(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5560{
5561 struct hwrm_cfa_l2_filter_alloc_output *resp;
5562 struct hwrm_cfa_l2_filter_alloc_input *req;
5563 u16 target_id = 0xffff;
5564 int rc;
5565
5566 if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5567 struct bnxt_pf_info *pf = &bp->pf;
5568
5569 if (fltr->base.vf_idx >= pf->active_vfs)
5570 return -EINVAL;
5571
5572 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5573 }
5574 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5575 if (rc)
5576 return rc;
5577
5578 req->target_id = cpu_to_le16(target_id);
5579 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5580
5581 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5582 req->flags |=
5583 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5584 req->dst_id = cpu_to_le16(fltr->base.fw_vnic_id);
5585 req->enables =
5586 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5587 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5588 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5589 ether_addr_copy(req->l2_addr, fltr->l2_key.dst_mac_addr);
5590 eth_broadcast_addr(req->l2_addr_mask);
5591
5592 if (fltr->l2_key.vlan) {
5593 req->enables |=
5594 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN |
5595 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK |
5596 CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS);
5597 req->num_vlans = 1;
5598 req->l2_ivlan = cpu_to_le16(fltr->l2_key.vlan);
5599 req->l2_ivlan_mask = cpu_to_le16(0xfff);
5600 }
5601
5602 resp = hwrm_req_hold(bp, req);
5603 rc = hwrm_req_send(bp, req);
5604 if (!rc) {
5605 fltr->base.filter_id = resp->l2_filter_id;
5606 set_bit(BNXT_FLTR_VALID, &fltr->base.state);
5607 }
5608 hwrm_req_drop(bp, req);
5609 return rc;
5610}
5611
5612int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
5613 struct bnxt_ntuple_filter *fltr)
5614{
5615 struct hwrm_cfa_ntuple_filter_free_input *req;
5616 int rc;
5617
5618 set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state);
5619 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
5620 if (rc)
5621 return rc;
5622
5623 req->ntuple_filter_id = fltr->base.filter_id;
5624 return hwrm_req_send(bp, req);
5625}
5626
5627#define BNXT_NTP_FLTR_FLAGS \
5628 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
5629 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
5630 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
5631 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
5632 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
5633 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
5634 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
5635 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
5636 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
5637 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
5638 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
5639 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
5640 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
5641
5642#define BNXT_NTP_TUNNEL_FLTR_FLAG \
5643 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
5644
5645void bnxt_fill_ipv6_mask(__be32 mask[4])
5646{
5647 int i;
5648
5649 for (i = 0; i < 4; i++)
5650 mask[i] = cpu_to_be32(~0);
5651}
5652
5653int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
5654 struct bnxt_ntuple_filter *fltr)
5655{
5656 struct hwrm_cfa_ntuple_filter_alloc_output *resp;
5657 struct hwrm_cfa_ntuple_filter_alloc_input *req;
5658 struct flow_keys *keys = &fltr->fkeys;
5659 struct bnxt_l2_filter *l2_fltr;
5660 struct bnxt_vnic_info *vnic;
5661 u32 flags = 0;
5662 int rc;
5663
5664 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
5665 if (rc)
5666 return rc;
5667
5668 l2_fltr = fltr->l2_fltr;
5669 req->l2_filter_id = l2_fltr->base.filter_id;
5670
5671
5672 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
5673 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
5674 req->dst_id = cpu_to_le16(fltr->base.rxq);
5675 } else {
5676 vnic = &bp->vnic_info[fltr->base.rxq + 1];
5677 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
5678 }
5679 req->flags = cpu_to_le32(flags);
5680 req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
5681
5682 req->ethertype = htons(ETH_P_IP);
5683 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
5684 req->ip_protocol = keys->basic.ip_proto;
5685
5686 if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
5687 req->ethertype = htons(ETH_P_IPV6);
5688 req->ip_addr_type =
5689 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
5690 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_IP) {
5691 *(struct in6_addr *)&req->src_ipaddr[0] =
5692 keys->addrs.v6addrs.src;
5693 bnxt_fill_ipv6_mask(req->src_ipaddr_mask);
5694 }
5695 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_DST_IP) {
5696 *(struct in6_addr *)&req->dst_ipaddr[0] =
5697 keys->addrs.v6addrs.dst;
5698 bnxt_fill_ipv6_mask(req->dst_ipaddr_mask);
5699 }
5700 } else {
5701 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_IP) {
5702 req->src_ipaddr[0] = keys->addrs.v4addrs.src;
5703 req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
5704 }
5705 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_DST_IP) {
5706 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
5707 req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
5708 }
5709 }
5710 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
5711 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
5712 req->tunnel_type =
5713 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
5714 }
5715
5716 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_PORT) {
5717 req->src_port = keys->ports.src;
5718 req->src_port_mask = cpu_to_be16(0xffff);
5719 }
5720 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_DST_PORT) {
5721 req->dst_port = keys->ports.dst;
5722 req->dst_port_mask = cpu_to_be16(0xffff);
5723 }
5724
5725 resp = hwrm_req_hold(bp, req);
5726 rc = hwrm_req_send(bp, req);
5727 if (!rc)
5728 fltr->base.filter_id = resp->ntuple_filter_id;
5729 hwrm_req_drop(bp, req);
5730 return rc;
5731}
5732
5733static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
5734 const u8 *mac_addr)
5735{
5736 struct bnxt_l2_filter *fltr;
5737 struct bnxt_l2_key key;
5738 int rc;
5739
5740 ether_addr_copy(key.dst_mac_addr, mac_addr);
5741 key.vlan = 0;
5742 fltr = bnxt_alloc_l2_filter(bp, &key, GFP_KERNEL);
5743 if (IS_ERR(fltr))
5744 return PTR_ERR(fltr);
5745
5746 fltr->base.fw_vnic_id = bp->vnic_info[vnic_id].fw_vnic_id;
5747 rc = bnxt_hwrm_l2_filter_alloc(bp, fltr);
5748 if (rc)
5749 bnxt_del_l2_filter(bp, fltr);
5750 else
5751 bp->vnic_info[vnic_id].l2_filters[idx] = fltr;
5752 return rc;
5753}
5754
5755static void bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
5756{
5757 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
5758
5759 /* Any associated ntuple filters will also be cleared by firmware. */
5760 for (i = 0; i < num_of_vnics; i++) {
5761 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5762
5763 for (j = 0; j < vnic->uc_filter_count; j++) {
5764 struct bnxt_l2_filter *fltr = vnic->l2_filters[j];
5765
5766 bnxt_hwrm_l2_filter_free(bp, fltr);
5767 bnxt_del_l2_filter(bp, fltr);
5768 }
5769 vnic->uc_filter_count = 0;
5770 }
5771}
5772
5773#define BNXT_DFLT_TUNL_TPA_BMAP \
5774 (VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE | \
5775 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 | \
5776 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6)
5777
5778static void bnxt_hwrm_vnic_update_tunl_tpa(struct bnxt *bp,
5779 struct hwrm_vnic_tpa_cfg_input *req)
5780{
5781 u32 tunl_tpa_bmap = BNXT_DFLT_TUNL_TPA_BMAP;
5782
5783 if (!(bp->fw_cap & BNXT_FW_CAP_VNIC_TUNNEL_TPA))
5784 return;
5785
5786 if (bp->vxlan_port)
5787 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN;
5788 if (bp->vxlan_gpe_port)
5789 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE;
5790 if (bp->nge_port)
5791 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE;
5792
5793 req->enables |= cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN);
5794 req->tnl_tpa_en_bitmap = cpu_to_le32(tunl_tpa_bmap);
5795}
5796
5797static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
5798{
5799 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5800 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
5801 struct hwrm_vnic_tpa_cfg_input *req;
5802 int rc;
5803
5804 if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
5805 return 0;
5806
5807 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
5808 if (rc)
5809 return rc;
5810
5811 if (tpa_flags) {
5812 u16 mss = bp->dev->mtu - 40;
5813 u32 nsegs, n, segs = 0, flags;
5814
5815 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
5816 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
5817 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
5818 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
5819 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
5820 if (tpa_flags & BNXT_FLAG_GRO)
5821 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
5822
5823 req->flags = cpu_to_le32(flags);
5824
5825 req->enables =
5826 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
5827 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
5828 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
5829
5830 /* Number of segs are log2 units, and first packet is not
5831 * included as part of this units.
5832 */
5833 if (mss <= BNXT_RX_PAGE_SIZE) {
5834 n = BNXT_RX_PAGE_SIZE / mss;
5835 nsegs = (MAX_SKB_FRAGS - 1) * n;
5836 } else {
5837 n = mss / BNXT_RX_PAGE_SIZE;
5838 if (mss & (BNXT_RX_PAGE_SIZE - 1))
5839 n++;
5840 nsegs = (MAX_SKB_FRAGS - n) / n;
5841 }
5842
5843 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5844 segs = MAX_TPA_SEGS_P5;
5845 max_aggs = bp->max_tpa;
5846 } else {
5847 segs = ilog2(nsegs);
5848 }
5849 req->max_agg_segs = cpu_to_le16(segs);
5850 req->max_aggs = cpu_to_le16(max_aggs);
5851
5852 req->min_agg_len = cpu_to_le32(512);
5853 bnxt_hwrm_vnic_update_tunl_tpa(bp, req);
5854 }
5855 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5856
5857 return hwrm_req_send(bp, req);
5858}
5859
5860static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5861{
5862 struct bnxt_ring_grp_info *grp_info;
5863
5864 grp_info = &bp->grp_info[ring->grp_idx];
5865 return grp_info->cp_fw_ring_id;
5866}
5867
5868static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5869{
5870 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5871 return rxr->rx_cpr->cp_ring_struct.fw_ring_id;
5872 else
5873 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5874}
5875
5876static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5877{
5878 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5879 return txr->tx_cpr->cp_ring_struct.fw_ring_id;
5880 else
5881 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5882}
5883
5884static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5885{
5886 int entries;
5887
5888 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5889 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5890 else
5891 entries = HW_HASH_INDEX_SIZE;
5892
5893 bp->rss_indir_tbl_entries = entries;
5894 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5895 GFP_KERNEL);
5896 if (!bp->rss_indir_tbl)
5897 return -ENOMEM;
5898 return 0;
5899}
5900
5901static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5902{
5903 u16 max_rings, max_entries, pad, i;
5904
5905 if (!bp->rx_nr_rings)
5906 return;
5907
5908 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5909 max_rings = bp->rx_nr_rings - 1;
5910 else
5911 max_rings = bp->rx_nr_rings;
5912
5913 max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5914
5915 for (i = 0; i < max_entries; i++)
5916 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5917
5918 pad = bp->rss_indir_tbl_entries - max_entries;
5919 if (pad)
5920 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5921}
5922
5923static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5924{
5925 u16 i, tbl_size, max_ring = 0;
5926
5927 if (!bp->rss_indir_tbl)
5928 return 0;
5929
5930 tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5931 for (i = 0; i < tbl_size; i++)
5932 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5933 return max_ring;
5934}
5935
5936int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5937{
5938 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5939 if (!rx_rings)
5940 return 0;
5941 return bnxt_calc_nr_ring_pages(rx_rings - 1,
5942 BNXT_RSS_TABLE_ENTRIES_P5);
5943 }
5944 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5945 return 2;
5946 return 1;
5947}
5948
5949static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5950{
5951 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5952 u16 i, j;
5953
5954 /* Fill the RSS indirection table with ring group ids */
5955 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5956 if (!no_rss)
5957 j = bp->rss_indir_tbl[i];
5958 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5959 }
5960}
5961
5962static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5963 struct bnxt_vnic_info *vnic)
5964{
5965 __le16 *ring_tbl = vnic->rss_table;
5966 struct bnxt_rx_ring_info *rxr;
5967 u16 tbl_size, i;
5968
5969 tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5970
5971 for (i = 0; i < tbl_size; i++) {
5972 u16 ring_id, j;
5973
5974 j = bp->rss_indir_tbl[i];
5975 rxr = &bp->rx_ring[j];
5976
5977 ring_id = rxr->rx_ring_struct.fw_ring_id;
5978 *ring_tbl++ = cpu_to_le16(ring_id);
5979 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5980 *ring_tbl++ = cpu_to_le16(ring_id);
5981 }
5982}
5983
5984static void
5985__bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
5986 struct bnxt_vnic_info *vnic)
5987{
5988 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5989 bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5990 else
5991 bnxt_fill_hw_rss_tbl(bp, vnic);
5992
5993 if (bp->rss_hash_delta) {
5994 req->hash_type = cpu_to_le32(bp->rss_hash_delta);
5995 if (bp->rss_hash_cfg & bp->rss_hash_delta)
5996 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
5997 else
5998 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
5999 } else {
6000 req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
6001 }
6002 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
6003 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
6004 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
6005}
6006
6007static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
6008{
6009 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6010 struct hwrm_vnic_rss_cfg_input *req;
6011 int rc;
6012
6013 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) ||
6014 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
6015 return 0;
6016
6017 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6018 if (rc)
6019 return rc;
6020
6021 if (set_rss)
6022 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6023 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6024 return hwrm_req_send(bp, req);
6025}
6026
6027static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
6028{
6029 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6030 struct hwrm_vnic_rss_cfg_input *req;
6031 dma_addr_t ring_tbl_map;
6032 u32 i, nr_ctxs;
6033 int rc;
6034
6035 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6036 if (rc)
6037 return rc;
6038
6039 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6040 if (!set_rss)
6041 return hwrm_req_send(bp, req);
6042
6043 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6044 ring_tbl_map = vnic->rss_table_dma_addr;
6045 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
6046
6047 hwrm_req_hold(bp, req);
6048 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
6049 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
6050 req->ring_table_pair_index = i;
6051 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
6052 rc = hwrm_req_send(bp, req);
6053 if (rc)
6054 goto exit;
6055 }
6056
6057exit:
6058 hwrm_req_drop(bp, req);
6059 return rc;
6060}
6061
6062static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
6063{
6064 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6065 struct hwrm_vnic_rss_qcfg_output *resp;
6066 struct hwrm_vnic_rss_qcfg_input *req;
6067
6068 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
6069 return;
6070
6071 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6072 /* all contexts configured to same hash_type, zero always exists */
6073 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6074 resp = hwrm_req_hold(bp, req);
6075 if (!hwrm_req_send(bp, req)) {
6076 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
6077 bp->rss_hash_delta = 0;
6078 }
6079 hwrm_req_drop(bp, req);
6080}
6081
6082static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
6083{
6084 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6085 struct hwrm_vnic_plcmodes_cfg_input *req;
6086 int rc;
6087
6088 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
6089 if (rc)
6090 return rc;
6091
6092 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
6093 req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
6094
6095 if (BNXT_RX_PAGE_MODE(bp)) {
6096 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
6097 } else {
6098 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
6099 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
6100 req->enables |=
6101 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
6102 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
6103 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
6104 }
6105 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
6106 return hwrm_req_send(bp, req);
6107}
6108
6109static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
6110 u16 ctx_idx)
6111{
6112 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
6113
6114 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
6115 return;
6116
6117 req->rss_cos_lb_ctx_id =
6118 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
6119
6120 hwrm_req_send(bp, req);
6121 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
6122}
6123
6124static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
6125{
6126 int i, j;
6127
6128 for (i = 0; i < bp->nr_vnics; i++) {
6129 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
6130
6131 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
6132 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
6133 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
6134 }
6135 }
6136 bp->rsscos_nr_ctxs = 0;
6137}
6138
6139static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
6140{
6141 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
6142 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
6143 int rc;
6144
6145 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
6146 if (rc)
6147 return rc;
6148
6149 resp = hwrm_req_hold(bp, req);
6150 rc = hwrm_req_send(bp, req);
6151 if (!rc)
6152 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
6153 le16_to_cpu(resp->rss_cos_lb_ctx_id);
6154 hwrm_req_drop(bp, req);
6155
6156 return rc;
6157}
6158
6159static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
6160{
6161 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
6162 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
6163 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
6164}
6165
6166int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
6167{
6168 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6169 struct hwrm_vnic_cfg_input *req;
6170 unsigned int ring = 0, grp_idx;
6171 u16 def_vlan = 0;
6172 int rc;
6173
6174 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
6175 if (rc)
6176 return rc;
6177
6178 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6179 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
6180
6181 req->default_rx_ring_id =
6182 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
6183 req->default_cmpl_ring_id =
6184 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
6185 req->enables =
6186 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
6187 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
6188 goto vnic_mru;
6189 }
6190 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
6191 /* Only RSS support for now TBD: COS & LB */
6192 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
6193 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6194 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6195 VNIC_CFG_REQ_ENABLES_MRU);
6196 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
6197 req->rss_rule =
6198 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
6199 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6200 VNIC_CFG_REQ_ENABLES_MRU);
6201 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
6202 } else {
6203 req->rss_rule = cpu_to_le16(0xffff);
6204 }
6205
6206 if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
6207 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
6208 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
6209 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
6210 } else {
6211 req->cos_rule = cpu_to_le16(0xffff);
6212 }
6213
6214 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
6215 ring = 0;
6216 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
6217 ring = vnic_id - 1;
6218 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
6219 ring = bp->rx_nr_rings - 1;
6220
6221 grp_idx = bp->rx_ring[ring].bnapi->index;
6222 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
6223 req->lb_rule = cpu_to_le16(0xffff);
6224vnic_mru:
6225 req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
6226
6227 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6228#ifdef CONFIG_BNXT_SRIOV
6229 if (BNXT_VF(bp))
6230 def_vlan = bp->vf.vlan;
6231#endif
6232 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
6233 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
6234 if (!vnic_id && bnxt_ulp_registered(bp->edev))
6235 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
6236
6237 return hwrm_req_send(bp, req);
6238}
6239
6240static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
6241{
6242 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
6243 struct hwrm_vnic_free_input *req;
6244
6245 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
6246 return;
6247
6248 req->vnic_id =
6249 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
6250
6251 hwrm_req_send(bp, req);
6252 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
6253 }
6254}
6255
6256static void bnxt_hwrm_vnic_free(struct bnxt *bp)
6257{
6258 u16 i;
6259
6260 for (i = 0; i < bp->nr_vnics; i++)
6261 bnxt_hwrm_vnic_free_one(bp, i);
6262}
6263
6264static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
6265 unsigned int start_rx_ring_idx,
6266 unsigned int nr_rings)
6267{
6268 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
6269 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6270 struct hwrm_vnic_alloc_output *resp;
6271 struct hwrm_vnic_alloc_input *req;
6272 int rc;
6273
6274 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
6275 if (rc)
6276 return rc;
6277
6278 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6279 goto vnic_no_ring_grps;
6280
6281 /* map ring groups to this vnic */
6282 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
6283 grp_idx = bp->rx_ring[i].bnapi->index;
6284 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
6285 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
6286 j, nr_rings);
6287 break;
6288 }
6289 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
6290 }
6291
6292vnic_no_ring_grps:
6293 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
6294 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
6295 if (vnic_id == 0)
6296 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
6297
6298 resp = hwrm_req_hold(bp, req);
6299 rc = hwrm_req_send(bp, req);
6300 if (!rc)
6301 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
6302 hwrm_req_drop(bp, req);
6303 return rc;
6304}
6305
6306static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
6307{
6308 struct hwrm_vnic_qcaps_output *resp;
6309 struct hwrm_vnic_qcaps_input *req;
6310 int rc;
6311
6312 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
6313 bp->flags &= ~BNXT_FLAG_ROCE_MIRROR_CAP;
6314 bp->rss_cap &= ~BNXT_RSS_CAP_NEW_RSS_CAP;
6315 if (bp->hwrm_spec_code < 0x10600)
6316 return 0;
6317
6318 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
6319 if (rc)
6320 return rc;
6321
6322 resp = hwrm_req_hold(bp, req);
6323 rc = hwrm_req_send(bp, req);
6324 if (!rc) {
6325 u32 flags = le32_to_cpu(resp->flags);
6326
6327 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
6328 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
6329 bp->rss_cap |= BNXT_RSS_CAP_NEW_RSS_CAP;
6330 if (flags &
6331 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
6332 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
6333
6334 /* Older P5 fw before EXT_HW_STATS support did not set
6335 * VLAN_STRIP_CAP properly.
6336 */
6337 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
6338 (BNXT_CHIP_P5(bp) &&
6339 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
6340 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
6341 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
6342 bp->rss_cap |= BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA;
6343 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED)
6344 bp->rss_cap |= BNXT_RSS_CAP_RSS_TCAM;
6345 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
6346 if (bp->max_tpa_v2) {
6347 if (BNXT_CHIP_P5(bp))
6348 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
6349 else
6350 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P7;
6351 }
6352 if (flags & VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP)
6353 bp->fw_cap |= BNXT_FW_CAP_VNIC_TUNNEL_TPA;
6354 }
6355 hwrm_req_drop(bp, req);
6356 return rc;
6357}
6358
6359static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
6360{
6361 struct hwrm_ring_grp_alloc_output *resp;
6362 struct hwrm_ring_grp_alloc_input *req;
6363 int rc;
6364 u16 i;
6365
6366 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6367 return 0;
6368
6369 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
6370 if (rc)
6371 return rc;
6372
6373 resp = hwrm_req_hold(bp, req);
6374 for (i = 0; i < bp->rx_nr_rings; i++) {
6375 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
6376
6377 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
6378 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
6379 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
6380 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
6381
6382 rc = hwrm_req_send(bp, req);
6383
6384 if (rc)
6385 break;
6386
6387 bp->grp_info[grp_idx].fw_grp_id =
6388 le32_to_cpu(resp->ring_group_id);
6389 }
6390 hwrm_req_drop(bp, req);
6391 return rc;
6392}
6393
6394static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
6395{
6396 struct hwrm_ring_grp_free_input *req;
6397 u16 i;
6398
6399 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
6400 return;
6401
6402 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
6403 return;
6404
6405 hwrm_req_hold(bp, req);
6406 for (i = 0; i < bp->cp_nr_rings; i++) {
6407 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
6408 continue;
6409 req->ring_group_id =
6410 cpu_to_le32(bp->grp_info[i].fw_grp_id);
6411
6412 hwrm_req_send(bp, req);
6413 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
6414 }
6415 hwrm_req_drop(bp, req);
6416}
6417
6418static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
6419 struct bnxt_ring_struct *ring,
6420 u32 ring_type, u32 map_index)
6421{
6422 struct hwrm_ring_alloc_output *resp;
6423 struct hwrm_ring_alloc_input *req;
6424 struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
6425 struct bnxt_ring_grp_info *grp_info;
6426 int rc, err = 0;
6427 u16 ring_id;
6428
6429 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
6430 if (rc)
6431 goto exit;
6432
6433 req->enables = 0;
6434 if (rmem->nr_pages > 1) {
6435 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
6436 /* Page size is in log2 units */
6437 req->page_size = BNXT_PAGE_SHIFT;
6438 req->page_tbl_depth = 1;
6439 } else {
6440 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]);
6441 }
6442 req->fbo = 0;
6443 /* Association of ring index with doorbell index and MSIX number */
6444 req->logical_id = cpu_to_le16(map_index);
6445
6446 switch (ring_type) {
6447 case HWRM_RING_ALLOC_TX: {
6448 struct bnxt_tx_ring_info *txr;
6449
6450 txr = container_of(ring, struct bnxt_tx_ring_info,
6451 tx_ring_struct);
6452 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
6453 /* Association of transmit ring with completion ring */
6454 grp_info = &bp->grp_info[ring->grp_idx];
6455 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
6456 req->length = cpu_to_le32(bp->tx_ring_mask + 1);
6457 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6458 req->queue_id = cpu_to_le16(ring->queue_id);
6459 if (bp->flags & BNXT_FLAG_TX_COAL_CMPL)
6460 req->cmpl_coal_cnt =
6461 RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64;
6462 break;
6463 }
6464 case HWRM_RING_ALLOC_RX:
6465 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6466 req->length = cpu_to_le32(bp->rx_ring_mask + 1);
6467 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6468 u16 flags = 0;
6469
6470 /* Association of rx ring with stats context */
6471 grp_info = &bp->grp_info[ring->grp_idx];
6472 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
6473 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6474 req->enables |= cpu_to_le32(
6475 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6476 if (NET_IP_ALIGN == 2)
6477 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
6478 req->flags = cpu_to_le16(flags);
6479 }
6480 break;
6481 case HWRM_RING_ALLOC_AGG:
6482 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6483 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
6484 /* Association of agg ring with rx ring */
6485 grp_info = &bp->grp_info[ring->grp_idx];
6486 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
6487 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
6488 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6489 req->enables |= cpu_to_le32(
6490 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
6491 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6492 } else {
6493 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6494 }
6495 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
6496 break;
6497 case HWRM_RING_ALLOC_CMPL:
6498 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
6499 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6500 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6501 /* Association of cp ring with nq */
6502 grp_info = &bp->grp_info[map_index];
6503 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
6504 req->cq_handle = cpu_to_le64(ring->handle);
6505 req->enables |= cpu_to_le32(
6506 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
6507 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
6508 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6509 }
6510 break;
6511 case HWRM_RING_ALLOC_NQ:
6512 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
6513 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6514 if (bp->flags & BNXT_FLAG_USING_MSIX)
6515 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6516 break;
6517 default:
6518 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
6519 ring_type);
6520 return -1;
6521 }
6522
6523 resp = hwrm_req_hold(bp, req);
6524 rc = hwrm_req_send(bp, req);
6525 err = le16_to_cpu(resp->error_code);
6526 ring_id = le16_to_cpu(resp->ring_id);
6527 hwrm_req_drop(bp, req);
6528
6529exit:
6530 if (rc || err) {
6531 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
6532 ring_type, rc, err);
6533 return -EIO;
6534 }
6535 ring->fw_ring_id = ring_id;
6536 return rc;
6537}
6538
6539static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
6540{
6541 int rc;
6542
6543 if (BNXT_PF(bp)) {
6544 struct hwrm_func_cfg_input *req;
6545
6546 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
6547 if (rc)
6548 return rc;
6549
6550 req->fid = cpu_to_le16(0xffff);
6551 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6552 req->async_event_cr = cpu_to_le16(idx);
6553 return hwrm_req_send(bp, req);
6554 } else {
6555 struct hwrm_func_vf_cfg_input *req;
6556
6557 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
6558 if (rc)
6559 return rc;
6560
6561 req->enables =
6562 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6563 req->async_event_cr = cpu_to_le16(idx);
6564 return hwrm_req_send(bp, req);
6565 }
6566}
6567
6568static void bnxt_set_db_mask(struct bnxt *bp, struct bnxt_db_info *db,
6569 u32 ring_type)
6570{
6571 switch (ring_type) {
6572 case HWRM_RING_ALLOC_TX:
6573 db->db_ring_mask = bp->tx_ring_mask;
6574 break;
6575 case HWRM_RING_ALLOC_RX:
6576 db->db_ring_mask = bp->rx_ring_mask;
6577 break;
6578 case HWRM_RING_ALLOC_AGG:
6579 db->db_ring_mask = bp->rx_agg_ring_mask;
6580 break;
6581 case HWRM_RING_ALLOC_CMPL:
6582 case HWRM_RING_ALLOC_NQ:
6583 db->db_ring_mask = bp->cp_ring_mask;
6584 break;
6585 }
6586 if (bp->flags & BNXT_FLAG_CHIP_P7) {
6587 db->db_epoch_mask = db->db_ring_mask + 1;
6588 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask);
6589 }
6590}
6591
6592static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
6593 u32 map_idx, u32 xid)
6594{
6595 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6596 switch (ring_type) {
6597 case HWRM_RING_ALLOC_TX:
6598 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
6599 break;
6600 case HWRM_RING_ALLOC_RX:
6601 case HWRM_RING_ALLOC_AGG:
6602 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
6603 break;
6604 case HWRM_RING_ALLOC_CMPL:
6605 db->db_key64 = DBR_PATH_L2;
6606 break;
6607 case HWRM_RING_ALLOC_NQ:
6608 db->db_key64 = DBR_PATH_L2;
6609 break;
6610 }
6611 db->db_key64 |= (u64)xid << DBR_XID_SFT;
6612
6613 if (bp->flags & BNXT_FLAG_CHIP_P7)
6614 db->db_key64 |= DBR_VALID;
6615
6616 db->doorbell = bp->bar1 + bp->db_offset;
6617 } else {
6618 db->doorbell = bp->bar1 + map_idx * 0x80;
6619 switch (ring_type) {
6620 case HWRM_RING_ALLOC_TX:
6621 db->db_key32 = DB_KEY_TX;
6622 break;
6623 case HWRM_RING_ALLOC_RX:
6624 case HWRM_RING_ALLOC_AGG:
6625 db->db_key32 = DB_KEY_RX;
6626 break;
6627 case HWRM_RING_ALLOC_CMPL:
6628 db->db_key32 = DB_KEY_CP;
6629 break;
6630 }
6631 }
6632 bnxt_set_db_mask(bp, db, ring_type);
6633}
6634
6635static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
6636{
6637 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
6638 int i, rc = 0;
6639 u32 type;
6640
6641 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6642 type = HWRM_RING_ALLOC_NQ;
6643 else
6644 type = HWRM_RING_ALLOC_CMPL;
6645 for (i = 0; i < bp->cp_nr_rings; i++) {
6646 struct bnxt_napi *bnapi = bp->bnapi[i];
6647 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6648 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
6649 u32 map_idx = ring->map_idx;
6650 unsigned int vector;
6651
6652 vector = bp->irq_tbl[map_idx].vector;
6653 disable_irq_nosync(vector);
6654 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6655 if (rc) {
6656 enable_irq(vector);
6657 goto err_out;
6658 }
6659 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
6660 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
6661 enable_irq(vector);
6662 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
6663
6664 if (!i) {
6665 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
6666 if (rc)
6667 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
6668 }
6669 }
6670
6671 type = HWRM_RING_ALLOC_TX;
6672 for (i = 0; i < bp->tx_nr_rings; i++) {
6673 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6674 struct bnxt_ring_struct *ring;
6675 u32 map_idx;
6676
6677 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6678 struct bnxt_cp_ring_info *cpr2 = txr->tx_cpr;
6679 struct bnxt_napi *bnapi = txr->bnapi;
6680 u32 type2 = HWRM_RING_ALLOC_CMPL;
6681
6682 ring = &cpr2->cp_ring_struct;
6683 ring->handle = BNXT_SET_NQ_HDL(cpr2);
6684 map_idx = bnapi->index;
6685 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
6686 if (rc)
6687 goto err_out;
6688 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
6689 ring->fw_ring_id);
6690 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
6691 }
6692 ring = &txr->tx_ring_struct;
6693 map_idx = i;
6694 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6695 if (rc)
6696 goto err_out;
6697 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
6698 }
6699
6700 type = HWRM_RING_ALLOC_RX;
6701 for (i = 0; i < bp->rx_nr_rings; i++) {
6702 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6703 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6704 struct bnxt_napi *bnapi = rxr->bnapi;
6705 u32 map_idx = bnapi->index;
6706
6707 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6708 if (rc)
6709 goto err_out;
6710 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
6711 /* If we have agg rings, post agg buffers first. */
6712 if (!agg_rings)
6713 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
6714 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
6715 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6716 struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr;
6717 u32 type2 = HWRM_RING_ALLOC_CMPL;
6718
6719 ring = &cpr2->cp_ring_struct;
6720 ring->handle = BNXT_SET_NQ_HDL(cpr2);
6721 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
6722 if (rc)
6723 goto err_out;
6724 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
6725 ring->fw_ring_id);
6726 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
6727 }
6728 }
6729
6730 if (agg_rings) {
6731 type = HWRM_RING_ALLOC_AGG;
6732 for (i = 0; i < bp->rx_nr_rings; i++) {
6733 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6734 struct bnxt_ring_struct *ring =
6735 &rxr->rx_agg_ring_struct;
6736 u32 grp_idx = ring->grp_idx;
6737 u32 map_idx = grp_idx + bp->rx_nr_rings;
6738
6739 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6740 if (rc)
6741 goto err_out;
6742
6743 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
6744 ring->fw_ring_id);
6745 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
6746 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
6747 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
6748 }
6749 }
6750err_out:
6751 return rc;
6752}
6753
6754static int hwrm_ring_free_send_msg(struct bnxt *bp,
6755 struct bnxt_ring_struct *ring,
6756 u32 ring_type, int cmpl_ring_id)
6757{
6758 struct hwrm_ring_free_output *resp;
6759 struct hwrm_ring_free_input *req;
6760 u16 error_code = 0;
6761 int rc;
6762
6763 if (BNXT_NO_FW_ACCESS(bp))
6764 return 0;
6765
6766 rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
6767 if (rc)
6768 goto exit;
6769
6770 req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
6771 req->ring_type = ring_type;
6772 req->ring_id = cpu_to_le16(ring->fw_ring_id);
6773
6774 resp = hwrm_req_hold(bp, req);
6775 rc = hwrm_req_send(bp, req);
6776 error_code = le16_to_cpu(resp->error_code);
6777 hwrm_req_drop(bp, req);
6778exit:
6779 if (rc || error_code) {
6780 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
6781 ring_type, rc, error_code);
6782 return -EIO;
6783 }
6784 return 0;
6785}
6786
6787static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
6788{
6789 u32 type;
6790 int i;
6791
6792 if (!bp->bnapi)
6793 return;
6794
6795 for (i = 0; i < bp->tx_nr_rings; i++) {
6796 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6797 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
6798
6799 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6800 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
6801
6802 hwrm_ring_free_send_msg(bp, ring,
6803 RING_FREE_REQ_RING_TYPE_TX,
6804 close_path ? cmpl_ring_id :
6805 INVALID_HW_RING_ID);
6806 ring->fw_ring_id = INVALID_HW_RING_ID;
6807 }
6808 }
6809
6810 for (i = 0; i < bp->rx_nr_rings; i++) {
6811 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6812 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6813 u32 grp_idx = rxr->bnapi->index;
6814
6815 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6816 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6817
6818 hwrm_ring_free_send_msg(bp, ring,
6819 RING_FREE_REQ_RING_TYPE_RX,
6820 close_path ? cmpl_ring_id :
6821 INVALID_HW_RING_ID);
6822 ring->fw_ring_id = INVALID_HW_RING_ID;
6823 bp->grp_info[grp_idx].rx_fw_ring_id =
6824 INVALID_HW_RING_ID;
6825 }
6826 }
6827
6828 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6829 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
6830 else
6831 type = RING_FREE_REQ_RING_TYPE_RX;
6832 for (i = 0; i < bp->rx_nr_rings; i++) {
6833 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6834 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
6835 u32 grp_idx = rxr->bnapi->index;
6836
6837 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6838 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6839
6840 hwrm_ring_free_send_msg(bp, ring, type,
6841 close_path ? cmpl_ring_id :
6842 INVALID_HW_RING_ID);
6843 ring->fw_ring_id = INVALID_HW_RING_ID;
6844 bp->grp_info[grp_idx].agg_fw_ring_id =
6845 INVALID_HW_RING_ID;
6846 }
6847 }
6848
6849 /* The completion rings are about to be freed. After that the
6850 * IRQ doorbell will not work anymore. So we need to disable
6851 * IRQ here.
6852 */
6853 bnxt_disable_int_sync(bp);
6854
6855 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6856 type = RING_FREE_REQ_RING_TYPE_NQ;
6857 else
6858 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
6859 for (i = 0; i < bp->cp_nr_rings; i++) {
6860 struct bnxt_napi *bnapi = bp->bnapi[i];
6861 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6862 struct bnxt_ring_struct *ring;
6863 int j;
6864
6865 for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) {
6866 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
6867
6868 ring = &cpr2->cp_ring_struct;
6869 if (ring->fw_ring_id == INVALID_HW_RING_ID)
6870 continue;
6871 hwrm_ring_free_send_msg(bp, ring,
6872 RING_FREE_REQ_RING_TYPE_L2_CMPL,
6873 INVALID_HW_RING_ID);
6874 ring->fw_ring_id = INVALID_HW_RING_ID;
6875 }
6876 ring = &cpr->cp_ring_struct;
6877 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6878 hwrm_ring_free_send_msg(bp, ring, type,
6879 INVALID_HW_RING_ID);
6880 ring->fw_ring_id = INVALID_HW_RING_ID;
6881 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
6882 }
6883 }
6884}
6885
6886static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6887 bool shared);
6888static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6889 bool shared);
6890
6891static int bnxt_hwrm_get_rings(struct bnxt *bp)
6892{
6893 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6894 struct hwrm_func_qcfg_output *resp;
6895 struct hwrm_func_qcfg_input *req;
6896 int rc;
6897
6898 if (bp->hwrm_spec_code < 0x10601)
6899 return 0;
6900
6901 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6902 if (rc)
6903 return rc;
6904
6905 req->fid = cpu_to_le16(0xffff);
6906 resp = hwrm_req_hold(bp, req);
6907 rc = hwrm_req_send(bp, req);
6908 if (rc) {
6909 hwrm_req_drop(bp, req);
6910 return rc;
6911 }
6912
6913 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6914 if (BNXT_NEW_RM(bp)) {
6915 u16 cp, stats;
6916
6917 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
6918 hw_resc->resv_hw_ring_grps =
6919 le32_to_cpu(resp->alloc_hw_ring_grps);
6920 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
6921 cp = le16_to_cpu(resp->alloc_cmpl_rings);
6922 stats = le16_to_cpu(resp->alloc_stat_ctx);
6923 hw_resc->resv_irqs = cp;
6924 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6925 int rx = hw_resc->resv_rx_rings;
6926 int tx = hw_resc->resv_tx_rings;
6927
6928 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6929 rx >>= 1;
6930 if (cp < (rx + tx)) {
6931 rc = __bnxt_trim_rings(bp, &rx, &tx, cp, false);
6932 if (rc)
6933 goto get_rings_exit;
6934 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6935 rx <<= 1;
6936 hw_resc->resv_rx_rings = rx;
6937 hw_resc->resv_tx_rings = tx;
6938 }
6939 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
6940 hw_resc->resv_hw_ring_grps = rx;
6941 }
6942 hw_resc->resv_cp_rings = cp;
6943 hw_resc->resv_stat_ctxs = stats;
6944 }
6945get_rings_exit:
6946 hwrm_req_drop(bp, req);
6947 return rc;
6948}
6949
6950int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6951{
6952 struct hwrm_func_qcfg_output *resp;
6953 struct hwrm_func_qcfg_input *req;
6954 int rc;
6955
6956 if (bp->hwrm_spec_code < 0x10601)
6957 return 0;
6958
6959 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6960 if (rc)
6961 return rc;
6962
6963 req->fid = cpu_to_le16(fid);
6964 resp = hwrm_req_hold(bp, req);
6965 rc = hwrm_req_send(bp, req);
6966 if (!rc)
6967 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6968
6969 hwrm_req_drop(bp, req);
6970 return rc;
6971}
6972
6973static bool bnxt_rfs_supported(struct bnxt *bp);
6974
6975static struct hwrm_func_cfg_input *
6976__bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6977 int ring_grps, int cp_rings, int stats, int vnics)
6978{
6979 struct hwrm_func_cfg_input *req;
6980 u32 enables = 0;
6981
6982 if (bnxt_hwrm_func_cfg_short_req_init(bp, &req))
6983 return NULL;
6984
6985 req->fid = cpu_to_le16(0xffff);
6986 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6987 req->num_tx_rings = cpu_to_le16(tx_rings);
6988 if (BNXT_NEW_RM(bp)) {
6989 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6990 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6991 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6992 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6993 enables |= tx_rings + ring_grps ?
6994 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6995 enables |= rx_rings ?
6996 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6997 } else {
6998 enables |= cp_rings ?
6999 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7000 enables |= ring_grps ?
7001 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
7002 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7003 }
7004 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
7005
7006 req->num_rx_rings = cpu_to_le16(rx_rings);
7007 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7008 u16 rss_ctx = bnxt_get_nr_rss_ctxs(bp, ring_grps);
7009
7010 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
7011 req->num_msix = cpu_to_le16(cp_rings);
7012 req->num_rsscos_ctxs = cpu_to_le16(rss_ctx);
7013 } else {
7014 req->num_cmpl_rings = cpu_to_le16(cp_rings);
7015 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
7016 req->num_rsscos_ctxs = cpu_to_le16(1);
7017 if (!(bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) &&
7018 bnxt_rfs_supported(bp))
7019 req->num_rsscos_ctxs =
7020 cpu_to_le16(ring_grps + 1);
7021 }
7022 req->num_stat_ctxs = cpu_to_le16(stats);
7023 req->num_vnics = cpu_to_le16(vnics);
7024 }
7025 req->enables = cpu_to_le32(enables);
7026 return req;
7027}
7028
7029static struct hwrm_func_vf_cfg_input *
7030__bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7031 int ring_grps, int cp_rings, int stats, int vnics)
7032{
7033 struct hwrm_func_vf_cfg_input *req;
7034 u32 enables = 0;
7035
7036 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
7037 return NULL;
7038
7039 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
7040 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
7041 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7042 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
7043 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7044 enables |= tx_rings + ring_grps ?
7045 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7046 } else {
7047 enables |= cp_rings ?
7048 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7049 enables |= ring_grps ?
7050 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
7051 }
7052 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
7053 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
7054
7055 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
7056 req->num_tx_rings = cpu_to_le16(tx_rings);
7057 req->num_rx_rings = cpu_to_le16(rx_rings);
7058 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7059 u16 rss_ctx = bnxt_get_nr_rss_ctxs(bp, ring_grps);
7060
7061 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
7062 req->num_rsscos_ctxs = cpu_to_le16(rss_ctx);
7063 } else {
7064 req->num_cmpl_rings = cpu_to_le16(cp_rings);
7065 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
7066 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
7067 }
7068 req->num_stat_ctxs = cpu_to_le16(stats);
7069 req->num_vnics = cpu_to_le16(vnics);
7070
7071 req->enables = cpu_to_le32(enables);
7072 return req;
7073}
7074
7075static int
7076bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7077 int ring_grps, int cp_rings, int stats, int vnics)
7078{
7079 struct hwrm_func_cfg_input *req;
7080 int rc;
7081
7082 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
7083 cp_rings, stats, vnics);
7084 if (!req)
7085 return -ENOMEM;
7086
7087 if (!req->enables) {
7088 hwrm_req_drop(bp, req);
7089 return 0;
7090 }
7091
7092 rc = hwrm_req_send(bp, req);
7093 if (rc)
7094 return rc;
7095
7096 if (bp->hwrm_spec_code < 0x10601)
7097 bp->hw_resc.resv_tx_rings = tx_rings;
7098
7099 return bnxt_hwrm_get_rings(bp);
7100}
7101
7102static int
7103bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7104 int ring_grps, int cp_rings, int stats, int vnics)
7105{
7106 struct hwrm_func_vf_cfg_input *req;
7107 int rc;
7108
7109 if (!BNXT_NEW_RM(bp)) {
7110 bp->hw_resc.resv_tx_rings = tx_rings;
7111 return 0;
7112 }
7113
7114 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
7115 cp_rings, stats, vnics);
7116 if (!req)
7117 return -ENOMEM;
7118
7119 rc = hwrm_req_send(bp, req);
7120 if (rc)
7121 return rc;
7122
7123 return bnxt_hwrm_get_rings(bp);
7124}
7125
7126static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
7127 int cp, int stat, int vnic)
7128{
7129 if (BNXT_PF(bp))
7130 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
7131 vnic);
7132 else
7133 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
7134 vnic);
7135}
7136
7137int bnxt_nq_rings_in_use(struct bnxt *bp)
7138{
7139 int cp = bp->cp_nr_rings;
7140 int ulp_msix, ulp_base;
7141
7142 ulp_msix = bnxt_get_ulp_msix_num(bp);
7143 if (ulp_msix) {
7144 ulp_base = bnxt_get_ulp_msix_base(bp);
7145 cp += ulp_msix;
7146 if ((ulp_base + ulp_msix) > cp)
7147 cp = ulp_base + ulp_msix;
7148 }
7149 return cp;
7150}
7151
7152static int bnxt_cp_rings_in_use(struct bnxt *bp)
7153{
7154 int cp;
7155
7156 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7157 return bnxt_nq_rings_in_use(bp);
7158
7159 cp = bp->tx_nr_rings + bp->rx_nr_rings;
7160 return cp;
7161}
7162
7163static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
7164{
7165 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
7166 int cp = bp->cp_nr_rings;
7167
7168 if (!ulp_stat)
7169 return cp;
7170
7171 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
7172 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
7173
7174 return cp + ulp_stat;
7175}
7176
7177/* Check if a default RSS map needs to be setup. This function is only
7178 * used on older firmware that does not require reserving RX rings.
7179 */
7180static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
7181{
7182 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7183
7184 /* The RSS map is valid for RX rings set to resv_rx_rings */
7185 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
7186 hw_resc->resv_rx_rings = bp->rx_nr_rings;
7187 if (!netif_is_rxfh_configured(bp->dev))
7188 bnxt_set_dflt_rss_indir_tbl(bp);
7189 }
7190}
7191
7192static bool bnxt_need_reserve_rings(struct bnxt *bp)
7193{
7194 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7195 int cp = bnxt_cp_rings_in_use(bp);
7196 int nq = bnxt_nq_rings_in_use(bp);
7197 int rx = bp->rx_nr_rings, stat;
7198 int vnic = 1, grp = rx;
7199
7200 if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
7201 bp->hwrm_spec_code >= 0x10601)
7202 return true;
7203
7204 /* Old firmware does not need RX ring reservations but we still
7205 * need to setup a default RSS map when needed. With new firmware
7206 * we go through RX ring reservations first and then set up the
7207 * RSS map for the successfully reserved RX rings when needed.
7208 */
7209 if (!BNXT_NEW_RM(bp)) {
7210 bnxt_check_rss_tbl_no_rmgr(bp);
7211 return false;
7212 }
7213 if ((bp->flags & BNXT_FLAG_RFS) &&
7214 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7215 vnic = rx + 1;
7216 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7217 rx <<= 1;
7218 stat = bnxt_get_func_stat_ctxs(bp);
7219 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
7220 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
7221 (hw_resc->resv_hw_ring_grps != grp &&
7222 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)))
7223 return true;
7224 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && BNXT_PF(bp) &&
7225 hw_resc->resv_irqs != nq)
7226 return true;
7227 return false;
7228}
7229
7230static int __bnxt_reserve_rings(struct bnxt *bp)
7231{
7232 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7233 int cp = bnxt_nq_rings_in_use(bp);
7234 int tx = bp->tx_nr_rings;
7235 int rx = bp->rx_nr_rings;
7236 int grp, rx_rings, rc;
7237 int vnic = 1, stat;
7238 bool sh = false;
7239 int tx_cp;
7240
7241 if (!bnxt_need_reserve_rings(bp))
7242 return 0;
7243
7244 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7245 sh = true;
7246 if ((bp->flags & BNXT_FLAG_RFS) &&
7247 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7248 vnic = rx + 1;
7249 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7250 rx <<= 1;
7251 grp = bp->rx_nr_rings;
7252 stat = bnxt_get_func_stat_ctxs(bp);
7253
7254 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
7255 if (rc)
7256 return rc;
7257
7258 tx = hw_resc->resv_tx_rings;
7259 if (BNXT_NEW_RM(bp)) {
7260 rx = hw_resc->resv_rx_rings;
7261 cp = hw_resc->resv_irqs;
7262 grp = hw_resc->resv_hw_ring_grps;
7263 vnic = hw_resc->resv_vnics;
7264 stat = hw_resc->resv_stat_ctxs;
7265 }
7266
7267 rx_rings = rx;
7268 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7269 if (rx >= 2) {
7270 rx_rings = rx >> 1;
7271 } else {
7272 if (netif_running(bp->dev))
7273 return -ENOMEM;
7274
7275 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
7276 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
7277 bp->dev->hw_features &= ~NETIF_F_LRO;
7278 bp->dev->features &= ~NETIF_F_LRO;
7279 bnxt_set_ring_params(bp);
7280 }
7281 }
7282 rx_rings = min_t(int, rx_rings, grp);
7283 cp = min_t(int, cp, bp->cp_nr_rings);
7284 if (stat > bnxt_get_ulp_stat_ctxs(bp))
7285 stat -= bnxt_get_ulp_stat_ctxs(bp);
7286 cp = min_t(int, cp, stat);
7287 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
7288 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7289 rx = rx_rings << 1;
7290 tx_cp = bnxt_num_tx_to_cp(bp, tx);
7291 cp = sh ? max_t(int, tx_cp, rx_rings) : tx_cp + rx_rings;
7292 bp->tx_nr_rings = tx;
7293
7294 /* If we cannot reserve all the RX rings, reset the RSS map only
7295 * if absolutely necessary
7296 */
7297 if (rx_rings != bp->rx_nr_rings) {
7298 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
7299 rx_rings, bp->rx_nr_rings);
7300 if (netif_is_rxfh_configured(bp->dev) &&
7301 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
7302 bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
7303 bnxt_get_max_rss_ring(bp) >= rx_rings)) {
7304 netdev_warn(bp->dev, "RSS table entries reverting to default\n");
7305 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
7306 }
7307 }
7308 bp->rx_nr_rings = rx_rings;
7309 bp->cp_nr_rings = cp;
7310
7311 if (!tx || !rx || !cp || !grp || !vnic || !stat)
7312 return -ENOMEM;
7313
7314 if (!netif_is_rxfh_configured(bp->dev))
7315 bnxt_set_dflt_rss_indir_tbl(bp);
7316
7317 return rc;
7318}
7319
7320static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7321 int ring_grps, int cp_rings, int stats,
7322 int vnics)
7323{
7324 struct hwrm_func_vf_cfg_input *req;
7325 u32 flags;
7326
7327 if (!BNXT_NEW_RM(bp))
7328 return 0;
7329
7330 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
7331 cp_rings, stats, vnics);
7332 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
7333 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7334 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7335 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7336 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
7337 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
7338 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7339 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7340
7341 req->flags = cpu_to_le32(flags);
7342 return hwrm_req_send_silent(bp, req);
7343}
7344
7345static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7346 int ring_grps, int cp_rings, int stats,
7347 int vnics)
7348{
7349 struct hwrm_func_cfg_input *req;
7350 u32 flags;
7351
7352 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
7353 cp_rings, stats, vnics);
7354 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
7355 if (BNXT_NEW_RM(bp)) {
7356 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7357 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7358 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7359 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
7360 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7361 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
7362 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
7363 else
7364 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7365 }
7366
7367 req->flags = cpu_to_le32(flags);
7368 return hwrm_req_send_silent(bp, req);
7369}
7370
7371static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7372 int ring_grps, int cp_rings, int stats,
7373 int vnics)
7374{
7375 if (bp->hwrm_spec_code < 0x10801)
7376 return 0;
7377
7378 if (BNXT_PF(bp))
7379 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
7380 ring_grps, cp_rings, stats,
7381 vnics);
7382
7383 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
7384 cp_rings, stats, vnics);
7385}
7386
7387static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
7388{
7389 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7390 struct hwrm_ring_aggint_qcaps_output *resp;
7391 struct hwrm_ring_aggint_qcaps_input *req;
7392 int rc;
7393
7394 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
7395 coal_cap->num_cmpl_dma_aggr_max = 63;
7396 coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
7397 coal_cap->cmpl_aggr_dma_tmr_max = 65535;
7398 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
7399 coal_cap->int_lat_tmr_min_max = 65535;
7400 coal_cap->int_lat_tmr_max_max = 65535;
7401 coal_cap->num_cmpl_aggr_int_max = 65535;
7402 coal_cap->timer_units = 80;
7403
7404 if (bp->hwrm_spec_code < 0x10902)
7405 return;
7406
7407 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
7408 return;
7409
7410 resp = hwrm_req_hold(bp, req);
7411 rc = hwrm_req_send_silent(bp, req);
7412 if (!rc) {
7413 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
7414 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
7415 coal_cap->num_cmpl_dma_aggr_max =
7416 le16_to_cpu(resp->num_cmpl_dma_aggr_max);
7417 coal_cap->num_cmpl_dma_aggr_during_int_max =
7418 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
7419 coal_cap->cmpl_aggr_dma_tmr_max =
7420 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
7421 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
7422 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
7423 coal_cap->int_lat_tmr_min_max =
7424 le16_to_cpu(resp->int_lat_tmr_min_max);
7425 coal_cap->int_lat_tmr_max_max =
7426 le16_to_cpu(resp->int_lat_tmr_max_max);
7427 coal_cap->num_cmpl_aggr_int_max =
7428 le16_to_cpu(resp->num_cmpl_aggr_int_max);
7429 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
7430 }
7431 hwrm_req_drop(bp, req);
7432}
7433
7434static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
7435{
7436 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7437
7438 return usec * 1000 / coal_cap->timer_units;
7439}
7440
7441static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
7442 struct bnxt_coal *hw_coal,
7443 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7444{
7445 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7446 u16 val, tmr, max, flags = hw_coal->flags;
7447 u32 cmpl_params = coal_cap->cmpl_params;
7448
7449 max = hw_coal->bufs_per_record * 128;
7450 if (hw_coal->budget)
7451 max = hw_coal->bufs_per_record * hw_coal->budget;
7452 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
7453
7454 val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
7455 req->num_cmpl_aggr_int = cpu_to_le16(val);
7456
7457 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
7458 req->num_cmpl_dma_aggr = cpu_to_le16(val);
7459
7460 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
7461 coal_cap->num_cmpl_dma_aggr_during_int_max);
7462 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
7463
7464 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
7465 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
7466 req->int_lat_tmr_max = cpu_to_le16(tmr);
7467
7468 /* min timer set to 1/2 of interrupt timer */
7469 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
7470 val = tmr / 2;
7471 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
7472 req->int_lat_tmr_min = cpu_to_le16(val);
7473 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7474 }
7475
7476 /* buf timer set to 1/4 of interrupt timer */
7477 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
7478 req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
7479
7480 if (cmpl_params &
7481 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
7482 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
7483 val = clamp_t(u16, tmr, 1,
7484 coal_cap->cmpl_aggr_dma_tmr_during_int_max);
7485 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
7486 req->enables |=
7487 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
7488 }
7489
7490 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
7491 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
7492 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
7493 req->flags = cpu_to_le16(flags);
7494 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
7495}
7496
7497static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
7498 struct bnxt_coal *hw_coal)
7499{
7500 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
7501 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7502 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7503 u32 nq_params = coal_cap->nq_params;
7504 u16 tmr;
7505 int rc;
7506
7507 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
7508 return 0;
7509
7510 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7511 if (rc)
7512 return rc;
7513
7514 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
7515 req->flags =
7516 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
7517
7518 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
7519 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
7520 req->int_lat_tmr_min = cpu_to_le16(tmr);
7521 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7522 return hwrm_req_send(bp, req);
7523}
7524
7525int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
7526{
7527 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
7528 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7529 struct bnxt_coal coal;
7530 int rc;
7531
7532 /* Tick values in micro seconds.
7533 * 1 coal_buf x bufs_per_record = 1 completion record.
7534 */
7535 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
7536
7537 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
7538 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
7539
7540 if (!bnapi->rx_ring)
7541 return -ENODEV;
7542
7543 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7544 if (rc)
7545 return rc;
7546
7547 bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
7548
7549 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
7550
7551 return hwrm_req_send(bp, req_rx);
7552}
7553
7554static int
7555bnxt_hwrm_set_rx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
7556 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7557{
7558 u16 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
7559
7560 req->ring_id = cpu_to_le16(ring_id);
7561 return hwrm_req_send(bp, req);
7562}
7563
7564static int
7565bnxt_hwrm_set_tx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
7566 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7567{
7568 struct bnxt_tx_ring_info *txr;
7569 int i, rc;
7570
7571 bnxt_for_each_napi_tx(i, bnapi, txr) {
7572 u16 ring_id;
7573
7574 ring_id = bnxt_cp_ring_for_tx(bp, txr);
7575 req->ring_id = cpu_to_le16(ring_id);
7576 rc = hwrm_req_send(bp, req);
7577 if (rc)
7578 return rc;
7579 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7580 return 0;
7581 }
7582 return 0;
7583}
7584
7585int bnxt_hwrm_set_coal(struct bnxt *bp)
7586{
7587 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx;
7588 int i, rc;
7589
7590 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7591 if (rc)
7592 return rc;
7593
7594 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7595 if (rc) {
7596 hwrm_req_drop(bp, req_rx);
7597 return rc;
7598 }
7599
7600 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
7601 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
7602
7603 hwrm_req_hold(bp, req_rx);
7604 hwrm_req_hold(bp, req_tx);
7605 for (i = 0; i < bp->cp_nr_rings; i++) {
7606 struct bnxt_napi *bnapi = bp->bnapi[i];
7607 struct bnxt_coal *hw_coal;
7608
7609 if (!bnapi->rx_ring)
7610 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
7611 else
7612 rc = bnxt_hwrm_set_rx_coal(bp, bnapi, req_rx);
7613 if (rc)
7614 break;
7615
7616 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7617 continue;
7618
7619 if (bnapi->rx_ring && bnapi->tx_ring[0]) {
7620 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
7621 if (rc)
7622 break;
7623 }
7624 if (bnapi->rx_ring)
7625 hw_coal = &bp->rx_coal;
7626 else
7627 hw_coal = &bp->tx_coal;
7628 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
7629 }
7630 hwrm_req_drop(bp, req_rx);
7631 hwrm_req_drop(bp, req_tx);
7632 return rc;
7633}
7634
7635static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
7636{
7637 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
7638 struct hwrm_stat_ctx_free_input *req;
7639 int i;
7640
7641 if (!bp->bnapi)
7642 return;
7643
7644 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7645 return;
7646
7647 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
7648 return;
7649 if (BNXT_FW_MAJ(bp) <= 20) {
7650 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
7651 hwrm_req_drop(bp, req);
7652 return;
7653 }
7654 hwrm_req_hold(bp, req0);
7655 }
7656 hwrm_req_hold(bp, req);
7657 for (i = 0; i < bp->cp_nr_rings; i++) {
7658 struct bnxt_napi *bnapi = bp->bnapi[i];
7659 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7660
7661 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
7662 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
7663 if (req0) {
7664 req0->stat_ctx_id = req->stat_ctx_id;
7665 hwrm_req_send(bp, req0);
7666 }
7667 hwrm_req_send(bp, req);
7668
7669 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
7670 }
7671 }
7672 hwrm_req_drop(bp, req);
7673 if (req0)
7674 hwrm_req_drop(bp, req0);
7675}
7676
7677static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
7678{
7679 struct hwrm_stat_ctx_alloc_output *resp;
7680 struct hwrm_stat_ctx_alloc_input *req;
7681 int rc, i;
7682
7683 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7684 return 0;
7685
7686 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
7687 if (rc)
7688 return rc;
7689
7690 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
7691 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
7692
7693 resp = hwrm_req_hold(bp, req);
7694 for (i = 0; i < bp->cp_nr_rings; i++) {
7695 struct bnxt_napi *bnapi = bp->bnapi[i];
7696 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7697
7698 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
7699
7700 rc = hwrm_req_send(bp, req);
7701 if (rc)
7702 break;
7703
7704 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
7705
7706 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
7707 }
7708 hwrm_req_drop(bp, req);
7709 return rc;
7710}
7711
7712static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
7713{
7714 struct hwrm_func_qcfg_output *resp;
7715 struct hwrm_func_qcfg_input *req;
7716 u16 flags;
7717 int rc;
7718
7719 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7720 if (rc)
7721 return rc;
7722
7723 req->fid = cpu_to_le16(0xffff);
7724 resp = hwrm_req_hold(bp, req);
7725 rc = hwrm_req_send(bp, req);
7726 if (rc)
7727 goto func_qcfg_exit;
7728
7729#ifdef CONFIG_BNXT_SRIOV
7730 if (BNXT_VF(bp)) {
7731 struct bnxt_vf_info *vf = &bp->vf;
7732
7733 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
7734 } else {
7735 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
7736 }
7737#endif
7738 flags = le16_to_cpu(resp->flags);
7739 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
7740 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
7741 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
7742 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
7743 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
7744 }
7745 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
7746 bp->flags |= BNXT_FLAG_MULTI_HOST;
7747
7748 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
7749 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
7750
7751 switch (resp->port_partition_type) {
7752 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
7753 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
7754 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
7755 bp->port_partition_type = resp->port_partition_type;
7756 break;
7757 }
7758 if (bp->hwrm_spec_code < 0x10707 ||
7759 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
7760 bp->br_mode = BRIDGE_MODE_VEB;
7761 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
7762 bp->br_mode = BRIDGE_MODE_VEPA;
7763 else
7764 bp->br_mode = BRIDGE_MODE_UNDEF;
7765
7766 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
7767 if (!bp->max_mtu)
7768 bp->max_mtu = BNXT_MAX_MTU;
7769
7770 if (bp->db_size)
7771 goto func_qcfg_exit;
7772
7773 bp->db_offset = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024;
7774 if (BNXT_CHIP_P5(bp)) {
7775 if (BNXT_PF(bp))
7776 bp->db_offset = DB_PF_OFFSET_P5;
7777 else
7778 bp->db_offset = DB_VF_OFFSET_P5;
7779 }
7780 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
7781 1024);
7782 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
7783 bp->db_size <= bp->db_offset)
7784 bp->db_size = pci_resource_len(bp->pdev, 2);
7785
7786func_qcfg_exit:
7787 hwrm_req_drop(bp, req);
7788 return rc;
7789}
7790
7791static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type *ctxm,
7792 u8 init_val, u8 init_offset,
7793 bool init_mask_set)
7794{
7795 ctxm->init_value = init_val;
7796 ctxm->init_offset = BNXT_CTX_INIT_INVALID_OFFSET;
7797 if (init_mask_set)
7798 ctxm->init_offset = init_offset * 4;
7799 else
7800 ctxm->init_value = 0;
7801}
7802
7803static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max)
7804{
7805 struct bnxt_ctx_mem_info *ctx = bp->ctx;
7806 u16 type;
7807
7808 for (type = 0; type < ctx_max; type++) {
7809 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
7810 int n = 1;
7811
7812 if (!ctxm->max_entries)
7813 continue;
7814
7815 if (ctxm->instance_bmap)
7816 n = hweight32(ctxm->instance_bmap);
7817 ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_KERNEL);
7818 if (!ctxm->pg_info)
7819 return -ENOMEM;
7820 }
7821 return 0;
7822}
7823
7824#define BNXT_CTX_INIT_VALID(flags) \
7825 (!!((flags) & \
7826 FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT))
7827
7828static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp)
7829{
7830 struct hwrm_func_backing_store_qcaps_v2_output *resp;
7831 struct hwrm_func_backing_store_qcaps_v2_input *req;
7832 struct bnxt_ctx_mem_info *ctx;
7833 u16 type;
7834 int rc;
7835
7836 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2);
7837 if (rc)
7838 return rc;
7839
7840 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7841 if (!ctx)
7842 return -ENOMEM;
7843 bp->ctx = ctx;
7844
7845 resp = hwrm_req_hold(bp, req);
7846
7847 for (type = 0; type < BNXT_CTX_V2_MAX; ) {
7848 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
7849 u8 init_val, init_off, i;
7850 __le32 *p;
7851 u32 flags;
7852
7853 req->type = cpu_to_le16(type);
7854 rc = hwrm_req_send(bp, req);
7855 if (rc)
7856 goto ctx_done;
7857 flags = le32_to_cpu(resp->flags);
7858 type = le16_to_cpu(resp->next_valid_type);
7859 if (!(flags & FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_TYPE_VALID))
7860 continue;
7861
7862 ctxm->type = le16_to_cpu(resp->type);
7863 ctxm->entry_size = le16_to_cpu(resp->entry_size);
7864 ctxm->flags = flags;
7865 ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map);
7866 ctxm->entry_multiple = resp->entry_multiple;
7867 ctxm->max_entries = le32_to_cpu(resp->max_num_entries);
7868 ctxm->min_entries = le32_to_cpu(resp->min_num_entries);
7869 init_val = resp->ctx_init_value;
7870 init_off = resp->ctx_init_offset;
7871 bnxt_init_ctx_initializer(ctxm, init_val, init_off,
7872 BNXT_CTX_INIT_VALID(flags));
7873 ctxm->split_entry_cnt = min_t(u8, resp->subtype_valid_cnt,
7874 BNXT_MAX_SPLIT_ENTRY);
7875 for (i = 0, p = &resp->split_entry_0; i < ctxm->split_entry_cnt;
7876 i++, p++)
7877 ctxm->split[i] = le32_to_cpu(*p);
7878 }
7879 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_V2_MAX);
7880
7881ctx_done:
7882 hwrm_req_drop(bp, req);
7883 return rc;
7884}
7885
7886static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
7887{
7888 struct hwrm_func_backing_store_qcaps_output *resp;
7889 struct hwrm_func_backing_store_qcaps_input *req;
7890 int rc;
7891
7892 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
7893 return 0;
7894
7895 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
7896 return bnxt_hwrm_func_backing_store_qcaps_v2(bp);
7897
7898 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
7899 if (rc)
7900 return rc;
7901
7902 resp = hwrm_req_hold(bp, req);
7903 rc = hwrm_req_send_silent(bp, req);
7904 if (!rc) {
7905 struct bnxt_ctx_mem_type *ctxm;
7906 struct bnxt_ctx_mem_info *ctx;
7907 u8 init_val, init_idx = 0;
7908 u16 init_mask;
7909
7910 ctx = bp->ctx;
7911 if (!ctx) {
7912 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7913 if (!ctx) {
7914 rc = -ENOMEM;
7915 goto ctx_err;
7916 }
7917 bp->ctx = ctx;
7918 }
7919 init_val = resp->ctx_kind_initializer;
7920 init_mask = le16_to_cpu(resp->ctx_init_mask);
7921
7922 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
7923 ctxm->max_entries = le32_to_cpu(resp->qp_max_entries);
7924 ctxm->qp_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
7925 ctxm->qp_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
7926 ctxm->qp_fast_qpmd_entries = le16_to_cpu(resp->fast_qpmd_qp_num_entries);
7927 ctxm->entry_size = le16_to_cpu(resp->qp_entry_size);
7928 bnxt_init_ctx_initializer(ctxm, init_val, resp->qp_init_offset,
7929 (init_mask & (1 << init_idx++)) != 0);
7930
7931 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
7932 ctxm->srq_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
7933 ctxm->max_entries = le32_to_cpu(resp->srq_max_entries);
7934 ctxm->entry_size = le16_to_cpu(resp->srq_entry_size);
7935 bnxt_init_ctx_initializer(ctxm, init_val, resp->srq_init_offset,
7936 (init_mask & (1 << init_idx++)) != 0);
7937
7938 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
7939 ctxm->cq_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
7940 ctxm->max_entries = le32_to_cpu(resp->cq_max_entries);
7941 ctxm->entry_size = le16_to_cpu(resp->cq_entry_size);
7942 bnxt_init_ctx_initializer(ctxm, init_val, resp->cq_init_offset,
7943 (init_mask & (1 << init_idx++)) != 0);
7944
7945 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
7946 ctxm->vnic_entries = le16_to_cpu(resp->vnic_max_vnic_entries);
7947 ctxm->max_entries = ctxm->vnic_entries +
7948 le16_to_cpu(resp->vnic_max_ring_table_entries);
7949 ctxm->entry_size = le16_to_cpu(resp->vnic_entry_size);
7950 bnxt_init_ctx_initializer(ctxm, init_val,
7951 resp->vnic_init_offset,
7952 (init_mask & (1 << init_idx++)) != 0);
7953
7954 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
7955 ctxm->max_entries = le32_to_cpu(resp->stat_max_entries);
7956 ctxm->entry_size = le16_to_cpu(resp->stat_entry_size);
7957 bnxt_init_ctx_initializer(ctxm, init_val,
7958 resp->stat_init_offset,
7959 (init_mask & (1 << init_idx++)) != 0);
7960
7961 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
7962 ctxm->entry_size = le16_to_cpu(resp->tqm_entry_size);
7963 ctxm->min_entries = le32_to_cpu(resp->tqm_min_entries_per_ring);
7964 ctxm->max_entries = le32_to_cpu(resp->tqm_max_entries_per_ring);
7965 ctxm->entry_multiple = resp->tqm_entries_multiple;
7966 if (!ctxm->entry_multiple)
7967 ctxm->entry_multiple = 1;
7968
7969 memcpy(&ctx->ctx_arr[BNXT_CTX_FTQM], ctxm, sizeof(*ctxm));
7970
7971 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
7972 ctxm->max_entries = le32_to_cpu(resp->mrav_max_entries);
7973 ctxm->entry_size = le16_to_cpu(resp->mrav_entry_size);
7974 ctxm->mrav_num_entries_units =
7975 le16_to_cpu(resp->mrav_num_entries_units);
7976 bnxt_init_ctx_initializer(ctxm, init_val,
7977 resp->mrav_init_offset,
7978 (init_mask & (1 << init_idx++)) != 0);
7979
7980 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
7981 ctxm->entry_size = le16_to_cpu(resp->tim_entry_size);
7982 ctxm->max_entries = le32_to_cpu(resp->tim_max_entries);
7983
7984 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
7985 if (!ctx->tqm_fp_rings_count)
7986 ctx->tqm_fp_rings_count = bp->max_q;
7987 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
7988 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
7989
7990 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
7991 memcpy(ctxm, &ctx->ctx_arr[BNXT_CTX_STQM], sizeof(*ctxm));
7992 ctxm->instance_bmap = (1 << ctx->tqm_fp_rings_count) - 1;
7993
7994 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_MAX);
7995 } else {
7996 rc = 0;
7997 }
7998ctx_err:
7999 hwrm_req_drop(bp, req);
8000 return rc;
8001}
8002
8003static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
8004 __le64 *pg_dir)
8005{
8006 if (!rmem->nr_pages)
8007 return;
8008
8009 BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
8010 if (rmem->depth >= 1) {
8011 if (rmem->depth == 2)
8012 *pg_attr |= 2;
8013 else
8014 *pg_attr |= 1;
8015 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
8016 } else {
8017 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
8018 }
8019}
8020
8021#define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \
8022 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \
8023 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \
8024 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \
8025 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \
8026 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
8027
8028static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
8029{
8030 struct hwrm_func_backing_store_cfg_input *req;
8031 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8032 struct bnxt_ctx_pg_info *ctx_pg;
8033 struct bnxt_ctx_mem_type *ctxm;
8034 void **__req = (void **)&req;
8035 u32 req_len = sizeof(*req);
8036 __le32 *num_entries;
8037 __le64 *pg_dir;
8038 u32 flags = 0;
8039 u8 *pg_attr;
8040 u32 ena;
8041 int rc;
8042 int i;
8043
8044 if (!ctx)
8045 return 0;
8046
8047 if (req_len > bp->hwrm_max_ext_req_len)
8048 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
8049 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
8050 if (rc)
8051 return rc;
8052
8053 req->enables = cpu_to_le32(enables);
8054 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
8055 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8056 ctx_pg = ctxm->pg_info;
8057 req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
8058 req->qp_num_qp1_entries = cpu_to_le16(ctxm->qp_qp1_entries);
8059 req->qp_num_l2_entries = cpu_to_le16(ctxm->qp_l2_entries);
8060 req->qp_entry_size = cpu_to_le16(ctxm->entry_size);
8061 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8062 &req->qpc_pg_size_qpc_lvl,
8063 &req->qpc_page_dir);
8064
8065 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD)
8066 req->qp_num_fast_qpmd_entries = cpu_to_le16(ctxm->qp_fast_qpmd_entries);
8067 }
8068 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
8069 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8070 ctx_pg = ctxm->pg_info;
8071 req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
8072 req->srq_num_l2_entries = cpu_to_le16(ctxm->srq_l2_entries);
8073 req->srq_entry_size = cpu_to_le16(ctxm->entry_size);
8074 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8075 &req->srq_pg_size_srq_lvl,
8076 &req->srq_page_dir);
8077 }
8078 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
8079 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8080 ctx_pg = ctxm->pg_info;
8081 req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
8082 req->cq_num_l2_entries = cpu_to_le16(ctxm->cq_l2_entries);
8083 req->cq_entry_size = cpu_to_le16(ctxm->entry_size);
8084 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8085 &req->cq_pg_size_cq_lvl,
8086 &req->cq_page_dir);
8087 }
8088 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
8089 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8090 ctx_pg = ctxm->pg_info;
8091 req->vnic_num_vnic_entries = cpu_to_le16(ctxm->vnic_entries);
8092 req->vnic_num_ring_table_entries =
8093 cpu_to_le16(ctxm->max_entries - ctxm->vnic_entries);
8094 req->vnic_entry_size = cpu_to_le16(ctxm->entry_size);
8095 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8096 &req->vnic_pg_size_vnic_lvl,
8097 &req->vnic_page_dir);
8098 }
8099 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
8100 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8101 ctx_pg = ctxm->pg_info;
8102 req->stat_num_entries = cpu_to_le32(ctxm->max_entries);
8103 req->stat_entry_size = cpu_to_le16(ctxm->entry_size);
8104 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8105 &req->stat_pg_size_stat_lvl,
8106 &req->stat_page_dir);
8107 }
8108 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
8109 u32 units;
8110
8111 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8112 ctx_pg = ctxm->pg_info;
8113 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
8114 units = ctxm->mrav_num_entries_units;
8115 if (units) {
8116 u32 num_mr, num_ah = ctxm->mrav_av_entries;
8117 u32 entries;
8118
8119 num_mr = ctx_pg->entries - num_ah;
8120 entries = ((num_mr / units) << 16) | (num_ah / units);
8121 req->mrav_num_entries = cpu_to_le32(entries);
8122 flags |= FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
8123 }
8124 req->mrav_entry_size = cpu_to_le16(ctxm->entry_size);
8125 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8126 &req->mrav_pg_size_mrav_lvl,
8127 &req->mrav_page_dir);
8128 }
8129 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
8130 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8131 ctx_pg = ctxm->pg_info;
8132 req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
8133 req->tim_entry_size = cpu_to_le16(ctxm->entry_size);
8134 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8135 &req->tim_pg_size_tim_lvl,
8136 &req->tim_page_dir);
8137 }
8138 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8139 for (i = 0, num_entries = &req->tqm_sp_num_entries,
8140 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
8141 pg_dir = &req->tqm_sp_page_dir,
8142 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP,
8143 ctx_pg = ctxm->pg_info;
8144 i < BNXT_MAX_TQM_RINGS;
8145 ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[i],
8146 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
8147 if (!(enables & ena))
8148 continue;
8149
8150 req->tqm_entry_size = cpu_to_le16(ctxm->entry_size);
8151 *num_entries = cpu_to_le32(ctx_pg->entries);
8152 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
8153 }
8154 req->flags = cpu_to_le32(flags);
8155 return hwrm_req_send(bp, req);
8156}
8157
8158static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
8159 struct bnxt_ctx_pg_info *ctx_pg)
8160{
8161 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8162
8163 rmem->page_size = BNXT_PAGE_SIZE;
8164 rmem->pg_arr = ctx_pg->ctx_pg_arr;
8165 rmem->dma_arr = ctx_pg->ctx_dma_arr;
8166 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
8167 if (rmem->depth >= 1)
8168 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
8169 return bnxt_alloc_ring(bp, rmem);
8170}
8171
8172static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
8173 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
8174 u8 depth, struct bnxt_ctx_mem_type *ctxm)
8175{
8176 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8177 int rc;
8178
8179 if (!mem_size)
8180 return -EINVAL;
8181
8182 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8183 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
8184 ctx_pg->nr_pages = 0;
8185 return -EINVAL;
8186 }
8187 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
8188 int nr_tbls, i;
8189
8190 rmem->depth = 2;
8191 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
8192 GFP_KERNEL);
8193 if (!ctx_pg->ctx_pg_tbl)
8194 return -ENOMEM;
8195 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
8196 rmem->nr_pages = nr_tbls;
8197 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8198 if (rc)
8199 return rc;
8200 for (i = 0; i < nr_tbls; i++) {
8201 struct bnxt_ctx_pg_info *pg_tbl;
8202
8203 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
8204 if (!pg_tbl)
8205 return -ENOMEM;
8206 ctx_pg->ctx_pg_tbl[i] = pg_tbl;
8207 rmem = &pg_tbl->ring_mem;
8208 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
8209 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
8210 rmem->depth = 1;
8211 rmem->nr_pages = MAX_CTX_PAGES;
8212 rmem->ctx_mem = ctxm;
8213 if (i == (nr_tbls - 1)) {
8214 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
8215
8216 if (rem)
8217 rmem->nr_pages = rem;
8218 }
8219 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
8220 if (rc)
8221 break;
8222 }
8223 } else {
8224 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8225 if (rmem->nr_pages > 1 || depth)
8226 rmem->depth = 1;
8227 rmem->ctx_mem = ctxm;
8228 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8229 }
8230 return rc;
8231}
8232
8233static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
8234 struct bnxt_ctx_pg_info *ctx_pg)
8235{
8236 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8237
8238 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
8239 ctx_pg->ctx_pg_tbl) {
8240 int i, nr_tbls = rmem->nr_pages;
8241
8242 for (i = 0; i < nr_tbls; i++) {
8243 struct bnxt_ctx_pg_info *pg_tbl;
8244 struct bnxt_ring_mem_info *rmem2;
8245
8246 pg_tbl = ctx_pg->ctx_pg_tbl[i];
8247 if (!pg_tbl)
8248 continue;
8249 rmem2 = &pg_tbl->ring_mem;
8250 bnxt_free_ring(bp, rmem2);
8251 ctx_pg->ctx_pg_arr[i] = NULL;
8252 kfree(pg_tbl);
8253 ctx_pg->ctx_pg_tbl[i] = NULL;
8254 }
8255 kfree(ctx_pg->ctx_pg_tbl);
8256 ctx_pg->ctx_pg_tbl = NULL;
8257 }
8258 bnxt_free_ring(bp, rmem);
8259 ctx_pg->nr_pages = 0;
8260}
8261
8262static int bnxt_setup_ctxm_pg_tbls(struct bnxt *bp,
8263 struct bnxt_ctx_mem_type *ctxm, u32 entries,
8264 u8 pg_lvl)
8265{
8266 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8267 int i, rc = 0, n = 1;
8268 u32 mem_size;
8269
8270 if (!ctxm->entry_size || !ctx_pg)
8271 return -EINVAL;
8272 if (ctxm->instance_bmap)
8273 n = hweight32(ctxm->instance_bmap);
8274 if (ctxm->entry_multiple)
8275 entries = roundup(entries, ctxm->entry_multiple);
8276 entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries);
8277 mem_size = entries * ctxm->entry_size;
8278 for (i = 0; i < n && !rc; i++) {
8279 ctx_pg[i].entries = entries;
8280 rc = bnxt_alloc_ctx_pg_tbls(bp, &ctx_pg[i], mem_size, pg_lvl,
8281 ctxm->init_value ? ctxm : NULL);
8282 }
8283 return rc;
8284}
8285
8286static int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt *bp,
8287 struct bnxt_ctx_mem_type *ctxm,
8288 bool last)
8289{
8290 struct hwrm_func_backing_store_cfg_v2_input *req;
8291 u32 instance_bmap = ctxm->instance_bmap;
8292 int i, j, rc = 0, n = 1;
8293 __le32 *p;
8294
8295 if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID) || !ctxm->pg_info)
8296 return 0;
8297
8298 if (instance_bmap)
8299 n = hweight32(ctxm->instance_bmap);
8300 else
8301 instance_bmap = 1;
8302
8303 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_CFG_V2);
8304 if (rc)
8305 return rc;
8306 hwrm_req_hold(bp, req);
8307 req->type = cpu_to_le16(ctxm->type);
8308 req->entry_size = cpu_to_le16(ctxm->entry_size);
8309 req->subtype_valid_cnt = ctxm->split_entry_cnt;
8310 for (i = 0, p = &req->split_entry_0; i < ctxm->split_entry_cnt; i++)
8311 p[i] = cpu_to_le32(ctxm->split[i]);
8312 for (i = 0, j = 0; j < n && !rc; i++) {
8313 struct bnxt_ctx_pg_info *ctx_pg;
8314
8315 if (!(instance_bmap & (1 << i)))
8316 continue;
8317 req->instance = cpu_to_le16(i);
8318 ctx_pg = &ctxm->pg_info[j++];
8319 if (!ctx_pg->entries)
8320 continue;
8321 req->num_entries = cpu_to_le32(ctx_pg->entries);
8322 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8323 &req->page_size_pbl_level,
8324 &req->page_dir);
8325 if (last && j == n)
8326 req->flags =
8327 cpu_to_le32(FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE);
8328 rc = hwrm_req_send(bp, req);
8329 }
8330 hwrm_req_drop(bp, req);
8331 return rc;
8332}
8333
8334static int bnxt_backing_store_cfg_v2(struct bnxt *bp, u32 ena)
8335{
8336 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8337 struct bnxt_ctx_mem_type *ctxm;
8338 u16 last_type;
8339 int rc = 0;
8340 u16 type;
8341
8342 if (!ena)
8343 return 0;
8344 else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM)
8345 last_type = BNXT_CTX_MAX - 1;
8346 else
8347 last_type = BNXT_CTX_L2_MAX - 1;
8348 ctx->ctx_arr[last_type].last = 1;
8349
8350 for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) {
8351 ctxm = &ctx->ctx_arr[type];
8352
8353 rc = bnxt_hwrm_func_backing_store_cfg_v2(bp, ctxm, ctxm->last);
8354 if (rc)
8355 return rc;
8356 }
8357 return 0;
8358}
8359
8360void bnxt_free_ctx_mem(struct bnxt *bp)
8361{
8362 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8363 u16 type;
8364
8365 if (!ctx)
8366 return;
8367
8368 for (type = 0; type < BNXT_CTX_V2_MAX; type++) {
8369 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8370 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8371 int i, n = 1;
8372
8373 if (!ctx_pg)
8374 continue;
8375 if (ctxm->instance_bmap)
8376 n = hweight32(ctxm->instance_bmap);
8377 for (i = 0; i < n; i++)
8378 bnxt_free_ctx_pg_tbls(bp, &ctx_pg[i]);
8379
8380 kfree(ctx_pg);
8381 ctxm->pg_info = NULL;
8382 }
8383
8384 ctx->flags &= ~BNXT_CTX_FLAG_INITED;
8385 kfree(ctx);
8386 bp->ctx = NULL;
8387}
8388
8389static int bnxt_alloc_ctx_mem(struct bnxt *bp)
8390{
8391 struct bnxt_ctx_mem_type *ctxm;
8392 struct bnxt_ctx_mem_info *ctx;
8393 u32 l2_qps, qp1_qps, max_qps;
8394 u32 ena, entries_sp, entries;
8395 u32 srqs, max_srqs, min;
8396 u32 num_mr, num_ah;
8397 u32 extra_srqs = 0;
8398 u32 extra_qps = 0;
8399 u32 fast_qpmd_qps;
8400 u8 pg_lvl = 1;
8401 int i, rc;
8402
8403 rc = bnxt_hwrm_func_backing_store_qcaps(bp);
8404 if (rc) {
8405 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
8406 rc);
8407 return rc;
8408 }
8409 ctx = bp->ctx;
8410 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
8411 return 0;
8412
8413 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8414 l2_qps = ctxm->qp_l2_entries;
8415 qp1_qps = ctxm->qp_qp1_entries;
8416 fast_qpmd_qps = ctxm->qp_fast_qpmd_entries;
8417 max_qps = ctxm->max_entries;
8418 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8419 srqs = ctxm->srq_l2_entries;
8420 max_srqs = ctxm->max_entries;
8421 ena = 0;
8422 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
8423 pg_lvl = 2;
8424 extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps);
8425 /* allocate extra qps if fw supports RoCE fast qp destroy feature */
8426 extra_qps += fast_qpmd_qps;
8427 extra_srqs = min_t(u32, 8192, max_srqs - srqs);
8428 if (fast_qpmd_qps)
8429 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD;
8430 }
8431
8432 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8433 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps,
8434 pg_lvl);
8435 if (rc)
8436 return rc;
8437
8438 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8439 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, srqs + extra_srqs, pg_lvl);
8440 if (rc)
8441 return rc;
8442
8443 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8444 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->cq_l2_entries +
8445 extra_qps * 2, pg_lvl);
8446 if (rc)
8447 return rc;
8448
8449 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8450 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
8451 if (rc)
8452 return rc;
8453
8454 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8455 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
8456 if (rc)
8457 return rc;
8458
8459 if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
8460 goto skip_rdma;
8461
8462 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8463 /* 128K extra is needed to accommodate static AH context
8464 * allocation by f/w.
8465 */
8466 num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256);
8467 num_ah = min_t(u32, num_mr, 1024 * 128);
8468 ctxm->split_entry_cnt = BNXT_CTX_MRAV_AV_SPLIT_ENTRY + 1;
8469 if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah)
8470 ctxm->mrav_av_entries = num_ah;
8471
8472 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, num_mr + num_ah, 2);
8473 if (rc)
8474 return rc;
8475 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
8476
8477 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8478 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 1);
8479 if (rc)
8480 return rc;
8481 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
8482
8483skip_rdma:
8484 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8485 min = ctxm->min_entries;
8486 entries_sp = ctx->ctx_arr[BNXT_CTX_VNIC].vnic_entries + l2_qps +
8487 2 * (extra_qps + qp1_qps) + min;
8488 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries_sp, 2);
8489 if (rc)
8490 return rc;
8491
8492 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
8493 entries = l2_qps + 2 * (extra_qps + qp1_qps);
8494 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries, 2);
8495 if (rc)
8496 return rc;
8497 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
8498 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
8499 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
8500
8501 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
8502 rc = bnxt_backing_store_cfg_v2(bp, ena);
8503 else
8504 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
8505 if (rc) {
8506 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
8507 rc);
8508 return rc;
8509 }
8510 ctx->flags |= BNXT_CTX_FLAG_INITED;
8511 return 0;
8512}
8513
8514int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
8515{
8516 struct hwrm_func_resource_qcaps_output *resp;
8517 struct hwrm_func_resource_qcaps_input *req;
8518 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8519 int rc;
8520
8521 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
8522 if (rc)
8523 return rc;
8524
8525 req->fid = cpu_to_le16(0xffff);
8526 resp = hwrm_req_hold(bp, req);
8527 rc = hwrm_req_send_silent(bp, req);
8528 if (rc)
8529 goto hwrm_func_resc_qcaps_exit;
8530
8531 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
8532 if (!all)
8533 goto hwrm_func_resc_qcaps_exit;
8534
8535 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
8536 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
8537 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
8538 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
8539 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
8540 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
8541 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
8542 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
8543 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
8544 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
8545 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
8546 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
8547 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
8548 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
8549 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
8550 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
8551
8552 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
8553 u16 max_msix = le16_to_cpu(resp->max_msix);
8554
8555 hw_resc->max_nqs = max_msix;
8556 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
8557 }
8558
8559 if (BNXT_PF(bp)) {
8560 struct bnxt_pf_info *pf = &bp->pf;
8561
8562 pf->vf_resv_strategy =
8563 le16_to_cpu(resp->vf_reservation_strategy);
8564 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
8565 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
8566 }
8567hwrm_func_resc_qcaps_exit:
8568 hwrm_req_drop(bp, req);
8569 return rc;
8570}
8571
8572static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
8573{
8574 struct hwrm_port_mac_ptp_qcfg_output *resp;
8575 struct hwrm_port_mac_ptp_qcfg_input *req;
8576 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
8577 bool phc_cfg;
8578 u8 flags;
8579 int rc;
8580
8581 if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5(bp)) {
8582 rc = -ENODEV;
8583 goto no_ptp;
8584 }
8585
8586 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
8587 if (rc)
8588 goto no_ptp;
8589
8590 req->port_id = cpu_to_le16(bp->pf.port_id);
8591 resp = hwrm_req_hold(bp, req);
8592 rc = hwrm_req_send(bp, req);
8593 if (rc)
8594 goto exit;
8595
8596 flags = resp->flags;
8597 if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
8598 rc = -ENODEV;
8599 goto exit;
8600 }
8601 if (!ptp) {
8602 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
8603 if (!ptp) {
8604 rc = -ENOMEM;
8605 goto exit;
8606 }
8607 ptp->bp = bp;
8608 bp->ptp_cfg = ptp;
8609 }
8610 if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
8611 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
8612 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
8613 } else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
8614 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
8615 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
8616 } else {
8617 rc = -ENODEV;
8618 goto exit;
8619 }
8620 phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
8621 rc = bnxt_ptp_init(bp, phc_cfg);
8622 if (rc)
8623 netdev_warn(bp->dev, "PTP initialization failed.\n");
8624exit:
8625 hwrm_req_drop(bp, req);
8626 if (!rc)
8627 return 0;
8628
8629no_ptp:
8630 bnxt_ptp_clear(bp);
8631 kfree(ptp);
8632 bp->ptp_cfg = NULL;
8633 return rc;
8634}
8635
8636static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
8637{
8638 struct hwrm_func_qcaps_output *resp;
8639 struct hwrm_func_qcaps_input *req;
8640 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8641 u32 flags, flags_ext, flags_ext2;
8642 int rc;
8643
8644 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
8645 if (rc)
8646 return rc;
8647
8648 req->fid = cpu_to_le16(0xffff);
8649 resp = hwrm_req_hold(bp, req);
8650 rc = hwrm_req_send(bp, req);
8651 if (rc)
8652 goto hwrm_func_qcaps_exit;
8653
8654 flags = le32_to_cpu(resp->flags);
8655 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
8656 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
8657 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
8658 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
8659 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
8660 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
8661 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
8662 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
8663 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
8664 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
8665 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
8666 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
8667 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
8668 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
8669 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
8670 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
8671 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
8672 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
8673
8674 flags_ext = le32_to_cpu(resp->flags_ext);
8675 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
8676 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
8677 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
8678 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
8679 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
8680 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
8681 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
8682 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
8683 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
8684 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
8685 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED)
8686 bp->fw_cap |= BNXT_FW_CAP_BACKING_STORE_V2;
8687 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP)
8688 bp->flags |= BNXT_FLAG_TX_COAL_CMPL;
8689
8690 flags_ext2 = le32_to_cpu(resp->flags_ext2);
8691 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
8692 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
8693 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED)
8694 bp->flags |= BNXT_FLAG_UDP_GSO_CAP;
8695
8696 bp->tx_push_thresh = 0;
8697 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
8698 BNXT_FW_MAJ(bp) > 217)
8699 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
8700
8701 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
8702 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
8703 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
8704 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
8705 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
8706 if (!hw_resc->max_hw_ring_grps)
8707 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
8708 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
8709 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
8710 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
8711
8712 if (BNXT_PF(bp)) {
8713 struct bnxt_pf_info *pf = &bp->pf;
8714
8715 pf->fw_fid = le16_to_cpu(resp->fid);
8716 pf->port_id = le16_to_cpu(resp->port_id);
8717 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
8718 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
8719 pf->max_vfs = le16_to_cpu(resp->max_vfs);
8720 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
8721 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
8722 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
8723 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
8724 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
8725 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
8726 bp->flags &= ~BNXT_FLAG_WOL_CAP;
8727 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
8728 bp->flags |= BNXT_FLAG_WOL_CAP;
8729 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
8730 bp->fw_cap |= BNXT_FW_CAP_PTP;
8731 } else {
8732 bnxt_ptp_clear(bp);
8733 kfree(bp->ptp_cfg);
8734 bp->ptp_cfg = NULL;
8735 }
8736 } else {
8737#ifdef CONFIG_BNXT_SRIOV
8738 struct bnxt_vf_info *vf = &bp->vf;
8739
8740 vf->fw_fid = le16_to_cpu(resp->fid);
8741 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
8742#endif
8743 }
8744
8745hwrm_func_qcaps_exit:
8746 hwrm_req_drop(bp, req);
8747 return rc;
8748}
8749
8750static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
8751{
8752 struct hwrm_dbg_qcaps_output *resp;
8753 struct hwrm_dbg_qcaps_input *req;
8754 int rc;
8755
8756 bp->fw_dbg_cap = 0;
8757 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
8758 return;
8759
8760 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
8761 if (rc)
8762 return;
8763
8764 req->fid = cpu_to_le16(0xffff);
8765 resp = hwrm_req_hold(bp, req);
8766 rc = hwrm_req_send(bp, req);
8767 if (rc)
8768 goto hwrm_dbg_qcaps_exit;
8769
8770 bp->fw_dbg_cap = le32_to_cpu(resp->flags);
8771
8772hwrm_dbg_qcaps_exit:
8773 hwrm_req_drop(bp, req);
8774}
8775
8776static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
8777
8778int bnxt_hwrm_func_qcaps(struct bnxt *bp)
8779{
8780 int rc;
8781
8782 rc = __bnxt_hwrm_func_qcaps(bp);
8783 if (rc)
8784 return rc;
8785
8786 bnxt_hwrm_dbg_qcaps(bp);
8787
8788 rc = bnxt_hwrm_queue_qportcfg(bp);
8789 if (rc) {
8790 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
8791 return rc;
8792 }
8793 if (bp->hwrm_spec_code >= 0x10803) {
8794 rc = bnxt_alloc_ctx_mem(bp);
8795 if (rc)
8796 return rc;
8797 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
8798 if (!rc)
8799 bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
8800 }
8801 return 0;
8802}
8803
8804static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
8805{
8806 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
8807 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
8808 u32 flags;
8809 int rc;
8810
8811 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
8812 return 0;
8813
8814 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
8815 if (rc)
8816 return rc;
8817
8818 resp = hwrm_req_hold(bp, req);
8819 rc = hwrm_req_send(bp, req);
8820 if (rc)
8821 goto hwrm_cfa_adv_qcaps_exit;
8822
8823 flags = le32_to_cpu(resp->flags);
8824 if (flags &
8825 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
8826 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
8827
8828hwrm_cfa_adv_qcaps_exit:
8829 hwrm_req_drop(bp, req);
8830 return rc;
8831}
8832
8833static int __bnxt_alloc_fw_health(struct bnxt *bp)
8834{
8835 if (bp->fw_health)
8836 return 0;
8837
8838 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
8839 if (!bp->fw_health)
8840 return -ENOMEM;
8841
8842 mutex_init(&bp->fw_health->lock);
8843 return 0;
8844}
8845
8846static int bnxt_alloc_fw_health(struct bnxt *bp)
8847{
8848 int rc;
8849
8850 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
8851 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
8852 return 0;
8853
8854 rc = __bnxt_alloc_fw_health(bp);
8855 if (rc) {
8856 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
8857 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
8858 return rc;
8859 }
8860
8861 return 0;
8862}
8863
8864static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
8865{
8866 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
8867 BNXT_GRCPF_REG_WINDOW_BASE_OUT +
8868 BNXT_FW_HEALTH_WIN_MAP_OFF);
8869}
8870
8871static void bnxt_inv_fw_health_reg(struct bnxt *bp)
8872{
8873 struct bnxt_fw_health *fw_health = bp->fw_health;
8874 u32 reg_type;
8875
8876 if (!fw_health)
8877 return;
8878
8879 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
8880 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
8881 fw_health->status_reliable = false;
8882
8883 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
8884 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
8885 fw_health->resets_reliable = false;
8886}
8887
8888static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
8889{
8890 void __iomem *hs;
8891 u32 status_loc;
8892 u32 reg_type;
8893 u32 sig;
8894
8895 if (bp->fw_health)
8896 bp->fw_health->status_reliable = false;
8897
8898 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
8899 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
8900
8901 sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
8902 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
8903 if (!bp->chip_num) {
8904 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
8905 bp->chip_num = readl(bp->bar0 +
8906 BNXT_FW_HEALTH_WIN_BASE +
8907 BNXT_GRC_REG_CHIP_NUM);
8908 }
8909 if (!BNXT_CHIP_P5(bp))
8910 return;
8911
8912 status_loc = BNXT_GRC_REG_STATUS_P5 |
8913 BNXT_FW_HEALTH_REG_TYPE_BAR0;
8914 } else {
8915 status_loc = readl(hs + offsetof(struct hcomm_status,
8916 fw_status_loc));
8917 }
8918
8919 if (__bnxt_alloc_fw_health(bp)) {
8920 netdev_warn(bp->dev, "no memory for firmware status checks\n");
8921 return;
8922 }
8923
8924 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
8925 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
8926 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
8927 __bnxt_map_fw_health_reg(bp, status_loc);
8928 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
8929 BNXT_FW_HEALTH_WIN_OFF(status_loc);
8930 }
8931
8932 bp->fw_health->status_reliable = true;
8933}
8934
8935static int bnxt_map_fw_health_regs(struct bnxt *bp)
8936{
8937 struct bnxt_fw_health *fw_health = bp->fw_health;
8938 u32 reg_base = 0xffffffff;
8939 int i;
8940
8941 bp->fw_health->status_reliable = false;
8942 bp->fw_health->resets_reliable = false;
8943 /* Only pre-map the monitoring GRC registers using window 3 */
8944 for (i = 0; i < 4; i++) {
8945 u32 reg = fw_health->regs[i];
8946
8947 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
8948 continue;
8949 if (reg_base == 0xffffffff)
8950 reg_base = reg & BNXT_GRC_BASE_MASK;
8951 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
8952 return -ERANGE;
8953 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
8954 }
8955 bp->fw_health->status_reliable = true;
8956 bp->fw_health->resets_reliable = true;
8957 if (reg_base == 0xffffffff)
8958 return 0;
8959
8960 __bnxt_map_fw_health_reg(bp, reg_base);
8961 return 0;
8962}
8963
8964static void bnxt_remap_fw_health_regs(struct bnxt *bp)
8965{
8966 if (!bp->fw_health)
8967 return;
8968
8969 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
8970 bp->fw_health->status_reliable = true;
8971 bp->fw_health->resets_reliable = true;
8972 } else {
8973 bnxt_try_map_fw_health_reg(bp);
8974 }
8975}
8976
8977static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
8978{
8979 struct bnxt_fw_health *fw_health = bp->fw_health;
8980 struct hwrm_error_recovery_qcfg_output *resp;
8981 struct hwrm_error_recovery_qcfg_input *req;
8982 int rc, i;
8983
8984 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
8985 return 0;
8986
8987 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
8988 if (rc)
8989 return rc;
8990
8991 resp = hwrm_req_hold(bp, req);
8992 rc = hwrm_req_send(bp, req);
8993 if (rc)
8994 goto err_recovery_out;
8995 fw_health->flags = le32_to_cpu(resp->flags);
8996 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
8997 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
8998 rc = -EINVAL;
8999 goto err_recovery_out;
9000 }
9001 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
9002 fw_health->master_func_wait_dsecs =
9003 le32_to_cpu(resp->master_func_wait_period);
9004 fw_health->normal_func_wait_dsecs =
9005 le32_to_cpu(resp->normal_func_wait_period);
9006 fw_health->post_reset_wait_dsecs =
9007 le32_to_cpu(resp->master_func_wait_period_after_reset);
9008 fw_health->post_reset_max_wait_dsecs =
9009 le32_to_cpu(resp->max_bailout_time_after_reset);
9010 fw_health->regs[BNXT_FW_HEALTH_REG] =
9011 le32_to_cpu(resp->fw_health_status_reg);
9012 fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
9013 le32_to_cpu(resp->fw_heartbeat_reg);
9014 fw_health->regs[BNXT_FW_RESET_CNT_REG] =
9015 le32_to_cpu(resp->fw_reset_cnt_reg);
9016 fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
9017 le32_to_cpu(resp->reset_inprogress_reg);
9018 fw_health->fw_reset_inprog_reg_mask =
9019 le32_to_cpu(resp->reset_inprogress_reg_mask);
9020 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
9021 if (fw_health->fw_reset_seq_cnt >= 16) {
9022 rc = -EINVAL;
9023 goto err_recovery_out;
9024 }
9025 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
9026 fw_health->fw_reset_seq_regs[i] =
9027 le32_to_cpu(resp->reset_reg[i]);
9028 fw_health->fw_reset_seq_vals[i] =
9029 le32_to_cpu(resp->reset_reg_val[i]);
9030 fw_health->fw_reset_seq_delay_msec[i] =
9031 resp->delay_after_reset[i];
9032 }
9033err_recovery_out:
9034 hwrm_req_drop(bp, req);
9035 if (!rc)
9036 rc = bnxt_map_fw_health_regs(bp);
9037 if (rc)
9038 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
9039 return rc;
9040}
9041
9042static int bnxt_hwrm_func_reset(struct bnxt *bp)
9043{
9044 struct hwrm_func_reset_input *req;
9045 int rc;
9046
9047 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
9048 if (rc)
9049 return rc;
9050
9051 req->enables = 0;
9052 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
9053 return hwrm_req_send(bp, req);
9054}
9055
9056static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
9057{
9058 struct hwrm_nvm_get_dev_info_output nvm_info;
9059
9060 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
9061 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
9062 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
9063 nvm_info.nvm_cfg_ver_upd);
9064}
9065
9066static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
9067{
9068 struct hwrm_queue_qportcfg_output *resp;
9069 struct hwrm_queue_qportcfg_input *req;
9070 u8 i, j, *qptr;
9071 bool no_rdma;
9072 int rc = 0;
9073
9074 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
9075 if (rc)
9076 return rc;
9077
9078 resp = hwrm_req_hold(bp, req);
9079 rc = hwrm_req_send(bp, req);
9080 if (rc)
9081 goto qportcfg_exit;
9082
9083 if (!resp->max_configurable_queues) {
9084 rc = -EINVAL;
9085 goto qportcfg_exit;
9086 }
9087 bp->max_tc = resp->max_configurable_queues;
9088 bp->max_lltc = resp->max_configurable_lossless_queues;
9089 if (bp->max_tc > BNXT_MAX_QUEUE)
9090 bp->max_tc = BNXT_MAX_QUEUE;
9091
9092 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
9093 qptr = &resp->queue_id0;
9094 for (i = 0, j = 0; i < bp->max_tc; i++) {
9095 bp->q_info[j].queue_id = *qptr;
9096 bp->q_ids[i] = *qptr++;
9097 bp->q_info[j].queue_profile = *qptr++;
9098 bp->tc_to_qidx[j] = j;
9099 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
9100 (no_rdma && BNXT_PF(bp)))
9101 j++;
9102 }
9103 bp->max_q = bp->max_tc;
9104 bp->max_tc = max_t(u8, j, 1);
9105
9106 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
9107 bp->max_tc = 1;
9108
9109 if (bp->max_lltc > bp->max_tc)
9110 bp->max_lltc = bp->max_tc;
9111
9112qportcfg_exit:
9113 hwrm_req_drop(bp, req);
9114 return rc;
9115}
9116
9117static int bnxt_hwrm_poll(struct bnxt *bp)
9118{
9119 struct hwrm_ver_get_input *req;
9120 int rc;
9121
9122 rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9123 if (rc)
9124 return rc;
9125
9126 req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9127 req->hwrm_intf_min = HWRM_VERSION_MINOR;
9128 req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9129
9130 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
9131 rc = hwrm_req_send(bp, req);
9132 return rc;
9133}
9134
9135static int bnxt_hwrm_ver_get(struct bnxt *bp)
9136{
9137 struct hwrm_ver_get_output *resp;
9138 struct hwrm_ver_get_input *req;
9139 u16 fw_maj, fw_min, fw_bld, fw_rsv;
9140 u32 dev_caps_cfg, hwrm_ver;
9141 int rc, len;
9142
9143 rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9144 if (rc)
9145 return rc;
9146
9147 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
9148 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
9149 req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9150 req->hwrm_intf_min = HWRM_VERSION_MINOR;
9151 req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9152
9153 resp = hwrm_req_hold(bp, req);
9154 rc = hwrm_req_send(bp, req);
9155 if (rc)
9156 goto hwrm_ver_get_exit;
9157
9158 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
9159
9160 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
9161 resp->hwrm_intf_min_8b << 8 |
9162 resp->hwrm_intf_upd_8b;
9163 if (resp->hwrm_intf_maj_8b < 1) {
9164 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
9165 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9166 resp->hwrm_intf_upd_8b);
9167 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
9168 }
9169
9170 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
9171 HWRM_VERSION_UPDATE;
9172
9173 if (bp->hwrm_spec_code > hwrm_ver)
9174 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9175 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
9176 HWRM_VERSION_UPDATE);
9177 else
9178 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9179 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9180 resp->hwrm_intf_upd_8b);
9181
9182 fw_maj = le16_to_cpu(resp->hwrm_fw_major);
9183 if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
9184 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
9185 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
9186 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
9187 len = FW_VER_STR_LEN;
9188 } else {
9189 fw_maj = resp->hwrm_fw_maj_8b;
9190 fw_min = resp->hwrm_fw_min_8b;
9191 fw_bld = resp->hwrm_fw_bld_8b;
9192 fw_rsv = resp->hwrm_fw_rsvd_8b;
9193 len = BC_HWRM_STR_LEN;
9194 }
9195 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
9196 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
9197 fw_rsv);
9198
9199 if (strlen(resp->active_pkg_name)) {
9200 int fw_ver_len = strlen(bp->fw_ver_str);
9201
9202 snprintf(bp->fw_ver_str + fw_ver_len,
9203 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
9204 resp->active_pkg_name);
9205 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
9206 }
9207
9208 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
9209 if (!bp->hwrm_cmd_timeout)
9210 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
9211 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
9212 if (!bp->hwrm_cmd_max_timeout)
9213 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
9214 else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
9215 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
9216 bp->hwrm_cmd_max_timeout / 1000);
9217
9218 if (resp->hwrm_intf_maj_8b >= 1) {
9219 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
9220 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
9221 }
9222 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
9223 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
9224
9225 bp->chip_num = le16_to_cpu(resp->chip_num);
9226 bp->chip_rev = resp->chip_rev;
9227 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
9228 !resp->chip_metal)
9229 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
9230
9231 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
9232 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
9233 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
9234 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
9235
9236 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
9237 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
9238
9239 if (dev_caps_cfg &
9240 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
9241 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
9242
9243 if (dev_caps_cfg &
9244 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
9245 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
9246
9247 if (dev_caps_cfg &
9248 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
9249 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
9250
9251hwrm_ver_get_exit:
9252 hwrm_req_drop(bp, req);
9253 return rc;
9254}
9255
9256int bnxt_hwrm_fw_set_time(struct bnxt *bp)
9257{
9258 struct hwrm_fw_set_time_input *req;
9259 struct tm tm;
9260 time64_t now = ktime_get_real_seconds();
9261 int rc;
9262
9263 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
9264 bp->hwrm_spec_code < 0x10400)
9265 return -EOPNOTSUPP;
9266
9267 time64_to_tm(now, 0, &tm);
9268 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
9269 if (rc)
9270 return rc;
9271
9272 req->year = cpu_to_le16(1900 + tm.tm_year);
9273 req->month = 1 + tm.tm_mon;
9274 req->day = tm.tm_mday;
9275 req->hour = tm.tm_hour;
9276 req->minute = tm.tm_min;
9277 req->second = tm.tm_sec;
9278 return hwrm_req_send(bp, req);
9279}
9280
9281static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
9282{
9283 u64 sw_tmp;
9284
9285 hw &= mask;
9286 sw_tmp = (*sw & ~mask) | hw;
9287 if (hw < (*sw & mask))
9288 sw_tmp += mask + 1;
9289 WRITE_ONCE(*sw, sw_tmp);
9290}
9291
9292static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
9293 int count, bool ignore_zero)
9294{
9295 int i;
9296
9297 for (i = 0; i < count; i++) {
9298 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
9299
9300 if (ignore_zero && !hw)
9301 continue;
9302
9303 if (masks[i] == -1ULL)
9304 sw_stats[i] = hw;
9305 else
9306 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
9307 }
9308}
9309
9310static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
9311{
9312 if (!stats->hw_stats)
9313 return;
9314
9315 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
9316 stats->hw_masks, stats->len / 8, false);
9317}
9318
9319static void bnxt_accumulate_all_stats(struct bnxt *bp)
9320{
9321 struct bnxt_stats_mem *ring0_stats;
9322 bool ignore_zero = false;
9323 int i;
9324
9325 /* Chip bug. Counter intermittently becomes 0. */
9326 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9327 ignore_zero = true;
9328
9329 for (i = 0; i < bp->cp_nr_rings; i++) {
9330 struct bnxt_napi *bnapi = bp->bnapi[i];
9331 struct bnxt_cp_ring_info *cpr;
9332 struct bnxt_stats_mem *stats;
9333
9334 cpr = &bnapi->cp_ring;
9335 stats = &cpr->stats;
9336 if (!i)
9337 ring0_stats = stats;
9338 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
9339 ring0_stats->hw_masks,
9340 ring0_stats->len / 8, ignore_zero);
9341 }
9342 if (bp->flags & BNXT_FLAG_PORT_STATS) {
9343 struct bnxt_stats_mem *stats = &bp->port_stats;
9344 __le64 *hw_stats = stats->hw_stats;
9345 u64 *sw_stats = stats->sw_stats;
9346 u64 *masks = stats->hw_masks;
9347 int cnt;
9348
9349 cnt = sizeof(struct rx_port_stats) / 8;
9350 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
9351
9352 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9353 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9354 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9355 cnt = sizeof(struct tx_port_stats) / 8;
9356 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
9357 }
9358 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
9359 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
9360 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
9361 }
9362}
9363
9364static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
9365{
9366 struct hwrm_port_qstats_input *req;
9367 struct bnxt_pf_info *pf = &bp->pf;
9368 int rc;
9369
9370 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
9371 return 0;
9372
9373 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
9374 return -EOPNOTSUPP;
9375
9376 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
9377 if (rc)
9378 return rc;
9379
9380 req->flags = flags;
9381 req->port_id = cpu_to_le16(pf->port_id);
9382 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
9383 BNXT_TX_PORT_STATS_BYTE_OFFSET);
9384 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
9385 return hwrm_req_send(bp, req);
9386}
9387
9388static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
9389{
9390 struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
9391 struct hwrm_queue_pri2cos_qcfg_input *req_qc;
9392 struct hwrm_port_qstats_ext_output *resp_qs;
9393 struct hwrm_port_qstats_ext_input *req_qs;
9394 struct bnxt_pf_info *pf = &bp->pf;
9395 u32 tx_stat_size;
9396 int rc;
9397
9398 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
9399 return 0;
9400
9401 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
9402 return -EOPNOTSUPP;
9403
9404 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
9405 if (rc)
9406 return rc;
9407
9408 req_qs->flags = flags;
9409 req_qs->port_id = cpu_to_le16(pf->port_id);
9410 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
9411 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
9412 tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
9413 sizeof(struct tx_port_stats_ext) : 0;
9414 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
9415 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
9416 resp_qs = hwrm_req_hold(bp, req_qs);
9417 rc = hwrm_req_send(bp, req_qs);
9418 if (!rc) {
9419 bp->fw_rx_stats_ext_size =
9420 le16_to_cpu(resp_qs->rx_stat_size) / 8;
9421 if (BNXT_FW_MAJ(bp) < 220 &&
9422 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
9423 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
9424
9425 bp->fw_tx_stats_ext_size = tx_stat_size ?
9426 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
9427 } else {
9428 bp->fw_rx_stats_ext_size = 0;
9429 bp->fw_tx_stats_ext_size = 0;
9430 }
9431 hwrm_req_drop(bp, req_qs);
9432
9433 if (flags)
9434 return rc;
9435
9436 if (bp->fw_tx_stats_ext_size <=
9437 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
9438 bp->pri2cos_valid = 0;
9439 return rc;
9440 }
9441
9442 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
9443 if (rc)
9444 return rc;
9445
9446 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
9447
9448 resp_qc = hwrm_req_hold(bp, req_qc);
9449 rc = hwrm_req_send(bp, req_qc);
9450 if (!rc) {
9451 u8 *pri2cos;
9452 int i, j;
9453
9454 pri2cos = &resp_qc->pri0_cos_queue_id;
9455 for (i = 0; i < 8; i++) {
9456 u8 queue_id = pri2cos[i];
9457 u8 queue_idx;
9458
9459 /* Per port queue IDs start from 0, 10, 20, etc */
9460 queue_idx = queue_id % 10;
9461 if (queue_idx > BNXT_MAX_QUEUE) {
9462 bp->pri2cos_valid = false;
9463 hwrm_req_drop(bp, req_qc);
9464 return rc;
9465 }
9466 for (j = 0; j < bp->max_q; j++) {
9467 if (bp->q_ids[j] == queue_id)
9468 bp->pri2cos_idx[i] = queue_idx;
9469 }
9470 }
9471 bp->pri2cos_valid = true;
9472 }
9473 hwrm_req_drop(bp, req_qc);
9474
9475 return rc;
9476}
9477
9478static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
9479{
9480 bnxt_hwrm_tunnel_dst_port_free(bp,
9481 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9482 bnxt_hwrm_tunnel_dst_port_free(bp,
9483 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9484}
9485
9486static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
9487{
9488 int rc, i;
9489 u32 tpa_flags = 0;
9490
9491 if (set_tpa)
9492 tpa_flags = bp->flags & BNXT_FLAG_TPA;
9493 else if (BNXT_NO_FW_ACCESS(bp))
9494 return 0;
9495 for (i = 0; i < bp->nr_vnics; i++) {
9496 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
9497 if (rc) {
9498 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
9499 i, rc);
9500 return rc;
9501 }
9502 }
9503 return 0;
9504}
9505
9506static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
9507{
9508 int i;
9509
9510 for (i = 0; i < bp->nr_vnics; i++)
9511 bnxt_hwrm_vnic_set_rss(bp, i, false);
9512}
9513
9514static void bnxt_clear_vnic(struct bnxt *bp)
9515{
9516 if (!bp->vnic_info)
9517 return;
9518
9519 bnxt_hwrm_clear_vnic_filter(bp);
9520 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) {
9521 /* clear all RSS setting before free vnic ctx */
9522 bnxt_hwrm_clear_vnic_rss(bp);
9523 bnxt_hwrm_vnic_ctx_free(bp);
9524 }
9525 /* before free the vnic, undo the vnic tpa settings */
9526 if (bp->flags & BNXT_FLAG_TPA)
9527 bnxt_set_tpa(bp, false);
9528 bnxt_hwrm_vnic_free(bp);
9529 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9530 bnxt_hwrm_vnic_ctx_free(bp);
9531}
9532
9533static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
9534 bool irq_re_init)
9535{
9536 bnxt_clear_vnic(bp);
9537 bnxt_hwrm_ring_free(bp, close_path);
9538 bnxt_hwrm_ring_grp_free(bp);
9539 if (irq_re_init) {
9540 bnxt_hwrm_stat_ctx_free(bp);
9541 bnxt_hwrm_free_tunnel_ports(bp);
9542 }
9543}
9544
9545static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
9546{
9547 struct hwrm_func_cfg_input *req;
9548 u8 evb_mode;
9549 int rc;
9550
9551 if (br_mode == BRIDGE_MODE_VEB)
9552 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
9553 else if (br_mode == BRIDGE_MODE_VEPA)
9554 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
9555 else
9556 return -EINVAL;
9557
9558 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
9559 if (rc)
9560 return rc;
9561
9562 req->fid = cpu_to_le16(0xffff);
9563 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
9564 req->evb_mode = evb_mode;
9565 return hwrm_req_send(bp, req);
9566}
9567
9568static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
9569{
9570 struct hwrm_func_cfg_input *req;
9571 int rc;
9572
9573 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
9574 return 0;
9575
9576 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
9577 if (rc)
9578 return rc;
9579
9580 req->fid = cpu_to_le16(0xffff);
9581 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
9582 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
9583 if (size == 128)
9584 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
9585
9586 return hwrm_req_send(bp, req);
9587}
9588
9589static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
9590{
9591 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
9592 int rc;
9593
9594 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
9595 goto skip_rss_ctx;
9596
9597 /* allocate context for vnic */
9598 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
9599 if (rc) {
9600 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
9601 vnic_id, rc);
9602 goto vnic_setup_err;
9603 }
9604 bp->rsscos_nr_ctxs++;
9605
9606 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9607 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
9608 if (rc) {
9609 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
9610 vnic_id, rc);
9611 goto vnic_setup_err;
9612 }
9613 bp->rsscos_nr_ctxs++;
9614 }
9615
9616skip_rss_ctx:
9617 /* configure default vnic, ring grp */
9618 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
9619 if (rc) {
9620 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
9621 vnic_id, rc);
9622 goto vnic_setup_err;
9623 }
9624
9625 /* Enable RSS hashing on vnic */
9626 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
9627 if (rc) {
9628 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
9629 vnic_id, rc);
9630 goto vnic_setup_err;
9631 }
9632
9633 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
9634 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
9635 if (rc) {
9636 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
9637 vnic_id, rc);
9638 }
9639 }
9640
9641vnic_setup_err:
9642 return rc;
9643}
9644
9645static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
9646{
9647 int rc, i, nr_ctxs;
9648
9649 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
9650 for (i = 0; i < nr_ctxs; i++) {
9651 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
9652 if (rc) {
9653 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
9654 vnic_id, i, rc);
9655 break;
9656 }
9657 bp->rsscos_nr_ctxs++;
9658 }
9659 if (i < nr_ctxs)
9660 return -ENOMEM;
9661
9662 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
9663 if (rc) {
9664 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
9665 vnic_id, rc);
9666 return rc;
9667 }
9668 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
9669 if (rc) {
9670 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
9671 vnic_id, rc);
9672 return rc;
9673 }
9674 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
9675 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
9676 if (rc) {
9677 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
9678 vnic_id, rc);
9679 }
9680 }
9681 return rc;
9682}
9683
9684static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
9685{
9686 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9687 return __bnxt_setup_vnic_p5(bp, vnic_id);
9688 else
9689 return __bnxt_setup_vnic(bp, vnic_id);
9690}
9691
9692static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
9693{
9694 int i, rc = 0;
9695
9696 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9697 return 0;
9698
9699 for (i = 0; i < bp->rx_nr_rings; i++) {
9700 struct bnxt_vnic_info *vnic;
9701 u16 vnic_id = i + 1;
9702 u16 ring_id = i;
9703
9704 if (vnic_id >= bp->nr_vnics)
9705 break;
9706
9707 vnic = &bp->vnic_info[vnic_id];
9708 vnic->flags |= BNXT_VNIC_RFS_FLAG;
9709 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
9710 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
9711 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
9712 if (rc) {
9713 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
9714 vnic_id, rc);
9715 break;
9716 }
9717 rc = bnxt_setup_vnic(bp, vnic_id);
9718 if (rc)
9719 break;
9720 }
9721 return rc;
9722}
9723
9724/* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
9725static bool bnxt_promisc_ok(struct bnxt *bp)
9726{
9727#ifdef CONFIG_BNXT_SRIOV
9728 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
9729 return false;
9730#endif
9731 return true;
9732}
9733
9734static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
9735{
9736 unsigned int rc = 0;
9737
9738 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
9739 if (rc) {
9740 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
9741 rc);
9742 return rc;
9743 }
9744
9745 rc = bnxt_hwrm_vnic_cfg(bp, 1);
9746 if (rc) {
9747 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
9748 rc);
9749 return rc;
9750 }
9751 return rc;
9752}
9753
9754static int bnxt_cfg_rx_mode(struct bnxt *);
9755static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
9756
9757static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
9758{
9759 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9760 int rc = 0;
9761 unsigned int rx_nr_rings = bp->rx_nr_rings;
9762
9763 if (irq_re_init) {
9764 rc = bnxt_hwrm_stat_ctx_alloc(bp);
9765 if (rc) {
9766 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
9767 rc);
9768 goto err_out;
9769 }
9770 }
9771
9772 rc = bnxt_hwrm_ring_alloc(bp);
9773 if (rc) {
9774 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
9775 goto err_out;
9776 }
9777
9778 rc = bnxt_hwrm_ring_grp_alloc(bp);
9779 if (rc) {
9780 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
9781 goto err_out;
9782 }
9783
9784 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
9785 rx_nr_rings--;
9786
9787 /* default vnic 0 */
9788 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
9789 if (rc) {
9790 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
9791 goto err_out;
9792 }
9793
9794 if (BNXT_VF(bp))
9795 bnxt_hwrm_func_qcfg(bp);
9796
9797 rc = bnxt_setup_vnic(bp, 0);
9798 if (rc)
9799 goto err_out;
9800 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
9801 bnxt_hwrm_update_rss_hash_cfg(bp);
9802
9803 if (bp->flags & BNXT_FLAG_RFS) {
9804 rc = bnxt_alloc_rfs_vnics(bp);
9805 if (rc)
9806 goto err_out;
9807 }
9808
9809 if (bp->flags & BNXT_FLAG_TPA) {
9810 rc = bnxt_set_tpa(bp, true);
9811 if (rc)
9812 goto err_out;
9813 }
9814
9815 if (BNXT_VF(bp))
9816 bnxt_update_vf_mac(bp);
9817
9818 /* Filter for default vnic 0 */
9819 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
9820 if (rc) {
9821 if (BNXT_VF(bp) && rc == -ENODEV)
9822 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
9823 else
9824 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
9825 goto err_out;
9826 }
9827 vnic->uc_filter_count = 1;
9828
9829 vnic->rx_mask = 0;
9830 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
9831 goto skip_rx_mask;
9832
9833 if (bp->dev->flags & IFF_BROADCAST)
9834 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
9835
9836 if (bp->dev->flags & IFF_PROMISC)
9837 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9838
9839 if (bp->dev->flags & IFF_ALLMULTI) {
9840 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9841 vnic->mc_list_count = 0;
9842 } else if (bp->dev->flags & IFF_MULTICAST) {
9843 u32 mask = 0;
9844
9845 bnxt_mc_list_updated(bp, &mask);
9846 vnic->rx_mask |= mask;
9847 }
9848
9849 rc = bnxt_cfg_rx_mode(bp);
9850 if (rc)
9851 goto err_out;
9852
9853skip_rx_mask:
9854 rc = bnxt_hwrm_set_coal(bp);
9855 if (rc)
9856 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
9857 rc);
9858
9859 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9860 rc = bnxt_setup_nitroa0_vnic(bp);
9861 if (rc)
9862 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
9863 rc);
9864 }
9865
9866 if (BNXT_VF(bp)) {
9867 bnxt_hwrm_func_qcfg(bp);
9868 netdev_update_features(bp->dev);
9869 }
9870
9871 return 0;
9872
9873err_out:
9874 bnxt_hwrm_resource_free(bp, 0, true);
9875
9876 return rc;
9877}
9878
9879static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
9880{
9881 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
9882 return 0;
9883}
9884
9885static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
9886{
9887 bnxt_init_cp_rings(bp);
9888 bnxt_init_rx_rings(bp);
9889 bnxt_init_tx_rings(bp);
9890 bnxt_init_ring_grps(bp, irq_re_init);
9891 bnxt_init_vnics(bp);
9892
9893 return bnxt_init_chip(bp, irq_re_init);
9894}
9895
9896static int bnxt_set_real_num_queues(struct bnxt *bp)
9897{
9898 int rc;
9899 struct net_device *dev = bp->dev;
9900
9901 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
9902 bp->tx_nr_rings_xdp);
9903 if (rc)
9904 return rc;
9905
9906 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
9907 if (rc)
9908 return rc;
9909
9910#ifdef CONFIG_RFS_ACCEL
9911 if (bp->flags & BNXT_FLAG_RFS)
9912 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
9913#endif
9914
9915 return rc;
9916}
9917
9918static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
9919 bool shared)
9920{
9921 int _rx = *rx, _tx = *tx;
9922
9923 if (shared) {
9924 *rx = min_t(int, _rx, max);
9925 *tx = min_t(int, _tx, max);
9926 } else {
9927 if (max < 2)
9928 return -ENOMEM;
9929
9930 while (_rx + _tx > max) {
9931 if (_rx > _tx && _rx > 1)
9932 _rx--;
9933 else if (_tx > 1)
9934 _tx--;
9935 }
9936 *rx = _rx;
9937 *tx = _tx;
9938 }
9939 return 0;
9940}
9941
9942static int __bnxt_num_tx_to_cp(struct bnxt *bp, int tx, int tx_sets, int tx_xdp)
9943{
9944 return (tx - tx_xdp) / tx_sets + tx_xdp;
9945}
9946
9947int bnxt_num_tx_to_cp(struct bnxt *bp, int tx)
9948{
9949 int tcs = bp->num_tc;
9950
9951 if (!tcs)
9952 tcs = 1;
9953 return __bnxt_num_tx_to_cp(bp, tx, tcs, bp->tx_nr_rings_xdp);
9954}
9955
9956static int bnxt_num_cp_to_tx(struct bnxt *bp, int tx_cp)
9957{
9958 int tcs = bp->num_tc;
9959
9960 return (tx_cp - bp->tx_nr_rings_xdp) * tcs +
9961 bp->tx_nr_rings_xdp;
9962}
9963
9964static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
9965 bool sh)
9966{
9967 int tx_cp = bnxt_num_tx_to_cp(bp, *tx);
9968
9969 if (tx_cp != *tx) {
9970 int tx_saved = tx_cp, rc;
9971
9972 rc = __bnxt_trim_rings(bp, rx, &tx_cp, max, sh);
9973 if (rc)
9974 return rc;
9975 if (tx_cp != tx_saved)
9976 *tx = bnxt_num_cp_to_tx(bp, tx_cp);
9977 return 0;
9978 }
9979 return __bnxt_trim_rings(bp, rx, tx, max, sh);
9980}
9981
9982static void bnxt_setup_msix(struct bnxt *bp)
9983{
9984 const int len = sizeof(bp->irq_tbl[0].name);
9985 struct net_device *dev = bp->dev;
9986 int tcs, i;
9987
9988 tcs = bp->num_tc;
9989 if (tcs) {
9990 int i, off, count;
9991
9992 for (i = 0; i < tcs; i++) {
9993 count = bp->tx_nr_rings_per_tc;
9994 off = BNXT_TC_TO_RING_BASE(bp, i);
9995 netdev_set_tc_queue(dev, i, count, off);
9996 }
9997 }
9998
9999 for (i = 0; i < bp->cp_nr_rings; i++) {
10000 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10001 char *attr;
10002
10003 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
10004 attr = "TxRx";
10005 else if (i < bp->rx_nr_rings)
10006 attr = "rx";
10007 else
10008 attr = "tx";
10009
10010 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
10011 attr, i);
10012 bp->irq_tbl[map_idx].handler = bnxt_msix;
10013 }
10014}
10015
10016static void bnxt_setup_inta(struct bnxt *bp)
10017{
10018 const int len = sizeof(bp->irq_tbl[0].name);
10019
10020 if (bp->num_tc) {
10021 netdev_reset_tc(bp->dev);
10022 bp->num_tc = 0;
10023 }
10024
10025 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
10026 0);
10027 bp->irq_tbl[0].handler = bnxt_inta;
10028}
10029
10030static int bnxt_init_int_mode(struct bnxt *bp);
10031
10032static int bnxt_setup_int_mode(struct bnxt *bp)
10033{
10034 int rc;
10035
10036 if (!bp->irq_tbl) {
10037 rc = bnxt_init_int_mode(bp);
10038 if (rc || !bp->irq_tbl)
10039 return rc ?: -ENODEV;
10040 }
10041
10042 if (bp->flags & BNXT_FLAG_USING_MSIX)
10043 bnxt_setup_msix(bp);
10044 else
10045 bnxt_setup_inta(bp);
10046
10047 rc = bnxt_set_real_num_queues(bp);
10048 return rc;
10049}
10050
10051static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
10052{
10053 return bp->hw_resc.max_rsscos_ctxs;
10054}
10055
10056static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
10057{
10058 return bp->hw_resc.max_vnics;
10059}
10060
10061unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
10062{
10063 return bp->hw_resc.max_stat_ctxs;
10064}
10065
10066unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
10067{
10068 return bp->hw_resc.max_cp_rings;
10069}
10070
10071static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
10072{
10073 unsigned int cp = bp->hw_resc.max_cp_rings;
10074
10075 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
10076 cp -= bnxt_get_ulp_msix_num(bp);
10077
10078 return cp;
10079}
10080
10081static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
10082{
10083 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10084
10085 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10086 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
10087
10088 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
10089}
10090
10091static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
10092{
10093 bp->hw_resc.max_irqs = max_irqs;
10094}
10095
10096unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
10097{
10098 unsigned int cp;
10099
10100 cp = bnxt_get_max_func_cp_rings_for_en(bp);
10101 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10102 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
10103 else
10104 return cp - bp->cp_nr_rings;
10105}
10106
10107unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
10108{
10109 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
10110}
10111
10112int bnxt_get_avail_msix(struct bnxt *bp, int num)
10113{
10114 int max_cp = bnxt_get_max_func_cp_rings(bp);
10115 int max_irq = bnxt_get_max_func_irqs(bp);
10116 int total_req = bp->cp_nr_rings + num;
10117 int max_idx, avail_msix;
10118
10119 max_idx = bp->total_irqs;
10120 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
10121 max_idx = min_t(int, bp->total_irqs, max_cp);
10122 avail_msix = max_idx - bp->cp_nr_rings;
10123 if (!BNXT_NEW_RM(bp) || avail_msix >= num)
10124 return avail_msix;
10125
10126 if (max_irq < total_req) {
10127 num = max_irq - bp->cp_nr_rings;
10128 if (num <= 0)
10129 return 0;
10130 }
10131 return num;
10132}
10133
10134static int bnxt_get_num_msix(struct bnxt *bp)
10135{
10136 if (!BNXT_NEW_RM(bp))
10137 return bnxt_get_max_func_irqs(bp);
10138
10139 return bnxt_nq_rings_in_use(bp);
10140}
10141
10142static int bnxt_init_msix(struct bnxt *bp)
10143{
10144 int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp;
10145 struct msix_entry *msix_ent;
10146
10147 total_vecs = bnxt_get_num_msix(bp);
10148 max = bnxt_get_max_func_irqs(bp);
10149 if (total_vecs > max)
10150 total_vecs = max;
10151
10152 if (!total_vecs)
10153 return 0;
10154
10155 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
10156 if (!msix_ent)
10157 return -ENOMEM;
10158
10159 for (i = 0; i < total_vecs; i++) {
10160 msix_ent[i].entry = i;
10161 msix_ent[i].vector = 0;
10162 }
10163
10164 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
10165 min = 2;
10166
10167 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
10168 ulp_msix = bnxt_get_ulp_msix_num(bp);
10169 if (total_vecs < 0 || total_vecs < ulp_msix) {
10170 rc = -ENODEV;
10171 goto msix_setup_exit;
10172 }
10173
10174 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
10175 if (bp->irq_tbl) {
10176 for (i = 0; i < total_vecs; i++)
10177 bp->irq_tbl[i].vector = msix_ent[i].vector;
10178
10179 bp->total_irqs = total_vecs;
10180 /* Trim rings based upon num of vectors allocated */
10181 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
10182 total_vecs - ulp_msix, min == 1);
10183 if (rc)
10184 goto msix_setup_exit;
10185
10186 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
10187 bp->cp_nr_rings = (min == 1) ?
10188 max_t(int, tx_cp, bp->rx_nr_rings) :
10189 tx_cp + bp->rx_nr_rings;
10190
10191 } else {
10192 rc = -ENOMEM;
10193 goto msix_setup_exit;
10194 }
10195 bp->flags |= BNXT_FLAG_USING_MSIX;
10196 kfree(msix_ent);
10197 return 0;
10198
10199msix_setup_exit:
10200 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
10201 kfree(bp->irq_tbl);
10202 bp->irq_tbl = NULL;
10203 pci_disable_msix(bp->pdev);
10204 kfree(msix_ent);
10205 return rc;
10206}
10207
10208static int bnxt_init_inta(struct bnxt *bp)
10209{
10210 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
10211 if (!bp->irq_tbl)
10212 return -ENOMEM;
10213
10214 bp->total_irqs = 1;
10215 bp->rx_nr_rings = 1;
10216 bp->tx_nr_rings = 1;
10217 bp->cp_nr_rings = 1;
10218 bp->flags |= BNXT_FLAG_SHARED_RINGS;
10219 bp->irq_tbl[0].vector = bp->pdev->irq;
10220 return 0;
10221}
10222
10223static int bnxt_init_int_mode(struct bnxt *bp)
10224{
10225 int rc = -ENODEV;
10226
10227 if (bp->flags & BNXT_FLAG_MSIX_CAP)
10228 rc = bnxt_init_msix(bp);
10229
10230 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
10231 /* fallback to INTA */
10232 rc = bnxt_init_inta(bp);
10233 }
10234 return rc;
10235}
10236
10237static void bnxt_clear_int_mode(struct bnxt *bp)
10238{
10239 if (bp->flags & BNXT_FLAG_USING_MSIX)
10240 pci_disable_msix(bp->pdev);
10241
10242 kfree(bp->irq_tbl);
10243 bp->irq_tbl = NULL;
10244 bp->flags &= ~BNXT_FLAG_USING_MSIX;
10245}
10246
10247int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
10248{
10249 bool irq_cleared = false;
10250 int tcs = bp->num_tc;
10251 int rc;
10252
10253 if (!bnxt_need_reserve_rings(bp))
10254 return 0;
10255
10256 if (irq_re_init && BNXT_NEW_RM(bp) &&
10257 bnxt_get_num_msix(bp) != bp->total_irqs) {
10258 bnxt_ulp_irq_stop(bp);
10259 bnxt_clear_int_mode(bp);
10260 irq_cleared = true;
10261 }
10262 rc = __bnxt_reserve_rings(bp);
10263 if (irq_cleared) {
10264 if (!rc)
10265 rc = bnxt_init_int_mode(bp);
10266 bnxt_ulp_irq_restart(bp, rc);
10267 }
10268 if (rc) {
10269 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
10270 return rc;
10271 }
10272 if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
10273 bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
10274 netdev_err(bp->dev, "tx ring reservation failure\n");
10275 netdev_reset_tc(bp->dev);
10276 bp->num_tc = 0;
10277 if (bp->tx_nr_rings_xdp)
10278 bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
10279 else
10280 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10281 return -ENOMEM;
10282 }
10283 return 0;
10284}
10285
10286static void bnxt_free_irq(struct bnxt *bp)
10287{
10288 struct bnxt_irq *irq;
10289 int i;
10290
10291#ifdef CONFIG_RFS_ACCEL
10292 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
10293 bp->dev->rx_cpu_rmap = NULL;
10294#endif
10295 if (!bp->irq_tbl || !bp->bnapi)
10296 return;
10297
10298 for (i = 0; i < bp->cp_nr_rings; i++) {
10299 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10300
10301 irq = &bp->irq_tbl[map_idx];
10302 if (irq->requested) {
10303 if (irq->have_cpumask) {
10304 irq_set_affinity_hint(irq->vector, NULL);
10305 free_cpumask_var(irq->cpu_mask);
10306 irq->have_cpumask = 0;
10307 }
10308 free_irq(irq->vector, bp->bnapi[i]);
10309 }
10310
10311 irq->requested = 0;
10312 }
10313}
10314
10315static int bnxt_request_irq(struct bnxt *bp)
10316{
10317 int i, j, rc = 0;
10318 unsigned long flags = 0;
10319#ifdef CONFIG_RFS_ACCEL
10320 struct cpu_rmap *rmap;
10321#endif
10322
10323 rc = bnxt_setup_int_mode(bp);
10324 if (rc) {
10325 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
10326 rc);
10327 return rc;
10328 }
10329#ifdef CONFIG_RFS_ACCEL
10330 rmap = bp->dev->rx_cpu_rmap;
10331#endif
10332 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
10333 flags = IRQF_SHARED;
10334
10335 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
10336 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10337 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
10338
10339#ifdef CONFIG_RFS_ACCEL
10340 if (rmap && bp->bnapi[i]->rx_ring) {
10341 rc = irq_cpu_rmap_add(rmap, irq->vector);
10342 if (rc)
10343 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
10344 j);
10345 j++;
10346 }
10347#endif
10348 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
10349 bp->bnapi[i]);
10350 if (rc)
10351 break;
10352
10353 netif_napi_set_irq(&bp->bnapi[i]->napi, irq->vector);
10354 irq->requested = 1;
10355
10356 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
10357 int numa_node = dev_to_node(&bp->pdev->dev);
10358
10359 irq->have_cpumask = 1;
10360 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
10361 irq->cpu_mask);
10362 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
10363 if (rc) {
10364 netdev_warn(bp->dev,
10365 "Set affinity failed, IRQ = %d\n",
10366 irq->vector);
10367 break;
10368 }
10369 }
10370 }
10371 return rc;
10372}
10373
10374static void bnxt_del_napi(struct bnxt *bp)
10375{
10376 int i;
10377
10378 if (!bp->bnapi)
10379 return;
10380
10381 for (i = 0; i < bp->rx_nr_rings; i++)
10382 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_RX, NULL);
10383 for (i = 0; i < bp->tx_nr_rings - bp->tx_nr_rings_xdp; i++)
10384 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_TX, NULL);
10385
10386 for (i = 0; i < bp->cp_nr_rings; i++) {
10387 struct bnxt_napi *bnapi = bp->bnapi[i];
10388
10389 __netif_napi_del(&bnapi->napi);
10390 }
10391 /* We called __netif_napi_del(), we need
10392 * to respect an RCU grace period before freeing napi structures.
10393 */
10394 synchronize_net();
10395}
10396
10397static void bnxt_init_napi(struct bnxt *bp)
10398{
10399 int i;
10400 unsigned int cp_nr_rings = bp->cp_nr_rings;
10401 struct bnxt_napi *bnapi;
10402
10403 if (bp->flags & BNXT_FLAG_USING_MSIX) {
10404 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
10405
10406 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10407 poll_fn = bnxt_poll_p5;
10408 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10409 cp_nr_rings--;
10410 for (i = 0; i < cp_nr_rings; i++) {
10411 bnapi = bp->bnapi[i];
10412 netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
10413 }
10414 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10415 bnapi = bp->bnapi[cp_nr_rings];
10416 netif_napi_add(bp->dev, &bnapi->napi,
10417 bnxt_poll_nitroa0);
10418 }
10419 } else {
10420 bnapi = bp->bnapi[0];
10421 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll);
10422 }
10423}
10424
10425static void bnxt_disable_napi(struct bnxt *bp)
10426{
10427 int i;
10428
10429 if (!bp->bnapi ||
10430 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
10431 return;
10432
10433 for (i = 0; i < bp->cp_nr_rings; i++) {
10434 struct bnxt_napi *bnapi = bp->bnapi[i];
10435 struct bnxt_cp_ring_info *cpr;
10436
10437 cpr = &bnapi->cp_ring;
10438 if (bnapi->tx_fault)
10439 cpr->sw_stats.tx.tx_resets++;
10440 if (bnapi->in_reset)
10441 cpr->sw_stats.rx.rx_resets++;
10442 napi_disable(&bnapi->napi);
10443 if (bnapi->rx_ring)
10444 cancel_work_sync(&cpr->dim.work);
10445 }
10446}
10447
10448static void bnxt_enable_napi(struct bnxt *bp)
10449{
10450 int i;
10451
10452 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
10453 for (i = 0; i < bp->cp_nr_rings; i++) {
10454 struct bnxt_napi *bnapi = bp->bnapi[i];
10455 struct bnxt_cp_ring_info *cpr;
10456
10457 bnapi->tx_fault = 0;
10458
10459 cpr = &bnapi->cp_ring;
10460 bnapi->in_reset = false;
10461
10462 if (bnapi->rx_ring) {
10463 INIT_WORK(&cpr->dim.work, bnxt_dim_work);
10464 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
10465 }
10466 napi_enable(&bnapi->napi);
10467 }
10468}
10469
10470void bnxt_tx_disable(struct bnxt *bp)
10471{
10472 int i;
10473 struct bnxt_tx_ring_info *txr;
10474
10475 if (bp->tx_ring) {
10476 for (i = 0; i < bp->tx_nr_rings; i++) {
10477 txr = &bp->tx_ring[i];
10478 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
10479 }
10480 }
10481 /* Make sure napi polls see @dev_state change */
10482 synchronize_net();
10483 /* Drop carrier first to prevent TX timeout */
10484 netif_carrier_off(bp->dev);
10485 /* Stop all TX queues */
10486 netif_tx_disable(bp->dev);
10487}
10488
10489void bnxt_tx_enable(struct bnxt *bp)
10490{
10491 int i;
10492 struct bnxt_tx_ring_info *txr;
10493
10494 for (i = 0; i < bp->tx_nr_rings; i++) {
10495 txr = &bp->tx_ring[i];
10496 WRITE_ONCE(txr->dev_state, 0);
10497 }
10498 /* Make sure napi polls see @dev_state change */
10499 synchronize_net();
10500 netif_tx_wake_all_queues(bp->dev);
10501 if (BNXT_LINK_IS_UP(bp))
10502 netif_carrier_on(bp->dev);
10503}
10504
10505static char *bnxt_report_fec(struct bnxt_link_info *link_info)
10506{
10507 u8 active_fec = link_info->active_fec_sig_mode &
10508 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
10509
10510 switch (active_fec) {
10511 default:
10512 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
10513 return "None";
10514 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
10515 return "Clause 74 BaseR";
10516 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
10517 return "Clause 91 RS(528,514)";
10518 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
10519 return "Clause 91 RS544_1XN";
10520 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
10521 return "Clause 91 RS(544,514)";
10522 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
10523 return "Clause 91 RS272_1XN";
10524 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
10525 return "Clause 91 RS(272,257)";
10526 }
10527}
10528
10529void bnxt_report_link(struct bnxt *bp)
10530{
10531 if (BNXT_LINK_IS_UP(bp)) {
10532 const char *signal = "";
10533 const char *flow_ctrl;
10534 const char *duplex;
10535 u32 speed;
10536 u16 fec;
10537
10538 netif_carrier_on(bp->dev);
10539 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
10540 if (speed == SPEED_UNKNOWN) {
10541 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
10542 return;
10543 }
10544 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
10545 duplex = "full";
10546 else
10547 duplex = "half";
10548 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
10549 flow_ctrl = "ON - receive & transmit";
10550 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
10551 flow_ctrl = "ON - transmit";
10552 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
10553 flow_ctrl = "ON - receive";
10554 else
10555 flow_ctrl = "none";
10556 if (bp->link_info.phy_qcfg_resp.option_flags &
10557 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
10558 u8 sig_mode = bp->link_info.active_fec_sig_mode &
10559 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
10560 switch (sig_mode) {
10561 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
10562 signal = "(NRZ) ";
10563 break;
10564 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
10565 signal = "(PAM4 56Gbps) ";
10566 break;
10567 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112:
10568 signal = "(PAM4 112Gbps) ";
10569 break;
10570 default:
10571 break;
10572 }
10573 }
10574 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
10575 speed, signal, duplex, flow_ctrl);
10576 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
10577 netdev_info(bp->dev, "EEE is %s\n",
10578 bp->eee.eee_active ? "active" :
10579 "not active");
10580 fec = bp->link_info.fec_cfg;
10581 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
10582 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
10583 (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
10584 bnxt_report_fec(&bp->link_info));
10585 } else {
10586 netif_carrier_off(bp->dev);
10587 netdev_err(bp->dev, "NIC Link is Down\n");
10588 }
10589}
10590
10591static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
10592{
10593 if (!resp->supported_speeds_auto_mode &&
10594 !resp->supported_speeds_force_mode &&
10595 !resp->supported_pam4_speeds_auto_mode &&
10596 !resp->supported_pam4_speeds_force_mode &&
10597 !resp->supported_speeds2_auto_mode &&
10598 !resp->supported_speeds2_force_mode)
10599 return true;
10600 return false;
10601}
10602
10603static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
10604{
10605 struct bnxt_link_info *link_info = &bp->link_info;
10606 struct hwrm_port_phy_qcaps_output *resp;
10607 struct hwrm_port_phy_qcaps_input *req;
10608 int rc = 0;
10609
10610 if (bp->hwrm_spec_code < 0x10201)
10611 return 0;
10612
10613 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
10614 if (rc)
10615 return rc;
10616
10617 resp = hwrm_req_hold(bp, req);
10618 rc = hwrm_req_send(bp, req);
10619 if (rc)
10620 goto hwrm_phy_qcaps_exit;
10621
10622 bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
10623 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
10624 struct ethtool_eee *eee = &bp->eee;
10625 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
10626
10627 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
10628 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
10629 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
10630 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
10631 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
10632 }
10633
10634 if (bp->hwrm_spec_code >= 0x10a01) {
10635 if (bnxt_phy_qcaps_no_speed(resp)) {
10636 link_info->phy_state = BNXT_PHY_STATE_DISABLED;
10637 netdev_warn(bp->dev, "Ethernet link disabled\n");
10638 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
10639 link_info->phy_state = BNXT_PHY_STATE_ENABLED;
10640 netdev_info(bp->dev, "Ethernet link enabled\n");
10641 /* Phy re-enabled, reprobe the speeds */
10642 link_info->support_auto_speeds = 0;
10643 link_info->support_pam4_auto_speeds = 0;
10644 link_info->support_auto_speeds2 = 0;
10645 }
10646 }
10647 if (resp->supported_speeds_auto_mode)
10648 link_info->support_auto_speeds =
10649 le16_to_cpu(resp->supported_speeds_auto_mode);
10650 if (resp->supported_pam4_speeds_auto_mode)
10651 link_info->support_pam4_auto_speeds =
10652 le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
10653 if (resp->supported_speeds2_auto_mode)
10654 link_info->support_auto_speeds2 =
10655 le16_to_cpu(resp->supported_speeds2_auto_mode);
10656
10657 bp->port_count = resp->port_cnt;
10658
10659hwrm_phy_qcaps_exit:
10660 hwrm_req_drop(bp, req);
10661 return rc;
10662}
10663
10664static bool bnxt_support_dropped(u16 advertising, u16 supported)
10665{
10666 u16 diff = advertising ^ supported;
10667
10668 return ((supported | diff) != supported);
10669}
10670
10671static bool bnxt_support_speed_dropped(struct bnxt_link_info *link_info)
10672{
10673 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
10674
10675 /* Check if any advertised speeds are no longer supported. The caller
10676 * holds the link_lock mutex, so we can modify link_info settings.
10677 */
10678 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
10679 if (bnxt_support_dropped(link_info->advertising,
10680 link_info->support_auto_speeds2)) {
10681 link_info->advertising = link_info->support_auto_speeds2;
10682 return true;
10683 }
10684 return false;
10685 }
10686 if (bnxt_support_dropped(link_info->advertising,
10687 link_info->support_auto_speeds)) {
10688 link_info->advertising = link_info->support_auto_speeds;
10689 return true;
10690 }
10691 if (bnxt_support_dropped(link_info->advertising_pam4,
10692 link_info->support_pam4_auto_speeds)) {
10693 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
10694 return true;
10695 }
10696 return false;
10697}
10698
10699int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
10700{
10701 struct bnxt_link_info *link_info = &bp->link_info;
10702 struct hwrm_port_phy_qcfg_output *resp;
10703 struct hwrm_port_phy_qcfg_input *req;
10704 u8 link_state = link_info->link_state;
10705 bool support_changed;
10706 int rc;
10707
10708 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
10709 if (rc)
10710 return rc;
10711
10712 resp = hwrm_req_hold(bp, req);
10713 rc = hwrm_req_send(bp, req);
10714 if (rc) {
10715 hwrm_req_drop(bp, req);
10716 if (BNXT_VF(bp) && rc == -ENODEV) {
10717 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
10718 rc = 0;
10719 }
10720 return rc;
10721 }
10722
10723 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
10724 link_info->phy_link_status = resp->link;
10725 link_info->duplex = resp->duplex_cfg;
10726 if (bp->hwrm_spec_code >= 0x10800)
10727 link_info->duplex = resp->duplex_state;
10728 link_info->pause = resp->pause;
10729 link_info->auto_mode = resp->auto_mode;
10730 link_info->auto_pause_setting = resp->auto_pause;
10731 link_info->lp_pause = resp->link_partner_adv_pause;
10732 link_info->force_pause_setting = resp->force_pause;
10733 link_info->duplex_setting = resp->duplex_cfg;
10734 if (link_info->phy_link_status == BNXT_LINK_LINK) {
10735 link_info->link_speed = le16_to_cpu(resp->link_speed);
10736 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
10737 link_info->active_lanes = resp->active_lanes;
10738 } else {
10739 link_info->link_speed = 0;
10740 link_info->active_lanes = 0;
10741 }
10742 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
10743 link_info->force_pam4_link_speed =
10744 le16_to_cpu(resp->force_pam4_link_speed);
10745 link_info->force_link_speed2 = le16_to_cpu(resp->force_link_speeds2);
10746 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
10747 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
10748 link_info->support_speeds2 = le16_to_cpu(resp->support_speeds2);
10749 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
10750 link_info->auto_pam4_link_speeds =
10751 le16_to_cpu(resp->auto_pam4_link_speed_mask);
10752 link_info->auto_link_speeds2 = le16_to_cpu(resp->auto_link_speeds2);
10753 link_info->lp_auto_link_speeds =
10754 le16_to_cpu(resp->link_partner_adv_speeds);
10755 link_info->lp_auto_pam4_link_speeds =
10756 resp->link_partner_pam4_adv_speeds;
10757 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
10758 link_info->phy_ver[0] = resp->phy_maj;
10759 link_info->phy_ver[1] = resp->phy_min;
10760 link_info->phy_ver[2] = resp->phy_bld;
10761 link_info->media_type = resp->media_type;
10762 link_info->phy_type = resp->phy_type;
10763 link_info->transceiver = resp->xcvr_pkg_type;
10764 link_info->phy_addr = resp->eee_config_phy_addr &
10765 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
10766 link_info->module_status = resp->module_status;
10767
10768 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
10769 struct ethtool_eee *eee = &bp->eee;
10770 u16 fw_speeds;
10771
10772 eee->eee_active = 0;
10773 if (resp->eee_config_phy_addr &
10774 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
10775 eee->eee_active = 1;
10776 fw_speeds = le16_to_cpu(
10777 resp->link_partner_adv_eee_link_speed_mask);
10778 eee->lp_advertised =
10779 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
10780 }
10781
10782 /* Pull initial EEE config */
10783 if (!chng_link_state) {
10784 if (resp->eee_config_phy_addr &
10785 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
10786 eee->eee_enabled = 1;
10787
10788 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
10789 eee->advertised =
10790 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
10791
10792 if (resp->eee_config_phy_addr &
10793 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
10794 __le32 tmr;
10795
10796 eee->tx_lpi_enabled = 1;
10797 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
10798 eee->tx_lpi_timer = le32_to_cpu(tmr) &
10799 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
10800 }
10801 }
10802 }
10803
10804 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
10805 if (bp->hwrm_spec_code >= 0x10504) {
10806 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
10807 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
10808 }
10809 /* TODO: need to add more logic to report VF link */
10810 if (chng_link_state) {
10811 if (link_info->phy_link_status == BNXT_LINK_LINK)
10812 link_info->link_state = BNXT_LINK_STATE_UP;
10813 else
10814 link_info->link_state = BNXT_LINK_STATE_DOWN;
10815 if (link_state != link_info->link_state)
10816 bnxt_report_link(bp);
10817 } else {
10818 /* always link down if not require to update link state */
10819 link_info->link_state = BNXT_LINK_STATE_DOWN;
10820 }
10821 hwrm_req_drop(bp, req);
10822
10823 if (!BNXT_PHY_CFG_ABLE(bp))
10824 return 0;
10825
10826 support_changed = bnxt_support_speed_dropped(link_info);
10827 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
10828 bnxt_hwrm_set_link_setting(bp, true, false);
10829 return 0;
10830}
10831
10832static void bnxt_get_port_module_status(struct bnxt *bp)
10833{
10834 struct bnxt_link_info *link_info = &bp->link_info;
10835 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
10836 u8 module_status;
10837
10838 if (bnxt_update_link(bp, true))
10839 return;
10840
10841 module_status = link_info->module_status;
10842 switch (module_status) {
10843 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
10844 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
10845 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
10846 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
10847 bp->pf.port_id);
10848 if (bp->hwrm_spec_code >= 0x10201) {
10849 netdev_warn(bp->dev, "Module part number %s\n",
10850 resp->phy_vendor_partnumber);
10851 }
10852 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
10853 netdev_warn(bp->dev, "TX is disabled\n");
10854 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
10855 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
10856 }
10857}
10858
10859static void
10860bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
10861{
10862 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
10863 if (bp->hwrm_spec_code >= 0x10201)
10864 req->auto_pause =
10865 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
10866 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
10867 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
10868 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
10869 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
10870 req->enables |=
10871 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
10872 } else {
10873 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
10874 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
10875 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
10876 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
10877 req->enables |=
10878 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
10879 if (bp->hwrm_spec_code >= 0x10201) {
10880 req->auto_pause = req->force_pause;
10881 req->enables |= cpu_to_le32(
10882 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
10883 }
10884 }
10885}
10886
10887static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
10888{
10889 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
10890 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
10891 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
10892 req->enables |=
10893 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK);
10894 req->auto_link_speeds2_mask = cpu_to_le16(bp->link_info.advertising);
10895 } else if (bp->link_info.advertising) {
10896 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
10897 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
10898 }
10899 if (bp->link_info.advertising_pam4) {
10900 req->enables |=
10901 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
10902 req->auto_link_pam4_speed_mask =
10903 cpu_to_le16(bp->link_info.advertising_pam4);
10904 }
10905 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
10906 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
10907 } else {
10908 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
10909 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
10910 req->force_link_speeds2 = cpu_to_le16(bp->link_info.req_link_speed);
10911 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2);
10912 netif_info(bp, link, bp->dev, "Forcing FW speed2: %d\n",
10913 (u32)bp->link_info.req_link_speed);
10914 } else if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
10915 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
10916 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
10917 } else {
10918 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
10919 }
10920 }
10921
10922 /* tell chimp that the setting takes effect immediately */
10923 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
10924}
10925
10926int bnxt_hwrm_set_pause(struct bnxt *bp)
10927{
10928 struct hwrm_port_phy_cfg_input *req;
10929 int rc;
10930
10931 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
10932 if (rc)
10933 return rc;
10934
10935 bnxt_hwrm_set_pause_common(bp, req);
10936
10937 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
10938 bp->link_info.force_link_chng)
10939 bnxt_hwrm_set_link_common(bp, req);
10940
10941 rc = hwrm_req_send(bp, req);
10942 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
10943 /* since changing of pause setting doesn't trigger any link
10944 * change event, the driver needs to update the current pause
10945 * result upon successfully return of the phy_cfg command
10946 */
10947 bp->link_info.pause =
10948 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
10949 bp->link_info.auto_pause_setting = 0;
10950 if (!bp->link_info.force_link_chng)
10951 bnxt_report_link(bp);
10952 }
10953 bp->link_info.force_link_chng = false;
10954 return rc;
10955}
10956
10957static void bnxt_hwrm_set_eee(struct bnxt *bp,
10958 struct hwrm_port_phy_cfg_input *req)
10959{
10960 struct ethtool_eee *eee = &bp->eee;
10961
10962 if (eee->eee_enabled) {
10963 u16 eee_speeds;
10964 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
10965
10966 if (eee->tx_lpi_enabled)
10967 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
10968 else
10969 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
10970
10971 req->flags |= cpu_to_le32(flags);
10972 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
10973 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
10974 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
10975 } else {
10976 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
10977 }
10978}
10979
10980int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
10981{
10982 struct hwrm_port_phy_cfg_input *req;
10983 int rc;
10984
10985 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
10986 if (rc)
10987 return rc;
10988
10989 if (set_pause)
10990 bnxt_hwrm_set_pause_common(bp, req);
10991
10992 bnxt_hwrm_set_link_common(bp, req);
10993
10994 if (set_eee)
10995 bnxt_hwrm_set_eee(bp, req);
10996 return hwrm_req_send(bp, req);
10997}
10998
10999static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
11000{
11001 struct hwrm_port_phy_cfg_input *req;
11002 int rc;
11003
11004 if (!BNXT_SINGLE_PF(bp))
11005 return 0;
11006
11007 if (pci_num_vf(bp->pdev) &&
11008 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
11009 return 0;
11010
11011 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11012 if (rc)
11013 return rc;
11014
11015 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
11016 rc = hwrm_req_send(bp, req);
11017 if (!rc) {
11018 mutex_lock(&bp->link_lock);
11019 /* Device is not obliged link down in certain scenarios, even
11020 * when forced. Setting the state unknown is consistent with
11021 * driver startup and will force link state to be reported
11022 * during subsequent open based on PORT_PHY_QCFG.
11023 */
11024 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
11025 mutex_unlock(&bp->link_lock);
11026 }
11027 return rc;
11028}
11029
11030static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11031{
11032#ifdef CONFIG_TEE_BNXT_FW
11033 int rc = tee_bnxt_fw_load();
11034
11035 if (rc)
11036 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11037
11038 return rc;
11039#else
11040 netdev_err(bp->dev, "OP-TEE not supported\n");
11041 return -ENODEV;
11042#endif
11043}
11044
11045static int bnxt_try_recover_fw(struct bnxt *bp)
11046{
11047 if (bp->fw_health && bp->fw_health->status_reliable) {
11048 int retry = 0, rc;
11049 u32 sts;
11050
11051 do {
11052 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11053 rc = bnxt_hwrm_poll(bp);
11054 if (!BNXT_FW_IS_BOOTING(sts) &&
11055 !BNXT_FW_IS_RECOVERING(sts))
11056 break;
11057 retry++;
11058 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
11059
11060 if (!BNXT_FW_IS_HEALTHY(sts)) {
11061 netdev_err(bp->dev,
11062 "Firmware not responding, status: 0x%x\n",
11063 sts);
11064 rc = -ENODEV;
11065 }
11066 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11067 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11068 return bnxt_fw_reset_via_optee(bp);
11069 }
11070 return rc;
11071 }
11072
11073 return -ENODEV;
11074}
11075
11076static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
11077{
11078 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11079
11080 if (!BNXT_NEW_RM(bp))
11081 return; /* no resource reservations required */
11082
11083 hw_resc->resv_cp_rings = 0;
11084 hw_resc->resv_stat_ctxs = 0;
11085 hw_resc->resv_irqs = 0;
11086 hw_resc->resv_tx_rings = 0;
11087 hw_resc->resv_rx_rings = 0;
11088 hw_resc->resv_hw_ring_grps = 0;
11089 hw_resc->resv_vnics = 0;
11090 if (!fw_reset) {
11091 bp->tx_nr_rings = 0;
11092 bp->rx_nr_rings = 0;
11093 }
11094}
11095
11096int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
11097{
11098 int rc;
11099
11100 if (!BNXT_NEW_RM(bp))
11101 return 0; /* no resource reservations required */
11102
11103 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
11104 if (rc)
11105 netdev_err(bp->dev, "resc_qcaps failed\n");
11106
11107 bnxt_clear_reservations(bp, fw_reset);
11108
11109 return rc;
11110}
11111
11112static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
11113{
11114 struct hwrm_func_drv_if_change_output *resp;
11115 struct hwrm_func_drv_if_change_input *req;
11116 bool fw_reset = !bp->irq_tbl;
11117 bool resc_reinit = false;
11118 int rc, retry = 0;
11119 u32 flags = 0;
11120
11121 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
11122 return 0;
11123
11124 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
11125 if (rc)
11126 return rc;
11127
11128 if (up)
11129 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
11130 resp = hwrm_req_hold(bp, req);
11131
11132 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
11133 while (retry < BNXT_FW_IF_RETRY) {
11134 rc = hwrm_req_send(bp, req);
11135 if (rc != -EAGAIN)
11136 break;
11137
11138 msleep(50);
11139 retry++;
11140 }
11141
11142 if (rc == -EAGAIN) {
11143 hwrm_req_drop(bp, req);
11144 return rc;
11145 } else if (!rc) {
11146 flags = le32_to_cpu(resp->flags);
11147 } else if (up) {
11148 rc = bnxt_try_recover_fw(bp);
11149 fw_reset = true;
11150 }
11151 hwrm_req_drop(bp, req);
11152 if (rc)
11153 return rc;
11154
11155 if (!up) {
11156 bnxt_inv_fw_health_reg(bp);
11157 return 0;
11158 }
11159
11160 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
11161 resc_reinit = true;
11162 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
11163 test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
11164 fw_reset = true;
11165 else
11166 bnxt_remap_fw_health_regs(bp);
11167
11168 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
11169 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
11170 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11171 return -ENODEV;
11172 }
11173 if (resc_reinit || fw_reset) {
11174 if (fw_reset) {
11175 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11176 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11177 bnxt_ulp_stop(bp);
11178 bnxt_free_ctx_mem(bp);
11179 bnxt_dcb_free(bp);
11180 rc = bnxt_fw_init_one(bp);
11181 if (rc) {
11182 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11183 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11184 return rc;
11185 }
11186 bnxt_clear_int_mode(bp);
11187 rc = bnxt_init_int_mode(bp);
11188 if (rc) {
11189 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11190 netdev_err(bp->dev, "init int mode failed\n");
11191 return rc;
11192 }
11193 }
11194 rc = bnxt_cancel_reservations(bp, fw_reset);
11195 }
11196 return rc;
11197}
11198
11199static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
11200{
11201 struct hwrm_port_led_qcaps_output *resp;
11202 struct hwrm_port_led_qcaps_input *req;
11203 struct bnxt_pf_info *pf = &bp->pf;
11204 int rc;
11205
11206 bp->num_leds = 0;
11207 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
11208 return 0;
11209
11210 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
11211 if (rc)
11212 return rc;
11213
11214 req->port_id = cpu_to_le16(pf->port_id);
11215 resp = hwrm_req_hold(bp, req);
11216 rc = hwrm_req_send(bp, req);
11217 if (rc) {
11218 hwrm_req_drop(bp, req);
11219 return rc;
11220 }
11221 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
11222 int i;
11223
11224 bp->num_leds = resp->num_leds;
11225 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
11226 bp->num_leds);
11227 for (i = 0; i < bp->num_leds; i++) {
11228 struct bnxt_led_info *led = &bp->leds[i];
11229 __le16 caps = led->led_state_caps;
11230
11231 if (!led->led_group_id ||
11232 !BNXT_LED_ALT_BLINK_CAP(caps)) {
11233 bp->num_leds = 0;
11234 break;
11235 }
11236 }
11237 }
11238 hwrm_req_drop(bp, req);
11239 return 0;
11240}
11241
11242int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
11243{
11244 struct hwrm_wol_filter_alloc_output *resp;
11245 struct hwrm_wol_filter_alloc_input *req;
11246 int rc;
11247
11248 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
11249 if (rc)
11250 return rc;
11251
11252 req->port_id = cpu_to_le16(bp->pf.port_id);
11253 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
11254 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
11255 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
11256
11257 resp = hwrm_req_hold(bp, req);
11258 rc = hwrm_req_send(bp, req);
11259 if (!rc)
11260 bp->wol_filter_id = resp->wol_filter_id;
11261 hwrm_req_drop(bp, req);
11262 return rc;
11263}
11264
11265int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
11266{
11267 struct hwrm_wol_filter_free_input *req;
11268 int rc;
11269
11270 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
11271 if (rc)
11272 return rc;
11273
11274 req->port_id = cpu_to_le16(bp->pf.port_id);
11275 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
11276 req->wol_filter_id = bp->wol_filter_id;
11277
11278 return hwrm_req_send(bp, req);
11279}
11280
11281static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
11282{
11283 struct hwrm_wol_filter_qcfg_output *resp;
11284 struct hwrm_wol_filter_qcfg_input *req;
11285 u16 next_handle = 0;
11286 int rc;
11287
11288 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
11289 if (rc)
11290 return rc;
11291
11292 req->port_id = cpu_to_le16(bp->pf.port_id);
11293 req->handle = cpu_to_le16(handle);
11294 resp = hwrm_req_hold(bp, req);
11295 rc = hwrm_req_send(bp, req);
11296 if (!rc) {
11297 next_handle = le16_to_cpu(resp->next_handle);
11298 if (next_handle != 0) {
11299 if (resp->wol_type ==
11300 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
11301 bp->wol = 1;
11302 bp->wol_filter_id = resp->wol_filter_id;
11303 }
11304 }
11305 }
11306 hwrm_req_drop(bp, req);
11307 return next_handle;
11308}
11309
11310static void bnxt_get_wol_settings(struct bnxt *bp)
11311{
11312 u16 handle = 0;
11313
11314 bp->wol = 0;
11315 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
11316 return;
11317
11318 do {
11319 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
11320 } while (handle && handle != 0xffff);
11321}
11322
11323static bool bnxt_eee_config_ok(struct bnxt *bp)
11324{
11325 struct ethtool_eee *eee = &bp->eee;
11326 struct bnxt_link_info *link_info = &bp->link_info;
11327
11328 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
11329 return true;
11330
11331 if (eee->eee_enabled) {
11332 u32 advertising =
11333 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
11334
11335 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
11336 eee->eee_enabled = 0;
11337 return false;
11338 }
11339 if (eee->advertised & ~advertising) {
11340 eee->advertised = advertising & eee->supported;
11341 return false;
11342 }
11343 }
11344 return true;
11345}
11346
11347static int bnxt_update_phy_setting(struct bnxt *bp)
11348{
11349 int rc;
11350 bool update_link = false;
11351 bool update_pause = false;
11352 bool update_eee = false;
11353 struct bnxt_link_info *link_info = &bp->link_info;
11354
11355 rc = bnxt_update_link(bp, true);
11356 if (rc) {
11357 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
11358 rc);
11359 return rc;
11360 }
11361 if (!BNXT_SINGLE_PF(bp))
11362 return 0;
11363
11364 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
11365 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
11366 link_info->req_flow_ctrl)
11367 update_pause = true;
11368 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
11369 link_info->force_pause_setting != link_info->req_flow_ctrl)
11370 update_pause = true;
11371 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
11372 if (BNXT_AUTO_MODE(link_info->auto_mode))
11373 update_link = true;
11374 if (bnxt_force_speed_updated(link_info))
11375 update_link = true;
11376 if (link_info->req_duplex != link_info->duplex_setting)
11377 update_link = true;
11378 } else {
11379 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
11380 update_link = true;
11381 if (bnxt_auto_speed_updated(link_info))
11382 update_link = true;
11383 }
11384
11385 /* The last close may have shutdown the link, so need to call
11386 * PHY_CFG to bring it back up.
11387 */
11388 if (!BNXT_LINK_IS_UP(bp))
11389 update_link = true;
11390
11391 if (!bnxt_eee_config_ok(bp))
11392 update_eee = true;
11393
11394 if (update_link)
11395 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
11396 else if (update_pause)
11397 rc = bnxt_hwrm_set_pause(bp);
11398 if (rc) {
11399 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
11400 rc);
11401 return rc;
11402 }
11403
11404 return rc;
11405}
11406
11407/* Common routine to pre-map certain register block to different GRC window.
11408 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
11409 * in PF and 3 windows in VF that can be customized to map in different
11410 * register blocks.
11411 */
11412static void bnxt_preset_reg_win(struct bnxt *bp)
11413{
11414 if (BNXT_PF(bp)) {
11415 /* CAG registers map to GRC window #4 */
11416 writel(BNXT_CAG_REG_BASE,
11417 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
11418 }
11419}
11420
11421static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
11422
11423static int bnxt_reinit_after_abort(struct bnxt *bp)
11424{
11425 int rc;
11426
11427 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11428 return -EBUSY;
11429
11430 if (bp->dev->reg_state == NETREG_UNREGISTERED)
11431 return -ENODEV;
11432
11433 rc = bnxt_fw_init_one(bp);
11434 if (!rc) {
11435 bnxt_clear_int_mode(bp);
11436 rc = bnxt_init_int_mode(bp);
11437 if (!rc) {
11438 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11439 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11440 }
11441 }
11442 return rc;
11443}
11444
11445static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
11446{
11447 int rc = 0;
11448
11449 bnxt_preset_reg_win(bp);
11450 netif_carrier_off(bp->dev);
11451 if (irq_re_init) {
11452 /* Reserve rings now if none were reserved at driver probe. */
11453 rc = bnxt_init_dflt_ring_mode(bp);
11454 if (rc) {
11455 netdev_err(bp->dev, "Failed to reserve default rings at open\n");
11456 return rc;
11457 }
11458 }
11459 rc = bnxt_reserve_rings(bp, irq_re_init);
11460 if (rc)
11461 return rc;
11462 if ((bp->flags & BNXT_FLAG_RFS) &&
11463 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
11464 /* disable RFS if falling back to INTA */
11465 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
11466 bp->flags &= ~BNXT_FLAG_RFS;
11467 }
11468
11469 rc = bnxt_alloc_mem(bp, irq_re_init);
11470 if (rc) {
11471 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
11472 goto open_err_free_mem;
11473 }
11474
11475 if (irq_re_init) {
11476 bnxt_init_napi(bp);
11477 rc = bnxt_request_irq(bp);
11478 if (rc) {
11479 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
11480 goto open_err_irq;
11481 }
11482 }
11483
11484 rc = bnxt_init_nic(bp, irq_re_init);
11485 if (rc) {
11486 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
11487 goto open_err_irq;
11488 }
11489
11490 bnxt_enable_napi(bp);
11491 bnxt_debug_dev_init(bp);
11492
11493 if (link_re_init) {
11494 mutex_lock(&bp->link_lock);
11495 rc = bnxt_update_phy_setting(bp);
11496 mutex_unlock(&bp->link_lock);
11497 if (rc) {
11498 netdev_warn(bp->dev, "failed to update phy settings\n");
11499 if (BNXT_SINGLE_PF(bp)) {
11500 bp->link_info.phy_retry = true;
11501 bp->link_info.phy_retry_expires =
11502 jiffies + 5 * HZ;
11503 }
11504 }
11505 }
11506
11507 if (irq_re_init)
11508 udp_tunnel_nic_reset_ntf(bp->dev);
11509
11510 if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
11511 if (!static_key_enabled(&bnxt_xdp_locking_key))
11512 static_branch_enable(&bnxt_xdp_locking_key);
11513 } else if (static_key_enabled(&bnxt_xdp_locking_key)) {
11514 static_branch_disable(&bnxt_xdp_locking_key);
11515 }
11516 set_bit(BNXT_STATE_OPEN, &bp->state);
11517 bnxt_enable_int(bp);
11518 /* Enable TX queues */
11519 bnxt_tx_enable(bp);
11520 mod_timer(&bp->timer, jiffies + bp->current_interval);
11521 /* Poll link status and check for SFP+ module status */
11522 mutex_lock(&bp->link_lock);
11523 bnxt_get_port_module_status(bp);
11524 mutex_unlock(&bp->link_lock);
11525
11526 /* VF-reps may need to be re-opened after the PF is re-opened */
11527 if (BNXT_PF(bp))
11528 bnxt_vf_reps_open(bp);
11529 bnxt_ptp_init_rtc(bp, true);
11530 bnxt_ptp_cfg_tstamp_filters(bp);
11531 return 0;
11532
11533open_err_irq:
11534 bnxt_del_napi(bp);
11535
11536open_err_free_mem:
11537 bnxt_free_skbs(bp);
11538 bnxt_free_irq(bp);
11539 bnxt_free_mem(bp, true);
11540 return rc;
11541}
11542
11543/* rtnl_lock held */
11544int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
11545{
11546 int rc = 0;
11547
11548 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
11549 rc = -EIO;
11550 if (!rc)
11551 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
11552 if (rc) {
11553 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
11554 dev_close(bp->dev);
11555 }
11556 return rc;
11557}
11558
11559/* rtnl_lock held, open the NIC half way by allocating all resources, but
11560 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline
11561 * self tests.
11562 */
11563int bnxt_half_open_nic(struct bnxt *bp)
11564{
11565 int rc = 0;
11566
11567 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
11568 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
11569 rc = -ENODEV;
11570 goto half_open_err;
11571 }
11572
11573 rc = bnxt_alloc_mem(bp, true);
11574 if (rc) {
11575 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
11576 goto half_open_err;
11577 }
11578 bnxt_init_napi(bp);
11579 set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
11580 rc = bnxt_init_nic(bp, true);
11581 if (rc) {
11582 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
11583 bnxt_del_napi(bp);
11584 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
11585 goto half_open_err;
11586 }
11587 return 0;
11588
11589half_open_err:
11590 bnxt_free_skbs(bp);
11591 bnxt_free_mem(bp, true);
11592 dev_close(bp->dev);
11593 return rc;
11594}
11595
11596/* rtnl_lock held, this call can only be made after a previous successful
11597 * call to bnxt_half_open_nic().
11598 */
11599void bnxt_half_close_nic(struct bnxt *bp)
11600{
11601 bnxt_hwrm_resource_free(bp, false, true);
11602 bnxt_del_napi(bp);
11603 bnxt_free_skbs(bp);
11604 bnxt_free_mem(bp, true);
11605 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
11606}
11607
11608void bnxt_reenable_sriov(struct bnxt *bp)
11609{
11610 if (BNXT_PF(bp)) {
11611 struct bnxt_pf_info *pf = &bp->pf;
11612 int n = pf->active_vfs;
11613
11614 if (n)
11615 bnxt_cfg_hw_sriov(bp, &n, true);
11616 }
11617}
11618
11619static int bnxt_open(struct net_device *dev)
11620{
11621 struct bnxt *bp = netdev_priv(dev);
11622 int rc;
11623
11624 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
11625 rc = bnxt_reinit_after_abort(bp);
11626 if (rc) {
11627 if (rc == -EBUSY)
11628 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
11629 else
11630 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
11631 return -ENODEV;
11632 }
11633 }
11634
11635 rc = bnxt_hwrm_if_change(bp, true);
11636 if (rc)
11637 return rc;
11638
11639 rc = __bnxt_open_nic(bp, true, true);
11640 if (rc) {
11641 bnxt_hwrm_if_change(bp, false);
11642 } else {
11643 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
11644 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11645 bnxt_ulp_start(bp, 0);
11646 bnxt_reenable_sriov(bp);
11647 }
11648 }
11649 }
11650
11651 return rc;
11652}
11653
11654static bool bnxt_drv_busy(struct bnxt *bp)
11655{
11656 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
11657 test_bit(BNXT_STATE_READ_STATS, &bp->state));
11658}
11659
11660static void bnxt_get_ring_stats(struct bnxt *bp,
11661 struct rtnl_link_stats64 *stats);
11662
11663static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
11664 bool link_re_init)
11665{
11666 /* Close the VF-reps before closing PF */
11667 if (BNXT_PF(bp))
11668 bnxt_vf_reps_close(bp);
11669
11670 /* Change device state to avoid TX queue wake up's */
11671 bnxt_tx_disable(bp);
11672
11673 clear_bit(BNXT_STATE_OPEN, &bp->state);
11674 smp_mb__after_atomic();
11675 while (bnxt_drv_busy(bp))
11676 msleep(20);
11677
11678 /* Flush rings and disable interrupts */
11679 bnxt_shutdown_nic(bp, irq_re_init);
11680
11681 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
11682
11683 bnxt_debug_dev_exit(bp);
11684 bnxt_disable_napi(bp);
11685 del_timer_sync(&bp->timer);
11686 bnxt_free_skbs(bp);
11687
11688 /* Save ring stats before shutdown */
11689 if (bp->bnapi && irq_re_init) {
11690 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
11691 bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev);
11692 }
11693 if (irq_re_init) {
11694 bnxt_free_irq(bp);
11695 bnxt_del_napi(bp);
11696 }
11697 bnxt_free_mem(bp, irq_re_init);
11698}
11699
11700void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
11701{
11702 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11703 /* If we get here, it means firmware reset is in progress
11704 * while we are trying to close. We can safely proceed with
11705 * the close because we are holding rtnl_lock(). Some firmware
11706 * messages may fail as we proceed to close. We set the
11707 * ABORT_ERR flag here so that the FW reset thread will later
11708 * abort when it gets the rtnl_lock() and sees the flag.
11709 */
11710 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
11711 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11712 }
11713
11714#ifdef CONFIG_BNXT_SRIOV
11715 if (bp->sriov_cfg) {
11716 int rc;
11717
11718 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
11719 !bp->sriov_cfg,
11720 BNXT_SRIOV_CFG_WAIT_TMO);
11721 if (!rc)
11722 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n");
11723 else if (rc < 0)
11724 netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n");
11725 }
11726#endif
11727 __bnxt_close_nic(bp, irq_re_init, link_re_init);
11728}
11729
11730static int bnxt_close(struct net_device *dev)
11731{
11732 struct bnxt *bp = netdev_priv(dev);
11733
11734 bnxt_close_nic(bp, true, true);
11735 bnxt_hwrm_shutdown_link(bp);
11736 bnxt_hwrm_if_change(bp, false);
11737 return 0;
11738}
11739
11740static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
11741 u16 *val)
11742{
11743 struct hwrm_port_phy_mdio_read_output *resp;
11744 struct hwrm_port_phy_mdio_read_input *req;
11745 int rc;
11746
11747 if (bp->hwrm_spec_code < 0x10a00)
11748 return -EOPNOTSUPP;
11749
11750 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
11751 if (rc)
11752 return rc;
11753
11754 req->port_id = cpu_to_le16(bp->pf.port_id);
11755 req->phy_addr = phy_addr;
11756 req->reg_addr = cpu_to_le16(reg & 0x1f);
11757 if (mdio_phy_id_is_c45(phy_addr)) {
11758 req->cl45_mdio = 1;
11759 req->phy_addr = mdio_phy_id_prtad(phy_addr);
11760 req->dev_addr = mdio_phy_id_devad(phy_addr);
11761 req->reg_addr = cpu_to_le16(reg);
11762 }
11763
11764 resp = hwrm_req_hold(bp, req);
11765 rc = hwrm_req_send(bp, req);
11766 if (!rc)
11767 *val = le16_to_cpu(resp->reg_data);
11768 hwrm_req_drop(bp, req);
11769 return rc;
11770}
11771
11772static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
11773 u16 val)
11774{
11775 struct hwrm_port_phy_mdio_write_input *req;
11776 int rc;
11777
11778 if (bp->hwrm_spec_code < 0x10a00)
11779 return -EOPNOTSUPP;
11780
11781 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
11782 if (rc)
11783 return rc;
11784
11785 req->port_id = cpu_to_le16(bp->pf.port_id);
11786 req->phy_addr = phy_addr;
11787 req->reg_addr = cpu_to_le16(reg & 0x1f);
11788 if (mdio_phy_id_is_c45(phy_addr)) {
11789 req->cl45_mdio = 1;
11790 req->phy_addr = mdio_phy_id_prtad(phy_addr);
11791 req->dev_addr = mdio_phy_id_devad(phy_addr);
11792 req->reg_addr = cpu_to_le16(reg);
11793 }
11794 req->reg_data = cpu_to_le16(val);
11795
11796 return hwrm_req_send(bp, req);
11797}
11798
11799/* rtnl_lock held */
11800static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
11801{
11802 struct mii_ioctl_data *mdio = if_mii(ifr);
11803 struct bnxt *bp = netdev_priv(dev);
11804 int rc;
11805
11806 switch (cmd) {
11807 case SIOCGMIIPHY:
11808 mdio->phy_id = bp->link_info.phy_addr;
11809
11810 fallthrough;
11811 case SIOCGMIIREG: {
11812 u16 mii_regval = 0;
11813
11814 if (!netif_running(dev))
11815 return -EAGAIN;
11816
11817 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
11818 &mii_regval);
11819 mdio->val_out = mii_regval;
11820 return rc;
11821 }
11822
11823 case SIOCSMIIREG:
11824 if (!netif_running(dev))
11825 return -EAGAIN;
11826
11827 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
11828 mdio->val_in);
11829
11830 case SIOCSHWTSTAMP:
11831 return bnxt_hwtstamp_set(dev, ifr);
11832
11833 case SIOCGHWTSTAMP:
11834 return bnxt_hwtstamp_get(dev, ifr);
11835
11836 default:
11837 /* do nothing */
11838 break;
11839 }
11840 return -EOPNOTSUPP;
11841}
11842
11843static void bnxt_get_ring_stats(struct bnxt *bp,
11844 struct rtnl_link_stats64 *stats)
11845{
11846 int i;
11847
11848 for (i = 0; i < bp->cp_nr_rings; i++) {
11849 struct bnxt_napi *bnapi = bp->bnapi[i];
11850 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
11851 u64 *sw = cpr->stats.sw_stats;
11852
11853 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
11854 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
11855 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
11856
11857 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
11858 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
11859 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
11860
11861 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
11862 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
11863 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
11864
11865 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
11866 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
11867 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
11868
11869 stats->rx_missed_errors +=
11870 BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
11871
11872 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
11873
11874 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
11875
11876 stats->rx_dropped +=
11877 cpr->sw_stats.rx.rx_netpoll_discards +
11878 cpr->sw_stats.rx.rx_oom_discards;
11879 }
11880}
11881
11882static void bnxt_add_prev_stats(struct bnxt *bp,
11883 struct rtnl_link_stats64 *stats)
11884{
11885 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
11886
11887 stats->rx_packets += prev_stats->rx_packets;
11888 stats->tx_packets += prev_stats->tx_packets;
11889 stats->rx_bytes += prev_stats->rx_bytes;
11890 stats->tx_bytes += prev_stats->tx_bytes;
11891 stats->rx_missed_errors += prev_stats->rx_missed_errors;
11892 stats->multicast += prev_stats->multicast;
11893 stats->rx_dropped += prev_stats->rx_dropped;
11894 stats->tx_dropped += prev_stats->tx_dropped;
11895}
11896
11897static void
11898bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
11899{
11900 struct bnxt *bp = netdev_priv(dev);
11901
11902 set_bit(BNXT_STATE_READ_STATS, &bp->state);
11903 /* Make sure bnxt_close_nic() sees that we are reading stats before
11904 * we check the BNXT_STATE_OPEN flag.
11905 */
11906 smp_mb__after_atomic();
11907 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11908 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
11909 *stats = bp->net_stats_prev;
11910 return;
11911 }
11912
11913 bnxt_get_ring_stats(bp, stats);
11914 bnxt_add_prev_stats(bp, stats);
11915
11916 if (bp->flags & BNXT_FLAG_PORT_STATS) {
11917 u64 *rx = bp->port_stats.sw_stats;
11918 u64 *tx = bp->port_stats.sw_stats +
11919 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
11920
11921 stats->rx_crc_errors =
11922 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
11923 stats->rx_frame_errors =
11924 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
11925 stats->rx_length_errors =
11926 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
11927 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
11928 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
11929 stats->rx_errors =
11930 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
11931 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
11932 stats->collisions =
11933 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
11934 stats->tx_fifo_errors =
11935 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
11936 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
11937 }
11938 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
11939}
11940
11941static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
11942 struct bnxt_total_ring_err_stats *stats,
11943 struct bnxt_cp_ring_info *cpr)
11944{
11945 struct bnxt_sw_stats *sw_stats = &cpr->sw_stats;
11946 u64 *hw_stats = cpr->stats.sw_stats;
11947
11948 stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors;
11949 stats->rx_total_resets += sw_stats->rx.rx_resets;
11950 stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors;
11951 stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards;
11952 stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards;
11953 stats->rx_total_ring_discards +=
11954 BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts);
11955 stats->tx_total_resets += sw_stats->tx.tx_resets;
11956 stats->tx_total_ring_discards +=
11957 BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts);
11958 stats->total_missed_irqs += sw_stats->cmn.missed_irqs;
11959}
11960
11961void bnxt_get_ring_err_stats(struct bnxt *bp,
11962 struct bnxt_total_ring_err_stats *stats)
11963{
11964 int i;
11965
11966 for (i = 0; i < bp->cp_nr_rings; i++)
11967 bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring);
11968}
11969
11970static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
11971{
11972 struct net_device *dev = bp->dev;
11973 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11974 struct netdev_hw_addr *ha;
11975 u8 *haddr;
11976 int mc_count = 0;
11977 bool update = false;
11978 int off = 0;
11979
11980 netdev_for_each_mc_addr(ha, dev) {
11981 if (mc_count >= BNXT_MAX_MC_ADDRS) {
11982 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11983 vnic->mc_list_count = 0;
11984 return false;
11985 }
11986 haddr = ha->addr;
11987 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
11988 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
11989 update = true;
11990 }
11991 off += ETH_ALEN;
11992 mc_count++;
11993 }
11994 if (mc_count)
11995 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11996
11997 if (mc_count != vnic->mc_list_count) {
11998 vnic->mc_list_count = mc_count;
11999 update = true;
12000 }
12001 return update;
12002}
12003
12004static bool bnxt_uc_list_updated(struct bnxt *bp)
12005{
12006 struct net_device *dev = bp->dev;
12007 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12008 struct netdev_hw_addr *ha;
12009 int off = 0;
12010
12011 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
12012 return true;
12013
12014 netdev_for_each_uc_addr(ha, dev) {
12015 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
12016 return true;
12017
12018 off += ETH_ALEN;
12019 }
12020 return false;
12021}
12022
12023static void bnxt_set_rx_mode(struct net_device *dev)
12024{
12025 struct bnxt *bp = netdev_priv(dev);
12026 struct bnxt_vnic_info *vnic;
12027 bool mc_update = false;
12028 bool uc_update;
12029 u32 mask;
12030
12031 if (!test_bit(BNXT_STATE_OPEN, &bp->state))
12032 return;
12033
12034 vnic = &bp->vnic_info[0];
12035 mask = vnic->rx_mask;
12036 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
12037 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
12038 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
12039 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
12040
12041 if (dev->flags & IFF_PROMISC)
12042 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12043
12044 uc_update = bnxt_uc_list_updated(bp);
12045
12046 if (dev->flags & IFF_BROADCAST)
12047 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
12048 if (dev->flags & IFF_ALLMULTI) {
12049 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12050 vnic->mc_list_count = 0;
12051 } else if (dev->flags & IFF_MULTICAST) {
12052 mc_update = bnxt_mc_list_updated(bp, &mask);
12053 }
12054
12055 if (mask != vnic->rx_mask || uc_update || mc_update) {
12056 vnic->rx_mask = mask;
12057
12058 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
12059 }
12060}
12061
12062static int bnxt_cfg_rx_mode(struct bnxt *bp)
12063{
12064 struct net_device *dev = bp->dev;
12065 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12066 struct netdev_hw_addr *ha;
12067 int i, off = 0, rc;
12068 bool uc_update;
12069
12070 netif_addr_lock_bh(dev);
12071 uc_update = bnxt_uc_list_updated(bp);
12072 netif_addr_unlock_bh(dev);
12073
12074 if (!uc_update)
12075 goto skip_uc;
12076
12077 for (i = 1; i < vnic->uc_filter_count; i++) {
12078 struct bnxt_l2_filter *fltr = vnic->l2_filters[i];
12079
12080 bnxt_hwrm_l2_filter_free(bp, fltr);
12081 bnxt_del_l2_filter(bp, fltr);
12082 }
12083
12084 vnic->uc_filter_count = 1;
12085
12086 netif_addr_lock_bh(dev);
12087 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
12088 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12089 } else {
12090 netdev_for_each_uc_addr(ha, dev) {
12091 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
12092 off += ETH_ALEN;
12093 vnic->uc_filter_count++;
12094 }
12095 }
12096 netif_addr_unlock_bh(dev);
12097
12098 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
12099 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
12100 if (rc) {
12101 if (BNXT_VF(bp) && rc == -ENODEV) {
12102 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12103 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
12104 else
12105 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
12106 rc = 0;
12107 } else {
12108 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
12109 }
12110 vnic->uc_filter_count = i;
12111 return rc;
12112 }
12113 }
12114 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12115 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
12116
12117skip_uc:
12118 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
12119 !bnxt_promisc_ok(bp))
12120 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12121 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
12122 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
12123 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
12124 rc);
12125 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
12126 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12127 vnic->mc_list_count = 0;
12128 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
12129 }
12130 if (rc)
12131 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
12132 rc);
12133
12134 return rc;
12135}
12136
12137static bool bnxt_can_reserve_rings(struct bnxt *bp)
12138{
12139#ifdef CONFIG_BNXT_SRIOV
12140 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
12141 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12142
12143 /* No minimum rings were provisioned by the PF. Don't
12144 * reserve rings by default when device is down.
12145 */
12146 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
12147 return true;
12148
12149 if (!netif_running(bp->dev))
12150 return false;
12151 }
12152#endif
12153 return true;
12154}
12155
12156/* If the chip and firmware supports RFS */
12157static bool bnxt_rfs_supported(struct bnxt *bp)
12158{
12159 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
12160 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
12161 return true;
12162 return false;
12163 }
12164 /* 212 firmware is broken for aRFS */
12165 if (BNXT_FW_MAJ(bp) == 212)
12166 return false;
12167 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
12168 return true;
12169 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
12170 return true;
12171 return false;
12172}
12173
12174/* If runtime conditions support RFS */
12175static bool bnxt_rfs_capable(struct bnxt *bp)
12176{
12177 int vnics, max_vnics, max_rss_ctxs;
12178
12179 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
12180 return bnxt_rfs_supported(bp);
12181 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
12182 return false;
12183
12184 vnics = 1 + bp->rx_nr_rings;
12185 max_vnics = bnxt_get_max_func_vnics(bp);
12186 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
12187
12188 /* RSS contexts not a limiting factor */
12189 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
12190 max_rss_ctxs = max_vnics;
12191 if (vnics > max_vnics || vnics > max_rss_ctxs) {
12192 if (bp->rx_nr_rings > 1)
12193 netdev_warn(bp->dev,
12194 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
12195 min(max_rss_ctxs - 1, max_vnics - 1));
12196 return false;
12197 }
12198
12199 if (!BNXT_NEW_RM(bp))
12200 return true;
12201
12202 if (vnics == bp->hw_resc.resv_vnics)
12203 return true;
12204
12205 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
12206 if (vnics <= bp->hw_resc.resv_vnics)
12207 return true;
12208
12209 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
12210 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
12211 return false;
12212}
12213
12214static netdev_features_t bnxt_fix_features(struct net_device *dev,
12215 netdev_features_t features)
12216{
12217 struct bnxt *bp = netdev_priv(dev);
12218 netdev_features_t vlan_features;
12219
12220 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
12221 features &= ~NETIF_F_NTUPLE;
12222
12223 if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
12224 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12225
12226 if (!(features & NETIF_F_GRO))
12227 features &= ~NETIF_F_GRO_HW;
12228
12229 if (features & NETIF_F_GRO_HW)
12230 features &= ~NETIF_F_LRO;
12231
12232 /* Both CTAG and STAG VLAN accelaration on the RX side have to be
12233 * turned on or off together.
12234 */
12235 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
12236 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
12237 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12238 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
12239 else if (vlan_features)
12240 features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12241 }
12242#ifdef CONFIG_BNXT_SRIOV
12243 if (BNXT_VF(bp) && bp->vf.vlan)
12244 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
12245#endif
12246 return features;
12247}
12248
12249static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
12250{
12251 struct bnxt *bp = netdev_priv(dev);
12252 u32 flags = bp->flags;
12253 u32 changes;
12254 int rc = 0;
12255 bool re_init = false;
12256 bool update_tpa = false;
12257
12258 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
12259 if (features & NETIF_F_GRO_HW)
12260 flags |= BNXT_FLAG_GRO;
12261 else if (features & NETIF_F_LRO)
12262 flags |= BNXT_FLAG_LRO;
12263
12264 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
12265 flags &= ~BNXT_FLAG_TPA;
12266
12267 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12268 flags |= BNXT_FLAG_STRIP_VLAN;
12269
12270 if (features & NETIF_F_NTUPLE)
12271 flags |= BNXT_FLAG_RFS;
12272
12273 changes = flags ^ bp->flags;
12274 if (changes & BNXT_FLAG_TPA) {
12275 update_tpa = true;
12276 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
12277 (flags & BNXT_FLAG_TPA) == 0 ||
12278 (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
12279 re_init = true;
12280 }
12281
12282 if (changes & ~BNXT_FLAG_TPA)
12283 re_init = true;
12284
12285 if (flags != bp->flags) {
12286 u32 old_flags = bp->flags;
12287
12288 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12289 bp->flags = flags;
12290 if (update_tpa)
12291 bnxt_set_ring_params(bp);
12292 return rc;
12293 }
12294
12295 if (re_init) {
12296 bnxt_close_nic(bp, false, false);
12297 bp->flags = flags;
12298 if (update_tpa)
12299 bnxt_set_ring_params(bp);
12300
12301 return bnxt_open_nic(bp, false, false);
12302 }
12303 if (update_tpa) {
12304 bp->flags = flags;
12305 rc = bnxt_set_tpa(bp,
12306 (flags & BNXT_FLAG_TPA) ?
12307 true : false);
12308 if (rc)
12309 bp->flags = old_flags;
12310 }
12311 }
12312 return rc;
12313}
12314
12315static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
12316 u8 **nextp)
12317{
12318 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
12319 struct hop_jumbo_hdr *jhdr;
12320 int hdr_count = 0;
12321 u8 *nexthdr;
12322 int start;
12323
12324 /* Check that there are at most 2 IPv6 extension headers, no
12325 * fragment header, and each is <= 64 bytes.
12326 */
12327 start = nw_off + sizeof(*ip6h);
12328 nexthdr = &ip6h->nexthdr;
12329 while (ipv6_ext_hdr(*nexthdr)) {
12330 struct ipv6_opt_hdr *hp;
12331 int hdrlen;
12332
12333 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
12334 *nexthdr == NEXTHDR_FRAGMENT)
12335 return false;
12336 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
12337 skb_headlen(skb), NULL);
12338 if (!hp)
12339 return false;
12340 if (*nexthdr == NEXTHDR_AUTH)
12341 hdrlen = ipv6_authlen(hp);
12342 else
12343 hdrlen = ipv6_optlen(hp);
12344
12345 if (hdrlen > 64)
12346 return false;
12347
12348 /* The ext header may be a hop-by-hop header inserted for
12349 * big TCP purposes. This will be removed before sending
12350 * from NIC, so do not count it.
12351 */
12352 if (*nexthdr == NEXTHDR_HOP) {
12353 if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
12354 goto increment_hdr;
12355
12356 jhdr = (struct hop_jumbo_hdr *)hp;
12357 if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
12358 jhdr->nexthdr != IPPROTO_TCP)
12359 goto increment_hdr;
12360
12361 goto next_hdr;
12362 }
12363increment_hdr:
12364 hdr_count++;
12365next_hdr:
12366 nexthdr = &hp->nexthdr;
12367 start += hdrlen;
12368 }
12369 if (nextp) {
12370 /* Caller will check inner protocol */
12371 if (skb->encapsulation) {
12372 *nextp = nexthdr;
12373 return true;
12374 }
12375 *nextp = NULL;
12376 }
12377 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
12378 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
12379}
12380
12381/* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
12382static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
12383{
12384 struct udphdr *uh = udp_hdr(skb);
12385 __be16 udp_port = uh->dest;
12386
12387 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port &&
12388 udp_port != bp->vxlan_gpe_port)
12389 return false;
12390 if (skb->inner_protocol == htons(ETH_P_TEB)) {
12391 struct ethhdr *eh = inner_eth_hdr(skb);
12392
12393 switch (eh->h_proto) {
12394 case htons(ETH_P_IP):
12395 return true;
12396 case htons(ETH_P_IPV6):
12397 return bnxt_exthdr_check(bp, skb,
12398 skb_inner_network_offset(skb),
12399 NULL);
12400 }
12401 } else if (skb->inner_protocol == htons(ETH_P_IP)) {
12402 return true;
12403 } else if (skb->inner_protocol == htons(ETH_P_IPV6)) {
12404 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
12405 NULL);
12406 }
12407 return false;
12408}
12409
12410static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
12411{
12412 switch (l4_proto) {
12413 case IPPROTO_UDP:
12414 return bnxt_udp_tunl_check(bp, skb);
12415 case IPPROTO_IPIP:
12416 return true;
12417 case IPPROTO_GRE: {
12418 switch (skb->inner_protocol) {
12419 default:
12420 return false;
12421 case htons(ETH_P_IP):
12422 return true;
12423 case htons(ETH_P_IPV6):
12424 fallthrough;
12425 }
12426 }
12427 case IPPROTO_IPV6:
12428 /* Check ext headers of inner ipv6 */
12429 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
12430 NULL);
12431 }
12432 return false;
12433}
12434
12435static netdev_features_t bnxt_features_check(struct sk_buff *skb,
12436 struct net_device *dev,
12437 netdev_features_t features)
12438{
12439 struct bnxt *bp = netdev_priv(dev);
12440 u8 *l4_proto;
12441
12442 features = vlan_features_check(skb, features);
12443 switch (vlan_get_protocol(skb)) {
12444 case htons(ETH_P_IP):
12445 if (!skb->encapsulation)
12446 return features;
12447 l4_proto = &ip_hdr(skb)->protocol;
12448 if (bnxt_tunl_check(bp, skb, *l4_proto))
12449 return features;
12450 break;
12451 case htons(ETH_P_IPV6):
12452 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
12453 &l4_proto))
12454 break;
12455 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
12456 return features;
12457 break;
12458 }
12459 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
12460}
12461
12462int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
12463 u32 *reg_buf)
12464{
12465 struct hwrm_dbg_read_direct_output *resp;
12466 struct hwrm_dbg_read_direct_input *req;
12467 __le32 *dbg_reg_buf;
12468 dma_addr_t mapping;
12469 int rc, i;
12470
12471 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
12472 if (rc)
12473 return rc;
12474
12475 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
12476 &mapping);
12477 if (!dbg_reg_buf) {
12478 rc = -ENOMEM;
12479 goto dbg_rd_reg_exit;
12480 }
12481
12482 req->host_dest_addr = cpu_to_le64(mapping);
12483
12484 resp = hwrm_req_hold(bp, req);
12485 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
12486 req->read_len32 = cpu_to_le32(num_words);
12487
12488 rc = hwrm_req_send(bp, req);
12489 if (rc || resp->error_code) {
12490 rc = -EIO;
12491 goto dbg_rd_reg_exit;
12492 }
12493 for (i = 0; i < num_words; i++)
12494 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
12495
12496dbg_rd_reg_exit:
12497 hwrm_req_drop(bp, req);
12498 return rc;
12499}
12500
12501static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
12502 u32 ring_id, u32 *prod, u32 *cons)
12503{
12504 struct hwrm_dbg_ring_info_get_output *resp;
12505 struct hwrm_dbg_ring_info_get_input *req;
12506 int rc;
12507
12508 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
12509 if (rc)
12510 return rc;
12511
12512 req->ring_type = ring_type;
12513 req->fw_ring_id = cpu_to_le32(ring_id);
12514 resp = hwrm_req_hold(bp, req);
12515 rc = hwrm_req_send(bp, req);
12516 if (!rc) {
12517 *prod = le32_to_cpu(resp->producer_index);
12518 *cons = le32_to_cpu(resp->consumer_index);
12519 }
12520 hwrm_req_drop(bp, req);
12521 return rc;
12522}
12523
12524static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
12525{
12526 struct bnxt_tx_ring_info *txr;
12527 int i = bnapi->index, j;
12528
12529 bnxt_for_each_napi_tx(j, bnapi, txr)
12530 netdev_info(bnapi->bp->dev, "[%d.%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
12531 i, j, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
12532 txr->tx_cons);
12533}
12534
12535static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
12536{
12537 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
12538 int i = bnapi->index;
12539
12540 if (!rxr)
12541 return;
12542
12543 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
12544 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
12545 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
12546 rxr->rx_sw_agg_prod);
12547}
12548
12549static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
12550{
12551 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
12552 int i = bnapi->index;
12553
12554 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
12555 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
12556}
12557
12558static void bnxt_dbg_dump_states(struct bnxt *bp)
12559{
12560 int i;
12561 struct bnxt_napi *bnapi;
12562
12563 for (i = 0; i < bp->cp_nr_rings; i++) {
12564 bnapi = bp->bnapi[i];
12565 if (netif_msg_drv(bp)) {
12566 bnxt_dump_tx_sw_state(bnapi);
12567 bnxt_dump_rx_sw_state(bnapi);
12568 bnxt_dump_cp_sw_state(bnapi);
12569 }
12570 }
12571}
12572
12573static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
12574{
12575 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
12576 struct hwrm_ring_reset_input *req;
12577 struct bnxt_napi *bnapi = rxr->bnapi;
12578 struct bnxt_cp_ring_info *cpr;
12579 u16 cp_ring_id;
12580 int rc;
12581
12582 rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
12583 if (rc)
12584 return rc;
12585
12586 cpr = &bnapi->cp_ring;
12587 cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
12588 req->cmpl_ring = cpu_to_le16(cp_ring_id);
12589 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
12590 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
12591 return hwrm_req_send_silent(bp, req);
12592}
12593
12594static void bnxt_reset_task(struct bnxt *bp, bool silent)
12595{
12596 if (!silent)
12597 bnxt_dbg_dump_states(bp);
12598 if (netif_running(bp->dev)) {
12599 int rc;
12600
12601 if (silent) {
12602 bnxt_close_nic(bp, false, false);
12603 bnxt_open_nic(bp, false, false);
12604 } else {
12605 bnxt_ulp_stop(bp);
12606 bnxt_close_nic(bp, true, false);
12607 rc = bnxt_open_nic(bp, true, false);
12608 bnxt_ulp_start(bp, rc);
12609 }
12610 }
12611}
12612
12613static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
12614{
12615 struct bnxt *bp = netdev_priv(dev);
12616
12617 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
12618 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
12619}
12620
12621static void bnxt_fw_health_check(struct bnxt *bp)
12622{
12623 struct bnxt_fw_health *fw_health = bp->fw_health;
12624 struct pci_dev *pdev = bp->pdev;
12625 u32 val;
12626
12627 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
12628 return;
12629
12630 /* Make sure it is enabled before checking the tmr_counter. */
12631 smp_rmb();
12632 if (fw_health->tmr_counter) {
12633 fw_health->tmr_counter--;
12634 return;
12635 }
12636
12637 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
12638 if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) {
12639 fw_health->arrests++;
12640 goto fw_reset;
12641 }
12642
12643 fw_health->last_fw_heartbeat = val;
12644
12645 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12646 if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) {
12647 fw_health->discoveries++;
12648 goto fw_reset;
12649 }
12650
12651 fw_health->tmr_counter = fw_health->tmr_multiplier;
12652 return;
12653
12654fw_reset:
12655 bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT);
12656}
12657
12658static void bnxt_timer(struct timer_list *t)
12659{
12660 struct bnxt *bp = from_timer(bp, t, timer);
12661 struct net_device *dev = bp->dev;
12662
12663 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
12664 return;
12665
12666 if (atomic_read(&bp->intr_sem) != 0)
12667 goto bnxt_restart_timer;
12668
12669 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
12670 bnxt_fw_health_check(bp);
12671
12672 if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks)
12673 bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT);
12674
12675 if (bnxt_tc_flower_enabled(bp))
12676 bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT);
12677
12678#ifdef CONFIG_RFS_ACCEL
12679 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count)
12680 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
12681#endif /*CONFIG_RFS_ACCEL*/
12682
12683 if (bp->link_info.phy_retry) {
12684 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
12685 bp->link_info.phy_retry = false;
12686 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
12687 } else {
12688 bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT);
12689 }
12690 }
12691
12692 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12693 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
12694
12695 if ((BNXT_CHIP_P5(bp)) && !bp->chip_rev && netif_carrier_ok(dev))
12696 bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT);
12697
12698bnxt_restart_timer:
12699 mod_timer(&bp->timer, jiffies + bp->current_interval);
12700}
12701
12702static void bnxt_rtnl_lock_sp(struct bnxt *bp)
12703{
12704 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
12705 * set. If the device is being closed, bnxt_close() may be holding
12706 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
12707 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
12708 */
12709 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12710 rtnl_lock();
12711}
12712
12713static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
12714{
12715 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12716 rtnl_unlock();
12717}
12718
12719/* Only called from bnxt_sp_task() */
12720static void bnxt_reset(struct bnxt *bp, bool silent)
12721{
12722 bnxt_rtnl_lock_sp(bp);
12723 if (test_bit(BNXT_STATE_OPEN, &bp->state))
12724 bnxt_reset_task(bp, silent);
12725 bnxt_rtnl_unlock_sp(bp);
12726}
12727
12728/* Only called from bnxt_sp_task() */
12729static void bnxt_rx_ring_reset(struct bnxt *bp)
12730{
12731 int i;
12732
12733 bnxt_rtnl_lock_sp(bp);
12734 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12735 bnxt_rtnl_unlock_sp(bp);
12736 return;
12737 }
12738 /* Disable and flush TPA before resetting the RX ring */
12739 if (bp->flags & BNXT_FLAG_TPA)
12740 bnxt_set_tpa(bp, false);
12741 for (i = 0; i < bp->rx_nr_rings; i++) {
12742 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
12743 struct bnxt_cp_ring_info *cpr;
12744 int rc;
12745
12746 if (!rxr->bnapi->in_reset)
12747 continue;
12748
12749 rc = bnxt_hwrm_rx_ring_reset(bp, i);
12750 if (rc) {
12751 if (rc == -EINVAL || rc == -EOPNOTSUPP)
12752 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
12753 else
12754 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
12755 rc);
12756 bnxt_reset_task(bp, true);
12757 break;
12758 }
12759 bnxt_free_one_rx_ring_skbs(bp, i);
12760 rxr->rx_prod = 0;
12761 rxr->rx_agg_prod = 0;
12762 rxr->rx_sw_agg_prod = 0;
12763 rxr->rx_next_cons = 0;
12764 rxr->bnapi->in_reset = false;
12765 bnxt_alloc_one_rx_ring(bp, i);
12766 cpr = &rxr->bnapi->cp_ring;
12767 cpr->sw_stats.rx.rx_resets++;
12768 if (bp->flags & BNXT_FLAG_AGG_RINGS)
12769 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
12770 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
12771 }
12772 if (bp->flags & BNXT_FLAG_TPA)
12773 bnxt_set_tpa(bp, true);
12774 bnxt_rtnl_unlock_sp(bp);
12775}
12776
12777static void bnxt_fw_reset_close(struct bnxt *bp)
12778{
12779 bnxt_ulp_stop(bp);
12780 /* When firmware is in fatal state, quiesce device and disable
12781 * bus master to prevent any potential bad DMAs before freeing
12782 * kernel memory.
12783 */
12784 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
12785 u16 val = 0;
12786
12787 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
12788 if (val == 0xffff)
12789 bp->fw_reset_min_dsecs = 0;
12790 bnxt_tx_disable(bp);
12791 bnxt_disable_napi(bp);
12792 bnxt_disable_int_sync(bp);
12793 bnxt_free_irq(bp);
12794 bnxt_clear_int_mode(bp);
12795 pci_disable_device(bp->pdev);
12796 }
12797 __bnxt_close_nic(bp, true, false);
12798 bnxt_vf_reps_free(bp);
12799 bnxt_clear_int_mode(bp);
12800 bnxt_hwrm_func_drv_unrgtr(bp);
12801 if (pci_is_enabled(bp->pdev))
12802 pci_disable_device(bp->pdev);
12803 bnxt_free_ctx_mem(bp);
12804}
12805
12806static bool is_bnxt_fw_ok(struct bnxt *bp)
12807{
12808 struct bnxt_fw_health *fw_health = bp->fw_health;
12809 bool no_heartbeat = false, has_reset = false;
12810 u32 val;
12811
12812 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
12813 if (val == fw_health->last_fw_heartbeat)
12814 no_heartbeat = true;
12815
12816 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12817 if (val != fw_health->last_fw_reset_cnt)
12818 has_reset = true;
12819
12820 if (!no_heartbeat && has_reset)
12821 return true;
12822
12823 return false;
12824}
12825
12826/* rtnl_lock is acquired before calling this function */
12827static void bnxt_force_fw_reset(struct bnxt *bp)
12828{
12829 struct bnxt_fw_health *fw_health = bp->fw_health;
12830 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
12831 u32 wait_dsecs;
12832
12833 if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
12834 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
12835 return;
12836
12837 if (ptp) {
12838 spin_lock_bh(&ptp->ptp_lock);
12839 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12840 spin_unlock_bh(&ptp->ptp_lock);
12841 } else {
12842 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12843 }
12844 bnxt_fw_reset_close(bp);
12845 wait_dsecs = fw_health->master_func_wait_dsecs;
12846 if (fw_health->primary) {
12847 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
12848 wait_dsecs = 0;
12849 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
12850 } else {
12851 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
12852 wait_dsecs = fw_health->normal_func_wait_dsecs;
12853 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12854 }
12855
12856 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
12857 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
12858 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
12859}
12860
12861void bnxt_fw_exception(struct bnxt *bp)
12862{
12863 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
12864 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
12865 bnxt_rtnl_lock_sp(bp);
12866 bnxt_force_fw_reset(bp);
12867 bnxt_rtnl_unlock_sp(bp);
12868}
12869
12870/* Returns the number of registered VFs, or 1 if VF configuration is pending, or
12871 * < 0 on error.
12872 */
12873static int bnxt_get_registered_vfs(struct bnxt *bp)
12874{
12875#ifdef CONFIG_BNXT_SRIOV
12876 int rc;
12877
12878 if (!BNXT_PF(bp))
12879 return 0;
12880
12881 rc = bnxt_hwrm_func_qcfg(bp);
12882 if (rc) {
12883 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
12884 return rc;
12885 }
12886 if (bp->pf.registered_vfs)
12887 return bp->pf.registered_vfs;
12888 if (bp->sriov_cfg)
12889 return 1;
12890#endif
12891 return 0;
12892}
12893
12894void bnxt_fw_reset(struct bnxt *bp)
12895{
12896 bnxt_rtnl_lock_sp(bp);
12897 if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
12898 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12899 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
12900 int n = 0, tmo;
12901
12902 if (ptp) {
12903 spin_lock_bh(&ptp->ptp_lock);
12904 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12905 spin_unlock_bh(&ptp->ptp_lock);
12906 } else {
12907 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12908 }
12909 if (bp->pf.active_vfs &&
12910 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
12911 n = bnxt_get_registered_vfs(bp);
12912 if (n < 0) {
12913 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
12914 n);
12915 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12916 dev_close(bp->dev);
12917 goto fw_reset_exit;
12918 } else if (n > 0) {
12919 u16 vf_tmo_dsecs = n * 10;
12920
12921 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
12922 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
12923 bp->fw_reset_state =
12924 BNXT_FW_RESET_STATE_POLL_VF;
12925 bnxt_queue_fw_reset_work(bp, HZ / 10);
12926 goto fw_reset_exit;
12927 }
12928 bnxt_fw_reset_close(bp);
12929 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12930 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
12931 tmo = HZ / 10;
12932 } else {
12933 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12934 tmo = bp->fw_reset_min_dsecs * HZ / 10;
12935 }
12936 bnxt_queue_fw_reset_work(bp, tmo);
12937 }
12938fw_reset_exit:
12939 bnxt_rtnl_unlock_sp(bp);
12940}
12941
12942static void bnxt_chk_missed_irq(struct bnxt *bp)
12943{
12944 int i;
12945
12946 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
12947 return;
12948
12949 for (i = 0; i < bp->cp_nr_rings; i++) {
12950 struct bnxt_napi *bnapi = bp->bnapi[i];
12951 struct bnxt_cp_ring_info *cpr;
12952 u32 fw_ring_id;
12953 int j;
12954
12955 if (!bnapi)
12956 continue;
12957
12958 cpr = &bnapi->cp_ring;
12959 for (j = 0; j < cpr->cp_ring_count; j++) {
12960 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
12961 u32 val[2];
12962
12963 if (cpr2->has_more_work || !bnxt_has_work(bp, cpr2))
12964 continue;
12965
12966 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
12967 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
12968 continue;
12969 }
12970 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
12971 bnxt_dbg_hwrm_ring_info_get(bp,
12972 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
12973 fw_ring_id, &val[0], &val[1]);
12974 cpr->sw_stats.cmn.missed_irqs++;
12975 }
12976 }
12977}
12978
12979static void bnxt_cfg_ntp_filters(struct bnxt *);
12980
12981static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
12982{
12983 struct bnxt_link_info *link_info = &bp->link_info;
12984
12985 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
12986 link_info->autoneg = BNXT_AUTONEG_SPEED;
12987 if (bp->hwrm_spec_code >= 0x10201) {
12988 if (link_info->auto_pause_setting &
12989 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
12990 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
12991 } else {
12992 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
12993 }
12994 bnxt_set_auto_speed(link_info);
12995 } else {
12996 bnxt_set_force_speed(link_info);
12997 link_info->req_duplex = link_info->duplex_setting;
12998 }
12999 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
13000 link_info->req_flow_ctrl =
13001 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
13002 else
13003 link_info->req_flow_ctrl = link_info->force_pause_setting;
13004}
13005
13006static void bnxt_fw_echo_reply(struct bnxt *bp)
13007{
13008 struct bnxt_fw_health *fw_health = bp->fw_health;
13009 struct hwrm_func_echo_response_input *req;
13010 int rc;
13011
13012 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
13013 if (rc)
13014 return;
13015 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
13016 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
13017 hwrm_req_send(bp, req);
13018}
13019
13020static void bnxt_sp_task(struct work_struct *work)
13021{
13022 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
13023
13024 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13025 smp_mb__after_atomic();
13026 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13027 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13028 return;
13029 }
13030
13031 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
13032 bnxt_cfg_rx_mode(bp);
13033
13034 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
13035 bnxt_cfg_ntp_filters(bp);
13036 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
13037 bnxt_hwrm_exec_fwd_req(bp);
13038 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
13039 netdev_info(bp->dev, "Receive PF driver unload event!\n");
13040 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
13041 bnxt_hwrm_port_qstats(bp, 0);
13042 bnxt_hwrm_port_qstats_ext(bp, 0);
13043 bnxt_accumulate_all_stats(bp);
13044 }
13045
13046 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
13047 int rc;
13048
13049 mutex_lock(&bp->link_lock);
13050 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
13051 &bp->sp_event))
13052 bnxt_hwrm_phy_qcaps(bp);
13053
13054 rc = bnxt_update_link(bp, true);
13055 if (rc)
13056 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
13057 rc);
13058
13059 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
13060 &bp->sp_event))
13061 bnxt_init_ethtool_link_settings(bp);
13062 mutex_unlock(&bp->link_lock);
13063 }
13064 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
13065 int rc;
13066
13067 mutex_lock(&bp->link_lock);
13068 rc = bnxt_update_phy_setting(bp);
13069 mutex_unlock(&bp->link_lock);
13070 if (rc) {
13071 netdev_warn(bp->dev, "update phy settings retry failed\n");
13072 } else {
13073 bp->link_info.phy_retry = false;
13074 netdev_info(bp->dev, "update phy settings retry succeeded\n");
13075 }
13076 }
13077 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
13078 mutex_lock(&bp->link_lock);
13079 bnxt_get_port_module_status(bp);
13080 mutex_unlock(&bp->link_lock);
13081 }
13082
13083 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
13084 bnxt_tc_flow_stats_work(bp);
13085
13086 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
13087 bnxt_chk_missed_irq(bp);
13088
13089 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
13090 bnxt_fw_echo_reply(bp);
13091
13092 if (test_and_clear_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event))
13093 bnxt_hwmon_notify_event(bp);
13094
13095 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They
13096 * must be the last functions to be called before exiting.
13097 */
13098 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
13099 bnxt_reset(bp, false);
13100
13101 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
13102 bnxt_reset(bp, true);
13103
13104 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
13105 bnxt_rx_ring_reset(bp);
13106
13107 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
13108 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
13109 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
13110 bnxt_devlink_health_fw_report(bp);
13111 else
13112 bnxt_fw_reset(bp);
13113 }
13114
13115 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
13116 if (!is_bnxt_fw_ok(bp))
13117 bnxt_devlink_health_fw_report(bp);
13118 }
13119
13120 smp_mb__before_atomic();
13121 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13122}
13123
13124static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13125 int *max_cp);
13126
13127/* Under rtnl_lock */
13128int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
13129 int tx_xdp)
13130{
13131 int max_rx, max_tx, max_cp, tx_sets = 1, tx_cp;
13132 int tx_rings_needed, stats;
13133 int rx_rings = rx;
13134 int cp, vnics;
13135
13136 if (tcs)
13137 tx_sets = tcs;
13138
13139 _bnxt_get_max_rings(bp, &max_rx, &max_tx, &max_cp);
13140
13141 if (max_rx < rx_rings)
13142 return -ENOMEM;
13143
13144 if (bp->flags & BNXT_FLAG_AGG_RINGS)
13145 rx_rings <<= 1;
13146
13147 tx_rings_needed = tx * tx_sets + tx_xdp;
13148 if (max_tx < tx_rings_needed)
13149 return -ENOMEM;
13150
13151 vnics = 1;
13152 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5_PLUS)) ==
13153 BNXT_FLAG_RFS)
13154 vnics += rx;
13155
13156 tx_cp = __bnxt_num_tx_to_cp(bp, tx_rings_needed, tx_sets, tx_xdp);
13157 cp = sh ? max_t(int, tx_cp, rx) : tx_cp + rx;
13158 if (max_cp < cp)
13159 return -ENOMEM;
13160 stats = cp;
13161 if (BNXT_NEW_RM(bp)) {
13162 cp += bnxt_get_ulp_msix_num(bp);
13163 stats += bnxt_get_ulp_stat_ctxs(bp);
13164 }
13165 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
13166 stats, vnics);
13167}
13168
13169static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
13170{
13171 if (bp->bar2) {
13172 pci_iounmap(pdev, bp->bar2);
13173 bp->bar2 = NULL;
13174 }
13175
13176 if (bp->bar1) {
13177 pci_iounmap(pdev, bp->bar1);
13178 bp->bar1 = NULL;
13179 }
13180
13181 if (bp->bar0) {
13182 pci_iounmap(pdev, bp->bar0);
13183 bp->bar0 = NULL;
13184 }
13185}
13186
13187static void bnxt_cleanup_pci(struct bnxt *bp)
13188{
13189 bnxt_unmap_bars(bp, bp->pdev);
13190 pci_release_regions(bp->pdev);
13191 if (pci_is_enabled(bp->pdev))
13192 pci_disable_device(bp->pdev);
13193}
13194
13195static void bnxt_init_dflt_coal(struct bnxt *bp)
13196{
13197 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
13198 struct bnxt_coal *coal;
13199 u16 flags = 0;
13200
13201 if (coal_cap->cmpl_params &
13202 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
13203 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
13204
13205 /* Tick values in micro seconds.
13206 * 1 coal_buf x bufs_per_record = 1 completion record.
13207 */
13208 coal = &bp->rx_coal;
13209 coal->coal_ticks = 10;
13210 coal->coal_bufs = 30;
13211 coal->coal_ticks_irq = 1;
13212 coal->coal_bufs_irq = 2;
13213 coal->idle_thresh = 50;
13214 coal->bufs_per_record = 2;
13215 coal->budget = 64; /* NAPI budget */
13216 coal->flags = flags;
13217
13218 coal = &bp->tx_coal;
13219 coal->coal_ticks = 28;
13220 coal->coal_bufs = 30;
13221 coal->coal_ticks_irq = 2;
13222 coal->coal_bufs_irq = 2;
13223 coal->bufs_per_record = 1;
13224 coal->flags = flags;
13225
13226 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
13227}
13228
13229/* FW that pre-reserves 1 VNIC per function */
13230static bool bnxt_fw_pre_resv_vnics(struct bnxt *bp)
13231{
13232 u16 fw_maj = BNXT_FW_MAJ(bp), fw_bld = BNXT_FW_BLD(bp);
13233
13234 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13235 (fw_maj > 218 || (fw_maj == 218 && fw_bld >= 18)))
13236 return true;
13237 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13238 (fw_maj > 216 || (fw_maj == 216 && fw_bld >= 172)))
13239 return true;
13240 return false;
13241}
13242
13243static int bnxt_fw_init_one_p1(struct bnxt *bp)
13244{
13245 int rc;
13246
13247 bp->fw_cap = 0;
13248 rc = bnxt_hwrm_ver_get(bp);
13249 /* FW may be unresponsive after FLR. FLR must complete within 100 msec
13250 * so wait before continuing with recovery.
13251 */
13252 if (rc)
13253 msleep(100);
13254 bnxt_try_map_fw_health_reg(bp);
13255 if (rc) {
13256 rc = bnxt_try_recover_fw(bp);
13257 if (rc)
13258 return rc;
13259 rc = bnxt_hwrm_ver_get(bp);
13260 if (rc)
13261 return rc;
13262 }
13263
13264 bnxt_nvm_cfg_ver_get(bp);
13265
13266 rc = bnxt_hwrm_func_reset(bp);
13267 if (rc)
13268 return -ENODEV;
13269
13270 bnxt_hwrm_fw_set_time(bp);
13271 return 0;
13272}
13273
13274static int bnxt_fw_init_one_p2(struct bnxt *bp)
13275{
13276 int rc;
13277
13278 /* Get the MAX capabilities for this function */
13279 rc = bnxt_hwrm_func_qcaps(bp);
13280 if (rc) {
13281 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
13282 rc);
13283 return -ENODEV;
13284 }
13285
13286 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
13287 if (rc)
13288 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
13289 rc);
13290
13291 if (bnxt_alloc_fw_health(bp)) {
13292 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
13293 } else {
13294 rc = bnxt_hwrm_error_recovery_qcfg(bp);
13295 if (rc)
13296 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
13297 rc);
13298 }
13299
13300 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
13301 if (rc)
13302 return -ENODEV;
13303
13304 if (bnxt_fw_pre_resv_vnics(bp))
13305 bp->fw_cap |= BNXT_FW_CAP_PRE_RESV_VNICS;
13306
13307 bnxt_hwrm_func_qcfg(bp);
13308 bnxt_hwrm_vnic_qcaps(bp);
13309 bnxt_hwrm_port_led_qcaps(bp);
13310 bnxt_ethtool_init(bp);
13311 if (bp->fw_cap & BNXT_FW_CAP_PTP)
13312 __bnxt_hwrm_ptp_qcfg(bp);
13313 bnxt_dcb_init(bp);
13314 bnxt_hwmon_init(bp);
13315 return 0;
13316}
13317
13318static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
13319{
13320 bp->rss_cap &= ~BNXT_RSS_CAP_UDP_RSS_CAP;
13321 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
13322 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
13323 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
13324 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
13325 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
13326 bp->rss_hash_delta = bp->rss_hash_cfg;
13327 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
13328 bp->rss_cap |= BNXT_RSS_CAP_UDP_RSS_CAP;
13329 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
13330 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
13331 }
13332}
13333
13334static void bnxt_set_dflt_rfs(struct bnxt *bp)
13335{
13336 struct net_device *dev = bp->dev;
13337
13338 dev->hw_features &= ~NETIF_F_NTUPLE;
13339 dev->features &= ~NETIF_F_NTUPLE;
13340 bp->flags &= ~BNXT_FLAG_RFS;
13341 if (bnxt_rfs_supported(bp)) {
13342 dev->hw_features |= NETIF_F_NTUPLE;
13343 if (bnxt_rfs_capable(bp)) {
13344 bp->flags |= BNXT_FLAG_RFS;
13345 dev->features |= NETIF_F_NTUPLE;
13346 }
13347 }
13348}
13349
13350static void bnxt_fw_init_one_p3(struct bnxt *bp)
13351{
13352 struct pci_dev *pdev = bp->pdev;
13353
13354 bnxt_set_dflt_rss_hash_type(bp);
13355 bnxt_set_dflt_rfs(bp);
13356
13357 bnxt_get_wol_settings(bp);
13358 if (bp->flags & BNXT_FLAG_WOL_CAP)
13359 device_set_wakeup_enable(&pdev->dev, bp->wol);
13360 else
13361 device_set_wakeup_capable(&pdev->dev, false);
13362
13363 bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
13364 bnxt_hwrm_coal_params_qcaps(bp);
13365}
13366
13367static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
13368
13369int bnxt_fw_init_one(struct bnxt *bp)
13370{
13371 int rc;
13372
13373 rc = bnxt_fw_init_one_p1(bp);
13374 if (rc) {
13375 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
13376 return rc;
13377 }
13378 rc = bnxt_fw_init_one_p2(bp);
13379 if (rc) {
13380 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
13381 return rc;
13382 }
13383 rc = bnxt_probe_phy(bp, false);
13384 if (rc)
13385 return rc;
13386 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
13387 if (rc)
13388 return rc;
13389
13390 bnxt_fw_init_one_p3(bp);
13391 return 0;
13392}
13393
13394static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
13395{
13396 struct bnxt_fw_health *fw_health = bp->fw_health;
13397 u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
13398 u32 val = fw_health->fw_reset_seq_vals[reg_idx];
13399 u32 reg_type, reg_off, delay_msecs;
13400
13401 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
13402 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
13403 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
13404 switch (reg_type) {
13405 case BNXT_FW_HEALTH_REG_TYPE_CFG:
13406 pci_write_config_dword(bp->pdev, reg_off, val);
13407 break;
13408 case BNXT_FW_HEALTH_REG_TYPE_GRC:
13409 writel(reg_off & BNXT_GRC_BASE_MASK,
13410 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
13411 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
13412 fallthrough;
13413 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
13414 writel(val, bp->bar0 + reg_off);
13415 break;
13416 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
13417 writel(val, bp->bar1 + reg_off);
13418 break;
13419 }
13420 if (delay_msecs) {
13421 pci_read_config_dword(bp->pdev, 0, &val);
13422 msleep(delay_msecs);
13423 }
13424}
13425
13426bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
13427{
13428 struct hwrm_func_qcfg_output *resp;
13429 struct hwrm_func_qcfg_input *req;
13430 bool result = true; /* firmware will enforce if unknown */
13431
13432 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
13433 return result;
13434
13435 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
13436 return result;
13437
13438 req->fid = cpu_to_le16(0xffff);
13439 resp = hwrm_req_hold(bp, req);
13440 if (!hwrm_req_send(bp, req))
13441 result = !!(le16_to_cpu(resp->flags) &
13442 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
13443 hwrm_req_drop(bp, req);
13444 return result;
13445}
13446
13447static void bnxt_reset_all(struct bnxt *bp)
13448{
13449 struct bnxt_fw_health *fw_health = bp->fw_health;
13450 int i, rc;
13451
13452 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
13453 bnxt_fw_reset_via_optee(bp);
13454 bp->fw_reset_timestamp = jiffies;
13455 return;
13456 }
13457
13458 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
13459 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
13460 bnxt_fw_reset_writel(bp, i);
13461 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
13462 struct hwrm_fw_reset_input *req;
13463
13464 rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
13465 if (!rc) {
13466 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
13467 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
13468 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
13469 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
13470 rc = hwrm_req_send(bp, req);
13471 }
13472 if (rc != -ENODEV)
13473 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
13474 }
13475 bp->fw_reset_timestamp = jiffies;
13476}
13477
13478static bool bnxt_fw_reset_timeout(struct bnxt *bp)
13479{
13480 return time_after(jiffies, bp->fw_reset_timestamp +
13481 (bp->fw_reset_max_dsecs * HZ / 10));
13482}
13483
13484static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
13485{
13486 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13487 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) {
13488 bnxt_ulp_start(bp, rc);
13489 bnxt_dl_health_fw_status_update(bp, false);
13490 }
13491 bp->fw_reset_state = 0;
13492 dev_close(bp->dev);
13493}
13494
13495static void bnxt_fw_reset_task(struct work_struct *work)
13496{
13497 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
13498 int rc = 0;
13499
13500 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
13501 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
13502 return;
13503 }
13504
13505 switch (bp->fw_reset_state) {
13506 case BNXT_FW_RESET_STATE_POLL_VF: {
13507 int n = bnxt_get_registered_vfs(bp);
13508 int tmo;
13509
13510 if (n < 0) {
13511 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
13512 n, jiffies_to_msecs(jiffies -
13513 bp->fw_reset_timestamp));
13514 goto fw_reset_abort;
13515 } else if (n > 0) {
13516 if (bnxt_fw_reset_timeout(bp)) {
13517 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13518 bp->fw_reset_state = 0;
13519 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
13520 n);
13521 return;
13522 }
13523 bnxt_queue_fw_reset_work(bp, HZ / 10);
13524 return;
13525 }
13526 bp->fw_reset_timestamp = jiffies;
13527 rtnl_lock();
13528 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
13529 bnxt_fw_reset_abort(bp, rc);
13530 rtnl_unlock();
13531 return;
13532 }
13533 bnxt_fw_reset_close(bp);
13534 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
13535 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
13536 tmo = HZ / 10;
13537 } else {
13538 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13539 tmo = bp->fw_reset_min_dsecs * HZ / 10;
13540 }
13541 rtnl_unlock();
13542 bnxt_queue_fw_reset_work(bp, tmo);
13543 return;
13544 }
13545 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
13546 u32 val;
13547
13548 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
13549 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
13550 !bnxt_fw_reset_timeout(bp)) {
13551 bnxt_queue_fw_reset_work(bp, HZ / 5);
13552 return;
13553 }
13554
13555 if (!bp->fw_health->primary) {
13556 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
13557
13558 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13559 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
13560 return;
13561 }
13562 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
13563 }
13564 fallthrough;
13565 case BNXT_FW_RESET_STATE_RESET_FW:
13566 bnxt_reset_all(bp);
13567 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13568 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
13569 return;
13570 case BNXT_FW_RESET_STATE_ENABLE_DEV:
13571 bnxt_inv_fw_health_reg(bp);
13572 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
13573 !bp->fw_reset_min_dsecs) {
13574 u16 val;
13575
13576 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
13577 if (val == 0xffff) {
13578 if (bnxt_fw_reset_timeout(bp)) {
13579 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
13580 rc = -ETIMEDOUT;
13581 goto fw_reset_abort;
13582 }
13583 bnxt_queue_fw_reset_work(bp, HZ / 1000);
13584 return;
13585 }
13586 }
13587 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
13588 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
13589 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
13590 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
13591 bnxt_dl_remote_reload(bp);
13592 if (pci_enable_device(bp->pdev)) {
13593 netdev_err(bp->dev, "Cannot re-enable PCI device\n");
13594 rc = -ENODEV;
13595 goto fw_reset_abort;
13596 }
13597 pci_set_master(bp->pdev);
13598 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
13599 fallthrough;
13600 case BNXT_FW_RESET_STATE_POLL_FW:
13601 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
13602 rc = bnxt_hwrm_poll(bp);
13603 if (rc) {
13604 if (bnxt_fw_reset_timeout(bp)) {
13605 netdev_err(bp->dev, "Firmware reset aborted\n");
13606 goto fw_reset_abort_status;
13607 }
13608 bnxt_queue_fw_reset_work(bp, HZ / 5);
13609 return;
13610 }
13611 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
13612 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
13613 fallthrough;
13614 case BNXT_FW_RESET_STATE_OPENING:
13615 while (!rtnl_trylock()) {
13616 bnxt_queue_fw_reset_work(bp, HZ / 10);
13617 return;
13618 }
13619 rc = bnxt_open(bp->dev);
13620 if (rc) {
13621 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
13622 bnxt_fw_reset_abort(bp, rc);
13623 rtnl_unlock();
13624 return;
13625 }
13626
13627 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
13628 bp->fw_health->enabled) {
13629 bp->fw_health->last_fw_reset_cnt =
13630 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
13631 }
13632 bp->fw_reset_state = 0;
13633 /* Make sure fw_reset_state is 0 before clearing the flag */
13634 smp_mb__before_atomic();
13635 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13636 bnxt_ulp_start(bp, 0);
13637 bnxt_reenable_sriov(bp);
13638 bnxt_vf_reps_alloc(bp);
13639 bnxt_vf_reps_open(bp);
13640 bnxt_ptp_reapply_pps(bp);
13641 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
13642 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
13643 bnxt_dl_health_fw_recovery_done(bp);
13644 bnxt_dl_health_fw_status_update(bp, true);
13645 }
13646 rtnl_unlock();
13647 break;
13648 }
13649 return;
13650
13651fw_reset_abort_status:
13652 if (bp->fw_health->status_reliable ||
13653 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
13654 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
13655
13656 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
13657 }
13658fw_reset_abort:
13659 rtnl_lock();
13660 bnxt_fw_reset_abort(bp, rc);
13661 rtnl_unlock();
13662}
13663
13664static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
13665{
13666 int rc;
13667 struct bnxt *bp = netdev_priv(dev);
13668
13669 SET_NETDEV_DEV(dev, &pdev->dev);
13670
13671 /* enable device (incl. PCI PM wakeup), and bus-mastering */
13672 rc = pci_enable_device(pdev);
13673 if (rc) {
13674 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
13675 goto init_err;
13676 }
13677
13678 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
13679 dev_err(&pdev->dev,
13680 "Cannot find PCI device base address, aborting\n");
13681 rc = -ENODEV;
13682 goto init_err_disable;
13683 }
13684
13685 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
13686 if (rc) {
13687 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
13688 goto init_err_disable;
13689 }
13690
13691 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
13692 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
13693 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
13694 rc = -EIO;
13695 goto init_err_release;
13696 }
13697
13698 pci_set_master(pdev);
13699
13700 bp->dev = dev;
13701 bp->pdev = pdev;
13702
13703 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
13704 * determines the BAR size.
13705 */
13706 bp->bar0 = pci_ioremap_bar(pdev, 0);
13707 if (!bp->bar0) {
13708 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
13709 rc = -ENOMEM;
13710 goto init_err_release;
13711 }
13712
13713 bp->bar2 = pci_ioremap_bar(pdev, 4);
13714 if (!bp->bar2) {
13715 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
13716 rc = -ENOMEM;
13717 goto init_err_release;
13718 }
13719
13720 INIT_WORK(&bp->sp_task, bnxt_sp_task);
13721 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
13722
13723 spin_lock_init(&bp->ntp_fltr_lock);
13724#if BITS_PER_LONG == 32
13725 spin_lock_init(&bp->db_lock);
13726#endif
13727
13728 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
13729 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
13730
13731 timer_setup(&bp->timer, bnxt_timer, 0);
13732 bp->current_interval = BNXT_TIMER_INTERVAL;
13733
13734 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
13735 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
13736
13737 clear_bit(BNXT_STATE_OPEN, &bp->state);
13738 return 0;
13739
13740init_err_release:
13741 bnxt_unmap_bars(bp, pdev);
13742 pci_release_regions(pdev);
13743
13744init_err_disable:
13745 pci_disable_device(pdev);
13746
13747init_err:
13748 return rc;
13749}
13750
13751/* rtnl_lock held */
13752static int bnxt_change_mac_addr(struct net_device *dev, void *p)
13753{
13754 struct sockaddr *addr = p;
13755 struct bnxt *bp = netdev_priv(dev);
13756 int rc = 0;
13757
13758 if (!is_valid_ether_addr(addr->sa_data))
13759 return -EADDRNOTAVAIL;
13760
13761 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
13762 return 0;
13763
13764 rc = bnxt_approve_mac(bp, addr->sa_data, true);
13765 if (rc)
13766 return rc;
13767
13768 eth_hw_addr_set(dev, addr->sa_data);
13769 if (netif_running(dev)) {
13770 bnxt_close_nic(bp, false, false);
13771 rc = bnxt_open_nic(bp, false, false);
13772 }
13773
13774 return rc;
13775}
13776
13777/* rtnl_lock held */
13778static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
13779{
13780 struct bnxt *bp = netdev_priv(dev);
13781
13782 if (netif_running(dev))
13783 bnxt_close_nic(bp, true, false);
13784
13785 dev->mtu = new_mtu;
13786 bnxt_set_ring_params(bp);
13787
13788 if (netif_running(dev))
13789 return bnxt_open_nic(bp, true, false);
13790
13791 return 0;
13792}
13793
13794int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
13795{
13796 struct bnxt *bp = netdev_priv(dev);
13797 bool sh = false;
13798 int rc, tx_cp;
13799
13800 if (tc > bp->max_tc) {
13801 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
13802 tc, bp->max_tc);
13803 return -EINVAL;
13804 }
13805
13806 if (bp->num_tc == tc)
13807 return 0;
13808
13809 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
13810 sh = true;
13811
13812 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
13813 sh, tc, bp->tx_nr_rings_xdp);
13814 if (rc)
13815 return rc;
13816
13817 /* Needs to close the device and do hw resource re-allocations */
13818 if (netif_running(bp->dev))
13819 bnxt_close_nic(bp, true, false);
13820
13821 if (tc) {
13822 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
13823 netdev_set_num_tc(dev, tc);
13824 bp->num_tc = tc;
13825 } else {
13826 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13827 netdev_reset_tc(dev);
13828 bp->num_tc = 0;
13829 }
13830 bp->tx_nr_rings += bp->tx_nr_rings_xdp;
13831 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
13832 bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) :
13833 tx_cp + bp->rx_nr_rings;
13834
13835 if (netif_running(bp->dev))
13836 return bnxt_open_nic(bp, true, false);
13837
13838 return 0;
13839}
13840
13841static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
13842 void *cb_priv)
13843{
13844 struct bnxt *bp = cb_priv;
13845
13846 if (!bnxt_tc_flower_enabled(bp) ||
13847 !tc_cls_can_offload_and_chain0(bp->dev, type_data))
13848 return -EOPNOTSUPP;
13849
13850 switch (type) {
13851 case TC_SETUP_CLSFLOWER:
13852 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
13853 default:
13854 return -EOPNOTSUPP;
13855 }
13856}
13857
13858LIST_HEAD(bnxt_block_cb_list);
13859
13860static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
13861 void *type_data)
13862{
13863 struct bnxt *bp = netdev_priv(dev);
13864
13865 switch (type) {
13866 case TC_SETUP_BLOCK:
13867 return flow_block_cb_setup_simple(type_data,
13868 &bnxt_block_cb_list,
13869 bnxt_setup_tc_block_cb,
13870 bp, bp, true);
13871 case TC_SETUP_QDISC_MQPRIO: {
13872 struct tc_mqprio_qopt *mqprio = type_data;
13873
13874 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
13875
13876 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
13877 }
13878 default:
13879 return -EOPNOTSUPP;
13880 }
13881}
13882
13883u32 bnxt_get_ntp_filter_idx(struct bnxt *bp, struct flow_keys *fkeys,
13884 const struct sk_buff *skb)
13885{
13886 struct bnxt_vnic_info *vnic;
13887
13888 if (skb)
13889 return skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
13890
13891 vnic = &bp->vnic_info[0];
13892 return bnxt_toeplitz(bp, fkeys, (void *)vnic->rss_hash_key);
13893}
13894
13895int bnxt_insert_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr,
13896 u32 idx)
13897{
13898 struct hlist_head *head;
13899 int bit_id;
13900
13901 spin_lock_bh(&bp->ntp_fltr_lock);
13902 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, BNXT_MAX_FLTR, 0);
13903 if (bit_id < 0) {
13904 spin_unlock_bh(&bp->ntp_fltr_lock);
13905 return -ENOMEM;
13906 }
13907
13908 fltr->base.sw_id = (u16)bit_id;
13909 fltr->base.type = BNXT_FLTR_TYPE_NTUPLE;
13910 fltr->base.flags |= BNXT_ACT_RING_DST;
13911 head = &bp->ntp_fltr_hash_tbl[idx];
13912 hlist_add_head_rcu(&fltr->base.hash, head);
13913 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
13914 bp->ntp_fltr_count++;
13915 spin_unlock_bh(&bp->ntp_fltr_lock);
13916 return 0;
13917}
13918
13919static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
13920 struct bnxt_ntuple_filter *f2)
13921{
13922 struct flow_keys *keys1 = &f1->fkeys;
13923 struct flow_keys *keys2 = &f2->fkeys;
13924
13925 if (f1->ntuple_flags != f2->ntuple_flags)
13926 return false;
13927
13928 if (keys1->basic.n_proto != keys2->basic.n_proto ||
13929 keys1->basic.ip_proto != keys2->basic.ip_proto)
13930 return false;
13931
13932 if (keys1->basic.n_proto == htons(ETH_P_IP)) {
13933 if (((f1->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_IP) &&
13934 keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src) ||
13935 ((f1->ntuple_flags & BNXT_NTUPLE_MATCH_DST_IP) &&
13936 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst))
13937 return false;
13938 } else {
13939 if (((f1->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_IP) &&
13940 memcmp(&keys1->addrs.v6addrs.src,
13941 &keys2->addrs.v6addrs.src,
13942 sizeof(keys1->addrs.v6addrs.src))) ||
13943 ((f1->ntuple_flags & BNXT_NTUPLE_MATCH_DST_IP) &&
13944 memcmp(&keys1->addrs.v6addrs.dst,
13945 &keys2->addrs.v6addrs.dst,
13946 sizeof(keys1->addrs.v6addrs.dst))))
13947 return false;
13948 }
13949
13950 if (((f1->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_PORT) &&
13951 keys1->ports.src != keys2->ports.src) ||
13952 ((f1->ntuple_flags & BNXT_NTUPLE_MATCH_DST_PORT) &&
13953 keys1->ports.dst != keys2->ports.dst))
13954 return false;
13955
13956 if (keys1->control.flags == keys2->control.flags &&
13957 f1->l2_fltr == f2->l2_fltr)
13958 return true;
13959
13960 return false;
13961}
13962
13963struct bnxt_ntuple_filter *
13964bnxt_lookup_ntp_filter_from_idx(struct bnxt *bp,
13965 struct bnxt_ntuple_filter *fltr, u32 idx)
13966{
13967 struct bnxt_ntuple_filter *f;
13968 struct hlist_head *head;
13969
13970 head = &bp->ntp_fltr_hash_tbl[idx];
13971 hlist_for_each_entry_rcu(f, head, base.hash) {
13972 if (bnxt_fltr_match(f, fltr))
13973 return f;
13974 }
13975 return NULL;
13976}
13977
13978#ifdef CONFIG_RFS_ACCEL
13979static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
13980 u16 rxq_index, u32 flow_id)
13981{
13982 struct bnxt *bp = netdev_priv(dev);
13983 struct bnxt_ntuple_filter *fltr, *new_fltr;
13984 struct flow_keys *fkeys;
13985 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
13986 struct bnxt_l2_filter *l2_fltr;
13987 int rc = 0, idx;
13988 u32 flags;
13989
13990 if (ether_addr_equal(dev->dev_addr, eth->h_dest)) {
13991 l2_fltr = bp->vnic_info[0].l2_filters[0];
13992 atomic_inc(&l2_fltr->refcnt);
13993 } else {
13994 struct bnxt_l2_key key;
13995
13996 ether_addr_copy(key.dst_mac_addr, eth->h_dest);
13997 key.vlan = 0;
13998 l2_fltr = bnxt_lookup_l2_filter_from_key(bp, &key);
13999 if (!l2_fltr)
14000 return -EINVAL;
14001 if (l2_fltr->base.flags & BNXT_ACT_FUNC_DST) {
14002 bnxt_del_l2_filter(bp, l2_fltr);
14003 return -EINVAL;
14004 }
14005 }
14006 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
14007 if (!new_fltr) {
14008 bnxt_del_l2_filter(bp, l2_fltr);
14009 return -ENOMEM;
14010 }
14011
14012 fkeys = &new_fltr->fkeys;
14013 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
14014 rc = -EPROTONOSUPPORT;
14015 goto err_free;
14016 }
14017
14018 if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
14019 fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
14020 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
14021 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
14022 rc = -EPROTONOSUPPORT;
14023 goto err_free;
14024 }
14025 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
14026 bp->hwrm_spec_code < 0x10601) {
14027 rc = -EPROTONOSUPPORT;
14028 goto err_free;
14029 }
14030 flags = fkeys->control.flags;
14031 if (((flags & FLOW_DIS_ENCAPSULATION) &&
14032 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
14033 rc = -EPROTONOSUPPORT;
14034 goto err_free;
14035 }
14036
14037 new_fltr->l2_fltr = l2_fltr;
14038 new_fltr->ntuple_flags = BNXT_NTUPLE_MATCH_ALL;
14039
14040 idx = bnxt_get_ntp_filter_idx(bp, fkeys, skb);
14041 rcu_read_lock();
14042 fltr = bnxt_lookup_ntp_filter_from_idx(bp, new_fltr, idx);
14043 if (fltr) {
14044 rc = fltr->base.sw_id;
14045 rcu_read_unlock();
14046 goto err_free;
14047 }
14048 rcu_read_unlock();
14049
14050 new_fltr->flow_id = flow_id;
14051 new_fltr->base.rxq = rxq_index;
14052 rc = bnxt_insert_ntp_filter(bp, new_fltr, idx);
14053 if (!rc) {
14054 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
14055 return new_fltr->base.sw_id;
14056 }
14057
14058err_free:
14059 bnxt_del_l2_filter(bp, l2_fltr);
14060 kfree(new_fltr);
14061 return rc;
14062}
14063#endif
14064
14065void bnxt_del_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr)
14066{
14067 spin_lock_bh(&bp->ntp_fltr_lock);
14068 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
14069 spin_unlock_bh(&bp->ntp_fltr_lock);
14070 return;
14071 }
14072 hlist_del_rcu(&fltr->base.hash);
14073 bp->ntp_fltr_count--;
14074 spin_unlock_bh(&bp->ntp_fltr_lock);
14075 bnxt_del_l2_filter(bp, fltr->l2_fltr);
14076 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
14077 kfree_rcu(fltr, base.rcu);
14078}
14079
14080static void bnxt_cfg_ntp_filters(struct bnxt *bp)
14081{
14082#ifdef CONFIG_RFS_ACCEL
14083 int i;
14084
14085 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
14086 struct hlist_head *head;
14087 struct hlist_node *tmp;
14088 struct bnxt_ntuple_filter *fltr;
14089 int rc;
14090
14091 head = &bp->ntp_fltr_hash_tbl[i];
14092 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
14093 bool del = false;
14094
14095 if (test_bit(BNXT_FLTR_VALID, &fltr->base.state)) {
14096 if (fltr->base.flags & BNXT_ACT_NO_AGING)
14097 continue;
14098 if (rps_may_expire_flow(bp->dev, fltr->base.rxq,
14099 fltr->flow_id,
14100 fltr->base.sw_id)) {
14101 bnxt_hwrm_cfa_ntuple_filter_free(bp,
14102 fltr);
14103 del = true;
14104 }
14105 } else {
14106 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
14107 fltr);
14108 if (rc)
14109 del = true;
14110 else
14111 set_bit(BNXT_FLTR_VALID, &fltr->base.state);
14112 }
14113
14114 if (del)
14115 bnxt_del_ntp_filter(bp, fltr);
14116 }
14117 }
14118#endif
14119}
14120
14121static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table,
14122 unsigned int entry, struct udp_tunnel_info *ti)
14123{
14124 struct bnxt *bp = netdev_priv(netdev);
14125 unsigned int cmd;
14126
14127 if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
14128 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
14129 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
14130 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE;
14131 else
14132 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE;
14133
14134 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd);
14135}
14136
14137static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table,
14138 unsigned int entry, struct udp_tunnel_info *ti)
14139{
14140 struct bnxt *bp = netdev_priv(netdev);
14141 unsigned int cmd;
14142
14143 if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
14144 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
14145 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
14146 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
14147 else
14148 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE;
14149
14150 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
14151}
14152
14153static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
14154 .set_port = bnxt_udp_tunnel_set_port,
14155 .unset_port = bnxt_udp_tunnel_unset_port,
14156 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
14157 UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
14158 .tables = {
14159 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
14160 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
14161 },
14162}, bnxt_udp_tunnels_p7 = {
14163 .set_port = bnxt_udp_tunnel_set_port,
14164 .unset_port = bnxt_udp_tunnel_unset_port,
14165 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
14166 UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
14167 .tables = {
14168 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
14169 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
14170 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN_GPE, },
14171 },
14172};
14173
14174static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
14175 struct net_device *dev, u32 filter_mask,
14176 int nlflags)
14177{
14178 struct bnxt *bp = netdev_priv(dev);
14179
14180 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
14181 nlflags, filter_mask, NULL);
14182}
14183
14184static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
14185 u16 flags, struct netlink_ext_ack *extack)
14186{
14187 struct bnxt *bp = netdev_priv(dev);
14188 struct nlattr *attr, *br_spec;
14189 int rem, rc = 0;
14190
14191 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
14192 return -EOPNOTSUPP;
14193
14194 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
14195 if (!br_spec)
14196 return -EINVAL;
14197
14198 nla_for_each_nested(attr, br_spec, rem) {
14199 u16 mode;
14200
14201 if (nla_type(attr) != IFLA_BRIDGE_MODE)
14202 continue;
14203
14204 mode = nla_get_u16(attr);
14205 if (mode == bp->br_mode)
14206 break;
14207
14208 rc = bnxt_hwrm_set_br_mode(bp, mode);
14209 if (!rc)
14210 bp->br_mode = mode;
14211 break;
14212 }
14213 return rc;
14214}
14215
14216int bnxt_get_port_parent_id(struct net_device *dev,
14217 struct netdev_phys_item_id *ppid)
14218{
14219 struct bnxt *bp = netdev_priv(dev);
14220
14221 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
14222 return -EOPNOTSUPP;
14223
14224 /* The PF and it's VF-reps only support the switchdev framework */
14225 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
14226 return -EOPNOTSUPP;
14227
14228 ppid->id_len = sizeof(bp->dsn);
14229 memcpy(ppid->id, bp->dsn, ppid->id_len);
14230
14231 return 0;
14232}
14233
14234static const struct net_device_ops bnxt_netdev_ops = {
14235 .ndo_open = bnxt_open,
14236 .ndo_start_xmit = bnxt_start_xmit,
14237 .ndo_stop = bnxt_close,
14238 .ndo_get_stats64 = bnxt_get_stats64,
14239 .ndo_set_rx_mode = bnxt_set_rx_mode,
14240 .ndo_eth_ioctl = bnxt_ioctl,
14241 .ndo_validate_addr = eth_validate_addr,
14242 .ndo_set_mac_address = bnxt_change_mac_addr,
14243 .ndo_change_mtu = bnxt_change_mtu,
14244 .ndo_fix_features = bnxt_fix_features,
14245 .ndo_set_features = bnxt_set_features,
14246 .ndo_features_check = bnxt_features_check,
14247 .ndo_tx_timeout = bnxt_tx_timeout,
14248#ifdef CONFIG_BNXT_SRIOV
14249 .ndo_get_vf_config = bnxt_get_vf_config,
14250 .ndo_set_vf_mac = bnxt_set_vf_mac,
14251 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
14252 .ndo_set_vf_rate = bnxt_set_vf_bw,
14253 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
14254 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
14255 .ndo_set_vf_trust = bnxt_set_vf_trust,
14256#endif
14257 .ndo_setup_tc = bnxt_setup_tc,
14258#ifdef CONFIG_RFS_ACCEL
14259 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
14260#endif
14261 .ndo_bpf = bnxt_xdp,
14262 .ndo_xdp_xmit = bnxt_xdp_xmit,
14263 .ndo_bridge_getlink = bnxt_bridge_getlink,
14264 .ndo_bridge_setlink = bnxt_bridge_setlink,
14265};
14266
14267static void bnxt_remove_one(struct pci_dev *pdev)
14268{
14269 struct net_device *dev = pci_get_drvdata(pdev);
14270 struct bnxt *bp = netdev_priv(dev);
14271
14272 if (BNXT_PF(bp))
14273 bnxt_sriov_disable(bp);
14274
14275 bnxt_rdma_aux_device_uninit(bp);
14276
14277 bnxt_ptp_clear(bp);
14278 unregister_netdev(dev);
14279 bnxt_free_l2_filters(bp, true);
14280 bnxt_free_ntp_fltrs(bp, true);
14281 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14282 /* Flush any pending tasks */
14283 cancel_work_sync(&bp->sp_task);
14284 cancel_delayed_work_sync(&bp->fw_reset_task);
14285 bp->sp_event = 0;
14286
14287 bnxt_dl_fw_reporters_destroy(bp);
14288 bnxt_dl_unregister(bp);
14289 bnxt_shutdown_tc(bp);
14290
14291 bnxt_clear_int_mode(bp);
14292 bnxt_hwrm_func_drv_unrgtr(bp);
14293 bnxt_free_hwrm_resources(bp);
14294 bnxt_hwmon_uninit(bp);
14295 bnxt_ethtool_free(bp);
14296 bnxt_dcb_free(bp);
14297 kfree(bp->ptp_cfg);
14298 bp->ptp_cfg = NULL;
14299 kfree(bp->fw_health);
14300 bp->fw_health = NULL;
14301 bnxt_cleanup_pci(bp);
14302 bnxt_free_ctx_mem(bp);
14303 kfree(bp->rss_indir_tbl);
14304 bp->rss_indir_tbl = NULL;
14305 bnxt_free_port_stats(bp);
14306 free_netdev(dev);
14307}
14308
14309static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
14310{
14311 int rc = 0;
14312 struct bnxt_link_info *link_info = &bp->link_info;
14313
14314 bp->phy_flags = 0;
14315 rc = bnxt_hwrm_phy_qcaps(bp);
14316 if (rc) {
14317 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
14318 rc);
14319 return rc;
14320 }
14321 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
14322 bp->dev->priv_flags |= IFF_SUPP_NOFCS;
14323 else
14324 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
14325 if (!fw_dflt)
14326 return 0;
14327
14328 mutex_lock(&bp->link_lock);
14329 rc = bnxt_update_link(bp, false);
14330 if (rc) {
14331 mutex_unlock(&bp->link_lock);
14332 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
14333 rc);
14334 return rc;
14335 }
14336
14337 /* Older firmware does not have supported_auto_speeds, so assume
14338 * that all supported speeds can be autonegotiated.
14339 */
14340 if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
14341 link_info->support_auto_speeds = link_info->support_speeds;
14342
14343 bnxt_init_ethtool_link_settings(bp);
14344 mutex_unlock(&bp->link_lock);
14345 return 0;
14346}
14347
14348static int bnxt_get_max_irq(struct pci_dev *pdev)
14349{
14350 u16 ctrl;
14351
14352 if (!pdev->msix_cap)
14353 return 1;
14354
14355 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
14356 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
14357}
14358
14359static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
14360 int *max_cp)
14361{
14362 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
14363 int max_ring_grps = 0, max_irq;
14364
14365 *max_tx = hw_resc->max_tx_rings;
14366 *max_rx = hw_resc->max_rx_rings;
14367 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
14368 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
14369 bnxt_get_ulp_msix_num(bp),
14370 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
14371 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
14372 *max_cp = min_t(int, *max_cp, max_irq);
14373 max_ring_grps = hw_resc->max_hw_ring_grps;
14374 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
14375 *max_cp -= 1;
14376 *max_rx -= 2;
14377 }
14378 if (bp->flags & BNXT_FLAG_AGG_RINGS)
14379 *max_rx >>= 1;
14380 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
14381 int rc;
14382
14383 rc = __bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
14384 if (rc) {
14385 *max_rx = 0;
14386 *max_tx = 0;
14387 }
14388 /* On P5 chips, max_cp output param should be available NQs */
14389 *max_cp = max_irq;
14390 }
14391 *max_rx = min_t(int, *max_rx, max_ring_grps);
14392}
14393
14394int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
14395{
14396 int rx, tx, cp;
14397
14398 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
14399 *max_rx = rx;
14400 *max_tx = tx;
14401 if (!rx || !tx || !cp)
14402 return -ENOMEM;
14403
14404 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
14405}
14406
14407static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
14408 bool shared)
14409{
14410 int rc;
14411
14412 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
14413 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
14414 /* Not enough rings, try disabling agg rings. */
14415 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
14416 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
14417 if (rc) {
14418 /* set BNXT_FLAG_AGG_RINGS back for consistency */
14419 bp->flags |= BNXT_FLAG_AGG_RINGS;
14420 return rc;
14421 }
14422 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
14423 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
14424 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
14425 bnxt_set_ring_params(bp);
14426 }
14427
14428 if (bp->flags & BNXT_FLAG_ROCE_CAP) {
14429 int max_cp, max_stat, max_irq;
14430
14431 /* Reserve minimum resources for RoCE */
14432 max_cp = bnxt_get_max_func_cp_rings(bp);
14433 max_stat = bnxt_get_max_func_stat_ctxs(bp);
14434 max_irq = bnxt_get_max_func_irqs(bp);
14435 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
14436 max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
14437 max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
14438 return 0;
14439
14440 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
14441 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
14442 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
14443 max_cp = min_t(int, max_cp, max_irq);
14444 max_cp = min_t(int, max_cp, max_stat);
14445 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
14446 if (rc)
14447 rc = 0;
14448 }
14449 return rc;
14450}
14451
14452/* In initial default shared ring setting, each shared ring must have a
14453 * RX/TX ring pair.
14454 */
14455static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
14456{
14457 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
14458 bp->rx_nr_rings = bp->cp_nr_rings;
14459 bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
14460 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
14461}
14462
14463static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
14464{
14465 int dflt_rings, max_rx_rings, max_tx_rings, rc;
14466
14467 if (!bnxt_can_reserve_rings(bp))
14468 return 0;
14469
14470 if (sh)
14471 bp->flags |= BNXT_FLAG_SHARED_RINGS;
14472 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
14473 /* Reduce default rings on multi-port cards so that total default
14474 * rings do not exceed CPU count.
14475 */
14476 if (bp->port_count > 1) {
14477 int max_rings =
14478 max_t(int, num_online_cpus() / bp->port_count, 1);
14479
14480 dflt_rings = min_t(int, dflt_rings, max_rings);
14481 }
14482 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
14483 if (rc)
14484 return rc;
14485 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
14486 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
14487 if (sh)
14488 bnxt_trim_dflt_sh_rings(bp);
14489 else
14490 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
14491 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
14492
14493 rc = __bnxt_reserve_rings(bp);
14494 if (rc && rc != -ENODEV)
14495 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
14496 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
14497 if (sh)
14498 bnxt_trim_dflt_sh_rings(bp);
14499
14500 /* Rings may have been trimmed, re-reserve the trimmed rings. */
14501 if (bnxt_need_reserve_rings(bp)) {
14502 rc = __bnxt_reserve_rings(bp);
14503 if (rc && rc != -ENODEV)
14504 netdev_warn(bp->dev, "2nd rings reservation failed.\n");
14505 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
14506 }
14507 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
14508 bp->rx_nr_rings++;
14509 bp->cp_nr_rings++;
14510 }
14511 if (rc) {
14512 bp->tx_nr_rings = 0;
14513 bp->rx_nr_rings = 0;
14514 }
14515 return rc;
14516}
14517
14518static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
14519{
14520 int rc;
14521
14522 if (bp->tx_nr_rings)
14523 return 0;
14524
14525 bnxt_ulp_irq_stop(bp);
14526 bnxt_clear_int_mode(bp);
14527 rc = bnxt_set_dflt_rings(bp, true);
14528 if (rc) {
14529 if (BNXT_VF(bp) && rc == -ENODEV)
14530 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
14531 else
14532 netdev_err(bp->dev, "Not enough rings available.\n");
14533 goto init_dflt_ring_err;
14534 }
14535 rc = bnxt_init_int_mode(bp);
14536 if (rc)
14537 goto init_dflt_ring_err;
14538
14539 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
14540
14541 bnxt_set_dflt_rfs(bp);
14542
14543init_dflt_ring_err:
14544 bnxt_ulp_irq_restart(bp, rc);
14545 return rc;
14546}
14547
14548int bnxt_restore_pf_fw_resources(struct bnxt *bp)
14549{
14550 int rc;
14551
14552 ASSERT_RTNL();
14553 bnxt_hwrm_func_qcaps(bp);
14554
14555 if (netif_running(bp->dev))
14556 __bnxt_close_nic(bp, true, false);
14557
14558 bnxt_ulp_irq_stop(bp);
14559 bnxt_clear_int_mode(bp);
14560 rc = bnxt_init_int_mode(bp);
14561 bnxt_ulp_irq_restart(bp, rc);
14562
14563 if (netif_running(bp->dev)) {
14564 if (rc)
14565 dev_close(bp->dev);
14566 else
14567 rc = bnxt_open_nic(bp, true, false);
14568 }
14569
14570 return rc;
14571}
14572
14573static int bnxt_init_mac_addr(struct bnxt *bp)
14574{
14575 int rc = 0;
14576
14577 if (BNXT_PF(bp)) {
14578 eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
14579 } else {
14580#ifdef CONFIG_BNXT_SRIOV
14581 struct bnxt_vf_info *vf = &bp->vf;
14582 bool strict_approval = true;
14583
14584 if (is_valid_ether_addr(vf->mac_addr)) {
14585 /* overwrite netdev dev_addr with admin VF MAC */
14586 eth_hw_addr_set(bp->dev, vf->mac_addr);
14587 /* Older PF driver or firmware may not approve this
14588 * correctly.
14589 */
14590 strict_approval = false;
14591 } else {
14592 eth_hw_addr_random(bp->dev);
14593 }
14594 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
14595#endif
14596 }
14597 return rc;
14598}
14599
14600static void bnxt_vpd_read_info(struct bnxt *bp)
14601{
14602 struct pci_dev *pdev = bp->pdev;
14603 unsigned int vpd_size, kw_len;
14604 int pos, size;
14605 u8 *vpd_data;
14606
14607 vpd_data = pci_vpd_alloc(pdev, &vpd_size);
14608 if (IS_ERR(vpd_data)) {
14609 pci_warn(pdev, "Unable to read VPD\n");
14610 return;
14611 }
14612
14613 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
14614 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
14615 if (pos < 0)
14616 goto read_sn;
14617
14618 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
14619 memcpy(bp->board_partno, &vpd_data[pos], size);
14620
14621read_sn:
14622 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
14623 PCI_VPD_RO_KEYWORD_SERIALNO,
14624 &kw_len);
14625 if (pos < 0)
14626 goto exit;
14627
14628 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
14629 memcpy(bp->board_serialno, &vpd_data[pos], size);
14630exit:
14631 kfree(vpd_data);
14632}
14633
14634static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
14635{
14636 struct pci_dev *pdev = bp->pdev;
14637 u64 qword;
14638
14639 qword = pci_get_dsn(pdev);
14640 if (!qword) {
14641 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
14642 return -EOPNOTSUPP;
14643 }
14644
14645 put_unaligned_le64(qword, dsn);
14646
14647 bp->flags |= BNXT_FLAG_DSN_VALID;
14648 return 0;
14649}
14650
14651static int bnxt_map_db_bar(struct bnxt *bp)
14652{
14653 if (!bp->db_size)
14654 return -ENODEV;
14655 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
14656 if (!bp->bar1)
14657 return -ENOMEM;
14658 return 0;
14659}
14660
14661void bnxt_print_device_info(struct bnxt *bp)
14662{
14663 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
14664 board_info[bp->board_idx].name,
14665 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
14666
14667 pcie_print_link_status(bp->pdev);
14668}
14669
14670static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
14671{
14672 struct net_device *dev;
14673 struct bnxt *bp;
14674 int rc, max_irqs;
14675
14676 if (pci_is_bridge(pdev))
14677 return -ENODEV;
14678
14679 /* Clear any pending DMA transactions from crash kernel
14680 * while loading driver in capture kernel.
14681 */
14682 if (is_kdump_kernel()) {
14683 pci_clear_master(pdev);
14684 pcie_flr(pdev);
14685 }
14686
14687 max_irqs = bnxt_get_max_irq(pdev);
14688 dev = alloc_etherdev_mqs(sizeof(*bp), max_irqs * BNXT_MAX_QUEUE,
14689 max_irqs);
14690 if (!dev)
14691 return -ENOMEM;
14692
14693 bp = netdev_priv(dev);
14694 bp->board_idx = ent->driver_data;
14695 bp->msg_enable = BNXT_DEF_MSG_ENABLE;
14696 bnxt_set_max_func_irqs(bp, max_irqs);
14697
14698 if (bnxt_vf_pciid(bp->board_idx))
14699 bp->flags |= BNXT_FLAG_VF;
14700
14701 /* No devlink port registration in case of a VF */
14702 if (BNXT_PF(bp))
14703 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
14704
14705 if (pdev->msix_cap)
14706 bp->flags |= BNXT_FLAG_MSIX_CAP;
14707
14708 rc = bnxt_init_board(pdev, dev);
14709 if (rc < 0)
14710 goto init_err_free;
14711
14712 dev->netdev_ops = &bnxt_netdev_ops;
14713 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
14714 dev->ethtool_ops = &bnxt_ethtool_ops;
14715 pci_set_drvdata(pdev, dev);
14716
14717 rc = bnxt_alloc_hwrm_resources(bp);
14718 if (rc)
14719 goto init_err_pci_clean;
14720
14721 mutex_init(&bp->hwrm_cmd_lock);
14722 mutex_init(&bp->link_lock);
14723
14724 rc = bnxt_fw_init_one_p1(bp);
14725 if (rc)
14726 goto init_err_pci_clean;
14727
14728 if (BNXT_PF(bp))
14729 bnxt_vpd_read_info(bp);
14730
14731 if (BNXT_CHIP_P5_PLUS(bp)) {
14732 bp->flags |= BNXT_FLAG_CHIP_P5_PLUS;
14733 if (BNXT_CHIP_P7(bp))
14734 bp->flags |= BNXT_FLAG_CHIP_P7;
14735 }
14736
14737 rc = bnxt_alloc_rss_indir_tbl(bp);
14738 if (rc)
14739 goto init_err_pci_clean;
14740
14741 rc = bnxt_fw_init_one_p2(bp);
14742 if (rc)
14743 goto init_err_pci_clean;
14744
14745 rc = bnxt_map_db_bar(bp);
14746 if (rc) {
14747 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
14748 rc);
14749 goto init_err_pci_clean;
14750 }
14751
14752 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
14753 NETIF_F_TSO | NETIF_F_TSO6 |
14754 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
14755 NETIF_F_GSO_IPXIP4 |
14756 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
14757 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
14758 NETIF_F_RXCSUM | NETIF_F_GRO;
14759 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
14760 dev->hw_features |= NETIF_F_GSO_UDP_L4;
14761
14762 if (BNXT_SUPPORTS_TPA(bp))
14763 dev->hw_features |= NETIF_F_LRO;
14764
14765 dev->hw_enc_features =
14766 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
14767 NETIF_F_TSO | NETIF_F_TSO6 |
14768 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
14769 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
14770 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
14771 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
14772 dev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
14773 if (bp->flags & BNXT_FLAG_CHIP_P7)
14774 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels_p7;
14775 else
14776 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
14777
14778 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
14779 NETIF_F_GSO_GRE_CSUM;
14780 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
14781 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
14782 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
14783 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
14784 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
14785 if (BNXT_SUPPORTS_TPA(bp))
14786 dev->hw_features |= NETIF_F_GRO_HW;
14787 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
14788 if (dev->features & NETIF_F_GRO_HW)
14789 dev->features &= ~NETIF_F_LRO;
14790 dev->priv_flags |= IFF_UNICAST_FLT;
14791
14792 netif_set_tso_max_size(dev, GSO_MAX_SIZE);
14793
14794 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
14795 NETDEV_XDP_ACT_RX_SG;
14796
14797#ifdef CONFIG_BNXT_SRIOV
14798 init_waitqueue_head(&bp->sriov_cfg_wait);
14799#endif
14800 if (BNXT_SUPPORTS_TPA(bp)) {
14801 bp->gro_func = bnxt_gro_func_5730x;
14802 if (BNXT_CHIP_P4(bp))
14803 bp->gro_func = bnxt_gro_func_5731x;
14804 else if (BNXT_CHIP_P5_PLUS(bp))
14805 bp->gro_func = bnxt_gro_func_5750x;
14806 }
14807 if (!BNXT_CHIP_P4_PLUS(bp))
14808 bp->flags |= BNXT_FLAG_DOUBLE_DB;
14809
14810 rc = bnxt_init_mac_addr(bp);
14811 if (rc) {
14812 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
14813 rc = -EADDRNOTAVAIL;
14814 goto init_err_pci_clean;
14815 }
14816
14817 if (BNXT_PF(bp)) {
14818 /* Read the adapter's DSN to use as the eswitch switch_id */
14819 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
14820 }
14821
14822 /* MTU range: 60 - FW defined max */
14823 dev->min_mtu = ETH_ZLEN;
14824 dev->max_mtu = bp->max_mtu;
14825
14826 rc = bnxt_probe_phy(bp, true);
14827 if (rc)
14828 goto init_err_pci_clean;
14829
14830 bnxt_init_l2_fltr_tbl(bp);
14831 bnxt_set_rx_skb_mode(bp, false);
14832 bnxt_set_tpa_flags(bp);
14833 bnxt_set_ring_params(bp);
14834 rc = bnxt_set_dflt_rings(bp, true);
14835 if (rc) {
14836 if (BNXT_VF(bp) && rc == -ENODEV) {
14837 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
14838 } else {
14839 netdev_err(bp->dev, "Not enough rings available.\n");
14840 rc = -ENOMEM;
14841 }
14842 goto init_err_pci_clean;
14843 }
14844
14845 bnxt_fw_init_one_p3(bp);
14846
14847 bnxt_init_dflt_coal(bp);
14848
14849 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
14850 bp->flags |= BNXT_FLAG_STRIP_VLAN;
14851
14852 rc = bnxt_init_int_mode(bp);
14853 if (rc)
14854 goto init_err_pci_clean;
14855
14856 /* No TC has been set yet and rings may have been trimmed due to
14857 * limited MSIX, so we re-initialize the TX rings per TC.
14858 */
14859 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
14860
14861 if (BNXT_PF(bp)) {
14862 if (!bnxt_pf_wq) {
14863 bnxt_pf_wq =
14864 create_singlethread_workqueue("bnxt_pf_wq");
14865 if (!bnxt_pf_wq) {
14866 dev_err(&pdev->dev, "Unable to create workqueue.\n");
14867 rc = -ENOMEM;
14868 goto init_err_pci_clean;
14869 }
14870 }
14871 rc = bnxt_init_tc(bp);
14872 if (rc)
14873 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
14874 rc);
14875 }
14876
14877 bnxt_inv_fw_health_reg(bp);
14878 rc = bnxt_dl_register(bp);
14879 if (rc)
14880 goto init_err_dl;
14881
14882 rc = register_netdev(dev);
14883 if (rc)
14884 goto init_err_cleanup;
14885
14886 bnxt_dl_fw_reporters_create(bp);
14887
14888 bnxt_rdma_aux_device_init(bp);
14889
14890 bnxt_print_device_info(bp);
14891
14892 pci_save_state(pdev);
14893
14894 return 0;
14895init_err_cleanup:
14896 bnxt_dl_unregister(bp);
14897init_err_dl:
14898 bnxt_shutdown_tc(bp);
14899 bnxt_clear_int_mode(bp);
14900
14901init_err_pci_clean:
14902 bnxt_hwrm_func_drv_unrgtr(bp);
14903 bnxt_free_hwrm_resources(bp);
14904 bnxt_hwmon_uninit(bp);
14905 bnxt_ethtool_free(bp);
14906 bnxt_ptp_clear(bp);
14907 kfree(bp->ptp_cfg);
14908 bp->ptp_cfg = NULL;
14909 kfree(bp->fw_health);
14910 bp->fw_health = NULL;
14911 bnxt_cleanup_pci(bp);
14912 bnxt_free_ctx_mem(bp);
14913 kfree(bp->rss_indir_tbl);
14914 bp->rss_indir_tbl = NULL;
14915
14916init_err_free:
14917 free_netdev(dev);
14918 return rc;
14919}
14920
14921static void bnxt_shutdown(struct pci_dev *pdev)
14922{
14923 struct net_device *dev = pci_get_drvdata(pdev);
14924 struct bnxt *bp;
14925
14926 if (!dev)
14927 return;
14928
14929 rtnl_lock();
14930 bp = netdev_priv(dev);
14931 if (!bp)
14932 goto shutdown_exit;
14933
14934 if (netif_running(dev))
14935 dev_close(dev);
14936
14937 bnxt_clear_int_mode(bp);
14938 pci_disable_device(pdev);
14939
14940 if (system_state == SYSTEM_POWER_OFF) {
14941 pci_wake_from_d3(pdev, bp->wol);
14942 pci_set_power_state(pdev, PCI_D3hot);
14943 }
14944
14945shutdown_exit:
14946 rtnl_unlock();
14947}
14948
14949#ifdef CONFIG_PM_SLEEP
14950static int bnxt_suspend(struct device *device)
14951{
14952 struct net_device *dev = dev_get_drvdata(device);
14953 struct bnxt *bp = netdev_priv(dev);
14954 int rc = 0;
14955
14956 rtnl_lock();
14957 bnxt_ulp_stop(bp);
14958 if (netif_running(dev)) {
14959 netif_device_detach(dev);
14960 rc = bnxt_close(dev);
14961 }
14962 bnxt_hwrm_func_drv_unrgtr(bp);
14963 pci_disable_device(bp->pdev);
14964 bnxt_free_ctx_mem(bp);
14965 rtnl_unlock();
14966 return rc;
14967}
14968
14969static int bnxt_resume(struct device *device)
14970{
14971 struct net_device *dev = dev_get_drvdata(device);
14972 struct bnxt *bp = netdev_priv(dev);
14973 int rc = 0;
14974
14975 rtnl_lock();
14976 rc = pci_enable_device(bp->pdev);
14977 if (rc) {
14978 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
14979 rc);
14980 goto resume_exit;
14981 }
14982 pci_set_master(bp->pdev);
14983 if (bnxt_hwrm_ver_get(bp)) {
14984 rc = -ENODEV;
14985 goto resume_exit;
14986 }
14987 rc = bnxt_hwrm_func_reset(bp);
14988 if (rc) {
14989 rc = -EBUSY;
14990 goto resume_exit;
14991 }
14992
14993 rc = bnxt_hwrm_func_qcaps(bp);
14994 if (rc)
14995 goto resume_exit;
14996
14997 bnxt_clear_reservations(bp, true);
14998
14999 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
15000 rc = -ENODEV;
15001 goto resume_exit;
15002 }
15003
15004 bnxt_get_wol_settings(bp);
15005 if (netif_running(dev)) {
15006 rc = bnxt_open(dev);
15007 if (!rc)
15008 netif_device_attach(dev);
15009 }
15010
15011resume_exit:
15012 bnxt_ulp_start(bp, rc);
15013 if (!rc)
15014 bnxt_reenable_sriov(bp);
15015 rtnl_unlock();
15016 return rc;
15017}
15018
15019static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
15020#define BNXT_PM_OPS (&bnxt_pm_ops)
15021
15022#else
15023
15024#define BNXT_PM_OPS NULL
15025
15026#endif /* CONFIG_PM_SLEEP */
15027
15028/**
15029 * bnxt_io_error_detected - called when PCI error is detected
15030 * @pdev: Pointer to PCI device
15031 * @state: The current pci connection state
15032 *
15033 * This function is called after a PCI bus error affecting
15034 * this device has been detected.
15035 */
15036static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
15037 pci_channel_state_t state)
15038{
15039 struct net_device *netdev = pci_get_drvdata(pdev);
15040 struct bnxt *bp = netdev_priv(netdev);
15041
15042 netdev_info(netdev, "PCI I/O error detected\n");
15043
15044 rtnl_lock();
15045 netif_device_detach(netdev);
15046
15047 bnxt_ulp_stop(bp);
15048
15049 if (state == pci_channel_io_perm_failure) {
15050 rtnl_unlock();
15051 return PCI_ERS_RESULT_DISCONNECT;
15052 }
15053
15054 if (state == pci_channel_io_frozen)
15055 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
15056
15057 if (netif_running(netdev))
15058 bnxt_close(netdev);
15059
15060 if (pci_is_enabled(pdev))
15061 pci_disable_device(pdev);
15062 bnxt_free_ctx_mem(bp);
15063 rtnl_unlock();
15064
15065 /* Request a slot slot reset. */
15066 return PCI_ERS_RESULT_NEED_RESET;
15067}
15068
15069/**
15070 * bnxt_io_slot_reset - called after the pci bus has been reset.
15071 * @pdev: Pointer to PCI device
15072 *
15073 * Restart the card from scratch, as if from a cold-boot.
15074 * At this point, the card has exprienced a hard reset,
15075 * followed by fixups by BIOS, and has its config space
15076 * set up identically to what it was at cold boot.
15077 */
15078static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
15079{
15080 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
15081 struct net_device *netdev = pci_get_drvdata(pdev);
15082 struct bnxt *bp = netdev_priv(netdev);
15083 int retry = 0;
15084 int err = 0;
15085 int off;
15086
15087 netdev_info(bp->dev, "PCI Slot Reset\n");
15088
15089 rtnl_lock();
15090
15091 if (pci_enable_device(pdev)) {
15092 dev_err(&pdev->dev,
15093 "Cannot re-enable PCI device after reset.\n");
15094 } else {
15095 pci_set_master(pdev);
15096 /* Upon fatal error, our device internal logic that latches to
15097 * BAR value is getting reset and will restore only upon
15098 * rewritting the BARs.
15099 *
15100 * As pci_restore_state() does not re-write the BARs if the
15101 * value is same as saved value earlier, driver needs to
15102 * write the BARs to 0 to force restore, in case of fatal error.
15103 */
15104 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
15105 &bp->state)) {
15106 for (off = PCI_BASE_ADDRESS_0;
15107 off <= PCI_BASE_ADDRESS_5; off += 4)
15108 pci_write_config_dword(bp->pdev, off, 0);
15109 }
15110 pci_restore_state(pdev);
15111 pci_save_state(pdev);
15112
15113 bnxt_inv_fw_health_reg(bp);
15114 bnxt_try_map_fw_health_reg(bp);
15115
15116 /* In some PCIe AER scenarios, firmware may take up to
15117 * 10 seconds to become ready in the worst case.
15118 */
15119 do {
15120 err = bnxt_try_recover_fw(bp);
15121 if (!err)
15122 break;
15123 retry++;
15124 } while (retry < BNXT_FW_SLOT_RESET_RETRY);
15125
15126 if (err) {
15127 dev_err(&pdev->dev, "Firmware not ready\n");
15128 goto reset_exit;
15129 }
15130
15131 err = bnxt_hwrm_func_reset(bp);
15132 if (!err)
15133 result = PCI_ERS_RESULT_RECOVERED;
15134
15135 bnxt_ulp_irq_stop(bp);
15136 bnxt_clear_int_mode(bp);
15137 err = bnxt_init_int_mode(bp);
15138 bnxt_ulp_irq_restart(bp, err);
15139 }
15140
15141reset_exit:
15142 bnxt_clear_reservations(bp, true);
15143 rtnl_unlock();
15144
15145 return result;
15146}
15147
15148/**
15149 * bnxt_io_resume - called when traffic can start flowing again.
15150 * @pdev: Pointer to PCI device
15151 *
15152 * This callback is called when the error recovery driver tells
15153 * us that its OK to resume normal operation.
15154 */
15155static void bnxt_io_resume(struct pci_dev *pdev)
15156{
15157 struct net_device *netdev = pci_get_drvdata(pdev);
15158 struct bnxt *bp = netdev_priv(netdev);
15159 int err;
15160
15161 netdev_info(bp->dev, "PCI Slot Resume\n");
15162 rtnl_lock();
15163
15164 err = bnxt_hwrm_func_qcaps(bp);
15165 if (!err && netif_running(netdev))
15166 err = bnxt_open(netdev);
15167
15168 bnxt_ulp_start(bp, err);
15169 if (!err) {
15170 bnxt_reenable_sriov(bp);
15171 netif_device_attach(netdev);
15172 }
15173
15174 rtnl_unlock();
15175}
15176
15177static const struct pci_error_handlers bnxt_err_handler = {
15178 .error_detected = bnxt_io_error_detected,
15179 .slot_reset = bnxt_io_slot_reset,
15180 .resume = bnxt_io_resume
15181};
15182
15183static struct pci_driver bnxt_pci_driver = {
15184 .name = DRV_MODULE_NAME,
15185 .id_table = bnxt_pci_tbl,
15186 .probe = bnxt_init_one,
15187 .remove = bnxt_remove_one,
15188 .shutdown = bnxt_shutdown,
15189 .driver.pm = BNXT_PM_OPS,
15190 .err_handler = &bnxt_err_handler,
15191#if defined(CONFIG_BNXT_SRIOV)
15192 .sriov_configure = bnxt_sriov_configure,
15193#endif
15194};
15195
15196static int __init bnxt_init(void)
15197{
15198 int err;
15199
15200 bnxt_debug_init();
15201 err = pci_register_driver(&bnxt_pci_driver);
15202 if (err) {
15203 bnxt_debug_exit();
15204 return err;
15205 }
15206
15207 return 0;
15208}
15209
15210static void __exit bnxt_exit(void)
15211{
15212 pci_unregister_driver(&bnxt_pci_driver);
15213 if (bnxt_pf_wq)
15214 destroy_workqueue(bnxt_pf_wq);
15215 bnxt_debug_exit();
15216}
15217
15218module_init(bnxt_init);
15219module_exit(bnxt_exit);