Loading...
1// SPDX-License-Identifier: GPL-2.0
2/* Marvell Octeon EP (EndPoint) Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8#include <linux/types.h>
9#include <linux/module.h>
10#include <linux/pci.h>
11#include <linux/netdevice.h>
12#include <linux/etherdevice.h>
13#include <linux/rtnetlink.h>
14#include <linux/vmalloc.h>
15
16#include "octep_config.h"
17#include "octep_main.h"
18#include "octep_ctrl_net.h"
19#include "octep_pfvf_mbox.h"
20
21#define OCTEP_INTR_POLL_TIME_MSECS 100
22struct workqueue_struct *octep_wq;
23
24/* Supported Devices */
25static const struct pci_device_id octep_pci_id_tbl[] = {
26 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN98_PF)},
27 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
28 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_PF)},
29 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_PF)},
30 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_PF)},
31 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_PF)},
32 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_PF)},
33 {0, },
34};
35MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
36
37MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
38MODULE_DESCRIPTION(OCTEP_DRV_STRING);
39MODULE_LICENSE("GPL");
40
41/**
42 * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
43 *
44 * @oct: Octeon device private data structure.
45 *
46 * Allocate resources to hold per Tx/Rx queue interrupt info.
47 * This is the information passed to interrupt handler, from which napi poll
48 * is scheduled and includes quick access to private data of Tx/Rx queue
49 * corresponding to the interrupt being handled.
50 *
51 * Return: 0, on successful allocation of resources for all queue interrupts.
52 * -1, if failed to allocate any resource.
53 */
54static int octep_alloc_ioq_vectors(struct octep_device *oct)
55{
56 int i;
57 struct octep_ioq_vector *ioq_vector;
58
59 for (i = 0; i < oct->num_oqs; i++) {
60 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
61 if (!oct->ioq_vector[i])
62 goto free_ioq_vector;
63
64 ioq_vector = oct->ioq_vector[i];
65 ioq_vector->iq = oct->iq[i];
66 ioq_vector->oq = oct->oq[i];
67 ioq_vector->octep_dev = oct;
68 }
69
70 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
71 return 0;
72
73free_ioq_vector:
74 while (i) {
75 i--;
76 vfree(oct->ioq_vector[i]);
77 oct->ioq_vector[i] = NULL;
78 }
79 return -1;
80}
81
82/**
83 * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
84 *
85 * @oct: Octeon device private data structure.
86 */
87static void octep_free_ioq_vectors(struct octep_device *oct)
88{
89 int i;
90
91 for (i = 0; i < oct->num_oqs; i++) {
92 if (oct->ioq_vector[i]) {
93 vfree(oct->ioq_vector[i]);
94 oct->ioq_vector[i] = NULL;
95 }
96 }
97 netdev_info(oct->netdev, "Freed IOQ Vectors\n");
98}
99
100/**
101 * octep_enable_msix_range() - enable MSI-x interrupts.
102 *
103 * @oct: Octeon device private data structure.
104 *
105 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
106 * for the Octeon device.
107 *
108 * Return: 0, on successfully enabling all MSI-x interrupts.
109 * -1, if failed to enable any MSI-x interrupt.
110 */
111static int octep_enable_msix_range(struct octep_device *oct)
112{
113 int num_msix, msix_allocated;
114 int i;
115
116 /* Generic interrupts apart from input/output queues */
117 num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
118 oct->msix_entries = kcalloc(num_msix,
119 sizeof(struct msix_entry), GFP_KERNEL);
120 if (!oct->msix_entries)
121 goto msix_alloc_err;
122
123 for (i = 0; i < num_msix; i++)
124 oct->msix_entries[i].entry = i;
125
126 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
127 num_msix, num_msix);
128 if (msix_allocated != num_msix) {
129 dev_err(&oct->pdev->dev,
130 "Failed to enable %d msix irqs; got only %d\n",
131 num_msix, msix_allocated);
132 goto enable_msix_err;
133 }
134 oct->num_irqs = msix_allocated;
135 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
136
137 return 0;
138
139enable_msix_err:
140 if (msix_allocated > 0)
141 pci_disable_msix(oct->pdev);
142 kfree(oct->msix_entries);
143 oct->msix_entries = NULL;
144msix_alloc_err:
145 return -1;
146}
147
148/**
149 * octep_disable_msix() - disable MSI-x interrupts.
150 *
151 * @oct: Octeon device private data structure.
152 *
153 * Disable MSI-x on the Octeon device.
154 */
155static void octep_disable_msix(struct octep_device *oct)
156{
157 pci_disable_msix(oct->pdev);
158 kfree(oct->msix_entries);
159 oct->msix_entries = NULL;
160 dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
161}
162
163/**
164 * octep_mbox_intr_handler() - common handler for pfvf mbox interrupts.
165 *
166 * @irq: Interrupt number.
167 * @data: interrupt data.
168 *
169 * this is common handler for pfvf mbox interrupts.
170 */
171static irqreturn_t octep_mbox_intr_handler(int irq, void *data)
172{
173 struct octep_device *oct = data;
174
175 return oct->hw_ops.mbox_intr_handler(oct);
176}
177
178/**
179 * octep_oei_intr_handler() - common handler for output endpoint interrupts.
180 *
181 * @irq: Interrupt number.
182 * @data: interrupt data.
183 *
184 * this is common handler for all output endpoint interrupts.
185 */
186static irqreturn_t octep_oei_intr_handler(int irq, void *data)
187{
188 struct octep_device *oct = data;
189
190 return oct->hw_ops.oei_intr_handler(oct);
191}
192
193/**
194 * octep_ire_intr_handler() - common handler for input ring error interrupts.
195 *
196 * @irq: Interrupt number.
197 * @data: interrupt data.
198 *
199 * this is common handler for input ring error interrupts.
200 */
201static irqreturn_t octep_ire_intr_handler(int irq, void *data)
202{
203 struct octep_device *oct = data;
204
205 return oct->hw_ops.ire_intr_handler(oct);
206}
207
208/**
209 * octep_ore_intr_handler() - common handler for output ring error interrupts.
210 *
211 * @irq: Interrupt number.
212 * @data: interrupt data.
213 *
214 * this is common handler for output ring error interrupts.
215 */
216static irqreturn_t octep_ore_intr_handler(int irq, void *data)
217{
218 struct octep_device *oct = data;
219
220 return oct->hw_ops.ore_intr_handler(oct);
221}
222
223/**
224 * octep_vfire_intr_handler() - common handler for vf input ring error interrupts.
225 *
226 * @irq: Interrupt number.
227 * @data: interrupt data.
228 *
229 * this is common handler for vf input ring error interrupts.
230 */
231static irqreturn_t octep_vfire_intr_handler(int irq, void *data)
232{
233 struct octep_device *oct = data;
234
235 return oct->hw_ops.vfire_intr_handler(oct);
236}
237
238/**
239 * octep_vfore_intr_handler() - common handler for vf output ring error interrupts.
240 *
241 * @irq: Interrupt number.
242 * @data: interrupt data.
243 *
244 * this is common handler for vf output ring error interrupts.
245 */
246static irqreturn_t octep_vfore_intr_handler(int irq, void *data)
247{
248 struct octep_device *oct = data;
249
250 return oct->hw_ops.vfore_intr_handler(oct);
251}
252
253/**
254 * octep_dma_intr_handler() - common handler for dpi dma related interrupts.
255 *
256 * @irq: Interrupt number.
257 * @data: interrupt data.
258 *
259 * this is common handler for dpi dma related interrupts.
260 */
261static irqreturn_t octep_dma_intr_handler(int irq, void *data)
262{
263 struct octep_device *oct = data;
264
265 return oct->hw_ops.dma_intr_handler(oct);
266}
267
268/**
269 * octep_dma_vf_intr_handler() - common handler for dpi dma transaction error interrupts for VFs.
270 *
271 * @irq: Interrupt number.
272 * @data: interrupt data.
273 *
274 * this is common handler for dpi dma transaction error interrupts for VFs.
275 */
276static irqreturn_t octep_dma_vf_intr_handler(int irq, void *data)
277{
278 struct octep_device *oct = data;
279
280 return oct->hw_ops.dma_vf_intr_handler(oct);
281}
282
283/**
284 * octep_pp_vf_intr_handler() - common handler for pp transaction error interrupts for VFs.
285 *
286 * @irq: Interrupt number.
287 * @data: interrupt data.
288 *
289 * this is common handler for pp transaction error interrupts for VFs.
290 */
291static irqreturn_t octep_pp_vf_intr_handler(int irq, void *data)
292{
293 struct octep_device *oct = data;
294
295 return oct->hw_ops.pp_vf_intr_handler(oct);
296}
297
298/**
299 * octep_misc_intr_handler() - common handler for mac related interrupts.
300 *
301 * @irq: Interrupt number.
302 * @data: interrupt data.
303 *
304 * this is common handler for mac related interrupts.
305 */
306static irqreturn_t octep_misc_intr_handler(int irq, void *data)
307{
308 struct octep_device *oct = data;
309
310 return oct->hw_ops.misc_intr_handler(oct);
311}
312
313/**
314 * octep_rsvd_intr_handler() - common handler for reserved interrupts (future use).
315 *
316 * @irq: Interrupt number.
317 * @data: interrupt data.
318 *
319 * this is common handler for all reserved interrupts.
320 */
321static irqreturn_t octep_rsvd_intr_handler(int irq, void *data)
322{
323 struct octep_device *oct = data;
324
325 return oct->hw_ops.rsvd_intr_handler(oct);
326}
327
328/**
329 * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
330 *
331 * @irq: Interrupt number.
332 * @data: interrupt data contains pointers to Tx/Rx queue private data
333 * and correspong NAPI context.
334 *
335 * this is common handler for all non-queue (generic) interrupts.
336 */
337static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
338{
339 struct octep_ioq_vector *ioq_vector = data;
340 struct octep_device *oct = ioq_vector->octep_dev;
341
342 return oct->hw_ops.ioq_intr_handler(ioq_vector);
343}
344
345/**
346 * octep_request_irqs() - Register interrupt handlers.
347 *
348 * @oct: Octeon device private data structure.
349 *
350 * Register handlers for all queue and non-queue interrupts.
351 *
352 * Return: 0, on successful registration of all interrupt handlers.
353 * -1, on any error.
354 */
355static int octep_request_irqs(struct octep_device *oct)
356{
357 struct net_device *netdev = oct->netdev;
358 struct octep_ioq_vector *ioq_vector;
359 struct msix_entry *msix_entry;
360 char **non_ioq_msix_names;
361 int num_non_ioq_msix;
362 int ret, i, j;
363
364 num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
365 non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);
366
367 oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix,
368 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
369 if (!oct->non_ioq_irq_names)
370 goto alloc_err;
371
372 /* First few MSI-X interrupts are non-queue interrupts */
373 for (i = 0; i < num_non_ioq_msix; i++) {
374 char *irq_name;
375
376 irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
377 msix_entry = &oct->msix_entries[i];
378
379 snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
380 "%s-%s", netdev->name, non_ioq_msix_names[i]);
381 if (!strncmp(non_ioq_msix_names[i], "epf_mbox_rint", strlen("epf_mbox_rint"))) {
382 ret = request_irq(msix_entry->vector,
383 octep_mbox_intr_handler, 0,
384 irq_name, oct);
385 } else if (!strncmp(non_ioq_msix_names[i], "epf_oei_rint",
386 strlen("epf_oei_rint"))) {
387 ret = request_irq(msix_entry->vector,
388 octep_oei_intr_handler, 0,
389 irq_name, oct);
390 } else if (!strncmp(non_ioq_msix_names[i], "epf_ire_rint",
391 strlen("epf_ire_rint"))) {
392 ret = request_irq(msix_entry->vector,
393 octep_ire_intr_handler, 0,
394 irq_name, oct);
395 } else if (!strncmp(non_ioq_msix_names[i], "epf_ore_rint",
396 strlen("epf_ore_rint"))) {
397 ret = request_irq(msix_entry->vector,
398 octep_ore_intr_handler, 0,
399 irq_name, oct);
400 } else if (!strncmp(non_ioq_msix_names[i], "epf_vfire_rint",
401 strlen("epf_vfire_rint"))) {
402 ret = request_irq(msix_entry->vector,
403 octep_vfire_intr_handler, 0,
404 irq_name, oct);
405 } else if (!strncmp(non_ioq_msix_names[i], "epf_vfore_rint",
406 strlen("epf_vfore_rint"))) {
407 ret = request_irq(msix_entry->vector,
408 octep_vfore_intr_handler, 0,
409 irq_name, oct);
410 } else if (!strncmp(non_ioq_msix_names[i], "epf_dma_rint",
411 strlen("epf_dma_rint"))) {
412 ret = request_irq(msix_entry->vector,
413 octep_dma_intr_handler, 0,
414 irq_name, oct);
415 } else if (!strncmp(non_ioq_msix_names[i], "epf_dma_vf_rint",
416 strlen("epf_dma_vf_rint"))) {
417 ret = request_irq(msix_entry->vector,
418 octep_dma_vf_intr_handler, 0,
419 irq_name, oct);
420 } else if (!strncmp(non_ioq_msix_names[i], "epf_pp_vf_rint",
421 strlen("epf_pp_vf_rint"))) {
422 ret = request_irq(msix_entry->vector,
423 octep_pp_vf_intr_handler, 0,
424 irq_name, oct);
425 } else if (!strncmp(non_ioq_msix_names[i], "epf_misc_rint",
426 strlen("epf_misc_rint"))) {
427 ret = request_irq(msix_entry->vector,
428 octep_misc_intr_handler, 0,
429 irq_name, oct);
430 } else {
431 ret = request_irq(msix_entry->vector,
432 octep_rsvd_intr_handler, 0,
433 irq_name, oct);
434 }
435
436 if (ret) {
437 netdev_err(netdev,
438 "request_irq failed for %s; err=%d",
439 irq_name, ret);
440 goto non_ioq_irq_err;
441 }
442 }
443
444 /* Request IRQs for Tx/Rx queues */
445 for (j = 0; j < oct->num_oqs; j++) {
446 ioq_vector = oct->ioq_vector[j];
447 msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
448
449 snprintf(ioq_vector->name, sizeof(ioq_vector->name),
450 "%s-q%d", netdev->name, j);
451 ret = request_irq(msix_entry->vector,
452 octep_ioq_intr_handler, 0,
453 ioq_vector->name, ioq_vector);
454 if (ret) {
455 netdev_err(netdev,
456 "request_irq failed for Q-%d; err=%d",
457 j, ret);
458 goto ioq_irq_err;
459 }
460
461 cpumask_set_cpu(j % num_online_cpus(),
462 &ioq_vector->affinity_mask);
463 irq_set_affinity_hint(msix_entry->vector,
464 &ioq_vector->affinity_mask);
465 }
466
467 return 0;
468ioq_irq_err:
469 while (j) {
470 --j;
471 ioq_vector = oct->ioq_vector[j];
472 msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
473
474 irq_set_affinity_hint(msix_entry->vector, NULL);
475 free_irq(msix_entry->vector, ioq_vector);
476 }
477non_ioq_irq_err:
478 while (i) {
479 --i;
480 free_irq(oct->msix_entries[i].vector, oct);
481 }
482 kfree(oct->non_ioq_irq_names);
483 oct->non_ioq_irq_names = NULL;
484alloc_err:
485 return -1;
486}
487
488/**
489 * octep_free_irqs() - free all registered interrupts.
490 *
491 * @oct: Octeon device private data structure.
492 *
493 * Free all queue and non-queue interrupts of the Octeon device.
494 */
495static void octep_free_irqs(struct octep_device *oct)
496{
497 int i;
498
499 /* First few MSI-X interrupts are non queue interrupts; free them */
500 for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
501 free_irq(oct->msix_entries[i].vector, oct);
502 kfree(oct->non_ioq_irq_names);
503
504 /* Free IRQs for Input/Output (Tx/Rx) queues */
505 for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
506 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
507 free_irq(oct->msix_entries[i].vector,
508 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
509 }
510 netdev_info(oct->netdev, "IRQs freed\n");
511}
512
513/**
514 * octep_setup_irqs() - setup interrupts for the Octeon device.
515 *
516 * @oct: Octeon device private data structure.
517 *
518 * Allocate data structures to hold per interrupt information, allocate/enable
519 * MSI-x interrupt and register interrupt handlers.
520 *
521 * Return: 0, on successful allocation and registration of all interrupts.
522 * -1, on any error.
523 */
524static int octep_setup_irqs(struct octep_device *oct)
525{
526 if (octep_alloc_ioq_vectors(oct))
527 goto ioq_vector_err;
528
529 if (octep_enable_msix_range(oct))
530 goto enable_msix_err;
531
532 if (octep_request_irqs(oct))
533 goto request_irq_err;
534
535 return 0;
536
537request_irq_err:
538 octep_disable_msix(oct);
539enable_msix_err:
540 octep_free_ioq_vectors(oct);
541ioq_vector_err:
542 return -1;
543}
544
545/**
546 * octep_clean_irqs() - free all interrupts and its resources.
547 *
548 * @oct: Octeon device private data structure.
549 */
550static void octep_clean_irqs(struct octep_device *oct)
551{
552 octep_free_irqs(oct);
553 octep_disable_msix(oct);
554 octep_free_ioq_vectors(oct);
555}
556
557/**
558 * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
559 *
560 * @iq: Octeon Tx queue data structure.
561 * @oq: Octeon Rx queue data structure.
562 */
563static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq)
564{
565 u32 pkts_pend = oq->pkts_pending;
566
567 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
568 if (iq->pkts_processed) {
569 writel(iq->pkts_processed, iq->inst_cnt_reg);
570 iq->pkt_in_done -= iq->pkts_processed;
571 iq->pkts_processed = 0;
572 }
573 if (oq->last_pkt_count - pkts_pend) {
574 writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg);
575 oq->last_pkt_count = pkts_pend;
576 }
577
578 /* Flush the previous wrties before writing to RESEND bit */
579 wmb();
580 writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
581 writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
582}
583
584/**
585 * octep_napi_poll() - NAPI poll function for Tx/Rx.
586 *
587 * @napi: pointer to napi context.
588 * @budget: max number of packets to be processed in single invocation.
589 */
590static int octep_napi_poll(struct napi_struct *napi, int budget)
591{
592 struct octep_ioq_vector *ioq_vector =
593 container_of(napi, struct octep_ioq_vector, napi);
594 u32 tx_pending, rx_done;
595
596 tx_pending = octep_iq_process_completions(ioq_vector->iq, budget);
597 rx_done = octep_oq_process_rx(ioq_vector->oq, budget);
598
599 /* need more polling if tx completion processing is still pending or
600 * processed at least 'budget' number of rx packets.
601 */
602 if (tx_pending || rx_done >= budget)
603 return budget;
604
605 napi_complete(napi);
606 octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
607 return rx_done;
608}
609
610/**
611 * octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
612 *
613 * @oct: Octeon device private data structure.
614 */
615static void octep_napi_add(struct octep_device *oct)
616{
617 int i;
618
619 for (i = 0; i < oct->num_oqs; i++) {
620 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
621 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
622 octep_napi_poll);
623 oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
624 }
625}
626
627/**
628 * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
629 *
630 * @oct: Octeon device private data structure.
631 */
632static void octep_napi_delete(struct octep_device *oct)
633{
634 int i;
635
636 for (i = 0; i < oct->num_oqs; i++) {
637 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
638 netif_napi_del(&oct->ioq_vector[i]->napi);
639 oct->oq[i]->napi = NULL;
640 }
641}
642
643/**
644 * octep_napi_enable() - enable NAPI for all Tx/Rx queues.
645 *
646 * @oct: Octeon device private data structure.
647 */
648static void octep_napi_enable(struct octep_device *oct)
649{
650 int i;
651
652 for (i = 0; i < oct->num_oqs; i++) {
653 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
654 napi_enable(&oct->ioq_vector[i]->napi);
655 }
656}
657
658/**
659 * octep_napi_disable() - disable NAPI for all Tx/Rx queues.
660 *
661 * @oct: Octeon device private data structure.
662 */
663static void octep_napi_disable(struct octep_device *oct)
664{
665 int i;
666
667 for (i = 0; i < oct->num_oqs; i++) {
668 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
669 napi_disable(&oct->ioq_vector[i]->napi);
670 }
671}
672
673static void octep_link_up(struct net_device *netdev)
674{
675 netif_carrier_on(netdev);
676 netif_tx_start_all_queues(netdev);
677}
678
679/**
680 * octep_open() - start the octeon network device.
681 *
682 * @netdev: pointer to kernel network device.
683 *
684 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
685 * and interrupts..
686 *
687 * Return: 0, on successfully setting up device and bring it up.
688 * -1, on any error.
689 */
690static int octep_open(struct net_device *netdev)
691{
692 struct octep_device *oct = netdev_priv(netdev);
693 int err, ret;
694
695 netdev_info(netdev, "Starting netdev ...\n");
696 netif_carrier_off(netdev);
697
698 oct->hw_ops.reset_io_queues(oct);
699
700 if (octep_setup_iqs(oct))
701 goto setup_iq_err;
702 if (octep_setup_oqs(oct))
703 goto setup_oq_err;
704 if (octep_setup_irqs(oct))
705 goto setup_irq_err;
706
707 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
708 if (err)
709 goto set_queues_err;
710 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
711 if (err)
712 goto set_queues_err;
713
714 octep_napi_add(oct);
715 octep_napi_enable(oct);
716
717 oct->link_info.admin_up = 1;
718 octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
719 false);
720 octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
721 false);
722 oct->poll_non_ioq_intr = false;
723
724 /* Enable the input and output queues for this Octeon device */
725 oct->hw_ops.enable_io_queues(oct);
726
727 /* Enable Octeon device interrupts */
728 oct->hw_ops.enable_interrupts(oct);
729
730 octep_oq_dbell_init(oct);
731
732 ret = octep_ctrl_net_get_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID);
733 if (ret > 0)
734 octep_link_up(netdev);
735
736 return 0;
737
738set_queues_err:
739 octep_clean_irqs(oct);
740setup_irq_err:
741 octep_free_oqs(oct);
742setup_oq_err:
743 octep_free_iqs(oct);
744setup_iq_err:
745 return -1;
746}
747
748/**
749 * octep_stop() - stop the octeon network device.
750 *
751 * @netdev: pointer to kernel network device.
752 *
753 * stop the device Tx/Rx operations, bring down the link and
754 * free up all resources allocated for Tx/Rx queues and interrupts.
755 */
756static int octep_stop(struct net_device *netdev)
757{
758 struct octep_device *oct = netdev_priv(netdev);
759
760 netdev_info(netdev, "Stopping the device ...\n");
761
762 octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
763 false);
764 octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
765 false);
766
767 /* Stop Tx from stack */
768 netif_tx_stop_all_queues(netdev);
769 netif_carrier_off(netdev);
770 netif_tx_disable(netdev);
771
772 oct->link_info.admin_up = 0;
773 oct->link_info.oper_up = 0;
774
775 oct->hw_ops.disable_interrupts(oct);
776 octep_napi_disable(oct);
777 octep_napi_delete(oct);
778
779 octep_clean_irqs(oct);
780 octep_clean_iqs(oct);
781
782 oct->hw_ops.disable_io_queues(oct);
783 oct->hw_ops.reset_io_queues(oct);
784 octep_free_oqs(oct);
785 octep_free_iqs(oct);
786
787 oct->poll_non_ioq_intr = true;
788 queue_delayed_work(octep_wq, &oct->intr_poll_task,
789 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
790
791 netdev_info(netdev, "Device stopped !!\n");
792 return 0;
793}
794
795/**
796 * octep_iq_full_check() - check if a Tx queue is full.
797 *
798 * @iq: Octeon Tx queue data structure.
799 *
800 * Return: 0, if the Tx queue is not full.
801 * 1, if the Tx queue is full.
802 */
803static inline int octep_iq_full_check(struct octep_iq *iq)
804{
805 if (likely((IQ_INSTR_SPACE(iq)) >
806 OCTEP_WAKE_QUEUE_THRESHOLD))
807 return 0;
808
809 /* Stop the queue if unable to send */
810 netif_stop_subqueue(iq->netdev, iq->q_no);
811
812 /* Allow for pending updates in write index
813 * from iq_process_completion in other cpus
814 * to reflect, in case queue gets free
815 * entries.
816 */
817 smp_mb();
818
819 /* check again and restart the queue, in case NAPI has just freed
820 * enough Tx ring entries.
821 */
822 if (unlikely(IQ_INSTR_SPACE(iq) >
823 OCTEP_WAKE_QUEUE_THRESHOLD)) {
824 netif_start_subqueue(iq->netdev, iq->q_no);
825 iq->stats->restart_cnt++;
826 return 0;
827 }
828
829 return 1;
830}
831
832/**
833 * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
834 *
835 * @skb: packet skbuff pointer.
836 * @netdev: kernel network device.
837 *
838 * Return: NETDEV_TX_BUSY, if Tx Queue is full.
839 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
840 */
841static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
842 struct net_device *netdev)
843{
844 struct octep_device *oct = netdev_priv(netdev);
845 netdev_features_t feat = netdev->features;
846 struct octep_tx_sglist_desc *sglist;
847 struct octep_tx_buffer *tx_buffer;
848 struct octep_tx_desc_hw *hw_desc;
849 struct skb_shared_info *shinfo;
850 struct octep_instr_hdr *ih;
851 struct octep_iq *iq;
852 skb_frag_t *frag;
853 u16 nr_frags, si;
854 int xmit_more;
855 u16 q_no, wi;
856
857 if (skb_put_padto(skb, ETH_ZLEN))
858 return NETDEV_TX_OK;
859
860 q_no = skb_get_queue_mapping(skb);
861 if (q_no >= oct->num_iqs) {
862 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
863 q_no = q_no % oct->num_iqs;
864 }
865
866 iq = oct->iq[q_no];
867
868 shinfo = skb_shinfo(skb);
869 nr_frags = shinfo->nr_frags;
870
871 wi = iq->host_write_index;
872 hw_desc = &iq->desc_ring[wi];
873 hw_desc->ih64 = 0;
874
875 tx_buffer = iq->buff_info + wi;
876 tx_buffer->skb = skb;
877
878 ih = &hw_desc->ih;
879 ih->pkind = oct->conf->fw_info.pkind;
880 ih->fsz = oct->conf->fw_info.fsz;
881 ih->tlen = skb->len + ih->fsz;
882
883 if (!nr_frags) {
884 tx_buffer->gather = 0;
885 tx_buffer->dma = dma_map_single(iq->dev, skb->data,
886 skb->len, DMA_TO_DEVICE);
887 if (dma_mapping_error(iq->dev, tx_buffer->dma))
888 goto dma_map_err;
889 hw_desc->dptr = tx_buffer->dma;
890 } else {
891 /* Scatter/Gather */
892 dma_addr_t dma;
893 u16 len;
894
895 sglist = tx_buffer->sglist;
896
897 ih->gsz = nr_frags + 1;
898 ih->gather = 1;
899 tx_buffer->gather = 1;
900
901 len = skb_headlen(skb);
902 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
903 if (dma_mapping_error(iq->dev, dma))
904 goto dma_map_err;
905
906 memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
907 sglist[0].len[3] = len;
908 sglist[0].dma_ptr[0] = dma;
909
910 si = 1; /* entry 0 is main skb, mapped above */
911 frag = &shinfo->frags[0];
912 while (nr_frags--) {
913 len = skb_frag_size(frag);
914 dma = skb_frag_dma_map(iq->dev, frag, 0,
915 len, DMA_TO_DEVICE);
916 if (dma_mapping_error(iq->dev, dma))
917 goto dma_map_sg_err;
918
919 sglist[si >> 2].len[3 - (si & 3)] = len;
920 sglist[si >> 2].dma_ptr[si & 3] = dma;
921
922 frag++;
923 si++;
924 }
925 hw_desc->dptr = tx_buffer->sglist_dma;
926 }
927
928 if (oct->conf->fw_info.tx_ol_flags) {
929 if ((feat & (NETIF_F_TSO)) && (skb_is_gso(skb))) {
930 hw_desc->txm.ol_flags = OCTEP_TX_OFFLOAD_CKSUM;
931 hw_desc->txm.ol_flags |= OCTEP_TX_OFFLOAD_TSO;
932 hw_desc->txm.gso_size = skb_shinfo(skb)->gso_size;
933 hw_desc->txm.gso_segs = skb_shinfo(skb)->gso_segs;
934 } else if (feat & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
935 hw_desc->txm.ol_flags = OCTEP_TX_OFFLOAD_CKSUM;
936 }
937 /* due to ESR txm will be swapped by hw */
938 hw_desc->txm64[0] = (__force u64)cpu_to_be64(hw_desc->txm64[0]);
939 }
940
941 xmit_more = netdev_xmit_more();
942
943 __netdev_tx_sent_queue(iq->netdev_q, skb->len, xmit_more);
944
945 skb_tx_timestamp(skb);
946 iq->fill_cnt++;
947 wi++;
948 iq->host_write_index = wi & iq->ring_size_mask;
949
950 /* octep_iq_full_check stops the queue and returns
951 * true if so, in case the queue has become full
952 * by inserting current packet. If so, we can
953 * go ahead and ring doorbell.
954 */
955 if (!octep_iq_full_check(iq) && xmit_more &&
956 iq->fill_cnt < iq->fill_threshold)
957 return NETDEV_TX_OK;
958
959 /* Flush the hw descriptor before writing to doorbell */
960 wmb();
961 /* Ring Doorbell to notify the NIC of new packets */
962 writel(iq->fill_cnt, iq->doorbell_reg);
963 iq->stats->instr_posted += iq->fill_cnt;
964 iq->fill_cnt = 0;
965 return NETDEV_TX_OK;
966
967dma_map_sg_err:
968 if (si > 0) {
969 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
970 sglist[0].len[3], DMA_TO_DEVICE);
971 sglist[0].len[3] = 0;
972 }
973 while (si > 1) {
974 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
975 sglist[si >> 2].len[3 - (si & 3)], DMA_TO_DEVICE);
976 sglist[si >> 2].len[3 - (si & 3)] = 0;
977 si--;
978 }
979 tx_buffer->gather = 0;
980dma_map_err:
981 dev_kfree_skb_any(skb);
982 return NETDEV_TX_OK;
983}
984
985/**
986 * octep_get_stats64() - Get Octeon network device statistics.
987 *
988 * @netdev: kernel network device.
989 * @stats: pointer to stats structure to be filled in.
990 */
991static void octep_get_stats64(struct net_device *netdev,
992 struct rtnl_link_stats64 *stats)
993{
994 struct octep_device *oct = netdev_priv(netdev);
995 u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
996 int q;
997
998 tx_packets = 0;
999 tx_bytes = 0;
1000 rx_packets = 0;
1001 rx_bytes = 0;
1002 for (q = 0; q < OCTEP_MAX_QUEUES; q++) {
1003 tx_packets += oct->stats_iq[q].instr_completed;
1004 tx_bytes += oct->stats_iq[q].bytes_sent;
1005 rx_packets += oct->stats_oq[q].packets;
1006 rx_bytes += oct->stats_oq[q].bytes;
1007 }
1008 stats->tx_packets = tx_packets;
1009 stats->tx_bytes = tx_bytes;
1010 stats->rx_packets = rx_packets;
1011 stats->rx_bytes = rx_bytes;
1012}
1013
1014/**
1015 * octep_tx_timeout_task - work queue task to Handle Tx queue timeout.
1016 *
1017 * @work: pointer to Tx queue timeout work_struct
1018 *
1019 * Stop and start the device so that it frees up all queue resources
1020 * and restarts the queues, that potentially clears a Tx queue timeout
1021 * condition.
1022 **/
1023static void octep_tx_timeout_task(struct work_struct *work)
1024{
1025 struct octep_device *oct = container_of(work, struct octep_device,
1026 tx_timeout_task);
1027 struct net_device *netdev = oct->netdev;
1028
1029 rtnl_lock();
1030 if (netif_running(netdev)) {
1031 octep_stop(netdev);
1032 octep_open(netdev);
1033 }
1034 rtnl_unlock();
1035}
1036
1037/**
1038 * octep_tx_timeout() - Handle Tx Queue timeout.
1039 *
1040 * @netdev: pointer to kernel network device.
1041 * @txqueue: Timed out Tx queue number.
1042 *
1043 * Schedule a work to handle Tx queue timeout.
1044 */
1045static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1046{
1047 struct octep_device *oct = netdev_priv(netdev);
1048
1049 queue_work(octep_wq, &oct->tx_timeout_task);
1050}
1051
1052static int octep_set_mac(struct net_device *netdev, void *p)
1053{
1054 struct octep_device *oct = netdev_priv(netdev);
1055 struct sockaddr *addr = (struct sockaddr *)p;
1056 int err;
1057
1058 if (!is_valid_ether_addr(addr->sa_data))
1059 return -EADDRNOTAVAIL;
1060
1061 err = octep_ctrl_net_set_mac_addr(oct, OCTEP_CTRL_NET_INVALID_VFID,
1062 addr->sa_data, true);
1063 if (err)
1064 return err;
1065
1066 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
1067 eth_hw_addr_set(netdev, addr->sa_data);
1068
1069 return 0;
1070}
1071
1072static int octep_change_mtu(struct net_device *netdev, int new_mtu)
1073{
1074 struct octep_device *oct = netdev_priv(netdev);
1075 struct octep_iface_link_info *link_info;
1076 int err = 0;
1077
1078 link_info = &oct->link_info;
1079 if (link_info->mtu == new_mtu)
1080 return 0;
1081
1082 err = octep_ctrl_net_set_mtu(oct, OCTEP_CTRL_NET_INVALID_VFID, new_mtu,
1083 true);
1084 if (!err) {
1085 oct->link_info.mtu = new_mtu;
1086 WRITE_ONCE(netdev->mtu, new_mtu);
1087 }
1088
1089 return err;
1090}
1091
1092static int octep_set_features(struct net_device *dev, netdev_features_t features)
1093{
1094 struct octep_ctrl_net_offloads offloads = { 0 };
1095 struct octep_device *oct = netdev_priv(dev);
1096 int err;
1097
1098 /* We only support features received from firmware */
1099 if ((features & dev->hw_features) != features)
1100 return -EINVAL;
1101
1102 if (features & NETIF_F_TSO)
1103 offloads.tx_offloads |= OCTEP_TX_OFFLOAD_TSO;
1104
1105 if (features & NETIF_F_TSO6)
1106 offloads.tx_offloads |= OCTEP_TX_OFFLOAD_TSO;
1107
1108 if (features & NETIF_F_IP_CSUM)
1109 offloads.tx_offloads |= OCTEP_TX_OFFLOAD_CKSUM;
1110
1111 if (features & NETIF_F_IPV6_CSUM)
1112 offloads.tx_offloads |= OCTEP_TX_OFFLOAD_CKSUM;
1113
1114 if (features & NETIF_F_RXCSUM)
1115 offloads.rx_offloads |= OCTEP_RX_OFFLOAD_CKSUM;
1116
1117 err = octep_ctrl_net_set_offloads(oct,
1118 OCTEP_CTRL_NET_INVALID_VFID,
1119 &offloads,
1120 true);
1121 if (!err)
1122 dev->features = features;
1123
1124 return err;
1125}
1126
1127static const struct net_device_ops octep_netdev_ops = {
1128 .ndo_open = octep_open,
1129 .ndo_stop = octep_stop,
1130 .ndo_start_xmit = octep_start_xmit,
1131 .ndo_get_stats64 = octep_get_stats64,
1132 .ndo_tx_timeout = octep_tx_timeout,
1133 .ndo_set_mac_address = octep_set_mac,
1134 .ndo_change_mtu = octep_change_mtu,
1135 .ndo_set_features = octep_set_features,
1136};
1137
1138/**
1139 * octep_intr_poll_task - work queue task to process non-ioq interrupts.
1140 *
1141 * @work: pointer to mbox work_struct
1142 *
1143 * Process non-ioq interrupts to handle control mailbox, pfvf mailbox.
1144 **/
1145static void octep_intr_poll_task(struct work_struct *work)
1146{
1147 struct octep_device *oct = container_of(work, struct octep_device,
1148 intr_poll_task.work);
1149
1150 if (!oct->poll_non_ioq_intr) {
1151 dev_info(&oct->pdev->dev, "Interrupt poll task stopped.\n");
1152 return;
1153 }
1154
1155 oct->hw_ops.poll_non_ioq_interrupts(oct);
1156 queue_delayed_work(octep_wq, &oct->intr_poll_task,
1157 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1158}
1159
1160/**
1161 * octep_hb_timeout_task - work queue task to check firmware heartbeat.
1162 *
1163 * @work: pointer to hb work_struct
1164 *
1165 * Check for heartbeat miss count. Uninitialize oct device if miss count
1166 * exceeds configured max heartbeat miss count.
1167 *
1168 **/
1169static void octep_hb_timeout_task(struct work_struct *work)
1170{
1171 struct octep_device *oct = container_of(work, struct octep_device,
1172 hb_task.work);
1173
1174 int miss_cnt;
1175
1176 miss_cnt = atomic_inc_return(&oct->hb_miss_cnt);
1177 if (miss_cnt < oct->conf->fw_info.hb_miss_count) {
1178 queue_delayed_work(octep_wq, &oct->hb_task,
1179 msecs_to_jiffies(oct->conf->fw_info.hb_interval));
1180 return;
1181 }
1182
1183 dev_err(&oct->pdev->dev, "Missed %u heartbeats. Uninitializing\n",
1184 miss_cnt);
1185 rtnl_lock();
1186 if (netif_running(oct->netdev))
1187 octep_stop(oct->netdev);
1188 rtnl_unlock();
1189}
1190
1191/**
1192 * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
1193 *
1194 * @work: pointer to ctrl mbox work_struct
1195 *
1196 * Poll ctrl mbox message queue and handle control messages from firmware.
1197 **/
1198static void octep_ctrl_mbox_task(struct work_struct *work)
1199{
1200 struct octep_device *oct = container_of(work, struct octep_device,
1201 ctrl_mbox_task);
1202
1203 octep_ctrl_net_recv_fw_messages(oct);
1204}
1205
1206static const char *octep_devid_to_str(struct octep_device *oct)
1207{
1208 switch (oct->chip_id) {
1209 case OCTEP_PCI_DEVICE_ID_CN98_PF:
1210 return "CN98XX";
1211 case OCTEP_PCI_DEVICE_ID_CN93_PF:
1212 return "CN93XX";
1213 case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
1214 return "CNF95N";
1215 case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
1216 return "CN10KA";
1217 case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
1218 return "CNF10KA";
1219 case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
1220 return "CNF10KB";
1221 case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
1222 return "CN10KB";
1223 default:
1224 return "Unsupported";
1225 }
1226}
1227
1228/**
1229 * octep_device_setup() - Setup Octeon Device.
1230 *
1231 * @oct: Octeon device private data structure.
1232 *
1233 * Setup Octeon device hardware operations, configuration, etc ...
1234 */
1235int octep_device_setup(struct octep_device *oct)
1236{
1237 struct pci_dev *pdev = oct->pdev;
1238 int i, ret;
1239
1240 /* allocate memory for oct->conf */
1241 oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);
1242 if (!oct->conf)
1243 return -ENOMEM;
1244
1245 /* Map BAR regions */
1246 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1247 oct->mmio[i].hw_addr =
1248 ioremap(pci_resource_start(oct->pdev, i * 2),
1249 pci_resource_len(oct->pdev, i * 2));
1250 if (!oct->mmio[i].hw_addr)
1251 goto unmap_prev;
1252
1253 oct->mmio[i].mapped = 1;
1254 }
1255
1256 oct->chip_id = pdev->device;
1257 oct->rev_id = pdev->revision;
1258 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
1259
1260 switch (oct->chip_id) {
1261 case OCTEP_PCI_DEVICE_ID_CN98_PF:
1262 case OCTEP_PCI_DEVICE_ID_CN93_PF:
1263 case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
1264 dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
1265 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct),
1266 OCTEP_MINOR_REV(oct));
1267 octep_device_setup_cn93_pf(oct);
1268 break;
1269 case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
1270 case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
1271 case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
1272 case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
1273 dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
1274 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct));
1275 octep_device_setup_cnxk_pf(oct);
1276 break;
1277 default:
1278 dev_err(&pdev->dev,
1279 "%s: unsupported device\n", __func__);
1280 goto unsupported_dev;
1281 }
1282
1283
1284 ret = octep_ctrl_net_init(oct);
1285 if (ret)
1286 return ret;
1287
1288 INIT_WORK(&oct->tx_timeout_task, octep_tx_timeout_task);
1289 INIT_WORK(&oct->ctrl_mbox_task, octep_ctrl_mbox_task);
1290 INIT_DELAYED_WORK(&oct->intr_poll_task, octep_intr_poll_task);
1291 oct->poll_non_ioq_intr = true;
1292 queue_delayed_work(octep_wq, &oct->intr_poll_task,
1293 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1294
1295 atomic_set(&oct->hb_miss_cnt, 0);
1296 INIT_DELAYED_WORK(&oct->hb_task, octep_hb_timeout_task);
1297
1298 return 0;
1299
1300unsupported_dev:
1301 i = OCTEP_MMIO_REGIONS;
1302unmap_prev:
1303 while (i--)
1304 iounmap(oct->mmio[i].hw_addr);
1305
1306 kfree(oct->conf);
1307 return -1;
1308}
1309
1310/**
1311 * octep_device_cleanup() - Cleanup Octeon Device.
1312 *
1313 * @oct: Octeon device private data structure.
1314 *
1315 * Cleanup Octeon device allocated resources.
1316 */
1317static void octep_device_cleanup(struct octep_device *oct)
1318{
1319 int i;
1320
1321 oct->poll_non_ioq_intr = false;
1322 cancel_delayed_work_sync(&oct->intr_poll_task);
1323 cancel_work_sync(&oct->ctrl_mbox_task);
1324
1325 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
1326
1327 for (i = 0; i < OCTEP_MAX_VF; i++) {
1328 vfree(oct->mbox[i]);
1329 oct->mbox[i] = NULL;
1330 }
1331
1332 octep_delete_pfvf_mbox(oct);
1333 octep_ctrl_net_uninit(oct);
1334 cancel_delayed_work_sync(&oct->hb_task);
1335
1336 oct->hw_ops.soft_reset(oct);
1337 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1338 if (oct->mmio[i].mapped)
1339 iounmap(oct->mmio[i].hw_addr);
1340 }
1341
1342 kfree(oct->conf);
1343 oct->conf = NULL;
1344}
1345
1346static bool get_fw_ready_status(struct pci_dev *pdev)
1347{
1348 u32 pos = 0;
1349 u16 vsec_id;
1350 u8 status;
1351
1352 while ((pos = pci_find_next_ext_capability(pdev, pos,
1353 PCI_EXT_CAP_ID_VNDR))) {
1354 pci_read_config_word(pdev, pos + 4, &vsec_id);
1355#define FW_STATUS_VSEC_ID 0xA3
1356 if (vsec_id != FW_STATUS_VSEC_ID)
1357 continue;
1358
1359 pci_read_config_byte(pdev, (pos + 8), &status);
1360 dev_info(&pdev->dev, "Firmware ready status = %u\n", status);
1361#define FW_STATUS_READY 1ULL
1362 return status == FW_STATUS_READY;
1363 }
1364 return false;
1365}
1366
1367/**
1368 * octep_probe() - Octeon PCI device probe handler.
1369 *
1370 * @pdev: PCI device structure.
1371 * @ent: entry in Octeon PCI device ID table.
1372 *
1373 * Initializes and enables the Octeon PCI device for network operations.
1374 * Initializes Octeon private data structure and registers a network device.
1375 */
1376static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1377{
1378 struct octep_device *octep_dev = NULL;
1379 struct net_device *netdev;
1380 int max_rx_pktlen;
1381 int err;
1382
1383 err = pci_enable_device(pdev);
1384 if (err) {
1385 dev_err(&pdev->dev, "Failed to enable PCI device\n");
1386 return err;
1387 }
1388
1389 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1390 if (err) {
1391 dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1392 goto err_dma_mask;
1393 }
1394
1395 err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME);
1396 if (err) {
1397 dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1398 goto err_pci_regions;
1399 }
1400
1401 pci_set_master(pdev);
1402
1403 if (!get_fw_ready_status(pdev)) {
1404 dev_notice(&pdev->dev, "Firmware not ready; defer probe.\n");
1405 err = -EPROBE_DEFER;
1406 goto err_alloc_netdev;
1407 }
1408
1409 netdev = alloc_etherdev_mq(sizeof(struct octep_device),
1410 OCTEP_MAX_QUEUES);
1411 if (!netdev) {
1412 dev_err(&pdev->dev, "Failed to allocate netdev\n");
1413 err = -ENOMEM;
1414 goto err_alloc_netdev;
1415 }
1416 SET_NETDEV_DEV(netdev, &pdev->dev);
1417
1418 octep_dev = netdev_priv(netdev);
1419 octep_dev->netdev = netdev;
1420 octep_dev->pdev = pdev;
1421 octep_dev->dev = &pdev->dev;
1422 pci_set_drvdata(pdev, octep_dev);
1423
1424 err = octep_device_setup(octep_dev);
1425 if (err) {
1426 dev_err(&pdev->dev, "Device setup failed\n");
1427 goto err_octep_config;
1428 }
1429
1430 err = octep_setup_pfvf_mbox(octep_dev);
1431 if (err) {
1432 dev_err(&pdev->dev, "PF-VF mailbox setup failed\n");
1433 goto register_dev_err;
1434 }
1435
1436 err = octep_ctrl_net_get_info(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1437 &octep_dev->conf->fw_info);
1438 if (err) {
1439 dev_err(&pdev->dev, "Failed to get firmware info\n");
1440 goto register_dev_err;
1441 }
1442 dev_info(&octep_dev->pdev->dev, "Heartbeat interval %u msecs Heartbeat miss count %u\n",
1443 octep_dev->conf->fw_info.hb_interval,
1444 octep_dev->conf->fw_info.hb_miss_count);
1445 queue_delayed_work(octep_wq, &octep_dev->hb_task,
1446 msecs_to_jiffies(octep_dev->conf->fw_info.hb_interval));
1447
1448 netdev->netdev_ops = &octep_netdev_ops;
1449 octep_set_ethtool_ops(netdev);
1450 netif_carrier_off(netdev);
1451
1452 netdev->hw_features = NETIF_F_SG;
1453 if (OCTEP_TX_IP_CSUM(octep_dev->conf->fw_info.tx_ol_flags))
1454 netdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
1455
1456 if (OCTEP_RX_IP_CSUM(octep_dev->conf->fw_info.rx_ol_flags))
1457 netdev->hw_features |= NETIF_F_RXCSUM;
1458
1459 max_rx_pktlen = octep_ctrl_net_get_mtu(octep_dev, OCTEP_CTRL_NET_INVALID_VFID);
1460 if (max_rx_pktlen < 0) {
1461 dev_err(&octep_dev->pdev->dev,
1462 "Failed to get max receive packet size; err = %d\n", max_rx_pktlen);
1463 err = max_rx_pktlen;
1464 goto register_dev_err;
1465 }
1466 netdev->min_mtu = OCTEP_MIN_MTU;
1467 netdev->max_mtu = max_rx_pktlen - (ETH_HLEN + ETH_FCS_LEN);
1468 netdev->mtu = OCTEP_DEFAULT_MTU;
1469
1470 if (OCTEP_TX_TSO(octep_dev->conf->fw_info.tx_ol_flags)) {
1471 netdev->hw_features |= NETIF_F_TSO;
1472 netif_set_tso_max_size(netdev, netdev->max_mtu);
1473 }
1474
1475 netdev->features |= netdev->hw_features;
1476 err = octep_ctrl_net_get_mac_addr(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1477 octep_dev->mac_addr);
1478 if (err) {
1479 dev_err(&pdev->dev, "Failed to get mac address\n");
1480 goto register_dev_err;
1481 }
1482 eth_hw_addr_set(netdev, octep_dev->mac_addr);
1483
1484 err = register_netdev(netdev);
1485 if (err) {
1486 dev_err(&pdev->dev, "Failed to register netdev\n");
1487 goto register_dev_err;
1488 }
1489 dev_info(&pdev->dev, "Device probe successful\n");
1490 return 0;
1491
1492register_dev_err:
1493 octep_device_cleanup(octep_dev);
1494err_octep_config:
1495 free_netdev(netdev);
1496err_alloc_netdev:
1497 pci_release_mem_regions(pdev);
1498err_pci_regions:
1499err_dma_mask:
1500 pci_disable_device(pdev);
1501 return err;
1502}
1503
1504static int octep_sriov_disable(struct octep_device *oct)
1505{
1506 struct pci_dev *pdev = oct->pdev;
1507
1508 if (pci_vfs_assigned(oct->pdev)) {
1509 dev_warn(&pdev->dev, "Can't disable SRIOV while VFs are assigned\n");
1510 return -EPERM;
1511 }
1512
1513 pci_disable_sriov(pdev);
1514 CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1515
1516 return 0;
1517}
1518
1519/**
1520 * octep_remove() - Remove Octeon PCI device from driver control.
1521 *
1522 * @pdev: PCI device structure of the Octeon device.
1523 *
1524 * Cleanup all resources allocated for the Octeon device.
1525 * Unregister from network device and disable the PCI device.
1526 */
1527static void octep_remove(struct pci_dev *pdev)
1528{
1529 struct octep_device *oct = pci_get_drvdata(pdev);
1530 struct net_device *netdev;
1531
1532 if (!oct)
1533 return;
1534
1535 netdev = oct->netdev;
1536 octep_sriov_disable(oct);
1537 if (netdev->reg_state == NETREG_REGISTERED)
1538 unregister_netdev(netdev);
1539
1540 cancel_work_sync(&oct->tx_timeout_task);
1541 octep_device_cleanup(oct);
1542 pci_release_mem_regions(pdev);
1543 free_netdev(netdev);
1544 pci_disable_device(pdev);
1545}
1546
1547static int octep_sriov_enable(struct octep_device *oct, int num_vfs)
1548{
1549 struct pci_dev *pdev = oct->pdev;
1550 int err;
1551
1552 CFG_GET_ACTIVE_VFS(oct->conf) = num_vfs;
1553 err = pci_enable_sriov(pdev, num_vfs);
1554 if (err) {
1555 dev_warn(&pdev->dev, "Failed to enable SRIOV err=%d\n", err);
1556 CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1557 return err;
1558 }
1559
1560 return num_vfs;
1561}
1562
1563static int octep_sriov_configure(struct pci_dev *pdev, int num_vfs)
1564{
1565 struct octep_device *oct = pci_get_drvdata(pdev);
1566 int max_nvfs;
1567
1568 if (num_vfs == 0)
1569 return octep_sriov_disable(oct);
1570
1571 max_nvfs = CFG_GET_MAX_VFS(oct->conf);
1572
1573 if (num_vfs > max_nvfs) {
1574 dev_err(&pdev->dev, "Invalid VF count Max supported VFs = %d\n",
1575 max_nvfs);
1576 return -EINVAL;
1577 }
1578
1579 return octep_sriov_enable(oct, num_vfs);
1580}
1581
1582static struct pci_driver octep_driver = {
1583 .name = OCTEP_DRV_NAME,
1584 .id_table = octep_pci_id_tbl,
1585 .probe = octep_probe,
1586 .remove = octep_remove,
1587 .sriov_configure = octep_sriov_configure,
1588};
1589
1590/**
1591 * octep_init_module() - Module initialiation.
1592 *
1593 * create common resource for the driver and register PCI driver.
1594 */
1595static int __init octep_init_module(void)
1596{
1597 int ret;
1598
1599 pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING);
1600
1601 /* work queue for all deferred tasks */
1602 octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME);
1603 if (!octep_wq) {
1604 pr_err("%s: Failed to create common workqueue\n",
1605 OCTEP_DRV_NAME);
1606 return -ENOMEM;
1607 }
1608
1609 ret = pci_register_driver(&octep_driver);
1610 if (ret < 0) {
1611 pr_err("%s: Failed to register PCI driver; err=%d\n",
1612 OCTEP_DRV_NAME, ret);
1613 destroy_workqueue(octep_wq);
1614 return ret;
1615 }
1616
1617 pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME);
1618
1619 return ret;
1620}
1621
1622/**
1623 * octep_exit_module() - Module exit routine.
1624 *
1625 * unregister the driver with PCI subsystem and cleanup common resources.
1626 */
1627static void __exit octep_exit_module(void)
1628{
1629 pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME);
1630
1631 pci_unregister_driver(&octep_driver);
1632 destroy_workqueue(octep_wq);
1633
1634 pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME);
1635}
1636
1637module_init(octep_init_module);
1638module_exit(octep_exit_module);
1// SPDX-License-Identifier: GPL-2.0
2/* Marvell Octeon EP (EndPoint) Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8#include <linux/types.h>
9#include <linux/module.h>
10#include <linux/pci.h>
11#include <linux/aer.h>
12#include <linux/netdevice.h>
13#include <linux/etherdevice.h>
14#include <linux/rtnetlink.h>
15#include <linux/vmalloc.h>
16
17#include "octep_config.h"
18#include "octep_main.h"
19#include "octep_ctrl_net.h"
20
21struct workqueue_struct *octep_wq;
22
23/* Supported Devices */
24static const struct pci_device_id octep_pci_id_tbl[] = {
25 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
26 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_PF)},
27 {0, },
28};
29MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
30
31MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
32MODULE_DESCRIPTION(OCTEP_DRV_STRING);
33MODULE_LICENSE("GPL");
34
35/**
36 * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
37 *
38 * @oct: Octeon device private data structure.
39 *
40 * Allocate resources to hold per Tx/Rx queue interrupt info.
41 * This is the information passed to interrupt handler, from which napi poll
42 * is scheduled and includes quick access to private data of Tx/Rx queue
43 * corresponding to the interrupt being handled.
44 *
45 * Return: 0, on successful allocation of resources for all queue interrupts.
46 * -1, if failed to allocate any resource.
47 */
48static int octep_alloc_ioq_vectors(struct octep_device *oct)
49{
50 int i;
51 struct octep_ioq_vector *ioq_vector;
52
53 for (i = 0; i < oct->num_oqs; i++) {
54 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
55 if (!oct->ioq_vector[i])
56 goto free_ioq_vector;
57
58 ioq_vector = oct->ioq_vector[i];
59 ioq_vector->iq = oct->iq[i];
60 ioq_vector->oq = oct->oq[i];
61 ioq_vector->octep_dev = oct;
62 }
63
64 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
65 return 0;
66
67free_ioq_vector:
68 while (i) {
69 i--;
70 vfree(oct->ioq_vector[i]);
71 oct->ioq_vector[i] = NULL;
72 }
73 return -1;
74}
75
76/**
77 * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
78 *
79 * @oct: Octeon device private data structure.
80 */
81static void octep_free_ioq_vectors(struct octep_device *oct)
82{
83 int i;
84
85 for (i = 0; i < oct->num_oqs; i++) {
86 if (oct->ioq_vector[i]) {
87 vfree(oct->ioq_vector[i]);
88 oct->ioq_vector[i] = NULL;
89 }
90 }
91 netdev_info(oct->netdev, "Freed IOQ Vectors\n");
92}
93
94/**
95 * octep_enable_msix_range() - enable MSI-x interrupts.
96 *
97 * @oct: Octeon device private data structure.
98 *
99 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
100 * for the Octeon device.
101 *
102 * Return: 0, on successfully enabling all MSI-x interrupts.
103 * -1, if failed to enable any MSI-x interrupt.
104 */
105static int octep_enable_msix_range(struct octep_device *oct)
106{
107 int num_msix, msix_allocated;
108 int i;
109
110 /* Generic interrupts apart from input/output queues */
111 num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
112 oct->msix_entries = kcalloc(num_msix,
113 sizeof(struct msix_entry), GFP_KERNEL);
114 if (!oct->msix_entries)
115 goto msix_alloc_err;
116
117 for (i = 0; i < num_msix; i++)
118 oct->msix_entries[i].entry = i;
119
120 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
121 num_msix, num_msix);
122 if (msix_allocated != num_msix) {
123 dev_err(&oct->pdev->dev,
124 "Failed to enable %d msix irqs; got only %d\n",
125 num_msix, msix_allocated);
126 goto enable_msix_err;
127 }
128 oct->num_irqs = msix_allocated;
129 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
130
131 return 0;
132
133enable_msix_err:
134 if (msix_allocated > 0)
135 pci_disable_msix(oct->pdev);
136 kfree(oct->msix_entries);
137 oct->msix_entries = NULL;
138msix_alloc_err:
139 return -1;
140}
141
142/**
143 * octep_disable_msix() - disable MSI-x interrupts.
144 *
145 * @oct: Octeon device private data structure.
146 *
147 * Disable MSI-x on the Octeon device.
148 */
149static void octep_disable_msix(struct octep_device *oct)
150{
151 pci_disable_msix(oct->pdev);
152 kfree(oct->msix_entries);
153 oct->msix_entries = NULL;
154 dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
155}
156
157/**
158 * octep_non_ioq_intr_handler() - common handler for all generic interrupts.
159 *
160 * @irq: Interrupt number.
161 * @data: interrupt data.
162 *
163 * this is common handler for all non-queue (generic) interrupts.
164 */
165static irqreturn_t octep_non_ioq_intr_handler(int irq, void *data)
166{
167 struct octep_device *oct = data;
168
169 return oct->hw_ops.non_ioq_intr_handler(oct);
170}
171
172/**
173 * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
174 *
175 * @irq: Interrupt number.
176 * @data: interrupt data contains pointers to Tx/Rx queue private data
177 * and correspong NAPI context.
178 *
179 * this is common handler for all non-queue (generic) interrupts.
180 */
181static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
182{
183 struct octep_ioq_vector *ioq_vector = data;
184 struct octep_device *oct = ioq_vector->octep_dev;
185
186 return oct->hw_ops.ioq_intr_handler(ioq_vector);
187}
188
189/**
190 * octep_request_irqs() - Register interrupt handlers.
191 *
192 * @oct: Octeon device private data structure.
193 *
194 * Register handlers for all queue and non-queue interrupts.
195 *
196 * Return: 0, on successful registration of all interrupt handlers.
197 * -1, on any error.
198 */
199static int octep_request_irqs(struct octep_device *oct)
200{
201 struct net_device *netdev = oct->netdev;
202 struct octep_ioq_vector *ioq_vector;
203 struct msix_entry *msix_entry;
204 char **non_ioq_msix_names;
205 int num_non_ioq_msix;
206 int ret, i, j;
207
208 num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
209 non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);
210
211 oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix,
212 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
213 if (!oct->non_ioq_irq_names)
214 goto alloc_err;
215
216 /* First few MSI-X interrupts are non-queue interrupts */
217 for (i = 0; i < num_non_ioq_msix; i++) {
218 char *irq_name;
219
220 irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
221 msix_entry = &oct->msix_entries[i];
222
223 snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
224 "%s-%s", netdev->name, non_ioq_msix_names[i]);
225 ret = request_irq(msix_entry->vector,
226 octep_non_ioq_intr_handler, 0,
227 irq_name, oct);
228 if (ret) {
229 netdev_err(netdev,
230 "request_irq failed for %s; err=%d",
231 irq_name, ret);
232 goto non_ioq_irq_err;
233 }
234 }
235
236 /* Request IRQs for Tx/Rx queues */
237 for (j = 0; j < oct->num_oqs; j++) {
238 ioq_vector = oct->ioq_vector[j];
239 msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
240
241 snprintf(ioq_vector->name, sizeof(ioq_vector->name),
242 "%s-q%d", netdev->name, j);
243 ret = request_irq(msix_entry->vector,
244 octep_ioq_intr_handler, 0,
245 ioq_vector->name, ioq_vector);
246 if (ret) {
247 netdev_err(netdev,
248 "request_irq failed for Q-%d; err=%d",
249 j, ret);
250 goto ioq_irq_err;
251 }
252
253 cpumask_set_cpu(j % num_online_cpus(),
254 &ioq_vector->affinity_mask);
255 irq_set_affinity_hint(msix_entry->vector,
256 &ioq_vector->affinity_mask);
257 }
258
259 return 0;
260ioq_irq_err:
261 while (j) {
262 --j;
263 ioq_vector = oct->ioq_vector[j];
264 msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
265
266 irq_set_affinity_hint(msix_entry->vector, NULL);
267 free_irq(msix_entry->vector, ioq_vector);
268 }
269non_ioq_irq_err:
270 while (i) {
271 --i;
272 free_irq(oct->msix_entries[i].vector, oct);
273 }
274 kfree(oct->non_ioq_irq_names);
275 oct->non_ioq_irq_names = NULL;
276alloc_err:
277 return -1;
278}
279
280/**
281 * octep_free_irqs() - free all registered interrupts.
282 *
283 * @oct: Octeon device private data structure.
284 *
285 * Free all queue and non-queue interrupts of the Octeon device.
286 */
287static void octep_free_irqs(struct octep_device *oct)
288{
289 int i;
290
291 /* First few MSI-X interrupts are non queue interrupts; free them */
292 for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
293 free_irq(oct->msix_entries[i].vector, oct);
294 kfree(oct->non_ioq_irq_names);
295
296 /* Free IRQs for Input/Output (Tx/Rx) queues */
297 for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
298 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
299 free_irq(oct->msix_entries[i].vector,
300 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
301 }
302 netdev_info(oct->netdev, "IRQs freed\n");
303}
304
305/**
306 * octep_setup_irqs() - setup interrupts for the Octeon device.
307 *
308 * @oct: Octeon device private data structure.
309 *
310 * Allocate data structures to hold per interrupt information, allocate/enable
311 * MSI-x interrupt and register interrupt handlers.
312 *
313 * Return: 0, on successful allocation and registration of all interrupts.
314 * -1, on any error.
315 */
316static int octep_setup_irqs(struct octep_device *oct)
317{
318 if (octep_alloc_ioq_vectors(oct))
319 goto ioq_vector_err;
320
321 if (octep_enable_msix_range(oct))
322 goto enable_msix_err;
323
324 if (octep_request_irqs(oct))
325 goto request_irq_err;
326
327 return 0;
328
329request_irq_err:
330 octep_disable_msix(oct);
331enable_msix_err:
332 octep_free_ioq_vectors(oct);
333ioq_vector_err:
334 return -1;
335}
336
337/**
338 * octep_clean_irqs() - free all interrupts and its resources.
339 *
340 * @oct: Octeon device private data structure.
341 */
342static void octep_clean_irqs(struct octep_device *oct)
343{
344 octep_free_irqs(oct);
345 octep_disable_msix(oct);
346 octep_free_ioq_vectors(oct);
347}
348
349/**
350 * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
351 *
352 * @iq: Octeon Tx queue data structure.
353 * @oq: Octeon Rx queue data structure.
354 */
355static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq)
356{
357 u32 pkts_pend = oq->pkts_pending;
358
359 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
360 if (iq->pkts_processed) {
361 writel(iq->pkts_processed, iq->inst_cnt_reg);
362 iq->pkt_in_done -= iq->pkts_processed;
363 iq->pkts_processed = 0;
364 }
365 if (oq->last_pkt_count - pkts_pend) {
366 writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg);
367 oq->last_pkt_count = pkts_pend;
368 }
369
370 /* Flush the previous wrties before writing to RESEND bit */
371 wmb();
372 writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
373 writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
374}
375
376/**
377 * octep_napi_poll() - NAPI poll function for Tx/Rx.
378 *
379 * @napi: pointer to napi context.
380 * @budget: max number of packets to be processed in single invocation.
381 */
382static int octep_napi_poll(struct napi_struct *napi, int budget)
383{
384 struct octep_ioq_vector *ioq_vector =
385 container_of(napi, struct octep_ioq_vector, napi);
386 u32 tx_pending, rx_done;
387
388 tx_pending = octep_iq_process_completions(ioq_vector->iq, budget);
389 rx_done = octep_oq_process_rx(ioq_vector->oq, budget);
390
391 /* need more polling if tx completion processing is still pending or
392 * processed at least 'budget' number of rx packets.
393 */
394 if (tx_pending || rx_done >= budget)
395 return budget;
396
397 napi_complete(napi);
398 octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
399 return rx_done;
400}
401
402/**
403 * octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
404 *
405 * @oct: Octeon device private data structure.
406 */
407static void octep_napi_add(struct octep_device *oct)
408{
409 int i;
410
411 for (i = 0; i < oct->num_oqs; i++) {
412 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
413 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
414 octep_napi_poll);
415 oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
416 }
417}
418
419/**
420 * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
421 *
422 * @oct: Octeon device private data structure.
423 */
424static void octep_napi_delete(struct octep_device *oct)
425{
426 int i;
427
428 for (i = 0; i < oct->num_oqs; i++) {
429 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
430 netif_napi_del(&oct->ioq_vector[i]->napi);
431 oct->oq[i]->napi = NULL;
432 }
433}
434
435/**
436 * octep_napi_enable() - enable NAPI for all Tx/Rx queues.
437 *
438 * @oct: Octeon device private data structure.
439 */
440static void octep_napi_enable(struct octep_device *oct)
441{
442 int i;
443
444 for (i = 0; i < oct->num_oqs; i++) {
445 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
446 napi_enable(&oct->ioq_vector[i]->napi);
447 }
448}
449
450/**
451 * octep_napi_disable() - disable NAPI for all Tx/Rx queues.
452 *
453 * @oct: Octeon device private data structure.
454 */
455static void octep_napi_disable(struct octep_device *oct)
456{
457 int i;
458
459 for (i = 0; i < oct->num_oqs; i++) {
460 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
461 napi_disable(&oct->ioq_vector[i]->napi);
462 }
463}
464
465static void octep_link_up(struct net_device *netdev)
466{
467 netif_carrier_on(netdev);
468 netif_tx_start_all_queues(netdev);
469}
470
471/**
472 * octep_open() - start the octeon network device.
473 *
474 * @netdev: pointer to kernel network device.
475 *
476 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
477 * and interrupts..
478 *
479 * Return: 0, on successfully setting up device and bring it up.
480 * -1, on any error.
481 */
482static int octep_open(struct net_device *netdev)
483{
484 struct octep_device *oct = netdev_priv(netdev);
485 int err, ret;
486
487 netdev_info(netdev, "Starting netdev ...\n");
488 netif_carrier_off(netdev);
489
490 oct->hw_ops.reset_io_queues(oct);
491
492 if (octep_setup_iqs(oct))
493 goto setup_iq_err;
494 if (octep_setup_oqs(oct))
495 goto setup_oq_err;
496 if (octep_setup_irqs(oct))
497 goto setup_irq_err;
498
499 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
500 if (err)
501 goto set_queues_err;
502 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
503 if (err)
504 goto set_queues_err;
505
506 octep_napi_add(oct);
507 octep_napi_enable(oct);
508
509 oct->link_info.admin_up = 1;
510 octep_set_rx_state(oct, true);
511
512 ret = octep_get_link_status(oct);
513 if (!ret)
514 octep_set_link_status(oct, true);
515
516 /* Enable the input and output queues for this Octeon device */
517 oct->hw_ops.enable_io_queues(oct);
518
519 /* Enable Octeon device interrupts */
520 oct->hw_ops.enable_interrupts(oct);
521
522 octep_oq_dbell_init(oct);
523
524 ret = octep_get_link_status(oct);
525 if (ret > 0)
526 octep_link_up(netdev);
527
528 return 0;
529
530set_queues_err:
531 octep_clean_irqs(oct);
532setup_irq_err:
533 octep_free_oqs(oct);
534setup_oq_err:
535 octep_free_iqs(oct);
536setup_iq_err:
537 return -1;
538}
539
540/**
541 * octep_stop() - stop the octeon network device.
542 *
543 * @netdev: pointer to kernel network device.
544 *
545 * stop the device Tx/Rx operations, bring down the link and
546 * free up all resources allocated for Tx/Rx queues and interrupts.
547 */
548static int octep_stop(struct net_device *netdev)
549{
550 struct octep_device *oct = netdev_priv(netdev);
551
552 netdev_info(netdev, "Stopping the device ...\n");
553
554 /* Stop Tx from stack */
555 netif_tx_stop_all_queues(netdev);
556 netif_carrier_off(netdev);
557 netif_tx_disable(netdev);
558
559 octep_set_link_status(oct, false);
560 octep_set_rx_state(oct, false);
561
562 oct->link_info.admin_up = 0;
563 oct->link_info.oper_up = 0;
564
565 oct->hw_ops.disable_interrupts(oct);
566 octep_napi_disable(oct);
567 octep_napi_delete(oct);
568
569 octep_clean_irqs(oct);
570 octep_clean_iqs(oct);
571
572 oct->hw_ops.disable_io_queues(oct);
573 oct->hw_ops.reset_io_queues(oct);
574 octep_free_oqs(oct);
575 octep_free_iqs(oct);
576 netdev_info(netdev, "Device stopped !!\n");
577 return 0;
578}
579
580/**
581 * octep_iq_full_check() - check if a Tx queue is full.
582 *
583 * @iq: Octeon Tx queue data structure.
584 *
585 * Return: 0, if the Tx queue is not full.
586 * 1, if the Tx queue is full.
587 */
588static inline int octep_iq_full_check(struct octep_iq *iq)
589{
590 if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >=
591 OCTEP_WAKE_QUEUE_THRESHOLD))
592 return 0;
593
594 /* Stop the queue if unable to send */
595 netif_stop_subqueue(iq->netdev, iq->q_no);
596
597 /* check again and restart the queue, in case NAPI has just freed
598 * enough Tx ring entries.
599 */
600 if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >=
601 OCTEP_WAKE_QUEUE_THRESHOLD)) {
602 netif_start_subqueue(iq->netdev, iq->q_no);
603 iq->stats.restart_cnt++;
604 return 0;
605 }
606
607 return 1;
608}
609
610/**
611 * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
612 *
613 * @skb: packet skbuff pointer.
614 * @netdev: kernel network device.
615 *
616 * Return: NETDEV_TX_BUSY, if Tx Queue is full.
617 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
618 */
619static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
620 struct net_device *netdev)
621{
622 struct octep_device *oct = netdev_priv(netdev);
623 struct octep_tx_sglist_desc *sglist;
624 struct octep_tx_buffer *tx_buffer;
625 struct octep_tx_desc_hw *hw_desc;
626 struct skb_shared_info *shinfo;
627 struct octep_instr_hdr *ih;
628 struct octep_iq *iq;
629 skb_frag_t *frag;
630 u16 nr_frags, si;
631 u16 q_no, wi;
632
633 q_no = skb_get_queue_mapping(skb);
634 if (q_no >= oct->num_iqs) {
635 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
636 q_no = q_no % oct->num_iqs;
637 }
638
639 iq = oct->iq[q_no];
640 if (octep_iq_full_check(iq)) {
641 iq->stats.tx_busy++;
642 return NETDEV_TX_BUSY;
643 }
644
645 shinfo = skb_shinfo(skb);
646 nr_frags = shinfo->nr_frags;
647
648 wi = iq->host_write_index;
649 hw_desc = &iq->desc_ring[wi];
650 hw_desc->ih64 = 0;
651
652 tx_buffer = iq->buff_info + wi;
653 tx_buffer->skb = skb;
654
655 ih = &hw_desc->ih;
656 ih->tlen = skb->len;
657 ih->pkind = oct->pkind;
658
659 if (!nr_frags) {
660 tx_buffer->gather = 0;
661 tx_buffer->dma = dma_map_single(iq->dev, skb->data,
662 skb->len, DMA_TO_DEVICE);
663 if (dma_mapping_error(iq->dev, tx_buffer->dma))
664 goto dma_map_err;
665 hw_desc->dptr = tx_buffer->dma;
666 } else {
667 /* Scatter/Gather */
668 dma_addr_t dma;
669 u16 len;
670
671 sglist = tx_buffer->sglist;
672
673 ih->gsz = nr_frags + 1;
674 ih->gather = 1;
675 tx_buffer->gather = 1;
676
677 len = skb_headlen(skb);
678 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
679 if (dma_mapping_error(iq->dev, dma))
680 goto dma_map_err;
681
682 dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma,
683 OCTEP_SGLIST_SIZE_PER_PKT,
684 DMA_TO_DEVICE);
685 memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
686 sglist[0].len[3] = len;
687 sglist[0].dma_ptr[0] = dma;
688
689 si = 1; /* entry 0 is main skb, mapped above */
690 frag = &shinfo->frags[0];
691 while (nr_frags--) {
692 len = skb_frag_size(frag);
693 dma = skb_frag_dma_map(iq->dev, frag, 0,
694 len, DMA_TO_DEVICE);
695 if (dma_mapping_error(iq->dev, dma))
696 goto dma_map_sg_err;
697
698 sglist[si >> 2].len[3 - (si & 3)] = len;
699 sglist[si >> 2].dma_ptr[si & 3] = dma;
700
701 frag++;
702 si++;
703 }
704 dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma,
705 OCTEP_SGLIST_SIZE_PER_PKT,
706 DMA_TO_DEVICE);
707
708 hw_desc->dptr = tx_buffer->sglist_dma;
709 }
710
711 /* Flush the hw descriptor before writing to doorbell */
712 wmb();
713
714 /* Ring Doorbell to notify the NIC there is a new packet */
715 writel(1, iq->doorbell_reg);
716 atomic_inc(&iq->instr_pending);
717 wi++;
718 if (wi == iq->max_count)
719 wi = 0;
720 iq->host_write_index = wi;
721
722 netdev_tx_sent_queue(iq->netdev_q, skb->len);
723 iq->stats.instr_posted++;
724 skb_tx_timestamp(skb);
725 return NETDEV_TX_OK;
726
727dma_map_sg_err:
728 if (si > 0) {
729 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
730 sglist[0].len[0], DMA_TO_DEVICE);
731 sglist[0].len[0] = 0;
732 }
733 while (si > 1) {
734 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
735 sglist[si >> 2].len[si & 3], DMA_TO_DEVICE);
736 sglist[si >> 2].len[si & 3] = 0;
737 si--;
738 }
739 tx_buffer->gather = 0;
740dma_map_err:
741 dev_kfree_skb_any(skb);
742 return NETDEV_TX_OK;
743}
744
745/**
746 * octep_get_stats64() - Get Octeon network device statistics.
747 *
748 * @netdev: kernel network device.
749 * @stats: pointer to stats structure to be filled in.
750 */
751static void octep_get_stats64(struct net_device *netdev,
752 struct rtnl_link_stats64 *stats)
753{
754 u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
755 struct octep_device *oct = netdev_priv(netdev);
756 int q;
757
758 octep_get_if_stats(oct);
759 tx_packets = 0;
760 tx_bytes = 0;
761 rx_packets = 0;
762 rx_bytes = 0;
763 for (q = 0; q < oct->num_oqs; q++) {
764 struct octep_iq *iq = oct->iq[q];
765 struct octep_oq *oq = oct->oq[q];
766
767 tx_packets += iq->stats.instr_completed;
768 tx_bytes += iq->stats.bytes_sent;
769 rx_packets += oq->stats.packets;
770 rx_bytes += oq->stats.bytes;
771 }
772 stats->tx_packets = tx_packets;
773 stats->tx_bytes = tx_bytes;
774 stats->rx_packets = rx_packets;
775 stats->rx_bytes = rx_bytes;
776 stats->multicast = oct->iface_rx_stats.mcast_pkts;
777 stats->rx_errors = oct->iface_rx_stats.err_pkts;
778 stats->collisions = oct->iface_tx_stats.xscol;
779 stats->tx_fifo_errors = oct->iface_tx_stats.undflw;
780}
781
782/**
783 * octep_tx_timeout_task - work queue task to Handle Tx queue timeout.
784 *
785 * @work: pointer to Tx queue timeout work_struct
786 *
787 * Stop and start the device so that it frees up all queue resources
788 * and restarts the queues, that potentially clears a Tx queue timeout
789 * condition.
790 **/
791static void octep_tx_timeout_task(struct work_struct *work)
792{
793 struct octep_device *oct = container_of(work, struct octep_device,
794 tx_timeout_task);
795 struct net_device *netdev = oct->netdev;
796
797 rtnl_lock();
798 if (netif_running(netdev)) {
799 octep_stop(netdev);
800 octep_open(netdev);
801 }
802 rtnl_unlock();
803}
804
805/**
806 * octep_tx_timeout() - Handle Tx Queue timeout.
807 *
808 * @netdev: pointer to kernel network device.
809 * @txqueue: Timed out Tx queue number.
810 *
811 * Schedule a work to handle Tx queue timeout.
812 */
813static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue)
814{
815 struct octep_device *oct = netdev_priv(netdev);
816
817 queue_work(octep_wq, &oct->tx_timeout_task);
818}
819
820static int octep_set_mac(struct net_device *netdev, void *p)
821{
822 struct octep_device *oct = netdev_priv(netdev);
823 struct sockaddr *addr = (struct sockaddr *)p;
824 int err;
825
826 if (!is_valid_ether_addr(addr->sa_data))
827 return -EADDRNOTAVAIL;
828
829 err = octep_set_mac_addr(oct, addr->sa_data);
830 if (err)
831 return err;
832
833 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
834 eth_hw_addr_set(netdev, addr->sa_data);
835
836 return 0;
837}
838
839static int octep_change_mtu(struct net_device *netdev, int new_mtu)
840{
841 struct octep_device *oct = netdev_priv(netdev);
842 struct octep_iface_link_info *link_info;
843 int err = 0;
844
845 link_info = &oct->link_info;
846 if (link_info->mtu == new_mtu)
847 return 0;
848
849 err = octep_set_mtu(oct, new_mtu);
850 if (!err) {
851 oct->link_info.mtu = new_mtu;
852 netdev->mtu = new_mtu;
853 }
854
855 return err;
856}
857
858static const struct net_device_ops octep_netdev_ops = {
859 .ndo_open = octep_open,
860 .ndo_stop = octep_stop,
861 .ndo_start_xmit = octep_start_xmit,
862 .ndo_get_stats64 = octep_get_stats64,
863 .ndo_tx_timeout = octep_tx_timeout,
864 .ndo_set_mac_address = octep_set_mac,
865 .ndo_change_mtu = octep_change_mtu,
866};
867
868/**
869 * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
870 *
871 * @work: pointer to ctrl mbox work_struct
872 *
873 * Poll ctrl mbox message queue and handle control messages from firmware.
874 **/
875static void octep_ctrl_mbox_task(struct work_struct *work)
876{
877 struct octep_device *oct = container_of(work, struct octep_device,
878 ctrl_mbox_task);
879 struct net_device *netdev = oct->netdev;
880 struct octep_ctrl_net_f2h_req req = {};
881 struct octep_ctrl_mbox_msg msg;
882 int ret = 0;
883
884 msg.msg = &req;
885 while (true) {
886 ret = octep_ctrl_mbox_recv(&oct->ctrl_mbox, &msg);
887 if (ret)
888 break;
889
890 switch (req.hdr.cmd) {
891 case OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS:
892 if (netif_running(netdev)) {
893 if (req.link.state) {
894 dev_info(&oct->pdev->dev, "netif_carrier_on\n");
895 netif_carrier_on(netdev);
896 } else {
897 dev_info(&oct->pdev->dev, "netif_carrier_off\n");
898 netif_carrier_off(netdev);
899 }
900 }
901 break;
902 default:
903 pr_info("Unknown mbox req : %u\n", req.hdr.cmd);
904 break;
905 }
906 }
907}
908
909static const char *octep_devid_to_str(struct octep_device *oct)
910{
911 switch (oct->chip_id) {
912 case OCTEP_PCI_DEVICE_ID_CN93_PF:
913 return "CN93XX";
914 case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
915 return "CNF95N";
916 default:
917 return "Unsupported";
918 }
919}
920
921/**
922 * octep_device_setup() - Setup Octeon Device.
923 *
924 * @oct: Octeon device private data structure.
925 *
926 * Setup Octeon device hardware operations, configuration, etc ...
927 */
928int octep_device_setup(struct octep_device *oct)
929{
930 struct octep_ctrl_mbox *ctrl_mbox;
931 struct pci_dev *pdev = oct->pdev;
932 int i, ret;
933
934 /* allocate memory for oct->conf */
935 oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);
936 if (!oct->conf)
937 return -ENOMEM;
938
939 /* Map BAR regions */
940 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
941 oct->mmio[i].hw_addr =
942 ioremap(pci_resource_start(oct->pdev, i * 2),
943 pci_resource_len(oct->pdev, i * 2));
944 oct->mmio[i].mapped = 1;
945 }
946
947 oct->chip_id = pdev->device;
948 oct->rev_id = pdev->revision;
949 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
950
951 switch (oct->chip_id) {
952 case OCTEP_PCI_DEVICE_ID_CN93_PF:
953 case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
954 dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
955 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct),
956 OCTEP_MINOR_REV(oct));
957 octep_device_setup_cn93_pf(oct);
958 break;
959 default:
960 dev_err(&pdev->dev,
961 "%s: unsupported device\n", __func__);
962 goto unsupported_dev;
963 }
964
965 oct->pkind = CFG_GET_IQ_PKIND(oct->conf);
966
967 /* Initialize control mbox */
968 ctrl_mbox = &oct->ctrl_mbox;
969 ctrl_mbox->barmem = CFG_GET_CTRL_MBOX_MEM_ADDR(oct->conf);
970 ret = octep_ctrl_mbox_init(ctrl_mbox);
971 if (ret) {
972 dev_err(&pdev->dev, "Failed to initialize control mbox\n");
973 goto unsupported_dev;
974 }
975 oct->ctrl_mbox_ifstats_offset = OCTEP_CTRL_MBOX_SZ(ctrl_mbox->h2fq.elem_sz,
976 ctrl_mbox->h2fq.elem_cnt,
977 ctrl_mbox->f2hq.elem_sz,
978 ctrl_mbox->f2hq.elem_cnt);
979
980 return 0;
981
982unsupported_dev:
983 for (i = 0; i < OCTEP_MMIO_REGIONS; i++)
984 iounmap(oct->mmio[i].hw_addr);
985
986 kfree(oct->conf);
987 return -1;
988}
989
990/**
991 * octep_device_cleanup() - Cleanup Octeon Device.
992 *
993 * @oct: Octeon device private data structure.
994 *
995 * Cleanup Octeon device allocated resources.
996 */
997static void octep_device_cleanup(struct octep_device *oct)
998{
999 int i;
1000
1001 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
1002
1003 for (i = 0; i < OCTEP_MAX_VF; i++) {
1004 vfree(oct->mbox[i]);
1005 oct->mbox[i] = NULL;
1006 }
1007
1008 octep_ctrl_mbox_uninit(&oct->ctrl_mbox);
1009
1010 oct->hw_ops.soft_reset(oct);
1011 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1012 if (oct->mmio[i].mapped)
1013 iounmap(oct->mmio[i].hw_addr);
1014 }
1015
1016 kfree(oct->conf);
1017 oct->conf = NULL;
1018}
1019
1020/**
1021 * octep_probe() - Octeon PCI device probe handler.
1022 *
1023 * @pdev: PCI device structure.
1024 * @ent: entry in Octeon PCI device ID table.
1025 *
1026 * Initializes and enables the Octeon PCI device for network operations.
1027 * Initializes Octeon private data structure and registers a network device.
1028 */
1029static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1030{
1031 struct octep_device *octep_dev = NULL;
1032 struct net_device *netdev;
1033 int err;
1034
1035 err = pci_enable_device(pdev);
1036 if (err) {
1037 dev_err(&pdev->dev, "Failed to enable PCI device\n");
1038 return err;
1039 }
1040
1041 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1042 if (err) {
1043 dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1044 goto err_dma_mask;
1045 }
1046
1047 err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME);
1048 if (err) {
1049 dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1050 goto err_pci_regions;
1051 }
1052
1053 pci_enable_pcie_error_reporting(pdev);
1054 pci_set_master(pdev);
1055
1056 netdev = alloc_etherdev_mq(sizeof(struct octep_device),
1057 OCTEP_MAX_QUEUES);
1058 if (!netdev) {
1059 dev_err(&pdev->dev, "Failed to allocate netdev\n");
1060 err = -ENOMEM;
1061 goto err_alloc_netdev;
1062 }
1063 SET_NETDEV_DEV(netdev, &pdev->dev);
1064
1065 octep_dev = netdev_priv(netdev);
1066 octep_dev->netdev = netdev;
1067 octep_dev->pdev = pdev;
1068 octep_dev->dev = &pdev->dev;
1069 pci_set_drvdata(pdev, octep_dev);
1070
1071 err = octep_device_setup(octep_dev);
1072 if (err) {
1073 dev_err(&pdev->dev, "Device setup failed\n");
1074 goto err_octep_config;
1075 }
1076 INIT_WORK(&octep_dev->tx_timeout_task, octep_tx_timeout_task);
1077 INIT_WORK(&octep_dev->ctrl_mbox_task, octep_ctrl_mbox_task);
1078
1079 netdev->netdev_ops = &octep_netdev_ops;
1080 octep_set_ethtool_ops(netdev);
1081 netif_carrier_off(netdev);
1082
1083 netdev->hw_features = NETIF_F_SG;
1084 netdev->features |= netdev->hw_features;
1085 netdev->min_mtu = OCTEP_MIN_MTU;
1086 netdev->max_mtu = OCTEP_MAX_MTU;
1087 netdev->mtu = OCTEP_DEFAULT_MTU;
1088
1089 err = octep_get_mac_addr(octep_dev, octep_dev->mac_addr);
1090 if (err) {
1091 dev_err(&pdev->dev, "Failed to get mac address\n");
1092 goto register_dev_err;
1093 }
1094 eth_hw_addr_set(netdev, octep_dev->mac_addr);
1095
1096 err = register_netdev(netdev);
1097 if (err) {
1098 dev_err(&pdev->dev, "Failed to register netdev\n");
1099 goto register_dev_err;
1100 }
1101 dev_info(&pdev->dev, "Device probe successful\n");
1102 return 0;
1103
1104register_dev_err:
1105 octep_device_cleanup(octep_dev);
1106err_octep_config:
1107 free_netdev(netdev);
1108err_alloc_netdev:
1109 pci_disable_pcie_error_reporting(pdev);
1110 pci_release_mem_regions(pdev);
1111err_pci_regions:
1112err_dma_mask:
1113 pci_disable_device(pdev);
1114 return err;
1115}
1116
1117/**
1118 * octep_remove() - Remove Octeon PCI device from driver control.
1119 *
1120 * @pdev: PCI device structure of the Octeon device.
1121 *
1122 * Cleanup all resources allocated for the Octeon device.
1123 * Unregister from network device and disable the PCI device.
1124 */
1125static void octep_remove(struct pci_dev *pdev)
1126{
1127 struct octep_device *oct = pci_get_drvdata(pdev);
1128 struct net_device *netdev;
1129
1130 if (!oct)
1131 return;
1132
1133 cancel_work_sync(&oct->tx_timeout_task);
1134 cancel_work_sync(&oct->ctrl_mbox_task);
1135 netdev = oct->netdev;
1136 if (netdev->reg_state == NETREG_REGISTERED)
1137 unregister_netdev(netdev);
1138
1139 octep_device_cleanup(oct);
1140 pci_release_mem_regions(pdev);
1141 free_netdev(netdev);
1142 pci_disable_pcie_error_reporting(pdev);
1143 pci_disable_device(pdev);
1144}
1145
1146static struct pci_driver octep_driver = {
1147 .name = OCTEP_DRV_NAME,
1148 .id_table = octep_pci_id_tbl,
1149 .probe = octep_probe,
1150 .remove = octep_remove,
1151};
1152
1153/**
1154 * octep_init_module() - Module initialiation.
1155 *
1156 * create common resource for the driver and register PCI driver.
1157 */
1158static int __init octep_init_module(void)
1159{
1160 int ret;
1161
1162 pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING);
1163
1164 /* work queue for all deferred tasks */
1165 octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME);
1166 if (!octep_wq) {
1167 pr_err("%s: Failed to create common workqueue\n",
1168 OCTEP_DRV_NAME);
1169 return -ENOMEM;
1170 }
1171
1172 ret = pci_register_driver(&octep_driver);
1173 if (ret < 0) {
1174 pr_err("%s: Failed to register PCI driver; err=%d\n",
1175 OCTEP_DRV_NAME, ret);
1176 destroy_workqueue(octep_wq);
1177 return ret;
1178 }
1179
1180 pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME);
1181
1182 return ret;
1183}
1184
1185/**
1186 * octep_exit_module() - Module exit routine.
1187 *
1188 * unregister the driver with PCI subsystem and cleanup common resources.
1189 */
1190static void __exit octep_exit_module(void)
1191{
1192 pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME);
1193
1194 pci_unregister_driver(&octep_driver);
1195 destroy_workqueue(octep_wq);
1196
1197 pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME);
1198}
1199
1200module_init(octep_init_module);
1201module_exit(octep_exit_module);