1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) Meta Platforms, Inc. and affiliates. */
  3
  4#include <linux/init.h>
  5#include <linux/module.h>
  6#include <linux/pci.h>
  7#include <linux/rtnetlink.h>
  8#include <linux/types.h>
  9
 10#include "fbnic.h"
 11#include "fbnic_drvinfo.h"
 12#include "fbnic_hw_stats.h"
 13#include "fbnic_netdev.h"
 14
 15char fbnic_driver_name[] = DRV_NAME;
 16
 17MODULE_DESCRIPTION(DRV_SUMMARY);
 18MODULE_LICENSE("GPL");
 19
 20static const struct fbnic_info fbnic_asic_info = {
 21	.max_num_queues = FBNIC_MAX_QUEUES,
 22	.bar_mask = BIT(0) | BIT(4)
 23};
 24
 25static const struct fbnic_info *fbnic_info_tbl[] = {
 26	[fbnic_board_asic] = &fbnic_asic_info,
 27};
 28
 29static const struct pci_device_id fbnic_pci_tbl[] = {
 30	{ PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) },
 31	/* Required last entry */
 32	{0, }
 33};
 34MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl);
 35
 36u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg)
 37{
 38	u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);
 39	u32 value;
 40
 41	if (!csr)
 42		return ~0U;
 43
 44	value = readl(csr + reg);
 45
 46	/* If any bits are 0 value should be valid */
 47	if (~value)
 48		return value;
 49
 50	/* All 1's may be valid if ZEROs register still works */
 51	if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0))
 52		return value;
 53
 54	/* Hardware is giving us all 1's reads, assume it is gone */
 55	WRITE_ONCE(fbd->uc_addr0, NULL);
 56	WRITE_ONCE(fbd->uc_addr4, NULL);
 57
 58	dev_err(fbd->dev,
 59		"Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
 60		reg, reg << 2);
 61
 62	/* Notify stack that device has lost (PCIe) link */
 63	if (!fbnic_init_failure(fbd))
 64		netif_device_detach(fbd->netdev);
 65
 66	return ~0U;
 67}
 68
 69bool fbnic_fw_present(struct fbnic_dev *fbd)
 70{
 71	return !!READ_ONCE(fbd->uc_addr4);
 72}
 73
 74void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val)
 75{
 76	u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
 77
 78	if (csr)
 79		writel(val, csr + reg);
 80}
 81
 82u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg)
 83{
 84	u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
 85	u32 value;
 86
 87	if (!csr)
 88		return ~0U;
 89
 90	value = readl(csr + reg);
 91
 92	/* If any bits are 0 value should be valid */
 93	if (~value)
 94		return value;
 95
 96	/* All 1's may be valid if ZEROs register still works */
 97	if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG))
 98		return value;
 99
100	/* Hardware is giving us all 1's reads, assume it is gone */
101	WRITE_ONCE(fbd->uc_addr0, NULL);
102	WRITE_ONCE(fbd->uc_addr4, NULL);
103
104	dev_err(fbd->dev,
105		"Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
106		reg, reg << 2);
107
108	/* Notify stack that device has lost (PCIe) link */
109	if (!fbnic_init_failure(fbd))
110		netif_device_detach(fbd->netdev);
111
112	return ~0U;
113}
114
115static void fbnic_service_task_start(struct fbnic_net *fbn)
116{
117	struct fbnic_dev *fbd = fbn->fbd;
118
119	schedule_delayed_work(&fbd->service_task, HZ);
120	phylink_resume(fbn->phylink);
121}
122
123static void fbnic_service_task_stop(struct fbnic_net *fbn)
124{
125	struct fbnic_dev *fbd = fbn->fbd;
126
127	phylink_suspend(fbn->phylink, fbnic_bmc_present(fbd));
128	cancel_delayed_work(&fbd->service_task);
129}
130
131void fbnic_up(struct fbnic_net *fbn)
132{
133	fbnic_enable(fbn);
134
135	fbnic_fill(fbn);
136
137	fbnic_rss_reinit_hw(fbn->fbd, fbn);
138
139	__fbnic_set_rx_mode(fbn->netdev);
140
141	/* Enable Tx/Rx processing */
142	fbnic_napi_enable(fbn);
143	netif_tx_start_all_queues(fbn->netdev);
144
145	fbnic_service_task_start(fbn);
146}
147
148static void fbnic_down_noidle(struct fbnic_net *fbn)
149{
150	fbnic_service_task_stop(fbn);
151
152	/* Disable Tx/Rx Processing */
153	fbnic_napi_disable(fbn);
154	netif_tx_disable(fbn->netdev);
155
156	fbnic_clear_rx_mode(fbn->netdev);
157	fbnic_clear_rules(fbn->fbd);
158	fbnic_rss_disable_hw(fbn->fbd);
159	fbnic_disable(fbn);
160}
161
162void fbnic_down(struct fbnic_net *fbn)
163{
164	fbnic_down_noidle(fbn);
165
166	fbnic_wait_all_queues_idle(fbn->fbd, false);
167
168	fbnic_flush(fbn);
169}
170
171static void fbnic_health_check(struct fbnic_dev *fbd)
172{
173	struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX];
174
175	/* As long as the heart is beating the FW is healty */
176	if (fbd->fw_heartbeat_enabled)
177		return;
178
179	/* If the Tx mailbox still has messages sitting in it then there likely
180	 * isn't anything we can do. We will wait until the mailbox is empty to
181	 * report the fault so we can collect the crashlog.
182	 */
183	if (tx_mbx->head != tx_mbx->tail)
184		return;
185
186	/* TBD: Need to add a more thorough recovery here.
187	 *	Specifically I need to verify what all the firmware will have
188	 *	changed since we had setup and it rebooted. May just need to
189	 *	perform a down/up. For now we will just reclaim ownership so
190	 *	the heartbeat can catch the next fault.
191	 */
192	fbnic_fw_xmit_ownership_msg(fbd, true);
193}
194
195static void fbnic_service_task(struct work_struct *work)
196{
197	struct fbnic_dev *fbd = container_of(to_delayed_work(work),
198					     struct fbnic_dev, service_task);
199
200	rtnl_lock();
201
202	fbnic_get_hw_stats32(fbd);
203
204	fbnic_fw_check_heartbeat(fbd);
205
206	fbnic_health_check(fbd);
207
208	if (netif_carrier_ok(fbd->netdev))
209		fbnic_napi_depletion_check(fbd->netdev);
210
211	if (netif_running(fbd->netdev))
212		schedule_delayed_work(&fbd->service_task, HZ);
213
214	rtnl_unlock();
215}
216
217/**
218 * fbnic_probe - Device Initialization Routine
219 * @pdev: PCI device information struct
220 * @ent: entry in fbnic_pci_tbl
221 *
222 * Initializes a PCI device identified by a pci_dev structure.
223 * The OS initialization, configuring of the adapter private structure,
224 * and a hardware reset occur.
225 *
226 * Return: 0 on success, negative on failure
227 **/
228static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
229{
230	const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data];
231	struct net_device *netdev;
232	struct fbnic_dev *fbd;
233	int err;
234
235	if (pdev->error_state != pci_channel_io_normal) {
236		dev_err(&pdev->dev,
237			"PCI device still in an error state. Unable to load...\n");
238		return -EIO;
239	}
240
241	err = pcim_enable_device(pdev);
242	if (err) {
243		dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);
244		return err;
245	}
246
247	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
248	if (err)
249		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
250	if (err) {
251		dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
252		return err;
253	}
254
255	err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name);
256	if (err) {
257		dev_err(&pdev->dev,
258			"pci_request_selected_regions failed: %d\n", err);
259		return err;
260	}
261
262	fbd = fbnic_devlink_alloc(pdev);
263	if (!fbd) {
264		dev_err(&pdev->dev, "Devlink allocation failed\n");
265		return -ENOMEM;
266	}
267
268	/* Populate driver with hardware-specific info and handlers */
269	fbd->max_num_queues = info->max_num_queues;
270
271	pci_set_master(pdev);
272	pci_save_state(pdev);
273
274	INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task);
275
276	err = fbnic_alloc_irqs(fbd);
277	if (err)
278		goto free_fbd;
279
280	err = fbnic_mac_init(fbd);
281	if (err) {
282		dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err);
283		goto free_irqs;
284	}
285
286	err = fbnic_fw_enable_mbx(fbd);
287	if (err) {
288		dev_err(&pdev->dev,
289			"Firmware mailbox initialization failure\n");
290		goto free_irqs;
291	}
292
293	fbnic_devlink_register(fbd);
294	fbnic_dbg_fbd_init(fbd);
295
296	/* Capture snapshot of hardware stats so netdev can calculate delta */
297	fbnic_reset_hw_stats(fbd);
298
299	if (!fbd->dsn) {
300		dev_warn(&pdev->dev, "Reading serial number failed\n");
301		goto init_failure_mode;
302	}
303
304	netdev = fbnic_netdev_alloc(fbd);
305	if (!netdev) {
306		dev_err(&pdev->dev, "Netdev allocation failed\n");
307		goto init_failure_mode;
308	}
309
310	err = fbnic_ptp_setup(fbd);
311	if (err)
312		goto ifm_free_netdev;
313
314	err = fbnic_netdev_register(netdev);
315	if (err) {
316		dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
317		goto ifm_destroy_ptp;
318	}
319
320	return 0;
321
322ifm_destroy_ptp:
323	fbnic_ptp_destroy(fbd);
324ifm_free_netdev:
325	fbnic_netdev_free(fbd);
326init_failure_mode:
327	dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n");
328	 /* Always return 0 even on error so devlink is registered to allow
329	  * firmware updates for fixes.
330	  */
331	return 0;
332free_irqs:
333	fbnic_free_irqs(fbd);
334free_fbd:
335	fbnic_devlink_free(fbd);
336
337	return err;
338}
339
340/**
341 * fbnic_remove - Device Removal Routine
342 * @pdev: PCI device information struct
343 *
344 * Called by the PCI subsystem to alert the driver that it should release
345 * a PCI device.  The could be caused by a Hot-Plug event, or because the
346 * driver is going to be removed from memory.
347 **/
348static void fbnic_remove(struct pci_dev *pdev)
349{
350	struct fbnic_dev *fbd = pci_get_drvdata(pdev);
351
352	if (!fbnic_init_failure(fbd)) {
353		struct net_device *netdev = fbd->netdev;
354
355		fbnic_netdev_unregister(netdev);
356		cancel_delayed_work_sync(&fbd->service_task);
357		fbnic_ptp_destroy(fbd);
358		fbnic_netdev_free(fbd);
359	}
360
361	fbnic_dbg_fbd_exit(fbd);
362	fbnic_devlink_unregister(fbd);
363	fbnic_fw_disable_mbx(fbd);
364	fbnic_free_irqs(fbd);
365
366	fbnic_devlink_free(fbd);
367}
368
369static int fbnic_pm_suspend(struct device *dev)
370{
371	struct fbnic_dev *fbd = dev_get_drvdata(dev);
372	struct net_device *netdev = fbd->netdev;
373
374	if (fbnic_init_failure(fbd))
375		goto null_uc_addr;
376
377	rtnl_lock();
378
379	netif_device_detach(netdev);
380
381	if (netif_running(netdev))
382		netdev->netdev_ops->ndo_stop(netdev);
383
384	rtnl_unlock();
385
386null_uc_addr:
387	fbnic_fw_disable_mbx(fbd);
388
389	/* Free the IRQs so they aren't trying to occupy sleeping CPUs */
390	fbnic_free_irqs(fbd);
391
392	/* Hardware is about to go away, so switch off MMIO access internally */
393	WRITE_ONCE(fbd->uc_addr0, NULL);
394	WRITE_ONCE(fbd->uc_addr4, NULL);
395
396	return 0;
397}
398
399static int __fbnic_pm_resume(struct device *dev)
400{
401	struct fbnic_dev *fbd = dev_get_drvdata(dev);
402	struct net_device *netdev = fbd->netdev;
403	void __iomem * const *iomap_table;
404	struct fbnic_net *fbn;
405	int err;
406
407	/* Restore MMIO access */
408	iomap_table = pcim_iomap_table(to_pci_dev(dev));
409	fbd->uc_addr0 = iomap_table[0];
410	fbd->uc_addr4 = iomap_table[4];
411
412	/* Rerequest the IRQs */
413	err = fbnic_alloc_irqs(fbd);
414	if (err)
415		goto err_invalidate_uc_addr;
416
417	fbd->mac->init_regs(fbd);
418
419	/* Re-enable mailbox */
420	err = fbnic_fw_enable_mbx(fbd);
421	if (err)
422		goto err_free_irqs;
423
424	/* No netdev means there isn't a network interface to bring up */
425	if (fbnic_init_failure(fbd))
426		return 0;
427
428	fbn = netdev_priv(netdev);
429
430	/* Reset the queues if needed */
431	fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);
432
433	rtnl_lock();
434
435	if (netif_running(netdev)) {
436		err = __fbnic_open(fbn);
437		if (err)
438			goto err_disable_mbx;
439	}
440
441	rtnl_unlock();
442
443	return 0;
444err_disable_mbx:
445	rtnl_unlock();
446	fbnic_fw_disable_mbx(fbd);
447err_free_irqs:
448	fbnic_free_irqs(fbd);
449err_invalidate_uc_addr:
450	WRITE_ONCE(fbd->uc_addr0, NULL);
451	WRITE_ONCE(fbd->uc_addr4, NULL);
452	return err;
453}
454
455static void __fbnic_pm_attach(struct device *dev)
456{
457	struct fbnic_dev *fbd = dev_get_drvdata(dev);
458	struct net_device *netdev = fbd->netdev;
459	struct fbnic_net *fbn;
460
461	if (fbnic_init_failure(fbd))
462		return;
463
464	fbn = netdev_priv(netdev);
465
466	if (netif_running(netdev))
467		fbnic_up(fbn);
468
469	netif_device_attach(netdev);
470}
471
472static int __maybe_unused fbnic_pm_resume(struct device *dev)
473{
474	int err;
475
476	err = __fbnic_pm_resume(dev);
477	if (!err)
478		__fbnic_pm_attach(dev);
479
480	return err;
481}
482
483static const struct dev_pm_ops fbnic_pm_ops = {
484	SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume)
485};
486
487static void fbnic_shutdown(struct pci_dev *pdev)
488{
489	fbnic_pm_suspend(&pdev->dev);
490}
491
492static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev,
493						 pci_channel_state_t state)
494{
495	/* Disconnect device if failure is not recoverable via reset */
496	if (state == pci_channel_io_perm_failure)
497		return PCI_ERS_RESULT_DISCONNECT;
498
499	fbnic_pm_suspend(&pdev->dev);
500
501	/* Request a slot reset */
502	return PCI_ERS_RESULT_NEED_RESET;
503}
504
505static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev)
506{
507	int err;
508
509	pci_set_power_state(pdev, PCI_D0);
510	pci_restore_state(pdev);
511	pci_save_state(pdev);
512
513	if (pci_enable_device_mem(pdev)) {
514		dev_err(&pdev->dev,
515			"Cannot re-enable PCI device after reset.\n");
516		return PCI_ERS_RESULT_DISCONNECT;
517	}
518
519	/* Restore device to previous state */
520	err = __fbnic_pm_resume(&pdev->dev);
521
522	return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
523}
524
525static void fbnic_err_resume(struct pci_dev *pdev)
526{
527	__fbnic_pm_attach(&pdev->dev);
528}
529
530static const struct pci_error_handlers fbnic_err_handler = {
531	.error_detected	= fbnic_err_error_detected,
532	.slot_reset	= fbnic_err_slot_reset,
533	.resume		= fbnic_err_resume,
534};
535
536static struct pci_driver fbnic_driver = {
537	.name		= fbnic_driver_name,
538	.id_table	= fbnic_pci_tbl,
539	.probe		= fbnic_probe,
540	.remove		= fbnic_remove,
541	.driver.pm	= &fbnic_pm_ops,
542	.shutdown	= fbnic_shutdown,
543	.err_handler	= &fbnic_err_handler,
544};
545
546/**
547 * fbnic_init_module - Driver Registration Routine
548 *
549 * The first routine called when the driver is loaded.  All it does is
550 * register with the PCI subsystem.
551 *
552 * Return: 0 on success, negative on failure
553 **/
554static int __init fbnic_init_module(void)
555{
556	int err;
557
558	fbnic_dbg_init();
559
560	err = pci_register_driver(&fbnic_driver);
561	if (err) {
562		fbnic_dbg_exit();
563		goto out;
564	}
565
566	pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name);
567out:
568	return err;
569}
570module_init(fbnic_init_module);
571
572/**
573 * fbnic_exit_module - Driver Exit Cleanup Routine
574 *
575 * Called just before the driver is removed from memory.
576 **/
577static void __exit fbnic_exit_module(void)
578{
579	pci_unregister_driver(&fbnic_driver);
580
581	fbnic_dbg_exit();
582}
583module_exit(fbnic_exit_module);