1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 *  ahci.c - AHCI SATA support
   4 *
   5 *  Maintained by:  Tejun Heo <tj@kernel.org>
   6 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
   7 *		    on emails.
   8 *
   9 *  Copyright 2004-2005 Red Hat, Inc.
  10 *
  11 * libata documentation is available via 'make {ps|pdf}docs',
  12 * as Documentation/driver-api/libata.rst
  13 *
  14 * AHCI hardware documentation:
  15 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
  16 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
  17 */
  18
  19#include <linux/kernel.h>
  20#include <linux/module.h>
  21#include <linux/pci.h>
  22#include <linux/blkdev.h>
  23#include <linux/delay.h>
  24#include <linux/interrupt.h>
  25#include <linux/dma-mapping.h>
  26#include <linux/device.h>
  27#include <linux/dmi.h>
  28#include <linux/gfp.h>
  29#include <scsi/scsi_host.h>
  30#include <scsi/scsi_cmnd.h>
  31#include <linux/libata.h>
  32#include <linux/ahci-remap.h>
  33#include <linux/io-64-nonatomic-lo-hi.h>
  34#include "ahci.h"
  35
  36#define DRV_NAME	"ahci"
  37#define DRV_VERSION	"3.0"
  38
  39enum {
  40	AHCI_PCI_BAR_STA2X11	= 0,
  41	AHCI_PCI_BAR_CAVIUM	= 0,
  42	AHCI_PCI_BAR_LOONGSON	= 0,
  43	AHCI_PCI_BAR_ENMOTUS	= 2,
  44	AHCI_PCI_BAR_CAVIUM_GEN5	= 4,
  45	AHCI_PCI_BAR_STANDARD	= 5,
  46};
  47
  48enum board_ids {
  49	/* board IDs by feature in alphabetical order */
  50	board_ahci,
  51	board_ahci_43bit_dma,
  52	board_ahci_ign_iferr,
  53	board_ahci_low_power,
  54	board_ahci_no_debounce_delay,
  55	board_ahci_nomsi,
  56	board_ahci_noncq,
  57	board_ahci_nosntf,
  58	board_ahci_yes_fbs,
  59
  60	/* board IDs for specific chipsets in alphabetical order */
  61	board_ahci_al,
  62	board_ahci_avn,
  63	board_ahci_mcp65,
  64	board_ahci_mcp77,
  65	board_ahci_mcp89,
  66	board_ahci_mv,
  67	board_ahci_sb600,
  68	board_ahci_sb700,	/* for SB700 and SB800 */
  69	board_ahci_vt8251,
  70
  71	/*
  72	 * board IDs for Intel chipsets that support more than 6 ports
  73	 * *and* end up needing the PCS quirk.
  74	 */
  75	board_ahci_pcs7,
  76
  77	/* aliases */
  78	board_ahci_mcp_linux	= board_ahci_mcp65,
  79	board_ahci_mcp67	= board_ahci_mcp65,
  80	board_ahci_mcp73	= board_ahci_mcp65,
  81	board_ahci_mcp79	= board_ahci_mcp77,
  82};
  83
  84static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
  85static void ahci_remove_one(struct pci_dev *dev);
  86static void ahci_shutdown_one(struct pci_dev *dev);
  87static void ahci_intel_pcs_quirk(struct pci_dev *pdev, struct ahci_host_priv *hpriv);
  88static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
  89				 unsigned long deadline);
  90static int ahci_avn_hardreset(struct ata_link *link, unsigned int *class,
  91			      unsigned long deadline);
  92static void ahci_mcp89_apple_enable(struct pci_dev *pdev);
  93static bool is_mcp89_apple(struct pci_dev *pdev);
  94static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
  95				unsigned long deadline);
  96#ifdef CONFIG_PM
  97static int ahci_pci_device_runtime_suspend(struct device *dev);
  98static int ahci_pci_device_runtime_resume(struct device *dev);
  99#ifdef CONFIG_PM_SLEEP
 100static int ahci_pci_device_suspend(struct device *dev);
 101static int ahci_pci_device_resume(struct device *dev);
 102#endif
 103#endif /* CONFIG_PM */
 104
 105static const struct scsi_host_template ahci_sht = {
 106	AHCI_SHT("ahci"),
 107};
 108
 109static struct ata_port_operations ahci_vt8251_ops = {
 110	.inherits		= &ahci_ops,
 111	.hardreset		= ahci_vt8251_hardreset,
 112};
 113
 114static struct ata_port_operations ahci_p5wdh_ops = {
 115	.inherits		= &ahci_ops,
 116	.hardreset		= ahci_p5wdh_hardreset,
 117};
 118
 119static struct ata_port_operations ahci_avn_ops = {
 120	.inherits		= &ahci_ops,
 121	.hardreset		= ahci_avn_hardreset,
 122};
 123
 124static const struct ata_port_info ahci_port_info[] = {
 125	/* by features */
 126	[board_ahci] = {
 127		.flags		= AHCI_FLAG_COMMON,
 128		.pio_mask	= ATA_PIO4,
 129		.udma_mask	= ATA_UDMA6,
 130		.port_ops	= &ahci_ops,
 131	},
 132	[board_ahci_43bit_dma] = {
 133		AHCI_HFLAGS	(AHCI_HFLAG_43BIT_ONLY),
 134		.flags		= AHCI_FLAG_COMMON,
 135		.pio_mask	= ATA_PIO4,
 136		.udma_mask	= ATA_UDMA6,
 137		.port_ops	= &ahci_ops,
 138	},
 139	[board_ahci_ign_iferr] = {
 140		AHCI_HFLAGS	(AHCI_HFLAG_IGN_IRQ_IF_ERR),
 141		.flags		= AHCI_FLAG_COMMON,
 142		.pio_mask	= ATA_PIO4,
 143		.udma_mask	= ATA_UDMA6,
 144		.port_ops	= &ahci_ops,
 145	},
 146	[board_ahci_low_power] = {
 147		AHCI_HFLAGS	(AHCI_HFLAG_USE_LPM_POLICY),
 148		.flags		= AHCI_FLAG_COMMON,
 149		.pio_mask	= ATA_PIO4,
 150		.udma_mask	= ATA_UDMA6,
 151		.port_ops	= &ahci_ops,
 152	},
 153	[board_ahci_no_debounce_delay] = {
 154		.flags		= AHCI_FLAG_COMMON,
 155		.link_flags	= ATA_LFLAG_NO_DEBOUNCE_DELAY,
 156		.pio_mask	= ATA_PIO4,
 157		.udma_mask	= ATA_UDMA6,
 158		.port_ops	= &ahci_ops,
 159	},
 160	[board_ahci_nomsi] = {
 161		AHCI_HFLAGS	(AHCI_HFLAG_NO_MSI),
 162		.flags		= AHCI_FLAG_COMMON,
 163		.pio_mask	= ATA_PIO4,
 164		.udma_mask	= ATA_UDMA6,
 165		.port_ops	= &ahci_ops,
 166	},
 167	[board_ahci_noncq] = {
 168		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ),
 169		.flags		= AHCI_FLAG_COMMON,
 170		.pio_mask	= ATA_PIO4,
 171		.udma_mask	= ATA_UDMA6,
 172		.port_ops	= &ahci_ops,
 173	},
 174	[board_ahci_nosntf] = {
 175		AHCI_HFLAGS	(AHCI_HFLAG_NO_SNTF),
 176		.flags		= AHCI_FLAG_COMMON,
 177		.pio_mask	= ATA_PIO4,
 178		.udma_mask	= ATA_UDMA6,
 179		.port_ops	= &ahci_ops,
 180	},
 181	[board_ahci_yes_fbs] = {
 182		AHCI_HFLAGS	(AHCI_HFLAG_YES_FBS),
 183		.flags		= AHCI_FLAG_COMMON,
 184		.pio_mask	= ATA_PIO4,
 185		.udma_mask	= ATA_UDMA6,
 186		.port_ops	= &ahci_ops,
 187	},
 188	/* by chipsets */
 189	[board_ahci_al] = {
 190		AHCI_HFLAGS	(AHCI_HFLAG_NO_PMP | AHCI_HFLAG_NO_MSI),
 191		.flags		= AHCI_FLAG_COMMON,
 192		.pio_mask	= ATA_PIO4,
 193		.udma_mask	= ATA_UDMA6,
 194		.port_ops	= &ahci_ops,
 195	},
 196	[board_ahci_avn] = {
 197		.flags		= AHCI_FLAG_COMMON,
 198		.pio_mask	= ATA_PIO4,
 199		.udma_mask	= ATA_UDMA6,
 200		.port_ops	= &ahci_avn_ops,
 201	},
 202	[board_ahci_mcp65] = {
 203		AHCI_HFLAGS	(AHCI_HFLAG_NO_FPDMA_AA | AHCI_HFLAG_NO_PMP |
 204				 AHCI_HFLAG_YES_NCQ),
 205		.flags		= AHCI_FLAG_COMMON | ATA_FLAG_NO_DIPM,
 206		.pio_mask	= ATA_PIO4,
 207		.udma_mask	= ATA_UDMA6,
 208		.port_ops	= &ahci_ops,
 209	},
 210	[board_ahci_mcp77] = {
 211		AHCI_HFLAGS	(AHCI_HFLAG_NO_FPDMA_AA | AHCI_HFLAG_NO_PMP),
 212		.flags		= AHCI_FLAG_COMMON,
 213		.pio_mask	= ATA_PIO4,
 214		.udma_mask	= ATA_UDMA6,
 215		.port_ops	= &ahci_ops,
 216	},
 217	[board_ahci_mcp89] = {
 218		AHCI_HFLAGS	(AHCI_HFLAG_NO_FPDMA_AA),
 219		.flags		= AHCI_FLAG_COMMON,
 220		.pio_mask	= ATA_PIO4,
 221		.udma_mask	= ATA_UDMA6,
 222		.port_ops	= &ahci_ops,
 223	},
 224	[board_ahci_mv] = {
 225		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
 226				 AHCI_HFLAG_MV_PATA | AHCI_HFLAG_NO_PMP),
 227		.flags		= ATA_FLAG_SATA | ATA_FLAG_PIO_DMA,
 228		.pio_mask	= ATA_PIO4,
 229		.udma_mask	= ATA_UDMA6,
 230		.port_ops	= &ahci_ops,
 231	},
 232	[board_ahci_sb600] = {
 233		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
 234				 AHCI_HFLAG_NO_MSI | AHCI_HFLAG_SECT255 |
 235				 AHCI_HFLAG_32BIT_ONLY),
 236		.flags		= AHCI_FLAG_COMMON,
 237		.pio_mask	= ATA_PIO4,
 238		.udma_mask	= ATA_UDMA6,
 239		.port_ops	= &ahci_pmp_retry_srst_ops,
 240	},
 241	[board_ahci_sb700] = {	/* for SB700 and SB800 */
 242		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL),
 243		.flags		= AHCI_FLAG_COMMON,
 244		.pio_mask	= ATA_PIO4,
 245		.udma_mask	= ATA_UDMA6,
 246		.port_ops	= &ahci_pmp_retry_srst_ops,
 247	},
 248	[board_ahci_vt8251] = {
 249		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
 250		.flags		= AHCI_FLAG_COMMON,
 251		.pio_mask	= ATA_PIO4,
 252		.udma_mask	= ATA_UDMA6,
 253		.port_ops	= &ahci_vt8251_ops,
 254	},
 255	[board_ahci_pcs7] = {
 256		.flags		= AHCI_FLAG_COMMON,
 257		.pio_mask	= ATA_PIO4,
 258		.udma_mask	= ATA_UDMA6,
 259		.port_ops	= &ahci_ops,
 260	},
 261};
 262
 263static const struct pci_device_id ahci_pci_tbl[] = {
 264	/* Intel */
 265	{ PCI_VDEVICE(INTEL, 0x06d6), board_ahci }, /* Comet Lake PCH-H RAID */
 266	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
 267	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
 268	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
 269	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
 270	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
 271	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
 272	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
 273	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
 274	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
 275	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
 276	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */
 277	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci_nosntf }, /* ICH8/Lewisburg RAID*/
 278	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */
 279	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */
 280	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */
 281	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */
 282	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */
 283	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
 284	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
 285	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
 286	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci_low_power }, /* ICH9M */
 287	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci_low_power }, /* ICH9M */
 288	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci_low_power }, /* ICH9M */
 289	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci_low_power }, /* ICH9M */
 290	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci_low_power }, /* ICH9M */
 291	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
 292	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci_low_power }, /* ICH9M */
 293	{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
 294	{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
 295	{ PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */
 296	{ PCI_VDEVICE(INTEL, 0x3a22), board_ahci }, /* ICH10 */
 297	{ PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */
 298	{ PCI_VDEVICE(INTEL, 0x3b22), board_ahci }, /* PCH AHCI */
 299	{ PCI_VDEVICE(INTEL, 0x3b23), board_ahci }, /* PCH AHCI */
 300	{ PCI_VDEVICE(INTEL, 0x3b24), board_ahci }, /* PCH RAID */
 301	{ PCI_VDEVICE(INTEL, 0x3b25), board_ahci }, /* PCH RAID */
 302	{ PCI_VDEVICE(INTEL, 0x3b29), board_ahci_low_power }, /* PCH M AHCI */
 303	{ PCI_VDEVICE(INTEL, 0x3b2b), board_ahci }, /* PCH RAID */
 304	{ PCI_VDEVICE(INTEL, 0x3b2c), board_ahci_low_power }, /* PCH M RAID */
 305	{ PCI_VDEVICE(INTEL, 0x3b2f), board_ahci }, /* PCH AHCI */
 306	{ PCI_VDEVICE(INTEL, 0x19b0), board_ahci_pcs7 }, /* DNV AHCI */
 307	{ PCI_VDEVICE(INTEL, 0x19b1), board_ahci_pcs7 }, /* DNV AHCI */
 308	{ PCI_VDEVICE(INTEL, 0x19b2), board_ahci_pcs7 }, /* DNV AHCI */
 309	{ PCI_VDEVICE(INTEL, 0x19b3), board_ahci_pcs7 }, /* DNV AHCI */
 310	{ PCI_VDEVICE(INTEL, 0x19b4), board_ahci_pcs7 }, /* DNV AHCI */
 311	{ PCI_VDEVICE(INTEL, 0x19b5), board_ahci_pcs7 }, /* DNV AHCI */
 312	{ PCI_VDEVICE(INTEL, 0x19b6), board_ahci_pcs7 }, /* DNV AHCI */
 313	{ PCI_VDEVICE(INTEL, 0x19b7), board_ahci_pcs7 }, /* DNV AHCI */
 314	{ PCI_VDEVICE(INTEL, 0x19bE), board_ahci_pcs7 }, /* DNV AHCI */
 315	{ PCI_VDEVICE(INTEL, 0x19bF), board_ahci_pcs7 }, /* DNV AHCI */
 316	{ PCI_VDEVICE(INTEL, 0x19c0), board_ahci_pcs7 }, /* DNV AHCI */
 317	{ PCI_VDEVICE(INTEL, 0x19c1), board_ahci_pcs7 }, /* DNV AHCI */
 318	{ PCI_VDEVICE(INTEL, 0x19c2), board_ahci_pcs7 }, /* DNV AHCI */
 319	{ PCI_VDEVICE(INTEL, 0x19c3), board_ahci_pcs7 }, /* DNV AHCI */
 320	{ PCI_VDEVICE(INTEL, 0x19c4), board_ahci_pcs7 }, /* DNV AHCI */
 321	{ PCI_VDEVICE(INTEL, 0x19c5), board_ahci_pcs7 }, /* DNV AHCI */
 322	{ PCI_VDEVICE(INTEL, 0x19c6), board_ahci_pcs7 }, /* DNV AHCI */
 323	{ PCI_VDEVICE(INTEL, 0x19c7), board_ahci_pcs7 }, /* DNV AHCI */
 324	{ PCI_VDEVICE(INTEL, 0x19cE), board_ahci_pcs7 }, /* DNV AHCI */
 325	{ PCI_VDEVICE(INTEL, 0x19cF), board_ahci_pcs7 }, /* DNV AHCI */
 326	{ PCI_VDEVICE(INTEL, 0x1c02), board_ahci }, /* CPT AHCI */
 327	{ PCI_VDEVICE(INTEL, 0x1c03), board_ahci_low_power }, /* CPT M AHCI */
 328	{ PCI_VDEVICE(INTEL, 0x1c04), board_ahci }, /* CPT RAID */
 329	{ PCI_VDEVICE(INTEL, 0x1c05), board_ahci_low_power }, /* CPT M RAID */
 330	{ PCI_VDEVICE(INTEL, 0x1c06), board_ahci }, /* CPT RAID */
 331	{ PCI_VDEVICE(INTEL, 0x1c07), board_ahci }, /* CPT RAID */
 332	{ PCI_VDEVICE(INTEL, 0x1d02), board_ahci }, /* PBG AHCI */
 333	{ PCI_VDEVICE(INTEL, 0x1d04), board_ahci }, /* PBG RAID */
 334	{ PCI_VDEVICE(INTEL, 0x1d06), board_ahci }, /* PBG RAID */
 335	{ PCI_VDEVICE(INTEL, 0x2323), board_ahci }, /* DH89xxCC AHCI */
 336	{ PCI_VDEVICE(INTEL, 0x1e02), board_ahci }, /* Panther Point AHCI */
 337	{ PCI_VDEVICE(INTEL, 0x1e03), board_ahci_low_power }, /* Panther M AHCI */
 338	{ PCI_VDEVICE(INTEL, 0x1e04), board_ahci }, /* Panther Point RAID */
 339	{ PCI_VDEVICE(INTEL, 0x1e05), board_ahci }, /* Panther Point RAID */
 340	{ PCI_VDEVICE(INTEL, 0x1e06), board_ahci }, /* Panther Point RAID */
 341	{ PCI_VDEVICE(INTEL, 0x1e07), board_ahci_low_power }, /* Panther M RAID */
 342	{ PCI_VDEVICE(INTEL, 0x1e0e), board_ahci }, /* Panther Point RAID */
 343	{ PCI_VDEVICE(INTEL, 0x8c02), board_ahci }, /* Lynx Point AHCI */
 344	{ PCI_VDEVICE(INTEL, 0x8c03), board_ahci_low_power }, /* Lynx M AHCI */
 345	{ PCI_VDEVICE(INTEL, 0x8c04), board_ahci }, /* Lynx Point RAID */
 346	{ PCI_VDEVICE(INTEL, 0x8c05), board_ahci_low_power }, /* Lynx M RAID */
 347	{ PCI_VDEVICE(INTEL, 0x8c06), board_ahci }, /* Lynx Point RAID */
 348	{ PCI_VDEVICE(INTEL, 0x8c07), board_ahci_low_power }, /* Lynx M RAID */
 349	{ PCI_VDEVICE(INTEL, 0x8c0e), board_ahci }, /* Lynx Point RAID */
 350	{ PCI_VDEVICE(INTEL, 0x8c0f), board_ahci_low_power }, /* Lynx M RAID */
 351	{ PCI_VDEVICE(INTEL, 0x9c02), board_ahci_low_power }, /* Lynx LP AHCI */
 352	{ PCI_VDEVICE(INTEL, 0x9c03), board_ahci_low_power }, /* Lynx LP AHCI */
 353	{ PCI_VDEVICE(INTEL, 0x9c04), board_ahci_low_power }, /* Lynx LP RAID */
 354	{ PCI_VDEVICE(INTEL, 0x9c05), board_ahci_low_power }, /* Lynx LP RAID */
 355	{ PCI_VDEVICE(INTEL, 0x9c06), board_ahci_low_power }, /* Lynx LP RAID */
 356	{ PCI_VDEVICE(INTEL, 0x9c07), board_ahci_low_power }, /* Lynx LP RAID */
 357	{ PCI_VDEVICE(INTEL, 0x9c0e), board_ahci_low_power }, /* Lynx LP RAID */
 358	{ PCI_VDEVICE(INTEL, 0x9c0f), board_ahci_low_power }, /* Lynx LP RAID */
 359	{ PCI_VDEVICE(INTEL, 0x9dd3), board_ahci_low_power }, /* Cannon Lake PCH-LP AHCI */
 360	{ PCI_VDEVICE(INTEL, 0x1f22), board_ahci }, /* Avoton AHCI */
 361	{ PCI_VDEVICE(INTEL, 0x1f23), board_ahci }, /* Avoton AHCI */
 362	{ PCI_VDEVICE(INTEL, 0x1f24), board_ahci }, /* Avoton RAID */
 363	{ PCI_VDEVICE(INTEL, 0x1f25), board_ahci }, /* Avoton RAID */
 364	{ PCI_VDEVICE(INTEL, 0x1f26), board_ahci }, /* Avoton RAID */
 365	{ PCI_VDEVICE(INTEL, 0x1f27), board_ahci }, /* Avoton RAID */
 366	{ PCI_VDEVICE(INTEL, 0x1f2e), board_ahci }, /* Avoton RAID */
 367	{ PCI_VDEVICE(INTEL, 0x1f2f), board_ahci }, /* Avoton RAID */
 368	{ PCI_VDEVICE(INTEL, 0x1f32), board_ahci_avn }, /* Avoton AHCI */
 369	{ PCI_VDEVICE(INTEL, 0x1f33), board_ahci_avn }, /* Avoton AHCI */
 370	{ PCI_VDEVICE(INTEL, 0x1f34), board_ahci_avn }, /* Avoton RAID */
 371	{ PCI_VDEVICE(INTEL, 0x1f35), board_ahci_avn }, /* Avoton RAID */
 372	{ PCI_VDEVICE(INTEL, 0x1f36), board_ahci_avn }, /* Avoton RAID */
 373	{ PCI_VDEVICE(INTEL, 0x1f37), board_ahci_avn }, /* Avoton RAID */
 374	{ PCI_VDEVICE(INTEL, 0x1f3e), board_ahci_avn }, /* Avoton RAID */
 375	{ PCI_VDEVICE(INTEL, 0x1f3f), board_ahci_avn }, /* Avoton RAID */
 376	{ PCI_VDEVICE(INTEL, 0x2823), board_ahci }, /* Wellsburg/Lewisburg AHCI*/
 377	{ PCI_VDEVICE(INTEL, 0x2826), board_ahci }, /* *burg SATA0 'RAID' */
 378	{ PCI_VDEVICE(INTEL, 0x2827), board_ahci }, /* *burg SATA1 'RAID' */
 379	{ PCI_VDEVICE(INTEL, 0x282f), board_ahci }, /* *burg SATA2 'RAID' */
 380	{ PCI_VDEVICE(INTEL, 0x43d4), board_ahci }, /* Rocket Lake PCH-H RAID */
 381	{ PCI_VDEVICE(INTEL, 0x43d5), board_ahci }, /* Rocket Lake PCH-H RAID */
 382	{ PCI_VDEVICE(INTEL, 0x43d6), board_ahci }, /* Rocket Lake PCH-H RAID */
 383	{ PCI_VDEVICE(INTEL, 0x43d7), board_ahci }, /* Rocket Lake PCH-H RAID */
 384	{ PCI_VDEVICE(INTEL, 0x8d02), board_ahci }, /* Wellsburg AHCI */
 385	{ PCI_VDEVICE(INTEL, 0x8d04), board_ahci }, /* Wellsburg RAID */
 386	{ PCI_VDEVICE(INTEL, 0x8d06), board_ahci }, /* Wellsburg RAID */
 387	{ PCI_VDEVICE(INTEL, 0x8d0e), board_ahci }, /* Wellsburg RAID */
 388	{ PCI_VDEVICE(INTEL, 0x8d62), board_ahci }, /* Wellsburg AHCI */
 389	{ PCI_VDEVICE(INTEL, 0x8d64), board_ahci }, /* Wellsburg RAID */
 390	{ PCI_VDEVICE(INTEL, 0x8d66), board_ahci }, /* Wellsburg RAID */
 391	{ PCI_VDEVICE(INTEL, 0x8d6e), board_ahci }, /* Wellsburg RAID */
 392	{ PCI_VDEVICE(INTEL, 0x23a3), board_ahci }, /* Coleto Creek AHCI */
 393	{ PCI_VDEVICE(INTEL, 0x9c83), board_ahci_low_power }, /* Wildcat LP AHCI */
 394	{ PCI_VDEVICE(INTEL, 0x9c85), board_ahci_low_power }, /* Wildcat LP RAID */
 395	{ PCI_VDEVICE(INTEL, 0x9c87), board_ahci_low_power }, /* Wildcat LP RAID */
 396	{ PCI_VDEVICE(INTEL, 0x9c8f), board_ahci_low_power }, /* Wildcat LP RAID */
 397	{ PCI_VDEVICE(INTEL, 0x8c82), board_ahci }, /* 9 Series AHCI */
 398	{ PCI_VDEVICE(INTEL, 0x8c83), board_ahci_low_power }, /* 9 Series M AHCI */
 399	{ PCI_VDEVICE(INTEL, 0x8c84), board_ahci }, /* 9 Series RAID */
 400	{ PCI_VDEVICE(INTEL, 0x8c85), board_ahci_low_power }, /* 9 Series M RAID */
 401	{ PCI_VDEVICE(INTEL, 0x8c86), board_ahci }, /* 9 Series RAID */
 402	{ PCI_VDEVICE(INTEL, 0x8c87), board_ahci_low_power }, /* 9 Series M RAID */
 403	{ PCI_VDEVICE(INTEL, 0x8c8e), board_ahci }, /* 9 Series RAID */
 404	{ PCI_VDEVICE(INTEL, 0x8c8f), board_ahci_low_power }, /* 9 Series M RAID */
 405	{ PCI_VDEVICE(INTEL, 0x9d03), board_ahci_low_power }, /* Sunrise LP AHCI */
 406	{ PCI_VDEVICE(INTEL, 0x9d05), board_ahci_low_power }, /* Sunrise LP RAID */
 407	{ PCI_VDEVICE(INTEL, 0x9d07), board_ahci_low_power }, /* Sunrise LP RAID */
 408	{ PCI_VDEVICE(INTEL, 0xa102), board_ahci }, /* Sunrise Point-H AHCI */
 409	{ PCI_VDEVICE(INTEL, 0xa103), board_ahci_low_power }, /* Sunrise M AHCI */
 410	{ PCI_VDEVICE(INTEL, 0xa105), board_ahci }, /* Sunrise Point-H RAID */
 411	{ PCI_VDEVICE(INTEL, 0xa106), board_ahci }, /* Sunrise Point-H RAID */
 412	{ PCI_VDEVICE(INTEL, 0xa107), board_ahci_low_power }, /* Sunrise M RAID */
 413	{ PCI_VDEVICE(INTEL, 0xa10f), board_ahci }, /* Sunrise Point-H RAID */
 414	{ PCI_VDEVICE(INTEL, 0xa182), board_ahci }, /* Lewisburg AHCI*/
 415	{ PCI_VDEVICE(INTEL, 0xa186), board_ahci }, /* Lewisburg RAID*/
 416	{ PCI_VDEVICE(INTEL, 0xa1d2), board_ahci }, /* Lewisburg RAID*/
 417	{ PCI_VDEVICE(INTEL, 0xa1d6), board_ahci }, /* Lewisburg RAID*/
 418	{ PCI_VDEVICE(INTEL, 0xa202), board_ahci }, /* Lewisburg AHCI*/
 419	{ PCI_VDEVICE(INTEL, 0xa206), board_ahci }, /* Lewisburg RAID*/
 420	{ PCI_VDEVICE(INTEL, 0xa252), board_ahci }, /* Lewisburg RAID*/
 421	{ PCI_VDEVICE(INTEL, 0xa256), board_ahci }, /* Lewisburg RAID*/
 422	{ PCI_VDEVICE(INTEL, 0xa356), board_ahci }, /* Cannon Lake PCH-H RAID */
 423	{ PCI_VDEVICE(INTEL, 0x06d7), board_ahci }, /* Comet Lake-H RAID */
 424	{ PCI_VDEVICE(INTEL, 0xa386), board_ahci }, /* Comet Lake PCH-V RAID */
 425	{ PCI_VDEVICE(INTEL, 0x0f22), board_ahci_low_power }, /* Bay Trail AHCI */
 426	{ PCI_VDEVICE(INTEL, 0x0f23), board_ahci_low_power }, /* Bay Trail AHCI */
 427	{ PCI_VDEVICE(INTEL, 0x22a3), board_ahci_low_power }, /* Cherry Tr. AHCI */
 428	{ PCI_VDEVICE(INTEL, 0x5ae3), board_ahci_low_power }, /* ApolloLake AHCI */
 429	{ PCI_VDEVICE(INTEL, 0x34d3), board_ahci_low_power }, /* Ice Lake LP AHCI */
 430	{ PCI_VDEVICE(INTEL, 0x02d3), board_ahci_low_power }, /* Comet Lake PCH-U AHCI */
 431	{ PCI_VDEVICE(INTEL, 0x02d7), board_ahci_low_power }, /* Comet Lake PCH RAID */
 432	/* Elkhart Lake IDs 0x4b60 & 0x4b62 https://sata-io.org/product/8803 not tested yet */
 433	{ PCI_VDEVICE(INTEL, 0x4b63), board_ahci_low_power }, /* Elkhart Lake AHCI */
 434	{ PCI_VDEVICE(INTEL, 0x7ae2), board_ahci_low_power }, /* Alder Lake-P AHCI */
 435
 436	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
 437	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
 438	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
 439	/* JMicron 362B and 362C have an AHCI function with IDE class code */
 440	{ PCI_VDEVICE(JMICRON, 0x2362), board_ahci_ign_iferr },
 441	{ PCI_VDEVICE(JMICRON, 0x236f), board_ahci_ign_iferr },
 442	/* May need to update quirk_jmicron_async_suspend() for additions */
 443
 444	/* ATI */
 445	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
 446	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb700 }, /* ATI SB700/800 */
 447	{ PCI_VDEVICE(ATI, 0x4391), board_ahci_sb700 }, /* ATI SB700/800 */
 448	{ PCI_VDEVICE(ATI, 0x4392), board_ahci_sb700 }, /* ATI SB700/800 */
 449	{ PCI_VDEVICE(ATI, 0x4393), board_ahci_sb700 }, /* ATI SB700/800 */
 450	{ PCI_VDEVICE(ATI, 0x4394), board_ahci_sb700 }, /* ATI SB700/800 */
 451	{ PCI_VDEVICE(ATI, 0x4395), board_ahci_sb700 }, /* ATI SB700/800 */
 452
 453	/* Amazon's Annapurna Labs support */
 454	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON_ANNAPURNA_LABS, 0x0031),
 455		.class = PCI_CLASS_STORAGE_SATA_AHCI,
 456		.class_mask = 0xffffff,
 457		board_ahci_al },
 458	/* AMD */
 459	{ PCI_VDEVICE(AMD, 0x7800), board_ahci }, /* AMD Hudson-2 */
 460	{ PCI_VDEVICE(AMD, 0x7801), board_ahci_no_debounce_delay }, /* AMD Hudson-2 (AHCI mode) */
 461	{ PCI_VDEVICE(AMD, 0x7900), board_ahci }, /* AMD CZ */
 462	{ PCI_VDEVICE(AMD, 0x7901), board_ahci_low_power }, /* AMD Green Sardine */
 463	/* AMD is using RAID class only for ahci controllers */
 464	{ PCI_VENDOR_ID_AMD, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
 465	  PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci },
 466
 467	/* Dell S140/S150 */
 468	{ PCI_VENDOR_ID_INTEL, PCI_ANY_ID, PCI_SUBVENDOR_ID_DELL, PCI_ANY_ID,
 469	  PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci },
 470
 471	/* VIA */
 472	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
 473	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
 474
 475	/* NVIDIA */
 476	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci_mcp65 },	/* MCP65 */
 477	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci_mcp65 },	/* MCP65 */
 478	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci_mcp65 },	/* MCP65 */
 479	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci_mcp65 },	/* MCP65 */
 480	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci_mcp65 },	/* MCP65 */
 481	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci_mcp65 },	/* MCP65 */
 482	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci_mcp65 },	/* MCP65 */
 483	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci_mcp65 },	/* MCP65 */
 484	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci_mcp67 },	/* MCP67 */
 485	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci_mcp67 },	/* MCP67 */
 486	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci_mcp67 },	/* MCP67 */
 487	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci_mcp67 },	/* MCP67 */
 488	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci_mcp67 },	/* MCP67 */
 489	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci_mcp67 },	/* MCP67 */
 490	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci_mcp67 },	/* MCP67 */
 491	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci_mcp67 },	/* MCP67 */
 492	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci_mcp67 },	/* MCP67 */
 493	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci_mcp67 },	/* MCP67 */
 494	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci_mcp67 },	/* MCP67 */
 495	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci_mcp67 },	/* MCP67 */
 496	{ PCI_VDEVICE(NVIDIA, 0x0580), board_ahci_mcp_linux },	/* Linux ID */
 497	{ PCI_VDEVICE(NVIDIA, 0x0581), board_ahci_mcp_linux },	/* Linux ID */
 498	{ PCI_VDEVICE(NVIDIA, 0x0582), board_ahci_mcp_linux },	/* Linux ID */
 499	{ PCI_VDEVICE(NVIDIA, 0x0583), board_ahci_mcp_linux },	/* Linux ID */
 500	{ PCI_VDEVICE(NVIDIA, 0x0584), board_ahci_mcp_linux },	/* Linux ID */
 501	{ PCI_VDEVICE(NVIDIA, 0x0585), board_ahci_mcp_linux },	/* Linux ID */
 502	{ PCI_VDEVICE(NVIDIA, 0x0586), board_ahci_mcp_linux },	/* Linux ID */
 503	{ PCI_VDEVICE(NVIDIA, 0x0587), board_ahci_mcp_linux },	/* Linux ID */
 504	{ PCI_VDEVICE(NVIDIA, 0x0588), board_ahci_mcp_linux },	/* Linux ID */
 505	{ PCI_VDEVICE(NVIDIA, 0x0589), board_ahci_mcp_linux },	/* Linux ID */
 506	{ PCI_VDEVICE(NVIDIA, 0x058a), board_ahci_mcp_linux },	/* Linux ID */
 507	{ PCI_VDEVICE(NVIDIA, 0x058b), board_ahci_mcp_linux },	/* Linux ID */
 508	{ PCI_VDEVICE(NVIDIA, 0x058c), board_ahci_mcp_linux },	/* Linux ID */
 509	{ PCI_VDEVICE(NVIDIA, 0x058d), board_ahci_mcp_linux },	/* Linux ID */
 510	{ PCI_VDEVICE(NVIDIA, 0x058e), board_ahci_mcp_linux },	/* Linux ID */
 511	{ PCI_VDEVICE(NVIDIA, 0x058f), board_ahci_mcp_linux },	/* Linux ID */
 512	{ PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci_mcp73 },	/* MCP73 */
 513	{ PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci_mcp73 },	/* MCP73 */
 514	{ PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci_mcp73 },	/* MCP73 */
 515	{ PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci_mcp73 },	/* MCP73 */
 516	{ PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci_mcp73 },	/* MCP73 */
 517	{ PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci_mcp73 },	/* MCP73 */
 518	{ PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci_mcp73 },	/* MCP73 */
 519	{ PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci_mcp73 },	/* MCP73 */
 520	{ PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci_mcp73 },	/* MCP73 */
 521	{ PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci_mcp73 },	/* MCP73 */
 522	{ PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci_mcp73 },	/* MCP73 */
 523	{ PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci_mcp73 },	/* MCP73 */
 524	{ PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci_mcp77 },	/* MCP77 */
 525	{ PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci_mcp77 },	/* MCP77 */
 526	{ PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci_mcp77 },	/* MCP77 */
 527	{ PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci_mcp77 },	/* MCP77 */
 528	{ PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci_mcp77 },	/* MCP77 */
 529	{ PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci_mcp77 },	/* MCP77 */
 530	{ PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci_mcp77 },	/* MCP77 */
 531	{ PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci_mcp77 },	/* MCP77 */
 532	{ PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci_mcp77 },	/* MCP77 */
 533	{ PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci_mcp77 },	/* MCP77 */
 534	{ PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci_mcp77 },	/* MCP77 */
 535	{ PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci_mcp77 },	/* MCP77 */
 536	{ PCI_VDEVICE(NVIDIA, 0x0ab4), board_ahci_mcp79 },	/* MCP79 */
 537	{ PCI_VDEVICE(NVIDIA, 0x0ab5), board_ahci_mcp79 },	/* MCP79 */
 538	{ PCI_VDEVICE(NVIDIA, 0x0ab6), board_ahci_mcp79 },	/* MCP79 */
 539	{ PCI_VDEVICE(NVIDIA, 0x0ab7), board_ahci_mcp79 },	/* MCP79 */
 540	{ PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci_mcp79 },	/* MCP79 */
 541	{ PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci_mcp79 },	/* MCP79 */
 542	{ PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci_mcp79 },	/* MCP79 */
 543	{ PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci_mcp79 },	/* MCP79 */
 544	{ PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci_mcp79 },	/* MCP79 */
 545	{ PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci_mcp79 },	/* MCP79 */
 546	{ PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci_mcp79 },	/* MCP79 */
 547	{ PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci_mcp79 },	/* MCP79 */
 548	{ PCI_VDEVICE(NVIDIA, 0x0d84), board_ahci_mcp89 },	/* MCP89 */
 549	{ PCI_VDEVICE(NVIDIA, 0x0d85), board_ahci_mcp89 },	/* MCP89 */
 550	{ PCI_VDEVICE(NVIDIA, 0x0d86), board_ahci_mcp89 },	/* MCP89 */
 551	{ PCI_VDEVICE(NVIDIA, 0x0d87), board_ahci_mcp89 },	/* MCP89 */
 552	{ PCI_VDEVICE(NVIDIA, 0x0d88), board_ahci_mcp89 },	/* MCP89 */
 553	{ PCI_VDEVICE(NVIDIA, 0x0d89), board_ahci_mcp89 },	/* MCP89 */
 554	{ PCI_VDEVICE(NVIDIA, 0x0d8a), board_ahci_mcp89 },	/* MCP89 */
 555	{ PCI_VDEVICE(NVIDIA, 0x0d8b), board_ahci_mcp89 },	/* MCP89 */
 556	{ PCI_VDEVICE(NVIDIA, 0x0d8c), board_ahci_mcp89 },	/* MCP89 */
 557	{ PCI_VDEVICE(NVIDIA, 0x0d8d), board_ahci_mcp89 },	/* MCP89 */
 558	{ PCI_VDEVICE(NVIDIA, 0x0d8e), board_ahci_mcp89 },	/* MCP89 */
 559	{ PCI_VDEVICE(NVIDIA, 0x0d8f), board_ahci_mcp89 },	/* MCP89 */
 560
 561	/* SiS */
 562	{ PCI_VDEVICE(SI, 0x1184), board_ahci },		/* SiS 966 */
 563	{ PCI_VDEVICE(SI, 0x1185), board_ahci },		/* SiS 968 */
 564	{ PCI_VDEVICE(SI, 0x0186), board_ahci },		/* SiS 968 */
 565
 566	/* ST Microelectronics */
 567	{ PCI_VDEVICE(STMICRO, 0xCC06), board_ahci },		/* ST ConneXt */
 568
 569	/* Marvell */
 570	{ PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },	/* 6145 */
 571	{ PCI_VDEVICE(MARVELL, 0x6121), board_ahci_mv },	/* 6121 */
 572	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9123),
 573	  .class = PCI_CLASS_STORAGE_SATA_AHCI,
 574	  .class_mask = 0xffffff,
 575	  .driver_data = board_ahci_yes_fbs },			/* 88se9128 */
 576	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9125),
 577	  .driver_data = board_ahci_yes_fbs },			/* 88se9125 */
 578	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_MARVELL_EXT, 0x9178,
 579			 PCI_VENDOR_ID_MARVELL_EXT, 0x9170),
 580	  .driver_data = board_ahci_yes_fbs },			/* 88se9170 */
 581	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x917a),
 582	  .driver_data = board_ahci_yes_fbs },			/* 88se9172 */
 583	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9172),
 584	  .driver_data = board_ahci_yes_fbs },			/* 88se9182 */
 585	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9182),
 586	  .driver_data = board_ahci_yes_fbs },			/* 88se9172 */
 587	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9192),
 588	  .driver_data = board_ahci_yes_fbs },			/* 88se9172 on some Gigabyte */
 589	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x91a0),
 590	  .driver_data = board_ahci_yes_fbs },
 591	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x91a2), 	/* 88se91a2 */
 592	  .driver_data = board_ahci_yes_fbs },
 593	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x91a3),
 594	  .driver_data = board_ahci_yes_fbs },
 595	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9230),
 596	  .driver_data = board_ahci_yes_fbs },
 597	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9235),
 598	  .driver_data = board_ahci_no_debounce_delay },
 599	{ PCI_DEVICE(PCI_VENDOR_ID_TTI, 0x0642), /* highpoint rocketraid 642L */
 600	  .driver_data = board_ahci_yes_fbs },
 601	{ PCI_DEVICE(PCI_VENDOR_ID_TTI, 0x0645), /* highpoint rocketraid 644L */
 602	  .driver_data = board_ahci_yes_fbs },
 603
 604	/* Promise */
 605	{ PCI_VDEVICE(PROMISE, 0x3f20), board_ahci },	/* PDC42819 */
 606	{ PCI_VDEVICE(PROMISE, 0x3781), board_ahci },   /* FastTrak TX8660 ahci-mode */
 607
 608	/* ASMedia */
 609	{ PCI_VDEVICE(ASMEDIA, 0x0601), board_ahci_43bit_dma },	/* ASM1060 */
 610	{ PCI_VDEVICE(ASMEDIA, 0x0602), board_ahci_43bit_dma },	/* ASM1060 */
 611	{ PCI_VDEVICE(ASMEDIA, 0x0611), board_ahci_43bit_dma },	/* ASM1061 */
 612	{ PCI_VDEVICE(ASMEDIA, 0x0612), board_ahci_43bit_dma },	/* ASM1061/1062 */
 613	{ PCI_VDEVICE(ASMEDIA, 0x0621), board_ahci_43bit_dma },	/* ASM1061R */
 614	{ PCI_VDEVICE(ASMEDIA, 0x0622), board_ahci_43bit_dma },	/* ASM1062R */
 615	{ PCI_VDEVICE(ASMEDIA, 0x0624), board_ahci_43bit_dma },	/* ASM1062+JMB575 */
 616	{ PCI_VDEVICE(ASMEDIA, 0x1062), board_ahci },	/* ASM1062A */
 617	{ PCI_VDEVICE(ASMEDIA, 0x1064), board_ahci },	/* ASM1064 */
 618	{ PCI_VDEVICE(ASMEDIA, 0x1164), board_ahci },   /* ASM1164 */
 619	{ PCI_VDEVICE(ASMEDIA, 0x1165), board_ahci },   /* ASM1165 */
 620	{ PCI_VDEVICE(ASMEDIA, 0x1166), board_ahci },   /* ASM1166 */
 621
 622	/*
 623	 * Samsung SSDs found on some macbooks.  NCQ times out if MSI is
 624	 * enabled.  https://bugzilla.kernel.org/show_bug.cgi?id=60731
 625	 */
 626	{ PCI_VDEVICE(SAMSUNG, 0x1600), board_ahci_nomsi },
 627	{ PCI_VDEVICE(SAMSUNG, 0xa800), board_ahci_nomsi },
 628
 629	/* Enmotus */
 630	{ PCI_DEVICE(0x1c44, 0x8000), board_ahci },
 631
 632	/* Loongson */
 633	{ PCI_VDEVICE(LOONGSON, 0x7a08), board_ahci },
 634
 635	/* Generic, PCI class code for AHCI */
 636	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
 637	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
 638
 639	{ }	/* terminate list */
 640};
 641
 642static const struct dev_pm_ops ahci_pci_pm_ops = {
 643	SET_SYSTEM_SLEEP_PM_OPS(ahci_pci_device_suspend, ahci_pci_device_resume)
 644	SET_RUNTIME_PM_OPS(ahci_pci_device_runtime_suspend,
 645			   ahci_pci_device_runtime_resume, NULL)
 646};
 647
 648static struct pci_driver ahci_pci_driver = {
 649	.name			= DRV_NAME,
 650	.id_table		= ahci_pci_tbl,
 651	.probe			= ahci_init_one,
 652	.remove			= ahci_remove_one,
 653	.shutdown		= ahci_shutdown_one,
 654	.driver = {
 655		.pm		= &ahci_pci_pm_ops,
 656	},
 657};
 658
 659#if IS_ENABLED(CONFIG_PATA_MARVELL)
 660static int marvell_enable;
 661#else
 662static int marvell_enable = 1;
 663#endif
 664module_param(marvell_enable, int, 0644);
 665MODULE_PARM_DESC(marvell_enable, "Marvell SATA via AHCI (1 = enabled)");
 666
 667static int mobile_lpm_policy = -1;
 668module_param(mobile_lpm_policy, int, 0644);
 669MODULE_PARM_DESC(mobile_lpm_policy, "Default LPM policy for mobile chipsets");
 670
 671static void ahci_pci_save_initial_config(struct pci_dev *pdev,
 672					 struct ahci_host_priv *hpriv)
 673{
 674	if (pdev->vendor == PCI_VENDOR_ID_ASMEDIA) {
 675		switch (pdev->device) {
 676		case 0x1166:
 677			dev_info(&pdev->dev, "ASM1166 has only six ports\n");
 678			hpriv->saved_port_map = 0x3f;
 679			break;
 680		case 0x1064:
 681			dev_info(&pdev->dev, "ASM1064 has only four ports\n");
 682			hpriv->saved_port_map = 0xf;
 683			break;
 684		}
 685	}
 686
 687	if (pdev->vendor == PCI_VENDOR_ID_JMICRON && pdev->device == 0x2361) {
 688		dev_info(&pdev->dev, "JMB361 has only one port\n");
 689		hpriv->saved_port_map = 1;
 690	}
 691
 692	/*
 693	 * Temporary Marvell 6145 hack: PATA port presence
 694	 * is asserted through the standard AHCI port
 695	 * presence register, as bit 4 (counting from 0)
 696	 */
 697	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
 698		if (pdev->device == 0x6121)
 699			hpriv->mask_port_map = 0x3;
 700		else
 701			hpriv->mask_port_map = 0xf;
 702		dev_info(&pdev->dev,
 703			  "Disabling your PATA port. Use the boot option 'ahci.marvell_enable=0' to avoid this.\n");
 704	}
 705
 706	ahci_save_initial_config(&pdev->dev, hpriv);
 707}
 708
 709static int ahci_pci_reset_controller(struct ata_host *host)
 710{
 711	struct pci_dev *pdev = to_pci_dev(host->dev);
 712	struct ahci_host_priv *hpriv = host->private_data;
 713	int rc;
 714
 715	rc = ahci_reset_controller(host);
 716	if (rc)
 717		return rc;
 718
 719	/*
 720	 * If platform firmware failed to enable ports, try to enable
 721	 * them here.
 722	 */
 723	ahci_intel_pcs_quirk(pdev, hpriv);
 724
 725	return 0;
 726}
 727
 728static void ahci_pci_init_controller(struct ata_host *host)
 729{
 730	struct ahci_host_priv *hpriv = host->private_data;
 731	struct pci_dev *pdev = to_pci_dev(host->dev);
 732	void __iomem *port_mmio;
 733	u32 tmp;
 734	int mv;
 735
 736	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
 737		if (pdev->device == 0x6121)
 738			mv = 2;
 739		else
 740			mv = 4;
 741		port_mmio = __ahci_port_base(hpriv, mv);
 742
 743		writel(0, port_mmio + PORT_IRQ_MASK);
 744
 745		/* clear port IRQ */
 746		tmp = readl(port_mmio + PORT_IRQ_STAT);
 747		dev_dbg(&pdev->dev, "PORT_IRQ_STAT 0x%x\n", tmp);
 748		if (tmp)
 749			writel(tmp, port_mmio + PORT_IRQ_STAT);
 750	}
 751
 752	ahci_init_controller(host);
 753}
 754
 755static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
 756				 unsigned long deadline)
 757{
 758	struct ata_port *ap = link->ap;
 759	struct ahci_host_priv *hpriv = ap->host->private_data;
 760	bool online;
 761	int rc;
 762
 763	hpriv->stop_engine(ap);
 764
 765	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
 766				 deadline, &online, NULL);
 767
 768	hpriv->start_engine(ap);
 769
 770	/* vt8251 doesn't clear BSY on signature FIS reception,
 771	 * request follow-up softreset.
 772	 */
 773	return online ? -EAGAIN : rc;
 774}
 775
 776static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
 777				unsigned long deadline)
 778{
 779	struct ata_port *ap = link->ap;
 780	struct ahci_port_priv *pp = ap->private_data;
 781	struct ahci_host_priv *hpriv = ap->host->private_data;
 782	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
 783	struct ata_taskfile tf;
 784	bool online;
 785	int rc;
 786
 787	hpriv->stop_engine(ap);
 788
 789	/* clear D2H reception area to properly wait for D2H FIS */
 790	ata_tf_init(link->device, &tf);
 791	tf.status = ATA_BUSY;
 792	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
 793
 794	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
 795				 deadline, &online, NULL);
 796
 797	hpriv->start_engine(ap);
 798
 799	/* The pseudo configuration device on SIMG4726 attached to
 800	 * ASUS P5W-DH Deluxe doesn't send signature FIS after
 801	 * hardreset if no device is attached to the first downstream
 802	 * port && the pseudo device locks up on SRST w/ PMP==0.  To
 803	 * work around this, wait for !BSY only briefly.  If BSY isn't
 804	 * cleared, perform CLO and proceed to IDENTIFY (achieved by
 805	 * ATA_LFLAG_NO_SRST and ATA_LFLAG_ASSUME_ATA).
 806	 *
 807	 * Wait for two seconds.  Devices attached to downstream port
 808	 * which can't process the following IDENTIFY after this will
 809	 * have to be reset again.  For most cases, this should
 810	 * suffice while making probing snappish enough.
 811	 */
 812	if (online) {
 813		rc = ata_wait_after_reset(link, jiffies + 2 * HZ,
 814					  ahci_check_ready);
 815		if (rc)
 816			ahci_kick_engine(ap);
 817	}
 818	return rc;
 819}
 820
 821/*
 822 * ahci_avn_hardreset - attempt more aggressive recovery of Avoton ports.
 823 *
 824 * It has been observed with some SSDs that the timing of events in the
 825 * link synchronization phase can leave the port in a state that can not
 826 * be recovered by a SATA-hard-reset alone.  The failing signature is
 827 * SStatus.DET stuck at 1 ("Device presence detected but Phy
 828 * communication not established").  It was found that unloading and
 829 * reloading the driver when this problem occurs allows the drive
 830 * connection to be recovered (DET advanced to 0x3).  The critical
 831 * component of reloading the driver is that the port state machines are
 832 * reset by bouncing "port enable" in the AHCI PCS configuration
 833 * register.  So, reproduce that effect by bouncing a port whenever we
 834 * see DET==1 after a reset.
 835 */
 836static int ahci_avn_hardreset(struct ata_link *link, unsigned int *class,
 837			      unsigned long deadline)
 838{
 839	const unsigned int *timing = sata_ehc_deb_timing(&link->eh_context);
 840	struct ata_port *ap = link->ap;
 841	struct ahci_port_priv *pp = ap->private_data;
 842	struct ahci_host_priv *hpriv = ap->host->private_data;
 843	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
 844	unsigned long tmo = deadline - jiffies;
 845	struct ata_taskfile tf;
 846	bool online;
 847	int rc, i;
 848
 849	hpriv->stop_engine(ap);
 850
 851	for (i = 0; i < 2; i++) {
 852		u16 val;
 853		u32 sstatus;
 854		int port = ap->port_no;
 855		struct ata_host *host = ap->host;
 856		struct pci_dev *pdev = to_pci_dev(host->dev);
 857
 858		/* clear D2H reception area to properly wait for D2H FIS */
 859		ata_tf_init(link->device, &tf);
 860		tf.status = ATA_BUSY;
 861		ata_tf_to_fis(&tf, 0, 0, d2h_fis);
 862
 863		rc = sata_link_hardreset(link, timing, deadline, &online,
 864				ahci_check_ready);
 865
 866		if (sata_scr_read(link, SCR_STATUS, &sstatus) != 0 ||
 867				(sstatus & 0xf) != 1)
 868			break;
 869
 870		ata_link_info(link,  "avn bounce port%d\n", port);
 871
 872		pci_read_config_word(pdev, 0x92, &val);
 873		val &= ~(1 << port);
 874		pci_write_config_word(pdev, 0x92, val);
 875		ata_msleep(ap, 1000);
 876		val |= 1 << port;
 877		pci_write_config_word(pdev, 0x92, val);
 878		deadline += tmo;
 879	}
 880
 881	hpriv->start_engine(ap);
 882
 883	if (online)
 884		*class = ahci_dev_classify(ap);
 885
 886	return rc;
 887}
 888
 889
 890#ifdef CONFIG_PM
 891static void ahci_pci_disable_interrupts(struct ata_host *host)
 892{
 893	struct ahci_host_priv *hpriv = host->private_data;
 894	void __iomem *mmio = hpriv->mmio;
 895	u32 ctl;
 896
 897	/* AHCI spec rev1.1 section 8.3.3:
 898	 * Software must disable interrupts prior to requesting a
 899	 * transition of the HBA to D3 state.
 900	 */
 901	ctl = readl(mmio + HOST_CTL);
 902	ctl &= ~HOST_IRQ_EN;
 903	writel(ctl, mmio + HOST_CTL);
 904	readl(mmio + HOST_CTL); /* flush */
 905}
 906
 907static int ahci_pci_device_runtime_suspend(struct device *dev)
 908{
 909	struct pci_dev *pdev = to_pci_dev(dev);
 910	struct ata_host *host = pci_get_drvdata(pdev);
 911
 912	ahci_pci_disable_interrupts(host);
 913	return 0;
 914}
 915
 916static int ahci_pci_device_runtime_resume(struct device *dev)
 917{
 918	struct pci_dev *pdev = to_pci_dev(dev);
 919	struct ata_host *host = pci_get_drvdata(pdev);
 920	int rc;
 921
 922	rc = ahci_pci_reset_controller(host);
 923	if (rc)
 924		return rc;
 925	ahci_pci_init_controller(host);
 926	return 0;
 927}
 928
 929#ifdef CONFIG_PM_SLEEP
 930static int ahci_pci_device_suspend(struct device *dev)
 931{
 932	struct pci_dev *pdev = to_pci_dev(dev);
 933	struct ata_host *host = pci_get_drvdata(pdev);
 934	struct ahci_host_priv *hpriv = host->private_data;
 935
 936	if (hpriv->flags & AHCI_HFLAG_NO_SUSPEND) {
 937		dev_err(&pdev->dev,
 938			"BIOS update required for suspend/resume\n");
 939		return -EIO;
 940	}
 941
 942	ahci_pci_disable_interrupts(host);
 943	ata_host_suspend(host, PMSG_SUSPEND);
 944	return 0;
 945}
 946
 947static int ahci_pci_device_resume(struct device *dev)
 948{
 949	struct pci_dev *pdev = to_pci_dev(dev);
 950	struct ata_host *host = pci_get_drvdata(pdev);
 951	int rc;
 952
 953	/* Apple BIOS helpfully mangles the registers on resume */
 954	if (is_mcp89_apple(pdev))
 955		ahci_mcp89_apple_enable(pdev);
 956
 957	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
 958		rc = ahci_pci_reset_controller(host);
 959		if (rc)
 960			return rc;
 961
 962		ahci_pci_init_controller(host);
 963	}
 964
 965	ata_host_resume(host);
 966
 967	return 0;
 968}
 969#endif
 970
 971#endif /* CONFIG_PM */
 972
 973static int ahci_configure_dma_masks(struct pci_dev *pdev,
 974				    struct ahci_host_priv *hpriv)
 975{
 976	int dma_bits;
 977	int rc;
 978
 979	if (hpriv->cap & HOST_CAP_64) {
 980		dma_bits = 64;
 981		if (hpriv->flags & AHCI_HFLAG_43BIT_ONLY)
 982			dma_bits = 43;
 983	} else {
 984		dma_bits = 32;
 985	}
 986
 987	/*
 988	 * If the device fixup already set the dma_mask to some non-standard
 989	 * value, don't extend it here. This happens on STA2X11, for example.
 990	 *
 991	 * XXX: manipulating the DMA mask from platform code is completely
 992	 * bogus, platform code should use dev->bus_dma_limit instead..
 993	 */
 994	if (pdev->dma_mask && pdev->dma_mask < DMA_BIT_MASK(32))
 995		return 0;
 996
 997	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_bits));
 998	if (rc)
 999		dev_err(&pdev->dev, "DMA enable failed\n");
1000	return rc;
1001}
1002
1003static void ahci_pci_print_info(struct ata_host *host)
1004{
1005	struct pci_dev *pdev = to_pci_dev(host->dev);
1006	u16 cc;
1007	const char *scc_s;
1008
1009	pci_read_config_word(pdev, 0x0a, &cc);
1010	if (cc == PCI_CLASS_STORAGE_IDE)
1011		scc_s = "IDE";
1012	else if (cc == PCI_CLASS_STORAGE_SATA)
1013		scc_s = "SATA";
1014	else if (cc == PCI_CLASS_STORAGE_RAID)
1015		scc_s = "RAID";
1016	else
1017		scc_s = "unknown";
1018
1019	ahci_print_info(host, scc_s);
1020}
1021
1022/* On ASUS P5W DH Deluxe, the second port of PCI device 00:1f.2 is
1023 * hardwired to on-board SIMG 4726.  The chipset is ICH8 and doesn't
1024 * support PMP and the 4726 either directly exports the device
1025 * attached to the first downstream port or acts as a hardware storage
1026 * controller and emulate a single ATA device (can be RAID 0/1 or some
1027 * other configuration).
1028 *
1029 * When there's no device attached to the first downstream port of the
1030 * 4726, "Config Disk" appears, which is a pseudo ATA device to
1031 * configure the 4726.  However, ATA emulation of the device is very
1032 * lame.  It doesn't send signature D2H Reg FIS after the initial
1033 * hardreset, pukes on SRST w/ PMP==0 and has bunch of other issues.
1034 *
1035 * The following function works around the problem by always using
1036 * hardreset on the port and not depending on receiving signature FIS
1037 * afterward.  If signature FIS isn't received soon, ATA class is
1038 * assumed without follow-up softreset.
1039 */
1040static void ahci_p5wdh_workaround(struct ata_host *host)
1041{
1042	static const struct dmi_system_id sysids[] = {
1043		{
1044			.ident = "P5W DH Deluxe",
1045			.matches = {
1046				DMI_MATCH(DMI_SYS_VENDOR,
1047					  "ASUSTEK COMPUTER INC"),
1048				DMI_MATCH(DMI_PRODUCT_NAME, "P5W DH Deluxe"),
1049			},
1050		},
1051		{ }
1052	};
1053	struct pci_dev *pdev = to_pci_dev(host->dev);
1054
1055	if (pdev->bus->number == 0 && pdev->devfn == PCI_DEVFN(0x1f, 2) &&
1056	    dmi_check_system(sysids)) {
1057		struct ata_port *ap = host->ports[1];
1058
1059		dev_info(&pdev->dev,
1060			 "enabling ASUS P5W DH Deluxe on-board SIMG4726 workaround\n");
1061
1062		ap->ops = &ahci_p5wdh_ops;
1063		ap->link.flags |= ATA_LFLAG_NO_SRST | ATA_LFLAG_ASSUME_ATA;
1064	}
1065}
1066
1067/*
1068 * Macbook7,1 firmware forcibly disables MCP89 AHCI and changes PCI ID when
1069 * booting in BIOS compatibility mode.  We restore the registers but not ID.
1070 */
1071static void ahci_mcp89_apple_enable(struct pci_dev *pdev)
1072{
1073	u32 val;
1074
1075	printk(KERN_INFO "ahci: enabling MCP89 AHCI mode\n");
1076
1077	pci_read_config_dword(pdev, 0xf8, &val);
1078	val |= 1 << 0x1b;
1079	/* the following changes the device ID, but appears not to affect function */
1080	/* val = (val & ~0xf0000000) | 0x80000000; */
1081	pci_write_config_dword(pdev, 0xf8, val);
1082
1083	pci_read_config_dword(pdev, 0x54c, &val);
1084	val |= 1 << 0xc;
1085	pci_write_config_dword(pdev, 0x54c, val);
1086
1087	pci_read_config_dword(pdev, 0x4a4, &val);
1088	val &= 0xff;
1089	val |= 0x01060100;
1090	pci_write_config_dword(pdev, 0x4a4, val);
1091
1092	pci_read_config_dword(pdev, 0x54c, &val);
1093	val &= ~(1 << 0xc);
1094	pci_write_config_dword(pdev, 0x54c, val);
1095
1096	pci_read_config_dword(pdev, 0xf8, &val);
1097	val &= ~(1 << 0x1b);
1098	pci_write_config_dword(pdev, 0xf8, val);
1099}
1100
1101static bool is_mcp89_apple(struct pci_dev *pdev)
1102{
1103	return pdev->vendor == PCI_VENDOR_ID_NVIDIA &&
1104		pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP89_SATA &&
1105		pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE &&
1106		pdev->subsystem_device == 0xcb89;
1107}
1108
1109/* only some SB600 ahci controllers can do 64bit DMA */
1110static bool ahci_sb600_enable_64bit(struct pci_dev *pdev)
1111{
1112	static const struct dmi_system_id sysids[] = {
1113		/*
1114		 * The oldest version known to be broken is 0901 and
1115		 * working is 1501 which was released on 2007-10-26.
1116		 * Enable 64bit DMA on 1501 and anything newer.
1117		 *
1118		 * Please read bko#9412 for more info.
1119		 */
1120		{
1121			.ident = "ASUS M2A-VM",
1122			.matches = {
1123				DMI_MATCH(DMI_BOARD_VENDOR,
1124					  "ASUSTeK Computer INC."),
1125				DMI_MATCH(DMI_BOARD_NAME, "M2A-VM"),
1126			},
1127			.driver_data = "20071026",	/* yyyymmdd */
1128		},
1129		/*
1130		 * All BIOS versions for the MSI K9A2 Platinum (MS-7376)
1131		 * support 64bit DMA.
1132		 *
1133		 * BIOS versions earlier than 1.5 had the Manufacturer DMI
1134		 * fields as "MICRO-STAR INTERANTIONAL CO.,LTD".
1135		 * This spelling mistake was fixed in BIOS version 1.5, so
1136		 * 1.5 and later have the Manufacturer as
1137		 * "MICRO-STAR INTERNATIONAL CO.,LTD".
1138		 * So try to match on DMI_BOARD_VENDOR of "MICRO-STAR INTER".
1139		 *
1140		 * BIOS versions earlier than 1.9 had a Board Product Name
1141		 * DMI field of "MS-7376". This was changed to be
1142		 * "K9A2 Platinum (MS-7376)" in version 1.9, but we can still
1143		 * match on DMI_BOARD_NAME of "MS-7376".
1144		 */
1145		{
1146			.ident = "MSI K9A2 Platinum",
1147			.matches = {
1148				DMI_MATCH(DMI_BOARD_VENDOR,
1149					  "MICRO-STAR INTER"),
1150				DMI_MATCH(DMI_BOARD_NAME, "MS-7376"),
1151			},
1152		},
1153		/*
1154		 * All BIOS versions for the MSI K9AGM2 (MS-7327) support
1155		 * 64bit DMA.
1156		 *
1157		 * This board also had the typo mentioned above in the
1158		 * Manufacturer DMI field (fixed in BIOS version 1.5), so
1159		 * match on DMI_BOARD_VENDOR of "MICRO-STAR INTER" again.
1160		 */
1161		{
1162			.ident = "MSI K9AGM2",
1163			.matches = {
1164				DMI_MATCH(DMI_BOARD_VENDOR,
1165					  "MICRO-STAR INTER"),
1166				DMI_MATCH(DMI_BOARD_NAME, "MS-7327"),
1167			},
1168		},
1169		/*
1170		 * All BIOS versions for the Asus M3A support 64bit DMA.
1171		 * (all release versions from 0301 to 1206 were tested)
1172		 */
1173		{
1174			.ident = "ASUS M3A",
1175			.matches = {
1176				DMI_MATCH(DMI_BOARD_VENDOR,
1177					  "ASUSTeK Computer INC."),
1178				DMI_MATCH(DMI_BOARD_NAME, "M3A"),
1179			},
1180		},
1181		{ }
1182	};
1183	const struct dmi_system_id *match;
1184	int year, month, date;
1185	char buf[9];
1186
1187	match = dmi_first_match(sysids);
1188	if (pdev->bus->number != 0 || pdev->devfn != PCI_DEVFN(0x12, 0) ||
1189	    !match)
1190		return false;
1191
1192	if (!match->driver_data)
1193		goto enable_64bit;
1194
1195	dmi_get_date(DMI_BIOS_DATE, &year, &month, &date);
1196	snprintf(buf, sizeof(buf), "%04d%02d%02d", year, month, date);
1197
1198	if (strcmp(buf, match->driver_data) >= 0)
1199		goto enable_64bit;
1200	else {
1201		dev_warn(&pdev->dev,
1202			 "%s: BIOS too old, forcing 32bit DMA, update BIOS\n",
1203			 match->ident);
1204		return false;
1205	}
1206
1207enable_64bit:
1208	dev_warn(&pdev->dev, "%s: enabling 64bit DMA\n", match->ident);
1209	return true;
1210}
1211
1212static bool ahci_broken_system_poweroff(struct pci_dev *pdev)
1213{
1214	static const struct dmi_system_id broken_systems[] = {
1215		{
1216			.ident = "HP Compaq nx6310",
1217			.matches = {
1218				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1219				DMI_MATCH(DMI_PRODUCT_NAME, "HP Compaq nx6310"),
1220			},
1221			/* PCI slot number of the controller */
1222			.driver_data = (void *)0x1FUL,
1223		},
1224		{
1225			.ident = "HP Compaq 6720s",
1226			.matches = {
1227				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1228				DMI_MATCH(DMI_PRODUCT_NAME, "HP Compaq 6720s"),
1229			},
1230			/* PCI slot number of the controller */
1231			.driver_data = (void *)0x1FUL,
1232		},
1233
1234		{ }	/* terminate list */
1235	};
1236	const struct dmi_system_id *dmi = dmi_first_match(broken_systems);
1237
1238	if (dmi) {
1239		unsigned long slot = (unsigned long)dmi->driver_data;
1240		/* apply the quirk only to on-board controllers */
1241		return slot == PCI_SLOT(pdev->devfn);
1242	}
1243
1244	return false;
1245}
1246
1247static bool ahci_broken_suspend(struct pci_dev *pdev)
1248{
1249	static const struct dmi_system_id sysids[] = {
1250		/*
1251		 * On HP dv[4-6] and HDX18 with earlier BIOSen, link
1252		 * to the harddisk doesn't become online after
1253		 * resuming from STR.  Warn and fail suspend.
1254		 *
1255		 * http://bugzilla.kernel.org/show_bug.cgi?id=12276
1256		 *
1257		 * Use dates instead of versions to match as HP is
1258		 * apparently recycling both product and version
1259		 * strings.
1260		 *
1261		 * http://bugzilla.kernel.org/show_bug.cgi?id=15462
1262		 */
1263		{
1264			.ident = "dv4",
1265			.matches = {
1266				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1267				DMI_MATCH(DMI_PRODUCT_NAME,
1268					  "HP Pavilion dv4 Notebook PC"),
1269			},
1270			.driver_data = "20090105",	/* F.30 */
1271		},
1272		{
1273			.ident = "dv5",
1274			.matches = {
1275				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1276				DMI_MATCH(DMI_PRODUCT_NAME,
1277					  "HP Pavilion dv5 Notebook PC"),
1278			},
1279			.driver_data = "20090506",	/* F.16 */
1280		},
1281		{
1282			.ident = "dv6",
1283			.matches = {
1284				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1285				DMI_MATCH(DMI_PRODUCT_NAME,
1286					  "HP Pavilion dv6 Notebook PC"),
1287			},
1288			.driver_data = "20090423",	/* F.21 */
1289		},
1290		{
1291			.ident = "HDX18",
1292			.matches = {
1293				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1294				DMI_MATCH(DMI_PRODUCT_NAME,
1295					  "HP HDX18 Notebook PC"),
1296			},
1297			.driver_data = "20090430",	/* F.23 */
1298		},
1299		/*
1300		 * Acer eMachines G725 has the same problem.  BIOS
1301		 * V1.03 is known to be broken.  V3.04 is known to
1302		 * work.  Between, there are V1.06, V2.06 and V3.03
1303		 * that we don't have much idea about.  For now,
1304		 * blacklist anything older than V3.04.
1305		 *
1306		 * http://bugzilla.kernel.org/show_bug.cgi?id=15104
1307		 */
1308		{
1309			.ident = "G725",
1310			.matches = {
1311				DMI_MATCH(DMI_SYS_VENDOR, "eMachines"),
1312				DMI_MATCH(DMI_PRODUCT_NAME, "eMachines G725"),
1313			},
1314			.driver_data = "20091216",	/* V3.04 */
1315		},
1316		{ }	/* terminate list */
1317	};
1318	const struct dmi_system_id *dmi = dmi_first_match(sysids);
1319	int year, month, date;
1320	char buf[9];
1321
1322	if (!dmi || pdev->bus->number || pdev->devfn != PCI_DEVFN(0x1f, 2))
1323		return false;
1324
1325	dmi_get_date(DMI_BIOS_DATE, &year, &month, &date);
1326	snprintf(buf, sizeof(buf), "%04d%02d%02d", year, month, date);
1327
1328	return strcmp(buf, dmi->driver_data) < 0;
1329}
1330
1331static bool ahci_broken_lpm(struct pci_dev *pdev)
1332{
1333	static const struct dmi_system_id sysids[] = {
1334		/* Various Lenovo 50 series have LPM issues with older BIOSen */
1335		{
1336			.matches = {
1337				DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
1338				DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad X250"),
1339			},
1340			.driver_data = "20180406", /* 1.31 */
1341		},
1342		{
1343			.matches = {
1344				DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
1345				DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad L450"),
1346			},
1347			.driver_data = "20180420", /* 1.28 */
1348		},
1349		{
1350			.matches = {
1351				DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
1352				DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad T450s"),
1353			},
1354			.driver_data = "20180315", /* 1.33 */
1355		},
1356		{
1357			.matches = {
1358				DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
1359				DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad W541"),
1360			},
1361			/*
1362			 * Note date based on release notes, 2.35 has been
1363			 * reported to be good, but I've been unable to get
1364			 * a hold of the reporter to get the DMI BIOS date.
1365			 * TODO: fix this.
1366			 */
1367			.driver_data = "20180310", /* 2.35 */
1368		},
1369		{ }	/* terminate list */
1370	};
1371	const struct dmi_system_id *dmi = dmi_first_match(sysids);
1372	int year, month, date;
1373	char buf[9];
1374
1375	if (!dmi)
1376		return false;
1377
1378	dmi_get_date(DMI_BIOS_DATE, &year, &month, &date);
1379	snprintf(buf, sizeof(buf), "%04d%02d%02d", year, month, date);
1380
1381	return strcmp(buf, dmi->driver_data) < 0;
1382}
1383
1384static bool ahci_broken_online(struct pci_dev *pdev)
1385{
1386#define ENCODE_BUSDEVFN(bus, slot, func)			\
1387	(void *)(unsigned long)(((bus) << 8) | PCI_DEVFN((slot), (func)))
1388	static const struct dmi_system_id sysids[] = {
1389		/*
1390		 * There are several gigabyte boards which use
1391		 * SIMG5723s configured as hardware RAID.  Certain
1392		 * 5723 firmware revisions shipped there keep the link
1393		 * online but fail to answer properly to SRST or
1394		 * IDENTIFY when no device is attached downstream
1395		 * causing libata to retry quite a few times leading
1396		 * to excessive detection delay.
1397		 *
1398		 * As these firmwares respond to the second reset try
1399		 * with invalid device signature, considering unknown
1400		 * sig as offline works around the problem acceptably.
1401		 */
1402		{
1403			.ident = "EP45-DQ6",
1404			.matches = {
1405				DMI_MATCH(DMI_BOARD_VENDOR,
1406					  "Gigabyte Technology Co., Ltd."),
1407				DMI_MATCH(DMI_BOARD_NAME, "EP45-DQ6"),
1408			},
1409			.driver_data = ENCODE_BUSDEVFN(0x0a, 0x00, 0),
1410		},
1411		{
1412			.ident = "EP45-DS5",
1413			.matches = {
1414				DMI_MATCH(DMI_BOARD_VENDOR,
1415					  "Gigabyte Technology Co., Ltd."),
1416				DMI_MATCH(DMI_BOARD_NAME, "EP45-DS5"),
1417			},
1418			.driver_data = ENCODE_BUSDEVFN(0x03, 0x00, 0),
1419		},
1420		{ }	/* terminate list */
1421	};
1422#undef ENCODE_BUSDEVFN
1423	const struct dmi_system_id *dmi = dmi_first_match(sysids);
1424	unsigned int val;
1425
1426	if (!dmi)
1427		return false;
1428
1429	val = (unsigned long)dmi->driver_data;
1430
1431	return pdev->bus->number == (val >> 8) && pdev->devfn == (val & 0xff);
1432}
1433
1434static bool ahci_broken_devslp(struct pci_dev *pdev)
1435{
1436	/* device with broken DEVSLP but still showing SDS capability */
1437	static const struct pci_device_id ids[] = {
1438		{ PCI_VDEVICE(INTEL, 0x0f23)}, /* Valleyview SoC */
1439		{}
1440	};
1441
1442	return pci_match_id(ids, pdev);
1443}
1444
1445#ifdef CONFIG_ATA_ACPI
1446static void ahci_gtf_filter_workaround(struct ata_host *host)
1447{
1448	static const struct dmi_system_id sysids[] = {
1449		/*
1450		 * Aspire 3810T issues a bunch of SATA enable commands
1451		 * via _GTF including an invalid one and one which is
1452		 * rejected by the device.  Among the successful ones
1453		 * is FPDMA non-zero offset enable which when enabled
1454		 * only on the drive side leads to NCQ command
1455		 * failures.  Filter it out.
1456		 */
1457		{
1458			.ident = "Aspire 3810T",
1459			.matches = {
1460				DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
1461				DMI_MATCH(DMI_PRODUCT_NAME, "Aspire 3810T"),
1462			},
1463			.driver_data = (void *)ATA_ACPI_FILTER_FPDMA_OFFSET,
1464		},
1465		{ }
1466	};
1467	const struct dmi_system_id *dmi = dmi_first_match(sysids);
1468	unsigned int filter;
1469	int i;
1470
1471	if (!dmi)
1472		return;
1473
1474	filter = (unsigned long)dmi->driver_data;
1475	dev_info(host->dev, "applying extra ACPI _GTF filter 0x%x for %s\n",
1476		 filter, dmi->ident);
1477
1478	for (i = 0; i < host->n_ports; i++) {
1479		struct ata_port *ap = host->ports[i];
1480		struct ata_link *link;
1481		struct ata_device *dev;
1482
1483		ata_for_each_link(link, ap, EDGE)
1484			ata_for_each_dev(dev, link, ALL)
1485				dev->gtf_filter |= filter;
1486	}
1487}
1488#else
1489static inline void ahci_gtf_filter_workaround(struct ata_host *host)
1490{}
1491#endif
1492
1493/*
1494 * On the Acer Aspire Switch Alpha 12, sometimes all SATA ports are detected
1495 * as DUMMY, or detected but eventually get a "link down" and never get up
1496 * again. When this happens, CAP.NP may hold a value of 0x00 or 0x01, and the
1497 * port_map may hold a value of 0x00.
1498 *
1499 * Overriding CAP.NP to 0x02 and the port_map to 0x7 will reveal all 3 ports
1500 * and can significantly reduce the occurrence of the problem.
1501 *
1502 * https://bugzilla.kernel.org/show_bug.cgi?id=189471
1503 */
1504static void acer_sa5_271_workaround(struct ahci_host_priv *hpriv,
1505				    struct pci_dev *pdev)
1506{
1507	static const struct dmi_system_id sysids[] = {
1508		{
1509			.ident = "Acer Switch Alpha 12",
1510			.matches = {
1511				DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
1512				DMI_MATCH(DMI_PRODUCT_NAME, "Switch SA5-271")
1513			},
1514		},
1515		{ }
1516	};
1517
1518	if (dmi_check_system(sysids)) {
1519		dev_info(&pdev->dev, "enabling Acer Switch Alpha 12 workaround\n");
1520		if ((hpriv->saved_cap & 0xC734FF00) == 0xC734FF00) {
1521			hpriv->port_map = 0x7;
1522			hpriv->cap = 0xC734FF02;
1523		}
1524	}
1525}
1526
1527#ifdef CONFIG_ARM64
1528/*
1529 * Due to ERRATA#22536, ThunderX needs to handle HOST_IRQ_STAT differently.
1530 * Workaround is to make sure all pending IRQs are served before leaving
1531 * handler.
1532 */
1533static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance)
1534{
1535	struct ata_host *host = dev_instance;
1536	struct ahci_host_priv *hpriv;
1537	unsigned int rc = 0;
1538	void __iomem *mmio;
1539	u32 irq_stat, irq_masked;
1540	unsigned int handled = 1;
1541
1542	hpriv = host->private_data;
1543	mmio = hpriv->mmio;
1544	irq_stat = readl(mmio + HOST_IRQ_STAT);
1545	if (!irq_stat)
1546		return IRQ_NONE;
1547
1548	do {
1549		irq_masked = irq_stat & hpriv->port_map;
1550		spin_lock(&host->lock);
1551		rc = ahci_handle_port_intr(host, irq_masked);
1552		if (!rc)
1553			handled = 0;
1554		writel(irq_stat, mmio + HOST_IRQ_STAT);
1555		irq_stat = readl(mmio + HOST_IRQ_STAT);
1556		spin_unlock(&host->lock);
1557	} while (irq_stat);
1558
1559	return IRQ_RETVAL(handled);
1560}
1561#endif
1562
1563static void ahci_remap_check(struct pci_dev *pdev, int bar,
1564		struct ahci_host_priv *hpriv)
1565{
1566	int i;
1567	u32 cap;
1568
1569	/*
1570	 * Check if this device might have remapped nvme devices.
1571	 */
1572	if (pdev->vendor != PCI_VENDOR_ID_INTEL ||
1573	    pci_resource_len(pdev, bar) < SZ_512K ||
1574	    bar != AHCI_PCI_BAR_STANDARD ||
1575	    !(readl(hpriv->mmio + AHCI_VSCAP) & 1))
1576		return;
1577
1578	cap = readq(hpriv->mmio + AHCI_REMAP_CAP);
1579	for (i = 0; i < AHCI_MAX_REMAP; i++) {
1580		if ((cap & (1 << i)) == 0)
1581			continue;
1582		if (readl(hpriv->mmio + ahci_remap_dcc(i))
1583				!= PCI_CLASS_STORAGE_EXPRESS)
1584			continue;
1585
1586		/* We've found a remapped device */
1587		hpriv->remapped_nvme++;
1588	}
1589
1590	if (!hpriv->remapped_nvme)
1591		return;
1592
1593	dev_warn(&pdev->dev, "Found %u remapped NVMe devices.\n",
1594		 hpriv->remapped_nvme);
1595	dev_warn(&pdev->dev,
1596		 "Switch your BIOS from RAID to AHCI mode to use them.\n");
1597
1598	/*
1599	 * Don't rely on the msi-x capability in the remap case,
1600	 * share the legacy interrupt across ahci and remapped devices.
1601	 */
1602	hpriv->flags |= AHCI_HFLAG_NO_MSI;
1603}
1604
1605static int ahci_get_irq_vector(struct ata_host *host, int port)
1606{
1607	return pci_irq_vector(to_pci_dev(host->dev), port);
1608}
1609
1610static int ahci_init_msi(struct pci_dev *pdev, unsigned int n_ports,
1611			struct ahci_host_priv *hpriv)
1612{
1613	int nvec;
1614
1615	if (hpriv->flags & AHCI_HFLAG_NO_MSI)
1616		return -ENODEV;
1617
1618	/*
1619	 * If number of MSIs is less than number of ports then Sharing Last
1620	 * Message mode could be enforced. In this case assume that advantage
1621	 * of multipe MSIs is negated and use single MSI mode instead.
1622	 */
1623	if (n_ports > 1) {
1624		nvec = pci_alloc_irq_vectors(pdev, n_ports, INT_MAX,
1625				PCI_IRQ_MSIX | PCI_IRQ_MSI);
1626		if (nvec > 0) {
1627			if (!(readl(hpriv->mmio + HOST_CTL) & HOST_MRSM)) {
1628				hpriv->get_irq_vector = ahci_get_irq_vector;
1629				hpriv->flags |= AHCI_HFLAG_MULTI_MSI;
1630				return nvec;
1631			}
1632
1633			/*
1634			 * Fallback to single MSI mode if the controller
1635			 * enforced MRSM mode.
1636			 */
1637			printk(KERN_INFO
1638				"ahci: MRSM is on, fallback to single MSI\n");
1639			pci_free_irq_vectors(pdev);
1640		}
1641	}
1642
1643	/*
1644	 * If the host is not capable of supporting per-port vectors, fall
1645	 * back to single MSI before finally attempting single MSI-X.
1646	 */
1647	nvec = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
1648	if (nvec == 1)
1649		return nvec;
1650	return pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX);
1651}
1652
1653static void ahci_update_initial_lpm_policy(struct ata_port *ap,
1654					   struct ahci_host_priv *hpriv)
1655{
1656	int policy = CONFIG_SATA_MOBILE_LPM_POLICY;
1657
1658
1659	/* Ignore processing for chipsets that don't use policy */
1660	if (!(hpriv->flags & AHCI_HFLAG_USE_LPM_POLICY))
1661		return;
1662
1663	/* user modified policy via module param */
1664	if (mobile_lpm_policy != -1) {
1665		policy = mobile_lpm_policy;
1666		goto update_policy;
1667	}
1668
1669	if (policy > ATA_LPM_MED_POWER && pm_suspend_default_s2idle()) {
1670		if (hpriv->cap & HOST_CAP_PART)
1671			policy = ATA_LPM_MIN_POWER_WITH_PARTIAL;
1672		else if (hpriv->cap & HOST_CAP_SSC)
1673			policy = ATA_LPM_MIN_POWER;
1674	}
1675
1676update_policy:
1677	if (policy >= ATA_LPM_UNKNOWN && policy <= ATA_LPM_MIN_POWER)
1678		ap->target_lpm_policy = policy;
1679}
1680
1681static void ahci_intel_pcs_quirk(struct pci_dev *pdev, struct ahci_host_priv *hpriv)
1682{
1683	const struct pci_device_id *id = pci_match_id(ahci_pci_tbl, pdev);
1684	u16 tmp16;
1685
1686	/*
1687	 * Only apply the 6-port PCS quirk for known legacy platforms.
1688	 */
1689	if (!id || id->vendor != PCI_VENDOR_ID_INTEL)
1690		return;
1691
1692	/* Skip applying the quirk on Denverton and beyond */
1693	if (((enum board_ids) id->driver_data) >= board_ahci_pcs7)
1694		return;
1695
1696	/*
1697	 * port_map is determined from PORTS_IMPL PCI register which is
1698	 * implemented as write or write-once register.  If the register
1699	 * isn't programmed, ahci automatically generates it from number
1700	 * of ports, which is good enough for PCS programming. It is
1701	 * otherwise expected that platform firmware enables the ports
1702	 * before the OS boots.
1703	 */
1704	pci_read_config_word(pdev, PCS_6, &tmp16);
1705	if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
1706		tmp16 |= hpriv->port_map;
1707		pci_write_config_word(pdev, PCS_6, tmp16);
1708	}
1709}
1710
1711static ssize_t remapped_nvme_show(struct device *dev,
1712				  struct device_attribute *attr,
1713				  char *buf)
1714{
1715	struct ata_host *host = dev_get_drvdata(dev);
1716	struct ahci_host_priv *hpriv = host->private_data;
1717
1718	return sysfs_emit(buf, "%u\n", hpriv->remapped_nvme);
1719}
1720
1721static DEVICE_ATTR_RO(remapped_nvme);
1722
1723static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1724{
1725	unsigned int board_id = ent->driver_data;
1726	struct ata_port_info pi = ahci_port_info[board_id];
1727	const struct ata_port_info *ppi[] = { &pi, NULL };
1728	struct device *dev = &pdev->dev;
1729	struct ahci_host_priv *hpriv;
1730	struct ata_host *host;
1731	int n_ports, i, rc;
1732	int ahci_pci_bar = AHCI_PCI_BAR_STANDARD;
1733
1734	WARN_ON((int)ATA_MAX_QUEUE > AHCI_MAX_CMDS);
1735
1736	ata_print_version_once(&pdev->dev, DRV_VERSION);
1737
1738	/* The AHCI driver can only drive the SATA ports, the PATA driver
1739	   can drive them all so if both drivers are selected make sure
1740	   AHCI stays out of the way */
1741	if (pdev->vendor == PCI_VENDOR_ID_MARVELL && !marvell_enable)
1742		return -ENODEV;
1743
1744	/* Apple BIOS on MCP89 prevents us using AHCI */
1745	if (is_mcp89_apple(pdev))
1746		ahci_mcp89_apple_enable(pdev);
1747
1748	/* Promise's PDC42819 is a SAS/SATA controller that has an AHCI mode.
1749	 * At the moment, we can only use the AHCI mode. Let the users know
1750	 * that for SAS drives they're out of luck.
1751	 */
1752	if (pdev->vendor == PCI_VENDOR_ID_PROMISE)
1753		dev_info(&pdev->dev,
1754			 "PDC42819 can only drive SATA devices with this driver\n");
1755
1756	/* Some devices use non-standard BARs */
1757	if (pdev->vendor == PCI_VENDOR_ID_STMICRO && pdev->device == 0xCC06)
1758		ahci_pci_bar = AHCI_PCI_BAR_STA2X11;
1759	else if (pdev->vendor == 0x1c44 && pdev->device == 0x8000)
1760		ahci_pci_bar = AHCI_PCI_BAR_ENMOTUS;
1761	else if (pdev->vendor == PCI_VENDOR_ID_CAVIUM) {
1762		if (pdev->device == 0xa01c)
1763			ahci_pci_bar = AHCI_PCI_BAR_CAVIUM;
1764		if (pdev->device == 0xa084)
1765			ahci_pci_bar = AHCI_PCI_BAR_CAVIUM_GEN5;
1766	} else if (pdev->vendor == PCI_VENDOR_ID_LOONGSON) {
1767		if (pdev->device == 0x7a08)
1768			ahci_pci_bar = AHCI_PCI_BAR_LOONGSON;
1769	}
1770
1771	/* acquire resources */
1772	rc = pcim_enable_device(pdev);
1773	if (rc)
1774		return rc;
1775
1776	if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
1777	    (pdev->device == 0x2652 || pdev->device == 0x2653)) {
1778		u8 map;
1779
1780		/* ICH6s share the same PCI ID for both piix and ahci
1781		 * modes.  Enabling ahci mode while MAP indicates
1782		 * combined mode is a bad idea.  Yield to ata_piix.
1783		 */
1784		pci_read_config_byte(pdev, ICH_MAP, &map);
1785		if (map & 0x3) {
1786			dev_info(&pdev->dev,
1787				 "controller is in combined mode, can't enable AHCI mode\n");
1788			return -ENODEV;
1789		}
1790	}
1791
1792	/* AHCI controllers often implement SFF compatible interface.
1793	 * Grab all PCI BARs just in case.
1794	 */
1795	rc = pcim_iomap_regions_request_all(pdev, 1 << ahci_pci_bar, DRV_NAME);
1796	if (rc == -EBUSY)
1797		pcim_pin_device(pdev);
1798	if (rc)
1799		return rc;
1800
1801	hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
1802	if (!hpriv)
1803		return -ENOMEM;
1804	hpriv->flags |= (unsigned long)pi.private_data;
1805
1806	/* MCP65 revision A1 and A2 can't do MSI */
1807	if (board_id == board_ahci_mcp65 &&
1808	    (pdev->revision == 0xa1 || pdev->revision == 0xa2))
1809		hpriv->flags |= AHCI_HFLAG_NO_MSI;
1810
1811	/* SB800 does NOT need the workaround to ignore SERR_INTERNAL */
1812	if (board_id == board_ahci_sb700 && pdev->revision >= 0x40)
1813		hpriv->flags &= ~AHCI_HFLAG_IGN_SERR_INTERNAL;
1814
1815	/* only some SB600s can do 64bit DMA */
1816	if (ahci_sb600_enable_64bit(pdev))
1817		hpriv->flags &= ~AHCI_HFLAG_32BIT_ONLY;
1818
1819	hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar];
1820
1821	/* detect remapped nvme devices */
1822	ahci_remap_check(pdev, ahci_pci_bar, hpriv);
1823
1824	sysfs_add_file_to_group(&pdev->dev.kobj,
1825				&dev_attr_remapped_nvme.attr,
1826				NULL);
1827
1828	/* must set flag prior to save config in order to take effect */
1829	if (ahci_broken_devslp(pdev))
1830		hpriv->flags |= AHCI_HFLAG_NO_DEVSLP;
1831
1832#ifdef CONFIG_ARM64
1833	if (pdev->vendor == PCI_VENDOR_ID_HUAWEI &&
1834	    pdev->device == 0xa235 &&
1835	    pdev->revision < 0x30)
1836		hpriv->flags |= AHCI_HFLAG_NO_SXS;
1837
1838	if (pdev->vendor == 0x177d && pdev->device == 0xa01c)
1839		hpriv->irq_handler = ahci_thunderx_irq_handler;
1840#endif
1841
1842	/* save initial config */
1843	ahci_pci_save_initial_config(pdev, hpriv);
1844
1845	/* prepare host */
1846	if (hpriv->cap & HOST_CAP_NCQ) {
1847		pi.flags |= ATA_FLAG_NCQ;
1848		/*
1849		 * Auto-activate optimization is supposed to be
1850		 * supported on all AHCI controllers indicating NCQ
1851		 * capability, but it seems to be broken on some
1852		 * chipsets including NVIDIAs.
1853		 */
1854		if (!(hpriv->flags & AHCI_HFLAG_NO_FPDMA_AA))
1855			pi.flags |= ATA_FLAG_FPDMA_AA;
1856
1857		/*
1858		 * All AHCI controllers should be forward-compatible
1859		 * with the new auxiliary field. This code should be
1860		 * conditionalized if any buggy AHCI controllers are
1861		 * encountered.
1862		 */
1863		pi.flags |= ATA_FLAG_FPDMA_AUX;
1864	}
1865
1866	if (hpriv->cap & HOST_CAP_PMP)
1867		pi.flags |= ATA_FLAG_PMP;
1868
1869	ahci_set_em_messages(hpriv, &pi);
1870
1871	if (ahci_broken_system_poweroff(pdev)) {
1872		pi.flags |= ATA_FLAG_NO_POWEROFF_SPINDOWN;
1873		dev_info(&pdev->dev,
1874			"quirky BIOS, skipping spindown on poweroff\n");
1875	}
1876
1877	if (ahci_broken_lpm(pdev)) {
1878		pi.flags |= ATA_FLAG_NO_LPM;
1879		dev_warn(&pdev->dev,
1880			 "BIOS update required for Link Power Management support\n");
1881	}
1882
1883	if (ahci_broken_suspend(pdev)) {
1884		hpriv->flags |= AHCI_HFLAG_NO_SUSPEND;
1885		dev_warn(&pdev->dev,
1886			 "BIOS update required for suspend/resume\n");
1887	}
1888
1889	if (ahci_broken_online(pdev)) {
1890		hpriv->flags |= AHCI_HFLAG_SRST_TOUT_IS_OFFLINE;
1891		dev_info(&pdev->dev,
1892			 "online status unreliable, applying workaround\n");
1893	}
1894
1895
1896	/* Acer SA5-271 workaround modifies private_data */
1897	acer_sa5_271_workaround(hpriv, pdev);
1898
1899	/* CAP.NP sometimes indicate the index of the last enabled
1900	 * port, at other times, that of the last possible port, so
1901	 * determining the maximum port number requires looking at
1902	 * both CAP.NP and port_map.
1903	 */
1904	n_ports = max(ahci_nr_ports(hpriv->cap), fls(hpriv->port_map));
1905
1906	host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
1907	if (!host)
1908		return -ENOMEM;
1909	host->private_data = hpriv;
1910
1911	if (ahci_init_msi(pdev, n_ports, hpriv) < 0) {
1912		/* legacy intx interrupts */
1913		pci_intx(pdev, 1);
1914	}
1915	hpriv->irq = pci_irq_vector(pdev, 0);
1916
1917	if (!(hpriv->cap & HOST_CAP_SSS) || ahci_ignore_sss)
1918		host->flags |= ATA_HOST_PARALLEL_SCAN;
1919	else
1920		dev_info(&pdev->dev, "SSS flag set, parallel bus scan disabled\n");
1921
1922	if (!(hpriv->cap & HOST_CAP_PART))
1923		host->flags |= ATA_HOST_NO_PART;
1924
1925	if (!(hpriv->cap & HOST_CAP_SSC))
1926		host->flags |= ATA_HOST_NO_SSC;
1927
1928	if (!(hpriv->cap2 & HOST_CAP2_SDS))
1929		host->flags |= ATA_HOST_NO_DEVSLP;
1930
1931	if (pi.flags & ATA_FLAG_EM)
1932		ahci_reset_em(host);
1933
1934	for (i = 0; i < host->n_ports; i++) {
1935		struct ata_port *ap = host->ports[i];
1936
1937		ata_port_pbar_desc(ap, ahci_pci_bar, -1, "abar");
1938		ata_port_pbar_desc(ap, ahci_pci_bar,
1939				   0x100 + ap->port_no * 0x80, "port");
1940
1941		/* set enclosure management message type */
1942		if (ap->flags & ATA_FLAG_EM)
1943			ap->em_message_type = hpriv->em_msg_type;
1944
1945		ahci_update_initial_lpm_policy(ap, hpriv);
1946
1947		/* disabled/not-implemented port */
1948		if (!(hpriv->port_map & (1 << i)))
1949			ap->ops = &ata_dummy_port_ops;
1950	}
1951
1952	/* apply workaround for ASUS P5W DH Deluxe mainboard */
1953	ahci_p5wdh_workaround(host);
1954
1955	/* apply gtf filter quirk */
1956	ahci_gtf_filter_workaround(host);
1957
1958	/* initialize adapter */
1959	rc = ahci_configure_dma_masks(pdev, hpriv);
1960	if (rc)
1961		return rc;
1962
1963	rc = ahci_pci_reset_controller(host);
1964	if (rc)
1965		return rc;
1966
1967	ahci_pci_init_controller(host);
1968	ahci_pci_print_info(host);
1969
1970	pci_set_master(pdev);
1971
1972	rc = ahci_host_activate(host, &ahci_sht);
1973	if (rc)
1974		return rc;
1975
1976	pm_runtime_put_noidle(&pdev->dev);
1977	return 0;
1978}
1979
1980static void ahci_shutdown_one(struct pci_dev *pdev)
1981{
1982	ata_pci_shutdown_one(pdev);
1983}
1984
1985static void ahci_remove_one(struct pci_dev *pdev)
1986{
1987	sysfs_remove_file_from_group(&pdev->dev.kobj,
1988				     &dev_attr_remapped_nvme.attr,
1989				     NULL);
1990	pm_runtime_get_noresume(&pdev->dev);
1991	ata_pci_remove_one(pdev);
1992}
1993
1994module_pci_driver(ahci_pci_driver);
1995
1996MODULE_AUTHOR("Jeff Garzik");
1997MODULE_DESCRIPTION("AHCI SATA low-level driver");
1998MODULE_LICENSE("GPL");
1999MODULE_DEVICE_TABLE(pci, ahci_pci_tbl);
2000MODULE_VERSION(DRV_VERSION);