1/*
   2 *  libahci.c - Common AHCI SATA low-level routines
   3 *
   4 *  Maintained by:  Tejun Heo <tj@kernel.org>
   5 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
   6 *		    on emails.
   7 *
   8 *  Copyright 2004-2005 Red Hat, Inc.
   9 *
  10 *
  11 *  This program is free software; you can redistribute it and/or modify
  12 *  it under the terms of the GNU General Public License as published by
  13 *  the Free Software Foundation; either version 2, or (at your option)
  14 *  any later version.
  15 *
  16 *  This program is distributed in the hope that it will be useful,
  17 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 *  GNU General Public License for more details.
  20 *
  21 *  You should have received a copy of the GNU General Public License
  22 *  along with this program; see the file COPYING.  If not, write to
  23 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  24 *
  25 *
  26 * libata documentation is available via 'make {ps|pdf}docs',
  27 * as Documentation/DocBook/libata.*
  28 *
  29 * AHCI hardware documentation:
  30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
  31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
  32 *
  33 */
  34
  35#include <linux/kernel.h>
  36#include <linux/gfp.h>
  37#include <linux/module.h>
 
  38#include <linux/blkdev.h>
  39#include <linux/delay.h>
  40#include <linux/interrupt.h>
  41#include <linux/dma-mapping.h>
  42#include <linux/device.h>
  43#include <scsi/scsi_host.h>
  44#include <scsi/scsi_cmnd.h>
  45#include <linux/libata.h>
  46#include "ahci.h"
  47#include "libata.h"
  48
  49static int ahci_skip_host_reset;
  50int ahci_ignore_sss;
  51EXPORT_SYMBOL_GPL(ahci_ignore_sss);
  52
  53module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
  54MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
  55
  56module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
  57MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
  58
  59static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
  60			unsigned hints);
  61static ssize_t ahci_led_show(struct ata_port *ap, char *buf);
  62static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
  63			      size_t size);
  64static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
  65					ssize_t size);
  66
  67
  68
  69static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
  70static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
  71static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
  72static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
  73static int ahci_port_start(struct ata_port *ap);
  74static void ahci_port_stop(struct ata_port *ap);
  75static void ahci_qc_prep(struct ata_queued_cmd *qc);
  76static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
  77static void ahci_freeze(struct ata_port *ap);
  78static void ahci_thaw(struct ata_port *ap);
  79static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep);
  80static void ahci_enable_fbs(struct ata_port *ap);
  81static void ahci_disable_fbs(struct ata_port *ap);
  82static void ahci_pmp_attach(struct ata_port *ap);
  83static void ahci_pmp_detach(struct ata_port *ap);
  84static int ahci_softreset(struct ata_link *link, unsigned int *class,
  85			  unsigned long deadline);
  86static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
  87			  unsigned long deadline);
  88static int ahci_hardreset(struct ata_link *link, unsigned int *class,
  89			  unsigned long deadline);
  90static void ahci_postreset(struct ata_link *link, unsigned int *class);
 
  91static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
  92static void ahci_dev_config(struct ata_device *dev);
  93#ifdef CONFIG_PM
  94static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
  95#endif
  96static ssize_t ahci_activity_show(struct ata_device *dev, char *buf);
  97static ssize_t ahci_activity_store(struct ata_device *dev,
  98				   enum sw_activity val);
  99static void ahci_init_sw_activity(struct ata_link *link);
 100
 101static ssize_t ahci_show_host_caps(struct device *dev,
 102				   struct device_attribute *attr, char *buf);
 103static ssize_t ahci_show_host_cap2(struct device *dev,
 104				   struct device_attribute *attr, char *buf);
 105static ssize_t ahci_show_host_version(struct device *dev,
 106				      struct device_attribute *attr, char *buf);
 107static ssize_t ahci_show_port_cmd(struct device *dev,
 108				  struct device_attribute *attr, char *buf);
 109static ssize_t ahci_read_em_buffer(struct device *dev,
 110				   struct device_attribute *attr, char *buf);
 111static ssize_t ahci_store_em_buffer(struct device *dev,
 112				    struct device_attribute *attr,
 113				    const char *buf, size_t size);
 114static ssize_t ahci_show_em_supported(struct device *dev,
 115				      struct device_attribute *attr, char *buf);
 116
 117static DEVICE_ATTR(ahci_host_caps, S_IRUGO, ahci_show_host_caps, NULL);
 118static DEVICE_ATTR(ahci_host_cap2, S_IRUGO, ahci_show_host_cap2, NULL);
 119static DEVICE_ATTR(ahci_host_version, S_IRUGO, ahci_show_host_version, NULL);
 120static DEVICE_ATTR(ahci_port_cmd, S_IRUGO, ahci_show_port_cmd, NULL);
 121static DEVICE_ATTR(em_buffer, S_IWUSR | S_IRUGO,
 122		   ahci_read_em_buffer, ahci_store_em_buffer);
 123static DEVICE_ATTR(em_message_supported, S_IRUGO, ahci_show_em_supported, NULL);
 124
 125struct device_attribute *ahci_shost_attrs[] = {
 126	&dev_attr_link_power_management_policy,
 127	&dev_attr_em_message_type,
 128	&dev_attr_em_message,
 129	&dev_attr_ahci_host_caps,
 130	&dev_attr_ahci_host_cap2,
 131	&dev_attr_ahci_host_version,
 132	&dev_attr_ahci_port_cmd,
 133	&dev_attr_em_buffer,
 134	&dev_attr_em_message_supported,
 135	NULL
 136};
 137EXPORT_SYMBOL_GPL(ahci_shost_attrs);
 138
 139struct device_attribute *ahci_sdev_attrs[] = {
 140	&dev_attr_sw_activity,
 141	&dev_attr_unload_heads,
 142	NULL
 143};
 144EXPORT_SYMBOL_GPL(ahci_sdev_attrs);
 145
 146struct ata_port_operations ahci_ops = {
 147	.inherits		= &sata_pmp_port_ops,
 148
 149	.qc_defer		= ahci_pmp_qc_defer,
 150	.qc_prep		= ahci_qc_prep,
 151	.qc_issue		= ahci_qc_issue,
 152	.qc_fill_rtf		= ahci_qc_fill_rtf,
 153
 154	.freeze			= ahci_freeze,
 155	.thaw			= ahci_thaw,
 156	.softreset		= ahci_softreset,
 157	.hardreset		= ahci_hardreset,
 158	.postreset		= ahci_postreset,
 159	.pmp_softreset		= ahci_softreset,
 160	.error_handler		= ahci_error_handler,
 161	.post_internal_cmd	= ahci_post_internal_cmd,
 162	.dev_config		= ahci_dev_config,
 163
 164	.scr_read		= ahci_scr_read,
 165	.scr_write		= ahci_scr_write,
 166	.pmp_attach		= ahci_pmp_attach,
 167	.pmp_detach		= ahci_pmp_detach,
 168
 169	.set_lpm		= ahci_set_lpm,
 170	.em_show		= ahci_led_show,
 171	.em_store		= ahci_led_store,
 172	.sw_activity_show	= ahci_activity_show,
 173	.sw_activity_store	= ahci_activity_store,
 174	.transmit_led_message	= ahci_transmit_led_message,
 175#ifdef CONFIG_PM
 176	.port_suspend		= ahci_port_suspend,
 177	.port_resume		= ahci_port_resume,
 178#endif
 179	.port_start		= ahci_port_start,
 180	.port_stop		= ahci_port_stop,
 181};
 182EXPORT_SYMBOL_GPL(ahci_ops);
 183
 184struct ata_port_operations ahci_pmp_retry_srst_ops = {
 185	.inherits		= &ahci_ops,
 186	.softreset		= ahci_pmp_retry_softreset,
 187};
 188EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
 189
 190static bool ahci_em_messages __read_mostly = true;
 191EXPORT_SYMBOL_GPL(ahci_em_messages);
 192module_param(ahci_em_messages, bool, 0444);
 193/* add other LED protocol types when they become supported */
 194MODULE_PARM_DESC(ahci_em_messages,
 195	"AHCI Enclosure Management Message control (0 = off, 1 = on)");
 196
 197/* device sleep idle timeout in ms */
 198static int devslp_idle_timeout __read_mostly = 1000;
 199module_param(devslp_idle_timeout, int, 0644);
 200MODULE_PARM_DESC(devslp_idle_timeout, "device sleep idle timeout");
 201
 202static void ahci_enable_ahci(void __iomem *mmio)
 203{
 204	int i;
 205	u32 tmp;
 206
 207	/* turn on AHCI_EN */
 208	tmp = readl(mmio + HOST_CTL);
 209	if (tmp & HOST_AHCI_EN)
 210		return;
 211
 212	/* Some controllers need AHCI_EN to be written multiple times.
 213	 * Try a few times before giving up.
 214	 */
 215	for (i = 0; i < 5; i++) {
 216		tmp |= HOST_AHCI_EN;
 217		writel(tmp, mmio + HOST_CTL);
 218		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
 219		if (tmp & HOST_AHCI_EN)
 220			return;
 221		msleep(10);
 222	}
 223
 224	WARN_ON(1);
 225}
 226
 227static ssize_t ahci_show_host_caps(struct device *dev,
 228				   struct device_attribute *attr, char *buf)
 229{
 230	struct Scsi_Host *shost = class_to_shost(dev);
 231	struct ata_port *ap = ata_shost_to_port(shost);
 232	struct ahci_host_priv *hpriv = ap->host->private_data;
 233
 234	return sprintf(buf, "%x\n", hpriv->cap);
 235}
 236
 237static ssize_t ahci_show_host_cap2(struct device *dev,
 238				   struct device_attribute *attr, char *buf)
 239{
 240	struct Scsi_Host *shost = class_to_shost(dev);
 241	struct ata_port *ap = ata_shost_to_port(shost);
 242	struct ahci_host_priv *hpriv = ap->host->private_data;
 243
 244	return sprintf(buf, "%x\n", hpriv->cap2);
 245}
 246
 247static ssize_t ahci_show_host_version(struct device *dev,
 248				   struct device_attribute *attr, char *buf)
 249{
 250	struct Scsi_Host *shost = class_to_shost(dev);
 251	struct ata_port *ap = ata_shost_to_port(shost);
 252	struct ahci_host_priv *hpriv = ap->host->private_data;
 253	void __iomem *mmio = hpriv->mmio;
 254
 255	return sprintf(buf, "%x\n", readl(mmio + HOST_VERSION));
 256}
 257
 258static ssize_t ahci_show_port_cmd(struct device *dev,
 259				  struct device_attribute *attr, char *buf)
 260{
 261	struct Scsi_Host *shost = class_to_shost(dev);
 262	struct ata_port *ap = ata_shost_to_port(shost);
 263	void __iomem *port_mmio = ahci_port_base(ap);
 264
 265	return sprintf(buf, "%x\n", readl(port_mmio + PORT_CMD));
 266}
 267
 268static ssize_t ahci_read_em_buffer(struct device *dev,
 269				   struct device_attribute *attr, char *buf)
 270{
 271	struct Scsi_Host *shost = class_to_shost(dev);
 272	struct ata_port *ap = ata_shost_to_port(shost);
 273	struct ahci_host_priv *hpriv = ap->host->private_data;
 274	void __iomem *mmio = hpriv->mmio;
 275	void __iomem *em_mmio = mmio + hpriv->em_loc;
 276	u32 em_ctl, msg;
 277	unsigned long flags;
 278	size_t count;
 279	int i;
 280
 281	spin_lock_irqsave(ap->lock, flags);
 282
 283	em_ctl = readl(mmio + HOST_EM_CTL);
 284	if (!(ap->flags & ATA_FLAG_EM) || em_ctl & EM_CTL_XMT ||
 285	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {
 286		spin_unlock_irqrestore(ap->lock, flags);
 287		return -EINVAL;
 288	}
 289
 290	if (!(em_ctl & EM_CTL_MR)) {
 291		spin_unlock_irqrestore(ap->lock, flags);
 292		return -EAGAIN;
 293	}
 294
 295	if (!(em_ctl & EM_CTL_SMB))
 296		em_mmio += hpriv->em_buf_sz;
 297
 298	count = hpriv->em_buf_sz;
 299
 300	/* the count should not be larger than PAGE_SIZE */
 301	if (count > PAGE_SIZE) {
 302		if (printk_ratelimit())
 303			ata_port_warn(ap,
 304				      "EM read buffer size too large: "
 305				      "buffer size %u, page size %lu\n",
 306				      hpriv->em_buf_sz, PAGE_SIZE);
 307		count = PAGE_SIZE;
 308	}
 309
 310	for (i = 0; i < count; i += 4) {
 311		msg = readl(em_mmio + i);
 312		buf[i] = msg & 0xff;
 313		buf[i + 1] = (msg >> 8) & 0xff;
 314		buf[i + 2] = (msg >> 16) & 0xff;
 315		buf[i + 3] = (msg >> 24) & 0xff;
 316	}
 317
 318	spin_unlock_irqrestore(ap->lock, flags);
 319
 320	return i;
 321}
 322
 323static ssize_t ahci_store_em_buffer(struct device *dev,
 324				    struct device_attribute *attr,
 325				    const char *buf, size_t size)
 326{
 327	struct Scsi_Host *shost = class_to_shost(dev);
 328	struct ata_port *ap = ata_shost_to_port(shost);
 329	struct ahci_host_priv *hpriv = ap->host->private_data;
 330	void __iomem *mmio = hpriv->mmio;
 331	void __iomem *em_mmio = mmio + hpriv->em_loc;
 332	const unsigned char *msg_buf = buf;
 333	u32 em_ctl, msg;
 334	unsigned long flags;
 335	int i;
 336
 337	/* check size validity */
 338	if (!(ap->flags & ATA_FLAG_EM) ||
 339	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) ||
 340	    size % 4 || size > hpriv->em_buf_sz)
 341		return -EINVAL;
 342
 343	spin_lock_irqsave(ap->lock, flags);
 344
 345	em_ctl = readl(mmio + HOST_EM_CTL);
 346	if (em_ctl & EM_CTL_TM) {
 347		spin_unlock_irqrestore(ap->lock, flags);
 348		return -EBUSY;
 349	}
 350
 351	for (i = 0; i < size; i += 4) {
 352		msg = msg_buf[i] | msg_buf[i + 1] << 8 |
 353		      msg_buf[i + 2] << 16 | msg_buf[i + 3] << 24;
 354		writel(msg, em_mmio + i);
 355	}
 356
 357	writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
 358
 359	spin_unlock_irqrestore(ap->lock, flags);
 360
 361	return size;
 362}
 363
 364static ssize_t ahci_show_em_supported(struct device *dev,
 365				      struct device_attribute *attr, char *buf)
 366{
 367	struct Scsi_Host *shost = class_to_shost(dev);
 368	struct ata_port *ap = ata_shost_to_port(shost);
 369	struct ahci_host_priv *hpriv = ap->host->private_data;
 370	void __iomem *mmio = hpriv->mmio;
 371	u32 em_ctl;
 372
 373	em_ctl = readl(mmio + HOST_EM_CTL);
 374
 375	return sprintf(buf, "%s%s%s%s\n",
 376		       em_ctl & EM_CTL_LED ? "led " : "",
 377		       em_ctl & EM_CTL_SAFTE ? "saf-te " : "",
 378		       em_ctl & EM_CTL_SES ? "ses-2 " : "",
 379		       em_ctl & EM_CTL_SGPIO ? "sgpio " : "");
 380}
 381
 382/**
 383 *	ahci_save_initial_config - Save and fixup initial config values
 384 *	@dev: target AHCI device
 385 *	@hpriv: host private area to store config values
 386 *	@force_port_map: force port map to a specified value
 387 *	@mask_port_map: mask out particular bits from port map
 388 *
 389 *	Some registers containing configuration info might be setup by
 390 *	BIOS and might be cleared on reset.  This function saves the
 391 *	initial values of those registers into @hpriv such that they
 392 *	can be restored after controller reset.
 393 *
 394 *	If inconsistent, config values are fixed up by this function.
 395 *
 396 *	If it is not set already this function sets hpriv->start_engine to
 397 *	ahci_start_engine.
 398 *
 399 *	LOCKING:
 400 *	None.
 401 */
 402void ahci_save_initial_config(struct device *dev,
 403			      struct ahci_host_priv *hpriv,
 404			      unsigned int force_port_map,
 405			      unsigned int mask_port_map)
 406{
 407	void __iomem *mmio = hpriv->mmio;
 408	u32 cap, cap2, vers, port_map;
 409	int i;
 410
 411	/* make sure AHCI mode is enabled before accessing CAP */
 412	ahci_enable_ahci(mmio);
 413
 414	/* Values prefixed with saved_ are written back to host after
 415	 * reset.  Values without are used for driver operation.
 416	 */
 417	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
 418	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
 419
 420	/* CAP2 register is only defined for AHCI 1.2 and later */
 421	vers = readl(mmio + HOST_VERSION);
 422	if ((vers >> 16) > 1 ||
 423	   ((vers >> 16) == 1 && (vers & 0xFFFF) >= 0x200))
 424		hpriv->saved_cap2 = cap2 = readl(mmio + HOST_CAP2);
 425	else
 426		hpriv->saved_cap2 = cap2 = 0;
 427
 428	/* some chips have errata preventing 64bit use */
 429	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
 430		dev_info(dev, "controller can't do 64bit DMA, forcing 32bit\n");
 431		cap &= ~HOST_CAP_64;
 432	}
 433
 434	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
 435		dev_info(dev, "controller can't do NCQ, turning off CAP_NCQ\n");
 436		cap &= ~HOST_CAP_NCQ;
 437	}
 438
 439	if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
 440		dev_info(dev, "controller can do NCQ, turning on CAP_NCQ\n");
 441		cap |= HOST_CAP_NCQ;
 442	}
 443
 444	if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
 445		dev_info(dev, "controller can't do PMP, turning off CAP_PMP\n");
 446		cap &= ~HOST_CAP_PMP;
 447	}
 448
 449	if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {
 450		dev_info(dev,
 451			 "controller can't do SNTF, turning off CAP_SNTF\n");
 452		cap &= ~HOST_CAP_SNTF;
 453	}
 454
 455	if ((cap2 & HOST_CAP2_SDS) && (hpriv->flags & AHCI_HFLAG_NO_DEVSLP)) {
 456		dev_info(dev,
 457			 "controller can't do DEVSLP, turning off\n");
 458		cap2 &= ~HOST_CAP2_SDS;
 459		cap2 &= ~HOST_CAP2_SADM;
 460	}
 461
 462	if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {
 463		dev_info(dev, "controller can do FBS, turning on CAP_FBS\n");
 464		cap |= HOST_CAP_FBS;
 465	}
 466
 467	if (force_port_map && port_map != force_port_map) {
 468		dev_info(dev, "forcing port_map 0x%x -> 0x%x\n",
 469			 port_map, force_port_map);
 470		port_map = force_port_map;
 471	}
 472
 473	if (mask_port_map) {
 474		dev_warn(dev, "masking port_map 0x%x -> 0x%x\n",
 475			port_map,
 476			port_map & mask_port_map);
 477		port_map &= mask_port_map;
 478	}
 479
 480	/* cross check port_map and cap.n_ports */
 481	if (port_map) {
 482		int map_ports = 0;
 483
 484		for (i = 0; i < AHCI_MAX_PORTS; i++)
 485			if (port_map & (1 << i))
 486				map_ports++;
 487
 488		/* If PI has more ports than n_ports, whine, clear
 489		 * port_map and let it be generated from n_ports.
 490		 */
 491		if (map_ports > ahci_nr_ports(cap)) {
 492			dev_warn(dev,
 493				 "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
 494				 port_map, ahci_nr_ports(cap));
 495			port_map = 0;
 496		}
 497	}
 498
 499	/* fabricate port_map from cap.nr_ports */
 500	if (!port_map) {
 501		port_map = (1 << ahci_nr_ports(cap)) - 1;
 502		dev_warn(dev, "forcing PORTS_IMPL to 0x%x\n", port_map);
 503
 504		/* write the fixed up value to the PI register */
 505		hpriv->saved_port_map = port_map;
 506	}
 507
 508	/* record values to use during operation */
 509	hpriv->cap = cap;
 510	hpriv->cap2 = cap2;
 511	hpriv->port_map = port_map;
 512
 513	if (!hpriv->start_engine)
 514		hpriv->start_engine = ahci_start_engine;
 515}
 516EXPORT_SYMBOL_GPL(ahci_save_initial_config);
 517
 518/**
 519 *	ahci_restore_initial_config - Restore initial config
 520 *	@host: target ATA host
 521 *
 522 *	Restore initial config stored by ahci_save_initial_config().
 523 *
 524 *	LOCKING:
 525 *	None.
 526 */
 527static void ahci_restore_initial_config(struct ata_host *host)
 528{
 529	struct ahci_host_priv *hpriv = host->private_data;
 530	void __iomem *mmio = hpriv->mmio;
 531
 532	writel(hpriv->saved_cap, mmio + HOST_CAP);
 533	if (hpriv->saved_cap2)
 534		writel(hpriv->saved_cap2, mmio + HOST_CAP2);
 535	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
 536	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
 537}
 538
 539static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
 540{
 541	static const int offset[] = {
 542		[SCR_STATUS]		= PORT_SCR_STAT,
 543		[SCR_CONTROL]		= PORT_SCR_CTL,
 544		[SCR_ERROR]		= PORT_SCR_ERR,
 545		[SCR_ACTIVE]		= PORT_SCR_ACT,
 546		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
 547	};
 548	struct ahci_host_priv *hpriv = ap->host->private_data;
 549
 550	if (sc_reg < ARRAY_SIZE(offset) &&
 551	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
 552		return offset[sc_reg];
 553	return 0;
 554}
 555
 556static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
 557{
 558	void __iomem *port_mmio = ahci_port_base(link->ap);
 559	int offset = ahci_scr_offset(link->ap, sc_reg);
 560
 561	if (offset) {
 562		*val = readl(port_mmio + offset);
 563		return 0;
 564	}
 565	return -EINVAL;
 566}
 567
 568static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
 569{
 570	void __iomem *port_mmio = ahci_port_base(link->ap);
 571	int offset = ahci_scr_offset(link->ap, sc_reg);
 572
 573	if (offset) {
 574		writel(val, port_mmio + offset);
 575		return 0;
 576	}
 577	return -EINVAL;
 578}
 579
 580void ahci_start_engine(struct ata_port *ap)
 581{
 582	void __iomem *port_mmio = ahci_port_base(ap);
 583	u32 tmp;
 584
 585	/* start DMA */
 586	tmp = readl(port_mmio + PORT_CMD);
 587	tmp |= PORT_CMD_START;
 588	writel(tmp, port_mmio + PORT_CMD);
 589	readl(port_mmio + PORT_CMD); /* flush */
 590}
 591EXPORT_SYMBOL_GPL(ahci_start_engine);
 592
 593int ahci_stop_engine(struct ata_port *ap)
 594{
 595	void __iomem *port_mmio = ahci_port_base(ap);
 596	u32 tmp;
 597
 598	tmp = readl(port_mmio + PORT_CMD);
 599
 600	/* check if the HBA is idle */
 601	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
 602		return 0;
 603
 604	/* setting HBA to idle */
 605	tmp &= ~PORT_CMD_START;
 606	writel(tmp, port_mmio + PORT_CMD);
 607
 608	/* wait for engine to stop. This could be as long as 500 msec */
 609	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
 610				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
 611	if (tmp & PORT_CMD_LIST_ON)
 612		return -EIO;
 613
 614	return 0;
 615}
 616EXPORT_SYMBOL_GPL(ahci_stop_engine);
 617
 618static void ahci_start_fis_rx(struct ata_port *ap)
 619{
 620	void __iomem *port_mmio = ahci_port_base(ap);
 621	struct ahci_host_priv *hpriv = ap->host->private_data;
 622	struct ahci_port_priv *pp = ap->private_data;
 623	u32 tmp;
 624
 625	/* set FIS registers */
 626	if (hpriv->cap & HOST_CAP_64)
 627		writel((pp->cmd_slot_dma >> 16) >> 16,
 628		       port_mmio + PORT_LST_ADDR_HI);
 629	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
 630
 631	if (hpriv->cap & HOST_CAP_64)
 632		writel((pp->rx_fis_dma >> 16) >> 16,
 633		       port_mmio + PORT_FIS_ADDR_HI);
 634	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
 635
 636	/* enable FIS reception */
 637	tmp = readl(port_mmio + PORT_CMD);
 638	tmp |= PORT_CMD_FIS_RX;
 639	writel(tmp, port_mmio + PORT_CMD);
 640
 641	/* flush */
 642	readl(port_mmio + PORT_CMD);
 643}
 644
 645static int ahci_stop_fis_rx(struct ata_port *ap)
 646{
 647	void __iomem *port_mmio = ahci_port_base(ap);
 648	u32 tmp;
 649
 650	/* disable FIS reception */
 651	tmp = readl(port_mmio + PORT_CMD);
 652	tmp &= ~PORT_CMD_FIS_RX;
 653	writel(tmp, port_mmio + PORT_CMD);
 654
 655	/* wait for completion, spec says 500ms, give it 1000 */
 656	tmp = ata_wait_register(ap, port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
 657				PORT_CMD_FIS_ON, 10, 1000);
 658	if (tmp & PORT_CMD_FIS_ON)
 659		return -EBUSY;
 660
 661	return 0;
 662}
 663
 664static void ahci_power_up(struct ata_port *ap)
 665{
 666	struct ahci_host_priv *hpriv = ap->host->private_data;
 667	void __iomem *port_mmio = ahci_port_base(ap);
 668	u32 cmd;
 669
 670	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 671
 672	/* spin up device */
 673	if (hpriv->cap & HOST_CAP_SSS) {
 674		cmd |= PORT_CMD_SPIN_UP;
 675		writel(cmd, port_mmio + PORT_CMD);
 676	}
 677
 678	/* wake up link */
 679	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
 680}
 681
 682static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
 683			unsigned int hints)
 684{
 685	struct ata_port *ap = link->ap;
 686	struct ahci_host_priv *hpriv = ap->host->private_data;
 687	struct ahci_port_priv *pp = ap->private_data;
 688	void __iomem *port_mmio = ahci_port_base(ap);
 689
 690	if (policy != ATA_LPM_MAX_POWER) {
 691		/*
 692		 * Disable interrupts on Phy Ready. This keeps us from
 693		 * getting woken up due to spurious phy ready
 694		 * interrupts.
 695		 */
 696		pp->intr_mask &= ~PORT_IRQ_PHYRDY;
 697		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 698
 699		sata_link_scr_lpm(link, policy, false);
 700	}
 701
 702	if (hpriv->cap & HOST_CAP_ALPM) {
 703		u32 cmd = readl(port_mmio + PORT_CMD);
 704
 705		if (policy == ATA_LPM_MAX_POWER || !(hints & ATA_LPM_HIPM)) {
 706			cmd &= ~(PORT_CMD_ASP | PORT_CMD_ALPE);
 707			cmd |= PORT_CMD_ICC_ACTIVE;
 708
 709			writel(cmd, port_mmio + PORT_CMD);
 710			readl(port_mmio + PORT_CMD);
 711
 712			/* wait 10ms to be sure we've come out of LPM state */
 713			ata_msleep(ap, 10);
 714		} else {
 715			cmd |= PORT_CMD_ALPE;
 716			if (policy == ATA_LPM_MIN_POWER)
 717				cmd |= PORT_CMD_ASP;
 718
 719			/* write out new cmd value */
 720			writel(cmd, port_mmio + PORT_CMD);
 721		}
 722	}
 723
 724	/* set aggressive device sleep */
 725	if ((hpriv->cap2 & HOST_CAP2_SDS) &&
 726	    (hpriv->cap2 & HOST_CAP2_SADM) &&
 727	    (link->device->flags & ATA_DFLAG_DEVSLP)) {
 728		if (policy == ATA_LPM_MIN_POWER)
 729			ahci_set_aggressive_devslp(ap, true);
 730		else
 731			ahci_set_aggressive_devslp(ap, false);
 732	}
 733
 734	if (policy == ATA_LPM_MAX_POWER) {
 735		sata_link_scr_lpm(link, policy, false);
 736
 737		/* turn PHYRDY IRQ back on */
 738		pp->intr_mask |= PORT_IRQ_PHYRDY;
 739		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 740	}
 741
 742	return 0;
 743}
 744
 745#ifdef CONFIG_PM
 746static void ahci_power_down(struct ata_port *ap)
 747{
 748	struct ahci_host_priv *hpriv = ap->host->private_data;
 749	void __iomem *port_mmio = ahci_port_base(ap);
 750	u32 cmd, scontrol;
 751
 752	if (!(hpriv->cap & HOST_CAP_SSS))
 753		return;
 754
 755	/* put device into listen mode, first set PxSCTL.DET to 0 */
 756	scontrol = readl(port_mmio + PORT_SCR_CTL);
 757	scontrol &= ~0xf;
 758	writel(scontrol, port_mmio + PORT_SCR_CTL);
 759
 760	/* then set PxCMD.SUD to 0 */
 761	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 762	cmd &= ~PORT_CMD_SPIN_UP;
 763	writel(cmd, port_mmio + PORT_CMD);
 764}
 765#endif
 766
 767static void ahci_start_port(struct ata_port *ap)
 768{
 769	struct ahci_host_priv *hpriv = ap->host->private_data;
 770	struct ahci_port_priv *pp = ap->private_data;
 771	struct ata_link *link;
 772	struct ahci_em_priv *emp;
 773	ssize_t rc;
 774	int i;
 775
 776	/* enable FIS reception */
 777	ahci_start_fis_rx(ap);
 778
 779	/* enable DMA */
 780	if (!(hpriv->flags & AHCI_HFLAG_DELAY_ENGINE))
 781		hpriv->start_engine(ap);
 782
 783	/* turn on LEDs */
 784	if (ap->flags & ATA_FLAG_EM) {
 785		ata_for_each_link(link, ap, EDGE) {
 786			emp = &pp->em_priv[link->pmp];
 787
 788			/* EM Transmit bit maybe busy during init */
 789			for (i = 0; i < EM_MAX_RETRY; i++) {
 790				rc = ap->ops->transmit_led_message(ap,
 791							       emp->led_state,
 792							       4);
 793				/*
 794				 * If busy, give a breather but do not
 795				 * release EH ownership by using msleep()
 796				 * instead of ata_msleep().  EM Transmit
 797				 * bit is busy for the whole host and
 798				 * releasing ownership will cause other
 799				 * ports to fail the same way.
 800				 */
 801				if (rc == -EBUSY)
 802					msleep(1);
 803				else
 804					break;
 805			}
 806		}
 807	}
 808
 809	if (ap->flags & ATA_FLAG_SW_ACTIVITY)
 810		ata_for_each_link(link, ap, EDGE)
 811			ahci_init_sw_activity(link);
 812
 813}
 814
 815static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
 816{
 817	int rc;
 818
 819	/* disable DMA */
 820	rc = ahci_stop_engine(ap);
 821	if (rc) {
 822		*emsg = "failed to stop engine";
 823		return rc;
 824	}
 825
 826	/* disable FIS reception */
 827	rc = ahci_stop_fis_rx(ap);
 828	if (rc) {
 829		*emsg = "failed stop FIS RX";
 830		return rc;
 831	}
 832
 833	return 0;
 834}
 835
 836int ahci_reset_controller(struct ata_host *host)
 837{
 838	struct ahci_host_priv *hpriv = host->private_data;
 839	void __iomem *mmio = hpriv->mmio;
 840	u32 tmp;
 841
 842	/* we must be in AHCI mode, before using anything
 843	 * AHCI-specific, such as HOST_RESET.
 844	 */
 845	ahci_enable_ahci(mmio);
 846
 847	/* global controller reset */
 848	if (!ahci_skip_host_reset) {
 849		tmp = readl(mmio + HOST_CTL);
 850		if ((tmp & HOST_RESET) == 0) {
 851			writel(tmp | HOST_RESET, mmio + HOST_CTL);
 852			readl(mmio + HOST_CTL); /* flush */
 853		}
 854
 855		/*
 856		 * to perform host reset, OS should set HOST_RESET
 857		 * and poll until this bit is read to be "0".
 858		 * reset must complete within 1 second, or
 859		 * the hardware should be considered fried.
 860		 */
 861		tmp = ata_wait_register(NULL, mmio + HOST_CTL, HOST_RESET,
 862					HOST_RESET, 10, 1000);
 863
 864		if (tmp & HOST_RESET) {
 865			dev_err(host->dev, "controller reset failed (0x%x)\n",
 866				tmp);
 867			return -EIO;
 868		}
 869
 870		/* turn on AHCI mode */
 871		ahci_enable_ahci(mmio);
 872
 873		/* Some registers might be cleared on reset.  Restore
 874		 * initial values.
 875		 */
 876		ahci_restore_initial_config(host);
 877	} else
 878		dev_info(host->dev, "skipping global host reset\n");
 879
 880	return 0;
 881}
 882EXPORT_SYMBOL_GPL(ahci_reset_controller);
 883
 884static void ahci_sw_activity(struct ata_link *link)
 885{
 886	struct ata_port *ap = link->ap;
 887	struct ahci_port_priv *pp = ap->private_data;
 888	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 889
 890	if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
 891		return;
 892
 893	emp->activity++;
 894	if (!timer_pending(&emp->timer))
 895		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));
 896}
 897
 898static void ahci_sw_activity_blink(unsigned long arg)
 899{
 900	struct ata_link *link = (struct ata_link *)arg;
 901	struct ata_port *ap = link->ap;
 902	struct ahci_port_priv *pp = ap->private_data;
 903	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 904	unsigned long led_message = emp->led_state;
 905	u32 activity_led_state;
 906	unsigned long flags;
 907
 908	led_message &= EM_MSG_LED_VALUE;
 909	led_message |= ap->port_no | (link->pmp << 8);
 910
 911	/* check to see if we've had activity.  If so,
 912	 * toggle state of LED and reset timer.  If not,
 913	 * turn LED to desired idle state.
 914	 */
 915	spin_lock_irqsave(ap->lock, flags);
 916	if (emp->saved_activity != emp->activity) {
 917		emp->saved_activity = emp->activity;
 918		/* get the current LED state */
 919		activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
 920
 921		if (activity_led_state)
 922			activity_led_state = 0;
 923		else
 924			activity_led_state = 1;
 925
 926		/* clear old state */
 927		led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
 928
 929		/* toggle state */
 930		led_message |= (activity_led_state << 16);
 931		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));
 932	} else {
 933		/* switch to idle */
 934		led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
 935		if (emp->blink_policy == BLINK_OFF)
 936			led_message |= (1 << 16);
 937	}
 938	spin_unlock_irqrestore(ap->lock, flags);
 939	ap->ops->transmit_led_message(ap, led_message, 4);
 940}
 941
 942static void ahci_init_sw_activity(struct ata_link *link)
 943{
 944	struct ata_port *ap = link->ap;
 945	struct ahci_port_priv *pp = ap->private_data;
 946	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 947
 948	/* init activity stats, setup timer */
 949	emp->saved_activity = emp->activity = 0;
 950	setup_timer(&emp->timer, ahci_sw_activity_blink, (unsigned long)link);
 951
 952	/* check our blink policy and set flag for link if it's enabled */
 953	if (emp->blink_policy)
 954		link->flags |= ATA_LFLAG_SW_ACTIVITY;
 955}
 956
 957int ahci_reset_em(struct ata_host *host)
 958{
 959	struct ahci_host_priv *hpriv = host->private_data;
 960	void __iomem *mmio = hpriv->mmio;
 961	u32 em_ctl;
 962
 963	em_ctl = readl(mmio + HOST_EM_CTL);
 964	if ((em_ctl & EM_CTL_TM) || (em_ctl & EM_CTL_RST))
 965		return -EINVAL;
 966
 967	writel(em_ctl | EM_CTL_RST, mmio + HOST_EM_CTL);
 968	return 0;
 969}
 970EXPORT_SYMBOL_GPL(ahci_reset_em);
 971
 972static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
 973					ssize_t size)
 974{
 975	struct ahci_host_priv *hpriv = ap->host->private_data;
 976	struct ahci_port_priv *pp = ap->private_data;
 977	void __iomem *mmio = hpriv->mmio;
 978	u32 em_ctl;
 979	u32 message[] = {0, 0};
 980	unsigned long flags;
 981	int pmp;
 982	struct ahci_em_priv *emp;
 983
 984	/* get the slot number from the message */
 985	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
 986	if (pmp < EM_MAX_SLOTS)
 987		emp = &pp->em_priv[pmp];
 988	else
 989		return -EINVAL;
 990
 991	spin_lock_irqsave(ap->lock, flags);
 992
 993	/*
 994	 * if we are still busy transmitting a previous message,
 995	 * do not allow
 996	 */
 997	em_ctl = readl(mmio + HOST_EM_CTL);
 998	if (em_ctl & EM_CTL_TM) {
 999		spin_unlock_irqrestore(ap->lock, flags);
1000		return -EBUSY;
1001	}
1002
1003	if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {
1004		/*
1005		 * create message header - this is all zero except for
1006		 * the message size, which is 4 bytes.
1007		 */
1008		message[0] |= (4 << 8);
1009
1010		/* ignore 0:4 of byte zero, fill in port info yourself */
1011		message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);
1012
1013		/* write message to EM_LOC */
1014		writel(message[0], mmio + hpriv->em_loc);
1015		writel(message[1], mmio + hpriv->em_loc+4);
1016
1017		/*
1018		 * tell hardware to transmit the message
1019		 */
1020		writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
1021	}
1022
1023	/* save off new led state for port/slot */
1024	emp->led_state = state;
1025
1026	spin_unlock_irqrestore(ap->lock, flags);
1027	return size;
1028}
1029
1030static ssize_t ahci_led_show(struct ata_port *ap, char *buf)
1031{
1032	struct ahci_port_priv *pp = ap->private_data;
1033	struct ata_link *link;
1034	struct ahci_em_priv *emp;
1035	int rc = 0;
1036
1037	ata_for_each_link(link, ap, EDGE) {
1038		emp = &pp->em_priv[link->pmp];
1039		rc += sprintf(buf, "%lx\n", emp->led_state);
1040	}
1041	return rc;
1042}
1043
1044static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
1045				size_t size)
1046{
1047	unsigned int state;
1048	int pmp;
1049	struct ahci_port_priv *pp = ap->private_data;
1050	struct ahci_em_priv *emp;
1051
1052	if (kstrtouint(buf, 0, &state) < 0)
1053		return -EINVAL;
1054
1055	/* get the slot number from the message */
1056	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1057	if (pmp < EM_MAX_SLOTS)
1058		emp = &pp->em_priv[pmp];
1059	else
1060		return -EINVAL;
1061
1062	/* mask off the activity bits if we are in sw_activity
1063	 * mode, user should turn off sw_activity before setting
1064	 * activity led through em_message
1065	 */
1066	if (emp->blink_policy)
1067		state &= ~EM_MSG_LED_VALUE_ACTIVITY;
1068
1069	return ap->ops->transmit_led_message(ap, state, size);
1070}
1071
1072static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
1073{
1074	struct ata_link *link = dev->link;
1075	struct ata_port *ap = link->ap;
1076	struct ahci_port_priv *pp = ap->private_data;
1077	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1078	u32 port_led_state = emp->led_state;
1079
1080	/* save the desired Activity LED behavior */
1081	if (val == OFF) {
1082		/* clear LFLAG */
1083		link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
1084
1085		/* set the LED to OFF */
1086		port_led_state &= EM_MSG_LED_VALUE_OFF;
1087		port_led_state |= (ap->port_no | (link->pmp << 8));
1088		ap->ops->transmit_led_message(ap, port_led_state, 4);
1089	} else {
1090		link->flags |= ATA_LFLAG_SW_ACTIVITY;
1091		if (val == BLINK_OFF) {
1092			/* set LED to ON for idle */
1093			port_led_state &= EM_MSG_LED_VALUE_OFF;
1094			port_led_state |= (ap->port_no | (link->pmp << 8));
1095			port_led_state |= EM_MSG_LED_VALUE_ON; /* check this */
1096			ap->ops->transmit_led_message(ap, port_led_state, 4);
1097		}
1098	}
1099	emp->blink_policy = val;
1100	return 0;
1101}
1102
1103static ssize_t ahci_activity_show(struct ata_device *dev, char *buf)
1104{
1105	struct ata_link *link = dev->link;
1106	struct ata_port *ap = link->ap;
1107	struct ahci_port_priv *pp = ap->private_data;
1108	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1109
1110	/* display the saved value of activity behavior for this
1111	 * disk.
1112	 */
1113	return sprintf(buf, "%d\n", emp->blink_policy);
1114}
1115
1116static void ahci_port_init(struct device *dev, struct ata_port *ap,
1117			   int port_no, void __iomem *mmio,
1118			   void __iomem *port_mmio)
1119{
1120	const char *emsg = NULL;
1121	int rc;
1122	u32 tmp;
1123
1124	/* make sure port is not active */
1125	rc = ahci_deinit_port(ap, &emsg);
1126	if (rc)
1127		dev_warn(dev, "%s (%d)\n", emsg, rc);
1128
1129	/* clear SError */
1130	tmp = readl(port_mmio + PORT_SCR_ERR);
1131	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
1132	writel(tmp, port_mmio + PORT_SCR_ERR);
1133
1134	/* clear port IRQ */
1135	tmp = readl(port_mmio + PORT_IRQ_STAT);
1136	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
1137	if (tmp)
1138		writel(tmp, port_mmio + PORT_IRQ_STAT);
1139
1140	writel(1 << port_no, mmio + HOST_IRQ_STAT);
1141}
1142
1143void ahci_init_controller(struct ata_host *host)
1144{
1145	struct ahci_host_priv *hpriv = host->private_data;
1146	void __iomem *mmio = hpriv->mmio;
1147	int i;
1148	void __iomem *port_mmio;
1149	u32 tmp;
1150
1151	for (i = 0; i < host->n_ports; i++) {
1152		struct ata_port *ap = host->ports[i];
1153
1154		port_mmio = ahci_port_base(ap);
1155		if (ata_port_is_dummy(ap))
1156			continue;
1157
1158		ahci_port_init(host->dev, ap, i, mmio, port_mmio);
1159	}
1160
1161	tmp = readl(mmio + HOST_CTL);
1162	VPRINTK("HOST_CTL 0x%x\n", tmp);
1163	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
1164	tmp = readl(mmio + HOST_CTL);
1165	VPRINTK("HOST_CTL 0x%x\n", tmp);
1166}
1167EXPORT_SYMBOL_GPL(ahci_init_controller);
1168
1169static void ahci_dev_config(struct ata_device *dev)
1170{
1171	struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
1172
1173	if (hpriv->flags & AHCI_HFLAG_SECT255) {
1174		dev->max_sectors = 255;
1175		ata_dev_info(dev,
1176			     "SB600 AHCI: limiting to 255 sectors per cmd\n");
1177	}
1178}
1179
1180unsigned int ahci_dev_classify(struct ata_port *ap)
1181{
1182	void __iomem *port_mmio = ahci_port_base(ap);
1183	struct ata_taskfile tf;
1184	u32 tmp;
1185
1186	tmp = readl(port_mmio + PORT_SIG);
1187	tf.lbah		= (tmp >> 24)	& 0xff;
1188	tf.lbam		= (tmp >> 16)	& 0xff;
1189	tf.lbal		= (tmp >> 8)	& 0xff;
1190	tf.nsect	= (tmp)		& 0xff;
1191
1192	return ata_dev_classify(&tf);
1193}
1194EXPORT_SYMBOL_GPL(ahci_dev_classify);
1195
1196void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
1197			u32 opts)
1198{
1199	dma_addr_t cmd_tbl_dma;
1200
1201	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
1202
1203	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
1204	pp->cmd_slot[tag].status = 0;
1205	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
1206	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1207}
1208EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot);
1209
1210int ahci_kick_engine(struct ata_port *ap)
1211{
1212	void __iomem *port_mmio = ahci_port_base(ap);
1213	struct ahci_host_priv *hpriv = ap->host->private_data;
1214	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1215	u32 tmp;
1216	int busy, rc;
1217
1218	/* stop engine */
1219	rc = ahci_stop_engine(ap);
1220	if (rc)
1221		goto out_restart;
1222
1223	/* need to do CLO?
1224	 * always do CLO if PMP is attached (AHCI-1.3 9.2)
1225	 */
1226	busy = status & (ATA_BUSY | ATA_DRQ);
1227	if (!busy && !sata_pmp_attached(ap)) {
1228		rc = 0;
1229		goto out_restart;
1230	}
1231
1232	if (!(hpriv->cap & HOST_CAP_CLO)) {
1233		rc = -EOPNOTSUPP;
1234		goto out_restart;
1235	}
1236
1237	/* perform CLO */
1238	tmp = readl(port_mmio + PORT_CMD);
1239	tmp |= PORT_CMD_CLO;
1240	writel(tmp, port_mmio + PORT_CMD);
1241
1242	rc = 0;
1243	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
1244				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
1245	if (tmp & PORT_CMD_CLO)
1246		rc = -EIO;
1247
1248	/* restart engine */
1249 out_restart:
1250	hpriv->start_engine(ap);
1251	return rc;
1252}
1253EXPORT_SYMBOL_GPL(ahci_kick_engine);
1254
1255static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
1256				struct ata_taskfile *tf, int is_cmd, u16 flags,
1257				unsigned long timeout_msec)
1258{
1259	const u32 cmd_fis_len = 5; /* five dwords */
1260	struct ahci_port_priv *pp = ap->private_data;
1261	void __iomem *port_mmio = ahci_port_base(ap);
1262	u8 *fis = pp->cmd_tbl;
1263	u32 tmp;
1264
1265	/* prep the command */
1266	ata_tf_to_fis(tf, pmp, is_cmd, fis);
1267	ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
1268
1269	/* issue & wait */
1270	writel(1, port_mmio + PORT_CMD_ISSUE);
1271
1272	if (timeout_msec) {
1273		tmp = ata_wait_register(ap, port_mmio + PORT_CMD_ISSUE,
1274					0x1, 0x1, 1, timeout_msec);
1275		if (tmp & 0x1) {
1276			ahci_kick_engine(ap);
1277			return -EBUSY;
1278		}
1279	} else
1280		readl(port_mmio + PORT_CMD_ISSUE);	/* flush */
1281
1282	return 0;
1283}
1284
1285int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1286		      int pmp, unsigned long deadline,
1287		      int (*check_ready)(struct ata_link *link))
1288{
1289	struct ata_port *ap = link->ap;
1290	struct ahci_host_priv *hpriv = ap->host->private_data;
1291	struct ahci_port_priv *pp = ap->private_data;
1292	const char *reason = NULL;
1293	unsigned long now, msecs;
1294	struct ata_taskfile tf;
1295	bool fbs_disabled = false;
1296	int rc;
1297
1298	DPRINTK("ENTER\n");
1299
1300	/* prepare for SRST (AHCI-1.1 10.4.1) */
1301	rc = ahci_kick_engine(ap);
1302	if (rc && rc != -EOPNOTSUPP)
1303		ata_link_warn(link, "failed to reset engine (errno=%d)\n", rc);
1304
1305	/*
1306	 * According to AHCI-1.2 9.3.9: if FBS is enable, software shall
1307	 * clear PxFBS.EN to '0' prior to issuing software reset to devices
1308	 * that is attached to port multiplier.
1309	 */
1310	if (!ata_is_host_link(link) && pp->fbs_enabled) {
1311		ahci_disable_fbs(ap);
1312		fbs_disabled = true;
1313	}
1314
1315	ata_tf_init(link->device, &tf);
1316
1317	/* issue the first D2H Register FIS */
1318	msecs = 0;
1319	now = jiffies;
1320	if (time_after(deadline, now))
1321		msecs = jiffies_to_msecs(deadline - now);
1322
1323	tf.ctl |= ATA_SRST;
1324	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1325				 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
1326		rc = -EIO;
1327		reason = "1st FIS failed";
1328		goto fail;
1329	}
1330
1331	/* spec says at least 5us, but be generous and sleep for 1ms */
1332	ata_msleep(ap, 1);
1333
1334	/* issue the second D2H Register FIS */
1335	tf.ctl &= ~ATA_SRST;
1336	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
1337
1338	/* wait for link to become ready */
1339	rc = ata_wait_after_reset(link, deadline, check_ready);
1340	if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {
1341		/*
1342		 * Workaround for cases where link online status can't
1343		 * be trusted.  Treat device readiness timeout as link
1344		 * offline.
1345		 */
1346		ata_link_info(link, "device not ready, treating as offline\n");
1347		*class = ATA_DEV_NONE;
1348	} else if (rc) {
1349		/* link occupied, -ENODEV too is an error */
1350		reason = "device not ready";
1351		goto fail;
1352	} else
1353		*class = ahci_dev_classify(ap);
1354
1355	/* re-enable FBS if disabled before */
1356	if (fbs_disabled)
1357		ahci_enable_fbs(ap);
1358
1359	DPRINTK("EXIT, class=%u\n", *class);
1360	return 0;
1361
1362 fail:
1363	ata_link_err(link, "softreset failed (%s)\n", reason);
1364	return rc;
1365}
1366
1367int ahci_check_ready(struct ata_link *link)
1368{
1369	void __iomem *port_mmio = ahci_port_base(link->ap);
1370	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1371
1372	return ata_check_ready(status);
1373}
1374EXPORT_SYMBOL_GPL(ahci_check_ready);
1375
1376static int ahci_softreset(struct ata_link *link, unsigned int *class,
1377			  unsigned long deadline)
1378{
1379	int pmp = sata_srst_pmp(link);
1380
1381	DPRINTK("ENTER\n");
1382
1383	return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
1384}
1385EXPORT_SYMBOL_GPL(ahci_do_softreset);
1386
1387static int ahci_bad_pmp_check_ready(struct ata_link *link)
1388{
1389	void __iomem *port_mmio = ahci_port_base(link->ap);
1390	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1391	u32 irq_status = readl(port_mmio + PORT_IRQ_STAT);
1392
1393	/*
1394	 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1395	 * which can save timeout delay.
1396	 */
1397	if (irq_status & PORT_IRQ_BAD_PMP)
1398		return -EIO;
1399
1400	return ata_check_ready(status);
1401}
1402
1403static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
1404				    unsigned long deadline)
1405{
1406	struct ata_port *ap = link->ap;
1407	void __iomem *port_mmio = ahci_port_base(ap);
1408	int pmp = sata_srst_pmp(link);
1409	int rc;
1410	u32 irq_sts;
1411
1412	DPRINTK("ENTER\n");
1413
1414	rc = ahci_do_softreset(link, class, pmp, deadline,
1415			       ahci_bad_pmp_check_ready);
1416
1417	/*
1418	 * Soft reset fails with IPMS set when PMP is enabled but
1419	 * SATA HDD/ODD is connected to SATA port, do soft reset
1420	 * again to port 0.
1421	 */
1422	if (rc == -EIO) {
1423		irq_sts = readl(port_mmio + PORT_IRQ_STAT);
1424		if (irq_sts & PORT_IRQ_BAD_PMP) {
1425			ata_link_warn(link,
1426					"applying PMP SRST workaround "
1427					"and retrying\n");
1428			rc = ahci_do_softreset(link, class, 0, deadline,
1429					       ahci_check_ready);
1430		}
1431	}
1432
1433	return rc;
1434}
1435
1436static int ahci_hardreset(struct ata_link *link, unsigned int *class,
1437			  unsigned long deadline)
1438{
1439	const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
1440	struct ata_port *ap = link->ap;
1441	struct ahci_port_priv *pp = ap->private_data;
1442	struct ahci_host_priv *hpriv = ap->host->private_data;
1443	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1444	struct ata_taskfile tf;
1445	bool online;
1446	int rc;
1447
1448	DPRINTK("ENTER\n");
1449
1450	ahci_stop_engine(ap);
1451
1452	/* clear D2H reception area to properly wait for D2H FIS */
1453	ata_tf_init(link->device, &tf);
1454	tf.command = ATA_BUSY;
1455	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1456
1457	rc = sata_link_hardreset(link, timing, deadline, &online,
1458				 ahci_check_ready);
1459
1460	hpriv->start_engine(ap);
1461
1462	if (online)
1463		*class = ahci_dev_classify(ap);
1464
1465	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1466	return rc;
1467}
1468
1469static void ahci_postreset(struct ata_link *link, unsigned int *class)
1470{
1471	struct ata_port *ap = link->ap;
1472	void __iomem *port_mmio = ahci_port_base(ap);
1473	u32 new_tmp, tmp;
1474
1475	ata_std_postreset(link, class);
1476
1477	/* Make sure port's ATAPI bit is set appropriately */
1478	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1479	if (*class == ATA_DEV_ATAPI)
1480		new_tmp |= PORT_CMD_ATAPI;
1481	else
1482		new_tmp &= ~PORT_CMD_ATAPI;
1483	if (new_tmp != tmp) {
1484		writel(new_tmp, port_mmio + PORT_CMD);
1485		readl(port_mmio + PORT_CMD); /* flush */
1486	}
1487}
1488
1489static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
1490{
1491	struct scatterlist *sg;
1492	struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
1493	unsigned int si;
1494
1495	VPRINTK("ENTER\n");
1496
1497	/*
1498	 * Next, the S/G list.
1499	 */
1500	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1501		dma_addr_t addr = sg_dma_address(sg);
1502		u32 sg_len = sg_dma_len(sg);
1503
1504		ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
1505		ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
1506		ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
1507	}
1508
1509	return si;
1510}
1511
1512static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
1513{
1514	struct ata_port *ap = qc->ap;
1515	struct ahci_port_priv *pp = ap->private_data;
1516
1517	if (!sata_pmp_attached(ap) || pp->fbs_enabled)
1518		return ata_std_qc_defer(qc);
1519	else
1520		return sata_pmp_qc_defer_cmd_switch(qc);
1521}
1522
1523static void ahci_qc_prep(struct ata_queued_cmd *qc)
1524{
1525	struct ata_port *ap = qc->ap;
1526	struct ahci_port_priv *pp = ap->private_data;
1527	int is_atapi = ata_is_atapi(qc->tf.protocol);
1528	void *cmd_tbl;
1529	u32 opts;
1530	const u32 cmd_fis_len = 5; /* five dwords */
1531	unsigned int n_elem;
1532
1533	/*
1534	 * Fill in command table information.  First, the header,
1535	 * a SATA Register - Host to Device command FIS.
1536	 */
1537	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
1538
1539	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
1540	if (is_atapi) {
1541		memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
1542		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1543	}
1544
1545	n_elem = 0;
1546	if (qc->flags & ATA_QCFLAG_DMAMAP)
1547		n_elem = ahci_fill_sg(qc, cmd_tbl);
1548
1549	/*
1550	 * Fill in command slot information.
1551	 */
1552	opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
1553	if (qc->tf.flags & ATA_TFLAG_WRITE)
1554		opts |= AHCI_CMD_WRITE;
1555	if (is_atapi)
1556		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1557
1558	ahci_fill_cmd_slot(pp, qc->tag, opts);
1559}
1560
1561static void ahci_fbs_dec_intr(struct ata_port *ap)
1562{
1563	struct ahci_port_priv *pp = ap->private_data;
1564	void __iomem *port_mmio = ahci_port_base(ap);
1565	u32 fbs = readl(port_mmio + PORT_FBS);
1566	int retries = 3;
1567
1568	DPRINTK("ENTER\n");
1569	BUG_ON(!pp->fbs_enabled);
1570
1571	/* time to wait for DEC is not specified by AHCI spec,
1572	 * add a retry loop for safety.
1573	 */
1574	writel(fbs | PORT_FBS_DEC, port_mmio + PORT_FBS);
1575	fbs = readl(port_mmio + PORT_FBS);
1576	while ((fbs & PORT_FBS_DEC) && retries--) {
1577		udelay(1);
1578		fbs = readl(port_mmio + PORT_FBS);
1579	}
1580
1581	if (fbs & PORT_FBS_DEC)
1582		dev_err(ap->host->dev, "failed to clear device error\n");
1583}
1584
1585static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1586{
1587	struct ahci_host_priv *hpriv = ap->host->private_data;
1588	struct ahci_port_priv *pp = ap->private_data;
1589	struct ata_eh_info *host_ehi = &ap->link.eh_info;
1590	struct ata_link *link = NULL;
1591	struct ata_queued_cmd *active_qc;
1592	struct ata_eh_info *active_ehi;
1593	bool fbs_need_dec = false;
1594	u32 serror;
1595
1596	/* determine active link with error */
1597	if (pp->fbs_enabled) {
1598		void __iomem *port_mmio = ahci_port_base(ap);
1599		u32 fbs = readl(port_mmio + PORT_FBS);
1600		int pmp = fbs >> PORT_FBS_DWE_OFFSET;
1601
1602		if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links)) {
 
1603			link = &ap->pmp_link[pmp];
1604			fbs_need_dec = true;
1605		}
1606
1607	} else
1608		ata_for_each_link(link, ap, EDGE)
1609			if (ata_link_active(link))
1610				break;
1611
1612	if (!link)
1613		link = &ap->link;
1614
1615	active_qc = ata_qc_from_tag(ap, link->active_tag);
1616	active_ehi = &link->eh_info;
1617
1618	/* record irq stat */
1619	ata_ehi_clear_desc(host_ehi);
1620	ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
1621
1622	/* AHCI needs SError cleared; otherwise, it might lock up */
1623	ahci_scr_read(&ap->link, SCR_ERROR, &serror);
1624	ahci_scr_write(&ap->link, SCR_ERROR, serror);
1625	host_ehi->serror |= serror;
1626
1627	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1628	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1629		irq_stat &= ~PORT_IRQ_IF_ERR;
1630
1631	if (irq_stat & PORT_IRQ_TF_ERR) {
1632		/* If qc is active, charge it; otherwise, the active
1633		 * link.  There's no active qc on NCQ errors.  It will
1634		 * be determined by EH by reading log page 10h.
1635		 */
1636		if (active_qc)
1637			active_qc->err_mask |= AC_ERR_DEV;
1638		else
1639			active_ehi->err_mask |= AC_ERR_DEV;
1640
1641		if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
1642			host_ehi->serror &= ~SERR_INTERNAL;
1643	}
1644
1645	if (irq_stat & PORT_IRQ_UNK_FIS) {
1646		u32 *unk = pp->rx_fis + RX_FIS_UNK;
1647
1648		active_ehi->err_mask |= AC_ERR_HSM;
1649		active_ehi->action |= ATA_EH_RESET;
1650		ata_ehi_push_desc(active_ehi,
1651				  "unknown FIS %08x %08x %08x %08x" ,
1652				  unk[0], unk[1], unk[2], unk[3]);
1653	}
1654
1655	if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
1656		active_ehi->err_mask |= AC_ERR_HSM;
1657		active_ehi->action |= ATA_EH_RESET;
1658		ata_ehi_push_desc(active_ehi, "incorrect PMP");
1659	}
1660
1661	if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
1662		host_ehi->err_mask |= AC_ERR_HOST_BUS;
1663		host_ehi->action |= ATA_EH_RESET;
1664		ata_ehi_push_desc(host_ehi, "host bus error");
1665	}
1666
1667	if (irq_stat & PORT_IRQ_IF_ERR) {
1668		if (fbs_need_dec)
1669			active_ehi->err_mask |= AC_ERR_DEV;
1670		else {
1671			host_ehi->err_mask |= AC_ERR_ATA_BUS;
1672			host_ehi->action |= ATA_EH_RESET;
1673		}
1674
1675		ata_ehi_push_desc(host_ehi, "interface fatal error");
1676	}
1677
1678	if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
1679		ata_ehi_hotplugged(host_ehi);
1680		ata_ehi_push_desc(host_ehi, "%s",
1681			irq_stat & PORT_IRQ_CONNECT ?
1682			"connection status changed" : "PHY RDY changed");
1683	}
1684
1685	/* okay, let's hand over to EH */
1686
1687	if (irq_stat & PORT_IRQ_FREEZE)
1688		ata_port_freeze(ap);
1689	else if (fbs_need_dec) {
1690		ata_link_abort(link);
1691		ahci_fbs_dec_intr(ap);
1692	} else
1693		ata_port_abort(ap);
1694}
1695
1696static void ahci_handle_port_interrupt(struct ata_port *ap,
1697				       void __iomem *port_mmio, u32 status)
1698{
 
1699	struct ata_eh_info *ehi = &ap->link.eh_info;
1700	struct ahci_port_priv *pp = ap->private_data;
1701	struct ahci_host_priv *hpriv = ap->host->private_data;
1702	int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
1703	u32 qc_active = 0;
1704	int rc;
1705
 
 
 
1706	/* ignore BAD_PMP while resetting */
1707	if (unlikely(resetting))
1708		status &= ~PORT_IRQ_BAD_PMP;
1709
1710	/* if LPM is enabled, PHYRDY doesn't mean anything */
1711	if (ap->link.lpm_policy > ATA_LPM_MAX_POWER) {
1712		status &= ~PORT_IRQ_PHYRDY;
1713		ahci_scr_write(&ap->link, SCR_ERROR, SERR_PHYRDY_CHG);
1714	}
1715
1716	if (unlikely(status & PORT_IRQ_ERROR)) {
1717		ahci_error_intr(ap, status);
1718		return;
1719	}
1720
1721	if (status & PORT_IRQ_SDB_FIS) {
1722		/* If SNotification is available, leave notification
1723		 * handling to sata_async_notification().  If not,
1724		 * emulate it by snooping SDB FIS RX area.
1725		 *
1726		 * Snooping FIS RX area is probably cheaper than
1727		 * poking SNotification but some constrollers which
1728		 * implement SNotification, ICH9 for example, don't
1729		 * store AN SDB FIS into receive area.
1730		 */
1731		if (hpriv->cap & HOST_CAP_SNTF)
1732			sata_async_notification(ap);
1733		else {
1734			/* If the 'N' bit in word 0 of the FIS is set,
1735			 * we just received asynchronous notification.
1736			 * Tell libata about it.
1737			 *
1738			 * Lack of SNotification should not appear in
1739			 * ahci 1.2, so the workaround is unnecessary
1740			 * when FBS is enabled.
1741			 */
1742			if (pp->fbs_enabled)
1743				WARN_ON_ONCE(1);
1744			else {
1745				const __le32 *f = pp->rx_fis + RX_FIS_SDB;
1746				u32 f0 = le32_to_cpu(f[0]);
1747				if (f0 & (1 << 15))
1748					sata_async_notification(ap);
1749			}
1750		}
1751	}
1752
1753	/* pp->active_link is not reliable once FBS is enabled, both
1754	 * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1755	 * NCQ and non-NCQ commands may be in flight at the same time.
1756	 */
1757	if (pp->fbs_enabled) {
1758		if (ap->qc_active) {
1759			qc_active = readl(port_mmio + PORT_SCR_ACT);
1760			qc_active |= readl(port_mmio + PORT_CMD_ISSUE);
1761		}
1762	} else {
1763		/* pp->active_link is valid iff any command is in flight */
1764		if (ap->qc_active && pp->active_link->sactive)
1765			qc_active = readl(port_mmio + PORT_SCR_ACT);
1766		else
1767			qc_active = readl(port_mmio + PORT_CMD_ISSUE);
1768	}
1769
1770
1771	rc = ata_qc_complete_multiple(ap, qc_active);
1772
1773	/* while resetting, invalid completions are expected */
1774	if (unlikely(rc < 0 && !resetting)) {
1775		ehi->err_mask |= AC_ERR_HSM;
1776		ehi->action |= ATA_EH_RESET;
1777		ata_port_freeze(ap);
1778	}
1779}
1780
1781static void ahci_port_intr(struct ata_port *ap)
1782{
1783	void __iomem *port_mmio = ahci_port_base(ap);
1784	u32 status;
1785
1786	status = readl(port_mmio + PORT_IRQ_STAT);
1787	writel(status, port_mmio + PORT_IRQ_STAT);
1788
1789	ahci_handle_port_interrupt(ap, port_mmio, status);
1790}
1791
1792irqreturn_t ahci_thread_fn(int irq, void *dev_instance)
1793{
1794	struct ata_port *ap = dev_instance;
1795	struct ahci_port_priv *pp = ap->private_data;
1796	void __iomem *port_mmio = ahci_port_base(ap);
1797	unsigned long flags;
1798	u32 status;
1799
1800	spin_lock_irqsave(&ap->host->lock, flags);
1801	status = pp->intr_status;
1802	if (status)
1803		pp->intr_status = 0;
1804	spin_unlock_irqrestore(&ap->host->lock, flags);
1805
1806	spin_lock_bh(ap->lock);
1807	ahci_handle_port_interrupt(ap, port_mmio, status);
1808	spin_unlock_bh(ap->lock);
1809
1810	return IRQ_HANDLED;
1811}
1812EXPORT_SYMBOL_GPL(ahci_thread_fn);
1813
1814static void ahci_hw_port_interrupt(struct ata_port *ap)
1815{
1816	void __iomem *port_mmio = ahci_port_base(ap);
1817	struct ahci_port_priv *pp = ap->private_data;
1818	u32 status;
1819
1820	status = readl(port_mmio + PORT_IRQ_STAT);
1821	writel(status, port_mmio + PORT_IRQ_STAT);
1822
1823	pp->intr_status |= status;
1824}
1825
1826irqreturn_t ahci_hw_interrupt(int irq, void *dev_instance)
1827{
1828	struct ata_port *ap_this = dev_instance;
1829	struct ahci_port_priv *pp = ap_this->private_data;
1830	struct ata_host *host = ap_this->host;
1831	struct ahci_host_priv *hpriv = host->private_data;
1832	void __iomem *mmio = hpriv->mmio;
1833	unsigned int i;
1834	u32 irq_stat, irq_masked;
1835
1836	VPRINTK("ENTER\n");
1837
1838	spin_lock(&host->lock);
1839
1840	irq_stat = readl(mmio + HOST_IRQ_STAT);
1841
1842	if (!irq_stat) {
1843		u32 status = pp->intr_status;
1844
1845		spin_unlock(&host->lock);
1846
1847		VPRINTK("EXIT\n");
1848
1849		return status ? IRQ_WAKE_THREAD : IRQ_NONE;
1850	}
1851
1852	irq_masked = irq_stat & hpriv->port_map;
1853
1854	for (i = 0; i < host->n_ports; i++) {
1855		struct ata_port *ap;
1856
1857		if (!(irq_masked & (1 << i)))
1858			continue;
1859
1860		ap = host->ports[i];
1861		if (ap) {
1862			ahci_hw_port_interrupt(ap);
1863			VPRINTK("port %u\n", i);
1864		} else {
1865			VPRINTK("port %u (no irq)\n", i);
1866			if (ata_ratelimit())
1867				dev_warn(host->dev,
1868					 "interrupt on disabled port %u\n", i);
1869		}
1870	}
1871
1872	writel(irq_stat, mmio + HOST_IRQ_STAT);
1873
1874	spin_unlock(&host->lock);
1875
1876	VPRINTK("EXIT\n");
1877
1878	return IRQ_WAKE_THREAD;
1879}
1880EXPORT_SYMBOL_GPL(ahci_hw_interrupt);
1881
1882irqreturn_t ahci_interrupt(int irq, void *dev_instance)
1883{
1884	struct ata_host *host = dev_instance;
1885	struct ahci_host_priv *hpriv;
1886	unsigned int i, handled = 0;
1887	void __iomem *mmio;
1888	u32 irq_stat, irq_masked;
1889
1890	VPRINTK("ENTER\n");
1891
1892	hpriv = host->private_data;
1893	mmio = hpriv->mmio;
1894
1895	/* sigh.  0xffffffff is a valid return from h/w */
1896	irq_stat = readl(mmio + HOST_IRQ_STAT);
1897	if (!irq_stat)
1898		return IRQ_NONE;
1899
1900	irq_masked = irq_stat & hpriv->port_map;
1901
1902	spin_lock(&host->lock);
1903
1904	for (i = 0; i < host->n_ports; i++) {
1905		struct ata_port *ap;
1906
1907		if (!(irq_masked & (1 << i)))
1908			continue;
1909
1910		ap = host->ports[i];
1911		if (ap) {
1912			ahci_port_intr(ap);
1913			VPRINTK("port %u\n", i);
1914		} else {
1915			VPRINTK("port %u (no irq)\n", i);
1916			if (ata_ratelimit())
1917				dev_warn(host->dev,
1918					 "interrupt on disabled port %u\n", i);
1919		}
1920
1921		handled = 1;
1922	}
1923
1924	/* HOST_IRQ_STAT behaves as level triggered latch meaning that
1925	 * it should be cleared after all the port events are cleared;
1926	 * otherwise, it will raise a spurious interrupt after each
1927	 * valid one.  Please read section 10.6.2 of ahci 1.1 for more
1928	 * information.
1929	 *
1930	 * Also, use the unmasked value to clear interrupt as spurious
1931	 * pending event on a dummy port might cause screaming IRQ.
1932	 */
1933	writel(irq_stat, mmio + HOST_IRQ_STAT);
1934
1935	spin_unlock(&host->lock);
1936
1937	VPRINTK("EXIT\n");
1938
1939	return IRQ_RETVAL(handled);
1940}
1941EXPORT_SYMBOL_GPL(ahci_interrupt);
1942
1943static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
1944{
1945	struct ata_port *ap = qc->ap;
1946	void __iomem *port_mmio = ahci_port_base(ap);
1947	struct ahci_port_priv *pp = ap->private_data;
1948
1949	/* Keep track of the currently active link.  It will be used
1950	 * in completion path to determine whether NCQ phase is in
1951	 * progress.
1952	 */
1953	pp->active_link = qc->dev->link;
1954
1955	if (qc->tf.protocol == ATA_PROT_NCQ)
1956		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
1957
1958	if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
1959		u32 fbs = readl(port_mmio + PORT_FBS);
1960		fbs &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
1961		fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
1962		writel(fbs, port_mmio + PORT_FBS);
1963		pp->fbs_last_dev = qc->dev->link->pmp;
1964	}
1965
1966	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
1967
1968	ahci_sw_activity(qc->dev->link);
1969
1970	return 0;
1971}
1972
1973static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
1974{
1975	struct ahci_port_priv *pp = qc->ap->private_data;
1976	u8 *rx_fis = pp->rx_fis;
1977
1978	if (pp->fbs_enabled)
1979		rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
1980
1981	/*
1982	 * After a successful execution of an ATA PIO data-in command,
1983	 * the device doesn't send D2H Reg FIS to update the TF and
1984	 * the host should take TF and E_Status from the preceding PIO
1985	 * Setup FIS.
1986	 */
1987	if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&
1988	    !(qc->flags & ATA_QCFLAG_FAILED)) {
1989		ata_tf_from_fis(rx_fis + RX_FIS_PIO_SETUP, &qc->result_tf);
1990		qc->result_tf.command = (rx_fis + RX_FIS_PIO_SETUP)[15];
1991	} else
1992		ata_tf_from_fis(rx_fis + RX_FIS_D2H_REG, &qc->result_tf);
1993
1994	return true;
1995}
1996
1997static void ahci_freeze(struct ata_port *ap)
1998{
1999	void __iomem *port_mmio = ahci_port_base(ap);
2000
2001	/* turn IRQ off */
2002	writel(0, port_mmio + PORT_IRQ_MASK);
2003}
2004
2005static void ahci_thaw(struct ata_port *ap)
2006{
2007	struct ahci_host_priv *hpriv = ap->host->private_data;
2008	void __iomem *mmio = hpriv->mmio;
2009	void __iomem *port_mmio = ahci_port_base(ap);
2010	u32 tmp;
2011	struct ahci_port_priv *pp = ap->private_data;
2012
2013	/* clear IRQ */
2014	tmp = readl(port_mmio + PORT_IRQ_STAT);
2015	writel(tmp, port_mmio + PORT_IRQ_STAT);
2016	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
2017
2018	/* turn IRQ back on */
2019	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2020}
2021
2022void ahci_error_handler(struct ata_port *ap)
2023{
2024	struct ahci_host_priv *hpriv = ap->host->private_data;
2025
2026	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
2027		/* restart engine */
2028		ahci_stop_engine(ap);
2029		hpriv->start_engine(ap);
2030	}
2031
2032	sata_pmp_error_handler(ap);
2033
2034	if (!ata_dev_enabled(ap->link.device))
2035		ahci_stop_engine(ap);
2036}
2037EXPORT_SYMBOL_GPL(ahci_error_handler);
2038
2039static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
2040{
2041	struct ata_port *ap = qc->ap;
2042
2043	/* make DMA engine forget about the failed command */
2044	if (qc->flags & ATA_QCFLAG_FAILED)
2045		ahci_kick_engine(ap);
2046}
2047
2048static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep)
2049{
2050	struct ahci_host_priv *hpriv = ap->host->private_data;
2051	void __iomem *port_mmio = ahci_port_base(ap);
2052	struct ata_device *dev = ap->link.device;
2053	u32 devslp, dm, dito, mdat, deto;
2054	int rc;
2055	unsigned int err_mask;
2056
2057	devslp = readl(port_mmio + PORT_DEVSLP);
2058	if (!(devslp & PORT_DEVSLP_DSP)) {
2059		dev_err(ap->host->dev, "port does not support device sleep\n");
2060		return;
2061	}
2062
2063	/* disable device sleep */
2064	if (!sleep) {
2065		if (devslp & PORT_DEVSLP_ADSE) {
2066			writel(devslp & ~PORT_DEVSLP_ADSE,
2067			       port_mmio + PORT_DEVSLP);
2068			err_mask = ata_dev_set_feature(dev,
2069						       SETFEATURES_SATA_DISABLE,
2070						       SATA_DEVSLP);
2071			if (err_mask && err_mask != AC_ERR_DEV)
2072				ata_dev_warn(dev, "failed to disable DEVSLP\n");
2073		}
2074		return;
2075	}
2076
2077	/* device sleep was already enabled */
2078	if (devslp & PORT_DEVSLP_ADSE)
2079		return;
2080
2081	/* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
2082	rc = ahci_stop_engine(ap);
2083	if (rc)
2084		return;
2085
2086	dm = (devslp & PORT_DEVSLP_DM_MASK) >> PORT_DEVSLP_DM_OFFSET;
2087	dito = devslp_idle_timeout / (dm + 1);
2088	if (dito > 0x3ff)
2089		dito = 0x3ff;
2090
2091	/* Use the nominal value 10 ms if the read MDAT is zero,
2092	 * the nominal value of DETO is 20 ms.
2093	 */
2094	if (dev->devslp_timing[ATA_LOG_DEVSLP_VALID] &
2095	    ATA_LOG_DEVSLP_VALID_MASK) {
2096		mdat = dev->devslp_timing[ATA_LOG_DEVSLP_MDAT] &
2097		       ATA_LOG_DEVSLP_MDAT_MASK;
2098		if (!mdat)
2099			mdat = 10;
2100		deto = dev->devslp_timing[ATA_LOG_DEVSLP_DETO];
2101		if (!deto)
2102			deto = 20;
2103	} else {
2104		mdat = 10;
2105		deto = 20;
2106	}
2107
2108	devslp |= ((dito << PORT_DEVSLP_DITO_OFFSET) |
2109		   (mdat << PORT_DEVSLP_MDAT_OFFSET) |
2110		   (deto << PORT_DEVSLP_DETO_OFFSET) |
2111		   PORT_DEVSLP_ADSE);
2112	writel(devslp, port_mmio + PORT_DEVSLP);
2113
2114	hpriv->start_engine(ap);
2115
2116	/* enable device sleep feature for the drive */
2117	err_mask = ata_dev_set_feature(dev,
2118				       SETFEATURES_SATA_ENABLE,
2119				       SATA_DEVSLP);
2120	if (err_mask && err_mask != AC_ERR_DEV)
2121		ata_dev_warn(dev, "failed to enable DEVSLP\n");
2122}
2123
2124static void ahci_enable_fbs(struct ata_port *ap)
2125{
2126	struct ahci_host_priv *hpriv = ap->host->private_data;
2127	struct ahci_port_priv *pp = ap->private_data;
2128	void __iomem *port_mmio = ahci_port_base(ap);
2129	u32 fbs;
2130	int rc;
2131
2132	if (!pp->fbs_supported)
2133		return;
2134
2135	fbs = readl(port_mmio + PORT_FBS);
2136	if (fbs & PORT_FBS_EN) {
2137		pp->fbs_enabled = true;
2138		pp->fbs_last_dev = -1; /* initialization */
2139		return;
2140	}
2141
2142	rc = ahci_stop_engine(ap);
2143	if (rc)
2144		return;
2145
2146	writel(fbs | PORT_FBS_EN, port_mmio + PORT_FBS);
2147	fbs = readl(port_mmio + PORT_FBS);
2148	if (fbs & PORT_FBS_EN) {
2149		dev_info(ap->host->dev, "FBS is enabled\n");
2150		pp->fbs_enabled = true;
2151		pp->fbs_last_dev = -1; /* initialization */
2152	} else
2153		dev_err(ap->host->dev, "Failed to enable FBS\n");
2154
2155	hpriv->start_engine(ap);
2156}
2157
2158static void ahci_disable_fbs(struct ata_port *ap)
2159{
2160	struct ahci_host_priv *hpriv = ap->host->private_data;
2161	struct ahci_port_priv *pp = ap->private_data;
2162	void __iomem *port_mmio = ahci_port_base(ap);
2163	u32 fbs;
2164	int rc;
2165
2166	if (!pp->fbs_supported)
2167		return;
2168
2169	fbs = readl(port_mmio + PORT_FBS);
2170	if ((fbs & PORT_FBS_EN) == 0) {
2171		pp->fbs_enabled = false;
2172		return;
2173	}
2174
2175	rc = ahci_stop_engine(ap);
2176	if (rc)
2177		return;
2178
2179	writel(fbs & ~PORT_FBS_EN, port_mmio + PORT_FBS);
2180	fbs = readl(port_mmio + PORT_FBS);
2181	if (fbs & PORT_FBS_EN)
2182		dev_err(ap->host->dev, "Failed to disable FBS\n");
2183	else {
2184		dev_info(ap->host->dev, "FBS is disabled\n");
2185		pp->fbs_enabled = false;
2186	}
2187
2188	hpriv->start_engine(ap);
2189}
2190
2191static void ahci_pmp_attach(struct ata_port *ap)
2192{
2193	void __iomem *port_mmio = ahci_port_base(ap);
2194	struct ahci_port_priv *pp = ap->private_data;
2195	u32 cmd;
2196
2197	cmd = readl(port_mmio + PORT_CMD);
2198	cmd |= PORT_CMD_PMP;
2199	writel(cmd, port_mmio + PORT_CMD);
2200
2201	ahci_enable_fbs(ap);
2202
2203	pp->intr_mask |= PORT_IRQ_BAD_PMP;
2204
2205	/*
2206	 * We must not change the port interrupt mask register if the
2207	 * port is marked frozen, the value in pp->intr_mask will be
2208	 * restored later when the port is thawed.
2209	 *
2210	 * Note that during initialization, the port is marked as
2211	 * frozen since the irq handler is not yet registered.
2212	 */
2213	if (!(ap->pflags & ATA_PFLAG_FROZEN))
2214		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2215}
2216
2217static void ahci_pmp_detach(struct ata_port *ap)
2218{
2219	void __iomem *port_mmio = ahci_port_base(ap);
2220	struct ahci_port_priv *pp = ap->private_data;
2221	u32 cmd;
2222
2223	ahci_disable_fbs(ap);
2224
2225	cmd = readl(port_mmio + PORT_CMD);
2226	cmd &= ~PORT_CMD_PMP;
2227	writel(cmd, port_mmio + PORT_CMD);
2228
2229	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
2230
2231	/* see comment above in ahci_pmp_attach() */
2232	if (!(ap->pflags & ATA_PFLAG_FROZEN))
2233		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2234}
2235
2236int ahci_port_resume(struct ata_port *ap)
2237{
2238	ahci_power_up(ap);
2239	ahci_start_port(ap);
2240
2241	if (sata_pmp_attached(ap))
2242		ahci_pmp_attach(ap);
2243	else
2244		ahci_pmp_detach(ap);
2245
2246	return 0;
2247}
2248EXPORT_SYMBOL_GPL(ahci_port_resume);
2249
2250#ifdef CONFIG_PM
2251static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
2252{
2253	const char *emsg = NULL;
2254	int rc;
2255
2256	rc = ahci_deinit_port(ap, &emsg);
2257	if (rc == 0)
2258		ahci_power_down(ap);
2259	else {
2260		ata_port_err(ap, "%s (%d)\n", emsg, rc);
2261		ata_port_freeze(ap);
2262	}
2263
2264	return rc;
2265}
2266#endif
2267
2268static int ahci_port_start(struct ata_port *ap)
2269{
2270	struct ahci_host_priv *hpriv = ap->host->private_data;
2271	struct device *dev = ap->host->dev;
2272	struct ahci_port_priv *pp;
2273	void *mem;
2274	dma_addr_t mem_dma;
2275	size_t dma_sz, rx_fis_sz;
2276
2277	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
2278	if (!pp)
2279		return -ENOMEM;
2280
2281	if (ap->host->n_ports > 1) {
2282		pp->irq_desc = devm_kzalloc(dev, 8, GFP_KERNEL);
2283		if (!pp->irq_desc) {
2284			devm_kfree(dev, pp);
2285			return -ENOMEM;
2286		}
2287		snprintf(pp->irq_desc, 8,
2288			 "%s%d", dev_driver_string(dev), ap->port_no);
2289	}
2290
2291	/* check FBS capability */
2292	if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
2293		void __iomem *port_mmio = ahci_port_base(ap);
2294		u32 cmd = readl(port_mmio + PORT_CMD);
2295		if (cmd & PORT_CMD_FBSCP)
2296			pp->fbs_supported = true;
2297		else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {
2298			dev_info(dev, "port %d can do FBS, forcing FBSCP\n",
2299				 ap->port_no);
2300			pp->fbs_supported = true;
2301		} else
2302			dev_warn(dev, "port %d is not capable of FBS\n",
2303				 ap->port_no);
2304	}
2305
2306	if (pp->fbs_supported) {
2307		dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
2308		rx_fis_sz = AHCI_RX_FIS_SZ * 16;
2309	} else {
2310		dma_sz = AHCI_PORT_PRIV_DMA_SZ;
2311		rx_fis_sz = AHCI_RX_FIS_SZ;
2312	}
2313
2314	mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
2315	if (!mem)
2316		return -ENOMEM;
2317	memset(mem, 0, dma_sz);
2318
2319	/*
2320	 * First item in chunk of DMA memory: 32-slot command table,
2321	 * 32 bytes each in size
2322	 */
2323	pp->cmd_slot = mem;
2324	pp->cmd_slot_dma = mem_dma;
2325
2326	mem += AHCI_CMD_SLOT_SZ;
2327	mem_dma += AHCI_CMD_SLOT_SZ;
2328
2329	/*
2330	 * Second item: Received-FIS area
2331	 */
2332	pp->rx_fis = mem;
2333	pp->rx_fis_dma = mem_dma;
2334
2335	mem += rx_fis_sz;
2336	mem_dma += rx_fis_sz;
2337
2338	/*
2339	 * Third item: data area for storing a single command
2340	 * and its scatter-gather table
2341	 */
2342	pp->cmd_tbl = mem;
2343	pp->cmd_tbl_dma = mem_dma;
2344
2345	/*
2346	 * Save off initial list of interrupts to be enabled.
2347	 * This could be changed later
2348	 */
2349	pp->intr_mask = DEF_PORT_IRQ;
2350
2351	/*
2352	 * Switch to per-port locking in case each port has its own MSI vector.
2353	 */
2354	if ((hpriv->flags & AHCI_HFLAG_MULTI_MSI)) {
2355		spin_lock_init(&pp->lock);
2356		ap->lock = &pp->lock;
2357	}
2358
2359	ap->private_data = pp;
2360
2361	/* engage engines, captain */
2362	return ahci_port_resume(ap);
2363}
2364
2365static void ahci_port_stop(struct ata_port *ap)
2366{
2367	const char *emsg = NULL;
2368	int rc;
2369
2370	/* de-initialize port */
2371	rc = ahci_deinit_port(ap, &emsg);
2372	if (rc)
2373		ata_port_warn(ap, "%s (%d)\n", emsg, rc);
2374}
2375
2376void ahci_print_info(struct ata_host *host, const char *scc_s)
2377{
2378	struct ahci_host_priv *hpriv = host->private_data;
2379	void __iomem *mmio = hpriv->mmio;
2380	u32 vers, cap, cap2, impl, speed;
2381	const char *speed_s;
2382
2383	vers = readl(mmio + HOST_VERSION);
2384	cap = hpriv->cap;
2385	cap2 = hpriv->cap2;
2386	impl = hpriv->port_map;
2387
2388	speed = (cap >> 20) & 0xf;
2389	if (speed == 1)
2390		speed_s = "1.5";
2391	else if (speed == 2)
2392		speed_s = "3";
2393	else if (speed == 3)
2394		speed_s = "6";
2395	else
2396		speed_s = "?";
2397
2398	dev_info(host->dev,
2399		"AHCI %02x%02x.%02x%02x "
2400		"%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2401		,
2402
2403		(vers >> 24) & 0xff,
2404		(vers >> 16) & 0xff,
2405		(vers >> 8) & 0xff,
2406		vers & 0xff,
2407
2408		((cap >> 8) & 0x1f) + 1,
2409		(cap & 0x1f) + 1,
2410		speed_s,
2411		impl,
2412		scc_s);
2413
2414	dev_info(host->dev,
2415		"flags: "
2416		"%s%s%s%s%s%s%s"
2417		"%s%s%s%s%s%s%s"
2418		"%s%s%s%s%s%s%s"
2419		"%s%s\n"
2420		,
2421
2422		cap & HOST_CAP_64 ? "64bit " : "",
2423		cap & HOST_CAP_NCQ ? "ncq " : "",
2424		cap & HOST_CAP_SNTF ? "sntf " : "",
2425		cap & HOST_CAP_MPS ? "ilck " : "",
2426		cap & HOST_CAP_SSS ? "stag " : "",
2427		cap & HOST_CAP_ALPM ? "pm " : "",
2428		cap & HOST_CAP_LED ? "led " : "",
2429		cap & HOST_CAP_CLO ? "clo " : "",
2430		cap & HOST_CAP_ONLY ? "only " : "",
2431		cap & HOST_CAP_PMP ? "pmp " : "",
2432		cap & HOST_CAP_FBS ? "fbs " : "",
2433		cap & HOST_CAP_PIO_MULTI ? "pio " : "",
2434		cap & HOST_CAP_SSC ? "slum " : "",
2435		cap & HOST_CAP_PART ? "part " : "",
2436		cap & HOST_CAP_CCC ? "ccc " : "",
2437		cap & HOST_CAP_EMS ? "ems " : "",
2438		cap & HOST_CAP_SXS ? "sxs " : "",
2439		cap2 & HOST_CAP2_DESO ? "deso " : "",
2440		cap2 & HOST_CAP2_SADM ? "sadm " : "",
2441		cap2 & HOST_CAP2_SDS ? "sds " : "",
2442		cap2 & HOST_CAP2_APST ? "apst " : "",
2443		cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
2444		cap2 & HOST_CAP2_BOH ? "boh " : ""
2445		);
2446}
2447EXPORT_SYMBOL_GPL(ahci_print_info);
2448
2449void ahci_set_em_messages(struct ahci_host_priv *hpriv,
2450			  struct ata_port_info *pi)
2451{
2452	u8 messages;
2453	void __iomem *mmio = hpriv->mmio;
2454	u32 em_loc = readl(mmio + HOST_EM_LOC);
2455	u32 em_ctl = readl(mmio + HOST_EM_CTL);
2456
2457	if (!ahci_em_messages || !(hpriv->cap & HOST_CAP_EMS))
2458		return;
2459
2460	messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
2461
2462	if (messages) {
2463		/* store em_loc */
2464		hpriv->em_loc = ((em_loc >> 16) * 4);
2465		hpriv->em_buf_sz = ((em_loc & 0xff) * 4);
2466		hpriv->em_msg_type = messages;
2467		pi->flags |= ATA_FLAG_EM;
2468		if (!(em_ctl & EM_CTL_ALHD))
2469			pi->flags |= ATA_FLAG_SW_ACTIVITY;
2470	}
2471}
2472EXPORT_SYMBOL_GPL(ahci_set_em_messages);
2473
2474MODULE_AUTHOR("Jeff Garzik");
2475MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2476MODULE_LICENSE("GPL");