1/*
   2 *  libahci.c - Common AHCI SATA low-level routines
   3 *
   4 *  Maintained by:  Tejun Heo <tj@kernel.org>
   5 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
   6 *		    on emails.
   7 *
   8 *  Copyright 2004-2005 Red Hat, Inc.
   9 *
  10 *
  11 *  This program is free software; you can redistribute it and/or modify
  12 *  it under the terms of the GNU General Public License as published by
  13 *  the Free Software Foundation; either version 2, or (at your option)
  14 *  any later version.
  15 *
  16 *  This program is distributed in the hope that it will be useful,
  17 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 *  GNU General Public License for more details.
  20 *
  21 *  You should have received a copy of the GNU General Public License
  22 *  along with this program; see the file COPYING.  If not, write to
  23 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  24 *
  25 *
  26 * libata documentation is available via 'make {ps|pdf}docs',
  27 * as Documentation/DocBook/libata.*
  28 *
  29 * AHCI hardware documentation:
  30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
  31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
  32 *
  33 */
  34
  35#include <linux/kernel.h>
  36#include <linux/gfp.h>
  37#include <linux/module.h>
 
  38#include <linux/blkdev.h>
  39#include <linux/delay.h>
  40#include <linux/interrupt.h>
  41#include <linux/dma-mapping.h>
  42#include <linux/device.h>
  43#include <scsi/scsi_host.h>
  44#include <scsi/scsi_cmnd.h>
  45#include <linux/libata.h>
  46#include <linux/pci.h>
  47#include "ahci.h"
  48#include "libata.h"
  49
  50static int ahci_skip_host_reset;
  51int ahci_ignore_sss;
  52EXPORT_SYMBOL_GPL(ahci_ignore_sss);
  53
  54module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
  55MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
  56
  57module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
  58MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
  59
  60static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
  61			unsigned hints);
  62static ssize_t ahci_led_show(struct ata_port *ap, char *buf);
  63static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
  64			      size_t size);
  65static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
  66					ssize_t size);
  67
  68
  69
  70static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
  71static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
  72static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
  73static int ahci_port_start(struct ata_port *ap);
  74static void ahci_port_stop(struct ata_port *ap);
  75static void ahci_qc_prep(struct ata_queued_cmd *qc);
  76static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
  77static void ahci_freeze(struct ata_port *ap);
  78static void ahci_thaw(struct ata_port *ap);
  79static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep);
  80static void ahci_enable_fbs(struct ata_port *ap);
  81static void ahci_disable_fbs(struct ata_port *ap);
  82static void ahci_pmp_attach(struct ata_port *ap);
  83static void ahci_pmp_detach(struct ata_port *ap);
  84static int ahci_softreset(struct ata_link *link, unsigned int *class,
  85			  unsigned long deadline);
  86static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
  87			  unsigned long deadline);
  88static int ahci_hardreset(struct ata_link *link, unsigned int *class,
  89			  unsigned long deadline);
  90static void ahci_postreset(struct ata_link *link, unsigned int *class);
  91static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
  92static void ahci_dev_config(struct ata_device *dev);
  93#ifdef CONFIG_PM
  94static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
  95#endif
  96static ssize_t ahci_activity_show(struct ata_device *dev, char *buf);
  97static ssize_t ahci_activity_store(struct ata_device *dev,
  98				   enum sw_activity val);
  99static void ahci_init_sw_activity(struct ata_link *link);
 100
 101static ssize_t ahci_show_host_caps(struct device *dev,
 102				   struct device_attribute *attr, char *buf);
 103static ssize_t ahci_show_host_cap2(struct device *dev,
 104				   struct device_attribute *attr, char *buf);
 105static ssize_t ahci_show_host_version(struct device *dev,
 106				      struct device_attribute *attr, char *buf);
 107static ssize_t ahci_show_port_cmd(struct device *dev,
 108				  struct device_attribute *attr, char *buf);
 109static ssize_t ahci_read_em_buffer(struct device *dev,
 110				   struct device_attribute *attr, char *buf);
 111static ssize_t ahci_store_em_buffer(struct device *dev,
 112				    struct device_attribute *attr,
 113				    const char *buf, size_t size);
 114static ssize_t ahci_show_em_supported(struct device *dev,
 115				      struct device_attribute *attr, char *buf);
 116static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance);
 117
 118static DEVICE_ATTR(ahci_host_caps, S_IRUGO, ahci_show_host_caps, NULL);
 119static DEVICE_ATTR(ahci_host_cap2, S_IRUGO, ahci_show_host_cap2, NULL);
 120static DEVICE_ATTR(ahci_host_version, S_IRUGO, ahci_show_host_version, NULL);
 121static DEVICE_ATTR(ahci_port_cmd, S_IRUGO, ahci_show_port_cmd, NULL);
 122static DEVICE_ATTR(em_buffer, S_IWUSR | S_IRUGO,
 123		   ahci_read_em_buffer, ahci_store_em_buffer);
 124static DEVICE_ATTR(em_message_supported, S_IRUGO, ahci_show_em_supported, NULL);
 125
 126struct device_attribute *ahci_shost_attrs[] = {
 127	&dev_attr_link_power_management_policy,
 128	&dev_attr_em_message_type,
 129	&dev_attr_em_message,
 130	&dev_attr_ahci_host_caps,
 131	&dev_attr_ahci_host_cap2,
 132	&dev_attr_ahci_host_version,
 133	&dev_attr_ahci_port_cmd,
 134	&dev_attr_em_buffer,
 135	&dev_attr_em_message_supported,
 136	NULL
 137};
 138EXPORT_SYMBOL_GPL(ahci_shost_attrs);
 139
 140struct device_attribute *ahci_sdev_attrs[] = {
 141	&dev_attr_sw_activity,
 142	&dev_attr_unload_heads,
 
 143	NULL
 144};
 145EXPORT_SYMBOL_GPL(ahci_sdev_attrs);
 146
 147struct ata_port_operations ahci_ops = {
 148	.inherits		= &sata_pmp_port_ops,
 149
 150	.qc_defer		= ahci_pmp_qc_defer,
 151	.qc_prep		= ahci_qc_prep,
 152	.qc_issue		= ahci_qc_issue,
 153	.qc_fill_rtf		= ahci_qc_fill_rtf,
 154
 155	.freeze			= ahci_freeze,
 156	.thaw			= ahci_thaw,
 157	.softreset		= ahci_softreset,
 158	.hardreset		= ahci_hardreset,
 159	.postreset		= ahci_postreset,
 160	.pmp_softreset		= ahci_softreset,
 161	.error_handler		= ahci_error_handler,
 162	.post_internal_cmd	= ahci_post_internal_cmd,
 163	.dev_config		= ahci_dev_config,
 164
 165	.scr_read		= ahci_scr_read,
 166	.scr_write		= ahci_scr_write,
 167	.pmp_attach		= ahci_pmp_attach,
 168	.pmp_detach		= ahci_pmp_detach,
 169
 170	.set_lpm		= ahci_set_lpm,
 171	.em_show		= ahci_led_show,
 172	.em_store		= ahci_led_store,
 173	.sw_activity_show	= ahci_activity_show,
 174	.sw_activity_store	= ahci_activity_store,
 175	.transmit_led_message	= ahci_transmit_led_message,
 176#ifdef CONFIG_PM
 177	.port_suspend		= ahci_port_suspend,
 178	.port_resume		= ahci_port_resume,
 179#endif
 180	.port_start		= ahci_port_start,
 181	.port_stop		= ahci_port_stop,
 182};
 183EXPORT_SYMBOL_GPL(ahci_ops);
 184
 185struct ata_port_operations ahci_pmp_retry_srst_ops = {
 186	.inherits		= &ahci_ops,
 187	.softreset		= ahci_pmp_retry_softreset,
 188};
 189EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
 190
 191static bool ahci_em_messages __read_mostly = true;
 192EXPORT_SYMBOL_GPL(ahci_em_messages);
 193module_param(ahci_em_messages, bool, 0444);
 194/* add other LED protocol types when they become supported */
 195MODULE_PARM_DESC(ahci_em_messages,
 196	"AHCI Enclosure Management Message control (0 = off, 1 = on)");
 197
 198/* device sleep idle timeout in ms */
 199static int devslp_idle_timeout __read_mostly = 1000;
 200module_param(devslp_idle_timeout, int, 0644);
 201MODULE_PARM_DESC(devslp_idle_timeout, "device sleep idle timeout");
 202
 203static void ahci_enable_ahci(void __iomem *mmio)
 204{
 205	int i;
 206	u32 tmp;
 207
 208	/* turn on AHCI_EN */
 209	tmp = readl(mmio + HOST_CTL);
 210	if (tmp & HOST_AHCI_EN)
 211		return;
 212
 213	/* Some controllers need AHCI_EN to be written multiple times.
 214	 * Try a few times before giving up.
 215	 */
 216	for (i = 0; i < 5; i++) {
 217		tmp |= HOST_AHCI_EN;
 218		writel(tmp, mmio + HOST_CTL);
 219		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
 220		if (tmp & HOST_AHCI_EN)
 221			return;
 222		msleep(10);
 223	}
 224
 225	WARN_ON(1);
 226}
 227
 228/**
 229 *	ahci_rpm_get_port - Make sure the port is powered on
 230 *	@ap: Port to power on
 231 *
 232 *	Whenever there is need to access the AHCI host registers outside of
 233 *	normal execution paths, call this function to make sure the host is
 234 *	actually powered on.
 235 */
 236static int ahci_rpm_get_port(struct ata_port *ap)
 237{
 238	return pm_runtime_get_sync(ap->dev);
 239}
 240
 241/**
 242 *	ahci_rpm_put_port - Undoes ahci_rpm_get_port()
 243 *	@ap: Port to power down
 244 *
 245 *	Undoes ahci_rpm_get_port() and possibly powers down the AHCI host
 246 *	if it has no more active users.
 247 */
 248static void ahci_rpm_put_port(struct ata_port *ap)
 249{
 250	pm_runtime_put(ap->dev);
 251}
 252
 253static ssize_t ahci_show_host_caps(struct device *dev,
 254				   struct device_attribute *attr, char *buf)
 255{
 256	struct Scsi_Host *shost = class_to_shost(dev);
 257	struct ata_port *ap = ata_shost_to_port(shost);
 258	struct ahci_host_priv *hpriv = ap->host->private_data;
 259
 260	return sprintf(buf, "%x\n", hpriv->cap);
 261}
 262
 263static ssize_t ahci_show_host_cap2(struct device *dev,
 264				   struct device_attribute *attr, char *buf)
 265{
 266	struct Scsi_Host *shost = class_to_shost(dev);
 267	struct ata_port *ap = ata_shost_to_port(shost);
 268	struct ahci_host_priv *hpriv = ap->host->private_data;
 269
 270	return sprintf(buf, "%x\n", hpriv->cap2);
 271}
 272
 273static ssize_t ahci_show_host_version(struct device *dev,
 274				   struct device_attribute *attr, char *buf)
 275{
 276	struct Scsi_Host *shost = class_to_shost(dev);
 277	struct ata_port *ap = ata_shost_to_port(shost);
 278	struct ahci_host_priv *hpriv = ap->host->private_data;
 279
 280	return sprintf(buf, "%x\n", hpriv->version);
 281}
 282
 283static ssize_t ahci_show_port_cmd(struct device *dev,
 284				  struct device_attribute *attr, char *buf)
 285{
 286	struct Scsi_Host *shost = class_to_shost(dev);
 287	struct ata_port *ap = ata_shost_to_port(shost);
 288	void __iomem *port_mmio = ahci_port_base(ap);
 289	ssize_t ret;
 290
 291	ahci_rpm_get_port(ap);
 292	ret = sprintf(buf, "%x\n", readl(port_mmio + PORT_CMD));
 293	ahci_rpm_put_port(ap);
 294
 295	return ret;
 296}
 297
 298static ssize_t ahci_read_em_buffer(struct device *dev,
 299				   struct device_attribute *attr, char *buf)
 300{
 301	struct Scsi_Host *shost = class_to_shost(dev);
 302	struct ata_port *ap = ata_shost_to_port(shost);
 303	struct ahci_host_priv *hpriv = ap->host->private_data;
 304	void __iomem *mmio = hpriv->mmio;
 305	void __iomem *em_mmio = mmio + hpriv->em_loc;
 306	u32 em_ctl, msg;
 307	unsigned long flags;
 308	size_t count;
 309	int i;
 310
 311	ahci_rpm_get_port(ap);
 312	spin_lock_irqsave(ap->lock, flags);
 313
 314	em_ctl = readl(mmio + HOST_EM_CTL);
 315	if (!(ap->flags & ATA_FLAG_EM) || em_ctl & EM_CTL_XMT ||
 316	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {
 317		spin_unlock_irqrestore(ap->lock, flags);
 318		ahci_rpm_put_port(ap);
 319		return -EINVAL;
 320	}
 321
 322	if (!(em_ctl & EM_CTL_MR)) {
 323		spin_unlock_irqrestore(ap->lock, flags);
 324		ahci_rpm_put_port(ap);
 325		return -EAGAIN;
 326	}
 327
 328	if (!(em_ctl & EM_CTL_SMB))
 329		em_mmio += hpriv->em_buf_sz;
 330
 331	count = hpriv->em_buf_sz;
 332
 333	/* the count should not be larger than PAGE_SIZE */
 334	if (count > PAGE_SIZE) {
 335		if (printk_ratelimit())
 336			ata_port_warn(ap,
 337				      "EM read buffer size too large: "
 338				      "buffer size %u, page size %lu\n",
 339				      hpriv->em_buf_sz, PAGE_SIZE);
 340		count = PAGE_SIZE;
 341	}
 342
 343	for (i = 0; i < count; i += 4) {
 344		msg = readl(em_mmio + i);
 345		buf[i] = msg & 0xff;
 346		buf[i + 1] = (msg >> 8) & 0xff;
 347		buf[i + 2] = (msg >> 16) & 0xff;
 348		buf[i + 3] = (msg >> 24) & 0xff;
 349	}
 350
 351	spin_unlock_irqrestore(ap->lock, flags);
 352	ahci_rpm_put_port(ap);
 353
 354	return i;
 355}
 356
 357static ssize_t ahci_store_em_buffer(struct device *dev,
 358				    struct device_attribute *attr,
 359				    const char *buf, size_t size)
 360{
 361	struct Scsi_Host *shost = class_to_shost(dev);
 362	struct ata_port *ap = ata_shost_to_port(shost);
 363	struct ahci_host_priv *hpriv = ap->host->private_data;
 364	void __iomem *mmio = hpriv->mmio;
 365	void __iomem *em_mmio = mmio + hpriv->em_loc;
 366	const unsigned char *msg_buf = buf;
 367	u32 em_ctl, msg;
 368	unsigned long flags;
 369	int i;
 370
 371	/* check size validity */
 372	if (!(ap->flags & ATA_FLAG_EM) ||
 373	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) ||
 374	    size % 4 || size > hpriv->em_buf_sz)
 375		return -EINVAL;
 376
 377	ahci_rpm_get_port(ap);
 378	spin_lock_irqsave(ap->lock, flags);
 379
 380	em_ctl = readl(mmio + HOST_EM_CTL);
 381	if (em_ctl & EM_CTL_TM) {
 382		spin_unlock_irqrestore(ap->lock, flags);
 383		ahci_rpm_put_port(ap);
 384		return -EBUSY;
 385	}
 386
 387	for (i = 0; i < size; i += 4) {
 388		msg = msg_buf[i] | msg_buf[i + 1] << 8 |
 389		      msg_buf[i + 2] << 16 | msg_buf[i + 3] << 24;
 390		writel(msg, em_mmio + i);
 391	}
 392
 393	writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
 394
 395	spin_unlock_irqrestore(ap->lock, flags);
 396	ahci_rpm_put_port(ap);
 397
 398	return size;
 399}
 400
 401static ssize_t ahci_show_em_supported(struct device *dev,
 402				      struct device_attribute *attr, char *buf)
 403{
 404	struct Scsi_Host *shost = class_to_shost(dev);
 405	struct ata_port *ap = ata_shost_to_port(shost);
 406	struct ahci_host_priv *hpriv = ap->host->private_data;
 407	void __iomem *mmio = hpriv->mmio;
 408	u32 em_ctl;
 409
 410	ahci_rpm_get_port(ap);
 411	em_ctl = readl(mmio + HOST_EM_CTL);
 412	ahci_rpm_put_port(ap);
 413
 414	return sprintf(buf, "%s%s%s%s\n",
 415		       em_ctl & EM_CTL_LED ? "led " : "",
 416		       em_ctl & EM_CTL_SAFTE ? "saf-te " : "",
 417		       em_ctl & EM_CTL_SES ? "ses-2 " : "",
 418		       em_ctl & EM_CTL_SGPIO ? "sgpio " : "");
 419}
 420
 421/**
 422 *	ahci_save_initial_config - Save and fixup initial config values
 423 *	@dev: target AHCI device
 424 *	@hpriv: host private area to store config values
 425 *
 426 *	Some registers containing configuration info might be setup by
 427 *	BIOS and might be cleared on reset.  This function saves the
 428 *	initial values of those registers into @hpriv such that they
 429 *	can be restored after controller reset.
 430 *
 431 *	If inconsistent, config values are fixed up by this function.
 432 *
 433 *	If it is not set already this function sets hpriv->start_engine to
 434 *	ahci_start_engine.
 435 *
 436 *	LOCKING:
 437 *	None.
 438 */
 439void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv)
 440{
 441	void __iomem *mmio = hpriv->mmio;
 442	u32 cap, cap2, vers, port_map;
 443	int i;
 444
 445	/* make sure AHCI mode is enabled before accessing CAP */
 446	ahci_enable_ahci(mmio);
 447
 448	/* Values prefixed with saved_ are written back to host after
 449	 * reset.  Values without are used for driver operation.
 450	 */
 451	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
 452	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
 453
 454	/* CAP2 register is only defined for AHCI 1.2 and later */
 455	vers = readl(mmio + HOST_VERSION);
 456	if ((vers >> 16) > 1 ||
 457	   ((vers >> 16) == 1 && (vers & 0xFFFF) >= 0x200))
 458		hpriv->saved_cap2 = cap2 = readl(mmio + HOST_CAP2);
 459	else
 460		hpriv->saved_cap2 = cap2 = 0;
 461
 462	/* some chips have errata preventing 64bit use */
 463	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
 464		dev_info(dev, "controller can't do 64bit DMA, forcing 32bit\n");
 465		cap &= ~HOST_CAP_64;
 466	}
 467
 468	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
 469		dev_info(dev, "controller can't do NCQ, turning off CAP_NCQ\n");
 470		cap &= ~HOST_CAP_NCQ;
 471	}
 472
 473	if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
 474		dev_info(dev, "controller can do NCQ, turning on CAP_NCQ\n");
 475		cap |= HOST_CAP_NCQ;
 476	}
 477
 478	if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
 479		dev_info(dev, "controller can't do PMP, turning off CAP_PMP\n");
 480		cap &= ~HOST_CAP_PMP;
 481	}
 482
 483	if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {
 484		dev_info(dev,
 485			 "controller can't do SNTF, turning off CAP_SNTF\n");
 486		cap &= ~HOST_CAP_SNTF;
 487	}
 488
 489	if ((cap2 & HOST_CAP2_SDS) && (hpriv->flags & AHCI_HFLAG_NO_DEVSLP)) {
 490		dev_info(dev,
 491			 "controller can't do DEVSLP, turning off\n");
 492		cap2 &= ~HOST_CAP2_SDS;
 493		cap2 &= ~HOST_CAP2_SADM;
 494	}
 495
 496	if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {
 497		dev_info(dev, "controller can do FBS, turning on CAP_FBS\n");
 498		cap |= HOST_CAP_FBS;
 499	}
 500
 501	if ((cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_NO_FBS)) {
 502		dev_info(dev, "controller can't do FBS, turning off CAP_FBS\n");
 503		cap &= ~HOST_CAP_FBS;
 504	}
 505
 
 
 
 
 
 506	if (hpriv->force_port_map && port_map != hpriv->force_port_map) {
 507		dev_info(dev, "forcing port_map 0x%x -> 0x%x\n",
 508			 port_map, hpriv->force_port_map);
 509		port_map = hpriv->force_port_map;
 510		hpriv->saved_port_map = port_map;
 511	}
 512
 513	if (hpriv->mask_port_map) {
 514		dev_warn(dev, "masking port_map 0x%x -> 0x%x\n",
 515			port_map,
 516			port_map & hpriv->mask_port_map);
 517		port_map &= hpriv->mask_port_map;
 518	}
 519
 520	/* cross check port_map and cap.n_ports */
 521	if (port_map) {
 522		int map_ports = 0;
 523
 524		for (i = 0; i < AHCI_MAX_PORTS; i++)
 525			if (port_map & (1 << i))
 526				map_ports++;
 527
 528		/* If PI has more ports than n_ports, whine, clear
 529		 * port_map and let it be generated from n_ports.
 530		 */
 531		if (map_ports > ahci_nr_ports(cap)) {
 532			dev_warn(dev,
 533				 "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
 534				 port_map, ahci_nr_ports(cap));
 535			port_map = 0;
 536		}
 537	}
 538
 539	/* fabricate port_map from cap.nr_ports for < AHCI 1.3 */
 540	if (!port_map && vers < 0x10300) {
 541		port_map = (1 << ahci_nr_ports(cap)) - 1;
 542		dev_warn(dev, "forcing PORTS_IMPL to 0x%x\n", port_map);
 543
 544		/* write the fixed up value to the PI register */
 545		hpriv->saved_port_map = port_map;
 546	}
 547
 548	/* record values to use during operation */
 549	hpriv->cap = cap;
 550	hpriv->cap2 = cap2;
 551	hpriv->version = readl(mmio + HOST_VERSION);
 552	hpriv->port_map = port_map;
 553
 554	if (!hpriv->start_engine)
 555		hpriv->start_engine = ahci_start_engine;
 556
 
 
 
 557	if (!hpriv->irq_handler)
 558		hpriv->irq_handler = ahci_single_level_irq_intr;
 559}
 560EXPORT_SYMBOL_GPL(ahci_save_initial_config);
 561
 562/**
 563 *	ahci_restore_initial_config - Restore initial config
 564 *	@host: target ATA host
 565 *
 566 *	Restore initial config stored by ahci_save_initial_config().
 567 *
 568 *	LOCKING:
 569 *	None.
 570 */
 571static void ahci_restore_initial_config(struct ata_host *host)
 572{
 573	struct ahci_host_priv *hpriv = host->private_data;
 574	void __iomem *mmio = hpriv->mmio;
 575
 576	writel(hpriv->saved_cap, mmio + HOST_CAP);
 577	if (hpriv->saved_cap2)
 578		writel(hpriv->saved_cap2, mmio + HOST_CAP2);
 579	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
 580	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
 581}
 582
 583static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
 584{
 585	static const int offset[] = {
 586		[SCR_STATUS]		= PORT_SCR_STAT,
 587		[SCR_CONTROL]		= PORT_SCR_CTL,
 588		[SCR_ERROR]		= PORT_SCR_ERR,
 589		[SCR_ACTIVE]		= PORT_SCR_ACT,
 590		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
 591	};
 592	struct ahci_host_priv *hpriv = ap->host->private_data;
 593
 594	if (sc_reg < ARRAY_SIZE(offset) &&
 595	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
 596		return offset[sc_reg];
 597	return 0;
 598}
 599
 600static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
 601{
 602	void __iomem *port_mmio = ahci_port_base(link->ap);
 603	int offset = ahci_scr_offset(link->ap, sc_reg);
 604
 605	if (offset) {
 606		*val = readl(port_mmio + offset);
 607		return 0;
 608	}
 609	return -EINVAL;
 610}
 611
 612static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
 613{
 614	void __iomem *port_mmio = ahci_port_base(link->ap);
 615	int offset = ahci_scr_offset(link->ap, sc_reg);
 616
 617	if (offset) {
 618		writel(val, port_mmio + offset);
 619		return 0;
 620	}
 621	return -EINVAL;
 622}
 623
 624void ahci_start_engine(struct ata_port *ap)
 625{
 626	void __iomem *port_mmio = ahci_port_base(ap);
 627	u32 tmp;
 628
 629	/* start DMA */
 630	tmp = readl(port_mmio + PORT_CMD);
 631	tmp |= PORT_CMD_START;
 632	writel(tmp, port_mmio + PORT_CMD);
 633	readl(port_mmio + PORT_CMD); /* flush */
 634}
 635EXPORT_SYMBOL_GPL(ahci_start_engine);
 636
 637int ahci_stop_engine(struct ata_port *ap)
 638{
 639	void __iomem *port_mmio = ahci_port_base(ap);
 640	struct ahci_host_priv *hpriv = ap->host->private_data;
 641	u32 tmp;
 642
 643	/*
 644	 * On some controllers, stopping a port's DMA engine while the port
 645	 * is in ALPM state (partial or slumber) results in failures on
 646	 * subsequent DMA engine starts.  For those controllers, put the
 647	 * port back in active state before stopping its DMA engine.
 648	 */
 649	if ((hpriv->flags & AHCI_HFLAG_WAKE_BEFORE_STOP) &&
 650	    (ap->link.lpm_policy > ATA_LPM_MAX_POWER) &&
 651	    ahci_set_lpm(&ap->link, ATA_LPM_MAX_POWER, ATA_LPM_WAKE_ONLY)) {
 652		dev_err(ap->host->dev, "Failed to wake up port before engine stop\n");
 653		return -EIO;
 654	}
 655
 656	tmp = readl(port_mmio + PORT_CMD);
 657
 658	/* check if the HBA is idle */
 659	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
 660		return 0;
 661
 
 
 
 
 
 
 
 
 
 
 662	/* setting HBA to idle */
 663	tmp &= ~PORT_CMD_START;
 664	writel(tmp, port_mmio + PORT_CMD);
 665
 666	/* wait for engine to stop. This could be as long as 500 msec */
 667	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
 668				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
 669	if (tmp & PORT_CMD_LIST_ON)
 670		return -EIO;
 671
 672	return 0;
 673}
 674EXPORT_SYMBOL_GPL(ahci_stop_engine);
 675
 676void ahci_start_fis_rx(struct ata_port *ap)
 677{
 678	void __iomem *port_mmio = ahci_port_base(ap);
 679	struct ahci_host_priv *hpriv = ap->host->private_data;
 680	struct ahci_port_priv *pp = ap->private_data;
 681	u32 tmp;
 682
 683	/* set FIS registers */
 684	if (hpriv->cap & HOST_CAP_64)
 685		writel((pp->cmd_slot_dma >> 16) >> 16,
 686		       port_mmio + PORT_LST_ADDR_HI);
 687	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
 688
 689	if (hpriv->cap & HOST_CAP_64)
 690		writel((pp->rx_fis_dma >> 16) >> 16,
 691		       port_mmio + PORT_FIS_ADDR_HI);
 692	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
 693
 694	/* enable FIS reception */
 695	tmp = readl(port_mmio + PORT_CMD);
 696	tmp |= PORT_CMD_FIS_RX;
 697	writel(tmp, port_mmio + PORT_CMD);
 698
 699	/* flush */
 700	readl(port_mmio + PORT_CMD);
 701}
 702EXPORT_SYMBOL_GPL(ahci_start_fis_rx);
 703
 704static int ahci_stop_fis_rx(struct ata_port *ap)
 705{
 706	void __iomem *port_mmio = ahci_port_base(ap);
 707	u32 tmp;
 708
 709	/* disable FIS reception */
 710	tmp = readl(port_mmio + PORT_CMD);
 711	tmp &= ~PORT_CMD_FIS_RX;
 712	writel(tmp, port_mmio + PORT_CMD);
 713
 714	/* wait for completion, spec says 500ms, give it 1000 */
 715	tmp = ata_wait_register(ap, port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
 716				PORT_CMD_FIS_ON, 10, 1000);
 717	if (tmp & PORT_CMD_FIS_ON)
 718		return -EBUSY;
 719
 720	return 0;
 721}
 722
 723static void ahci_power_up(struct ata_port *ap)
 724{
 725	struct ahci_host_priv *hpriv = ap->host->private_data;
 726	void __iomem *port_mmio = ahci_port_base(ap);
 727	u32 cmd;
 728
 729	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 730
 731	/* spin up device */
 732	if (hpriv->cap & HOST_CAP_SSS) {
 733		cmd |= PORT_CMD_SPIN_UP;
 734		writel(cmd, port_mmio + PORT_CMD);
 735	}
 736
 737	/* wake up link */
 738	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
 739}
 740
 741static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
 742			unsigned int hints)
 743{
 744	struct ata_port *ap = link->ap;
 745	struct ahci_host_priv *hpriv = ap->host->private_data;
 746	struct ahci_port_priv *pp = ap->private_data;
 747	void __iomem *port_mmio = ahci_port_base(ap);
 748
 749	if (policy != ATA_LPM_MAX_POWER) {
 750		/* wakeup flag only applies to the max power policy */
 751		hints &= ~ATA_LPM_WAKE_ONLY;
 752
 753		/*
 754		 * Disable interrupts on Phy Ready. This keeps us from
 755		 * getting woken up due to spurious phy ready
 756		 * interrupts.
 757		 */
 758		pp->intr_mask &= ~PORT_IRQ_PHYRDY;
 759		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 760
 761		sata_link_scr_lpm(link, policy, false);
 762	}
 763
 764	if (hpriv->cap & HOST_CAP_ALPM) {
 765		u32 cmd = readl(port_mmio + PORT_CMD);
 766
 767		if (policy == ATA_LPM_MAX_POWER || !(hints & ATA_LPM_HIPM)) {
 768			if (!(hints & ATA_LPM_WAKE_ONLY))
 769				cmd &= ~(PORT_CMD_ASP | PORT_CMD_ALPE);
 770			cmd |= PORT_CMD_ICC_ACTIVE;
 771
 772			writel(cmd, port_mmio + PORT_CMD);
 773			readl(port_mmio + PORT_CMD);
 774
 775			/* wait 10ms to be sure we've come out of LPM state */
 776			ata_msleep(ap, 10);
 777
 778			if (hints & ATA_LPM_WAKE_ONLY)
 779				return 0;
 780		} else {
 781			cmd |= PORT_CMD_ALPE;
 782			if (policy == ATA_LPM_MIN_POWER)
 783				cmd |= PORT_CMD_ASP;
 
 
 784
 785			/* write out new cmd value */
 786			writel(cmd, port_mmio + PORT_CMD);
 787		}
 788	}
 789
 790	/* set aggressive device sleep */
 791	if ((hpriv->cap2 & HOST_CAP2_SDS) &&
 792	    (hpriv->cap2 & HOST_CAP2_SADM) &&
 793	    (link->device->flags & ATA_DFLAG_DEVSLP)) {
 794		if (policy == ATA_LPM_MIN_POWER)
 
 795			ahci_set_aggressive_devslp(ap, true);
 796		else
 797			ahci_set_aggressive_devslp(ap, false);
 798	}
 799
 800	if (policy == ATA_LPM_MAX_POWER) {
 801		sata_link_scr_lpm(link, policy, false);
 802
 803		/* turn PHYRDY IRQ back on */
 804		pp->intr_mask |= PORT_IRQ_PHYRDY;
 805		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 806	}
 807
 808	return 0;
 809}
 810
 811#ifdef CONFIG_PM
 812static void ahci_power_down(struct ata_port *ap)
 813{
 814	struct ahci_host_priv *hpriv = ap->host->private_data;
 815	void __iomem *port_mmio = ahci_port_base(ap);
 816	u32 cmd, scontrol;
 817
 818	if (!(hpriv->cap & HOST_CAP_SSS))
 819		return;
 820
 821	/* put device into listen mode, first set PxSCTL.DET to 0 */
 822	scontrol = readl(port_mmio + PORT_SCR_CTL);
 823	scontrol &= ~0xf;
 824	writel(scontrol, port_mmio + PORT_SCR_CTL);
 825
 826	/* then set PxCMD.SUD to 0 */
 827	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 828	cmd &= ~PORT_CMD_SPIN_UP;
 829	writel(cmd, port_mmio + PORT_CMD);
 830}
 831#endif
 832
 833static void ahci_start_port(struct ata_port *ap)
 834{
 835	struct ahci_host_priv *hpriv = ap->host->private_data;
 836	struct ahci_port_priv *pp = ap->private_data;
 837	struct ata_link *link;
 838	struct ahci_em_priv *emp;
 839	ssize_t rc;
 840	int i;
 841
 842	/* enable FIS reception */
 843	ahci_start_fis_rx(ap);
 844
 845	/* enable DMA */
 846	if (!(hpriv->flags & AHCI_HFLAG_DELAY_ENGINE))
 847		hpriv->start_engine(ap);
 848
 849	/* turn on LEDs */
 850	if (ap->flags & ATA_FLAG_EM) {
 851		ata_for_each_link(link, ap, EDGE) {
 852			emp = &pp->em_priv[link->pmp];
 853
 854			/* EM Transmit bit maybe busy during init */
 855			for (i = 0; i < EM_MAX_RETRY; i++) {
 856				rc = ap->ops->transmit_led_message(ap,
 857							       emp->led_state,
 858							       4);
 859				/*
 860				 * If busy, give a breather but do not
 861				 * release EH ownership by using msleep()
 862				 * instead of ata_msleep().  EM Transmit
 863				 * bit is busy for the whole host and
 864				 * releasing ownership will cause other
 865				 * ports to fail the same way.
 866				 */
 867				if (rc == -EBUSY)
 868					msleep(1);
 869				else
 870					break;
 871			}
 872		}
 873	}
 874
 875	if (ap->flags & ATA_FLAG_SW_ACTIVITY)
 876		ata_for_each_link(link, ap, EDGE)
 877			ahci_init_sw_activity(link);
 878
 879}
 880
 881static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
 882{
 883	int rc;
 
 884
 885	/* disable DMA */
 886	rc = ahci_stop_engine(ap);
 887	if (rc) {
 888		*emsg = "failed to stop engine";
 889		return rc;
 890	}
 891
 892	/* disable FIS reception */
 893	rc = ahci_stop_fis_rx(ap);
 894	if (rc) {
 895		*emsg = "failed stop FIS RX";
 896		return rc;
 897	}
 898
 899	return 0;
 900}
 901
 902int ahci_reset_controller(struct ata_host *host)
 903{
 904	struct ahci_host_priv *hpriv = host->private_data;
 905	void __iomem *mmio = hpriv->mmio;
 906	u32 tmp;
 907
 908	/* we must be in AHCI mode, before using anything
 909	 * AHCI-specific, such as HOST_RESET.
 910	 */
 911	ahci_enable_ahci(mmio);
 912
 913	/* global controller reset */
 914	if (!ahci_skip_host_reset) {
 915		tmp = readl(mmio + HOST_CTL);
 916		if ((tmp & HOST_RESET) == 0) {
 917			writel(tmp | HOST_RESET, mmio + HOST_CTL);
 918			readl(mmio + HOST_CTL); /* flush */
 919		}
 920
 921		/*
 922		 * to perform host reset, OS should set HOST_RESET
 923		 * and poll until this bit is read to be "0".
 924		 * reset must complete within 1 second, or
 925		 * the hardware should be considered fried.
 926		 */
 927		tmp = ata_wait_register(NULL, mmio + HOST_CTL, HOST_RESET,
 928					HOST_RESET, 10, 1000);
 929
 930		if (tmp & HOST_RESET) {
 931			dev_err(host->dev, "controller reset failed (0x%x)\n",
 932				tmp);
 933			return -EIO;
 934		}
 935
 936		/* turn on AHCI mode */
 937		ahci_enable_ahci(mmio);
 938
 939		/* Some registers might be cleared on reset.  Restore
 940		 * initial values.
 941		 */
 942		ahci_restore_initial_config(host);
 
 943	} else
 944		dev_info(host->dev, "skipping global host reset\n");
 945
 946	return 0;
 947}
 948EXPORT_SYMBOL_GPL(ahci_reset_controller);
 949
 950static void ahci_sw_activity(struct ata_link *link)
 951{
 952	struct ata_port *ap = link->ap;
 953	struct ahci_port_priv *pp = ap->private_data;
 954	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 955
 956	if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
 957		return;
 958
 959	emp->activity++;
 960	if (!timer_pending(&emp->timer))
 961		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));
 962}
 963
 964static void ahci_sw_activity_blink(unsigned long arg)
 965{
 966	struct ata_link *link = (struct ata_link *)arg;
 
 967	struct ata_port *ap = link->ap;
 968	struct ahci_port_priv *pp = ap->private_data;
 969	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 970	unsigned long led_message = emp->led_state;
 971	u32 activity_led_state;
 972	unsigned long flags;
 973
 974	led_message &= EM_MSG_LED_VALUE;
 975	led_message |= ap->port_no | (link->pmp << 8);
 976
 977	/* check to see if we've had activity.  If so,
 978	 * toggle state of LED and reset timer.  If not,
 979	 * turn LED to desired idle state.
 980	 */
 981	spin_lock_irqsave(ap->lock, flags);
 982	if (emp->saved_activity != emp->activity) {
 983		emp->saved_activity = emp->activity;
 984		/* get the current LED state */
 985		activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
 986
 987		if (activity_led_state)
 988			activity_led_state = 0;
 989		else
 990			activity_led_state = 1;
 991
 992		/* clear old state */
 993		led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
 994
 995		/* toggle state */
 996		led_message |= (activity_led_state << 16);
 997		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));
 998	} else {
 999		/* switch to idle */
1000		led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
1001		if (emp->blink_policy == BLINK_OFF)
1002			led_message |= (1 << 16);
1003	}
1004	spin_unlock_irqrestore(ap->lock, flags);
1005	ap->ops->transmit_led_message(ap, led_message, 4);
1006}
1007
1008static void ahci_init_sw_activity(struct ata_link *link)
1009{
1010	struct ata_port *ap = link->ap;
1011	struct ahci_port_priv *pp = ap->private_data;
1012	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1013
1014	/* init activity stats, setup timer */
1015	emp->saved_activity = emp->activity = 0;
1016	setup_timer(&emp->timer, ahci_sw_activity_blink, (unsigned long)link);
 
1017
1018	/* check our blink policy and set flag for link if it's enabled */
1019	if (emp->blink_policy)
1020		link->flags |= ATA_LFLAG_SW_ACTIVITY;
1021}
1022
1023int ahci_reset_em(struct ata_host *host)
1024{
1025	struct ahci_host_priv *hpriv = host->private_data;
1026	void __iomem *mmio = hpriv->mmio;
1027	u32 em_ctl;
1028
1029	em_ctl = readl(mmio + HOST_EM_CTL);
1030	if ((em_ctl & EM_CTL_TM) || (em_ctl & EM_CTL_RST))
1031		return -EINVAL;
1032
1033	writel(em_ctl | EM_CTL_RST, mmio + HOST_EM_CTL);
1034	return 0;
1035}
1036EXPORT_SYMBOL_GPL(ahci_reset_em);
1037
1038static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
1039					ssize_t size)
1040{
1041	struct ahci_host_priv *hpriv = ap->host->private_data;
1042	struct ahci_port_priv *pp = ap->private_data;
1043	void __iomem *mmio = hpriv->mmio;
1044	u32 em_ctl;
1045	u32 message[] = {0, 0};
1046	unsigned long flags;
1047	int pmp;
1048	struct ahci_em_priv *emp;
1049
1050	/* get the slot number from the message */
1051	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1052	if (pmp < EM_MAX_SLOTS)
1053		emp = &pp->em_priv[pmp];
1054	else
1055		return -EINVAL;
1056
1057	ahci_rpm_get_port(ap);
1058	spin_lock_irqsave(ap->lock, flags);
1059
1060	/*
1061	 * if we are still busy transmitting a previous message,
1062	 * do not allow
1063	 */
1064	em_ctl = readl(mmio + HOST_EM_CTL);
1065	if (em_ctl & EM_CTL_TM) {
1066		spin_unlock_irqrestore(ap->lock, flags);
1067		ahci_rpm_put_port(ap);
1068		return -EBUSY;
1069	}
1070
1071	if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {
1072		/*
1073		 * create message header - this is all zero except for
1074		 * the message size, which is 4 bytes.
1075		 */
1076		message[0] |= (4 << 8);
1077
1078		/* ignore 0:4 of byte zero, fill in port info yourself */
1079		message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);
1080
1081		/* write message to EM_LOC */
1082		writel(message[0], mmio + hpriv->em_loc);
1083		writel(message[1], mmio + hpriv->em_loc+4);
1084
1085		/*
1086		 * tell hardware to transmit the message
1087		 */
1088		writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
1089	}
1090
1091	/* save off new led state for port/slot */
1092	emp->led_state = state;
1093
1094	spin_unlock_irqrestore(ap->lock, flags);
1095	ahci_rpm_put_port(ap);
1096
1097	return size;
1098}
1099
1100static ssize_t ahci_led_show(struct ata_port *ap, char *buf)
1101{
1102	struct ahci_port_priv *pp = ap->private_data;
1103	struct ata_link *link;
1104	struct ahci_em_priv *emp;
1105	int rc = 0;
1106
1107	ata_for_each_link(link, ap, EDGE) {
1108		emp = &pp->em_priv[link->pmp];
1109		rc += sprintf(buf, "%lx\n", emp->led_state);
1110	}
1111	return rc;
1112}
1113
1114static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
1115				size_t size)
1116{
1117	unsigned int state;
1118	int pmp;
1119	struct ahci_port_priv *pp = ap->private_data;
1120	struct ahci_em_priv *emp;
1121
1122	if (kstrtouint(buf, 0, &state) < 0)
1123		return -EINVAL;
1124
1125	/* get the slot number from the message */
1126	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1127	if (pmp < EM_MAX_SLOTS)
 
1128		emp = &pp->em_priv[pmp];
1129	else
1130		return -EINVAL;
 
1131
1132	/* mask off the activity bits if we are in sw_activity
1133	 * mode, user should turn off sw_activity before setting
1134	 * activity led through em_message
1135	 */
1136	if (emp->blink_policy)
1137		state &= ~EM_MSG_LED_VALUE_ACTIVITY;
1138
1139	return ap->ops->transmit_led_message(ap, state, size);
1140}
1141
1142static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
1143{
1144	struct ata_link *link = dev->link;
1145	struct ata_port *ap = link->ap;
1146	struct ahci_port_priv *pp = ap->private_data;
1147	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1148	u32 port_led_state = emp->led_state;
1149
1150	/* save the desired Activity LED behavior */
1151	if (val == OFF) {
1152		/* clear LFLAG */
1153		link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
1154
1155		/* set the LED to OFF */
1156		port_led_state &= EM_MSG_LED_VALUE_OFF;
1157		port_led_state |= (ap->port_no | (link->pmp << 8));
1158		ap->ops->transmit_led_message(ap, port_led_state, 4);
1159	} else {
1160		link->flags |= ATA_LFLAG_SW_ACTIVITY;
1161		if (val == BLINK_OFF) {
1162			/* set LED to ON for idle */
1163			port_led_state &= EM_MSG_LED_VALUE_OFF;
1164			port_led_state |= (ap->port_no | (link->pmp << 8));
1165			port_led_state |= EM_MSG_LED_VALUE_ON; /* check this */
1166			ap->ops->transmit_led_message(ap, port_led_state, 4);
1167		}
1168	}
1169	emp->blink_policy = val;
1170	return 0;
1171}
1172
1173static ssize_t ahci_activity_show(struct ata_device *dev, char *buf)
1174{
1175	struct ata_link *link = dev->link;
1176	struct ata_port *ap = link->ap;
1177	struct ahci_port_priv *pp = ap->private_data;
1178	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1179
1180	/* display the saved value of activity behavior for this
1181	 * disk.
1182	 */
1183	return sprintf(buf, "%d\n", emp->blink_policy);
1184}
1185
1186static void ahci_port_init(struct device *dev, struct ata_port *ap,
1187			   int port_no, void __iomem *mmio,
1188			   void __iomem *port_mmio)
1189{
1190	struct ahci_host_priv *hpriv = ap->host->private_data;
1191	const char *emsg = NULL;
1192	int rc;
1193	u32 tmp;
1194
1195	/* make sure port is not active */
1196	rc = ahci_deinit_port(ap, &emsg);
1197	if (rc)
1198		dev_warn(dev, "%s (%d)\n", emsg, rc);
1199
1200	/* clear SError */
1201	tmp = readl(port_mmio + PORT_SCR_ERR);
1202	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
1203	writel(tmp, port_mmio + PORT_SCR_ERR);
1204
1205	/* clear port IRQ */
1206	tmp = readl(port_mmio + PORT_IRQ_STAT);
1207	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
1208	if (tmp)
1209		writel(tmp, port_mmio + PORT_IRQ_STAT);
1210
1211	writel(1 << port_no, mmio + HOST_IRQ_STAT);
1212
1213	/* mark esata ports */
1214	tmp = readl(port_mmio + PORT_CMD);
1215	if ((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS))
1216		ap->pflags |= ATA_PFLAG_EXTERNAL;
1217}
1218
1219void ahci_init_controller(struct ata_host *host)
1220{
1221	struct ahci_host_priv *hpriv = host->private_data;
1222	void __iomem *mmio = hpriv->mmio;
1223	int i;
1224	void __iomem *port_mmio;
1225	u32 tmp;
1226
1227	for (i = 0; i < host->n_ports; i++) {
1228		struct ata_port *ap = host->ports[i];
1229
1230		port_mmio = ahci_port_base(ap);
1231		if (ata_port_is_dummy(ap))
1232			continue;
1233
1234		ahci_port_init(host->dev, ap, i, mmio, port_mmio);
1235	}
1236
1237	tmp = readl(mmio + HOST_CTL);
1238	VPRINTK("HOST_CTL 0x%x\n", tmp);
1239	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
1240	tmp = readl(mmio + HOST_CTL);
1241	VPRINTK("HOST_CTL 0x%x\n", tmp);
1242}
1243EXPORT_SYMBOL_GPL(ahci_init_controller);
1244
1245static void ahci_dev_config(struct ata_device *dev)
1246{
1247	struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
1248
1249	if (hpriv->flags & AHCI_HFLAG_SECT255) {
1250		dev->max_sectors = 255;
1251		ata_dev_info(dev,
1252			     "SB600 AHCI: limiting to 255 sectors per cmd\n");
1253	}
1254}
1255
1256unsigned int ahci_dev_classify(struct ata_port *ap)
1257{
1258	void __iomem *port_mmio = ahci_port_base(ap);
1259	struct ata_taskfile tf;
1260	u32 tmp;
1261
1262	tmp = readl(port_mmio + PORT_SIG);
1263	tf.lbah		= (tmp >> 24)	& 0xff;
1264	tf.lbam		= (tmp >> 16)	& 0xff;
1265	tf.lbal		= (tmp >> 8)	& 0xff;
1266	tf.nsect	= (tmp)		& 0xff;
1267
1268	return ata_dev_classify(&tf);
1269}
1270EXPORT_SYMBOL_GPL(ahci_dev_classify);
1271
1272void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
1273			u32 opts)
1274{
1275	dma_addr_t cmd_tbl_dma;
1276
1277	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
1278
1279	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
1280	pp->cmd_slot[tag].status = 0;
1281	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
1282	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1283}
1284EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot);
1285
1286int ahci_kick_engine(struct ata_port *ap)
1287{
1288	void __iomem *port_mmio = ahci_port_base(ap);
1289	struct ahci_host_priv *hpriv = ap->host->private_data;
1290	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1291	u32 tmp;
1292	int busy, rc;
1293
1294	/* stop engine */
1295	rc = ahci_stop_engine(ap);
1296	if (rc)
1297		goto out_restart;
1298
1299	/* need to do CLO?
1300	 * always do CLO if PMP is attached (AHCI-1.3 9.2)
1301	 */
1302	busy = status & (ATA_BUSY | ATA_DRQ);
1303	if (!busy && !sata_pmp_attached(ap)) {
1304		rc = 0;
1305		goto out_restart;
1306	}
1307
1308	if (!(hpriv->cap & HOST_CAP_CLO)) {
1309		rc = -EOPNOTSUPP;
1310		goto out_restart;
1311	}
1312
1313	/* perform CLO */
1314	tmp = readl(port_mmio + PORT_CMD);
1315	tmp |= PORT_CMD_CLO;
1316	writel(tmp, port_mmio + PORT_CMD);
1317
1318	rc = 0;
1319	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
1320				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
1321	if (tmp & PORT_CMD_CLO)
1322		rc = -EIO;
1323
1324	/* restart engine */
1325 out_restart:
1326	hpriv->start_engine(ap);
1327	return rc;
1328}
1329EXPORT_SYMBOL_GPL(ahci_kick_engine);
1330
1331static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
1332				struct ata_taskfile *tf, int is_cmd, u16 flags,
1333				unsigned long timeout_msec)
1334{
1335	const u32 cmd_fis_len = 5; /* five dwords */
1336	struct ahci_port_priv *pp = ap->private_data;
1337	void __iomem *port_mmio = ahci_port_base(ap);
1338	u8 *fis = pp->cmd_tbl;
1339	u32 tmp;
1340
1341	/* prep the command */
1342	ata_tf_to_fis(tf, pmp, is_cmd, fis);
1343	ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
1344
1345	/* set port value for softreset of Port Multiplier */
1346	if (pp->fbs_enabled && pp->fbs_last_dev != pmp) {
1347		tmp = readl(port_mmio + PORT_FBS);
1348		tmp &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
1349		tmp |= pmp << PORT_FBS_DEV_OFFSET;
1350		writel(tmp, port_mmio + PORT_FBS);
1351		pp->fbs_last_dev = pmp;
1352	}
1353
1354	/* issue & wait */
1355	writel(1, port_mmio + PORT_CMD_ISSUE);
1356
1357	if (timeout_msec) {
1358		tmp = ata_wait_register(ap, port_mmio + PORT_CMD_ISSUE,
1359					0x1, 0x1, 1, timeout_msec);
1360		if (tmp & 0x1) {
1361			ahci_kick_engine(ap);
1362			return -EBUSY;
1363		}
1364	} else
1365		readl(port_mmio + PORT_CMD_ISSUE);	/* flush */
1366
1367	return 0;
1368}
1369
1370int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1371		      int pmp, unsigned long deadline,
1372		      int (*check_ready)(struct ata_link *link))
1373{
1374	struct ata_port *ap = link->ap;
1375	struct ahci_host_priv *hpriv = ap->host->private_data;
1376	struct ahci_port_priv *pp = ap->private_data;
1377	const char *reason = NULL;
1378	unsigned long now, msecs;
1379	struct ata_taskfile tf;
1380	bool fbs_disabled = false;
1381	int rc;
1382
1383	DPRINTK("ENTER\n");
1384
1385	/* prepare for SRST (AHCI-1.1 10.4.1) */
1386	rc = ahci_kick_engine(ap);
1387	if (rc && rc != -EOPNOTSUPP)
1388		ata_link_warn(link, "failed to reset engine (errno=%d)\n", rc);
1389
1390	/*
1391	 * According to AHCI-1.2 9.3.9: if FBS is enable, software shall
1392	 * clear PxFBS.EN to '0' prior to issuing software reset to devices
1393	 * that is attached to port multiplier.
1394	 */
1395	if (!ata_is_host_link(link) && pp->fbs_enabled) {
1396		ahci_disable_fbs(ap);
1397		fbs_disabled = true;
1398	}
1399
1400	ata_tf_init(link->device, &tf);
1401
1402	/* issue the first D2H Register FIS */
1403	msecs = 0;
1404	now = jiffies;
1405	if (time_after(deadline, now))
1406		msecs = jiffies_to_msecs(deadline - now);
1407
1408	tf.ctl |= ATA_SRST;
1409	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1410				 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
1411		rc = -EIO;
1412		reason = "1st FIS failed";
1413		goto fail;
1414	}
1415
1416	/* spec says at least 5us, but be generous and sleep for 1ms */
1417	ata_msleep(ap, 1);
1418
1419	/* issue the second D2H Register FIS */
1420	tf.ctl &= ~ATA_SRST;
1421	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
1422
1423	/* wait for link to become ready */
1424	rc = ata_wait_after_reset(link, deadline, check_ready);
1425	if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {
1426		/*
1427		 * Workaround for cases where link online status can't
1428		 * be trusted.  Treat device readiness timeout as link
1429		 * offline.
1430		 */
1431		ata_link_info(link, "device not ready, treating as offline\n");
1432		*class = ATA_DEV_NONE;
1433	} else if (rc) {
1434		/* link occupied, -ENODEV too is an error */
1435		reason = "device not ready";
1436		goto fail;
1437	} else
1438		*class = ahci_dev_classify(ap);
1439
1440	/* re-enable FBS if disabled before */
1441	if (fbs_disabled)
1442		ahci_enable_fbs(ap);
1443
1444	DPRINTK("EXIT, class=%u\n", *class);
1445	return 0;
1446
1447 fail:
1448	ata_link_err(link, "softreset failed (%s)\n", reason);
1449	return rc;
1450}
1451
1452int ahci_check_ready(struct ata_link *link)
1453{
1454	void __iomem *port_mmio = ahci_port_base(link->ap);
1455	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1456
1457	return ata_check_ready(status);
1458}
1459EXPORT_SYMBOL_GPL(ahci_check_ready);
1460
1461static int ahci_softreset(struct ata_link *link, unsigned int *class,
1462			  unsigned long deadline)
1463{
1464	int pmp = sata_srst_pmp(link);
1465
1466	DPRINTK("ENTER\n");
1467
1468	return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
1469}
1470EXPORT_SYMBOL_GPL(ahci_do_softreset);
1471
1472static int ahci_bad_pmp_check_ready(struct ata_link *link)
1473{
1474	void __iomem *port_mmio = ahci_port_base(link->ap);
1475	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1476	u32 irq_status = readl(port_mmio + PORT_IRQ_STAT);
1477
1478	/*
1479	 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1480	 * which can save timeout delay.
1481	 */
1482	if (irq_status & PORT_IRQ_BAD_PMP)
1483		return -EIO;
1484
1485	return ata_check_ready(status);
1486}
1487
1488static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
1489				    unsigned long deadline)
1490{
1491	struct ata_port *ap = link->ap;
1492	void __iomem *port_mmio = ahci_port_base(ap);
1493	int pmp = sata_srst_pmp(link);
1494	int rc;
1495	u32 irq_sts;
1496
1497	DPRINTK("ENTER\n");
1498
1499	rc = ahci_do_softreset(link, class, pmp, deadline,
1500			       ahci_bad_pmp_check_ready);
1501
1502	/*
1503	 * Soft reset fails with IPMS set when PMP is enabled but
1504	 * SATA HDD/ODD is connected to SATA port, do soft reset
1505	 * again to port 0.
1506	 */
1507	if (rc == -EIO) {
1508		irq_sts = readl(port_mmio + PORT_IRQ_STAT);
1509		if (irq_sts & PORT_IRQ_BAD_PMP) {
1510			ata_link_warn(link,
1511					"applying PMP SRST workaround "
1512					"and retrying\n");
1513			rc = ahci_do_softreset(link, class, 0, deadline,
1514					       ahci_check_ready);
1515		}
1516	}
1517
1518	return rc;
1519}
1520
1521static int ahci_hardreset(struct ata_link *link, unsigned int *class,
1522			  unsigned long deadline)
1523{
1524	const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
1525	struct ata_port *ap = link->ap;
1526	struct ahci_port_priv *pp = ap->private_data;
1527	struct ahci_host_priv *hpriv = ap->host->private_data;
1528	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1529	struct ata_taskfile tf;
1530	bool online;
1531	int rc;
1532
1533	DPRINTK("ENTER\n");
1534
1535	ahci_stop_engine(ap);
1536
1537	/* clear D2H reception area to properly wait for D2H FIS */
1538	ata_tf_init(link->device, &tf);
1539	tf.command = ATA_BUSY;
1540	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1541
1542	rc = sata_link_hardreset(link, timing, deadline, &online,
1543				 ahci_check_ready);
1544
1545	hpriv->start_engine(ap);
1546
1547	if (online)
1548		*class = ahci_dev_classify(ap);
1549
1550	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1551	return rc;
1552}
 
 
 
 
 
 
 
 
 
1553
1554static void ahci_postreset(struct ata_link *link, unsigned int *class)
1555{
1556	struct ata_port *ap = link->ap;
1557	void __iomem *port_mmio = ahci_port_base(ap);
1558	u32 new_tmp, tmp;
1559
1560	ata_std_postreset(link, class);
1561
1562	/* Make sure port's ATAPI bit is set appropriately */
1563	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1564	if (*class == ATA_DEV_ATAPI)
1565		new_tmp |= PORT_CMD_ATAPI;
1566	else
1567		new_tmp &= ~PORT_CMD_ATAPI;
1568	if (new_tmp != tmp) {
1569		writel(new_tmp, port_mmio + PORT_CMD);
1570		readl(port_mmio + PORT_CMD); /* flush */
1571	}
1572}
1573
1574static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
1575{
1576	struct scatterlist *sg;
1577	struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
1578	unsigned int si;
1579
1580	VPRINTK("ENTER\n");
1581
1582	/*
1583	 * Next, the S/G list.
1584	 */
1585	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1586		dma_addr_t addr = sg_dma_address(sg);
1587		u32 sg_len = sg_dma_len(sg);
1588
1589		ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
1590		ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
1591		ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
1592	}
1593
1594	return si;
1595}
1596
1597static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
1598{
1599	struct ata_port *ap = qc->ap;
1600	struct ahci_port_priv *pp = ap->private_data;
1601
1602	if (!sata_pmp_attached(ap) || pp->fbs_enabled)
1603		return ata_std_qc_defer(qc);
1604	else
1605		return sata_pmp_qc_defer_cmd_switch(qc);
1606}
1607
1608static void ahci_qc_prep(struct ata_queued_cmd *qc)
1609{
1610	struct ata_port *ap = qc->ap;
1611	struct ahci_port_priv *pp = ap->private_data;
1612	int is_atapi = ata_is_atapi(qc->tf.protocol);
1613	void *cmd_tbl;
1614	u32 opts;
1615	const u32 cmd_fis_len = 5; /* five dwords */
1616	unsigned int n_elem;
1617
1618	/*
1619	 * Fill in command table information.  First, the header,
1620	 * a SATA Register - Host to Device command FIS.
1621	 */
1622	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
1623
1624	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
1625	if (is_atapi) {
1626		memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
1627		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1628	}
1629
1630	n_elem = 0;
1631	if (qc->flags & ATA_QCFLAG_DMAMAP)
1632		n_elem = ahci_fill_sg(qc, cmd_tbl);
1633
1634	/*
1635	 * Fill in command slot information.
1636	 */
1637	opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
1638	if (qc->tf.flags & ATA_TFLAG_WRITE)
1639		opts |= AHCI_CMD_WRITE;
1640	if (is_atapi)
1641		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1642
1643	ahci_fill_cmd_slot(pp, qc->tag, opts);
 
 
1644}
1645
1646static void ahci_fbs_dec_intr(struct ata_port *ap)
1647{
1648	struct ahci_port_priv *pp = ap->private_data;
1649	void __iomem *port_mmio = ahci_port_base(ap);
1650	u32 fbs = readl(port_mmio + PORT_FBS);
1651	int retries = 3;
1652
1653	DPRINTK("ENTER\n");
1654	BUG_ON(!pp->fbs_enabled);
1655
1656	/* time to wait for DEC is not specified by AHCI spec,
1657	 * add a retry loop for safety.
1658	 */
1659	writel(fbs | PORT_FBS_DEC, port_mmio + PORT_FBS);
1660	fbs = readl(port_mmio + PORT_FBS);
1661	while ((fbs & PORT_FBS_DEC) && retries--) {
1662		udelay(1);
1663		fbs = readl(port_mmio + PORT_FBS);
1664	}
1665
1666	if (fbs & PORT_FBS_DEC)
1667		dev_err(ap->host->dev, "failed to clear device error\n");
1668}
1669
1670static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1671{
1672	struct ahci_host_priv *hpriv = ap->host->private_data;
1673	struct ahci_port_priv *pp = ap->private_data;
1674	struct ata_eh_info *host_ehi = &ap->link.eh_info;
1675	struct ata_link *link = NULL;
1676	struct ata_queued_cmd *active_qc;
1677	struct ata_eh_info *active_ehi;
1678	bool fbs_need_dec = false;
1679	u32 serror;
1680
1681	/* determine active link with error */
1682	if (pp->fbs_enabled) {
1683		void __iomem *port_mmio = ahci_port_base(ap);
1684		u32 fbs = readl(port_mmio + PORT_FBS);
1685		int pmp = fbs >> PORT_FBS_DWE_OFFSET;
1686
1687		if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links)) {
1688			link = &ap->pmp_link[pmp];
1689			fbs_need_dec = true;
1690		}
1691
1692	} else
1693		ata_for_each_link(link, ap, EDGE)
1694			if (ata_link_active(link))
1695				break;
1696
1697	if (!link)
1698		link = &ap->link;
1699
1700	active_qc = ata_qc_from_tag(ap, link->active_tag);
1701	active_ehi = &link->eh_info;
1702
1703	/* record irq stat */
1704	ata_ehi_clear_desc(host_ehi);
1705	ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
1706
1707	/* AHCI needs SError cleared; otherwise, it might lock up */
1708	ahci_scr_read(&ap->link, SCR_ERROR, &serror);
1709	ahci_scr_write(&ap->link, SCR_ERROR, serror);
1710	host_ehi->serror |= serror;
1711
1712	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1713	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1714		irq_stat &= ~PORT_IRQ_IF_ERR;
1715
1716	if (irq_stat & PORT_IRQ_TF_ERR) {
1717		/* If qc is active, charge it; otherwise, the active
1718		 * link.  There's no active qc on NCQ errors.  It will
1719		 * be determined by EH by reading log page 10h.
1720		 */
1721		if (active_qc)
1722			active_qc->err_mask |= AC_ERR_DEV;
1723		else
1724			active_ehi->err_mask |= AC_ERR_DEV;
1725
1726		if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
1727			host_ehi->serror &= ~SERR_INTERNAL;
1728	}
1729
1730	if (irq_stat & PORT_IRQ_UNK_FIS) {
1731		u32 *unk = pp->rx_fis + RX_FIS_UNK;
1732
1733		active_ehi->err_mask |= AC_ERR_HSM;
1734		active_ehi->action |= ATA_EH_RESET;
1735		ata_ehi_push_desc(active_ehi,
1736				  "unknown FIS %08x %08x %08x %08x" ,
1737				  unk[0], unk[1], unk[2], unk[3]);
1738	}
1739
1740	if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
1741		active_ehi->err_mask |= AC_ERR_HSM;
1742		active_ehi->action |= ATA_EH_RESET;
1743		ata_ehi_push_desc(active_ehi, "incorrect PMP");
1744	}
1745
1746	if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
1747		host_ehi->err_mask |= AC_ERR_HOST_BUS;
1748		host_ehi->action |= ATA_EH_RESET;
1749		ata_ehi_push_desc(host_ehi, "host bus error");
1750	}
1751
1752	if (irq_stat & PORT_IRQ_IF_ERR) {
1753		if (fbs_need_dec)
1754			active_ehi->err_mask |= AC_ERR_DEV;
1755		else {
1756			host_ehi->err_mask |= AC_ERR_ATA_BUS;
1757			host_ehi->action |= ATA_EH_RESET;
1758		}
1759
1760		ata_ehi_push_desc(host_ehi, "interface fatal error");
1761	}
1762
1763	if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
1764		ata_ehi_hotplugged(host_ehi);
1765		ata_ehi_push_desc(host_ehi, "%s",
1766			irq_stat & PORT_IRQ_CONNECT ?
1767			"connection status changed" : "PHY RDY changed");
1768	}
1769
1770	/* okay, let's hand over to EH */
1771
1772	if (irq_stat & PORT_IRQ_FREEZE)
1773		ata_port_freeze(ap);
1774	else if (fbs_need_dec) {
1775		ata_link_abort(link);
1776		ahci_fbs_dec_intr(ap);
1777	} else
1778		ata_port_abort(ap);
1779}
1780
1781static void ahci_handle_port_interrupt(struct ata_port *ap,
1782				       void __iomem *port_mmio, u32 status)
1783{
1784	struct ata_eh_info *ehi = &ap->link.eh_info;
1785	struct ahci_port_priv *pp = ap->private_data;
1786	struct ahci_host_priv *hpriv = ap->host->private_data;
1787	int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
1788	u32 qc_active = 0;
1789	int rc;
1790
1791	/* ignore BAD_PMP while resetting */
1792	if (unlikely(resetting))
1793		status &= ~PORT_IRQ_BAD_PMP;
1794
1795	if (sata_lpm_ignore_phy_events(&ap->link)) {
1796		status &= ~PORT_IRQ_PHYRDY;
1797		ahci_scr_write(&ap->link, SCR_ERROR, SERR_PHYRDY_CHG);
1798	}
1799
1800	if (unlikely(status & PORT_IRQ_ERROR)) {
1801		ahci_error_intr(ap, status);
1802		return;
1803	}
1804
1805	if (status & PORT_IRQ_SDB_FIS) {
1806		/* If SNotification is available, leave notification
1807		 * handling to sata_async_notification().  If not,
1808		 * emulate it by snooping SDB FIS RX area.
1809		 *
1810		 * Snooping FIS RX area is probably cheaper than
1811		 * poking SNotification but some constrollers which
1812		 * implement SNotification, ICH9 for example, don't
1813		 * store AN SDB FIS into receive area.
1814		 */
1815		if (hpriv->cap & HOST_CAP_SNTF)
1816			sata_async_notification(ap);
1817		else {
1818			/* If the 'N' bit in word 0 of the FIS is set,
1819			 * we just received asynchronous notification.
1820			 * Tell libata about it.
1821			 *
1822			 * Lack of SNotification should not appear in
1823			 * ahci 1.2, so the workaround is unnecessary
1824			 * when FBS is enabled.
1825			 */
1826			if (pp->fbs_enabled)
1827				WARN_ON_ONCE(1);
1828			else {
1829				const __le32 *f = pp->rx_fis + RX_FIS_SDB;
1830				u32 f0 = le32_to_cpu(f[0]);
1831				if (f0 & (1 << 15))
1832					sata_async_notification(ap);
1833			}
1834		}
1835	}
1836
1837	/* pp->active_link is not reliable once FBS is enabled, both
1838	 * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1839	 * NCQ and non-NCQ commands may be in flight at the same time.
1840	 */
1841	if (pp->fbs_enabled) {
1842		if (ap->qc_active) {
1843			qc_active = readl(port_mmio + PORT_SCR_ACT);
1844			qc_active |= readl(port_mmio + PORT_CMD_ISSUE);
1845		}
1846	} else {
1847		/* pp->active_link is valid iff any command is in flight */
1848		if (ap->qc_active && pp->active_link->sactive)
1849			qc_active = readl(port_mmio + PORT_SCR_ACT);
1850		else
1851			qc_active = readl(port_mmio + PORT_CMD_ISSUE);
1852	}
1853
1854
1855	rc = ata_qc_complete_multiple(ap, qc_active);
1856
1857	/* while resetting, invalid completions are expected */
1858	if (unlikely(rc < 0 && !resetting)) {
1859		ehi->err_mask |= AC_ERR_HSM;
1860		ehi->action |= ATA_EH_RESET;
1861		ata_port_freeze(ap);
1862	}
1863}
1864
1865static void ahci_port_intr(struct ata_port *ap)
1866{
1867	void __iomem *port_mmio = ahci_port_base(ap);
1868	u32 status;
1869
1870	status = readl(port_mmio + PORT_IRQ_STAT);
1871	writel(status, port_mmio + PORT_IRQ_STAT);
1872
1873	ahci_handle_port_interrupt(ap, port_mmio, status);
1874}
1875
1876static irqreturn_t ahci_multi_irqs_intr_hard(int irq, void *dev_instance)
1877{
1878	struct ata_port *ap = dev_instance;
1879	void __iomem *port_mmio = ahci_port_base(ap);
1880	u32 status;
1881
1882	VPRINTK("ENTER\n");
1883
1884	status = readl(port_mmio + PORT_IRQ_STAT);
1885	writel(status, port_mmio + PORT_IRQ_STAT);
1886
1887	spin_lock(ap->lock);
1888	ahci_handle_port_interrupt(ap, port_mmio, status);
1889	spin_unlock(ap->lock);
1890
1891	VPRINTK("EXIT\n");
1892
1893	return IRQ_HANDLED;
1894}
1895
1896u32 ahci_handle_port_intr(struct ata_host *host, u32 irq_masked)
1897{
1898	unsigned int i, handled = 0;
1899
1900	for (i = 0; i < host->n_ports; i++) {
1901		struct ata_port *ap;
1902
1903		if (!(irq_masked & (1 << i)))
1904			continue;
1905
1906		ap = host->ports[i];
1907		if (ap) {
1908			ahci_port_intr(ap);
1909			VPRINTK("port %u\n", i);
1910		} else {
1911			VPRINTK("port %u (no irq)\n", i);
1912			if (ata_ratelimit())
1913				dev_warn(host->dev,
1914					 "interrupt on disabled port %u\n", i);
1915		}
1916
1917		handled = 1;
1918	}
1919
1920	return handled;
1921}
1922EXPORT_SYMBOL_GPL(ahci_handle_port_intr);
1923
1924static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance)
1925{
1926	struct ata_host *host = dev_instance;
1927	struct ahci_host_priv *hpriv;
1928	unsigned int rc = 0;
1929	void __iomem *mmio;
1930	u32 irq_stat, irq_masked;
1931
1932	VPRINTK("ENTER\n");
1933
1934	hpriv = host->private_data;
1935	mmio = hpriv->mmio;
1936
1937	/* sigh.  0xffffffff is a valid return from h/w */
1938	irq_stat = readl(mmio + HOST_IRQ_STAT);
1939	if (!irq_stat)
1940		return IRQ_NONE;
1941
1942	irq_masked = irq_stat & hpriv->port_map;
1943
1944	spin_lock(&host->lock);
1945
1946	rc = ahci_handle_port_intr(host, irq_masked);
1947
1948	/* HOST_IRQ_STAT behaves as level triggered latch meaning that
1949	 * it should be cleared after all the port events are cleared;
1950	 * otherwise, it will raise a spurious interrupt after each
1951	 * valid one.  Please read section 10.6.2 of ahci 1.1 for more
1952	 * information.
1953	 *
1954	 * Also, use the unmasked value to clear interrupt as spurious
1955	 * pending event on a dummy port might cause screaming IRQ.
1956	 */
1957	writel(irq_stat, mmio + HOST_IRQ_STAT);
1958
1959	spin_unlock(&host->lock);
1960
1961	VPRINTK("EXIT\n");
1962
1963	return IRQ_RETVAL(rc);
1964}
1965
1966unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
1967{
1968	struct ata_port *ap = qc->ap;
1969	void __iomem *port_mmio = ahci_port_base(ap);
1970	struct ahci_port_priv *pp = ap->private_data;
1971
1972	/* Keep track of the currently active link.  It will be used
1973	 * in completion path to determine whether NCQ phase is in
1974	 * progress.
1975	 */
1976	pp->active_link = qc->dev->link;
1977
1978	if (qc->tf.protocol == ATA_PROT_NCQ)
1979		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
1980
1981	if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
1982		u32 fbs = readl(port_mmio + PORT_FBS);
1983		fbs &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
1984		fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
1985		writel(fbs, port_mmio + PORT_FBS);
1986		pp->fbs_last_dev = qc->dev->link->pmp;
1987	}
1988
1989	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
1990
1991	ahci_sw_activity(qc->dev->link);
1992
1993	return 0;
1994}
1995EXPORT_SYMBOL_GPL(ahci_qc_issue);
1996
1997static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
1998{
1999	struct ahci_port_priv *pp = qc->ap->private_data;
2000	u8 *rx_fis = pp->rx_fis;
2001
2002	if (pp->fbs_enabled)
2003		rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
2004
2005	/*
2006	 * After a successful execution of an ATA PIO data-in command,
2007	 * the device doesn't send D2H Reg FIS to update the TF and
2008	 * the host should take TF and E_Status from the preceding PIO
2009	 * Setup FIS.
2010	 */
2011	if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&
2012	    !(qc->flags & ATA_QCFLAG_FAILED)) {
2013		ata_tf_from_fis(rx_fis + RX_FIS_PIO_SETUP, &qc->result_tf);
2014		qc->result_tf.command = (rx_fis + RX_FIS_PIO_SETUP)[15];
2015	} else
2016		ata_tf_from_fis(rx_fis + RX_FIS_D2H_REG, &qc->result_tf);
2017
2018	return true;
2019}
2020
2021static void ahci_freeze(struct ata_port *ap)
2022{
2023	void __iomem *port_mmio = ahci_port_base(ap);
2024
2025	/* turn IRQ off */
2026	writel(0, port_mmio + PORT_IRQ_MASK);
2027}
2028
2029static void ahci_thaw(struct ata_port *ap)
2030{
2031	struct ahci_host_priv *hpriv = ap->host->private_data;
2032	void __iomem *mmio = hpriv->mmio;
2033	void __iomem *port_mmio = ahci_port_base(ap);
2034	u32 tmp;
2035	struct ahci_port_priv *pp = ap->private_data;
2036
2037	/* clear IRQ */
2038	tmp = readl(port_mmio + PORT_IRQ_STAT);
2039	writel(tmp, port_mmio + PORT_IRQ_STAT);
2040	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
2041
2042	/* turn IRQ back on */
2043	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2044}
2045
2046void ahci_error_handler(struct ata_port *ap)
2047{
2048	struct ahci_host_priv *hpriv = ap->host->private_data;
2049
2050	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
2051		/* restart engine */
2052		ahci_stop_engine(ap);
2053		hpriv->start_engine(ap);
2054	}
2055
2056	sata_pmp_error_handler(ap);
2057
2058	if (!ata_dev_enabled(ap->link.device))
2059		ahci_stop_engine(ap);
2060}
2061EXPORT_SYMBOL_GPL(ahci_error_handler);
2062
2063static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
2064{
2065	struct ata_port *ap = qc->ap;
2066
2067	/* make DMA engine forget about the failed command */
2068	if (qc->flags & ATA_QCFLAG_FAILED)
2069		ahci_kick_engine(ap);
2070}
2071
2072static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep)
2073{
2074	struct ahci_host_priv *hpriv = ap->host->private_data;
2075	void __iomem *port_mmio = ahci_port_base(ap);
2076	struct ata_device *dev = ap->link.device;
2077	u32 devslp, dm, dito, mdat, deto;
2078	int rc;
2079	unsigned int err_mask;
2080
2081	devslp = readl(port_mmio + PORT_DEVSLP);
2082	if (!(devslp & PORT_DEVSLP_DSP)) {
2083		dev_info(ap->host->dev, "port does not support device sleep\n");
2084		return;
2085	}
2086
2087	/* disable device sleep */
2088	if (!sleep) {
2089		if (devslp & PORT_DEVSLP_ADSE) {
2090			writel(devslp & ~PORT_DEVSLP_ADSE,
2091			       port_mmio + PORT_DEVSLP);
2092			err_mask = ata_dev_set_feature(dev,
2093						       SETFEATURES_SATA_DISABLE,
2094						       SATA_DEVSLP);
2095			if (err_mask && err_mask != AC_ERR_DEV)
2096				ata_dev_warn(dev, "failed to disable DEVSLP\n");
2097		}
2098		return;
2099	}
2100
2101	/* device sleep was already enabled */
2102	if (devslp & PORT_DEVSLP_ADSE)
 
 
 
 
 
 
 
2103		return;
2104
2105	/* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
2106	rc = ahci_stop_engine(ap);
2107	if (rc)
2108		return;
2109
2110	dm = (devslp & PORT_DEVSLP_DM_MASK) >> PORT_DEVSLP_DM_OFFSET;
2111	dito = devslp_idle_timeout / (dm + 1);
2112	if (dito > 0x3ff)
2113		dito = 0x3ff;
2114
2115	/* Use the nominal value 10 ms if the read MDAT is zero,
2116	 * the nominal value of DETO is 20 ms.
2117	 */
2118	if (dev->devslp_timing[ATA_LOG_DEVSLP_VALID] &
2119	    ATA_LOG_DEVSLP_VALID_MASK) {
2120		mdat = dev->devslp_timing[ATA_LOG_DEVSLP_MDAT] &
2121		       ATA_LOG_DEVSLP_MDAT_MASK;
2122		if (!mdat)
2123			mdat = 10;
2124		deto = dev->devslp_timing[ATA_LOG_DEVSLP_DETO];
2125		if (!deto)
2126			deto = 20;
2127	} else {
2128		mdat = 10;
2129		deto = 20;
2130	}
2131
 
 
2132	devslp |= ((dito << PORT_DEVSLP_DITO_OFFSET) |
2133		   (mdat << PORT_DEVSLP_MDAT_OFFSET) |
2134		   (deto << PORT_DEVSLP_DETO_OFFSET) |
2135		   PORT_DEVSLP_ADSE);
2136	writel(devslp, port_mmio + PORT_DEVSLP);
2137
2138	hpriv->start_engine(ap);
2139
2140	/* enable device sleep feature for the drive */
2141	err_mask = ata_dev_set_feature(dev,
2142				       SETFEATURES_SATA_ENABLE,
2143				       SATA_DEVSLP);
2144	if (err_mask && err_mask != AC_ERR_DEV)
2145		ata_dev_warn(dev, "failed to enable DEVSLP\n");
2146}
2147
2148static void ahci_enable_fbs(struct ata_port *ap)
2149{
2150	struct ahci_host_priv *hpriv = ap->host->private_data;
2151	struct ahci_port_priv *pp = ap->private_data;
2152	void __iomem *port_mmio = ahci_port_base(ap);
2153	u32 fbs;
2154	int rc;
2155
2156	if (!pp->fbs_supported)
2157		return;
2158
2159	fbs = readl(port_mmio + PORT_FBS);
2160	if (fbs & PORT_FBS_EN) {
2161		pp->fbs_enabled = true;
2162		pp->fbs_last_dev = -1; /* initialization */
2163		return;
2164	}
2165
2166	rc = ahci_stop_engine(ap);
2167	if (rc)
2168		return;
2169
2170	writel(fbs | PORT_FBS_EN, port_mmio + PORT_FBS);
2171	fbs = readl(port_mmio + PORT_FBS);
2172	if (fbs & PORT_FBS_EN) {
2173		dev_info(ap->host->dev, "FBS is enabled\n");
2174		pp->fbs_enabled = true;
2175		pp->fbs_last_dev = -1; /* initialization */
2176	} else
2177		dev_err(ap->host->dev, "Failed to enable FBS\n");
2178
2179	hpriv->start_engine(ap);
2180}
2181
2182static void ahci_disable_fbs(struct ata_port *ap)
2183{
2184	struct ahci_host_priv *hpriv = ap->host->private_data;
2185	struct ahci_port_priv *pp = ap->private_data;
2186	void __iomem *port_mmio = ahci_port_base(ap);
2187	u32 fbs;
2188	int rc;
2189
2190	if (!pp->fbs_supported)
2191		return;
2192
2193	fbs = readl(port_mmio + PORT_FBS);
2194	if ((fbs & PORT_FBS_EN) == 0) {
2195		pp->fbs_enabled = false;
2196		return;
2197	}
2198
2199	rc = ahci_stop_engine(ap);
2200	if (rc)
2201		return;
2202
2203	writel(fbs & ~PORT_FBS_EN, port_mmio + PORT_FBS);
2204	fbs = readl(port_mmio + PORT_FBS);
2205	if (fbs & PORT_FBS_EN)
2206		dev_err(ap->host->dev, "Failed to disable FBS\n");
2207	else {
2208		dev_info(ap->host->dev, "FBS is disabled\n");
2209		pp->fbs_enabled = false;
2210	}
2211
2212	hpriv->start_engine(ap);
2213}
2214
2215static void ahci_pmp_attach(struct ata_port *ap)
2216{
2217	void __iomem *port_mmio = ahci_port_base(ap);
2218	struct ahci_port_priv *pp = ap->private_data;
2219	u32 cmd;
2220
2221	cmd = readl(port_mmio + PORT_CMD);
2222	cmd |= PORT_CMD_PMP;
2223	writel(cmd, port_mmio + PORT_CMD);
2224
2225	ahci_enable_fbs(ap);
2226
2227	pp->intr_mask |= PORT_IRQ_BAD_PMP;
2228
2229	/*
2230	 * We must not change the port interrupt mask register if the
2231	 * port is marked frozen, the value in pp->intr_mask will be
2232	 * restored later when the port is thawed.
2233	 *
2234	 * Note that during initialization, the port is marked as
2235	 * frozen since the irq handler is not yet registered.
2236	 */
2237	if (!(ap->pflags & ATA_PFLAG_FROZEN))
2238		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2239}
2240
2241static void ahci_pmp_detach(struct ata_port *ap)
2242{
2243	void __iomem *port_mmio = ahci_port_base(ap);
2244	struct ahci_port_priv *pp = ap->private_data;
2245	u32 cmd;
2246
2247	ahci_disable_fbs(ap);
2248
2249	cmd = readl(port_mmio + PORT_CMD);
2250	cmd &= ~PORT_CMD_PMP;
2251	writel(cmd, port_mmio + PORT_CMD);
2252
2253	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
2254
2255	/* see comment above in ahci_pmp_attach() */
2256	if (!(ap->pflags & ATA_PFLAG_FROZEN))
2257		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2258}
2259
2260int ahci_port_resume(struct ata_port *ap)
2261{
2262	ahci_rpm_get_port(ap);
2263
2264	ahci_power_up(ap);
2265	ahci_start_port(ap);
2266
2267	if (sata_pmp_attached(ap))
2268		ahci_pmp_attach(ap);
2269	else
2270		ahci_pmp_detach(ap);
2271
2272	return 0;
2273}
2274EXPORT_SYMBOL_GPL(ahci_port_resume);
2275
2276#ifdef CONFIG_PM
2277static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
2278{
2279	const char *emsg = NULL;
2280	int rc;
2281
2282	rc = ahci_deinit_port(ap, &emsg);
2283	if (rc == 0)
2284		ahci_power_down(ap);
2285	else {
2286		ata_port_err(ap, "%s (%d)\n", emsg, rc);
2287		ata_port_freeze(ap);
2288	}
2289
2290	ahci_rpm_put_port(ap);
2291	return rc;
2292}
2293#endif
2294
2295static int ahci_port_start(struct ata_port *ap)
2296{
2297	struct ahci_host_priv *hpriv = ap->host->private_data;
2298	struct device *dev = ap->host->dev;
2299	struct ahci_port_priv *pp;
2300	void *mem;
2301	dma_addr_t mem_dma;
2302	size_t dma_sz, rx_fis_sz;
2303
2304	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
2305	if (!pp)
2306		return -ENOMEM;
2307
2308	if (ap->host->n_ports > 1) {
2309		pp->irq_desc = devm_kzalloc(dev, 8, GFP_KERNEL);
2310		if (!pp->irq_desc) {
2311			devm_kfree(dev, pp);
2312			return -ENOMEM;
2313		}
2314		snprintf(pp->irq_desc, 8,
2315			 "%s%d", dev_driver_string(dev), ap->port_no);
2316	}
2317
2318	/* check FBS capability */
2319	if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
2320		void __iomem *port_mmio = ahci_port_base(ap);
2321		u32 cmd = readl(port_mmio + PORT_CMD);
2322		if (cmd & PORT_CMD_FBSCP)
2323			pp->fbs_supported = true;
2324		else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {
2325			dev_info(dev, "port %d can do FBS, forcing FBSCP\n",
2326				 ap->port_no);
2327			pp->fbs_supported = true;
2328		} else
2329			dev_warn(dev, "port %d is not capable of FBS\n",
2330				 ap->port_no);
2331	}
2332
2333	if (pp->fbs_supported) {
2334		dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
2335		rx_fis_sz = AHCI_RX_FIS_SZ * 16;
2336	} else {
2337		dma_sz = AHCI_PORT_PRIV_DMA_SZ;
2338		rx_fis_sz = AHCI_RX_FIS_SZ;
2339	}
2340
2341	mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
2342	if (!mem)
2343		return -ENOMEM;
2344	memset(mem, 0, dma_sz);
2345
2346	/*
2347	 * First item in chunk of DMA memory: 32-slot command table,
2348	 * 32 bytes each in size
2349	 */
2350	pp->cmd_slot = mem;
2351	pp->cmd_slot_dma = mem_dma;
2352
2353	mem += AHCI_CMD_SLOT_SZ;
2354	mem_dma += AHCI_CMD_SLOT_SZ;
2355
2356	/*
2357	 * Second item: Received-FIS area
2358	 */
2359	pp->rx_fis = mem;
2360	pp->rx_fis_dma = mem_dma;
2361
2362	mem += rx_fis_sz;
2363	mem_dma += rx_fis_sz;
2364
2365	/*
2366	 * Third item: data area for storing a single command
2367	 * and its scatter-gather table
2368	 */
2369	pp->cmd_tbl = mem;
2370	pp->cmd_tbl_dma = mem_dma;
2371
2372	/*
2373	 * Save off initial list of interrupts to be enabled.
2374	 * This could be changed later
2375	 */
2376	pp->intr_mask = DEF_PORT_IRQ;
2377
2378	/*
2379	 * Switch to per-port locking in case each port has its own MSI vector.
2380	 */
2381	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
2382		spin_lock_init(&pp->lock);
2383		ap->lock = &pp->lock;
2384	}
2385
2386	ap->private_data = pp;
2387
2388	/* engage engines, captain */
2389	return ahci_port_resume(ap);
2390}
2391
2392static void ahci_port_stop(struct ata_port *ap)
2393{
2394	const char *emsg = NULL;
 
 
2395	int rc;
2396
2397	/* de-initialize port */
2398	rc = ahci_deinit_port(ap, &emsg);
2399	if (rc)
2400		ata_port_warn(ap, "%s (%d)\n", emsg, rc);
 
 
 
 
 
 
 
 
2401}
2402
2403void ahci_print_info(struct ata_host *host, const char *scc_s)
2404{
2405	struct ahci_host_priv *hpriv = host->private_data;
2406	u32 vers, cap, cap2, impl, speed;
2407	const char *speed_s;
2408
2409	vers = hpriv->version;
2410	cap = hpriv->cap;
2411	cap2 = hpriv->cap2;
2412	impl = hpriv->port_map;
2413
2414	speed = (cap >> 20) & 0xf;
2415	if (speed == 1)
2416		speed_s = "1.5";
2417	else if (speed == 2)
2418		speed_s = "3";
2419	else if (speed == 3)
2420		speed_s = "6";
2421	else
2422		speed_s = "?";
2423
2424	dev_info(host->dev,
2425		"AHCI %02x%02x.%02x%02x "
2426		"%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2427		,
2428
2429		(vers >> 24) & 0xff,
2430		(vers >> 16) & 0xff,
2431		(vers >> 8) & 0xff,
2432		vers & 0xff,
2433
2434		((cap >> 8) & 0x1f) + 1,
2435		(cap & 0x1f) + 1,
2436		speed_s,
2437		impl,
2438		scc_s);
2439
2440	dev_info(host->dev,
2441		"flags: "
2442		"%s%s%s%s%s%s%s"
2443		"%s%s%s%s%s%s%s"
2444		"%s%s%s%s%s%s%s"
2445		"%s%s\n"
2446		,
2447
2448		cap & HOST_CAP_64 ? "64bit " : "",
2449		cap & HOST_CAP_NCQ ? "ncq " : "",
2450		cap & HOST_CAP_SNTF ? "sntf " : "",
2451		cap & HOST_CAP_MPS ? "ilck " : "",
2452		cap & HOST_CAP_SSS ? "stag " : "",
2453		cap & HOST_CAP_ALPM ? "pm " : "",
2454		cap & HOST_CAP_LED ? "led " : "",
2455		cap & HOST_CAP_CLO ? "clo " : "",
2456		cap & HOST_CAP_ONLY ? "only " : "",
2457		cap & HOST_CAP_PMP ? "pmp " : "",
2458		cap & HOST_CAP_FBS ? "fbs " : "",
2459		cap & HOST_CAP_PIO_MULTI ? "pio " : "",
2460		cap & HOST_CAP_SSC ? "slum " : "",
2461		cap & HOST_CAP_PART ? "part " : "",
2462		cap & HOST_CAP_CCC ? "ccc " : "",
2463		cap & HOST_CAP_EMS ? "ems " : "",
2464		cap & HOST_CAP_SXS ? "sxs " : "",
2465		cap2 & HOST_CAP2_DESO ? "deso " : "",
2466		cap2 & HOST_CAP2_SADM ? "sadm " : "",
2467		cap2 & HOST_CAP2_SDS ? "sds " : "",
2468		cap2 & HOST_CAP2_APST ? "apst " : "",
2469		cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
2470		cap2 & HOST_CAP2_BOH ? "boh " : ""
2471		);
2472}
2473EXPORT_SYMBOL_GPL(ahci_print_info);
2474
2475void ahci_set_em_messages(struct ahci_host_priv *hpriv,
2476			  struct ata_port_info *pi)
2477{
2478	u8 messages;
2479	void __iomem *mmio = hpriv->mmio;
2480	u32 em_loc = readl(mmio + HOST_EM_LOC);
2481	u32 em_ctl = readl(mmio + HOST_EM_CTL);
2482
2483	if (!ahci_em_messages || !(hpriv->cap & HOST_CAP_EMS))
2484		return;
2485
2486	messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
2487
2488	if (messages) {
2489		/* store em_loc */
2490		hpriv->em_loc = ((em_loc >> 16) * 4);
2491		hpriv->em_buf_sz = ((em_loc & 0xff) * 4);
2492		hpriv->em_msg_type = messages;
2493		pi->flags |= ATA_FLAG_EM;
2494		if (!(em_ctl & EM_CTL_ALHD))
2495			pi->flags |= ATA_FLAG_SW_ACTIVITY;
2496	}
2497}
2498EXPORT_SYMBOL_GPL(ahci_set_em_messages);
2499
2500static int ahci_host_activate_multi_irqs(struct ata_host *host,
2501					 struct scsi_host_template *sht)
2502{
2503	struct ahci_host_priv *hpriv = host->private_data;
2504	int i, rc;
2505
2506	rc = ata_host_start(host);
2507	if (rc)
2508		return rc;
2509	/*
2510	 * Requests IRQs according to AHCI-1.1 when multiple MSIs were
2511	 * allocated. That is one MSI per port, starting from @irq.
2512	 */
2513	for (i = 0; i < host->n_ports; i++) {
2514		struct ahci_port_priv *pp = host->ports[i]->private_data;
2515		int irq = ahci_irq_vector(hpriv, i);
2516
2517		/* Do not receive interrupts sent by dummy ports */
2518		if (!pp) {
2519			disable_irq(irq + i);
2520			continue;
2521		}
2522
2523		rc = devm_request_irq(host->dev, irq, ahci_multi_irqs_intr_hard,
2524				0, pp->irq_desc, host->ports[i]);
2525
2526		if (rc)
2527			return rc;
2528		ata_port_desc(host->ports[i], "irq %d", irq);
2529	}
2530
2531	return ata_host_register(host, sht);
2532}
2533
2534/**
2535 *	ahci_host_activate - start AHCI host, request IRQs and register it
2536 *	@host: target ATA host
2537 *	@sht: scsi_host_template to use when registering the host
2538 *
2539 *	LOCKING:
2540 *	Inherited from calling layer (may sleep).
2541 *
2542 *	RETURNS:
2543 *	0 on success, -errno otherwise.
2544 */
2545int ahci_host_activate(struct ata_host *host, struct scsi_host_template *sht)
2546{
2547	struct ahci_host_priv *hpriv = host->private_data;
2548	int irq = hpriv->irq;
2549	int rc;
2550
2551	if (hpriv->flags & (AHCI_HFLAG_MULTI_MSI | AHCI_HFLAG_MULTI_MSIX)) {
2552		if (hpriv->irq_handler)
2553			dev_warn(host->dev, "both AHCI_HFLAG_MULTI_MSI flag set \
2554				 and custom irq handler implemented\n");
 
 
 
 
 
 
2555
2556		rc = ahci_host_activate_multi_irqs(host, sht);
2557	} else {
2558		rc = ata_host_activate(host, irq, hpriv->irq_handler,
2559				       IRQF_SHARED, sht);
2560	}
2561
2562
2563	return rc;
2564}
2565EXPORT_SYMBOL_GPL(ahci_host_activate);
2566
2567MODULE_AUTHOR("Jeff Garzik");
2568MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2569MODULE_LICENSE("GPL");