1/*
   2 *  libahci.c - Common AHCI SATA low-level routines
   3 *
   4 *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
   5 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
   6 *		    on emails.
   7 *
   8 *  Copyright 2004-2005 Red Hat, Inc.
   9 *
  10 *
  11 *  This program is free software; you can redistribute it and/or modify
  12 *  it under the terms of the GNU General Public License as published by
  13 *  the Free Software Foundation; either version 2, or (at your option)
  14 *  any later version.
  15 *
  16 *  This program is distributed in the hope that it will be useful,
  17 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 *  GNU General Public License for more details.
  20 *
  21 *  You should have received a copy of the GNU General Public License
  22 *  along with this program; see the file COPYING.  If not, write to
  23 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  24 *
  25 *
  26 * libata documentation is available via 'make {ps|pdf}docs',
  27 * as Documentation/DocBook/libata.*
  28 *
  29 * AHCI hardware documentation:
  30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
  31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
  32 *
  33 */
  34
  35#include <linux/kernel.h>
  36#include <linux/gfp.h>
  37#include <linux/module.h>
  38#include <linux/init.h>
  39#include <linux/blkdev.h>
  40#include <linux/delay.h>
  41#include <linux/interrupt.h>
  42#include <linux/dma-mapping.h>
  43#include <linux/device.h>
  44#include <scsi/scsi_host.h>
  45#include <scsi/scsi_cmnd.h>
  46#include <linux/libata.h>
 
  47#include "ahci.h"
 
  48
  49static int ahci_skip_host_reset;
  50int ahci_ignore_sss;
  51EXPORT_SYMBOL_GPL(ahci_ignore_sss);
  52
  53module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
  54MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
  55
  56module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
  57MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
  58
  59static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
  60			unsigned hints);
  61static ssize_t ahci_led_show(struct ata_port *ap, char *buf);
  62static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
  63			      size_t size);
  64static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
  65					ssize_t size);
  66
  67
  68
  69static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
  70static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
  71static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
  72static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
  73static int ahci_port_start(struct ata_port *ap);
  74static void ahci_port_stop(struct ata_port *ap);
  75static void ahci_qc_prep(struct ata_queued_cmd *qc);
  76static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
  77static void ahci_freeze(struct ata_port *ap);
  78static void ahci_thaw(struct ata_port *ap);
 
  79static void ahci_enable_fbs(struct ata_port *ap);
  80static void ahci_disable_fbs(struct ata_port *ap);
  81static void ahci_pmp_attach(struct ata_port *ap);
  82static void ahci_pmp_detach(struct ata_port *ap);
  83static int ahci_softreset(struct ata_link *link, unsigned int *class,
  84			  unsigned long deadline);
  85static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
  86			  unsigned long deadline);
  87static int ahci_hardreset(struct ata_link *link, unsigned int *class,
  88			  unsigned long deadline);
  89static void ahci_postreset(struct ata_link *link, unsigned int *class);
  90static void ahci_error_handler(struct ata_port *ap);
  91static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
  92static void ahci_dev_config(struct ata_device *dev);
  93#ifdef CONFIG_PM
  94static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
  95#endif
  96static ssize_t ahci_activity_show(struct ata_device *dev, char *buf);
  97static ssize_t ahci_activity_store(struct ata_device *dev,
  98				   enum sw_activity val);
  99static void ahci_init_sw_activity(struct ata_link *link);
 100
 101static ssize_t ahci_show_host_caps(struct device *dev,
 102				   struct device_attribute *attr, char *buf);
 103static ssize_t ahci_show_host_cap2(struct device *dev,
 104				   struct device_attribute *attr, char *buf);
 105static ssize_t ahci_show_host_version(struct device *dev,
 106				      struct device_attribute *attr, char *buf);
 107static ssize_t ahci_show_port_cmd(struct device *dev,
 108				  struct device_attribute *attr, char *buf);
 109static ssize_t ahci_read_em_buffer(struct device *dev,
 110				   struct device_attribute *attr, char *buf);
 111static ssize_t ahci_store_em_buffer(struct device *dev,
 112				    struct device_attribute *attr,
 113				    const char *buf, size_t size);
 114static ssize_t ahci_show_em_supported(struct device *dev,
 115				      struct device_attribute *attr, char *buf);
 
 116
 117static DEVICE_ATTR(ahci_host_caps, S_IRUGO, ahci_show_host_caps, NULL);
 118static DEVICE_ATTR(ahci_host_cap2, S_IRUGO, ahci_show_host_cap2, NULL);
 119static DEVICE_ATTR(ahci_host_version, S_IRUGO, ahci_show_host_version, NULL);
 120static DEVICE_ATTR(ahci_port_cmd, S_IRUGO, ahci_show_port_cmd, NULL);
 121static DEVICE_ATTR(em_buffer, S_IWUSR | S_IRUGO,
 122		   ahci_read_em_buffer, ahci_store_em_buffer);
 123static DEVICE_ATTR(em_message_supported, S_IRUGO, ahci_show_em_supported, NULL);
 124
 125struct device_attribute *ahci_shost_attrs[] = {
 126	&dev_attr_link_power_management_policy,
 127	&dev_attr_em_message_type,
 128	&dev_attr_em_message,
 129	&dev_attr_ahci_host_caps,
 130	&dev_attr_ahci_host_cap2,
 131	&dev_attr_ahci_host_version,
 132	&dev_attr_ahci_port_cmd,
 133	&dev_attr_em_buffer,
 134	&dev_attr_em_message_supported,
 135	NULL
 136};
 137EXPORT_SYMBOL_GPL(ahci_shost_attrs);
 138
 139struct device_attribute *ahci_sdev_attrs[] = {
 140	&dev_attr_sw_activity,
 141	&dev_attr_unload_heads,
 
 142	NULL
 143};
 144EXPORT_SYMBOL_GPL(ahci_sdev_attrs);
 145
 146struct ata_port_operations ahci_ops = {
 147	.inherits		= &sata_pmp_port_ops,
 148
 149	.qc_defer		= ahci_pmp_qc_defer,
 150	.qc_prep		= ahci_qc_prep,
 151	.qc_issue		= ahci_qc_issue,
 152	.qc_fill_rtf		= ahci_qc_fill_rtf,
 153
 154	.freeze			= ahci_freeze,
 155	.thaw			= ahci_thaw,
 156	.softreset		= ahci_softreset,
 157	.hardreset		= ahci_hardreset,
 158	.postreset		= ahci_postreset,
 159	.pmp_softreset		= ahci_softreset,
 160	.error_handler		= ahci_error_handler,
 161	.post_internal_cmd	= ahci_post_internal_cmd,
 162	.dev_config		= ahci_dev_config,
 163
 164	.scr_read		= ahci_scr_read,
 165	.scr_write		= ahci_scr_write,
 166	.pmp_attach		= ahci_pmp_attach,
 167	.pmp_detach		= ahci_pmp_detach,
 168
 169	.set_lpm		= ahci_set_lpm,
 170	.em_show		= ahci_led_show,
 171	.em_store		= ahci_led_store,
 172	.sw_activity_show	= ahci_activity_show,
 173	.sw_activity_store	= ahci_activity_store,
 
 174#ifdef CONFIG_PM
 175	.port_suspend		= ahci_port_suspend,
 176	.port_resume		= ahci_port_resume,
 177#endif
 178	.port_start		= ahci_port_start,
 179	.port_stop		= ahci_port_stop,
 180};
 181EXPORT_SYMBOL_GPL(ahci_ops);
 182
 183struct ata_port_operations ahci_pmp_retry_srst_ops = {
 184	.inherits		= &ahci_ops,
 185	.softreset		= ahci_pmp_retry_softreset,
 186};
 187EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
 188
 189int ahci_em_messages = 1;
 190EXPORT_SYMBOL_GPL(ahci_em_messages);
 191module_param(ahci_em_messages, int, 0444);
 192/* add other LED protocol types when they become supported */
 193MODULE_PARM_DESC(ahci_em_messages,
 194	"AHCI Enclosure Management Message control (0 = off, 1 = on)");
 195
 
 
 
 
 
 196static void ahci_enable_ahci(void __iomem *mmio)
 197{
 198	int i;
 199	u32 tmp;
 200
 201	/* turn on AHCI_EN */
 202	tmp = readl(mmio + HOST_CTL);
 203	if (tmp & HOST_AHCI_EN)
 204		return;
 205
 206	/* Some controllers need AHCI_EN to be written multiple times.
 207	 * Try a few times before giving up.
 208	 */
 209	for (i = 0; i < 5; i++) {
 210		tmp |= HOST_AHCI_EN;
 211		writel(tmp, mmio + HOST_CTL);
 212		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
 213		if (tmp & HOST_AHCI_EN)
 214			return;
 215		msleep(10);
 216	}
 217
 218	WARN_ON(1);
 219}
 220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 221static ssize_t ahci_show_host_caps(struct device *dev,
 222				   struct device_attribute *attr, char *buf)
 223{
 224	struct Scsi_Host *shost = class_to_shost(dev);
 225	struct ata_port *ap = ata_shost_to_port(shost);
 226	struct ahci_host_priv *hpriv = ap->host->private_data;
 227
 228	return sprintf(buf, "%x\n", hpriv->cap);
 229}
 230
 231static ssize_t ahci_show_host_cap2(struct device *dev,
 232				   struct device_attribute *attr, char *buf)
 233{
 234	struct Scsi_Host *shost = class_to_shost(dev);
 235	struct ata_port *ap = ata_shost_to_port(shost);
 236	struct ahci_host_priv *hpriv = ap->host->private_data;
 237
 238	return sprintf(buf, "%x\n", hpriv->cap2);
 239}
 240
 241static ssize_t ahci_show_host_version(struct device *dev,
 242				   struct device_attribute *attr, char *buf)
 243{
 244	struct Scsi_Host *shost = class_to_shost(dev);
 245	struct ata_port *ap = ata_shost_to_port(shost);
 246	struct ahci_host_priv *hpriv = ap->host->private_data;
 247	void __iomem *mmio = hpriv->mmio;
 248
 249	return sprintf(buf, "%x\n", readl(mmio + HOST_VERSION));
 250}
 251
 252static ssize_t ahci_show_port_cmd(struct device *dev,
 253				  struct device_attribute *attr, char *buf)
 254{
 255	struct Scsi_Host *shost = class_to_shost(dev);
 256	struct ata_port *ap = ata_shost_to_port(shost);
 257	void __iomem *port_mmio = ahci_port_base(ap);
 
 
 
 
 
 258
 259	return sprintf(buf, "%x\n", readl(port_mmio + PORT_CMD));
 260}
 261
 262static ssize_t ahci_read_em_buffer(struct device *dev,
 263				   struct device_attribute *attr, char *buf)
 264{
 265	struct Scsi_Host *shost = class_to_shost(dev);
 266	struct ata_port *ap = ata_shost_to_port(shost);
 267	struct ahci_host_priv *hpriv = ap->host->private_data;
 268	void __iomem *mmio = hpriv->mmio;
 269	void __iomem *em_mmio = mmio + hpriv->em_loc;
 270	u32 em_ctl, msg;
 271	unsigned long flags;
 272	size_t count;
 273	int i;
 274
 
 275	spin_lock_irqsave(ap->lock, flags);
 276
 277	em_ctl = readl(mmio + HOST_EM_CTL);
 278	if (!(ap->flags & ATA_FLAG_EM) || em_ctl & EM_CTL_XMT ||
 279	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {
 280		spin_unlock_irqrestore(ap->lock, flags);
 
 281		return -EINVAL;
 282	}
 283
 284	if (!(em_ctl & EM_CTL_MR)) {
 285		spin_unlock_irqrestore(ap->lock, flags);
 
 286		return -EAGAIN;
 287	}
 288
 289	if (!(em_ctl & EM_CTL_SMB))
 290		em_mmio += hpriv->em_buf_sz;
 291
 292	count = hpriv->em_buf_sz;
 293
 294	/* the count should not be larger than PAGE_SIZE */
 295	if (count > PAGE_SIZE) {
 296		if (printk_ratelimit())
 297			ata_port_warn(ap,
 298				      "EM read buffer size too large: "
 299				      "buffer size %u, page size %lu\n",
 300				      hpriv->em_buf_sz, PAGE_SIZE);
 301		count = PAGE_SIZE;
 302	}
 303
 304	for (i = 0; i < count; i += 4) {
 305		msg = readl(em_mmio + i);
 306		buf[i] = msg & 0xff;
 307		buf[i + 1] = (msg >> 8) & 0xff;
 308		buf[i + 2] = (msg >> 16) & 0xff;
 309		buf[i + 3] = (msg >> 24) & 0xff;
 310	}
 311
 312	spin_unlock_irqrestore(ap->lock, flags);
 
 313
 314	return i;
 315}
 316
 317static ssize_t ahci_store_em_buffer(struct device *dev,
 318				    struct device_attribute *attr,
 319				    const char *buf, size_t size)
 320{
 321	struct Scsi_Host *shost = class_to_shost(dev);
 322	struct ata_port *ap = ata_shost_to_port(shost);
 323	struct ahci_host_priv *hpriv = ap->host->private_data;
 324	void __iomem *mmio = hpriv->mmio;
 325	void __iomem *em_mmio = mmio + hpriv->em_loc;
 326	const unsigned char *msg_buf = buf;
 327	u32 em_ctl, msg;
 328	unsigned long flags;
 329	int i;
 330
 331	/* check size validity */
 332	if (!(ap->flags & ATA_FLAG_EM) ||
 333	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) ||
 334	    size % 4 || size > hpriv->em_buf_sz)
 335		return -EINVAL;
 336
 
 337	spin_lock_irqsave(ap->lock, flags);
 338
 339	em_ctl = readl(mmio + HOST_EM_CTL);
 340	if (em_ctl & EM_CTL_TM) {
 341		spin_unlock_irqrestore(ap->lock, flags);
 
 342		return -EBUSY;
 343	}
 344
 345	for (i = 0; i < size; i += 4) {
 346		msg = msg_buf[i] | msg_buf[i + 1] << 8 |
 347		      msg_buf[i + 2] << 16 | msg_buf[i + 3] << 24;
 348		writel(msg, em_mmio + i);
 349	}
 350
 351	writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
 352
 353	spin_unlock_irqrestore(ap->lock, flags);
 
 354
 355	return size;
 356}
 357
 358static ssize_t ahci_show_em_supported(struct device *dev,
 359				      struct device_attribute *attr, char *buf)
 360{
 361	struct Scsi_Host *shost = class_to_shost(dev);
 362	struct ata_port *ap = ata_shost_to_port(shost);
 363	struct ahci_host_priv *hpriv = ap->host->private_data;
 364	void __iomem *mmio = hpriv->mmio;
 365	u32 em_ctl;
 366
 
 367	em_ctl = readl(mmio + HOST_EM_CTL);
 
 368
 369	return sprintf(buf, "%s%s%s%s\n",
 370		       em_ctl & EM_CTL_LED ? "led " : "",
 371		       em_ctl & EM_CTL_SAFTE ? "saf-te " : "",
 372		       em_ctl & EM_CTL_SES ? "ses-2 " : "",
 373		       em_ctl & EM_CTL_SGPIO ? "sgpio " : "");
 374}
 375
 376/**
 377 *	ahci_save_initial_config - Save and fixup initial config values
 378 *	@dev: target AHCI device
 379 *	@hpriv: host private area to store config values
 380 *	@force_port_map: force port map to a specified value
 381 *	@mask_port_map: mask out particular bits from port map
 382 *
 383 *	Some registers containing configuration info might be setup by
 384 *	BIOS and might be cleared on reset.  This function saves the
 385 *	initial values of those registers into @hpriv such that they
 386 *	can be restored after controller reset.
 387 *
 388 *	If inconsistent, config values are fixed up by this function.
 389 *
 
 
 
 390 *	LOCKING:
 391 *	None.
 392 */
 393void ahci_save_initial_config(struct device *dev,
 394			      struct ahci_host_priv *hpriv,
 395			      unsigned int force_port_map,
 396			      unsigned int mask_port_map)
 397{
 398	void __iomem *mmio = hpriv->mmio;
 399	u32 cap, cap2, vers, port_map;
 400	int i;
 401
 402	/* make sure AHCI mode is enabled before accessing CAP */
 403	ahci_enable_ahci(mmio);
 404
 405	/* Values prefixed with saved_ are written back to host after
 406	 * reset.  Values without are used for driver operation.
 407	 */
 408	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
 409	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
 410
 411	/* CAP2 register is only defined for AHCI 1.2 and later */
 412	vers = readl(mmio + HOST_VERSION);
 413	if ((vers >> 16) > 1 ||
 414	   ((vers >> 16) == 1 && (vers & 0xFFFF) >= 0x200))
 415		hpriv->saved_cap2 = cap2 = readl(mmio + HOST_CAP2);
 416	else
 417		hpriv->saved_cap2 = cap2 = 0;
 418
 419	/* some chips have errata preventing 64bit use */
 420	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
 421		dev_info(dev, "controller can't do 64bit DMA, forcing 32bit\n");
 422		cap &= ~HOST_CAP_64;
 423	}
 424
 425	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
 426		dev_info(dev, "controller can't do NCQ, turning off CAP_NCQ\n");
 427		cap &= ~HOST_CAP_NCQ;
 428	}
 429
 430	if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
 431		dev_info(dev, "controller can do NCQ, turning on CAP_NCQ\n");
 432		cap |= HOST_CAP_NCQ;
 433	}
 434
 435	if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
 436		dev_info(dev, "controller can't do PMP, turning off CAP_PMP\n");
 437		cap &= ~HOST_CAP_PMP;
 438	}
 439
 440	if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {
 441		dev_info(dev,
 442			 "controller can't do SNTF, turning off CAP_SNTF\n");
 443		cap &= ~HOST_CAP_SNTF;
 444	}
 445
 
 
 
 
 
 
 
 446	if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {
 447		dev_info(dev, "controller can do FBS, turning on CAP_FBS\n");
 448		cap |= HOST_CAP_FBS;
 449	}
 450
 451	if (force_port_map && port_map != force_port_map) {
 
 
 
 
 
 
 
 
 
 
 452		dev_info(dev, "forcing port_map 0x%x -> 0x%x\n",
 453			 port_map, force_port_map);
 454		port_map = force_port_map;
 
 455	}
 456
 457	if (mask_port_map) {
 458		dev_warn(dev, "masking port_map 0x%x -> 0x%x\n",
 459			port_map,
 460			port_map & mask_port_map);
 461		port_map &= mask_port_map;
 462	}
 463
 464	/* cross check port_map and cap.n_ports */
 465	if (port_map) {
 466		int map_ports = 0;
 467
 468		for (i = 0; i < AHCI_MAX_PORTS; i++)
 469			if (port_map & (1 << i))
 470				map_ports++;
 471
 472		/* If PI has more ports than n_ports, whine, clear
 473		 * port_map and let it be generated from n_ports.
 474		 */
 475		if (map_ports > ahci_nr_ports(cap)) {
 476			dev_warn(dev,
 477				 "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
 478				 port_map, ahci_nr_ports(cap));
 479			port_map = 0;
 480		}
 481	}
 482
 483	/* fabricate port_map from cap.nr_ports */
 484	if (!port_map) {
 485		port_map = (1 << ahci_nr_ports(cap)) - 1;
 486		dev_warn(dev, "forcing PORTS_IMPL to 0x%x\n", port_map);
 487
 488		/* write the fixed up value to the PI register */
 489		hpriv->saved_port_map = port_map;
 490	}
 491
 492	/* record values to use during operation */
 493	hpriv->cap = cap;
 494	hpriv->cap2 = cap2;
 
 495	hpriv->port_map = port_map;
 
 
 
 
 
 
 
 
 
 496}
 497EXPORT_SYMBOL_GPL(ahci_save_initial_config);
 498
 499/**
 500 *	ahci_restore_initial_config - Restore initial config
 501 *	@host: target ATA host
 502 *
 503 *	Restore initial config stored by ahci_save_initial_config().
 504 *
 505 *	LOCKING:
 506 *	None.
 507 */
 508static void ahci_restore_initial_config(struct ata_host *host)
 509{
 510	struct ahci_host_priv *hpriv = host->private_data;
 511	void __iomem *mmio = hpriv->mmio;
 512
 513	writel(hpriv->saved_cap, mmio + HOST_CAP);
 514	if (hpriv->saved_cap2)
 515		writel(hpriv->saved_cap2, mmio + HOST_CAP2);
 516	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
 517	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
 518}
 519
 520static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
 521{
 522	static const int offset[] = {
 523		[SCR_STATUS]		= PORT_SCR_STAT,
 524		[SCR_CONTROL]		= PORT_SCR_CTL,
 525		[SCR_ERROR]		= PORT_SCR_ERR,
 526		[SCR_ACTIVE]		= PORT_SCR_ACT,
 527		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
 528	};
 529	struct ahci_host_priv *hpriv = ap->host->private_data;
 530
 531	if (sc_reg < ARRAY_SIZE(offset) &&
 532	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
 533		return offset[sc_reg];
 534	return 0;
 535}
 536
 537static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
 538{
 539	void __iomem *port_mmio = ahci_port_base(link->ap);
 540	int offset = ahci_scr_offset(link->ap, sc_reg);
 541
 542	if (offset) {
 543		*val = readl(port_mmio + offset);
 544		return 0;
 545	}
 546	return -EINVAL;
 547}
 548
 549static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
 550{
 551	void __iomem *port_mmio = ahci_port_base(link->ap);
 552	int offset = ahci_scr_offset(link->ap, sc_reg);
 553
 554	if (offset) {
 555		writel(val, port_mmio + offset);
 556		return 0;
 557	}
 558	return -EINVAL;
 559}
 560
 561void ahci_start_engine(struct ata_port *ap)
 562{
 563	void __iomem *port_mmio = ahci_port_base(ap);
 564	u32 tmp;
 565
 566	/* start DMA */
 567	tmp = readl(port_mmio + PORT_CMD);
 568	tmp |= PORT_CMD_START;
 569	writel(tmp, port_mmio + PORT_CMD);
 570	readl(port_mmio + PORT_CMD); /* flush */
 571}
 572EXPORT_SYMBOL_GPL(ahci_start_engine);
 573
 574int ahci_stop_engine(struct ata_port *ap)
 575{
 576	void __iomem *port_mmio = ahci_port_base(ap);
 
 577	u32 tmp;
 578
 
 
 
 
 
 
 
 
 
 
 
 
 
 579	tmp = readl(port_mmio + PORT_CMD);
 580
 581	/* check if the HBA is idle */
 582	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
 583		return 0;
 584
 
 
 
 
 
 
 
 
 
 
 585	/* setting HBA to idle */
 586	tmp &= ~PORT_CMD_START;
 587	writel(tmp, port_mmio + PORT_CMD);
 588
 589	/* wait for engine to stop. This could be as long as 500 msec */
 590	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
 591				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
 592	if (tmp & PORT_CMD_LIST_ON)
 593		return -EIO;
 594
 595	return 0;
 596}
 597EXPORT_SYMBOL_GPL(ahci_stop_engine);
 598
 599static void ahci_start_fis_rx(struct ata_port *ap)
 600{
 601	void __iomem *port_mmio = ahci_port_base(ap);
 602	struct ahci_host_priv *hpriv = ap->host->private_data;
 603	struct ahci_port_priv *pp = ap->private_data;
 604	u32 tmp;
 605
 606	/* set FIS registers */
 607	if (hpriv->cap & HOST_CAP_64)
 608		writel((pp->cmd_slot_dma >> 16) >> 16,
 609		       port_mmio + PORT_LST_ADDR_HI);
 610	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
 611
 612	if (hpriv->cap & HOST_CAP_64)
 613		writel((pp->rx_fis_dma >> 16) >> 16,
 614		       port_mmio + PORT_FIS_ADDR_HI);
 615	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
 616
 617	/* enable FIS reception */
 618	tmp = readl(port_mmio + PORT_CMD);
 619	tmp |= PORT_CMD_FIS_RX;
 620	writel(tmp, port_mmio + PORT_CMD);
 621
 622	/* flush */
 623	readl(port_mmio + PORT_CMD);
 624}
 
 625
 626static int ahci_stop_fis_rx(struct ata_port *ap)
 627{
 628	void __iomem *port_mmio = ahci_port_base(ap);
 629	u32 tmp;
 630
 631	/* disable FIS reception */
 632	tmp = readl(port_mmio + PORT_CMD);
 633	tmp &= ~PORT_CMD_FIS_RX;
 634	writel(tmp, port_mmio + PORT_CMD);
 635
 636	/* wait for completion, spec says 500ms, give it 1000 */
 637	tmp = ata_wait_register(ap, port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
 638				PORT_CMD_FIS_ON, 10, 1000);
 639	if (tmp & PORT_CMD_FIS_ON)
 640		return -EBUSY;
 641
 642	return 0;
 643}
 644
 645static void ahci_power_up(struct ata_port *ap)
 646{
 647	struct ahci_host_priv *hpriv = ap->host->private_data;
 648	void __iomem *port_mmio = ahci_port_base(ap);
 649	u32 cmd;
 650
 651	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 652
 653	/* spin up device */
 654	if (hpriv->cap & HOST_CAP_SSS) {
 655		cmd |= PORT_CMD_SPIN_UP;
 656		writel(cmd, port_mmio + PORT_CMD);
 657	}
 658
 659	/* wake up link */
 660	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
 661}
 662
 663static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
 664			unsigned int hints)
 665{
 666	struct ata_port *ap = link->ap;
 667	struct ahci_host_priv *hpriv = ap->host->private_data;
 668	struct ahci_port_priv *pp = ap->private_data;
 669	void __iomem *port_mmio = ahci_port_base(ap);
 670
 671	if (policy != ATA_LPM_MAX_POWER) {
 
 
 
 672		/*
 673		 * Disable interrupts on Phy Ready. This keeps us from
 674		 * getting woken up due to spurious phy ready
 675		 * interrupts.
 676		 */
 677		pp->intr_mask &= ~PORT_IRQ_PHYRDY;
 678		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 679
 680		sata_link_scr_lpm(link, policy, false);
 681	}
 682
 683	if (hpriv->cap & HOST_CAP_ALPM) {
 684		u32 cmd = readl(port_mmio + PORT_CMD);
 685
 686		if (policy == ATA_LPM_MAX_POWER || !(hints & ATA_LPM_HIPM)) {
 687			cmd &= ~(PORT_CMD_ASP | PORT_CMD_ALPE);
 
 688			cmd |= PORT_CMD_ICC_ACTIVE;
 689
 690			writel(cmd, port_mmio + PORT_CMD);
 691			readl(port_mmio + PORT_CMD);
 692
 693			/* wait 10ms to be sure we've come out of LPM state */
 694			ata_msleep(ap, 10);
 
 
 
 695		} else {
 696			cmd |= PORT_CMD_ALPE;
 697			if (policy == ATA_LPM_MIN_POWER)
 698				cmd |= PORT_CMD_ASP;
 699
 700			/* write out new cmd value */
 701			writel(cmd, port_mmio + PORT_CMD);
 702		}
 703	}
 704
 
 
 
 
 
 
 
 
 
 
 705	if (policy == ATA_LPM_MAX_POWER) {
 706		sata_link_scr_lpm(link, policy, false);
 707
 708		/* turn PHYRDY IRQ back on */
 709		pp->intr_mask |= PORT_IRQ_PHYRDY;
 710		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 711	}
 712
 713	return 0;
 714}
 715
 716#ifdef CONFIG_PM
 717static void ahci_power_down(struct ata_port *ap)
 718{
 719	struct ahci_host_priv *hpriv = ap->host->private_data;
 720	void __iomem *port_mmio = ahci_port_base(ap);
 721	u32 cmd, scontrol;
 722
 723	if (!(hpriv->cap & HOST_CAP_SSS))
 724		return;
 725
 726	/* put device into listen mode, first set PxSCTL.DET to 0 */
 727	scontrol = readl(port_mmio + PORT_SCR_CTL);
 728	scontrol &= ~0xf;
 729	writel(scontrol, port_mmio + PORT_SCR_CTL);
 730
 731	/* then set PxCMD.SUD to 0 */
 732	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 733	cmd &= ~PORT_CMD_SPIN_UP;
 734	writel(cmd, port_mmio + PORT_CMD);
 735}
 736#endif
 737
 738static void ahci_start_port(struct ata_port *ap)
 739{
 740	struct ahci_host_priv *hpriv = ap->host->private_data;
 741	struct ahci_port_priv *pp = ap->private_data;
 742	struct ata_link *link;
 743	struct ahci_em_priv *emp;
 744	ssize_t rc;
 745	int i;
 746
 747	/* enable FIS reception */
 748	ahci_start_fis_rx(ap);
 749
 750	/* enable DMA */
 751	if (!(hpriv->flags & AHCI_HFLAG_DELAY_ENGINE))
 752		ahci_start_engine(ap);
 753
 754	/* turn on LEDs */
 755	if (ap->flags & ATA_FLAG_EM) {
 756		ata_for_each_link(link, ap, EDGE) {
 757			emp = &pp->em_priv[link->pmp];
 758
 759			/* EM Transmit bit maybe busy during init */
 760			for (i = 0; i < EM_MAX_RETRY; i++) {
 761				rc = ahci_transmit_led_message(ap,
 762							       emp->led_state,
 763							       4);
 
 
 
 
 
 
 
 
 764				if (rc == -EBUSY)
 765					ata_msleep(ap, 1);
 766				else
 767					break;
 768			}
 769		}
 770	}
 771
 772	if (ap->flags & ATA_FLAG_SW_ACTIVITY)
 773		ata_for_each_link(link, ap, EDGE)
 774			ahci_init_sw_activity(link);
 775
 776}
 777
 778static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
 779{
 780	int rc;
 
 781
 782	/* disable DMA */
 783	rc = ahci_stop_engine(ap);
 784	if (rc) {
 785		*emsg = "failed to stop engine";
 786		return rc;
 787	}
 788
 789	/* disable FIS reception */
 790	rc = ahci_stop_fis_rx(ap);
 791	if (rc) {
 792		*emsg = "failed stop FIS RX";
 793		return rc;
 794	}
 795
 796	return 0;
 797}
 798
 799int ahci_reset_controller(struct ata_host *host)
 800{
 801	struct ahci_host_priv *hpriv = host->private_data;
 802	void __iomem *mmio = hpriv->mmio;
 803	u32 tmp;
 804
 805	/* we must be in AHCI mode, before using anything
 806	 * AHCI-specific, such as HOST_RESET.
 807	 */
 808	ahci_enable_ahci(mmio);
 809
 810	/* global controller reset */
 811	if (!ahci_skip_host_reset) {
 812		tmp = readl(mmio + HOST_CTL);
 813		if ((tmp & HOST_RESET) == 0) {
 814			writel(tmp | HOST_RESET, mmio + HOST_CTL);
 815			readl(mmio + HOST_CTL); /* flush */
 816		}
 817
 818		/*
 819		 * to perform host reset, OS should set HOST_RESET
 820		 * and poll until this bit is read to be "0".
 821		 * reset must complete within 1 second, or
 822		 * the hardware should be considered fried.
 823		 */
 824		tmp = ata_wait_register(NULL, mmio + HOST_CTL, HOST_RESET,
 825					HOST_RESET, 10, 1000);
 826
 827		if (tmp & HOST_RESET) {
 828			dev_err(host->dev, "controller reset failed (0x%x)\n",
 829				tmp);
 830			return -EIO;
 831		}
 832
 833		/* turn on AHCI mode */
 834		ahci_enable_ahci(mmio);
 835
 836		/* Some registers might be cleared on reset.  Restore
 837		 * initial values.
 838		 */
 839		ahci_restore_initial_config(host);
 
 840	} else
 841		dev_info(host->dev, "skipping global host reset\n");
 842
 843	return 0;
 844}
 845EXPORT_SYMBOL_GPL(ahci_reset_controller);
 846
 847static void ahci_sw_activity(struct ata_link *link)
 848{
 849	struct ata_port *ap = link->ap;
 850	struct ahci_port_priv *pp = ap->private_data;
 851	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 852
 853	if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
 854		return;
 855
 856	emp->activity++;
 857	if (!timer_pending(&emp->timer))
 858		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));
 859}
 860
 861static void ahci_sw_activity_blink(unsigned long arg)
 862{
 863	struct ata_link *link = (struct ata_link *)arg;
 
 864	struct ata_port *ap = link->ap;
 865	struct ahci_port_priv *pp = ap->private_data;
 866	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 867	unsigned long led_message = emp->led_state;
 868	u32 activity_led_state;
 869	unsigned long flags;
 870
 871	led_message &= EM_MSG_LED_VALUE;
 872	led_message |= ap->port_no | (link->pmp << 8);
 873
 874	/* check to see if we've had activity.  If so,
 875	 * toggle state of LED and reset timer.  If not,
 876	 * turn LED to desired idle state.
 877	 */
 878	spin_lock_irqsave(ap->lock, flags);
 879	if (emp->saved_activity != emp->activity) {
 880		emp->saved_activity = emp->activity;
 881		/* get the current LED state */
 882		activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
 883
 884		if (activity_led_state)
 885			activity_led_state = 0;
 886		else
 887			activity_led_state = 1;
 888
 889		/* clear old state */
 890		led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
 891
 892		/* toggle state */
 893		led_message |= (activity_led_state << 16);
 894		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));
 895	} else {
 896		/* switch to idle */
 897		led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
 898		if (emp->blink_policy == BLINK_OFF)
 899			led_message |= (1 << 16);
 900	}
 901	spin_unlock_irqrestore(ap->lock, flags);
 902	ahci_transmit_led_message(ap, led_message, 4);
 903}
 904
 905static void ahci_init_sw_activity(struct ata_link *link)
 906{
 907	struct ata_port *ap = link->ap;
 908	struct ahci_port_priv *pp = ap->private_data;
 909	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 910
 911	/* init activity stats, setup timer */
 912	emp->saved_activity = emp->activity = 0;
 913	setup_timer(&emp->timer, ahci_sw_activity_blink, (unsigned long)link);
 
 914
 915	/* check our blink policy and set flag for link if it's enabled */
 916	if (emp->blink_policy)
 917		link->flags |= ATA_LFLAG_SW_ACTIVITY;
 918}
 919
 920int ahci_reset_em(struct ata_host *host)
 921{
 922	struct ahci_host_priv *hpriv = host->private_data;
 923	void __iomem *mmio = hpriv->mmio;
 924	u32 em_ctl;
 925
 926	em_ctl = readl(mmio + HOST_EM_CTL);
 927	if ((em_ctl & EM_CTL_TM) || (em_ctl & EM_CTL_RST))
 928		return -EINVAL;
 929
 930	writel(em_ctl | EM_CTL_RST, mmio + HOST_EM_CTL);
 931	return 0;
 932}
 933EXPORT_SYMBOL_GPL(ahci_reset_em);
 934
 935static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
 936					ssize_t size)
 937{
 938	struct ahci_host_priv *hpriv = ap->host->private_data;
 939	struct ahci_port_priv *pp = ap->private_data;
 940	void __iomem *mmio = hpriv->mmio;
 941	u32 em_ctl;
 942	u32 message[] = {0, 0};
 943	unsigned long flags;
 944	int pmp;
 945	struct ahci_em_priv *emp;
 946
 947	/* get the slot number from the message */
 948	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
 949	if (pmp < EM_MAX_SLOTS)
 950		emp = &pp->em_priv[pmp];
 951	else
 952		return -EINVAL;
 953
 
 954	spin_lock_irqsave(ap->lock, flags);
 955
 956	/*
 957	 * if we are still busy transmitting a previous message,
 958	 * do not allow
 959	 */
 960	em_ctl = readl(mmio + HOST_EM_CTL);
 961	if (em_ctl & EM_CTL_TM) {
 962		spin_unlock_irqrestore(ap->lock, flags);
 
 963		return -EBUSY;
 964	}
 965
 966	if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {
 967		/*
 968		 * create message header - this is all zero except for
 969		 * the message size, which is 4 bytes.
 970		 */
 971		message[0] |= (4 << 8);
 972
 973		/* ignore 0:4 of byte zero, fill in port info yourself */
 974		message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);
 975
 976		/* write message to EM_LOC */
 977		writel(message[0], mmio + hpriv->em_loc);
 978		writel(message[1], mmio + hpriv->em_loc+4);
 979
 980		/*
 981		 * tell hardware to transmit the message
 982		 */
 983		writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
 984	}
 985
 986	/* save off new led state for port/slot */
 987	emp->led_state = state;
 988
 989	spin_unlock_irqrestore(ap->lock, flags);
 
 
 990	return size;
 991}
 992
 993static ssize_t ahci_led_show(struct ata_port *ap, char *buf)
 994{
 995	struct ahci_port_priv *pp = ap->private_data;
 996	struct ata_link *link;
 997	struct ahci_em_priv *emp;
 998	int rc = 0;
 999
1000	ata_for_each_link(link, ap, EDGE) {
1001		emp = &pp->em_priv[link->pmp];
1002		rc += sprintf(buf, "%lx\n", emp->led_state);
1003	}
1004	return rc;
1005}
1006
1007static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
1008				size_t size)
1009{
1010	int state;
1011	int pmp;
1012	struct ahci_port_priv *pp = ap->private_data;
1013	struct ahci_em_priv *emp;
1014
1015	state = simple_strtoul(buf, NULL, 0);
 
1016
1017	/* get the slot number from the message */
1018	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1019	if (pmp < EM_MAX_SLOTS)
1020		emp = &pp->em_priv[pmp];
1021	else
1022		return -EINVAL;
1023
1024	/* mask off the activity bits if we are in sw_activity
1025	 * mode, user should turn off sw_activity before setting
1026	 * activity led through em_message
1027	 */
1028	if (emp->blink_policy)
1029		state &= ~EM_MSG_LED_VALUE_ACTIVITY;
1030
1031	return ahci_transmit_led_message(ap, state, size);
1032}
1033
1034static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
1035{
1036	struct ata_link *link = dev->link;
1037	struct ata_port *ap = link->ap;
1038	struct ahci_port_priv *pp = ap->private_data;
1039	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1040	u32 port_led_state = emp->led_state;
1041
1042	/* save the desired Activity LED behavior */
1043	if (val == OFF) {
1044		/* clear LFLAG */
1045		link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
1046
1047		/* set the LED to OFF */
1048		port_led_state &= EM_MSG_LED_VALUE_OFF;
1049		port_led_state |= (ap->port_no | (link->pmp << 8));
1050		ahci_transmit_led_message(ap, port_led_state, 4);
1051	} else {
1052		link->flags |= ATA_LFLAG_SW_ACTIVITY;
1053		if (val == BLINK_OFF) {
1054			/* set LED to ON for idle */
1055			port_led_state &= EM_MSG_LED_VALUE_OFF;
1056			port_led_state |= (ap->port_no | (link->pmp << 8));
1057			port_led_state |= EM_MSG_LED_VALUE_ON; /* check this */
1058			ahci_transmit_led_message(ap, port_led_state, 4);
1059		}
1060	}
1061	emp->blink_policy = val;
1062	return 0;
1063}
1064
1065static ssize_t ahci_activity_show(struct ata_device *dev, char *buf)
1066{
1067	struct ata_link *link = dev->link;
1068	struct ata_port *ap = link->ap;
1069	struct ahci_port_priv *pp = ap->private_data;
1070	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1071
1072	/* display the saved value of activity behavior for this
1073	 * disk.
1074	 */
1075	return sprintf(buf, "%d\n", emp->blink_policy);
1076}
1077
1078static void ahci_port_init(struct device *dev, struct ata_port *ap,
1079			   int port_no, void __iomem *mmio,
1080			   void __iomem *port_mmio)
1081{
 
1082	const char *emsg = NULL;
1083	int rc;
1084	u32 tmp;
1085
1086	/* make sure port is not active */
1087	rc = ahci_deinit_port(ap, &emsg);
1088	if (rc)
1089		dev_warn(dev, "%s (%d)\n", emsg, rc);
1090
1091	/* clear SError */
1092	tmp = readl(port_mmio + PORT_SCR_ERR);
1093	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
1094	writel(tmp, port_mmio + PORT_SCR_ERR);
1095
1096	/* clear port IRQ */
1097	tmp = readl(port_mmio + PORT_IRQ_STAT);
1098	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
1099	if (tmp)
1100		writel(tmp, port_mmio + PORT_IRQ_STAT);
1101
1102	writel(1 << port_no, mmio + HOST_IRQ_STAT);
 
 
 
 
 
1103}
1104
1105void ahci_init_controller(struct ata_host *host)
1106{
1107	struct ahci_host_priv *hpriv = host->private_data;
1108	void __iomem *mmio = hpriv->mmio;
1109	int i;
1110	void __iomem *port_mmio;
1111	u32 tmp;
1112
1113	for (i = 0; i < host->n_ports; i++) {
1114		struct ata_port *ap = host->ports[i];
1115
1116		port_mmio = ahci_port_base(ap);
1117		if (ata_port_is_dummy(ap))
1118			continue;
1119
1120		ahci_port_init(host->dev, ap, i, mmio, port_mmio);
1121	}
1122
1123	tmp = readl(mmio + HOST_CTL);
1124	VPRINTK("HOST_CTL 0x%x\n", tmp);
1125	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
1126	tmp = readl(mmio + HOST_CTL);
1127	VPRINTK("HOST_CTL 0x%x\n", tmp);
1128}
1129EXPORT_SYMBOL_GPL(ahci_init_controller);
1130
1131static void ahci_dev_config(struct ata_device *dev)
1132{
1133	struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
1134
1135	if (hpriv->flags & AHCI_HFLAG_SECT255) {
1136		dev->max_sectors = 255;
1137		ata_dev_info(dev,
1138			     "SB600 AHCI: limiting to 255 sectors per cmd\n");
1139	}
1140}
1141
1142static unsigned int ahci_dev_classify(struct ata_port *ap)
1143{
1144	void __iomem *port_mmio = ahci_port_base(ap);
1145	struct ata_taskfile tf;
1146	u32 tmp;
1147
1148	tmp = readl(port_mmio + PORT_SIG);
1149	tf.lbah		= (tmp >> 24)	& 0xff;
1150	tf.lbam		= (tmp >> 16)	& 0xff;
1151	tf.lbal		= (tmp >> 8)	& 0xff;
1152	tf.nsect	= (tmp)		& 0xff;
1153
1154	return ata_dev_classify(&tf);
1155}
 
1156
1157void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
1158			u32 opts)
1159{
1160	dma_addr_t cmd_tbl_dma;
1161
1162	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
1163
1164	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
1165	pp->cmd_slot[tag].status = 0;
1166	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
1167	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1168}
1169EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot);
1170
1171int ahci_kick_engine(struct ata_port *ap)
1172{
1173	void __iomem *port_mmio = ahci_port_base(ap);
1174	struct ahci_host_priv *hpriv = ap->host->private_data;
1175	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1176	u32 tmp;
1177	int busy, rc;
1178
1179	/* stop engine */
1180	rc = ahci_stop_engine(ap);
1181	if (rc)
1182		goto out_restart;
1183
1184	/* need to do CLO?
1185	 * always do CLO if PMP is attached (AHCI-1.3 9.2)
1186	 */
1187	busy = status & (ATA_BUSY | ATA_DRQ);
1188	if (!busy && !sata_pmp_attached(ap)) {
1189		rc = 0;
1190		goto out_restart;
1191	}
1192
1193	if (!(hpriv->cap & HOST_CAP_CLO)) {
1194		rc = -EOPNOTSUPP;
1195		goto out_restart;
1196	}
1197
1198	/* perform CLO */
1199	tmp = readl(port_mmio + PORT_CMD);
1200	tmp |= PORT_CMD_CLO;
1201	writel(tmp, port_mmio + PORT_CMD);
1202
1203	rc = 0;
1204	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
1205				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
1206	if (tmp & PORT_CMD_CLO)
1207		rc = -EIO;
1208
1209	/* restart engine */
1210 out_restart:
1211	ahci_start_engine(ap);
1212	return rc;
1213}
1214EXPORT_SYMBOL_GPL(ahci_kick_engine);
1215
1216static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
1217				struct ata_taskfile *tf, int is_cmd, u16 flags,
1218				unsigned long timeout_msec)
1219{
1220	const u32 cmd_fis_len = 5; /* five dwords */
1221	struct ahci_port_priv *pp = ap->private_data;
1222	void __iomem *port_mmio = ahci_port_base(ap);
1223	u8 *fis = pp->cmd_tbl;
1224	u32 tmp;
1225
1226	/* prep the command */
1227	ata_tf_to_fis(tf, pmp, is_cmd, fis);
1228	ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
1229
 
 
 
 
 
 
 
 
 
1230	/* issue & wait */
1231	writel(1, port_mmio + PORT_CMD_ISSUE);
1232
1233	if (timeout_msec) {
1234		tmp = ata_wait_register(ap, port_mmio + PORT_CMD_ISSUE,
1235					0x1, 0x1, 1, timeout_msec);
1236		if (tmp & 0x1) {
1237			ahci_kick_engine(ap);
1238			return -EBUSY;
1239		}
1240	} else
1241		readl(port_mmio + PORT_CMD_ISSUE);	/* flush */
1242
1243	return 0;
1244}
1245
1246int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1247		      int pmp, unsigned long deadline,
1248		      int (*check_ready)(struct ata_link *link))
1249{
1250	struct ata_port *ap = link->ap;
1251	struct ahci_host_priv *hpriv = ap->host->private_data;
 
1252	const char *reason = NULL;
1253	unsigned long now, msecs;
1254	struct ata_taskfile tf;
 
1255	int rc;
1256
1257	DPRINTK("ENTER\n");
1258
1259	/* prepare for SRST (AHCI-1.1 10.4.1) */
1260	rc = ahci_kick_engine(ap);
1261	if (rc && rc != -EOPNOTSUPP)
1262		ata_link_warn(link, "failed to reset engine (errno=%d)\n", rc);
1263
 
 
 
 
 
 
 
 
 
 
1264	ata_tf_init(link->device, &tf);
1265
1266	/* issue the first D2H Register FIS */
1267	msecs = 0;
1268	now = jiffies;
1269	if (time_after(deadline, now))
1270		msecs = jiffies_to_msecs(deadline - now);
1271
1272	tf.ctl |= ATA_SRST;
1273	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1274				 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
1275		rc = -EIO;
1276		reason = "1st FIS failed";
1277		goto fail;
1278	}
1279
1280	/* spec says at least 5us, but be generous and sleep for 1ms */
1281	ata_msleep(ap, 1);
1282
1283	/* issue the second D2H Register FIS */
1284	tf.ctl &= ~ATA_SRST;
1285	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
1286
1287	/* wait for link to become ready */
1288	rc = ata_wait_after_reset(link, deadline, check_ready);
1289	if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {
1290		/*
1291		 * Workaround for cases where link online status can't
1292		 * be trusted.  Treat device readiness timeout as link
1293		 * offline.
1294		 */
1295		ata_link_info(link, "device not ready, treating as offline\n");
1296		*class = ATA_DEV_NONE;
1297	} else if (rc) {
1298		/* link occupied, -ENODEV too is an error */
1299		reason = "device not ready";
1300		goto fail;
1301	} else
1302		*class = ahci_dev_classify(ap);
1303
 
 
 
 
1304	DPRINTK("EXIT, class=%u\n", *class);
1305	return 0;
1306
1307 fail:
1308	ata_link_err(link, "softreset failed (%s)\n", reason);
1309	return rc;
1310}
1311
1312int ahci_check_ready(struct ata_link *link)
1313{
1314	void __iomem *port_mmio = ahci_port_base(link->ap);
1315	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1316
1317	return ata_check_ready(status);
1318}
1319EXPORT_SYMBOL_GPL(ahci_check_ready);
1320
1321static int ahci_softreset(struct ata_link *link, unsigned int *class,
1322			  unsigned long deadline)
1323{
1324	int pmp = sata_srst_pmp(link);
1325
1326	DPRINTK("ENTER\n");
1327
1328	return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
1329}
1330EXPORT_SYMBOL_GPL(ahci_do_softreset);
1331
1332static int ahci_bad_pmp_check_ready(struct ata_link *link)
1333{
1334	void __iomem *port_mmio = ahci_port_base(link->ap);
1335	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1336	u32 irq_status = readl(port_mmio + PORT_IRQ_STAT);
1337
1338	/*
1339	 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1340	 * which can save timeout delay.
1341	 */
1342	if (irq_status & PORT_IRQ_BAD_PMP)
1343		return -EIO;
1344
1345	return ata_check_ready(status);
1346}
1347
1348int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
1349				unsigned long deadline)
1350{
1351	struct ata_port *ap = link->ap;
1352	void __iomem *port_mmio = ahci_port_base(ap);
1353	int pmp = sata_srst_pmp(link);
1354	int rc;
1355	u32 irq_sts;
1356
1357	DPRINTK("ENTER\n");
1358
1359	rc = ahci_do_softreset(link, class, pmp, deadline,
1360			       ahci_bad_pmp_check_ready);
1361
1362	/*
1363	 * Soft reset fails with IPMS set when PMP is enabled but
1364	 * SATA HDD/ODD is connected to SATA port, do soft reset
1365	 * again to port 0.
1366	 */
1367	if (rc == -EIO) {
1368		irq_sts = readl(port_mmio + PORT_IRQ_STAT);
1369		if (irq_sts & PORT_IRQ_BAD_PMP) {
1370			ata_link_printk(link, KERN_WARNING,
1371					"applying PMP SRST workaround "
1372					"and retrying\n");
1373			rc = ahci_do_softreset(link, class, 0, deadline,
1374					       ahci_check_ready);
1375		}
1376	}
1377
1378	return rc;
1379}
1380
1381static int ahci_hardreset(struct ata_link *link, unsigned int *class,
1382			  unsigned long deadline)
1383{
1384	const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
1385	struct ata_port *ap = link->ap;
1386	struct ahci_port_priv *pp = ap->private_data;
 
1387	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1388	struct ata_taskfile tf;
1389	bool online;
1390	int rc;
1391
1392	DPRINTK("ENTER\n");
1393
1394	ahci_stop_engine(ap);
1395
1396	/* clear D2H reception area to properly wait for D2H FIS */
1397	ata_tf_init(link->device, &tf);
1398	tf.command = 0x80;
1399	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1400
1401	rc = sata_link_hardreset(link, timing, deadline, &online,
1402				 ahci_check_ready);
1403
1404	ahci_start_engine(ap);
1405
1406	if (online)
1407		*class = ahci_dev_classify(ap);
1408
1409	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1410	return rc;
1411}
 
 
 
 
 
 
 
 
 
1412
1413static void ahci_postreset(struct ata_link *link, unsigned int *class)
1414{
1415	struct ata_port *ap = link->ap;
1416	void __iomem *port_mmio = ahci_port_base(ap);
1417	u32 new_tmp, tmp;
1418
1419	ata_std_postreset(link, class);
1420
1421	/* Make sure port's ATAPI bit is set appropriately */
1422	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1423	if (*class == ATA_DEV_ATAPI)
1424		new_tmp |= PORT_CMD_ATAPI;
1425	else
1426		new_tmp &= ~PORT_CMD_ATAPI;
1427	if (new_tmp != tmp) {
1428		writel(new_tmp, port_mmio + PORT_CMD);
1429		readl(port_mmio + PORT_CMD); /* flush */
1430	}
1431}
1432
1433static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
1434{
1435	struct scatterlist *sg;
1436	struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
1437	unsigned int si;
1438
1439	VPRINTK("ENTER\n");
1440
1441	/*
1442	 * Next, the S/G list.
1443	 */
1444	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1445		dma_addr_t addr = sg_dma_address(sg);
1446		u32 sg_len = sg_dma_len(sg);
1447
1448		ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
1449		ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
1450		ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
1451	}
1452
1453	return si;
1454}
1455
1456static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
1457{
1458	struct ata_port *ap = qc->ap;
1459	struct ahci_port_priv *pp = ap->private_data;
1460
1461	if (!sata_pmp_attached(ap) || pp->fbs_enabled)
1462		return ata_std_qc_defer(qc);
1463	else
1464		return sata_pmp_qc_defer_cmd_switch(qc);
1465}
1466
1467static void ahci_qc_prep(struct ata_queued_cmd *qc)
1468{
1469	struct ata_port *ap = qc->ap;
1470	struct ahci_port_priv *pp = ap->private_data;
1471	int is_atapi = ata_is_atapi(qc->tf.protocol);
1472	void *cmd_tbl;
1473	u32 opts;
1474	const u32 cmd_fis_len = 5; /* five dwords */
1475	unsigned int n_elem;
1476
1477	/*
1478	 * Fill in command table information.  First, the header,
1479	 * a SATA Register - Host to Device command FIS.
1480	 */
1481	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
1482
1483	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
1484	if (is_atapi) {
1485		memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
1486		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1487	}
1488
1489	n_elem = 0;
1490	if (qc->flags & ATA_QCFLAG_DMAMAP)
1491		n_elem = ahci_fill_sg(qc, cmd_tbl);
1492
1493	/*
1494	 * Fill in command slot information.
1495	 */
1496	opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
1497	if (qc->tf.flags & ATA_TFLAG_WRITE)
1498		opts |= AHCI_CMD_WRITE;
1499	if (is_atapi)
1500		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1501
1502	ahci_fill_cmd_slot(pp, qc->tag, opts);
1503}
1504
1505static void ahci_fbs_dec_intr(struct ata_port *ap)
1506{
1507	struct ahci_port_priv *pp = ap->private_data;
1508	void __iomem *port_mmio = ahci_port_base(ap);
1509	u32 fbs = readl(port_mmio + PORT_FBS);
1510	int retries = 3;
1511
1512	DPRINTK("ENTER\n");
1513	BUG_ON(!pp->fbs_enabled);
1514
1515	/* time to wait for DEC is not specified by AHCI spec,
1516	 * add a retry loop for safety.
1517	 */
1518	writel(fbs | PORT_FBS_DEC, port_mmio + PORT_FBS);
1519	fbs = readl(port_mmio + PORT_FBS);
1520	while ((fbs & PORT_FBS_DEC) && retries--) {
1521		udelay(1);
1522		fbs = readl(port_mmio + PORT_FBS);
1523	}
1524
1525	if (fbs & PORT_FBS_DEC)
1526		dev_err(ap->host->dev, "failed to clear device error\n");
1527}
1528
1529static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1530{
1531	struct ahci_host_priv *hpriv = ap->host->private_data;
1532	struct ahci_port_priv *pp = ap->private_data;
1533	struct ata_eh_info *host_ehi = &ap->link.eh_info;
1534	struct ata_link *link = NULL;
1535	struct ata_queued_cmd *active_qc;
1536	struct ata_eh_info *active_ehi;
1537	bool fbs_need_dec = false;
1538	u32 serror;
1539
1540	/* determine active link with error */
1541	if (pp->fbs_enabled) {
1542		void __iomem *port_mmio = ahci_port_base(ap);
1543		u32 fbs = readl(port_mmio + PORT_FBS);
1544		int pmp = fbs >> PORT_FBS_DWE_OFFSET;
1545
1546		if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links) &&
1547		    ata_link_online(&ap->pmp_link[pmp])) {
1548			link = &ap->pmp_link[pmp];
1549			fbs_need_dec = true;
1550		}
1551
1552	} else
1553		ata_for_each_link(link, ap, EDGE)
1554			if (ata_link_active(link))
1555				break;
1556
1557	if (!link)
1558		link = &ap->link;
1559
1560	active_qc = ata_qc_from_tag(ap, link->active_tag);
1561	active_ehi = &link->eh_info;
1562
1563	/* record irq stat */
1564	ata_ehi_clear_desc(host_ehi);
1565	ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
1566
1567	/* AHCI needs SError cleared; otherwise, it might lock up */
1568	ahci_scr_read(&ap->link, SCR_ERROR, &serror);
1569	ahci_scr_write(&ap->link, SCR_ERROR, serror);
1570	host_ehi->serror |= serror;
1571
1572	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1573	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1574		irq_stat &= ~PORT_IRQ_IF_ERR;
1575
1576	if (irq_stat & PORT_IRQ_TF_ERR) {
1577		/* If qc is active, charge it; otherwise, the active
1578		 * link.  There's no active qc on NCQ errors.  It will
1579		 * be determined by EH by reading log page 10h.
1580		 */
1581		if (active_qc)
1582			active_qc->err_mask |= AC_ERR_DEV;
1583		else
1584			active_ehi->err_mask |= AC_ERR_DEV;
1585
1586		if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
1587			host_ehi->serror &= ~SERR_INTERNAL;
1588	}
1589
1590	if (irq_stat & PORT_IRQ_UNK_FIS) {
1591		u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);
1592
1593		active_ehi->err_mask |= AC_ERR_HSM;
1594		active_ehi->action |= ATA_EH_RESET;
1595		ata_ehi_push_desc(active_ehi,
1596				  "unknown FIS %08x %08x %08x %08x" ,
1597				  unk[0], unk[1], unk[2], unk[3]);
1598	}
1599
1600	if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
1601		active_ehi->err_mask |= AC_ERR_HSM;
1602		active_ehi->action |= ATA_EH_RESET;
1603		ata_ehi_push_desc(active_ehi, "incorrect PMP");
1604	}
1605
1606	if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
1607		host_ehi->err_mask |= AC_ERR_HOST_BUS;
1608		host_ehi->action |= ATA_EH_RESET;
1609		ata_ehi_push_desc(host_ehi, "host bus error");
1610	}
1611
1612	if (irq_stat & PORT_IRQ_IF_ERR) {
1613		if (fbs_need_dec)
1614			active_ehi->err_mask |= AC_ERR_DEV;
1615		else {
1616			host_ehi->err_mask |= AC_ERR_ATA_BUS;
1617			host_ehi->action |= ATA_EH_RESET;
1618		}
1619
1620		ata_ehi_push_desc(host_ehi, "interface fatal error");
1621	}
1622
1623	if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
1624		ata_ehi_hotplugged(host_ehi);
1625		ata_ehi_push_desc(host_ehi, "%s",
1626			irq_stat & PORT_IRQ_CONNECT ?
1627			"connection status changed" : "PHY RDY changed");
1628	}
1629
1630	/* okay, let's hand over to EH */
1631
1632	if (irq_stat & PORT_IRQ_FREEZE)
1633		ata_port_freeze(ap);
1634	else if (fbs_need_dec) {
1635		ata_link_abort(link);
1636		ahci_fbs_dec_intr(ap);
1637	} else
1638		ata_port_abort(ap);
1639}
1640
1641static void ahci_port_intr(struct ata_port *ap)
 
1642{
1643	void __iomem *port_mmio = ahci_port_base(ap);
1644	struct ata_eh_info *ehi = &ap->link.eh_info;
1645	struct ahci_port_priv *pp = ap->private_data;
1646	struct ahci_host_priv *hpriv = ap->host->private_data;
1647	int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
1648	u32 status, qc_active = 0;
1649	int rc;
1650
1651	status = readl(port_mmio + PORT_IRQ_STAT);
1652	writel(status, port_mmio + PORT_IRQ_STAT);
1653
1654	/* ignore BAD_PMP while resetting */
1655	if (unlikely(resetting))
1656		status &= ~PORT_IRQ_BAD_PMP;
1657
1658	/* if LPM is enabled, PHYRDY doesn't mean anything */
1659	if (ap->link.lpm_policy > ATA_LPM_MAX_POWER) {
1660		status &= ~PORT_IRQ_PHYRDY;
1661		ahci_scr_write(&ap->link, SCR_ERROR, SERR_PHYRDY_CHG);
1662	}
1663
1664	if (unlikely(status & PORT_IRQ_ERROR)) {
1665		ahci_error_intr(ap, status);
1666		return;
1667	}
1668
1669	if (status & PORT_IRQ_SDB_FIS) {
1670		/* If SNotification is available, leave notification
1671		 * handling to sata_async_notification().  If not,
1672		 * emulate it by snooping SDB FIS RX area.
1673		 *
1674		 * Snooping FIS RX area is probably cheaper than
1675		 * poking SNotification but some constrollers which
1676		 * implement SNotification, ICH9 for example, don't
1677		 * store AN SDB FIS into receive area.
1678		 */
1679		if (hpriv->cap & HOST_CAP_SNTF)
1680			sata_async_notification(ap);
1681		else {
1682			/* If the 'N' bit in word 0 of the FIS is set,
1683			 * we just received asynchronous notification.
1684			 * Tell libata about it.
1685			 *
1686			 * Lack of SNotification should not appear in
1687			 * ahci 1.2, so the workaround is unnecessary
1688			 * when FBS is enabled.
1689			 */
1690			if (pp->fbs_enabled)
1691				WARN_ON_ONCE(1);
1692			else {
1693				const __le32 *f = pp->rx_fis + RX_FIS_SDB;
1694				u32 f0 = le32_to_cpu(f[0]);
1695				if (f0 & (1 << 15))
1696					sata_async_notification(ap);
1697			}
1698		}
1699	}
1700
1701	/* pp->active_link is not reliable once FBS is enabled, both
1702	 * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1703	 * NCQ and non-NCQ commands may be in flight at the same time.
1704	 */
1705	if (pp->fbs_enabled) {
1706		if (ap->qc_active) {
1707			qc_active = readl(port_mmio + PORT_SCR_ACT);
1708			qc_active |= readl(port_mmio + PORT_CMD_ISSUE);
1709		}
1710	} else {
1711		/* pp->active_link is valid iff any command is in flight */
1712		if (ap->qc_active && pp->active_link->sactive)
1713			qc_active = readl(port_mmio + PORT_SCR_ACT);
1714		else
1715			qc_active = readl(port_mmio + PORT_CMD_ISSUE);
1716	}
1717
1718
1719	rc = ata_qc_complete_multiple(ap, qc_active);
1720
1721	/* while resetting, invalid completions are expected */
1722	if (unlikely(rc < 0 && !resetting)) {
1723		ehi->err_mask |= AC_ERR_HSM;
1724		ehi->action |= ATA_EH_RESET;
1725		ata_port_freeze(ap);
1726	}
1727}
1728
1729irqreturn_t ahci_interrupt(int irq, void *dev_instance)
1730{
1731	struct ata_host *host = dev_instance;
1732	struct ahci_host_priv *hpriv;
1733	unsigned int i, handled = 0;
1734	void __iomem *mmio;
1735	u32 irq_stat, irq_masked;
 
 
 
 
 
 
 
 
 
1736
1737	VPRINTK("ENTER\n");
1738
1739	hpriv = host->private_data;
1740	mmio = hpriv->mmio;
1741
1742	/* sigh.  0xffffffff is a valid return from h/w */
1743	irq_stat = readl(mmio + HOST_IRQ_STAT);
1744	if (!irq_stat)
1745		return IRQ_NONE;
1746
1747	irq_masked = irq_stat & hpriv->port_map;
1748
1749	spin_lock(&host->lock);
 
 
 
 
 
1750
1751	for (i = 0; i < host->n_ports; i++) {
1752		struct ata_port *ap;
1753
1754		if (!(irq_masked & (1 << i)))
1755			continue;
1756
1757		ap = host->ports[i];
1758		if (ap) {
1759			ahci_port_intr(ap);
1760			VPRINTK("port %u\n", i);
1761		} else {
1762			VPRINTK("port %u (no irq)\n", i);
1763			if (ata_ratelimit())
1764				dev_warn(host->dev,
1765					 "interrupt on disabled port %u\n", i);
1766		}
1767
1768		handled = 1;
1769	}
1770
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1771	/* HOST_IRQ_STAT behaves as level triggered latch meaning that
1772	 * it should be cleared after all the port events are cleared;
1773	 * otherwise, it will raise a spurious interrupt after each
1774	 * valid one.  Please read section 10.6.2 of ahci 1.1 for more
1775	 * information.
1776	 *
1777	 * Also, use the unmasked value to clear interrupt as spurious
1778	 * pending event on a dummy port might cause screaming IRQ.
1779	 */
1780	writel(irq_stat, mmio + HOST_IRQ_STAT);
1781
1782	spin_unlock(&host->lock);
1783
1784	VPRINTK("EXIT\n");
1785
1786	return IRQ_RETVAL(handled);
1787}
1788EXPORT_SYMBOL_GPL(ahci_interrupt);
1789
1790static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
1791{
1792	struct ata_port *ap = qc->ap;
1793	void __iomem *port_mmio = ahci_port_base(ap);
1794	struct ahci_port_priv *pp = ap->private_data;
1795
1796	/* Keep track of the currently active link.  It will be used
1797	 * in completion path to determine whether NCQ phase is in
1798	 * progress.
1799	 */
1800	pp->active_link = qc->dev->link;
1801
1802	if (qc->tf.protocol == ATA_PROT_NCQ)
1803		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
1804
1805	if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
1806		u32 fbs = readl(port_mmio + PORT_FBS);
1807		fbs &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
1808		fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
1809		writel(fbs, port_mmio + PORT_FBS);
1810		pp->fbs_last_dev = qc->dev->link->pmp;
1811	}
1812
1813	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
1814
1815	ahci_sw_activity(qc->dev->link);
1816
1817	return 0;
1818}
 
1819
1820static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
1821{
1822	struct ahci_port_priv *pp = qc->ap->private_data;
1823	u8 *rx_fis = pp->rx_fis;
1824
1825	if (pp->fbs_enabled)
1826		rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
1827
1828	/*
1829	 * After a successful execution of an ATA PIO data-in command,
1830	 * the device doesn't send D2H Reg FIS to update the TF and
1831	 * the host should take TF and E_Status from the preceding PIO
1832	 * Setup FIS.
1833	 */
1834	if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&
1835	    !(qc->flags & ATA_QCFLAG_FAILED)) {
1836		ata_tf_from_fis(rx_fis + RX_FIS_PIO_SETUP, &qc->result_tf);
1837		qc->result_tf.command = (rx_fis + RX_FIS_PIO_SETUP)[15];
1838	} else
1839		ata_tf_from_fis(rx_fis + RX_FIS_D2H_REG, &qc->result_tf);
1840
1841	return true;
1842}
1843
1844static void ahci_freeze(struct ata_port *ap)
1845{
1846	void __iomem *port_mmio = ahci_port_base(ap);
1847
1848	/* turn IRQ off */
1849	writel(0, port_mmio + PORT_IRQ_MASK);
1850}
1851
1852static void ahci_thaw(struct ata_port *ap)
1853{
1854	struct ahci_host_priv *hpriv = ap->host->private_data;
1855	void __iomem *mmio = hpriv->mmio;
1856	void __iomem *port_mmio = ahci_port_base(ap);
1857	u32 tmp;
1858	struct ahci_port_priv *pp = ap->private_data;
1859
1860	/* clear IRQ */
1861	tmp = readl(port_mmio + PORT_IRQ_STAT);
1862	writel(tmp, port_mmio + PORT_IRQ_STAT);
1863	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
1864
1865	/* turn IRQ back on */
1866	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
1867}
1868
1869static void ahci_error_handler(struct ata_port *ap)
1870{
 
 
1871	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
1872		/* restart engine */
1873		ahci_stop_engine(ap);
1874		ahci_start_engine(ap);
1875	}
1876
1877	sata_pmp_error_handler(ap);
1878
1879	if (!ata_dev_enabled(ap->link.device))
1880		ahci_stop_engine(ap);
1881}
 
1882
1883static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
1884{
1885	struct ata_port *ap = qc->ap;
1886
1887	/* make DMA engine forget about the failed command */
1888	if (qc->flags & ATA_QCFLAG_FAILED)
1889		ahci_kick_engine(ap);
1890}
1891
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1892static void ahci_enable_fbs(struct ata_port *ap)
1893{
 
1894	struct ahci_port_priv *pp = ap->private_data;
1895	void __iomem *port_mmio = ahci_port_base(ap);
1896	u32 fbs;
1897	int rc;
1898
1899	if (!pp->fbs_supported)
1900		return;
1901
1902	fbs = readl(port_mmio + PORT_FBS);
1903	if (fbs & PORT_FBS_EN) {
1904		pp->fbs_enabled = true;
1905		pp->fbs_last_dev = -1; /* initialization */
1906		return;
1907	}
1908
1909	rc = ahci_stop_engine(ap);
1910	if (rc)
1911		return;
1912
1913	writel(fbs | PORT_FBS_EN, port_mmio + PORT_FBS);
1914	fbs = readl(port_mmio + PORT_FBS);
1915	if (fbs & PORT_FBS_EN) {
1916		dev_info(ap->host->dev, "FBS is enabled\n");
1917		pp->fbs_enabled = true;
1918		pp->fbs_last_dev = -1; /* initialization */
1919	} else
1920		dev_err(ap->host->dev, "Failed to enable FBS\n");
1921
1922	ahci_start_engine(ap);
1923}
1924
1925static void ahci_disable_fbs(struct ata_port *ap)
1926{
 
1927	struct ahci_port_priv *pp = ap->private_data;
1928	void __iomem *port_mmio = ahci_port_base(ap);
1929	u32 fbs;
1930	int rc;
1931
1932	if (!pp->fbs_supported)
1933		return;
1934
1935	fbs = readl(port_mmio + PORT_FBS);
1936	if ((fbs & PORT_FBS_EN) == 0) {
1937		pp->fbs_enabled = false;
1938		return;
1939	}
1940
1941	rc = ahci_stop_engine(ap);
1942	if (rc)
1943		return;
1944
1945	writel(fbs & ~PORT_FBS_EN, port_mmio + PORT_FBS);
1946	fbs = readl(port_mmio + PORT_FBS);
1947	if (fbs & PORT_FBS_EN)
1948		dev_err(ap->host->dev, "Failed to disable FBS\n");
1949	else {
1950		dev_info(ap->host->dev, "FBS is disabled\n");
1951		pp->fbs_enabled = false;
1952	}
1953
1954	ahci_start_engine(ap);
1955}
1956
1957static void ahci_pmp_attach(struct ata_port *ap)
1958{
1959	void __iomem *port_mmio = ahci_port_base(ap);
1960	struct ahci_port_priv *pp = ap->private_data;
1961	u32 cmd;
1962
1963	cmd = readl(port_mmio + PORT_CMD);
1964	cmd |= PORT_CMD_PMP;
1965	writel(cmd, port_mmio + PORT_CMD);
1966
1967	ahci_enable_fbs(ap);
1968
1969	pp->intr_mask |= PORT_IRQ_BAD_PMP;
1970
1971	/*
1972	 * We must not change the port interrupt mask register if the
1973	 * port is marked frozen, the value in pp->intr_mask will be
1974	 * restored later when the port is thawed.
1975	 *
1976	 * Note that during initialization, the port is marked as
1977	 * frozen since the irq handler is not yet registered.
1978	 */
1979	if (!(ap->pflags & ATA_PFLAG_FROZEN))
1980		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
1981}
1982
1983static void ahci_pmp_detach(struct ata_port *ap)
1984{
1985	void __iomem *port_mmio = ahci_port_base(ap);
1986	struct ahci_port_priv *pp = ap->private_data;
1987	u32 cmd;
1988
1989	ahci_disable_fbs(ap);
1990
1991	cmd = readl(port_mmio + PORT_CMD);
1992	cmd &= ~PORT_CMD_PMP;
1993	writel(cmd, port_mmio + PORT_CMD);
1994
1995	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
1996
1997	/* see comment above in ahci_pmp_attach() */
1998	if (!(ap->pflags & ATA_PFLAG_FROZEN))
1999		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2000}
2001
2002int ahci_port_resume(struct ata_port *ap)
2003{
 
 
2004	ahci_power_up(ap);
2005	ahci_start_port(ap);
2006
2007	if (sata_pmp_attached(ap))
2008		ahci_pmp_attach(ap);
2009	else
2010		ahci_pmp_detach(ap);
2011
2012	return 0;
2013}
2014EXPORT_SYMBOL_GPL(ahci_port_resume);
2015
2016#ifdef CONFIG_PM
2017static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
2018{
2019	const char *emsg = NULL;
2020	int rc;
2021
2022	rc = ahci_deinit_port(ap, &emsg);
2023	if (rc == 0)
2024		ahci_power_down(ap);
2025	else {
2026		ata_port_err(ap, "%s (%d)\n", emsg, rc);
2027		ata_port_freeze(ap);
2028	}
2029
 
2030	return rc;
2031}
2032#endif
2033
2034static int ahci_port_start(struct ata_port *ap)
2035{
2036	struct ahci_host_priv *hpriv = ap->host->private_data;
2037	struct device *dev = ap->host->dev;
2038	struct ahci_port_priv *pp;
2039	void *mem;
2040	dma_addr_t mem_dma;
2041	size_t dma_sz, rx_fis_sz;
2042
2043	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
2044	if (!pp)
2045		return -ENOMEM;
2046
 
 
 
 
 
 
 
 
 
 
2047	/* check FBS capability */
2048	if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
2049		void __iomem *port_mmio = ahci_port_base(ap);
2050		u32 cmd = readl(port_mmio + PORT_CMD);
2051		if (cmd & PORT_CMD_FBSCP)
2052			pp->fbs_supported = true;
2053		else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {
2054			dev_info(dev, "port %d can do FBS, forcing FBSCP\n",
2055				 ap->port_no);
2056			pp->fbs_supported = true;
2057		} else
2058			dev_warn(dev, "port %d is not capable of FBS\n",
2059				 ap->port_no);
2060	}
2061
2062	if (pp->fbs_supported) {
2063		dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
2064		rx_fis_sz = AHCI_RX_FIS_SZ * 16;
2065	} else {
2066		dma_sz = AHCI_PORT_PRIV_DMA_SZ;
2067		rx_fis_sz = AHCI_RX_FIS_SZ;
2068	}
2069
2070	mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
2071	if (!mem)
2072		return -ENOMEM;
2073	memset(mem, 0, dma_sz);
2074
2075	/*
2076	 * First item in chunk of DMA memory: 32-slot command table,
2077	 * 32 bytes each in size
2078	 */
2079	pp->cmd_slot = mem;
2080	pp->cmd_slot_dma = mem_dma;
2081
2082	mem += AHCI_CMD_SLOT_SZ;
2083	mem_dma += AHCI_CMD_SLOT_SZ;
2084
2085	/*
2086	 * Second item: Received-FIS area
2087	 */
2088	pp->rx_fis = mem;
2089	pp->rx_fis_dma = mem_dma;
2090
2091	mem += rx_fis_sz;
2092	mem_dma += rx_fis_sz;
2093
2094	/*
2095	 * Third item: data area for storing a single command
2096	 * and its scatter-gather table
2097	 */
2098	pp->cmd_tbl = mem;
2099	pp->cmd_tbl_dma = mem_dma;
2100
2101	/*
2102	 * Save off initial list of interrupts to be enabled.
2103	 * This could be changed later
2104	 */
2105	pp->intr_mask = DEF_PORT_IRQ;
2106
 
 
 
 
 
 
 
 
2107	ap->private_data = pp;
2108
2109	/* engage engines, captain */
2110	return ahci_port_resume(ap);
2111}
2112
2113static void ahci_port_stop(struct ata_port *ap)
2114{
2115	const char *emsg = NULL;
 
 
2116	int rc;
2117
2118	/* de-initialize port */
2119	rc = ahci_deinit_port(ap, &emsg);
2120	if (rc)
2121		ata_port_warn(ap, "%s (%d)\n", emsg, rc);
 
 
 
 
 
 
2122}
2123
2124void ahci_print_info(struct ata_host *host, const char *scc_s)
2125{
2126	struct ahci_host_priv *hpriv = host->private_data;
2127	void __iomem *mmio = hpriv->mmio;
2128	u32 vers, cap, cap2, impl, speed;
2129	const char *speed_s;
2130
2131	vers = readl(mmio + HOST_VERSION);
2132	cap = hpriv->cap;
2133	cap2 = hpriv->cap2;
2134	impl = hpriv->port_map;
2135
2136	speed = (cap >> 20) & 0xf;
2137	if (speed == 1)
2138		speed_s = "1.5";
2139	else if (speed == 2)
2140		speed_s = "3";
2141	else if (speed == 3)
2142		speed_s = "6";
2143	else
2144		speed_s = "?";
2145
2146	dev_info(host->dev,
2147		"AHCI %02x%02x.%02x%02x "
2148		"%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2149		,
2150
2151		(vers >> 24) & 0xff,
2152		(vers >> 16) & 0xff,
2153		(vers >> 8) & 0xff,
2154		vers & 0xff,
2155
2156		((cap >> 8) & 0x1f) + 1,
2157		(cap & 0x1f) + 1,
2158		speed_s,
2159		impl,
2160		scc_s);
2161
2162	dev_info(host->dev,
2163		"flags: "
2164		"%s%s%s%s%s%s%s"
2165		"%s%s%s%s%s%s%s"
2166		"%s%s%s%s%s%s\n"
 
2167		,
2168
2169		cap & HOST_CAP_64 ? "64bit " : "",
2170		cap & HOST_CAP_NCQ ? "ncq " : "",
2171		cap & HOST_CAP_SNTF ? "sntf " : "",
2172		cap & HOST_CAP_MPS ? "ilck " : "",
2173		cap & HOST_CAP_SSS ? "stag " : "",
2174		cap & HOST_CAP_ALPM ? "pm " : "",
2175		cap & HOST_CAP_LED ? "led " : "",
2176		cap & HOST_CAP_CLO ? "clo " : "",
2177		cap & HOST_CAP_ONLY ? "only " : "",
2178		cap & HOST_CAP_PMP ? "pmp " : "",
2179		cap & HOST_CAP_FBS ? "fbs " : "",
2180		cap & HOST_CAP_PIO_MULTI ? "pio " : "",
2181		cap & HOST_CAP_SSC ? "slum " : "",
2182		cap & HOST_CAP_PART ? "part " : "",
2183		cap & HOST_CAP_CCC ? "ccc " : "",
2184		cap & HOST_CAP_EMS ? "ems " : "",
2185		cap & HOST_CAP_SXS ? "sxs " : "",
 
 
 
2186		cap2 & HOST_CAP2_APST ? "apst " : "",
2187		cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
2188		cap2 & HOST_CAP2_BOH ? "boh " : ""
2189		);
2190}
2191EXPORT_SYMBOL_GPL(ahci_print_info);
2192
2193void ahci_set_em_messages(struct ahci_host_priv *hpriv,
2194			  struct ata_port_info *pi)
2195{
2196	u8 messages;
2197	void __iomem *mmio = hpriv->mmio;
2198	u32 em_loc = readl(mmio + HOST_EM_LOC);
2199	u32 em_ctl = readl(mmio + HOST_EM_CTL);
2200
2201	if (!ahci_em_messages || !(hpriv->cap & HOST_CAP_EMS))
2202		return;
2203
2204	messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
2205
2206	if (messages) {
2207		/* store em_loc */
2208		hpriv->em_loc = ((em_loc >> 16) * 4);
2209		hpriv->em_buf_sz = ((em_loc & 0xff) * 4);
2210		hpriv->em_msg_type = messages;
2211		pi->flags |= ATA_FLAG_EM;
2212		if (!(em_ctl & EM_CTL_ALHD))
2213			pi->flags |= ATA_FLAG_SW_ACTIVITY;
2214	}
2215}
2216EXPORT_SYMBOL_GPL(ahci_set_em_messages);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2217
2218MODULE_AUTHOR("Jeff Garzik");
2219MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2220MODULE_LICENSE("GPL");