1/*
   2 *  libahci.c - Common AHCI SATA low-level routines
   3 *
   4 *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
   5 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
   6 *		    on emails.
   7 *
   8 *  Copyright 2004-2005 Red Hat, Inc.
   9 *
  10 *
  11 *  This program is free software; you can redistribute it and/or modify
  12 *  it under the terms of the GNU General Public License as published by
  13 *  the Free Software Foundation; either version 2, or (at your option)
  14 *  any later version.
  15 *
  16 *  This program is distributed in the hope that it will be useful,
  17 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 *  GNU General Public License for more details.
  20 *
  21 *  You should have received a copy of the GNU General Public License
  22 *  along with this program; see the file COPYING.  If not, write to
  23 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  24 *
  25 *
  26 * libata documentation is available via 'make {ps|pdf}docs',
  27 * as Documentation/DocBook/libata.*
  28 *
  29 * AHCI hardware documentation:
  30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
  31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
  32 *
  33 */
  34
 
  35#include <linux/kernel.h>
  36#include <linux/gfp.h>
  37#include <linux/module.h>
  38#include <linux/init.h>
  39#include <linux/blkdev.h>
  40#include <linux/delay.h>
  41#include <linux/interrupt.h>
  42#include <linux/dma-mapping.h>
  43#include <linux/device.h>
  44#include <scsi/scsi_host.h>
  45#include <scsi/scsi_cmnd.h>
  46#include <linux/libata.h>
 
  47#include "ahci.h"
 
  48
  49static int ahci_skip_host_reset;
  50int ahci_ignore_sss;
  51EXPORT_SYMBOL_GPL(ahci_ignore_sss);
  52
  53module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
  54MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
  55
  56module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
  57MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
  58
  59static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
  60			unsigned hints);
  61static ssize_t ahci_led_show(struct ata_port *ap, char *buf);
  62static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
  63			      size_t size);
  64static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
  65					ssize_t size);
  66
  67
  68
  69static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
  70static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
  71static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
  72static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
  73static int ahci_port_start(struct ata_port *ap);
  74static void ahci_port_stop(struct ata_port *ap);
  75static void ahci_qc_prep(struct ata_queued_cmd *qc);
  76static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
  77static void ahci_freeze(struct ata_port *ap);
  78static void ahci_thaw(struct ata_port *ap);
 
  79static void ahci_enable_fbs(struct ata_port *ap);
  80static void ahci_disable_fbs(struct ata_port *ap);
  81static void ahci_pmp_attach(struct ata_port *ap);
  82static void ahci_pmp_detach(struct ata_port *ap);
  83static int ahci_softreset(struct ata_link *link, unsigned int *class,
  84			  unsigned long deadline);
  85static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
  86			  unsigned long deadline);
  87static int ahci_hardreset(struct ata_link *link, unsigned int *class,
  88			  unsigned long deadline);
  89static void ahci_postreset(struct ata_link *link, unsigned int *class);
  90static void ahci_error_handler(struct ata_port *ap);
  91static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
  92static void ahci_dev_config(struct ata_device *dev);
  93#ifdef CONFIG_PM
  94static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
  95#endif
  96static ssize_t ahci_activity_show(struct ata_device *dev, char *buf);
  97static ssize_t ahci_activity_store(struct ata_device *dev,
  98				   enum sw_activity val);
  99static void ahci_init_sw_activity(struct ata_link *link);
 100
 101static ssize_t ahci_show_host_caps(struct device *dev,
 102				   struct device_attribute *attr, char *buf);
 103static ssize_t ahci_show_host_cap2(struct device *dev,
 104				   struct device_attribute *attr, char *buf);
 105static ssize_t ahci_show_host_version(struct device *dev,
 106				      struct device_attribute *attr, char *buf);
 107static ssize_t ahci_show_port_cmd(struct device *dev,
 108				  struct device_attribute *attr, char *buf);
 109static ssize_t ahci_read_em_buffer(struct device *dev,
 110				   struct device_attribute *attr, char *buf);
 111static ssize_t ahci_store_em_buffer(struct device *dev,
 112				    struct device_attribute *attr,
 113				    const char *buf, size_t size);
 114static ssize_t ahci_show_em_supported(struct device *dev,
 115				      struct device_attribute *attr, char *buf);
 
 116
 117static DEVICE_ATTR(ahci_host_caps, S_IRUGO, ahci_show_host_caps, NULL);
 118static DEVICE_ATTR(ahci_host_cap2, S_IRUGO, ahci_show_host_cap2, NULL);
 119static DEVICE_ATTR(ahci_host_version, S_IRUGO, ahci_show_host_version, NULL);
 120static DEVICE_ATTR(ahci_port_cmd, S_IRUGO, ahci_show_port_cmd, NULL);
 121static DEVICE_ATTR(em_buffer, S_IWUSR | S_IRUGO,
 122		   ahci_read_em_buffer, ahci_store_em_buffer);
 123static DEVICE_ATTR(em_message_supported, S_IRUGO, ahci_show_em_supported, NULL);
 124
 125struct device_attribute *ahci_shost_attrs[] = {
 126	&dev_attr_link_power_management_policy,
 127	&dev_attr_em_message_type,
 128	&dev_attr_em_message,
 129	&dev_attr_ahci_host_caps,
 130	&dev_attr_ahci_host_cap2,
 131	&dev_attr_ahci_host_version,
 132	&dev_attr_ahci_port_cmd,
 133	&dev_attr_em_buffer,
 134	&dev_attr_em_message_supported,
 135	NULL
 136};
 137EXPORT_SYMBOL_GPL(ahci_shost_attrs);
 138
 139struct device_attribute *ahci_sdev_attrs[] = {
 140	&dev_attr_sw_activity,
 141	&dev_attr_unload_heads,
 
 
 
 142	NULL
 143};
 144EXPORT_SYMBOL_GPL(ahci_sdev_attrs);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 145
 146struct ata_port_operations ahci_ops = {
 147	.inherits		= &sata_pmp_port_ops,
 148
 149	.qc_defer		= ahci_pmp_qc_defer,
 150	.qc_prep		= ahci_qc_prep,
 151	.qc_issue		= ahci_qc_issue,
 152	.qc_fill_rtf		= ahci_qc_fill_rtf,
 153
 154	.freeze			= ahci_freeze,
 155	.thaw			= ahci_thaw,
 156	.softreset		= ahci_softreset,
 157	.hardreset		= ahci_hardreset,
 158	.postreset		= ahci_postreset,
 159	.pmp_softreset		= ahci_softreset,
 160	.error_handler		= ahci_error_handler,
 161	.post_internal_cmd	= ahci_post_internal_cmd,
 162	.dev_config		= ahci_dev_config,
 163
 164	.scr_read		= ahci_scr_read,
 165	.scr_write		= ahci_scr_write,
 166	.pmp_attach		= ahci_pmp_attach,
 167	.pmp_detach		= ahci_pmp_detach,
 168
 169	.set_lpm		= ahci_set_lpm,
 170	.em_show		= ahci_led_show,
 171	.em_store		= ahci_led_store,
 172	.sw_activity_show	= ahci_activity_show,
 173	.sw_activity_store	= ahci_activity_store,
 
 174#ifdef CONFIG_PM
 175	.port_suspend		= ahci_port_suspend,
 176	.port_resume		= ahci_port_resume,
 177#endif
 178	.port_start		= ahci_port_start,
 179	.port_stop		= ahci_port_stop,
 180};
 181EXPORT_SYMBOL_GPL(ahci_ops);
 182
 183struct ata_port_operations ahci_pmp_retry_srst_ops = {
 184	.inherits		= &ahci_ops,
 185	.softreset		= ahci_pmp_retry_softreset,
 186};
 187EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
 188
 189int ahci_em_messages = 1;
 190EXPORT_SYMBOL_GPL(ahci_em_messages);
 191module_param(ahci_em_messages, int, 0444);
 192/* add other LED protocol types when they become supported */
 193MODULE_PARM_DESC(ahci_em_messages,
 194	"AHCI Enclosure Management Message control (0 = off, 1 = on)");
 195
 
 
 
 
 
 196static void ahci_enable_ahci(void __iomem *mmio)
 197{
 198	int i;
 199	u32 tmp;
 200
 201	/* turn on AHCI_EN */
 202	tmp = readl(mmio + HOST_CTL);
 203	if (tmp & HOST_AHCI_EN)
 204		return;
 205
 206	/* Some controllers need AHCI_EN to be written multiple times.
 207	 * Try a few times before giving up.
 208	 */
 209	for (i = 0; i < 5; i++) {
 210		tmp |= HOST_AHCI_EN;
 211		writel(tmp, mmio + HOST_CTL);
 212		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
 213		if (tmp & HOST_AHCI_EN)
 214			return;
 215		msleep(10);
 216	}
 217
 218	WARN_ON(1);
 219}
 220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 221static ssize_t ahci_show_host_caps(struct device *dev,
 222				   struct device_attribute *attr, char *buf)
 223{
 224	struct Scsi_Host *shost = class_to_shost(dev);
 225	struct ata_port *ap = ata_shost_to_port(shost);
 226	struct ahci_host_priv *hpriv = ap->host->private_data;
 227
 228	return sprintf(buf, "%x\n", hpriv->cap);
 229}
 230
 231static ssize_t ahci_show_host_cap2(struct device *dev,
 232				   struct device_attribute *attr, char *buf)
 233{
 234	struct Scsi_Host *shost = class_to_shost(dev);
 235	struct ata_port *ap = ata_shost_to_port(shost);
 236	struct ahci_host_priv *hpriv = ap->host->private_data;
 237
 238	return sprintf(buf, "%x\n", hpriv->cap2);
 239}
 240
 241static ssize_t ahci_show_host_version(struct device *dev,
 242				   struct device_attribute *attr, char *buf)
 243{
 244	struct Scsi_Host *shost = class_to_shost(dev);
 245	struct ata_port *ap = ata_shost_to_port(shost);
 246	struct ahci_host_priv *hpriv = ap->host->private_data;
 247	void __iomem *mmio = hpriv->mmio;
 248
 249	return sprintf(buf, "%x\n", readl(mmio + HOST_VERSION));
 250}
 251
 252static ssize_t ahci_show_port_cmd(struct device *dev,
 253				  struct device_attribute *attr, char *buf)
 254{
 255	struct Scsi_Host *shost = class_to_shost(dev);
 256	struct ata_port *ap = ata_shost_to_port(shost);
 257	void __iomem *port_mmio = ahci_port_base(ap);
 
 
 
 
 
 258
 259	return sprintf(buf, "%x\n", readl(port_mmio + PORT_CMD));
 260}
 261
 262static ssize_t ahci_read_em_buffer(struct device *dev,
 263				   struct device_attribute *attr, char *buf)
 264{
 265	struct Scsi_Host *shost = class_to_shost(dev);
 266	struct ata_port *ap = ata_shost_to_port(shost);
 267	struct ahci_host_priv *hpriv = ap->host->private_data;
 268	void __iomem *mmio = hpriv->mmio;
 269	void __iomem *em_mmio = mmio + hpriv->em_loc;
 270	u32 em_ctl, msg;
 271	unsigned long flags;
 272	size_t count;
 273	int i;
 274
 
 275	spin_lock_irqsave(ap->lock, flags);
 276
 277	em_ctl = readl(mmio + HOST_EM_CTL);
 278	if (!(ap->flags & ATA_FLAG_EM) || em_ctl & EM_CTL_XMT ||
 279	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {
 280		spin_unlock_irqrestore(ap->lock, flags);
 
 281		return -EINVAL;
 282	}
 283
 284	if (!(em_ctl & EM_CTL_MR)) {
 285		spin_unlock_irqrestore(ap->lock, flags);
 
 286		return -EAGAIN;
 287	}
 288
 289	if (!(em_ctl & EM_CTL_SMB))
 290		em_mmio += hpriv->em_buf_sz;
 291
 292	count = hpriv->em_buf_sz;
 293
 294	/* the count should not be larger than PAGE_SIZE */
 295	if (count > PAGE_SIZE) {
 296		if (printk_ratelimit())
 297			ata_port_warn(ap,
 298				      "EM read buffer size too large: "
 299				      "buffer size %u, page size %lu\n",
 300				      hpriv->em_buf_sz, PAGE_SIZE);
 301		count = PAGE_SIZE;
 302	}
 303
 304	for (i = 0; i < count; i += 4) {
 305		msg = readl(em_mmio + i);
 306		buf[i] = msg & 0xff;
 307		buf[i + 1] = (msg >> 8) & 0xff;
 308		buf[i + 2] = (msg >> 16) & 0xff;
 309		buf[i + 3] = (msg >> 24) & 0xff;
 310	}
 311
 312	spin_unlock_irqrestore(ap->lock, flags);
 
 313
 314	return i;
 315}
 316
 317static ssize_t ahci_store_em_buffer(struct device *dev,
 318				    struct device_attribute *attr,
 319				    const char *buf, size_t size)
 320{
 321	struct Scsi_Host *shost = class_to_shost(dev);
 322	struct ata_port *ap = ata_shost_to_port(shost);
 323	struct ahci_host_priv *hpriv = ap->host->private_data;
 324	void __iomem *mmio = hpriv->mmio;
 325	void __iomem *em_mmio = mmio + hpriv->em_loc;
 326	const unsigned char *msg_buf = buf;
 327	u32 em_ctl, msg;
 328	unsigned long flags;
 329	int i;
 330
 331	/* check size validity */
 332	if (!(ap->flags & ATA_FLAG_EM) ||
 333	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) ||
 334	    size % 4 || size > hpriv->em_buf_sz)
 335		return -EINVAL;
 336
 
 337	spin_lock_irqsave(ap->lock, flags);
 338
 339	em_ctl = readl(mmio + HOST_EM_CTL);
 340	if (em_ctl & EM_CTL_TM) {
 341		spin_unlock_irqrestore(ap->lock, flags);
 
 342		return -EBUSY;
 343	}
 344
 345	for (i = 0; i < size; i += 4) {
 346		msg = msg_buf[i] | msg_buf[i + 1] << 8 |
 347		      msg_buf[i + 2] << 16 | msg_buf[i + 3] << 24;
 348		writel(msg, em_mmio + i);
 349	}
 350
 351	writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
 352
 353	spin_unlock_irqrestore(ap->lock, flags);
 
 354
 355	return size;
 356}
 357
 358static ssize_t ahci_show_em_supported(struct device *dev,
 359				      struct device_attribute *attr, char *buf)
 360{
 361	struct Scsi_Host *shost = class_to_shost(dev);
 362	struct ata_port *ap = ata_shost_to_port(shost);
 363	struct ahci_host_priv *hpriv = ap->host->private_data;
 364	void __iomem *mmio = hpriv->mmio;
 365	u32 em_ctl;
 366
 
 367	em_ctl = readl(mmio + HOST_EM_CTL);
 
 368
 369	return sprintf(buf, "%s%s%s%s\n",
 370		       em_ctl & EM_CTL_LED ? "led " : "",
 371		       em_ctl & EM_CTL_SAFTE ? "saf-te " : "",
 372		       em_ctl & EM_CTL_SES ? "ses-2 " : "",
 373		       em_ctl & EM_CTL_SGPIO ? "sgpio " : "");
 374}
 375
 376/**
 377 *	ahci_save_initial_config - Save and fixup initial config values
 378 *	@dev: target AHCI device
 379 *	@hpriv: host private area to store config values
 380 *	@force_port_map: force port map to a specified value
 381 *	@mask_port_map: mask out particular bits from port map
 382 *
 383 *	Some registers containing configuration info might be setup by
 384 *	BIOS and might be cleared on reset.  This function saves the
 385 *	initial values of those registers into @hpriv such that they
 386 *	can be restored after controller reset.
 387 *
 388 *	If inconsistent, config values are fixed up by this function.
 389 *
 
 
 
 390 *	LOCKING:
 391 *	None.
 392 */
 393void ahci_save_initial_config(struct device *dev,
 394			      struct ahci_host_priv *hpriv,
 395			      unsigned int force_port_map,
 396			      unsigned int mask_port_map)
 397{
 398	void __iomem *mmio = hpriv->mmio;
 399	u32 cap, cap2, vers, port_map;
 
 
 400	int i;
 401
 402	/* make sure AHCI mode is enabled before accessing CAP */
 403	ahci_enable_ahci(mmio);
 404
 405	/* Values prefixed with saved_ are written back to host after
 406	 * reset.  Values without are used for driver operation.
 
 407	 */
 408	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
 409	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
 
 
 
 
 
 
 
 
 410
 411	/* CAP2 register is only defined for AHCI 1.2 and later */
 412	vers = readl(mmio + HOST_VERSION);
 413	if ((vers >> 16) > 1 ||
 414	   ((vers >> 16) == 1 && (vers & 0xFFFF) >= 0x200))
 415		hpriv->saved_cap2 = cap2 = readl(mmio + HOST_CAP2);
 416	else
 417		hpriv->saved_cap2 = cap2 = 0;
 418
 419	/* some chips have errata preventing 64bit use */
 420	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
 421		dev_info(dev, "controller can't do 64bit DMA, forcing 32bit\n");
 422		cap &= ~HOST_CAP_64;
 423	}
 424
 425	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
 426		dev_info(dev, "controller can't do NCQ, turning off CAP_NCQ\n");
 427		cap &= ~HOST_CAP_NCQ;
 428	}
 429
 430	if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
 431		dev_info(dev, "controller can do NCQ, turning on CAP_NCQ\n");
 432		cap |= HOST_CAP_NCQ;
 433	}
 434
 435	if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
 436		dev_info(dev, "controller can't do PMP, turning off CAP_PMP\n");
 437		cap &= ~HOST_CAP_PMP;
 438	}
 439
 440	if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {
 441		dev_info(dev,
 442			 "controller can't do SNTF, turning off CAP_SNTF\n");
 443		cap &= ~HOST_CAP_SNTF;
 444	}
 445
 
 
 
 
 
 
 
 446	if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {
 447		dev_info(dev, "controller can do FBS, turning on CAP_FBS\n");
 448		cap |= HOST_CAP_FBS;
 449	}
 450
 451	if (force_port_map && port_map != force_port_map) {
 452		dev_info(dev, "forcing port_map 0x%x -> 0x%x\n",
 453			 port_map, force_port_map);
 454		port_map = force_port_map;
 
 
 
 
 455	}
 456
 457	if (mask_port_map) {
 458		dev_warn(dev, "masking port_map 0x%x -> 0x%x\n",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 459			port_map,
 460			port_map & mask_port_map);
 461		port_map &= mask_port_map;
 462	}
 463
 464	/* cross check port_map and cap.n_ports */
 465	if (port_map) {
 466		int map_ports = 0;
 467
 468		for (i = 0; i < AHCI_MAX_PORTS; i++)
 469			if (port_map & (1 << i))
 470				map_ports++;
 471
 472		/* If PI has more ports than n_ports, whine, clear
 473		 * port_map and let it be generated from n_ports.
 474		 */
 475		if (map_ports > ahci_nr_ports(cap)) {
 476			dev_warn(dev,
 477				 "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
 478				 port_map, ahci_nr_ports(cap));
 479			port_map = 0;
 480		}
 481	}
 482
 483	/* fabricate port_map from cap.nr_ports */
 484	if (!port_map) {
 485		port_map = (1 << ahci_nr_ports(cap)) - 1;
 486		dev_warn(dev, "forcing PORTS_IMPL to 0x%x\n", port_map);
 487
 488		/* write the fixed up value to the PI register */
 489		hpriv->saved_port_map = port_map;
 490	}
 491
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 492	/* record values to use during operation */
 493	hpriv->cap = cap;
 494	hpriv->cap2 = cap2;
 
 495	hpriv->port_map = port_map;
 
 
 
 
 
 
 
 
 
 496}
 497EXPORT_SYMBOL_GPL(ahci_save_initial_config);
 498
 499/**
 500 *	ahci_restore_initial_config - Restore initial config
 501 *	@host: target ATA host
 502 *
 503 *	Restore initial config stored by ahci_save_initial_config().
 504 *
 505 *	LOCKING:
 506 *	None.
 507 */
 508static void ahci_restore_initial_config(struct ata_host *host)
 509{
 510	struct ahci_host_priv *hpriv = host->private_data;
 
 511	void __iomem *mmio = hpriv->mmio;
 
 
 512
 513	writel(hpriv->saved_cap, mmio + HOST_CAP);
 514	if (hpriv->saved_cap2)
 515		writel(hpriv->saved_cap2, mmio + HOST_CAP2);
 516	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
 517	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
 
 
 
 
 
 518}
 519
 520static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
 521{
 522	static const int offset[] = {
 523		[SCR_STATUS]		= PORT_SCR_STAT,
 524		[SCR_CONTROL]		= PORT_SCR_CTL,
 525		[SCR_ERROR]		= PORT_SCR_ERR,
 526		[SCR_ACTIVE]		= PORT_SCR_ACT,
 527		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
 528	};
 529	struct ahci_host_priv *hpriv = ap->host->private_data;
 530
 531	if (sc_reg < ARRAY_SIZE(offset) &&
 532	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
 533		return offset[sc_reg];
 534	return 0;
 535}
 536
 537static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
 538{
 539	void __iomem *port_mmio = ahci_port_base(link->ap);
 540	int offset = ahci_scr_offset(link->ap, sc_reg);
 541
 542	if (offset) {
 543		*val = readl(port_mmio + offset);
 544		return 0;
 545	}
 546	return -EINVAL;
 547}
 548
 549static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
 550{
 551	void __iomem *port_mmio = ahci_port_base(link->ap);
 552	int offset = ahci_scr_offset(link->ap, sc_reg);
 553
 554	if (offset) {
 555		writel(val, port_mmio + offset);
 556		return 0;
 557	}
 558	return -EINVAL;
 559}
 560
 561void ahci_start_engine(struct ata_port *ap)
 562{
 563	void __iomem *port_mmio = ahci_port_base(ap);
 564	u32 tmp;
 565
 566	/* start DMA */
 567	tmp = readl(port_mmio + PORT_CMD);
 568	tmp |= PORT_CMD_START;
 569	writel(tmp, port_mmio + PORT_CMD);
 570	readl(port_mmio + PORT_CMD); /* flush */
 571}
 572EXPORT_SYMBOL_GPL(ahci_start_engine);
 573
 574int ahci_stop_engine(struct ata_port *ap)
 575{
 576	void __iomem *port_mmio = ahci_port_base(ap);
 
 577	u32 tmp;
 578
 
 
 
 
 
 
 
 
 
 
 
 
 
 579	tmp = readl(port_mmio + PORT_CMD);
 580
 581	/* check if the HBA is idle */
 582	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
 583		return 0;
 584
 
 
 
 
 
 
 
 
 
 
 585	/* setting HBA to idle */
 586	tmp &= ~PORT_CMD_START;
 587	writel(tmp, port_mmio + PORT_CMD);
 588
 589	/* wait for engine to stop. This could be as long as 500 msec */
 590	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
 591				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
 592	if (tmp & PORT_CMD_LIST_ON)
 593		return -EIO;
 594
 595	return 0;
 596}
 597EXPORT_SYMBOL_GPL(ahci_stop_engine);
 598
 599static void ahci_start_fis_rx(struct ata_port *ap)
 600{
 601	void __iomem *port_mmio = ahci_port_base(ap);
 602	struct ahci_host_priv *hpriv = ap->host->private_data;
 603	struct ahci_port_priv *pp = ap->private_data;
 604	u32 tmp;
 605
 606	/* set FIS registers */
 607	if (hpriv->cap & HOST_CAP_64)
 608		writel((pp->cmd_slot_dma >> 16) >> 16,
 609		       port_mmio + PORT_LST_ADDR_HI);
 610	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
 611
 612	if (hpriv->cap & HOST_CAP_64)
 613		writel((pp->rx_fis_dma >> 16) >> 16,
 614		       port_mmio + PORT_FIS_ADDR_HI);
 615	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
 616
 617	/* enable FIS reception */
 618	tmp = readl(port_mmio + PORT_CMD);
 619	tmp |= PORT_CMD_FIS_RX;
 620	writel(tmp, port_mmio + PORT_CMD);
 621
 622	/* flush */
 623	readl(port_mmio + PORT_CMD);
 624}
 
 625
 626static int ahci_stop_fis_rx(struct ata_port *ap)
 627{
 628	void __iomem *port_mmio = ahci_port_base(ap);
 629	u32 tmp;
 630
 631	/* disable FIS reception */
 632	tmp = readl(port_mmio + PORT_CMD);
 633	tmp &= ~PORT_CMD_FIS_RX;
 634	writel(tmp, port_mmio + PORT_CMD);
 635
 636	/* wait for completion, spec says 500ms, give it 1000 */
 637	tmp = ata_wait_register(ap, port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
 638				PORT_CMD_FIS_ON, 10, 1000);
 639	if (tmp & PORT_CMD_FIS_ON)
 640		return -EBUSY;
 641
 642	return 0;
 643}
 644
 645static void ahci_power_up(struct ata_port *ap)
 646{
 647	struct ahci_host_priv *hpriv = ap->host->private_data;
 648	void __iomem *port_mmio = ahci_port_base(ap);
 649	u32 cmd;
 650
 651	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 652
 653	/* spin up device */
 654	if (hpriv->cap & HOST_CAP_SSS) {
 655		cmd |= PORT_CMD_SPIN_UP;
 656		writel(cmd, port_mmio + PORT_CMD);
 657	}
 658
 659	/* wake up link */
 660	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
 661}
 662
 663static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
 664			unsigned int hints)
 665{
 666	struct ata_port *ap = link->ap;
 667	struct ahci_host_priv *hpriv = ap->host->private_data;
 668	struct ahci_port_priv *pp = ap->private_data;
 669	void __iomem *port_mmio = ahci_port_base(ap);
 670
 671	if (policy != ATA_LPM_MAX_POWER) {
 
 
 
 672		/*
 673		 * Disable interrupts on Phy Ready. This keeps us from
 674		 * getting woken up due to spurious phy ready
 675		 * interrupts.
 676		 */
 677		pp->intr_mask &= ~PORT_IRQ_PHYRDY;
 678		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 679
 680		sata_link_scr_lpm(link, policy, false);
 681	}
 682
 683	if (hpriv->cap & HOST_CAP_ALPM) {
 684		u32 cmd = readl(port_mmio + PORT_CMD);
 685
 686		if (policy == ATA_LPM_MAX_POWER || !(hints & ATA_LPM_HIPM)) {
 687			cmd &= ~(PORT_CMD_ASP | PORT_CMD_ALPE);
 
 688			cmd |= PORT_CMD_ICC_ACTIVE;
 689
 690			writel(cmd, port_mmio + PORT_CMD);
 691			readl(port_mmio + PORT_CMD);
 692
 693			/* wait 10ms to be sure we've come out of LPM state */
 694			ata_msleep(ap, 10);
 
 
 
 695		} else {
 696			cmd |= PORT_CMD_ALPE;
 697			if (policy == ATA_LPM_MIN_POWER)
 698				cmd |= PORT_CMD_ASP;
 
 
 699
 700			/* write out new cmd value */
 701			writel(cmd, port_mmio + PORT_CMD);
 702		}
 703	}
 704
 
 
 
 
 
 
 
 
 
 
 
 705	if (policy == ATA_LPM_MAX_POWER) {
 706		sata_link_scr_lpm(link, policy, false);
 707
 708		/* turn PHYRDY IRQ back on */
 709		pp->intr_mask |= PORT_IRQ_PHYRDY;
 710		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 711	}
 712
 713	return 0;
 714}
 715
 716#ifdef CONFIG_PM
 717static void ahci_power_down(struct ata_port *ap)
 718{
 719	struct ahci_host_priv *hpriv = ap->host->private_data;
 720	void __iomem *port_mmio = ahci_port_base(ap);
 721	u32 cmd, scontrol;
 722
 723	if (!(hpriv->cap & HOST_CAP_SSS))
 724		return;
 725
 726	/* put device into listen mode, first set PxSCTL.DET to 0 */
 727	scontrol = readl(port_mmio + PORT_SCR_CTL);
 728	scontrol &= ~0xf;
 729	writel(scontrol, port_mmio + PORT_SCR_CTL);
 730
 731	/* then set PxCMD.SUD to 0 */
 732	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 733	cmd &= ~PORT_CMD_SPIN_UP;
 734	writel(cmd, port_mmio + PORT_CMD);
 735}
 736#endif
 737
 738static void ahci_start_port(struct ata_port *ap)
 739{
 
 740	struct ahci_port_priv *pp = ap->private_data;
 741	struct ata_link *link;
 742	struct ahci_em_priv *emp;
 743	ssize_t rc;
 744	int i;
 745
 746	/* enable FIS reception */
 747	ahci_start_fis_rx(ap);
 748
 749	/* enable DMA */
 750	ahci_start_engine(ap);
 
 751
 752	/* turn on LEDs */
 753	if (ap->flags & ATA_FLAG_EM) {
 754		ata_for_each_link(link, ap, EDGE) {
 755			emp = &pp->em_priv[link->pmp];
 756
 757			/* EM Transmit bit maybe busy during init */
 758			for (i = 0; i < EM_MAX_RETRY; i++) {
 759				rc = ahci_transmit_led_message(ap,
 760							       emp->led_state,
 761							       4);
 
 
 
 
 
 
 
 
 762				if (rc == -EBUSY)
 763					ata_msleep(ap, 1);
 764				else
 765					break;
 766			}
 767		}
 768	}
 769
 770	if (ap->flags & ATA_FLAG_SW_ACTIVITY)
 771		ata_for_each_link(link, ap, EDGE)
 772			ahci_init_sw_activity(link);
 773
 774}
 775
 776static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
 777{
 778	int rc;
 
 779
 780	/* disable DMA */
 781	rc = ahci_stop_engine(ap);
 782	if (rc) {
 783		*emsg = "failed to stop engine";
 784		return rc;
 785	}
 786
 787	/* disable FIS reception */
 788	rc = ahci_stop_fis_rx(ap);
 789	if (rc) {
 790		*emsg = "failed stop FIS RX";
 791		return rc;
 792	}
 793
 794	return 0;
 795}
 796
 797int ahci_reset_controller(struct ata_host *host)
 798{
 799	struct ahci_host_priv *hpriv = host->private_data;
 800	void __iomem *mmio = hpriv->mmio;
 801	u32 tmp;
 802
 803	/* we must be in AHCI mode, before using anything
 804	 * AHCI-specific, such as HOST_RESET.
 805	 */
 806	ahci_enable_ahci(mmio);
 807
 808	/* global controller reset */
 809	if (!ahci_skip_host_reset) {
 810		tmp = readl(mmio + HOST_CTL);
 811		if ((tmp & HOST_RESET) == 0) {
 812			writel(tmp | HOST_RESET, mmio + HOST_CTL);
 813			readl(mmio + HOST_CTL); /* flush */
 814		}
 815
 816		/*
 817		 * to perform host reset, OS should set HOST_RESET
 818		 * and poll until this bit is read to be "0".
 819		 * reset must complete within 1 second, or
 820		 * the hardware should be considered fried.
 821		 */
 822		tmp = ata_wait_register(NULL, mmio + HOST_CTL, HOST_RESET,
 823					HOST_RESET, 10, 1000);
 824
 825		if (tmp & HOST_RESET) {
 826			dev_err(host->dev, "controller reset failed (0x%x)\n",
 827				tmp);
 828			return -EIO;
 829		}
 830
 831		/* turn on AHCI mode */
 832		ahci_enable_ahci(mmio);
 833
 834		/* Some registers might be cleared on reset.  Restore
 835		 * initial values.
 836		 */
 837		ahci_restore_initial_config(host);
 
 838	} else
 839		dev_info(host->dev, "skipping global host reset\n");
 840
 841	return 0;
 842}
 843EXPORT_SYMBOL_GPL(ahci_reset_controller);
 844
 845static void ahci_sw_activity(struct ata_link *link)
 846{
 847	struct ata_port *ap = link->ap;
 848	struct ahci_port_priv *pp = ap->private_data;
 849	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 850
 851	if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
 852		return;
 853
 854	emp->activity++;
 855	if (!timer_pending(&emp->timer))
 856		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));
 857}
 858
 859static void ahci_sw_activity_blink(unsigned long arg)
 860{
 861	struct ata_link *link = (struct ata_link *)arg;
 
 862	struct ata_port *ap = link->ap;
 863	struct ahci_port_priv *pp = ap->private_data;
 864	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 865	unsigned long led_message = emp->led_state;
 866	u32 activity_led_state;
 867	unsigned long flags;
 868
 869	led_message &= EM_MSG_LED_VALUE;
 870	led_message |= ap->port_no | (link->pmp << 8);
 871
 872	/* check to see if we've had activity.  If so,
 873	 * toggle state of LED and reset timer.  If not,
 874	 * turn LED to desired idle state.
 875	 */
 876	spin_lock_irqsave(ap->lock, flags);
 877	if (emp->saved_activity != emp->activity) {
 878		emp->saved_activity = emp->activity;
 879		/* get the current LED state */
 880		activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
 881
 882		if (activity_led_state)
 883			activity_led_state = 0;
 884		else
 885			activity_led_state = 1;
 886
 887		/* clear old state */
 888		led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
 889
 890		/* toggle state */
 891		led_message |= (activity_led_state << 16);
 892		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));
 893	} else {
 894		/* switch to idle */
 895		led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
 896		if (emp->blink_policy == BLINK_OFF)
 897			led_message |= (1 << 16);
 898	}
 899	spin_unlock_irqrestore(ap->lock, flags);
 900	ahci_transmit_led_message(ap, led_message, 4);
 901}
 902
 903static void ahci_init_sw_activity(struct ata_link *link)
 904{
 905	struct ata_port *ap = link->ap;
 906	struct ahci_port_priv *pp = ap->private_data;
 907	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
 908
 909	/* init activity stats, setup timer */
 910	emp->saved_activity = emp->activity = 0;
 911	setup_timer(&emp->timer, ahci_sw_activity_blink, (unsigned long)link);
 
 912
 913	/* check our blink policy and set flag for link if it's enabled */
 914	if (emp->blink_policy)
 915		link->flags |= ATA_LFLAG_SW_ACTIVITY;
 916}
 917
 918int ahci_reset_em(struct ata_host *host)
 919{
 920	struct ahci_host_priv *hpriv = host->private_data;
 921	void __iomem *mmio = hpriv->mmio;
 922	u32 em_ctl;
 923
 924	em_ctl = readl(mmio + HOST_EM_CTL);
 925	if ((em_ctl & EM_CTL_TM) || (em_ctl & EM_CTL_RST))
 926		return -EINVAL;
 927
 928	writel(em_ctl | EM_CTL_RST, mmio + HOST_EM_CTL);
 929	return 0;
 930}
 931EXPORT_SYMBOL_GPL(ahci_reset_em);
 932
 933static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
 934					ssize_t size)
 935{
 936	struct ahci_host_priv *hpriv = ap->host->private_data;
 937	struct ahci_port_priv *pp = ap->private_data;
 938	void __iomem *mmio = hpriv->mmio;
 939	u32 em_ctl;
 940	u32 message[] = {0, 0};
 941	unsigned long flags;
 942	int pmp;
 943	struct ahci_em_priv *emp;
 944
 945	/* get the slot number from the message */
 946	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
 947	if (pmp < EM_MAX_SLOTS)
 948		emp = &pp->em_priv[pmp];
 949	else
 950		return -EINVAL;
 951
 
 952	spin_lock_irqsave(ap->lock, flags);
 953
 954	/*
 955	 * if we are still busy transmitting a previous message,
 956	 * do not allow
 957	 */
 958	em_ctl = readl(mmio + HOST_EM_CTL);
 959	if (em_ctl & EM_CTL_TM) {
 960		spin_unlock_irqrestore(ap->lock, flags);
 
 961		return -EBUSY;
 962	}
 963
 964	if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {
 965		/*
 966		 * create message header - this is all zero except for
 967		 * the message size, which is 4 bytes.
 968		 */
 969		message[0] |= (4 << 8);
 970
 971		/* ignore 0:4 of byte zero, fill in port info yourself */
 972		message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);
 973
 974		/* write message to EM_LOC */
 975		writel(message[0], mmio + hpriv->em_loc);
 976		writel(message[1], mmio + hpriv->em_loc+4);
 977
 978		/*
 979		 * tell hardware to transmit the message
 980		 */
 981		writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
 982	}
 983
 984	/* save off new led state for port/slot */
 985	emp->led_state = state;
 986
 987	spin_unlock_irqrestore(ap->lock, flags);
 
 
 988	return size;
 989}
 990
 991static ssize_t ahci_led_show(struct ata_port *ap, char *buf)
 992{
 993	struct ahci_port_priv *pp = ap->private_data;
 994	struct ata_link *link;
 995	struct ahci_em_priv *emp;
 996	int rc = 0;
 997
 998	ata_for_each_link(link, ap, EDGE) {
 999		emp = &pp->em_priv[link->pmp];
1000		rc += sprintf(buf, "%lx\n", emp->led_state);
1001	}
1002	return rc;
1003}
1004
1005static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
1006				size_t size)
1007{
1008	int state;
1009	int pmp;
1010	struct ahci_port_priv *pp = ap->private_data;
1011	struct ahci_em_priv *emp;
1012
1013	state = simple_strtoul(buf, NULL, 0);
 
1014
1015	/* get the slot number from the message */
1016	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1017	if (pmp < EM_MAX_SLOTS)
 
1018		emp = &pp->em_priv[pmp];
1019	else
1020		return -EINVAL;
 
1021
1022	/* mask off the activity bits if we are in sw_activity
1023	 * mode, user should turn off sw_activity before setting
1024	 * activity led through em_message
1025	 */
1026	if (emp->blink_policy)
1027		state &= ~EM_MSG_LED_VALUE_ACTIVITY;
1028
1029	return ahci_transmit_led_message(ap, state, size);
1030}
1031
1032static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
1033{
1034	struct ata_link *link = dev->link;
1035	struct ata_port *ap = link->ap;
1036	struct ahci_port_priv *pp = ap->private_data;
1037	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1038	u32 port_led_state = emp->led_state;
1039
1040	/* save the desired Activity LED behavior */
1041	if (val == OFF) {
1042		/* clear LFLAG */
1043		link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
1044
1045		/* set the LED to OFF */
1046		port_led_state &= EM_MSG_LED_VALUE_OFF;
1047		port_led_state |= (ap->port_no | (link->pmp << 8));
1048		ahci_transmit_led_message(ap, port_led_state, 4);
1049	} else {
1050		link->flags |= ATA_LFLAG_SW_ACTIVITY;
1051		if (val == BLINK_OFF) {
1052			/* set LED to ON for idle */
1053			port_led_state &= EM_MSG_LED_VALUE_OFF;
1054			port_led_state |= (ap->port_no | (link->pmp << 8));
1055			port_led_state |= EM_MSG_LED_VALUE_ON; /* check this */
1056			ahci_transmit_led_message(ap, port_led_state, 4);
1057		}
1058	}
1059	emp->blink_policy = val;
1060	return 0;
1061}
1062
1063static ssize_t ahci_activity_show(struct ata_device *dev, char *buf)
1064{
1065	struct ata_link *link = dev->link;
1066	struct ata_port *ap = link->ap;
1067	struct ahci_port_priv *pp = ap->private_data;
1068	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1069
1070	/* display the saved value of activity behavior for this
1071	 * disk.
1072	 */
1073	return sprintf(buf, "%d\n", emp->blink_policy);
1074}
1075
1076static void ahci_port_init(struct device *dev, struct ata_port *ap,
1077			   int port_no, void __iomem *mmio,
1078			   void __iomem *port_mmio)
1079{
 
1080	const char *emsg = NULL;
1081	int rc;
1082	u32 tmp;
1083
1084	/* make sure port is not active */
1085	rc = ahci_deinit_port(ap, &emsg);
1086	if (rc)
1087		dev_warn(dev, "%s (%d)\n", emsg, rc);
1088
1089	/* clear SError */
1090	tmp = readl(port_mmio + PORT_SCR_ERR);
1091	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
1092	writel(tmp, port_mmio + PORT_SCR_ERR);
1093
1094	/* clear port IRQ */
1095	tmp = readl(port_mmio + PORT_IRQ_STAT);
1096	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
1097	if (tmp)
1098		writel(tmp, port_mmio + PORT_IRQ_STAT);
1099
1100	writel(1 << port_no, mmio + HOST_IRQ_STAT);
 
 
 
 
 
1101}
1102
1103void ahci_init_controller(struct ata_host *host)
1104{
1105	struct ahci_host_priv *hpriv = host->private_data;
1106	void __iomem *mmio = hpriv->mmio;
1107	int i;
1108	void __iomem *port_mmio;
1109	u32 tmp;
1110
1111	for (i = 0; i < host->n_ports; i++) {
1112		struct ata_port *ap = host->ports[i];
1113
1114		port_mmio = ahci_port_base(ap);
1115		if (ata_port_is_dummy(ap))
1116			continue;
1117
1118		ahci_port_init(host->dev, ap, i, mmio, port_mmio);
1119	}
1120
1121	tmp = readl(mmio + HOST_CTL);
1122	VPRINTK("HOST_CTL 0x%x\n", tmp);
1123	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
1124	tmp = readl(mmio + HOST_CTL);
1125	VPRINTK("HOST_CTL 0x%x\n", tmp);
1126}
1127EXPORT_SYMBOL_GPL(ahci_init_controller);
1128
1129static void ahci_dev_config(struct ata_device *dev)
1130{
1131	struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
1132
1133	if (hpriv->flags & AHCI_HFLAG_SECT255) {
1134		dev->max_sectors = 255;
1135		ata_dev_info(dev,
1136			     "SB600 AHCI: limiting to 255 sectors per cmd\n");
1137	}
1138}
1139
1140static unsigned int ahci_dev_classify(struct ata_port *ap)
1141{
1142	void __iomem *port_mmio = ahci_port_base(ap);
1143	struct ata_taskfile tf;
1144	u32 tmp;
1145
1146	tmp = readl(port_mmio + PORT_SIG);
1147	tf.lbah		= (tmp >> 24)	& 0xff;
1148	tf.lbam		= (tmp >> 16)	& 0xff;
1149	tf.lbal		= (tmp >> 8)	& 0xff;
1150	tf.nsect	= (tmp)		& 0xff;
1151
1152	return ata_dev_classify(&tf);
1153}
 
1154
1155void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
1156			u32 opts)
1157{
1158	dma_addr_t cmd_tbl_dma;
1159
1160	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
1161
1162	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
1163	pp->cmd_slot[tag].status = 0;
1164	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
1165	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1166}
1167EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot);
1168
1169int ahci_kick_engine(struct ata_port *ap)
1170{
1171	void __iomem *port_mmio = ahci_port_base(ap);
1172	struct ahci_host_priv *hpriv = ap->host->private_data;
1173	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1174	u32 tmp;
1175	int busy, rc;
1176
1177	/* stop engine */
1178	rc = ahci_stop_engine(ap);
1179	if (rc)
1180		goto out_restart;
1181
1182	/* need to do CLO?
1183	 * always do CLO if PMP is attached (AHCI-1.3 9.2)
1184	 */
1185	busy = status & (ATA_BUSY | ATA_DRQ);
1186	if (!busy && !sata_pmp_attached(ap)) {
1187		rc = 0;
1188		goto out_restart;
1189	}
1190
1191	if (!(hpriv->cap & HOST_CAP_CLO)) {
1192		rc = -EOPNOTSUPP;
1193		goto out_restart;
1194	}
1195
1196	/* perform CLO */
1197	tmp = readl(port_mmio + PORT_CMD);
1198	tmp |= PORT_CMD_CLO;
1199	writel(tmp, port_mmio + PORT_CMD);
1200
1201	rc = 0;
1202	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
1203				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
1204	if (tmp & PORT_CMD_CLO)
1205		rc = -EIO;
1206
1207	/* restart engine */
1208 out_restart:
1209	ahci_start_engine(ap);
1210	return rc;
1211}
1212EXPORT_SYMBOL_GPL(ahci_kick_engine);
1213
1214static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
1215				struct ata_taskfile *tf, int is_cmd, u16 flags,
1216				unsigned long timeout_msec)
1217{
1218	const u32 cmd_fis_len = 5; /* five dwords */
1219	struct ahci_port_priv *pp = ap->private_data;
1220	void __iomem *port_mmio = ahci_port_base(ap);
1221	u8 *fis = pp->cmd_tbl;
1222	u32 tmp;
1223
1224	/* prep the command */
1225	ata_tf_to_fis(tf, pmp, is_cmd, fis);
1226	ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
1227
 
 
 
 
 
 
 
 
 
1228	/* issue & wait */
1229	writel(1, port_mmio + PORT_CMD_ISSUE);
1230
1231	if (timeout_msec) {
1232		tmp = ata_wait_register(ap, port_mmio + PORT_CMD_ISSUE,
1233					0x1, 0x1, 1, timeout_msec);
1234		if (tmp & 0x1) {
1235			ahci_kick_engine(ap);
1236			return -EBUSY;
1237		}
1238	} else
1239		readl(port_mmio + PORT_CMD_ISSUE);	/* flush */
1240
1241	return 0;
1242}
1243
1244int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1245		      int pmp, unsigned long deadline,
1246		      int (*check_ready)(struct ata_link *link))
1247{
1248	struct ata_port *ap = link->ap;
1249	struct ahci_host_priv *hpriv = ap->host->private_data;
 
1250	const char *reason = NULL;
1251	unsigned long now, msecs;
1252	struct ata_taskfile tf;
 
1253	int rc;
1254
1255	DPRINTK("ENTER\n");
1256
1257	/* prepare for SRST (AHCI-1.1 10.4.1) */
1258	rc = ahci_kick_engine(ap);
1259	if (rc && rc != -EOPNOTSUPP)
1260		ata_link_warn(link, "failed to reset engine (errno=%d)\n", rc);
1261
 
 
 
 
 
 
 
 
 
 
1262	ata_tf_init(link->device, &tf);
1263
1264	/* issue the first D2H Register FIS */
1265	msecs = 0;
1266	now = jiffies;
1267	if (time_after(deadline, now))
1268		msecs = jiffies_to_msecs(deadline - now);
1269
1270	tf.ctl |= ATA_SRST;
1271	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1272				 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
1273		rc = -EIO;
1274		reason = "1st FIS failed";
1275		goto fail;
1276	}
1277
1278	/* spec says at least 5us, but be generous and sleep for 1ms */
1279	ata_msleep(ap, 1);
1280
1281	/* issue the second D2H Register FIS */
1282	tf.ctl &= ~ATA_SRST;
1283	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
1284
1285	/* wait for link to become ready */
1286	rc = ata_wait_after_reset(link, deadline, check_ready);
1287	if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {
1288		/*
1289		 * Workaround for cases where link online status can't
1290		 * be trusted.  Treat device readiness timeout as link
1291		 * offline.
1292		 */
1293		ata_link_info(link, "device not ready, treating as offline\n");
1294		*class = ATA_DEV_NONE;
1295	} else if (rc) {
1296		/* link occupied, -ENODEV too is an error */
1297		reason = "device not ready";
1298		goto fail;
1299	} else
1300		*class = ahci_dev_classify(ap);
1301
1302	DPRINTK("EXIT, class=%u\n", *class);
 
 
 
1303	return 0;
1304
1305 fail:
1306	ata_link_err(link, "softreset failed (%s)\n", reason);
1307	return rc;
1308}
1309
1310int ahci_check_ready(struct ata_link *link)
1311{
1312	void __iomem *port_mmio = ahci_port_base(link->ap);
1313	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1314
1315	return ata_check_ready(status);
1316}
1317EXPORT_SYMBOL_GPL(ahci_check_ready);
1318
1319static int ahci_softreset(struct ata_link *link, unsigned int *class,
1320			  unsigned long deadline)
1321{
1322	int pmp = sata_srst_pmp(link);
1323
1324	DPRINTK("ENTER\n");
1325
1326	return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
1327}
1328EXPORT_SYMBOL_GPL(ahci_do_softreset);
1329
1330static int ahci_bad_pmp_check_ready(struct ata_link *link)
1331{
1332	void __iomem *port_mmio = ahci_port_base(link->ap);
1333	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1334	u32 irq_status = readl(port_mmio + PORT_IRQ_STAT);
1335
1336	/*
1337	 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1338	 * which can save timeout delay.
1339	 */
1340	if (irq_status & PORT_IRQ_BAD_PMP)
1341		return -EIO;
1342
1343	return ata_check_ready(status);
1344}
1345
1346int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
1347				unsigned long deadline)
1348{
1349	struct ata_port *ap = link->ap;
1350	void __iomem *port_mmio = ahci_port_base(ap);
1351	int pmp = sata_srst_pmp(link);
1352	int rc;
1353	u32 irq_sts;
1354
1355	DPRINTK("ENTER\n");
1356
1357	rc = ahci_do_softreset(link, class, pmp, deadline,
1358			       ahci_bad_pmp_check_ready);
1359
1360	/*
1361	 * Soft reset fails with IPMS set when PMP is enabled but
1362	 * SATA HDD/ODD is connected to SATA port, do soft reset
1363	 * again to port 0.
1364	 */
1365	if (rc == -EIO) {
1366		irq_sts = readl(port_mmio + PORT_IRQ_STAT);
1367		if (irq_sts & PORT_IRQ_BAD_PMP) {
1368			ata_link_printk(link, KERN_WARNING,
1369					"applying PMP SRST workaround "
1370					"and retrying\n");
1371			rc = ahci_do_softreset(link, class, 0, deadline,
1372					       ahci_check_ready);
1373		}
1374	}
1375
1376	return rc;
1377}
1378
1379static int ahci_hardreset(struct ata_link *link, unsigned int *class,
1380			  unsigned long deadline)
1381{
1382	const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
1383	struct ata_port *ap = link->ap;
1384	struct ahci_port_priv *pp = ap->private_data;
 
1385	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1386	struct ata_taskfile tf;
1387	bool online;
1388	int rc;
1389
1390	DPRINTK("ENTER\n");
1391
1392	ahci_stop_engine(ap);
1393
1394	/* clear D2H reception area to properly wait for D2H FIS */
1395	ata_tf_init(link->device, &tf);
1396	tf.command = 0x80;
1397	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1398
1399	rc = sata_link_hardreset(link, timing, deadline, &online,
1400				 ahci_check_ready);
1401
1402	ahci_start_engine(ap);
1403
1404	if (online)
1405		*class = ahci_dev_classify(ap);
1406
1407	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1408	return rc;
1409}
 
 
 
 
 
 
 
 
 
1410
1411static void ahci_postreset(struct ata_link *link, unsigned int *class)
1412{
1413	struct ata_port *ap = link->ap;
1414	void __iomem *port_mmio = ahci_port_base(ap);
1415	u32 new_tmp, tmp;
1416
1417	ata_std_postreset(link, class);
1418
1419	/* Make sure port's ATAPI bit is set appropriately */
1420	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1421	if (*class == ATA_DEV_ATAPI)
1422		new_tmp |= PORT_CMD_ATAPI;
1423	else
1424		new_tmp &= ~PORT_CMD_ATAPI;
1425	if (new_tmp != tmp) {
1426		writel(new_tmp, port_mmio + PORT_CMD);
1427		readl(port_mmio + PORT_CMD); /* flush */
1428	}
1429}
1430
1431static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
1432{
1433	struct scatterlist *sg;
1434	struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
1435	unsigned int si;
1436
1437	VPRINTK("ENTER\n");
1438
1439	/*
1440	 * Next, the S/G list.
1441	 */
1442	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1443		dma_addr_t addr = sg_dma_address(sg);
1444		u32 sg_len = sg_dma_len(sg);
1445
1446		ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
1447		ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
1448		ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
1449	}
1450
1451	return si;
1452}
1453
1454static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
1455{
1456	struct ata_port *ap = qc->ap;
1457	struct ahci_port_priv *pp = ap->private_data;
1458
1459	if (!sata_pmp_attached(ap) || pp->fbs_enabled)
1460		return ata_std_qc_defer(qc);
1461	else
1462		return sata_pmp_qc_defer_cmd_switch(qc);
1463}
1464
1465static void ahci_qc_prep(struct ata_queued_cmd *qc)
1466{
1467	struct ata_port *ap = qc->ap;
1468	struct ahci_port_priv *pp = ap->private_data;
1469	int is_atapi = ata_is_atapi(qc->tf.protocol);
1470	void *cmd_tbl;
1471	u32 opts;
1472	const u32 cmd_fis_len = 5; /* five dwords */
1473	unsigned int n_elem;
1474
1475	/*
1476	 * Fill in command table information.  First, the header,
1477	 * a SATA Register - Host to Device command FIS.
1478	 */
1479	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
1480
1481	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
1482	if (is_atapi) {
1483		memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
1484		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1485	}
1486
1487	n_elem = 0;
1488	if (qc->flags & ATA_QCFLAG_DMAMAP)
1489		n_elem = ahci_fill_sg(qc, cmd_tbl);
1490
1491	/*
1492	 * Fill in command slot information.
1493	 */
1494	opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
1495	if (qc->tf.flags & ATA_TFLAG_WRITE)
1496		opts |= AHCI_CMD_WRITE;
1497	if (is_atapi)
1498		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1499
1500	ahci_fill_cmd_slot(pp, qc->tag, opts);
 
 
1501}
1502
1503static void ahci_fbs_dec_intr(struct ata_port *ap)
1504{
1505	struct ahci_port_priv *pp = ap->private_data;
1506	void __iomem *port_mmio = ahci_port_base(ap);
1507	u32 fbs = readl(port_mmio + PORT_FBS);
1508	int retries = 3;
1509
1510	DPRINTK("ENTER\n");
1511	BUG_ON(!pp->fbs_enabled);
1512
1513	/* time to wait for DEC is not specified by AHCI spec,
1514	 * add a retry loop for safety.
1515	 */
1516	writel(fbs | PORT_FBS_DEC, port_mmio + PORT_FBS);
1517	fbs = readl(port_mmio + PORT_FBS);
1518	while ((fbs & PORT_FBS_DEC) && retries--) {
1519		udelay(1);
1520		fbs = readl(port_mmio + PORT_FBS);
1521	}
1522
1523	if (fbs & PORT_FBS_DEC)
1524		dev_err(ap->host->dev, "failed to clear device error\n");
1525}
1526
1527static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1528{
1529	struct ahci_host_priv *hpriv = ap->host->private_data;
1530	struct ahci_port_priv *pp = ap->private_data;
1531	struct ata_eh_info *host_ehi = &ap->link.eh_info;
1532	struct ata_link *link = NULL;
1533	struct ata_queued_cmd *active_qc;
1534	struct ata_eh_info *active_ehi;
1535	bool fbs_need_dec = false;
1536	u32 serror;
1537
1538	/* determine active link with error */
1539	if (pp->fbs_enabled) {
1540		void __iomem *port_mmio = ahci_port_base(ap);
1541		u32 fbs = readl(port_mmio + PORT_FBS);
1542		int pmp = fbs >> PORT_FBS_DWE_OFFSET;
1543
1544		if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links) &&
1545		    ata_link_online(&ap->pmp_link[pmp])) {
1546			link = &ap->pmp_link[pmp];
1547			fbs_need_dec = true;
1548		}
1549
1550	} else
1551		ata_for_each_link(link, ap, EDGE)
1552			if (ata_link_active(link))
1553				break;
1554
1555	if (!link)
1556		link = &ap->link;
1557
1558	active_qc = ata_qc_from_tag(ap, link->active_tag);
1559	active_ehi = &link->eh_info;
1560
1561	/* record irq stat */
1562	ata_ehi_clear_desc(host_ehi);
1563	ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
1564
1565	/* AHCI needs SError cleared; otherwise, it might lock up */
1566	ahci_scr_read(&ap->link, SCR_ERROR, &serror);
1567	ahci_scr_write(&ap->link, SCR_ERROR, serror);
1568	host_ehi->serror |= serror;
1569
1570	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1571	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1572		irq_stat &= ~PORT_IRQ_IF_ERR;
1573
1574	if (irq_stat & PORT_IRQ_TF_ERR) {
1575		/* If qc is active, charge it; otherwise, the active
1576		 * link.  There's no active qc on NCQ errors.  It will
1577		 * be determined by EH by reading log page 10h.
1578		 */
1579		if (active_qc)
1580			active_qc->err_mask |= AC_ERR_DEV;
1581		else
1582			active_ehi->err_mask |= AC_ERR_DEV;
1583
1584		if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
1585			host_ehi->serror &= ~SERR_INTERNAL;
1586	}
1587
1588	if (irq_stat & PORT_IRQ_UNK_FIS) {
1589		u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);
1590
1591		active_ehi->err_mask |= AC_ERR_HSM;
1592		active_ehi->action |= ATA_EH_RESET;
1593		ata_ehi_push_desc(active_ehi,
1594				  "unknown FIS %08x %08x %08x %08x" ,
1595				  unk[0], unk[1], unk[2], unk[3]);
1596	}
1597
1598	if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
1599		active_ehi->err_mask |= AC_ERR_HSM;
1600		active_ehi->action |= ATA_EH_RESET;
1601		ata_ehi_push_desc(active_ehi, "incorrect PMP");
1602	}
1603
1604	if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
1605		host_ehi->err_mask |= AC_ERR_HOST_BUS;
1606		host_ehi->action |= ATA_EH_RESET;
1607		ata_ehi_push_desc(host_ehi, "host bus error");
1608	}
1609
1610	if (irq_stat & PORT_IRQ_IF_ERR) {
1611		if (fbs_need_dec)
1612			active_ehi->err_mask |= AC_ERR_DEV;
1613		else {
1614			host_ehi->err_mask |= AC_ERR_ATA_BUS;
1615			host_ehi->action |= ATA_EH_RESET;
1616		}
1617
1618		ata_ehi_push_desc(host_ehi, "interface fatal error");
1619	}
1620
1621	if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
1622		ata_ehi_hotplugged(host_ehi);
1623		ata_ehi_push_desc(host_ehi, "%s",
1624			irq_stat & PORT_IRQ_CONNECT ?
1625			"connection status changed" : "PHY RDY changed");
1626	}
1627
1628	/* okay, let's hand over to EH */
1629
1630	if (irq_stat & PORT_IRQ_FREEZE)
1631		ata_port_freeze(ap);
1632	else if (fbs_need_dec) {
1633		ata_link_abort(link);
1634		ahci_fbs_dec_intr(ap);
1635	} else
1636		ata_port_abort(ap);
1637}
1638
1639static void ahci_port_intr(struct ata_port *ap)
 
1640{
1641	void __iomem *port_mmio = ahci_port_base(ap);
1642	struct ata_eh_info *ehi = &ap->link.eh_info;
1643	struct ahci_port_priv *pp = ap->private_data;
1644	struct ahci_host_priv *hpriv = ap->host->private_data;
1645	int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
1646	u32 status, qc_active = 0;
1647	int rc;
1648
1649	status = readl(port_mmio + PORT_IRQ_STAT);
1650	writel(status, port_mmio + PORT_IRQ_STAT);
1651
1652	/* ignore BAD_PMP while resetting */
1653	if (unlikely(resetting))
1654		status &= ~PORT_IRQ_BAD_PMP;
1655
1656	/* if LPM is enabled, PHYRDY doesn't mean anything */
1657	if (ap->link.lpm_policy > ATA_LPM_MAX_POWER) {
1658		status &= ~PORT_IRQ_PHYRDY;
1659		ahci_scr_write(&ap->link, SCR_ERROR, SERR_PHYRDY_CHG);
1660	}
1661
1662	if (unlikely(status & PORT_IRQ_ERROR)) {
1663		ahci_error_intr(ap, status);
1664		return;
1665	}
1666
1667	if (status & PORT_IRQ_SDB_FIS) {
1668		/* If SNotification is available, leave notification
1669		 * handling to sata_async_notification().  If not,
1670		 * emulate it by snooping SDB FIS RX area.
1671		 *
1672		 * Snooping FIS RX area is probably cheaper than
1673		 * poking SNotification but some constrollers which
1674		 * implement SNotification, ICH9 for example, don't
1675		 * store AN SDB FIS into receive area.
1676		 */
1677		if (hpriv->cap & HOST_CAP_SNTF)
1678			sata_async_notification(ap);
1679		else {
1680			/* If the 'N' bit in word 0 of the FIS is set,
1681			 * we just received asynchronous notification.
1682			 * Tell libata about it.
1683			 *
1684			 * Lack of SNotification should not appear in
1685			 * ahci 1.2, so the workaround is unnecessary
1686			 * when FBS is enabled.
1687			 */
1688			if (pp->fbs_enabled)
1689				WARN_ON_ONCE(1);
1690			else {
1691				const __le32 *f = pp->rx_fis + RX_FIS_SDB;
1692				u32 f0 = le32_to_cpu(f[0]);
1693				if (f0 & (1 << 15))
1694					sata_async_notification(ap);
1695			}
1696		}
1697	}
1698
1699	/* pp->active_link is not reliable once FBS is enabled, both
1700	 * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1701	 * NCQ and non-NCQ commands may be in flight at the same time.
1702	 */
1703	if (pp->fbs_enabled) {
1704		if (ap->qc_active) {
1705			qc_active = readl(port_mmio + PORT_SCR_ACT);
1706			qc_active |= readl(port_mmio + PORT_CMD_ISSUE);
1707		}
1708	} else {
1709		/* pp->active_link is valid iff any command is in flight */
1710		if (ap->qc_active && pp->active_link->sactive)
1711			qc_active = readl(port_mmio + PORT_SCR_ACT);
1712		else
1713			qc_active = readl(port_mmio + PORT_CMD_ISSUE);
1714	}
1715
1716
1717	rc = ata_qc_complete_multiple(ap, qc_active);
1718
1719	/* while resetting, invalid completions are expected */
1720	if (unlikely(rc < 0 && !resetting)) {
1721		ehi->err_mask |= AC_ERR_HSM;
1722		ehi->action |= ATA_EH_RESET;
1723		ata_port_freeze(ap);
1724	}
1725}
1726
1727irqreturn_t ahci_interrupt(int irq, void *dev_instance)
1728{
1729	struct ata_host *host = dev_instance;
1730	struct ahci_host_priv *hpriv;
1731	unsigned int i, handled = 0;
1732	void __iomem *mmio;
1733	u32 irq_stat, irq_masked;
1734
1735	VPRINTK("ENTER\n");
 
1736
1737	hpriv = host->private_data;
1738	mmio = hpriv->mmio;
1739
1740	/* sigh.  0xffffffff is a valid return from h/w */
1741	irq_stat = readl(mmio + HOST_IRQ_STAT);
1742	if (!irq_stat)
1743		return IRQ_NONE;
 
1744
1745	irq_masked = irq_stat & hpriv->port_map;
 
1746
1747	spin_lock(&host->lock);
 
 
 
 
 
 
 
 
 
1748
1749	for (i = 0; i < host->n_ports; i++) {
1750		struct ata_port *ap;
1751
1752		if (!(irq_masked & (1 << i)))
1753			continue;
1754
1755		ap = host->ports[i];
1756		if (ap) {
1757			ahci_port_intr(ap);
1758			VPRINTK("port %u\n", i);
1759		} else {
1760			VPRINTK("port %u (no irq)\n", i);
1761			if (ata_ratelimit())
1762				dev_warn(host->dev,
1763					 "interrupt on disabled port %u\n", i);
1764		}
1765
1766		handled = 1;
1767	}
1768
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1769	/* HOST_IRQ_STAT behaves as level triggered latch meaning that
1770	 * it should be cleared after all the port events are cleared;
1771	 * otherwise, it will raise a spurious interrupt after each
1772	 * valid one.  Please read section 10.6.2 of ahci 1.1 for more
1773	 * information.
1774	 *
1775	 * Also, use the unmasked value to clear interrupt as spurious
1776	 * pending event on a dummy port might cause screaming IRQ.
1777	 */
1778	writel(irq_stat, mmio + HOST_IRQ_STAT);
1779
1780	spin_unlock(&host->lock);
1781
1782	VPRINTK("EXIT\n");
1783
1784	return IRQ_RETVAL(handled);
1785}
1786EXPORT_SYMBOL_GPL(ahci_interrupt);
1787
1788static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
1789{
1790	struct ata_port *ap = qc->ap;
1791	void __iomem *port_mmio = ahci_port_base(ap);
1792	struct ahci_port_priv *pp = ap->private_data;
1793
1794	/* Keep track of the currently active link.  It will be used
1795	 * in completion path to determine whether NCQ phase is in
1796	 * progress.
1797	 */
1798	pp->active_link = qc->dev->link;
1799
1800	if (qc->tf.protocol == ATA_PROT_NCQ)
1801		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
1802
1803	if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
1804		u32 fbs = readl(port_mmio + PORT_FBS);
1805		fbs &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
1806		fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
1807		writel(fbs, port_mmio + PORT_FBS);
1808		pp->fbs_last_dev = qc->dev->link->pmp;
1809	}
1810
1811	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
1812
1813	ahci_sw_activity(qc->dev->link);
1814
1815	return 0;
1816}
 
1817
1818static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
1819{
1820	struct ahci_port_priv *pp = qc->ap->private_data;
1821	u8 *rx_fis = pp->rx_fis;
1822
1823	if (pp->fbs_enabled)
1824		rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
1825
1826	/*
1827	 * After a successful execution of an ATA PIO data-in command,
1828	 * the device doesn't send D2H Reg FIS to update the TF and
1829	 * the host should take TF and E_Status from the preceding PIO
1830	 * Setup FIS.
1831	 */
1832	if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&
1833	    !(qc->flags & ATA_QCFLAG_FAILED)) {
1834		ata_tf_from_fis(rx_fis + RX_FIS_PIO_SETUP, &qc->result_tf);
1835		qc->result_tf.command = (rx_fis + RX_FIS_PIO_SETUP)[15];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1836	} else
1837		ata_tf_from_fis(rx_fis + RX_FIS_D2H_REG, &qc->result_tf);
1838
1839	return true;
1840}
1841
1842static void ahci_freeze(struct ata_port *ap)
1843{
1844	void __iomem *port_mmio = ahci_port_base(ap);
1845
1846	/* turn IRQ off */
1847	writel(0, port_mmio + PORT_IRQ_MASK);
1848}
1849
1850static void ahci_thaw(struct ata_port *ap)
1851{
1852	struct ahci_host_priv *hpriv = ap->host->private_data;
1853	void __iomem *mmio = hpriv->mmio;
1854	void __iomem *port_mmio = ahci_port_base(ap);
1855	u32 tmp;
1856	struct ahci_port_priv *pp = ap->private_data;
1857
1858	/* clear IRQ */
1859	tmp = readl(port_mmio + PORT_IRQ_STAT);
1860	writel(tmp, port_mmio + PORT_IRQ_STAT);
1861	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
1862
1863	/* turn IRQ back on */
1864	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
1865}
1866
1867static void ahci_error_handler(struct ata_port *ap)
1868{
1869	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
 
 
1870		/* restart engine */
1871		ahci_stop_engine(ap);
1872		ahci_start_engine(ap);
1873	}
1874
1875	sata_pmp_error_handler(ap);
1876
1877	if (!ata_dev_enabled(ap->link.device))
1878		ahci_stop_engine(ap);
1879}
 
1880
1881static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
1882{
1883	struct ata_port *ap = qc->ap;
1884
1885	/* make DMA engine forget about the failed command */
1886	if (qc->flags & ATA_QCFLAG_FAILED)
1887		ahci_kick_engine(ap);
1888}
1889
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1890static void ahci_enable_fbs(struct ata_port *ap)
1891{
 
1892	struct ahci_port_priv *pp = ap->private_data;
1893	void __iomem *port_mmio = ahci_port_base(ap);
1894	u32 fbs;
1895	int rc;
1896
1897	if (!pp->fbs_supported)
1898		return;
1899
1900	fbs = readl(port_mmio + PORT_FBS);
1901	if (fbs & PORT_FBS_EN) {
1902		pp->fbs_enabled = true;
1903		pp->fbs_last_dev = -1; /* initialization */
1904		return;
1905	}
1906
1907	rc = ahci_stop_engine(ap);
1908	if (rc)
1909		return;
1910
1911	writel(fbs | PORT_FBS_EN, port_mmio + PORT_FBS);
1912	fbs = readl(port_mmio + PORT_FBS);
1913	if (fbs & PORT_FBS_EN) {
1914		dev_info(ap->host->dev, "FBS is enabled\n");
1915		pp->fbs_enabled = true;
1916		pp->fbs_last_dev = -1; /* initialization */
1917	} else
1918		dev_err(ap->host->dev, "Failed to enable FBS\n");
1919
1920	ahci_start_engine(ap);
1921}
1922
1923static void ahci_disable_fbs(struct ata_port *ap)
1924{
 
1925	struct ahci_port_priv *pp = ap->private_data;
1926	void __iomem *port_mmio = ahci_port_base(ap);
1927	u32 fbs;
1928	int rc;
1929
1930	if (!pp->fbs_supported)
1931		return;
1932
1933	fbs = readl(port_mmio + PORT_FBS);
1934	if ((fbs & PORT_FBS_EN) == 0) {
1935		pp->fbs_enabled = false;
1936		return;
1937	}
1938
1939	rc = ahci_stop_engine(ap);
1940	if (rc)
1941		return;
1942
1943	writel(fbs & ~PORT_FBS_EN, port_mmio + PORT_FBS);
1944	fbs = readl(port_mmio + PORT_FBS);
1945	if (fbs & PORT_FBS_EN)
1946		dev_err(ap->host->dev, "Failed to disable FBS\n");
1947	else {
1948		dev_info(ap->host->dev, "FBS is disabled\n");
1949		pp->fbs_enabled = false;
1950	}
1951
1952	ahci_start_engine(ap);
1953}
1954
1955static void ahci_pmp_attach(struct ata_port *ap)
1956{
1957	void __iomem *port_mmio = ahci_port_base(ap);
1958	struct ahci_port_priv *pp = ap->private_data;
1959	u32 cmd;
1960
1961	cmd = readl(port_mmio + PORT_CMD);
1962	cmd |= PORT_CMD_PMP;
1963	writel(cmd, port_mmio + PORT_CMD);
1964
1965	ahci_enable_fbs(ap);
1966
1967	pp->intr_mask |= PORT_IRQ_BAD_PMP;
1968
1969	/*
1970	 * We must not change the port interrupt mask register if the
1971	 * port is marked frozen, the value in pp->intr_mask will be
1972	 * restored later when the port is thawed.
1973	 *
1974	 * Note that during initialization, the port is marked as
1975	 * frozen since the irq handler is not yet registered.
1976	 */
1977	if (!(ap->pflags & ATA_PFLAG_FROZEN))
1978		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
1979}
1980
1981static void ahci_pmp_detach(struct ata_port *ap)
1982{
1983	void __iomem *port_mmio = ahci_port_base(ap);
1984	struct ahci_port_priv *pp = ap->private_data;
1985	u32 cmd;
1986
1987	ahci_disable_fbs(ap);
1988
1989	cmd = readl(port_mmio + PORT_CMD);
1990	cmd &= ~PORT_CMD_PMP;
1991	writel(cmd, port_mmio + PORT_CMD);
1992
1993	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
1994
1995	/* see comment above in ahci_pmp_attach() */
1996	if (!(ap->pflags & ATA_PFLAG_FROZEN))
1997		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
1998}
1999
2000int ahci_port_resume(struct ata_port *ap)
2001{
 
 
2002	ahci_power_up(ap);
2003	ahci_start_port(ap);
2004
2005	if (sata_pmp_attached(ap))
2006		ahci_pmp_attach(ap);
2007	else
2008		ahci_pmp_detach(ap);
2009
2010	return 0;
2011}
2012EXPORT_SYMBOL_GPL(ahci_port_resume);
2013
2014#ifdef CONFIG_PM
 
 
 
 
 
 
 
 
 
 
 
 
2015static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
2016{
2017	const char *emsg = NULL;
2018	int rc;
2019
2020	rc = ahci_deinit_port(ap, &emsg);
2021	if (rc == 0)
2022		ahci_power_down(ap);
2023	else {
2024		ata_port_err(ap, "%s (%d)\n", emsg, rc);
2025		ahci_start_port(ap);
2026	}
2027
 
 
 
 
2028	return rc;
2029}
2030#endif
2031
2032static int ahci_port_start(struct ata_port *ap)
2033{
2034	struct ahci_host_priv *hpriv = ap->host->private_data;
2035	struct device *dev = ap->host->dev;
2036	struct ahci_port_priv *pp;
2037	void *mem;
2038	dma_addr_t mem_dma;
2039	size_t dma_sz, rx_fis_sz;
2040
2041	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
2042	if (!pp)
2043		return -ENOMEM;
2044
 
 
 
 
 
 
 
 
 
 
2045	/* check FBS capability */
2046	if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
2047		void __iomem *port_mmio = ahci_port_base(ap);
2048		u32 cmd = readl(port_mmio + PORT_CMD);
2049		if (cmd & PORT_CMD_FBSCP)
2050			pp->fbs_supported = true;
2051		else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {
2052			dev_info(dev, "port %d can do FBS, forcing FBSCP\n",
2053				 ap->port_no);
2054			pp->fbs_supported = true;
2055		} else
2056			dev_warn(dev, "port %d is not capable of FBS\n",
2057				 ap->port_no);
2058	}
2059
2060	if (pp->fbs_supported) {
2061		dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
2062		rx_fis_sz = AHCI_RX_FIS_SZ * 16;
2063	} else {
2064		dma_sz = AHCI_PORT_PRIV_DMA_SZ;
2065		rx_fis_sz = AHCI_RX_FIS_SZ;
2066	}
2067
2068	mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
2069	if (!mem)
2070		return -ENOMEM;
2071	memset(mem, 0, dma_sz);
2072
2073	/*
2074	 * First item in chunk of DMA memory: 32-slot command table,
2075	 * 32 bytes each in size
2076	 */
2077	pp->cmd_slot = mem;
2078	pp->cmd_slot_dma = mem_dma;
2079
2080	mem += AHCI_CMD_SLOT_SZ;
2081	mem_dma += AHCI_CMD_SLOT_SZ;
2082
2083	/*
2084	 * Second item: Received-FIS area
2085	 */
2086	pp->rx_fis = mem;
2087	pp->rx_fis_dma = mem_dma;
2088
2089	mem += rx_fis_sz;
2090	mem_dma += rx_fis_sz;
2091
2092	/*
2093	 * Third item: data area for storing a single command
2094	 * and its scatter-gather table
2095	 */
2096	pp->cmd_tbl = mem;
2097	pp->cmd_tbl_dma = mem_dma;
2098
2099	/*
2100	 * Save off initial list of interrupts to be enabled.
2101	 * This could be changed later
2102	 */
2103	pp->intr_mask = DEF_PORT_IRQ;
2104
 
 
 
 
 
 
 
 
2105	ap->private_data = pp;
2106
2107	/* engage engines, captain */
2108	return ahci_port_resume(ap);
2109}
2110
2111static void ahci_port_stop(struct ata_port *ap)
2112{
2113	const char *emsg = NULL;
 
 
2114	int rc;
2115
2116	/* de-initialize port */
2117	rc = ahci_deinit_port(ap, &emsg);
2118	if (rc)
2119		ata_port_warn(ap, "%s (%d)\n", emsg, rc);
 
 
 
 
 
 
 
 
2120}
2121
2122void ahci_print_info(struct ata_host *host, const char *scc_s)
2123{
2124	struct ahci_host_priv *hpriv = host->private_data;
2125	void __iomem *mmio = hpriv->mmio;
2126	u32 vers, cap, cap2, impl, speed;
2127	const char *speed_s;
2128
2129	vers = readl(mmio + HOST_VERSION);
2130	cap = hpriv->cap;
2131	cap2 = hpriv->cap2;
2132	impl = hpriv->port_map;
2133
2134	speed = (cap >> 20) & 0xf;
2135	if (speed == 1)
2136		speed_s = "1.5";
2137	else if (speed == 2)
2138		speed_s = "3";
2139	else if (speed == 3)
2140		speed_s = "6";
2141	else
2142		speed_s = "?";
2143
2144	dev_info(host->dev,
2145		"AHCI %02x%02x.%02x%02x "
2146		"%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2147		,
2148
2149		(vers >> 24) & 0xff,
2150		(vers >> 16) & 0xff,
2151		(vers >> 8) & 0xff,
2152		vers & 0xff,
2153
2154		((cap >> 8) & 0x1f) + 1,
2155		(cap & 0x1f) + 1,
2156		speed_s,
2157		impl,
2158		scc_s);
2159
2160	dev_info(host->dev,
2161		"flags: "
2162		"%s%s%s%s%s%s%s"
2163		"%s%s%s%s%s%s%s"
2164		"%s%s%s%s%s%s\n"
 
2165		,
2166
2167		cap & HOST_CAP_64 ? "64bit " : "",
2168		cap & HOST_CAP_NCQ ? "ncq " : "",
2169		cap & HOST_CAP_SNTF ? "sntf " : "",
2170		cap & HOST_CAP_MPS ? "ilck " : "",
2171		cap & HOST_CAP_SSS ? "stag " : "",
2172		cap & HOST_CAP_ALPM ? "pm " : "",
2173		cap & HOST_CAP_LED ? "led " : "",
2174		cap & HOST_CAP_CLO ? "clo " : "",
2175		cap & HOST_CAP_ONLY ? "only " : "",
2176		cap & HOST_CAP_PMP ? "pmp " : "",
2177		cap & HOST_CAP_FBS ? "fbs " : "",
2178		cap & HOST_CAP_PIO_MULTI ? "pio " : "",
2179		cap & HOST_CAP_SSC ? "slum " : "",
2180		cap & HOST_CAP_PART ? "part " : "",
2181		cap & HOST_CAP_CCC ? "ccc " : "",
2182		cap & HOST_CAP_EMS ? "ems " : "",
2183		cap & HOST_CAP_SXS ? "sxs " : "",
 
 
 
2184		cap2 & HOST_CAP2_APST ? "apst " : "",
2185		cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
2186		cap2 & HOST_CAP2_BOH ? "boh " : ""
2187		);
2188}
2189EXPORT_SYMBOL_GPL(ahci_print_info);
2190
2191void ahci_set_em_messages(struct ahci_host_priv *hpriv,
2192			  struct ata_port_info *pi)
2193{
2194	u8 messages;
2195	void __iomem *mmio = hpriv->mmio;
2196	u32 em_loc = readl(mmio + HOST_EM_LOC);
2197	u32 em_ctl = readl(mmio + HOST_EM_CTL);
2198
2199	if (!ahci_em_messages || !(hpriv->cap & HOST_CAP_EMS))
2200		return;
2201
2202	messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
2203
2204	if (messages) {
2205		/* store em_loc */
2206		hpriv->em_loc = ((em_loc >> 16) * 4);
2207		hpriv->em_buf_sz = ((em_loc & 0xff) * 4);
2208		hpriv->em_msg_type = messages;
2209		pi->flags |= ATA_FLAG_EM;
2210		if (!(em_ctl & EM_CTL_ALHD))
2211			pi->flags |= ATA_FLAG_SW_ACTIVITY;
2212	}
2213}
2214EXPORT_SYMBOL_GPL(ahci_set_em_messages);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2215
2216MODULE_AUTHOR("Jeff Garzik");
2217MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2218MODULE_LICENSE("GPL");