1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c)  2019 Intel Corporation */
   3
   4#include "igc.h"
   5
   6#include <linux/module.h>
   7#include <linux/device.h>
   8#include <linux/pci.h>
   9#include <linux/ptp_classify.h>
  10#include <linux/clocksource.h>
  11#include <linux/ktime.h>
  12#include <linux/delay.h>
  13#include <linux/iopoll.h>
  14
  15#define INCVALUE_MASK		0x7fffffff
  16#define ISGN			0x80000000
  17
  18#define IGC_PTP_TX_TIMEOUT		(HZ * 15)
  19
  20#define IGC_PTM_STAT_SLEEP		2
  21#define IGC_PTM_STAT_TIMEOUT		100
  22
  23/* SYSTIM read access for I225 */
  24void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts)
  25{
  26	struct igc_hw *hw = &adapter->hw;
  27	u32 sec, nsec;
  28
  29	/* The timestamp is latched when SYSTIML is read. */
  30	nsec = rd32(IGC_SYSTIML);
  31	sec = rd32(IGC_SYSTIMH);
  32
  33	ts->tv_sec = sec;
  34	ts->tv_nsec = nsec;
  35}
  36
  37static void igc_ptp_write_i225(struct igc_adapter *adapter,
  38			       const struct timespec64 *ts)
  39{
  40	struct igc_hw *hw = &adapter->hw;
  41
  42	wr32(IGC_SYSTIML, ts->tv_nsec);
  43	wr32(IGC_SYSTIMH, ts->tv_sec);
  44}
  45
  46static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm)
  47{
  48	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  49					       ptp_caps);
  50	struct igc_hw *hw = &igc->hw;
  51	int neg_adj = 0;
  52	u64 rate;
  53	u32 inca;
  54
  55	if (scaled_ppm < 0) {
  56		neg_adj = 1;
  57		scaled_ppm = -scaled_ppm;
  58	}
  59	rate = scaled_ppm;
  60	rate <<= 14;
  61	rate = div_u64(rate, 78125);
  62
  63	inca = rate & INCVALUE_MASK;
  64	if (neg_adj)
  65		inca |= ISGN;
  66
  67	wr32(IGC_TIMINCA, inca);
  68
  69	return 0;
  70}
  71
  72static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta)
  73{
  74	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  75					       ptp_caps);
  76	struct timespec64 now, then = ns_to_timespec64(delta);
  77	unsigned long flags;
  78
  79	spin_lock_irqsave(&igc->tmreg_lock, flags);
  80
  81	igc_ptp_read(igc, &now);
  82	now = timespec64_add(now, then);
  83	igc_ptp_write_i225(igc, (const struct timespec64 *)&now);
  84
  85	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
  86
  87	return 0;
  88}
  89
  90static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp,
  91				   struct timespec64 *ts,
  92				   struct ptp_system_timestamp *sts)
  93{
  94	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  95					       ptp_caps);
  96	struct igc_hw *hw = &igc->hw;
  97	unsigned long flags;
  98
  99	spin_lock_irqsave(&igc->tmreg_lock, flags);
 100
 101	ptp_read_system_prets(sts);
 102	ts->tv_nsec = rd32(IGC_SYSTIML);
 103	ts->tv_sec = rd32(IGC_SYSTIMH);
 104	ptp_read_system_postts(sts);
 105
 106	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 107
 108	return 0;
 109}
 110
 111static int igc_ptp_settime_i225(struct ptp_clock_info *ptp,
 112				const struct timespec64 *ts)
 113{
 114	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
 115					       ptp_caps);
 116	unsigned long flags;
 117
 118	spin_lock_irqsave(&igc->tmreg_lock, flags);
 119
 120	igc_ptp_write_i225(igc, ts);
 121
 122	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 123
 124	return 0;
 125}
 126
 127static void igc_pin_direction(int pin, int input, u32 *ctrl, u32 *ctrl_ext)
 128{
 129	u32 *ptr = pin < 2 ? ctrl : ctrl_ext;
 130	static const u32 mask[IGC_N_SDP] = {
 131		IGC_CTRL_SDP0_DIR,
 132		IGC_CTRL_SDP1_DIR,
 133		IGC_CTRL_EXT_SDP2_DIR,
 134		IGC_CTRL_EXT_SDP3_DIR,
 135	};
 136
 137	if (input)
 138		*ptr &= ~mask[pin];
 139	else
 140		*ptr |= mask[pin];
 141}
 142
 143static void igc_pin_perout(struct igc_adapter *igc, int chan, int pin, int freq)
 144{
 145	static const u32 igc_aux0_sel_sdp[IGC_N_SDP] = {
 146		IGC_AUX0_SEL_SDP0, IGC_AUX0_SEL_SDP1, IGC_AUX0_SEL_SDP2, IGC_AUX0_SEL_SDP3,
 147	};
 148	static const u32 igc_aux1_sel_sdp[IGC_N_SDP] = {
 149		IGC_AUX1_SEL_SDP0, IGC_AUX1_SEL_SDP1, IGC_AUX1_SEL_SDP2, IGC_AUX1_SEL_SDP3,
 150	};
 151	static const u32 igc_ts_sdp_en[IGC_N_SDP] = {
 152		IGC_TS_SDP0_EN, IGC_TS_SDP1_EN, IGC_TS_SDP2_EN, IGC_TS_SDP3_EN,
 153	};
 154	static const u32 igc_ts_sdp_sel_tt0[IGC_N_SDP] = {
 155		IGC_TS_SDP0_SEL_TT0, IGC_TS_SDP1_SEL_TT0,
 156		IGC_TS_SDP2_SEL_TT0, IGC_TS_SDP3_SEL_TT0,
 157	};
 158	static const u32 igc_ts_sdp_sel_tt1[IGC_N_SDP] = {
 159		IGC_TS_SDP0_SEL_TT1, IGC_TS_SDP1_SEL_TT1,
 160		IGC_TS_SDP2_SEL_TT1, IGC_TS_SDP3_SEL_TT1,
 161	};
 162	static const u32 igc_ts_sdp_sel_fc0[IGC_N_SDP] = {
 163		IGC_TS_SDP0_SEL_FC0, IGC_TS_SDP1_SEL_FC0,
 164		IGC_TS_SDP2_SEL_FC0, IGC_TS_SDP3_SEL_FC0,
 165	};
 166	static const u32 igc_ts_sdp_sel_fc1[IGC_N_SDP] = {
 167		IGC_TS_SDP0_SEL_FC1, IGC_TS_SDP1_SEL_FC1,
 168		IGC_TS_SDP2_SEL_FC1, IGC_TS_SDP3_SEL_FC1,
 169	};
 170	static const u32 igc_ts_sdp_sel_clr[IGC_N_SDP] = {
 171		IGC_TS_SDP0_SEL_FC1, IGC_TS_SDP1_SEL_FC1,
 172		IGC_TS_SDP2_SEL_FC1, IGC_TS_SDP3_SEL_FC1,
 173	};
 174	struct igc_hw *hw = &igc->hw;
 175	u32 ctrl, ctrl_ext, tssdp = 0;
 176
 177	ctrl = rd32(IGC_CTRL);
 178	ctrl_ext = rd32(IGC_CTRL_EXT);
 179	tssdp = rd32(IGC_TSSDP);
 180
 181	igc_pin_direction(pin, 0, &ctrl, &ctrl_ext);
 182
 183	/* Make sure this pin is not enabled as an input. */
 184	if ((tssdp & IGC_AUX0_SEL_SDP3) == igc_aux0_sel_sdp[pin])
 185		tssdp &= ~IGC_AUX0_TS_SDP_EN;
 186
 187	if ((tssdp & IGC_AUX1_SEL_SDP3) == igc_aux1_sel_sdp[pin])
 188		tssdp &= ~IGC_AUX1_TS_SDP_EN;
 189
 190	tssdp &= ~igc_ts_sdp_sel_clr[pin];
 191	if (freq) {
 192		if (chan == 1)
 193			tssdp |= igc_ts_sdp_sel_fc1[pin];
 194		else
 195			tssdp |= igc_ts_sdp_sel_fc0[pin];
 196	} else {
 197		if (chan == 1)
 198			tssdp |= igc_ts_sdp_sel_tt1[pin];
 199		else
 200			tssdp |= igc_ts_sdp_sel_tt0[pin];
 201	}
 202	tssdp |= igc_ts_sdp_en[pin];
 203
 204	wr32(IGC_TSSDP, tssdp);
 205	wr32(IGC_CTRL, ctrl);
 206	wr32(IGC_CTRL_EXT, ctrl_ext);
 207}
 208
 209static void igc_pin_extts(struct igc_adapter *igc, int chan, int pin)
 210{
 211	static const u32 igc_aux0_sel_sdp[IGC_N_SDP] = {
 212		IGC_AUX0_SEL_SDP0, IGC_AUX0_SEL_SDP1, IGC_AUX0_SEL_SDP2, IGC_AUX0_SEL_SDP3,
 213	};
 214	static const u32 igc_aux1_sel_sdp[IGC_N_SDP] = {
 215		IGC_AUX1_SEL_SDP0, IGC_AUX1_SEL_SDP1, IGC_AUX1_SEL_SDP2, IGC_AUX1_SEL_SDP3,
 216	};
 217	static const u32 igc_ts_sdp_en[IGC_N_SDP] = {
 218		IGC_TS_SDP0_EN, IGC_TS_SDP1_EN, IGC_TS_SDP2_EN, IGC_TS_SDP3_EN,
 219	};
 220	struct igc_hw *hw = &igc->hw;
 221	u32 ctrl, ctrl_ext, tssdp = 0;
 222
 223	ctrl = rd32(IGC_CTRL);
 224	ctrl_ext = rd32(IGC_CTRL_EXT);
 225	tssdp = rd32(IGC_TSSDP);
 226
 227	igc_pin_direction(pin, 1, &ctrl, &ctrl_ext);
 228
 229	/* Make sure this pin is not enabled as an output. */
 230	tssdp &= ~igc_ts_sdp_en[pin];
 231
 232	if (chan == 1) {
 233		tssdp &= ~IGC_AUX1_SEL_SDP3;
 234		tssdp |= igc_aux1_sel_sdp[pin] | IGC_AUX1_TS_SDP_EN;
 235	} else {
 236		tssdp &= ~IGC_AUX0_SEL_SDP3;
 237		tssdp |= igc_aux0_sel_sdp[pin] | IGC_AUX0_TS_SDP_EN;
 238	}
 239
 240	wr32(IGC_TSSDP, tssdp);
 241	wr32(IGC_CTRL, ctrl);
 242	wr32(IGC_CTRL_EXT, ctrl_ext);
 243}
 244
 245static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp,
 246				       struct ptp_clock_request *rq, int on)
 247{
 248	struct igc_adapter *igc =
 249		container_of(ptp, struct igc_adapter, ptp_caps);
 250	struct igc_hw *hw = &igc->hw;
 251	unsigned long flags;
 252	struct timespec64 ts;
 253	int use_freq = 0, pin = -1;
 254	u32 tsim, tsauxc, tsauxc_mask, tsim_mask, trgttiml, trgttimh, freqout;
 255	s64 ns;
 256
 257	switch (rq->type) {
 258	case PTP_CLK_REQ_EXTTS:
 259		/* Reject requests with unsupported flags */
 260		if (rq->extts.flags & ~(PTP_ENABLE_FEATURE |
 261					PTP_RISING_EDGE |
 262					PTP_FALLING_EDGE |
 263					PTP_STRICT_FLAGS))
 264			return -EOPNOTSUPP;
 265
 266		/* Reject requests failing to enable both edges. */
 267		if ((rq->extts.flags & PTP_STRICT_FLAGS) &&
 268		    (rq->extts.flags & PTP_ENABLE_FEATURE) &&
 269		    (rq->extts.flags & PTP_EXTTS_EDGES) != PTP_EXTTS_EDGES)
 270			return -EOPNOTSUPP;
 271
 272		if (on) {
 273			pin = ptp_find_pin(igc->ptp_clock, PTP_PF_EXTTS,
 274					   rq->extts.index);
 275			if (pin < 0)
 276				return -EBUSY;
 277		}
 278		if (rq->extts.index == 1) {
 279			tsauxc_mask = IGC_TSAUXC_EN_TS1;
 280			tsim_mask = IGC_TSICR_AUTT1;
 281		} else {
 282			tsauxc_mask = IGC_TSAUXC_EN_TS0;
 283			tsim_mask = IGC_TSICR_AUTT0;
 284		}
 285		spin_lock_irqsave(&igc->tmreg_lock, flags);
 286		tsauxc = rd32(IGC_TSAUXC);
 287		tsim = rd32(IGC_TSIM);
 288		if (on) {
 289			igc_pin_extts(igc, rq->extts.index, pin);
 290			tsauxc |= tsauxc_mask;
 291			tsim |= tsim_mask;
 292		} else {
 293			tsauxc &= ~tsauxc_mask;
 294			tsim &= ~tsim_mask;
 295		}
 296		wr32(IGC_TSAUXC, tsauxc);
 297		wr32(IGC_TSIM, tsim);
 298		spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 299		return 0;
 300
 301	case PTP_CLK_REQ_PEROUT:
 302		/* Reject requests with unsupported flags */
 303		if (rq->perout.flags)
 304			return -EOPNOTSUPP;
 305
 306		if (on) {
 307			pin = ptp_find_pin(igc->ptp_clock, PTP_PF_PEROUT,
 308					   rq->perout.index);
 309			if (pin < 0)
 310				return -EBUSY;
 311		}
 312		ts.tv_sec = rq->perout.period.sec;
 313		ts.tv_nsec = rq->perout.period.nsec;
 314		ns = timespec64_to_ns(&ts);
 315		ns = ns >> 1;
 316		if (on && (ns <= 70000000LL || ns == 125000000LL ||
 317			   ns == 250000000LL || ns == 500000000LL)) {
 318			if (ns < 8LL)
 319				return -EINVAL;
 320			use_freq = 1;
 321		}
 322		ts = ns_to_timespec64(ns);
 323		if (rq->perout.index == 1) {
 324			if (use_freq) {
 325				tsauxc_mask = IGC_TSAUXC_EN_CLK1 | IGC_TSAUXC_ST1;
 326				tsim_mask = 0;
 327			} else {
 328				tsauxc_mask = IGC_TSAUXC_EN_TT1;
 329				tsim_mask = IGC_TSICR_TT1;
 330			}
 331			trgttiml = IGC_TRGTTIML1;
 332			trgttimh = IGC_TRGTTIMH1;
 333			freqout = IGC_FREQOUT1;
 334		} else {
 335			if (use_freq) {
 336				tsauxc_mask = IGC_TSAUXC_EN_CLK0 | IGC_TSAUXC_ST0;
 337				tsim_mask = 0;
 338			} else {
 339				tsauxc_mask = IGC_TSAUXC_EN_TT0;
 340				tsim_mask = IGC_TSICR_TT0;
 341			}
 342			trgttiml = IGC_TRGTTIML0;
 343			trgttimh = IGC_TRGTTIMH0;
 344			freqout = IGC_FREQOUT0;
 345		}
 346		spin_lock_irqsave(&igc->tmreg_lock, flags);
 347		tsauxc = rd32(IGC_TSAUXC);
 348		tsim = rd32(IGC_TSIM);
 349		if (rq->perout.index == 1) {
 350			tsauxc &= ~(IGC_TSAUXC_EN_TT1 | IGC_TSAUXC_EN_CLK1 |
 351				    IGC_TSAUXC_ST1);
 352			tsim &= ~IGC_TSICR_TT1;
 353		} else {
 354			tsauxc &= ~(IGC_TSAUXC_EN_TT0 | IGC_TSAUXC_EN_CLK0 |
 355				    IGC_TSAUXC_ST0);
 356			tsim &= ~IGC_TSICR_TT0;
 357		}
 358		if (on) {
 359			struct timespec64 safe_start;
 360			int i = rq->perout.index;
 361
 362			igc_pin_perout(igc, i, pin, use_freq);
 363			igc_ptp_read(igc, &safe_start);
 364
 365			/* PPS output start time is triggered by Target time(TT)
 366			 * register. Programming any past time value into TT
 367			 * register will cause PPS to never start. Need to make
 368			 * sure we program the TT register a time ahead in
 369			 * future. There isn't a stringent need to fire PPS out
 370			 * right away. Adding +2 seconds should take care of
 371			 * corner cases. Let's say if the SYSTIML is close to
 372			 * wrap up and the timer keeps ticking as we program the
 373			 * register, adding +2seconds is safe bet.
 374			 */
 375			safe_start.tv_sec += 2;
 376
 377			if (rq->perout.start.sec < safe_start.tv_sec)
 378				igc->perout[i].start.tv_sec = safe_start.tv_sec;
 379			else
 380				igc->perout[i].start.tv_sec = rq->perout.start.sec;
 381			igc->perout[i].start.tv_nsec = rq->perout.start.nsec;
 382			igc->perout[i].period.tv_sec = ts.tv_sec;
 383			igc->perout[i].period.tv_nsec = ts.tv_nsec;
 384			wr32(trgttimh, (u32)igc->perout[i].start.tv_sec);
 385			/* For now, always select timer 0 as source. */
 386			wr32(trgttiml, (u32)(igc->perout[i].start.tv_nsec |
 387					     IGC_TT_IO_TIMER_SEL_SYSTIM0));
 388			if (use_freq)
 389				wr32(freqout, ns);
 390			tsauxc |= tsauxc_mask;
 391			tsim |= tsim_mask;
 392		}
 393		wr32(IGC_TSAUXC, tsauxc);
 394		wr32(IGC_TSIM, tsim);
 395		spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 396		return 0;
 397
 398	case PTP_CLK_REQ_PPS:
 399		spin_lock_irqsave(&igc->tmreg_lock, flags);
 400		tsim = rd32(IGC_TSIM);
 401		if (on)
 402			tsim |= IGC_TSICR_SYS_WRAP;
 403		else
 404			tsim &= ~IGC_TSICR_SYS_WRAP;
 405		igc->pps_sys_wrap_on = on;
 406		wr32(IGC_TSIM, tsim);
 407		spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 408		return 0;
 409
 410	default:
 411		break;
 412	}
 413
 414	return -EOPNOTSUPP;
 415}
 416
 417static int igc_ptp_verify_pin(struct ptp_clock_info *ptp, unsigned int pin,
 418			      enum ptp_pin_function func, unsigned int chan)
 419{
 420	switch (func) {
 421	case PTP_PF_NONE:
 422	case PTP_PF_EXTTS:
 423	case PTP_PF_PEROUT:
 424		break;
 425	case PTP_PF_PHYSYNC:
 426		return -1;
 427	}
 428	return 0;
 429}
 430
 431/**
 432 * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp
 433 * @adapter: board private structure
 434 * @hwtstamps: timestamp structure to update
 435 * @systim: unsigned 64bit system time value
 436 *
 437 * We need to convert the system time value stored in the RX/TXSTMP registers
 438 * into a hwtstamp which can be used by the upper level timestamping functions.
 439 *
 440 * Returns 0 on success.
 441 **/
 442static int igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
 443				      struct skb_shared_hwtstamps *hwtstamps,
 444				      u64 systim)
 445{
 446	switch (adapter->hw.mac.type) {
 447	case igc_i225:
 448		memset(hwtstamps, 0, sizeof(*hwtstamps));
 449		/* Upper 32 bits contain s, lower 32 bits contain ns. */
 450		hwtstamps->hwtstamp = ktime_set(systim >> 32,
 451						systim & 0xFFFFFFFF);
 452		break;
 453	default:
 454		return -EINVAL;
 455	}
 456	return 0;
 457}
 458
 459/**
 460 * igc_ptp_rx_pktstamp - Retrieve timestamp from Rx packet buffer
 461 * @adapter: Pointer to adapter the packet buffer belongs to
 462 * @buf: Pointer to start of timestamp in HW format (2 32-bit words)
 463 *
 464 * This function retrieves and converts the timestamp stored at @buf
 465 * to ktime_t, adjusting for hardware latencies.
 466 *
 467 * Returns timestamp value.
 468 */
 469ktime_t igc_ptp_rx_pktstamp(struct igc_adapter *adapter, __le32 *buf)
 470{
 471	ktime_t timestamp;
 472	u32 secs, nsecs;
 473	int adjust;
 474
 475	nsecs = le32_to_cpu(buf[0]);
 476	secs = le32_to_cpu(buf[1]);
 477
 478	timestamp = ktime_set(secs, nsecs);
 479
 480	/* Adjust timestamp for the RX latency based on link speed */
 481	switch (adapter->link_speed) {
 482	case SPEED_10:
 483		adjust = IGC_I225_RX_LATENCY_10;
 484		break;
 485	case SPEED_100:
 486		adjust = IGC_I225_RX_LATENCY_100;
 487		break;
 488	case SPEED_1000:
 489		adjust = IGC_I225_RX_LATENCY_1000;
 490		break;
 491	case SPEED_2500:
 492		adjust = IGC_I225_RX_LATENCY_2500;
 493		break;
 494	default:
 495		adjust = 0;
 496		netdev_warn_once(adapter->netdev, "Imprecise timestamp\n");
 497		break;
 498	}
 499
 500	return ktime_sub_ns(timestamp, adjust);
 501}
 502
 503static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter)
 504{
 505	struct igc_hw *hw = &adapter->hw;
 506	u32 val;
 507	int i;
 508
 509	wr32(IGC_TSYNCRXCTL, 0);
 510
 511	for (i = 0; i < adapter->num_rx_queues; i++) {
 512		val = rd32(IGC_SRRCTL(i));
 513		val &= ~IGC_SRRCTL_TIMESTAMP;
 514		wr32(IGC_SRRCTL(i), val);
 515	}
 516
 517	val = rd32(IGC_RXPBS);
 518	val &= ~IGC_RXPBS_CFG_TS_EN;
 519	wr32(IGC_RXPBS, val);
 520}
 521
 522static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
 523{
 524	struct igc_hw *hw = &adapter->hw;
 525	u32 val;
 526	int i;
 527
 528	val = rd32(IGC_RXPBS);
 529	val |= IGC_RXPBS_CFG_TS_EN;
 530	wr32(IGC_RXPBS, val);
 531
 532	for (i = 0; i < adapter->num_rx_queues; i++) {
 533		val = rd32(IGC_SRRCTL(i));
 534		/* Enable retrieving timestamps from timer 0, the
 535		 * "adjustable clock" and timer 1 the "free running
 536		 * clock".
 537		 */
 538		val |= IGC_SRRCTL_TIMER1SEL(1) | IGC_SRRCTL_TIMER0SEL(0) |
 539		       IGC_SRRCTL_TIMESTAMP;
 540		wr32(IGC_SRRCTL(i), val);
 541	}
 542
 543	val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL |
 544	      IGC_TSYNCRXCTL_RXSYNSIG;
 545	wr32(IGC_TSYNCRXCTL, val);
 546}
 547
 548static void igc_ptp_clear_tx_tstamp(struct igc_adapter *adapter)
 549{
 550	unsigned long flags;
 551	int i;
 552
 553	spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
 554
 555	for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
 556		struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
 557
 558		dev_kfree_skb_any(tstamp->skb);
 559		tstamp->skb = NULL;
 560	}
 561
 562	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
 563}
 564
 565static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
 566{
 567	struct igc_hw *hw = &adapter->hw;
 568	int i;
 569
 570	/* Clear the flags first to avoid new packets to be enqueued
 571	 * for TX timestamping.
 572	 */
 573	for (i = 0; i < adapter->num_tx_queues; i++) {
 574		struct igc_ring *tx_ring = adapter->tx_ring[i];
 575
 576		clear_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags);
 577	}
 578
 579	/* Now we can clean the pending TX timestamp requests. */
 580	igc_ptp_clear_tx_tstamp(adapter);
 581
 582	wr32(IGC_TSYNCTXCTL, 0);
 583}
 584
 585static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter)
 586{
 587	struct igc_hw *hw = &adapter->hw;
 588	int i;
 589
 590	wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG);
 591
 592	/* Read TXSTMP registers to discard any timestamp previously stored. */
 593	rd32(IGC_TXSTMPL);
 594	rd32(IGC_TXSTMPH);
 595
 596	/* The hardware is ready to accept TX timestamp requests,
 597	 * notify the transmit path.
 598	 */
 599	for (i = 0; i < adapter->num_tx_queues; i++) {
 600		struct igc_ring *tx_ring = adapter->tx_ring[i];
 601
 602		set_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags);
 603	}
 604
 605}
 606
 607/**
 608 * igc_ptp_set_timestamp_mode - setup hardware for timestamping
 609 * @adapter: networking device structure
 610 * @config: hwtstamp configuration
 611 *
 612 * Return: 0 in case of success, negative errno code otherwise.
 613 */
 614static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
 615				      struct hwtstamp_config *config)
 616{
 617	switch (config->tx_type) {
 618	case HWTSTAMP_TX_OFF:
 619		igc_ptp_disable_tx_timestamp(adapter);
 620		break;
 621	case HWTSTAMP_TX_ON:
 622		igc_ptp_enable_tx_timestamp(adapter);
 623		break;
 624	default:
 625		return -ERANGE;
 626	}
 627
 628	switch (config->rx_filter) {
 629	case HWTSTAMP_FILTER_NONE:
 630		igc_ptp_disable_rx_timestamp(adapter);
 631		break;
 632	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 633	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 634	case HWTSTAMP_FILTER_PTP_V2_EVENT:
 635	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 636	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 637	case HWTSTAMP_FILTER_PTP_V2_SYNC:
 638	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 639	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 640	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 641	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 642	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 643	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 644	case HWTSTAMP_FILTER_NTP_ALL:
 645	case HWTSTAMP_FILTER_ALL:
 646		igc_ptp_enable_rx_timestamp(adapter);
 647		config->rx_filter = HWTSTAMP_FILTER_ALL;
 648		break;
 649	default:
 650		return -ERANGE;
 651	}
 652
 653	return 0;
 654}
 655
 656/* Requires adapter->ptp_tx_lock held by caller. */
 657static void igc_ptp_tx_timeout(struct igc_adapter *adapter,
 658			       struct igc_tx_timestamp_request *tstamp)
 659{
 660	dev_kfree_skb_any(tstamp->skb);
 661	tstamp->skb = NULL;
 662	adapter->tx_hwtstamp_timeouts++;
 663
 664	netdev_warn(adapter->netdev, "Tx timestamp timeout\n");
 665}
 666
 667void igc_ptp_tx_hang(struct igc_adapter *adapter)
 668{
 669	struct igc_tx_timestamp_request *tstamp;
 670	struct igc_hw *hw = &adapter->hw;
 671	unsigned long flags;
 672	bool found = false;
 673	int i;
 674
 675	spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
 676
 677	for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
 678		tstamp = &adapter->tx_tstamp[i];
 679
 680		if (!tstamp->skb)
 681			continue;
 682
 683		if (time_is_after_jiffies(tstamp->start + IGC_PTP_TX_TIMEOUT))
 684			continue;
 685
 686		igc_ptp_tx_timeout(adapter, tstamp);
 687		found = true;
 688	}
 689
 690	if (found) {
 691		/* Reading the high register of the first set of timestamp registers
 692		 * clears all the equivalent bits in the TSYNCTXCTL register.
 693		 */
 694		rd32(IGC_TXSTMPH_0);
 695	}
 696
 697	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
 698}
 699
 700static void igc_ptp_tx_reg_to_stamp(struct igc_adapter *adapter,
 701				    struct igc_tx_timestamp_request *tstamp, u64 regval)
 702{
 703	struct skb_shared_hwtstamps shhwtstamps;
 704	struct sk_buff *skb;
 705	int adjust = 0;
 706
 707	skb = tstamp->skb;
 708	if (!skb)
 709		return;
 710
 711	if (igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval))
 712		return;
 713
 714	switch (adapter->link_speed) {
 715	case SPEED_10:
 716		adjust = IGC_I225_TX_LATENCY_10;
 717		break;
 718	case SPEED_100:
 719		adjust = IGC_I225_TX_LATENCY_100;
 720		break;
 721	case SPEED_1000:
 722		adjust = IGC_I225_TX_LATENCY_1000;
 723		break;
 724	case SPEED_2500:
 725		adjust = IGC_I225_TX_LATENCY_2500;
 726		break;
 727	}
 728
 729	shhwtstamps.hwtstamp =
 730		ktime_add_ns(shhwtstamps.hwtstamp, adjust);
 731
 732	tstamp->skb = NULL;
 733
 734	skb_tstamp_tx(skb, &shhwtstamps);
 735	dev_kfree_skb_any(skb);
 736}
 737
 738/**
 739 * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp
 740 * @adapter: Board private structure
 741 *
 742 * Check against the ready mask for which of the timestamp register
 743 * sets are ready to be retrieved, then retrieve that and notify the
 744 * rest of the stack.
 745 *
 746 * Context: Expects adapter->ptp_tx_lock to be held by caller.
 747 */
 748static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
 749{
 750	struct igc_hw *hw = &adapter->hw;
 751	u64 regval;
 752	u32 mask;
 753	int i;
 754
 755	mask = rd32(IGC_TSYNCTXCTL) & IGC_TSYNCTXCTL_TXTT_ANY;
 756	if (mask & IGC_TSYNCTXCTL_TXTT_0) {
 757		regval = rd32(IGC_TXSTMPL);
 758		regval |= (u64)rd32(IGC_TXSTMPH) << 32;
 759	} else {
 760		/* There's a bug in the hardware that could cause
 761		 * missing interrupts for TX timestamping. The issue
 762		 * is that for new interrupts to be triggered, the
 763		 * IGC_TXSTMPH_0 register must be read.
 764		 *
 765		 * To avoid discarding a valid timestamp that just
 766		 * happened at the "wrong" time, we need to confirm
 767		 * that there was no timestamp captured, we do that by
 768		 * assuming that no two timestamps in sequence have
 769		 * the same nanosecond value.
 770		 *
 771		 * So, we read the "low" register, read the "high"
 772		 * register (to latch a new timestamp) and read the
 773		 * "low" register again, if "old" and "new" versions
 774		 * of the "low" register are different, a valid
 775		 * timestamp was captured, we can read the "high"
 776		 * register again.
 777		 */
 778		u32 txstmpl_old, txstmpl_new;
 779
 780		txstmpl_old = rd32(IGC_TXSTMPL);
 781		rd32(IGC_TXSTMPH);
 782		txstmpl_new = rd32(IGC_TXSTMPL);
 783
 784		if (txstmpl_old == txstmpl_new)
 785			goto done;
 786
 787		regval = txstmpl_new;
 788		regval |= (u64)rd32(IGC_TXSTMPH) << 32;
 789	}
 790
 791	igc_ptp_tx_reg_to_stamp(adapter, &adapter->tx_tstamp[0], regval);
 792
 793done:
 794	/* Now that the problematic first register was handled, we can
 795	 * use retrieve the timestamps from the other registers
 796	 * (starting from '1') with less complications.
 797	 */
 798	for (i = 1; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
 799		struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
 800
 801		if (!(tstamp->mask & mask))
 802			continue;
 803
 804		regval = rd32(tstamp->regl);
 805		regval |= (u64)rd32(tstamp->regh) << 32;
 806
 807		igc_ptp_tx_reg_to_stamp(adapter, tstamp, regval);
 808	}
 809}
 810
 811/**
 812 * igc_ptp_tx_tstamp_event
 813 * @adapter: board private structure
 814 *
 815 * Called when a TX timestamp interrupt happens to retrieve the
 816 * timestamp and send it up to the socket.
 817 */
 818void igc_ptp_tx_tstamp_event(struct igc_adapter *adapter)
 819{
 820	unsigned long flags;
 821
 822	spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
 823
 824	igc_ptp_tx_hwtstamp(adapter);
 825
 826	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
 827}
 828
 829/**
 830 * igc_ptp_set_ts_config - set hardware time stamping config
 831 * @netdev: network interface device structure
 832 * @ifr: interface request data
 833 *
 834 **/
 835int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
 836{
 837	struct igc_adapter *adapter = netdev_priv(netdev);
 838	struct hwtstamp_config config;
 839	int err;
 840
 841	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
 842		return -EFAULT;
 843
 844	err = igc_ptp_set_timestamp_mode(adapter, &config);
 845	if (err)
 846		return err;
 847
 848	/* save these settings for future reference */
 849	memcpy(&adapter->tstamp_config, &config,
 850	       sizeof(adapter->tstamp_config));
 851
 852	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
 853		-EFAULT : 0;
 854}
 855
 856/**
 857 * igc_ptp_get_ts_config - get hardware time stamping config
 858 * @netdev: network interface device structure
 859 * @ifr: interface request data
 860 *
 861 * Get the hwtstamp_config settings to return to the user. Rather than attempt
 862 * to deconstruct the settings from the registers, just return a shadow copy
 863 * of the last known settings.
 864 **/
 865int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
 866{
 867	struct igc_adapter *adapter = netdev_priv(netdev);
 868	struct hwtstamp_config *config = &adapter->tstamp_config;
 869
 870	return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
 871		-EFAULT : 0;
 872}
 873
 874/* The two conditions below must be met for cross timestamping via
 875 * PCIe PTM:
 876 *
 877 * 1. We have an way to convert the timestamps in the PTM messages
 878 *    to something related to the system clocks (right now, only
 879 *    X86 systems with support for the Always Running Timer allow that);
 880 *
 881 * 2. We have PTM enabled in the path from the device to the PCIe root port.
 882 */
 883static bool igc_is_crosststamp_supported(struct igc_adapter *adapter)
 884{
 885	if (!IS_ENABLED(CONFIG_X86_TSC))
 886		return false;
 887
 888	/* FIXME: it was noticed that enabling support for PCIe PTM in
 889	 * some i225-V models could cause lockups when bringing the
 890	 * interface up/down. There should be no downsides to
 891	 * disabling crosstimestamping support for i225-V, as it
 892	 * doesn't have any PTP support. That way we gain some time
 893	 * while root causing the issue.
 894	 */
 895	if (adapter->pdev->device == IGC_DEV_ID_I225_V)
 896		return false;
 897
 898	return pcie_ptm_enabled(adapter->pdev);
 899}
 900
 901static struct system_counterval_t igc_device_tstamp_to_system(u64 tstamp)
 902{
 903#if IS_ENABLED(CONFIG_X86_TSC) && !defined(CONFIG_UML)
 904	return convert_art_ns_to_tsc(tstamp);
 905#else
 906	return (struct system_counterval_t) { };
 907#endif
 908}
 909
 910static void igc_ptm_log_error(struct igc_adapter *adapter, u32 ptm_stat)
 911{
 912	struct net_device *netdev = adapter->netdev;
 913
 914	switch (ptm_stat) {
 915	case IGC_PTM_STAT_RET_ERR:
 916		netdev_err(netdev, "PTM Error: Root port timeout\n");
 917		break;
 918	case IGC_PTM_STAT_BAD_PTM_RES:
 919		netdev_err(netdev, "PTM Error: Bad response, PTM Response Data expected\n");
 920		break;
 921	case IGC_PTM_STAT_T4M1_OVFL:
 922		netdev_err(netdev, "PTM Error: T4 minus T1 overflow\n");
 923		break;
 924	case IGC_PTM_STAT_ADJUST_1ST:
 925		netdev_err(netdev, "PTM Error: 1588 timer adjusted during first PTM cycle\n");
 926		break;
 927	case IGC_PTM_STAT_ADJUST_CYC:
 928		netdev_err(netdev, "PTM Error: 1588 timer adjusted during non-first PTM cycle\n");
 929		break;
 930	default:
 931		netdev_err(netdev, "PTM Error: Unknown error (%#x)\n", ptm_stat);
 932		break;
 933	}
 934}
 935
 936static int igc_phc_get_syncdevicetime(ktime_t *device,
 937				      struct system_counterval_t *system,
 938				      void *ctx)
 939{
 940	u32 stat, t2_curr_h, t2_curr_l, ctrl;
 941	struct igc_adapter *adapter = ctx;
 942	struct igc_hw *hw = &adapter->hw;
 943	int err, count = 100;
 944	ktime_t t1, t2_curr;
 945
 946	/* Get a snapshot of system clocks to use as historic value. */
 947	ktime_get_snapshot(&adapter->snapshot);
 948
 949	do {
 950		/* Doing this in a loop because in the event of a
 951		 * badly timed (ha!) system clock adjustment, we may
 952		 * get PTM errors from the PCI root, but these errors
 953		 * are transitory. Repeating the process returns valid
 954		 * data eventually.
 955		 */
 956
 957		/* To "manually" start the PTM cycle we need to clear and
 958		 * then set again the TRIG bit.
 959		 */
 960		ctrl = rd32(IGC_PTM_CTRL);
 961		ctrl &= ~IGC_PTM_CTRL_TRIG;
 962		wr32(IGC_PTM_CTRL, ctrl);
 963		ctrl |= IGC_PTM_CTRL_TRIG;
 964		wr32(IGC_PTM_CTRL, ctrl);
 965
 966		/* The cycle only starts "for real" when software notifies
 967		 * that it has read the registers, this is done by setting
 968		 * VALID bit.
 969		 */
 970		wr32(IGC_PTM_STAT, IGC_PTM_STAT_VALID);
 971
 972		err = readx_poll_timeout(rd32, IGC_PTM_STAT, stat,
 973					 stat, IGC_PTM_STAT_SLEEP,
 974					 IGC_PTM_STAT_TIMEOUT);
 975		if (err < 0) {
 976			netdev_err(adapter->netdev, "Timeout reading IGC_PTM_STAT register\n");
 977			return err;
 978		}
 979
 980		if ((stat & IGC_PTM_STAT_VALID) == IGC_PTM_STAT_VALID)
 981			break;
 982
 983		if (stat & ~IGC_PTM_STAT_VALID) {
 984			/* An error occurred, log it. */
 985			igc_ptm_log_error(adapter, stat);
 986			/* The STAT register is write-1-to-clear (W1C),
 987			 * so write the previous error status to clear it.
 988			 */
 989			wr32(IGC_PTM_STAT, stat);
 990			continue;
 991		}
 992	} while (--count);
 993
 994	if (!count) {
 995		netdev_err(adapter->netdev, "Exceeded number of tries for PTM cycle\n");
 996		return -ETIMEDOUT;
 997	}
 998
 999	t1 = ktime_set(rd32(IGC_PTM_T1_TIM0_H), rd32(IGC_PTM_T1_TIM0_L));
1000
1001	t2_curr_l = rd32(IGC_PTM_CURR_T2_L);
1002	t2_curr_h = rd32(IGC_PTM_CURR_T2_H);
1003
1004	/* FIXME: When the register that tells the endianness of the
1005	 * PTM registers are implemented, check them here and add the
1006	 * appropriate conversion.
1007	 */
1008	t2_curr_h = swab32(t2_curr_h);
1009
1010	t2_curr = ((s64)t2_curr_h << 32 | t2_curr_l);
1011
1012	*device = t1;
1013	*system = igc_device_tstamp_to_system(t2_curr);
1014
1015	return 0;
1016}
1017
1018static int igc_ptp_getcrosststamp(struct ptp_clock_info *ptp,
1019				  struct system_device_crosststamp *cts)
1020{
1021	struct igc_adapter *adapter = container_of(ptp, struct igc_adapter,
1022						   ptp_caps);
1023
1024	return get_device_system_crosststamp(igc_phc_get_syncdevicetime,
1025					     adapter, &adapter->snapshot, cts);
1026}
1027
1028static int igc_ptp_getcyclesx64(struct ptp_clock_info *ptp,
1029				struct timespec64 *ts,
1030				struct ptp_system_timestamp *sts)
1031{
1032	struct igc_adapter *igc = container_of(ptp, struct igc_adapter, ptp_caps);
1033	struct igc_hw *hw = &igc->hw;
1034	unsigned long flags;
1035
1036	spin_lock_irqsave(&igc->free_timer_lock, flags);
1037
1038	ptp_read_system_prets(sts);
1039	ts->tv_nsec = rd32(IGC_SYSTIML_1);
1040	ts->tv_sec = rd32(IGC_SYSTIMH_1);
1041	ptp_read_system_postts(sts);
1042
1043	spin_unlock_irqrestore(&igc->free_timer_lock, flags);
1044
1045	return 0;
1046}
1047
1048/**
1049 * igc_ptp_init - Initialize PTP functionality
1050 * @adapter: Board private structure
1051 *
1052 * This function is called at device probe to initialize the PTP
1053 * functionality.
1054 */
1055void igc_ptp_init(struct igc_adapter *adapter)
1056{
1057	struct net_device *netdev = adapter->netdev;
1058	struct igc_tx_timestamp_request *tstamp;
1059	struct igc_hw *hw = &adapter->hw;
1060	int i;
1061
1062	tstamp = &adapter->tx_tstamp[0];
1063	tstamp->mask = IGC_TSYNCTXCTL_TXTT_0;
1064	tstamp->regl = IGC_TXSTMPL_0;
1065	tstamp->regh = IGC_TXSTMPH_0;
1066	tstamp->flags = 0;
1067
1068	tstamp = &adapter->tx_tstamp[1];
1069	tstamp->mask = IGC_TSYNCTXCTL_TXTT_1;
1070	tstamp->regl = IGC_TXSTMPL_1;
1071	tstamp->regh = IGC_TXSTMPH_1;
1072	tstamp->flags = IGC_TX_FLAGS_TSTAMP_1;
1073
1074	tstamp = &adapter->tx_tstamp[2];
1075	tstamp->mask = IGC_TSYNCTXCTL_TXTT_2;
1076	tstamp->regl = IGC_TXSTMPL_2;
1077	tstamp->regh = IGC_TXSTMPH_2;
1078	tstamp->flags = IGC_TX_FLAGS_TSTAMP_2;
1079
1080	tstamp = &adapter->tx_tstamp[3];
1081	tstamp->mask = IGC_TSYNCTXCTL_TXTT_3;
1082	tstamp->regl = IGC_TXSTMPL_3;
1083	tstamp->regh = IGC_TXSTMPH_3;
1084	tstamp->flags = IGC_TX_FLAGS_TSTAMP_3;
1085
1086	switch (hw->mac.type) {
1087	case igc_i225:
1088		for (i = 0; i < IGC_N_SDP; i++) {
1089			struct ptp_pin_desc *ppd = &adapter->sdp_config[i];
1090
1091			snprintf(ppd->name, sizeof(ppd->name), "SDP%d", i);
1092			ppd->index = i;
1093			ppd->func = PTP_PF_NONE;
1094		}
1095		snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
1096		adapter->ptp_caps.owner = THIS_MODULE;
1097		adapter->ptp_caps.max_adj = 62499999;
1098		adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
1099		adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
1100		adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
1101		adapter->ptp_caps.getcyclesx64 = igc_ptp_getcyclesx64;
1102		adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
1103		adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
1104		adapter->ptp_caps.pps = 1;
1105		adapter->ptp_caps.pin_config = adapter->sdp_config;
1106		adapter->ptp_caps.n_ext_ts = IGC_N_EXTTS;
1107		adapter->ptp_caps.n_per_out = IGC_N_PEROUT;
1108		adapter->ptp_caps.n_pins = IGC_N_SDP;
1109		adapter->ptp_caps.verify = igc_ptp_verify_pin;
1110
1111		if (!igc_is_crosststamp_supported(adapter))
1112			break;
1113
1114		adapter->ptp_caps.getcrosststamp = igc_ptp_getcrosststamp;
1115		break;
1116	default:
1117		adapter->ptp_clock = NULL;
1118		return;
1119	}
1120
1121	spin_lock_init(&adapter->ptp_tx_lock);
1122	spin_lock_init(&adapter->free_timer_lock);
1123	spin_lock_init(&adapter->tmreg_lock);
1124
1125	adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
1126	adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
1127
1128	adapter->prev_ptp_time = ktime_to_timespec64(ktime_get_real());
1129	adapter->ptp_reset_start = ktime_get();
1130
1131	adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
1132						&adapter->pdev->dev);
1133	if (IS_ERR(adapter->ptp_clock)) {
1134		adapter->ptp_clock = NULL;
1135		netdev_err(netdev, "ptp_clock_register failed\n");
1136	} else if (adapter->ptp_clock) {
1137		netdev_info(netdev, "PHC added\n");
1138		adapter->ptp_flags |= IGC_PTP_ENABLED;
1139	}
1140}
1141
1142static void igc_ptp_time_save(struct igc_adapter *adapter)
1143{
1144	igc_ptp_read(adapter, &adapter->prev_ptp_time);
1145	adapter->ptp_reset_start = ktime_get();
1146}
1147
1148static void igc_ptp_time_restore(struct igc_adapter *adapter)
1149{
1150	struct timespec64 ts = adapter->prev_ptp_time;
1151	ktime_t delta;
1152
1153	delta = ktime_sub(ktime_get(), adapter->ptp_reset_start);
1154
1155	timespec64_add_ns(&ts, ktime_to_ns(delta));
1156
1157	igc_ptp_write_i225(adapter, &ts);
1158}
1159
1160static void igc_ptm_stop(struct igc_adapter *adapter)
1161{
1162	struct igc_hw *hw = &adapter->hw;
1163	u32 ctrl;
1164
1165	ctrl = rd32(IGC_PTM_CTRL);
1166	ctrl &= ~IGC_PTM_CTRL_EN;
1167
1168	wr32(IGC_PTM_CTRL, ctrl);
1169}
1170
1171/**
1172 * igc_ptp_suspend - Disable PTP work items and prepare for suspend
1173 * @adapter: Board private structure
1174 *
1175 * This function stops the overflow check work and PTP Tx timestamp work, and
1176 * will prepare the device for OS suspend.
1177 */
1178void igc_ptp_suspend(struct igc_adapter *adapter)
1179{
1180	if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
1181		return;
1182
1183	igc_ptp_clear_tx_tstamp(adapter);
1184
1185	if (pci_device_is_present(adapter->pdev)) {
1186		igc_ptp_time_save(adapter);
1187		igc_ptm_stop(adapter);
1188	}
1189}
1190
1191/**
1192 * igc_ptp_stop - Disable PTP device and stop the overflow check.
1193 * @adapter: Board private structure.
1194 *
1195 * This function stops the PTP support and cancels the delayed work.
1196 **/
1197void igc_ptp_stop(struct igc_adapter *adapter)
1198{
1199	igc_ptp_suspend(adapter);
1200
1201	if (adapter->ptp_clock) {
1202		ptp_clock_unregister(adapter->ptp_clock);
1203		netdev_info(adapter->netdev, "PHC removed\n");
1204		adapter->ptp_flags &= ~IGC_PTP_ENABLED;
1205	}
1206}
1207
1208/**
1209 * igc_ptp_reset - Re-enable the adapter for PTP following a reset.
1210 * @adapter: Board private structure.
1211 *
1212 * This function handles the reset work required to re-enable the PTP device.
1213 **/
1214void igc_ptp_reset(struct igc_adapter *adapter)
1215{
1216	struct igc_hw *hw = &adapter->hw;
1217	u32 cycle_ctrl, ctrl;
1218	unsigned long flags;
1219	u32 timadj;
1220
1221	/* reset the tstamp_config */
1222	igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
1223
1224	spin_lock_irqsave(&adapter->tmreg_lock, flags);
1225
1226	switch (adapter->hw.mac.type) {
1227	case igc_i225:
1228		timadj = rd32(IGC_TIMADJ);
1229		timadj |= IGC_TIMADJ_ADJUST_METH;
1230		wr32(IGC_TIMADJ, timadj);
1231
1232		wr32(IGC_TSAUXC, 0x0);
1233		wr32(IGC_TSSDP, 0x0);
1234		wr32(IGC_TSIM,
1235		     IGC_TSICR_INTERRUPTS |
1236		     (adapter->pps_sys_wrap_on ? IGC_TSICR_SYS_WRAP : 0));
1237		wr32(IGC_IMS, IGC_IMS_TS);
1238
1239		if (!igc_is_crosststamp_supported(adapter))
1240			break;
1241
1242		wr32(IGC_PCIE_DIG_DELAY, IGC_PCIE_DIG_DELAY_DEFAULT);
1243		wr32(IGC_PCIE_PHY_DELAY, IGC_PCIE_PHY_DELAY_DEFAULT);
1244
1245		cycle_ctrl = IGC_PTM_CYCLE_CTRL_CYC_TIME(IGC_PTM_CYC_TIME_DEFAULT);
1246
1247		wr32(IGC_PTM_CYCLE_CTRL, cycle_ctrl);
1248
1249		ctrl = IGC_PTM_CTRL_EN |
1250			IGC_PTM_CTRL_START_NOW |
1251			IGC_PTM_CTRL_SHRT_CYC(IGC_PTM_SHORT_CYC_DEFAULT) |
1252			IGC_PTM_CTRL_PTM_TO(IGC_PTM_TIMEOUT_DEFAULT) |
1253			IGC_PTM_CTRL_TRIG;
1254
1255		wr32(IGC_PTM_CTRL, ctrl);
1256
1257		/* Force the first cycle to run. */
1258		wr32(IGC_PTM_STAT, IGC_PTM_STAT_VALID);
1259
1260		break;
1261	default:
1262		/* No work to do. */
1263		goto out;
1264	}
1265
1266	/* Re-initialize the timer. */
1267	if (hw->mac.type == igc_i225) {
1268		igc_ptp_time_restore(adapter);
1269	} else {
1270		timecounter_init(&adapter->tc, &adapter->cc,
1271				 ktime_to_ns(ktime_get_real()));
1272	}
1273out:
1274	spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
1275
1276	wrfl();
1277}