Loading...
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2021, Intel Corporation. */
3
4#include "ice.h"
5#include "ice_lib.h"
6#include "ice_trace.h"
7
8#define E810_OUT_PROP_DELAY_NS 1
9
10#define UNKNOWN_INCVAL_E82X 0x100000000ULL
11
12static const struct ptp_pin_desc ice_pin_desc_e810t[] = {
13 /* name idx func chan */
14 { "GNSS", GNSS, PTP_PF_EXTTS, 0, { 0, } },
15 { "SMA1", SMA1, PTP_PF_NONE, 1, { 0, } },
16 { "U.FL1", UFL1, PTP_PF_NONE, 1, { 0, } },
17 { "SMA2", SMA2, PTP_PF_NONE, 2, { 0, } },
18 { "U.FL2", UFL2, PTP_PF_NONE, 2, { 0, } },
19};
20
21/**
22 * ice_get_sma_config_e810t
23 * @hw: pointer to the hw struct
24 * @ptp_pins: pointer to the ptp_pin_desc struture
25 *
26 * Read the configuration of the SMA control logic and put it into the
27 * ptp_pin_desc structure
28 */
29static int
30ice_get_sma_config_e810t(struct ice_hw *hw, struct ptp_pin_desc *ptp_pins)
31{
32 u8 data, i;
33 int status;
34
35 /* Read initial pin state */
36 status = ice_read_sma_ctrl_e810t(hw, &data);
37 if (status)
38 return status;
39
40 /* initialize with defaults */
41 for (i = 0; i < NUM_PTP_PINS_E810T; i++) {
42 strscpy(ptp_pins[i].name, ice_pin_desc_e810t[i].name,
43 sizeof(ptp_pins[i].name));
44 ptp_pins[i].index = ice_pin_desc_e810t[i].index;
45 ptp_pins[i].func = ice_pin_desc_e810t[i].func;
46 ptp_pins[i].chan = ice_pin_desc_e810t[i].chan;
47 }
48
49 /* Parse SMA1/UFL1 */
50 switch (data & ICE_SMA1_MASK_E810T) {
51 case ICE_SMA1_MASK_E810T:
52 default:
53 ptp_pins[SMA1].func = PTP_PF_NONE;
54 ptp_pins[UFL1].func = PTP_PF_NONE;
55 break;
56 case ICE_SMA1_DIR_EN_E810T:
57 ptp_pins[SMA1].func = PTP_PF_PEROUT;
58 ptp_pins[UFL1].func = PTP_PF_NONE;
59 break;
60 case ICE_SMA1_TX_EN_E810T:
61 ptp_pins[SMA1].func = PTP_PF_EXTTS;
62 ptp_pins[UFL1].func = PTP_PF_NONE;
63 break;
64 case 0:
65 ptp_pins[SMA1].func = PTP_PF_EXTTS;
66 ptp_pins[UFL1].func = PTP_PF_PEROUT;
67 break;
68 }
69
70 /* Parse SMA2/UFL2 */
71 switch (data & ICE_SMA2_MASK_E810T) {
72 case ICE_SMA2_MASK_E810T:
73 default:
74 ptp_pins[SMA2].func = PTP_PF_NONE;
75 ptp_pins[UFL2].func = PTP_PF_NONE;
76 break;
77 case (ICE_SMA2_TX_EN_E810T | ICE_SMA2_UFL2_RX_DIS_E810T):
78 ptp_pins[SMA2].func = PTP_PF_EXTTS;
79 ptp_pins[UFL2].func = PTP_PF_NONE;
80 break;
81 case (ICE_SMA2_DIR_EN_E810T | ICE_SMA2_UFL2_RX_DIS_E810T):
82 ptp_pins[SMA2].func = PTP_PF_PEROUT;
83 ptp_pins[UFL2].func = PTP_PF_NONE;
84 break;
85 case (ICE_SMA2_DIR_EN_E810T | ICE_SMA2_TX_EN_E810T):
86 ptp_pins[SMA2].func = PTP_PF_NONE;
87 ptp_pins[UFL2].func = PTP_PF_EXTTS;
88 break;
89 case ICE_SMA2_DIR_EN_E810T:
90 ptp_pins[SMA2].func = PTP_PF_PEROUT;
91 ptp_pins[UFL2].func = PTP_PF_EXTTS;
92 break;
93 }
94
95 return 0;
96}
97
98/**
99 * ice_ptp_set_sma_config_e810t
100 * @hw: pointer to the hw struct
101 * @ptp_pins: pointer to the ptp_pin_desc struture
102 *
103 * Set the configuration of the SMA control logic based on the configuration in
104 * num_pins parameter
105 */
106static int
107ice_ptp_set_sma_config_e810t(struct ice_hw *hw,
108 const struct ptp_pin_desc *ptp_pins)
109{
110 int status;
111 u8 data;
112
113 /* SMA1 and UFL1 cannot be set to TX at the same time */
114 if (ptp_pins[SMA1].func == PTP_PF_PEROUT &&
115 ptp_pins[UFL1].func == PTP_PF_PEROUT)
116 return -EINVAL;
117
118 /* SMA2 and UFL2 cannot be set to RX at the same time */
119 if (ptp_pins[SMA2].func == PTP_PF_EXTTS &&
120 ptp_pins[UFL2].func == PTP_PF_EXTTS)
121 return -EINVAL;
122
123 /* Read initial pin state value */
124 status = ice_read_sma_ctrl_e810t(hw, &data);
125 if (status)
126 return status;
127
128 /* Set the right sate based on the desired configuration */
129 data &= ~ICE_SMA1_MASK_E810T;
130 if (ptp_pins[SMA1].func == PTP_PF_NONE &&
131 ptp_pins[UFL1].func == PTP_PF_NONE) {
132 dev_info(ice_hw_to_dev(hw), "SMA1 + U.FL1 disabled");
133 data |= ICE_SMA1_MASK_E810T;
134 } else if (ptp_pins[SMA1].func == PTP_PF_EXTTS &&
135 ptp_pins[UFL1].func == PTP_PF_NONE) {
136 dev_info(ice_hw_to_dev(hw), "SMA1 RX");
137 data |= ICE_SMA1_TX_EN_E810T;
138 } else if (ptp_pins[SMA1].func == PTP_PF_NONE &&
139 ptp_pins[UFL1].func == PTP_PF_PEROUT) {
140 /* U.FL 1 TX will always enable SMA 1 RX */
141 dev_info(ice_hw_to_dev(hw), "SMA1 RX + U.FL1 TX");
142 } else if (ptp_pins[SMA1].func == PTP_PF_EXTTS &&
143 ptp_pins[UFL1].func == PTP_PF_PEROUT) {
144 dev_info(ice_hw_to_dev(hw), "SMA1 RX + U.FL1 TX");
145 } else if (ptp_pins[SMA1].func == PTP_PF_PEROUT &&
146 ptp_pins[UFL1].func == PTP_PF_NONE) {
147 dev_info(ice_hw_to_dev(hw), "SMA1 TX");
148 data |= ICE_SMA1_DIR_EN_E810T;
149 }
150
151 data &= ~ICE_SMA2_MASK_E810T;
152 if (ptp_pins[SMA2].func == PTP_PF_NONE &&
153 ptp_pins[UFL2].func == PTP_PF_NONE) {
154 dev_info(ice_hw_to_dev(hw), "SMA2 + U.FL2 disabled");
155 data |= ICE_SMA2_MASK_E810T;
156 } else if (ptp_pins[SMA2].func == PTP_PF_EXTTS &&
157 ptp_pins[UFL2].func == PTP_PF_NONE) {
158 dev_info(ice_hw_to_dev(hw), "SMA2 RX");
159 data |= (ICE_SMA2_TX_EN_E810T |
160 ICE_SMA2_UFL2_RX_DIS_E810T);
161 } else if (ptp_pins[SMA2].func == PTP_PF_NONE &&
162 ptp_pins[UFL2].func == PTP_PF_EXTTS) {
163 dev_info(ice_hw_to_dev(hw), "UFL2 RX");
164 data |= (ICE_SMA2_DIR_EN_E810T | ICE_SMA2_TX_EN_E810T);
165 } else if (ptp_pins[SMA2].func == PTP_PF_PEROUT &&
166 ptp_pins[UFL2].func == PTP_PF_NONE) {
167 dev_info(ice_hw_to_dev(hw), "SMA2 TX");
168 data |= (ICE_SMA2_DIR_EN_E810T |
169 ICE_SMA2_UFL2_RX_DIS_E810T);
170 } else if (ptp_pins[SMA2].func == PTP_PF_PEROUT &&
171 ptp_pins[UFL2].func == PTP_PF_EXTTS) {
172 dev_info(ice_hw_to_dev(hw), "SMA2 TX + U.FL2 RX");
173 data |= ICE_SMA2_DIR_EN_E810T;
174 }
175
176 return ice_write_sma_ctrl_e810t(hw, data);
177}
178
179/**
180 * ice_ptp_set_sma_e810t
181 * @info: the driver's PTP info structure
182 * @pin: pin index in kernel structure
183 * @func: Pin function to be set (PTP_PF_NONE, PTP_PF_EXTTS or PTP_PF_PEROUT)
184 *
185 * Set the configuration of a single SMA pin
186 */
187static int
188ice_ptp_set_sma_e810t(struct ptp_clock_info *info, unsigned int pin,
189 enum ptp_pin_function func)
190{
191 struct ptp_pin_desc ptp_pins[NUM_PTP_PINS_E810T];
192 struct ice_pf *pf = ptp_info_to_pf(info);
193 struct ice_hw *hw = &pf->hw;
194 int err;
195
196 if (pin < SMA1 || func > PTP_PF_PEROUT)
197 return -EOPNOTSUPP;
198
199 err = ice_get_sma_config_e810t(hw, ptp_pins);
200 if (err)
201 return err;
202
203 /* Disable the same function on the other pin sharing the channel */
204 if (pin == SMA1 && ptp_pins[UFL1].func == func)
205 ptp_pins[UFL1].func = PTP_PF_NONE;
206 if (pin == UFL1 && ptp_pins[SMA1].func == func)
207 ptp_pins[SMA1].func = PTP_PF_NONE;
208
209 if (pin == SMA2 && ptp_pins[UFL2].func == func)
210 ptp_pins[UFL2].func = PTP_PF_NONE;
211 if (pin == UFL2 && ptp_pins[SMA2].func == func)
212 ptp_pins[SMA2].func = PTP_PF_NONE;
213
214 /* Set up new pin function in the temp table */
215 ptp_pins[pin].func = func;
216
217 return ice_ptp_set_sma_config_e810t(hw, ptp_pins);
218}
219
220/**
221 * ice_verify_pin_e810t
222 * @info: the driver's PTP info structure
223 * @pin: Pin index
224 * @func: Assigned function
225 * @chan: Assigned channel
226 *
227 * Verify if pin supports requested pin function. If the Check pins consistency.
228 * Reconfigure the SMA logic attached to the given pin to enable its
229 * desired functionality
230 */
231static int
232ice_verify_pin_e810t(struct ptp_clock_info *info, unsigned int pin,
233 enum ptp_pin_function func, unsigned int chan)
234{
235 /* Don't allow channel reassignment */
236 if (chan != ice_pin_desc_e810t[pin].chan)
237 return -EOPNOTSUPP;
238
239 /* Check if functions are properly assigned */
240 switch (func) {
241 case PTP_PF_NONE:
242 break;
243 case PTP_PF_EXTTS:
244 if (pin == UFL1)
245 return -EOPNOTSUPP;
246 break;
247 case PTP_PF_PEROUT:
248 if (pin == UFL2 || pin == GNSS)
249 return -EOPNOTSUPP;
250 break;
251 case PTP_PF_PHYSYNC:
252 return -EOPNOTSUPP;
253 }
254
255 return ice_ptp_set_sma_e810t(info, pin, func);
256}
257
258/**
259 * ice_ptp_cfg_tx_interrupt - Configure Tx timestamp interrupt for the device
260 * @pf: Board private structure
261 *
262 * Program the device to respond appropriately to the Tx timestamp interrupt
263 * cause.
264 */
265static void ice_ptp_cfg_tx_interrupt(struct ice_pf *pf)
266{
267 struct ice_hw *hw = &pf->hw;
268 bool enable;
269 u32 val;
270
271 switch (pf->ptp.tx_interrupt_mode) {
272 case ICE_PTP_TX_INTERRUPT_ALL:
273 /* React to interrupts across all quads. */
274 wr32(hw, PFINT_TSYN_MSK + (0x4 * hw->pf_id), (u32)0x1f);
275 enable = true;
276 break;
277 case ICE_PTP_TX_INTERRUPT_NONE:
278 /* Do not react to interrupts on any quad. */
279 wr32(hw, PFINT_TSYN_MSK + (0x4 * hw->pf_id), (u32)0x0);
280 enable = false;
281 break;
282 case ICE_PTP_TX_INTERRUPT_SELF:
283 default:
284 enable = pf->ptp.tstamp_config.tx_type == HWTSTAMP_TX_ON;
285 break;
286 }
287
288 /* Configure the Tx timestamp interrupt */
289 val = rd32(hw, PFINT_OICR_ENA);
290 if (enable)
291 val |= PFINT_OICR_TSYN_TX_M;
292 else
293 val &= ~PFINT_OICR_TSYN_TX_M;
294 wr32(hw, PFINT_OICR_ENA, val);
295}
296
297/**
298 * ice_set_rx_tstamp - Enable or disable Rx timestamping
299 * @pf: The PF pointer to search in
300 * @on: bool value for whether timestamps are enabled or disabled
301 */
302static void ice_set_rx_tstamp(struct ice_pf *pf, bool on)
303{
304 struct ice_vsi *vsi;
305 u16 i;
306
307 vsi = ice_get_main_vsi(pf);
308 if (!vsi || !vsi->rx_rings)
309 return;
310
311 /* Set the timestamp flag for all the Rx rings */
312 ice_for_each_rxq(vsi, i) {
313 if (!vsi->rx_rings[i])
314 continue;
315 vsi->rx_rings[i]->ptp_rx = on;
316 }
317}
318
319/**
320 * ice_ptp_disable_timestamp_mode - Disable current timestamp mode
321 * @pf: Board private structure
322 *
323 * Called during preparation for reset to temporarily disable timestamping on
324 * the device. Called during remove to disable timestamping while cleaning up
325 * driver resources.
326 */
327static void ice_ptp_disable_timestamp_mode(struct ice_pf *pf)
328{
329 struct ice_hw *hw = &pf->hw;
330 u32 val;
331
332 val = rd32(hw, PFINT_OICR_ENA);
333 val &= ~PFINT_OICR_TSYN_TX_M;
334 wr32(hw, PFINT_OICR_ENA, val);
335
336 ice_set_rx_tstamp(pf, false);
337}
338
339/**
340 * ice_ptp_restore_timestamp_mode - Restore timestamp configuration
341 * @pf: Board private structure
342 *
343 * Called at the end of rebuild to restore timestamp configuration after
344 * a device reset.
345 */
346void ice_ptp_restore_timestamp_mode(struct ice_pf *pf)
347{
348 struct ice_hw *hw = &pf->hw;
349 bool enable_rx;
350
351 ice_ptp_cfg_tx_interrupt(pf);
352
353 enable_rx = pf->ptp.tstamp_config.rx_filter == HWTSTAMP_FILTER_ALL;
354 ice_set_rx_tstamp(pf, enable_rx);
355
356 /* Trigger an immediate software interrupt to ensure that timestamps
357 * which occurred during reset are handled now.
358 */
359 wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M);
360 ice_flush(hw);
361}
362
363/**
364 * ice_ptp_read_src_clk_reg - Read the source clock register
365 * @pf: Board private structure
366 * @sts: Optional parameter for holding a pair of system timestamps from
367 * the system clock. Will be ignored if NULL is given.
368 */
369static u64
370ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts)
371{
372 struct ice_hw *hw = &pf->hw;
373 u32 hi, lo, lo2;
374 u8 tmr_idx;
375
376 tmr_idx = ice_get_ptp_src_clock_index(hw);
377 /* Read the system timestamp pre PHC read */
378 ptp_read_system_prets(sts);
379
380 lo = rd32(hw, GLTSYN_TIME_L(tmr_idx));
381
382 /* Read the system timestamp post PHC read */
383 ptp_read_system_postts(sts);
384
385 hi = rd32(hw, GLTSYN_TIME_H(tmr_idx));
386 lo2 = rd32(hw, GLTSYN_TIME_L(tmr_idx));
387
388 if (lo2 < lo) {
389 /* if TIME_L rolled over read TIME_L again and update
390 * system timestamps
391 */
392 ptp_read_system_prets(sts);
393 lo = rd32(hw, GLTSYN_TIME_L(tmr_idx));
394 ptp_read_system_postts(sts);
395 hi = rd32(hw, GLTSYN_TIME_H(tmr_idx));
396 }
397
398 return ((u64)hi << 32) | lo;
399}
400
401/**
402 * ice_ptp_extend_32b_ts - Convert a 32b nanoseconds timestamp to 64b
403 * @cached_phc_time: recently cached copy of PHC time
404 * @in_tstamp: Ingress/egress 32b nanoseconds timestamp value
405 *
406 * Hardware captures timestamps which contain only 32 bits of nominal
407 * nanoseconds, as opposed to the 64bit timestamps that the stack expects.
408 * Note that the captured timestamp values may be 40 bits, but the lower
409 * 8 bits are sub-nanoseconds and generally discarded.
410 *
411 * Extend the 32bit nanosecond timestamp using the following algorithm and
412 * assumptions:
413 *
414 * 1) have a recently cached copy of the PHC time
415 * 2) assume that the in_tstamp was captured 2^31 nanoseconds (~2.1
416 * seconds) before or after the PHC time was captured.
417 * 3) calculate the delta between the cached time and the timestamp
418 * 4) if the delta is smaller than 2^31 nanoseconds, then the timestamp was
419 * captured after the PHC time. In this case, the full timestamp is just
420 * the cached PHC time plus the delta.
421 * 5) otherwise, if the delta is larger than 2^31 nanoseconds, then the
422 * timestamp was captured *before* the PHC time, i.e. because the PHC
423 * cache was updated after the timestamp was captured by hardware. In this
424 * case, the full timestamp is the cached time minus the inverse delta.
425 *
426 * This algorithm works even if the PHC time was updated after a Tx timestamp
427 * was requested, but before the Tx timestamp event was reported from
428 * hardware.
429 *
430 * This calculation primarily relies on keeping the cached PHC time up to
431 * date. If the timestamp was captured more than 2^31 nanoseconds after the
432 * PHC time, it is possible that the lower 32bits of PHC time have
433 * overflowed more than once, and we might generate an incorrect timestamp.
434 *
435 * This is prevented by (a) periodically updating the cached PHC time once
436 * a second, and (b) discarding any Tx timestamp packet if it has waited for
437 * a timestamp for more than one second.
438 */
439static u64 ice_ptp_extend_32b_ts(u64 cached_phc_time, u32 in_tstamp)
440{
441 u32 delta, phc_time_lo;
442 u64 ns;
443
444 /* Extract the lower 32 bits of the PHC time */
445 phc_time_lo = (u32)cached_phc_time;
446
447 /* Calculate the delta between the lower 32bits of the cached PHC
448 * time and the in_tstamp value
449 */
450 delta = (in_tstamp - phc_time_lo);
451
452 /* Do not assume that the in_tstamp is always more recent than the
453 * cached PHC time. If the delta is large, it indicates that the
454 * in_tstamp was taken in the past, and should be converted
455 * forward.
456 */
457 if (delta > (U32_MAX / 2)) {
458 /* reverse the delta calculation here */
459 delta = (phc_time_lo - in_tstamp);
460 ns = cached_phc_time - delta;
461 } else {
462 ns = cached_phc_time + delta;
463 }
464
465 return ns;
466}
467
468/**
469 * ice_ptp_extend_40b_ts - Convert a 40b timestamp to 64b nanoseconds
470 * @pf: Board private structure
471 * @in_tstamp: Ingress/egress 40b timestamp value
472 *
473 * The Tx and Rx timestamps are 40 bits wide, including 32 bits of nominal
474 * nanoseconds, 7 bits of sub-nanoseconds, and a valid bit.
475 *
476 * *--------------------------------------------------------------*
477 * | 32 bits of nanoseconds | 7 high bits of sub ns underflow | v |
478 * *--------------------------------------------------------------*
479 *
480 * The low bit is an indicator of whether the timestamp is valid. The next
481 * 7 bits are a capture of the upper 7 bits of the sub-nanosecond underflow,
482 * and the remaining 32 bits are the lower 32 bits of the PHC timer.
483 *
484 * It is assumed that the caller verifies the timestamp is valid prior to
485 * calling this function.
486 *
487 * Extract the 32bit nominal nanoseconds and extend them. Use the cached PHC
488 * time stored in the device private PTP structure as the basis for timestamp
489 * extension.
490 *
491 * See ice_ptp_extend_32b_ts for a detailed explanation of the extension
492 * algorithm.
493 */
494static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp)
495{
496 const u64 mask = GENMASK_ULL(31, 0);
497 unsigned long discard_time;
498
499 /* Discard the hardware timestamp if the cached PHC time is too old */
500 discard_time = pf->ptp.cached_phc_jiffies + msecs_to_jiffies(2000);
501 if (time_is_before_jiffies(discard_time)) {
502 pf->ptp.tx_hwtstamp_discarded++;
503 return 0;
504 }
505
506 return ice_ptp_extend_32b_ts(pf->ptp.cached_phc_time,
507 (in_tstamp >> 8) & mask);
508}
509
510/**
511 * ice_ptp_is_tx_tracker_up - Check if Tx tracker is ready for new timestamps
512 * @tx: the PTP Tx timestamp tracker to check
513 *
514 * Check that a given PTP Tx timestamp tracker is up, i.e. that it is ready
515 * to accept new timestamp requests.
516 *
517 * Assumes the tx->lock spinlock is already held.
518 */
519static bool
520ice_ptp_is_tx_tracker_up(struct ice_ptp_tx *tx)
521{
522 lockdep_assert_held(&tx->lock);
523
524 return tx->init && !tx->calibrating;
525}
526
527/**
528 * ice_ptp_req_tx_single_tstamp - Request Tx timestamp for a port from FW
529 * @tx: the PTP Tx timestamp tracker
530 * @idx: index of the timestamp to request
531 */
532void ice_ptp_req_tx_single_tstamp(struct ice_ptp_tx *tx, u8 idx)
533{
534 struct ice_ptp_port *ptp_port;
535 struct sk_buff *skb;
536 struct ice_pf *pf;
537
538 if (!tx->init)
539 return;
540
541 ptp_port = container_of(tx, struct ice_ptp_port, tx);
542 pf = ptp_port_to_pf(ptp_port);
543
544 /* Drop packets which have waited for more than 2 seconds */
545 if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) {
546 /* Count the number of Tx timestamps that timed out */
547 pf->ptp.tx_hwtstamp_timeouts++;
548
549 skb = tx->tstamps[idx].skb;
550 tx->tstamps[idx].skb = NULL;
551 clear_bit(idx, tx->in_use);
552
553 dev_kfree_skb_any(skb);
554 return;
555 }
556
557 ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);
558
559 /* Write TS index to read to the PF register so the FW can read it */
560 wr32(&pf->hw, PF_SB_ATQBAL,
561 TS_LL_READ_TS_INTR | FIELD_PREP(TS_LL_READ_TS_IDX, idx) |
562 TS_LL_READ_TS);
563 tx->last_ll_ts_idx_read = idx;
564}
565
566/**
567 * ice_ptp_complete_tx_single_tstamp - Complete Tx timestamp for a port
568 * @tx: the PTP Tx timestamp tracker
569 */
570void ice_ptp_complete_tx_single_tstamp(struct ice_ptp_tx *tx)
571{
572 struct skb_shared_hwtstamps shhwtstamps = {};
573 u8 idx = tx->last_ll_ts_idx_read;
574 struct ice_ptp_port *ptp_port;
575 u64 raw_tstamp, tstamp;
576 bool drop_ts = false;
577 struct sk_buff *skb;
578 struct ice_pf *pf;
579 u32 val;
580
581 if (!tx->init || tx->last_ll_ts_idx_read < 0)
582 return;
583
584 ptp_port = container_of(tx, struct ice_ptp_port, tx);
585 pf = ptp_port_to_pf(ptp_port);
586
587 ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);
588
589 val = rd32(&pf->hw, PF_SB_ATQBAL);
590
591 /* When the bit is cleared, the TS is ready in the register */
592 if (val & TS_LL_READ_TS) {
593 dev_err(ice_pf_to_dev(pf), "Failed to get the Tx tstamp - FW not ready");
594 return;
595 }
596
597 /* High 8 bit value of the TS is on the bits 16:23 */
598 raw_tstamp = FIELD_GET(TS_LL_READ_TS_HIGH, val);
599 raw_tstamp <<= 32;
600
601 /* Read the low 32 bit value */
602 raw_tstamp |= (u64)rd32(&pf->hw, PF_SB_ATQBAH);
603
604 /* Devices using this interface always verify the timestamp differs
605 * relative to the last cached timestamp value.
606 */
607 if (raw_tstamp == tx->tstamps[idx].cached_tstamp)
608 return;
609
610 tx->tstamps[idx].cached_tstamp = raw_tstamp;
611 clear_bit(idx, tx->in_use);
612 skb = tx->tstamps[idx].skb;
613 tx->tstamps[idx].skb = NULL;
614 if (test_and_clear_bit(idx, tx->stale))
615 drop_ts = true;
616
617 if (!skb)
618 return;
619
620 if (drop_ts) {
621 dev_kfree_skb_any(skb);
622 return;
623 }
624
625 /* Extend the timestamp using cached PHC time */
626 tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp);
627 if (tstamp) {
628 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
629 ice_trace(tx_tstamp_complete, skb, idx);
630 }
631
632 skb_tstamp_tx(skb, &shhwtstamps);
633 dev_kfree_skb_any(skb);
634}
635
636/**
637 * ice_ptp_process_tx_tstamp - Process Tx timestamps for a port
638 * @tx: the PTP Tx timestamp tracker
639 *
640 * Process timestamps captured by the PHY associated with this port. To do
641 * this, loop over each index with a waiting skb.
642 *
643 * If a given index has a valid timestamp, perform the following steps:
644 *
645 * 1) check that the timestamp request is not stale
646 * 2) check that a timestamp is ready and available in the PHY memory bank
647 * 3) read and copy the timestamp out of the PHY register
648 * 4) unlock the index by clearing the associated in_use bit
649 * 5) check if the timestamp is stale, and discard if so
650 * 6) extend the 40 bit timestamp value to get a 64 bit timestamp value
651 * 7) send this 64 bit timestamp to the stack
652 *
653 * Note that we do not hold the tracking lock while reading the Tx timestamp.
654 * This is because reading the timestamp requires taking a mutex that might
655 * sleep.
656 *
657 * The only place where we set in_use is when a new timestamp is initiated
658 * with a slot index. This is only called in the hard xmit routine where an
659 * SKB has a request flag set. The only places where we clear this bit is this
660 * function, or during teardown when the Tx timestamp tracker is being
661 * removed. A timestamp index will never be re-used until the in_use bit for
662 * that index is cleared.
663 *
664 * If a Tx thread starts a new timestamp, we might not begin processing it
665 * right away but we will notice it at the end when we re-queue the task.
666 *
667 * If a Tx thread starts a new timestamp just after this function exits, the
668 * interrupt for that timestamp should re-trigger this function once
669 * a timestamp is ready.
670 *
671 * In cases where the PTP hardware clock was directly adjusted, some
672 * timestamps may not be able to safely use the timestamp extension math. In
673 * this case, software will set the stale bit for any outstanding Tx
674 * timestamps when the clock is adjusted. Then this function will discard
675 * those captured timestamps instead of sending them to the stack.
676 *
677 * If a Tx packet has been waiting for more than 2 seconds, it is not possible
678 * to correctly extend the timestamp using the cached PHC time. It is
679 * extremely unlikely that a packet will ever take this long to timestamp. If
680 * we detect a Tx timestamp request that has waited for this long we assume
681 * the packet will never be sent by hardware and discard it without reading
682 * the timestamp register.
683 */
684static void ice_ptp_process_tx_tstamp(struct ice_ptp_tx *tx)
685{
686 struct ice_ptp_port *ptp_port;
687 unsigned long flags;
688 struct ice_pf *pf;
689 struct ice_hw *hw;
690 u64 tstamp_ready;
691 bool link_up;
692 int err;
693 u8 idx;
694
695 ptp_port = container_of(tx, struct ice_ptp_port, tx);
696 pf = ptp_port_to_pf(ptp_port);
697 hw = &pf->hw;
698
699 /* Read the Tx ready status first */
700 if (tx->has_ready_bitmap) {
701 err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready);
702 if (err)
703 return;
704 }
705
706 /* Drop packets if the link went down */
707 link_up = ptp_port->link_up;
708
709 for_each_set_bit(idx, tx->in_use, tx->len) {
710 struct skb_shared_hwtstamps shhwtstamps = {};
711 u8 phy_idx = idx + tx->offset;
712 u64 raw_tstamp = 0, tstamp;
713 bool drop_ts = !link_up;
714 struct sk_buff *skb;
715
716 /* Drop packets which have waited for more than 2 seconds */
717 if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) {
718 drop_ts = true;
719
720 /* Count the number of Tx timestamps that timed out */
721 pf->ptp.tx_hwtstamp_timeouts++;
722 }
723
724 /* Only read a timestamp from the PHY if its marked as ready
725 * by the tstamp_ready register. This avoids unnecessary
726 * reading of timestamps which are not yet valid. This is
727 * important as we must read all timestamps which are valid
728 * and only timestamps which are valid during each interrupt.
729 * If we do not, the hardware logic for generating a new
730 * interrupt can get stuck on some devices.
731 */
732 if (tx->has_ready_bitmap &&
733 !(tstamp_ready & BIT_ULL(phy_idx))) {
734 if (drop_ts)
735 goto skip_ts_read;
736
737 continue;
738 }
739
740 ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);
741
742 err = ice_read_phy_tstamp(hw, tx->block, phy_idx, &raw_tstamp);
743 if (err && !drop_ts)
744 continue;
745
746 ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);
747
748 /* For PHYs which don't implement a proper timestamp ready
749 * bitmap, verify that the timestamp value is different
750 * from the last cached timestamp. If it is not, skip this for
751 * now assuming it hasn't yet been captured by hardware.
752 */
753 if (!drop_ts && !tx->has_ready_bitmap &&
754 raw_tstamp == tx->tstamps[idx].cached_tstamp)
755 continue;
756
757 /* Discard any timestamp value without the valid bit set */
758 if (!(raw_tstamp & ICE_PTP_TS_VALID))
759 drop_ts = true;
760
761skip_ts_read:
762 spin_lock_irqsave(&tx->lock, flags);
763 if (!tx->has_ready_bitmap && raw_tstamp)
764 tx->tstamps[idx].cached_tstamp = raw_tstamp;
765 clear_bit(idx, tx->in_use);
766 skb = tx->tstamps[idx].skb;
767 tx->tstamps[idx].skb = NULL;
768 if (test_and_clear_bit(idx, tx->stale))
769 drop_ts = true;
770 spin_unlock_irqrestore(&tx->lock, flags);
771
772 /* It is unlikely but possible that the SKB will have been
773 * flushed at this point due to link change or teardown.
774 */
775 if (!skb)
776 continue;
777
778 if (drop_ts) {
779 dev_kfree_skb_any(skb);
780 continue;
781 }
782
783 /* Extend the timestamp using cached PHC time */
784 tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp);
785 if (tstamp) {
786 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
787 ice_trace(tx_tstamp_complete, skb, idx);
788 }
789
790 skb_tstamp_tx(skb, &shhwtstamps);
791 dev_kfree_skb_any(skb);
792 }
793}
794
795/**
796 * ice_ptp_tx_tstamp_owner - Process Tx timestamps for all ports on the device
797 * @pf: Board private structure
798 */
799static enum ice_tx_tstamp_work ice_ptp_tx_tstamp_owner(struct ice_pf *pf)
800{
801 struct ice_ptp_port *port;
802 unsigned int i;
803
804 mutex_lock(&pf->ptp.ports_owner.lock);
805 list_for_each_entry(port, &pf->ptp.ports_owner.ports, list_member) {
806 struct ice_ptp_tx *tx = &port->tx;
807
808 if (!tx || !tx->init)
809 continue;
810
811 ice_ptp_process_tx_tstamp(tx);
812 }
813 mutex_unlock(&pf->ptp.ports_owner.lock);
814
815 for (i = 0; i < ICE_MAX_QUAD; i++) {
816 u64 tstamp_ready;
817 int err;
818
819 /* Read the Tx ready status first */
820 err = ice_get_phy_tx_tstamp_ready(&pf->hw, i, &tstamp_ready);
821 if (err)
822 break;
823 else if (tstamp_ready)
824 return ICE_TX_TSTAMP_WORK_PENDING;
825 }
826
827 return ICE_TX_TSTAMP_WORK_DONE;
828}
829
830/**
831 * ice_ptp_tx_tstamp - Process Tx timestamps for this function.
832 * @tx: Tx tracking structure to initialize
833 *
834 * Returns: ICE_TX_TSTAMP_WORK_PENDING if there are any outstanding incomplete
835 * Tx timestamps, or ICE_TX_TSTAMP_WORK_DONE otherwise.
836 */
837static enum ice_tx_tstamp_work ice_ptp_tx_tstamp(struct ice_ptp_tx *tx)
838{
839 bool more_timestamps;
840 unsigned long flags;
841
842 if (!tx->init)
843 return ICE_TX_TSTAMP_WORK_DONE;
844
845 /* Process the Tx timestamp tracker */
846 ice_ptp_process_tx_tstamp(tx);
847
848 /* Check if there are outstanding Tx timestamps */
849 spin_lock_irqsave(&tx->lock, flags);
850 more_timestamps = tx->init && !bitmap_empty(tx->in_use, tx->len);
851 spin_unlock_irqrestore(&tx->lock, flags);
852
853 if (more_timestamps)
854 return ICE_TX_TSTAMP_WORK_PENDING;
855
856 return ICE_TX_TSTAMP_WORK_DONE;
857}
858
859/**
860 * ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps
861 * @tx: Tx tracking structure to initialize
862 *
863 * Assumes that the length has already been initialized. Do not call directly,
864 * use the ice_ptp_init_tx_* instead.
865 */
866static int
867ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx)
868{
869 unsigned long *in_use, *stale;
870 struct ice_tx_tstamp *tstamps;
871
872 tstamps = kcalloc(tx->len, sizeof(*tstamps), GFP_KERNEL);
873 in_use = bitmap_zalloc(tx->len, GFP_KERNEL);
874 stale = bitmap_zalloc(tx->len, GFP_KERNEL);
875
876 if (!tstamps || !in_use || !stale) {
877 kfree(tstamps);
878 bitmap_free(in_use);
879 bitmap_free(stale);
880
881 return -ENOMEM;
882 }
883
884 tx->tstamps = tstamps;
885 tx->in_use = in_use;
886 tx->stale = stale;
887 tx->init = 1;
888 tx->last_ll_ts_idx_read = -1;
889
890 spin_lock_init(&tx->lock);
891
892 return 0;
893}
894
895/**
896 * ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker
897 * @pf: Board private structure
898 * @tx: the tracker to flush
899 *
900 * Called during teardown when a Tx tracker is being removed.
901 */
902static void
903ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
904{
905 struct ice_hw *hw = &pf->hw;
906 unsigned long flags;
907 u64 tstamp_ready;
908 int err;
909 u8 idx;
910
911 err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready);
912 if (err) {
913 dev_dbg(ice_pf_to_dev(pf), "Failed to get the Tx tstamp ready bitmap for block %u, err %d\n",
914 tx->block, err);
915
916 /* If we fail to read the Tx timestamp ready bitmap just
917 * skip clearing the PHY timestamps.
918 */
919 tstamp_ready = 0;
920 }
921
922 for_each_set_bit(idx, tx->in_use, tx->len) {
923 u8 phy_idx = idx + tx->offset;
924 struct sk_buff *skb;
925
926 /* In case this timestamp is ready, we need to clear it. */
927 if (!hw->reset_ongoing && (tstamp_ready & BIT_ULL(phy_idx)))
928 ice_clear_phy_tstamp(hw, tx->block, phy_idx);
929
930 spin_lock_irqsave(&tx->lock, flags);
931 skb = tx->tstamps[idx].skb;
932 tx->tstamps[idx].skb = NULL;
933 clear_bit(idx, tx->in_use);
934 clear_bit(idx, tx->stale);
935 spin_unlock_irqrestore(&tx->lock, flags);
936
937 /* Count the number of Tx timestamps flushed */
938 pf->ptp.tx_hwtstamp_flushed++;
939
940 /* Free the SKB after we've cleared the bit */
941 dev_kfree_skb_any(skb);
942 }
943}
944
945/**
946 * ice_ptp_mark_tx_tracker_stale - Mark unfinished timestamps as stale
947 * @tx: the tracker to mark
948 *
949 * Mark currently outstanding Tx timestamps as stale. This prevents sending
950 * their timestamp value to the stack. This is required to prevent extending
951 * the 40bit hardware timestamp incorrectly.
952 *
953 * This should be called when the PTP clock is modified such as after a set
954 * time request.
955 */
956static void
957ice_ptp_mark_tx_tracker_stale(struct ice_ptp_tx *tx)
958{
959 unsigned long flags;
960
961 spin_lock_irqsave(&tx->lock, flags);
962 bitmap_or(tx->stale, tx->stale, tx->in_use, tx->len);
963 spin_unlock_irqrestore(&tx->lock, flags);
964}
965
966/**
967 * ice_ptp_flush_all_tx_tracker - Flush all timestamp trackers on this clock
968 * @pf: Board private structure
969 *
970 * Called by the clock owner to flush all the Tx timestamp trackers associated
971 * with the clock.
972 */
973static void
974ice_ptp_flush_all_tx_tracker(struct ice_pf *pf)
975{
976 struct ice_ptp_port *port;
977
978 list_for_each_entry(port, &pf->ptp.ports_owner.ports, list_member)
979 ice_ptp_flush_tx_tracker(ptp_port_to_pf(port), &port->tx);
980}
981
982/**
983 * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker
984 * @pf: Board private structure
985 * @tx: Tx tracking structure to release
986 *
987 * Free memory associated with the Tx timestamp tracker.
988 */
989static void
990ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
991{
992 unsigned long flags;
993
994 spin_lock_irqsave(&tx->lock, flags);
995 tx->init = 0;
996 spin_unlock_irqrestore(&tx->lock, flags);
997
998 /* wait for potentially outstanding interrupt to complete */
999 synchronize_irq(pf->oicr_irq.virq);
1000
1001 ice_ptp_flush_tx_tracker(pf, tx);
1002
1003 kfree(tx->tstamps);
1004 tx->tstamps = NULL;
1005
1006 bitmap_free(tx->in_use);
1007 tx->in_use = NULL;
1008
1009 bitmap_free(tx->stale);
1010 tx->stale = NULL;
1011
1012 tx->len = 0;
1013}
1014
1015/**
1016 * ice_ptp_init_tx_e82x - Initialize tracking for Tx timestamps
1017 * @pf: Board private structure
1018 * @tx: the Tx tracking structure to initialize
1019 * @port: the port this structure tracks
1020 *
1021 * Initialize the Tx timestamp tracker for this port. For generic MAC devices,
1022 * the timestamp block is shared for all ports in the same quad. To avoid
1023 * ports using the same timestamp index, logically break the block of
1024 * registers into chunks based on the port number.
1025 */
1026static int
1027ice_ptp_init_tx_e82x(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port)
1028{
1029 tx->block = port / ICE_PORTS_PER_QUAD;
1030 tx->offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT_E82X;
1031 tx->len = INDEX_PER_PORT_E82X;
1032 tx->has_ready_bitmap = 1;
1033
1034 return ice_ptp_alloc_tx_tracker(tx);
1035}
1036
1037/**
1038 * ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps
1039 * @pf: Board private structure
1040 * @tx: the Tx tracking structure to initialize
1041 *
1042 * Initialize the Tx timestamp tracker for this PF. For E810 devices, each
1043 * port has its own block of timestamps, independent of the other ports.
1044 */
1045static int
1046ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)
1047{
1048 tx->block = pf->hw.port_info->lport;
1049 tx->offset = 0;
1050 tx->len = INDEX_PER_PORT_E810;
1051 /* The E810 PHY does not provide a timestamp ready bitmap. Instead,
1052 * verify new timestamps against cached copy of the last read
1053 * timestamp.
1054 */
1055 tx->has_ready_bitmap = 0;
1056
1057 return ice_ptp_alloc_tx_tracker(tx);
1058}
1059
1060/**
1061 * ice_ptp_update_cached_phctime - Update the cached PHC time values
1062 * @pf: Board specific private structure
1063 *
1064 * This function updates the system time values which are cached in the PF
1065 * structure and the Rx rings.
1066 *
1067 * This function must be called periodically to ensure that the cached value
1068 * is never more than 2 seconds old.
1069 *
1070 * Note that the cached copy in the PF PTP structure is always updated, even
1071 * if we can't update the copy in the Rx rings.
1072 *
1073 * Return:
1074 * * 0 - OK, successfully updated
1075 * * -EAGAIN - PF was busy, need to reschedule the update
1076 */
1077static int ice_ptp_update_cached_phctime(struct ice_pf *pf)
1078{
1079 struct device *dev = ice_pf_to_dev(pf);
1080 unsigned long update_before;
1081 u64 systime;
1082 int i;
1083
1084 update_before = pf->ptp.cached_phc_jiffies + msecs_to_jiffies(2000);
1085 if (pf->ptp.cached_phc_time &&
1086 time_is_before_jiffies(update_before)) {
1087 unsigned long time_taken = jiffies - pf->ptp.cached_phc_jiffies;
1088
1089 dev_warn(dev, "%u msecs passed between update to cached PHC time\n",
1090 jiffies_to_msecs(time_taken));
1091 pf->ptp.late_cached_phc_updates++;
1092 }
1093
1094 /* Read the current PHC time */
1095 systime = ice_ptp_read_src_clk_reg(pf, NULL);
1096
1097 /* Update the cached PHC time stored in the PF structure */
1098 WRITE_ONCE(pf->ptp.cached_phc_time, systime);
1099 WRITE_ONCE(pf->ptp.cached_phc_jiffies, jiffies);
1100
1101 if (test_and_set_bit(ICE_CFG_BUSY, pf->state))
1102 return -EAGAIN;
1103
1104 ice_for_each_vsi(pf, i) {
1105 struct ice_vsi *vsi = pf->vsi[i];
1106 int j;
1107
1108 if (!vsi)
1109 continue;
1110
1111 if (vsi->type != ICE_VSI_PF)
1112 continue;
1113
1114 ice_for_each_rxq(vsi, j) {
1115 if (!vsi->rx_rings[j])
1116 continue;
1117 WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime);
1118 }
1119 }
1120 clear_bit(ICE_CFG_BUSY, pf->state);
1121
1122 return 0;
1123}
1124
1125/**
1126 * ice_ptp_reset_cached_phctime - Reset cached PHC time after an update
1127 * @pf: Board specific private structure
1128 *
1129 * This function must be called when the cached PHC time is no longer valid,
1130 * such as after a time adjustment. It marks any currently outstanding Tx
1131 * timestamps as stale and updates the cached PHC time for both the PF and Rx
1132 * rings.
1133 *
1134 * If updating the PHC time cannot be done immediately, a warning message is
1135 * logged and the work item is scheduled immediately to minimize the window
1136 * with a wrong cached timestamp.
1137 */
1138static void ice_ptp_reset_cached_phctime(struct ice_pf *pf)
1139{
1140 struct device *dev = ice_pf_to_dev(pf);
1141 int err;
1142
1143 /* Update the cached PHC time immediately if possible, otherwise
1144 * schedule the work item to execute soon.
1145 */
1146 err = ice_ptp_update_cached_phctime(pf);
1147 if (err) {
1148 /* If another thread is updating the Rx rings, we won't
1149 * properly reset them here. This could lead to reporting of
1150 * invalid timestamps, but there isn't much we can do.
1151 */
1152 dev_warn(dev, "%s: ICE_CFG_BUSY, unable to immediately update cached PHC time\n",
1153 __func__);
1154
1155 /* Queue the work item to update the Rx rings when possible */
1156 kthread_queue_delayed_work(pf->ptp.kworker, &pf->ptp.work,
1157 msecs_to_jiffies(10));
1158 }
1159
1160 /* Mark any outstanding timestamps as stale, since they might have
1161 * been captured in hardware before the time update. This could lead
1162 * to us extending them with the wrong cached value resulting in
1163 * incorrect timestamp values.
1164 */
1165 ice_ptp_mark_tx_tracker_stale(&pf->ptp.port.tx);
1166}
1167
1168/**
1169 * ice_ptp_read_time - Read the time from the device
1170 * @pf: Board private structure
1171 * @ts: timespec structure to hold the current time value
1172 * @sts: Optional parameter for holding a pair of system timestamps from
1173 * the system clock. Will be ignored if NULL is given.
1174 *
1175 * This function reads the source clock registers and stores them in a timespec.
1176 * However, since the registers are 64 bits of nanoseconds, we must convert the
1177 * result to a timespec before we can return.
1178 */
1179static void
1180ice_ptp_read_time(struct ice_pf *pf, struct timespec64 *ts,
1181 struct ptp_system_timestamp *sts)
1182{
1183 u64 time_ns = ice_ptp_read_src_clk_reg(pf, sts);
1184
1185 *ts = ns_to_timespec64(time_ns);
1186}
1187
1188/**
1189 * ice_ptp_write_init - Set PHC time to provided value
1190 * @pf: Board private structure
1191 * @ts: timespec structure that holds the new time value
1192 *
1193 * Set the PHC time to the specified time provided in the timespec.
1194 */
1195static int ice_ptp_write_init(struct ice_pf *pf, struct timespec64 *ts)
1196{
1197 u64 ns = timespec64_to_ns(ts);
1198 struct ice_hw *hw = &pf->hw;
1199
1200 return ice_ptp_init_time(hw, ns);
1201}
1202
1203/**
1204 * ice_ptp_write_adj - Adjust PHC clock time atomically
1205 * @pf: Board private structure
1206 * @adj: Adjustment in nanoseconds
1207 *
1208 * Perform an atomic adjustment of the PHC time by the specified number of
1209 * nanoseconds.
1210 */
1211static int ice_ptp_write_adj(struct ice_pf *pf, s32 adj)
1212{
1213 struct ice_hw *hw = &pf->hw;
1214
1215 return ice_ptp_adj_clock(hw, adj);
1216}
1217
1218/**
1219 * ice_base_incval - Get base timer increment value
1220 * @pf: Board private structure
1221 *
1222 * Look up the base timer increment value for this device. The base increment
1223 * value is used to define the nominal clock tick rate. This increment value
1224 * is programmed during device initialization. It is also used as the basis
1225 * for calculating adjustments using scaled_ppm.
1226 */
1227static u64 ice_base_incval(struct ice_pf *pf)
1228{
1229 struct ice_hw *hw = &pf->hw;
1230 u64 incval;
1231
1232 if (ice_is_e810(hw))
1233 incval = ICE_PTP_NOMINAL_INCVAL_E810;
1234 else if (ice_e82x_time_ref(hw) < NUM_ICE_TIME_REF_FREQ)
1235 incval = ice_e82x_nominal_incval(ice_e82x_time_ref(hw));
1236 else
1237 incval = UNKNOWN_INCVAL_E82X;
1238
1239 dev_dbg(ice_pf_to_dev(pf), "PTP: using base increment value of 0x%016llx\n",
1240 incval);
1241
1242 return incval;
1243}
1244
1245/**
1246 * ice_ptp_check_tx_fifo - Check whether Tx FIFO is in an OK state
1247 * @port: PTP port for which Tx FIFO is checked
1248 */
1249static int ice_ptp_check_tx_fifo(struct ice_ptp_port *port)
1250{
1251 int quad = port->port_num / ICE_PORTS_PER_QUAD;
1252 int offs = port->port_num % ICE_PORTS_PER_QUAD;
1253 struct ice_pf *pf;
1254 struct ice_hw *hw;
1255 u32 val, phy_sts;
1256 int err;
1257
1258 pf = ptp_port_to_pf(port);
1259 hw = &pf->hw;
1260
1261 if (port->tx_fifo_busy_cnt == FIFO_OK)
1262 return 0;
1263
1264 /* need to read FIFO state */
1265 if (offs == 0 || offs == 1)
1266 err = ice_read_quad_reg_e82x(hw, quad, Q_REG_FIFO01_STATUS,
1267 &val);
1268 else
1269 err = ice_read_quad_reg_e82x(hw, quad, Q_REG_FIFO23_STATUS,
1270 &val);
1271
1272 if (err) {
1273 dev_err(ice_pf_to_dev(pf), "PTP failed to check port %d Tx FIFO, err %d\n",
1274 port->port_num, err);
1275 return err;
1276 }
1277
1278 if (offs & 0x1)
1279 phy_sts = FIELD_GET(Q_REG_FIFO13_M, val);
1280 else
1281 phy_sts = FIELD_GET(Q_REG_FIFO02_M, val);
1282
1283 if (phy_sts & FIFO_EMPTY) {
1284 port->tx_fifo_busy_cnt = FIFO_OK;
1285 return 0;
1286 }
1287
1288 port->tx_fifo_busy_cnt++;
1289
1290 dev_dbg(ice_pf_to_dev(pf), "Try %d, port %d FIFO not empty\n",
1291 port->tx_fifo_busy_cnt, port->port_num);
1292
1293 if (port->tx_fifo_busy_cnt == ICE_PTP_FIFO_NUM_CHECKS) {
1294 dev_dbg(ice_pf_to_dev(pf),
1295 "Port %d Tx FIFO still not empty; resetting quad %d\n",
1296 port->port_num, quad);
1297 ice_ptp_reset_ts_memory_quad_e82x(hw, quad);
1298 port->tx_fifo_busy_cnt = FIFO_OK;
1299 return 0;
1300 }
1301
1302 return -EAGAIN;
1303}
1304
1305/**
1306 * ice_ptp_wait_for_offsets - Check for valid Tx and Rx offsets
1307 * @work: Pointer to the kthread_work structure for this task
1308 *
1309 * Check whether hardware has completed measuring the Tx and Rx offset values
1310 * used to configure and enable vernier timestamp calibration.
1311 *
1312 * Once the offset in either direction is measured, configure the associated
1313 * registers with the calibrated offset values and enable timestamping. The Tx
1314 * and Rx directions are configured independently as soon as their associated
1315 * offsets are known.
1316 *
1317 * This function reschedules itself until both Tx and Rx calibration have
1318 * completed.
1319 */
1320static void ice_ptp_wait_for_offsets(struct kthread_work *work)
1321{
1322 struct ice_ptp_port *port;
1323 struct ice_pf *pf;
1324 struct ice_hw *hw;
1325 int tx_err;
1326 int rx_err;
1327
1328 port = container_of(work, struct ice_ptp_port, ov_work.work);
1329 pf = ptp_port_to_pf(port);
1330 hw = &pf->hw;
1331
1332 if (ice_is_reset_in_progress(pf->state)) {
1333 /* wait for device driver to complete reset */
1334 kthread_queue_delayed_work(pf->ptp.kworker,
1335 &port->ov_work,
1336 msecs_to_jiffies(100));
1337 return;
1338 }
1339
1340 tx_err = ice_ptp_check_tx_fifo(port);
1341 if (!tx_err)
1342 tx_err = ice_phy_cfg_tx_offset_e82x(hw, port->port_num);
1343 rx_err = ice_phy_cfg_rx_offset_e82x(hw, port->port_num);
1344 if (tx_err || rx_err) {
1345 /* Tx and/or Rx offset not yet configured, try again later */
1346 kthread_queue_delayed_work(pf->ptp.kworker,
1347 &port->ov_work,
1348 msecs_to_jiffies(100));
1349 return;
1350 }
1351}
1352
1353/**
1354 * ice_ptp_port_phy_stop - Stop timestamping for a PHY port
1355 * @ptp_port: PTP port to stop
1356 */
1357static int
1358ice_ptp_port_phy_stop(struct ice_ptp_port *ptp_port)
1359{
1360 struct ice_pf *pf = ptp_port_to_pf(ptp_port);
1361 u8 port = ptp_port->port_num;
1362 struct ice_hw *hw = &pf->hw;
1363 int err;
1364
1365 if (ice_is_e810(hw))
1366 return 0;
1367
1368 mutex_lock(&ptp_port->ps_lock);
1369
1370 kthread_cancel_delayed_work_sync(&ptp_port->ov_work);
1371
1372 err = ice_stop_phy_timer_e82x(hw, port, true);
1373 if (err)
1374 dev_err(ice_pf_to_dev(pf), "PTP failed to set PHY port %d down, err %d\n",
1375 port, err);
1376
1377 mutex_unlock(&ptp_port->ps_lock);
1378
1379 return err;
1380}
1381
1382/**
1383 * ice_ptp_port_phy_restart - (Re)start and calibrate PHY timestamping
1384 * @ptp_port: PTP port for which the PHY start is set
1385 *
1386 * Start the PHY timestamping block, and initiate Vernier timestamping
1387 * calibration. If timestamping cannot be calibrated (such as if link is down)
1388 * then disable the timestamping block instead.
1389 */
1390static int
1391ice_ptp_port_phy_restart(struct ice_ptp_port *ptp_port)
1392{
1393 struct ice_pf *pf = ptp_port_to_pf(ptp_port);
1394 u8 port = ptp_port->port_num;
1395 struct ice_hw *hw = &pf->hw;
1396 unsigned long flags;
1397 int err;
1398
1399 if (ice_is_e810(hw))
1400 return 0;
1401
1402 if (!ptp_port->link_up)
1403 return ice_ptp_port_phy_stop(ptp_port);
1404
1405 mutex_lock(&ptp_port->ps_lock);
1406
1407 kthread_cancel_delayed_work_sync(&ptp_port->ov_work);
1408
1409 /* temporarily disable Tx timestamps while calibrating PHY offset */
1410 spin_lock_irqsave(&ptp_port->tx.lock, flags);
1411 ptp_port->tx.calibrating = true;
1412 spin_unlock_irqrestore(&ptp_port->tx.lock, flags);
1413 ptp_port->tx_fifo_busy_cnt = 0;
1414
1415 /* Start the PHY timer in Vernier mode */
1416 err = ice_start_phy_timer_e82x(hw, port);
1417 if (err)
1418 goto out_unlock;
1419
1420 /* Enable Tx timestamps right away */
1421 spin_lock_irqsave(&ptp_port->tx.lock, flags);
1422 ptp_port->tx.calibrating = false;
1423 spin_unlock_irqrestore(&ptp_port->tx.lock, flags);
1424
1425 kthread_queue_delayed_work(pf->ptp.kworker, &ptp_port->ov_work, 0);
1426
1427out_unlock:
1428 if (err)
1429 dev_err(ice_pf_to_dev(pf), "PTP failed to set PHY port %d up, err %d\n",
1430 port, err);
1431
1432 mutex_unlock(&ptp_port->ps_lock);
1433
1434 return err;
1435}
1436
1437/**
1438 * ice_ptp_link_change - Reconfigure PTP after link status change
1439 * @pf: Board private structure
1440 * @port: Port for which the PHY start is set
1441 * @linkup: Link is up or down
1442 */
1443void ice_ptp_link_change(struct ice_pf *pf, u8 port, bool linkup)
1444{
1445 struct ice_ptp_port *ptp_port;
1446 struct ice_hw *hw = &pf->hw;
1447
1448 if (pf->ptp.state != ICE_PTP_READY)
1449 return;
1450
1451 if (WARN_ON_ONCE(port >= ICE_NUM_EXTERNAL_PORTS))
1452 return;
1453
1454 ptp_port = &pf->ptp.port;
1455 if (WARN_ON_ONCE(ptp_port->port_num != port))
1456 return;
1457
1458 /* Update cached link status for this port immediately */
1459 ptp_port->link_up = linkup;
1460
1461 switch (hw->phy_model) {
1462 case ICE_PHY_E810:
1463 /* Do not reconfigure E810 PHY */
1464 return;
1465 case ICE_PHY_E82X:
1466 ice_ptp_port_phy_restart(ptp_port);
1467 return;
1468 default:
1469 dev_warn(ice_pf_to_dev(pf), "%s: Unknown PHY type\n", __func__);
1470 }
1471}
1472
1473/**
1474 * ice_ptp_cfg_phy_interrupt - Configure PHY interrupt settings
1475 * @pf: PF private structure
1476 * @ena: bool value to enable or disable interrupt
1477 * @threshold: Minimum number of packets at which intr is triggered
1478 *
1479 * Utility function to enable or disable Tx timestamp interrupt and threshold
1480 */
1481static int ice_ptp_cfg_phy_interrupt(struct ice_pf *pf, bool ena, u32 threshold)
1482{
1483 struct ice_hw *hw = &pf->hw;
1484 int err = 0;
1485 int quad;
1486 u32 val;
1487
1488 ice_ptp_reset_ts_memory(hw);
1489
1490 for (quad = 0; quad < ICE_MAX_QUAD; quad++) {
1491 err = ice_read_quad_reg_e82x(hw, quad, Q_REG_TX_MEM_GBL_CFG,
1492 &val);
1493 if (err)
1494 break;
1495
1496 if (ena) {
1497 val |= Q_REG_TX_MEM_GBL_CFG_INTR_ENA_M;
1498 val &= ~Q_REG_TX_MEM_GBL_CFG_INTR_THR_M;
1499 val |= FIELD_PREP(Q_REG_TX_MEM_GBL_CFG_INTR_THR_M,
1500 threshold);
1501 } else {
1502 val &= ~Q_REG_TX_MEM_GBL_CFG_INTR_ENA_M;
1503 }
1504
1505 err = ice_write_quad_reg_e82x(hw, quad, Q_REG_TX_MEM_GBL_CFG,
1506 val);
1507 if (err)
1508 break;
1509 }
1510
1511 if (err)
1512 dev_err(ice_pf_to_dev(pf), "PTP failed in intr ena, err %d\n",
1513 err);
1514 return err;
1515}
1516
1517/**
1518 * ice_ptp_reset_phy_timestamping - Reset PHY timestamping block
1519 * @pf: Board private structure
1520 */
1521static void ice_ptp_reset_phy_timestamping(struct ice_pf *pf)
1522{
1523 ice_ptp_port_phy_restart(&pf->ptp.port);
1524}
1525
1526/**
1527 * ice_ptp_restart_all_phy - Restart all PHYs to recalibrate timestamping
1528 * @pf: Board private structure
1529 */
1530static void ice_ptp_restart_all_phy(struct ice_pf *pf)
1531{
1532 struct list_head *entry;
1533
1534 list_for_each(entry, &pf->ptp.ports_owner.ports) {
1535 struct ice_ptp_port *port = list_entry(entry,
1536 struct ice_ptp_port,
1537 list_member);
1538
1539 if (port->link_up)
1540 ice_ptp_port_phy_restart(port);
1541 }
1542}
1543
1544/**
1545 * ice_ptp_adjfine - Adjust clock increment rate
1546 * @info: the driver's PTP info structure
1547 * @scaled_ppm: Parts per million with 16-bit fractional field
1548 *
1549 * Adjust the frequency of the clock by the indicated scaled ppm from the
1550 * base frequency.
1551 */
1552static int ice_ptp_adjfine(struct ptp_clock_info *info, long scaled_ppm)
1553{
1554 struct ice_pf *pf = ptp_info_to_pf(info);
1555 struct ice_hw *hw = &pf->hw;
1556 u64 incval;
1557 int err;
1558
1559 incval = adjust_by_scaled_ppm(ice_base_incval(pf), scaled_ppm);
1560 err = ice_ptp_write_incval_locked(hw, incval);
1561 if (err) {
1562 dev_err(ice_pf_to_dev(pf), "PTP failed to set incval, err %d\n",
1563 err);
1564 return -EIO;
1565 }
1566
1567 return 0;
1568}
1569
1570/**
1571 * ice_ptp_extts_event - Process PTP external clock event
1572 * @pf: Board private structure
1573 */
1574void ice_ptp_extts_event(struct ice_pf *pf)
1575{
1576 struct ptp_clock_event event;
1577 struct ice_hw *hw = &pf->hw;
1578 u8 chan, tmr_idx;
1579 u32 hi, lo;
1580
1581 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
1582 /* Event time is captured by one of the two matched registers
1583 * GLTSYN_EVNT_L: 32 LSB of sampled time event
1584 * GLTSYN_EVNT_H: 32 MSB of sampled time event
1585 * Event is defined in GLTSYN_EVNT_0 register
1586 */
1587 for (chan = 0; chan < GLTSYN_EVNT_H_IDX_MAX; chan++) {
1588 /* Check if channel is enabled */
1589 if (pf->ptp.ext_ts_irq & (1 << chan)) {
1590 lo = rd32(hw, GLTSYN_EVNT_L(chan, tmr_idx));
1591 hi = rd32(hw, GLTSYN_EVNT_H(chan, tmr_idx));
1592 event.timestamp = (((u64)hi) << 32) | lo;
1593 event.type = PTP_CLOCK_EXTTS;
1594 event.index = chan;
1595
1596 /* Fire event */
1597 ptp_clock_event(pf->ptp.clock, &event);
1598 pf->ptp.ext_ts_irq &= ~(1 << chan);
1599 }
1600 }
1601}
1602
1603/**
1604 * ice_ptp_cfg_extts - Configure EXTTS pin and channel
1605 * @pf: Board private structure
1606 * @ena: true to enable; false to disable
1607 * @chan: GPIO channel (0-3)
1608 * @gpio_pin: GPIO pin
1609 * @extts_flags: request flags from the ptp_extts_request.flags
1610 */
1611static int
1612ice_ptp_cfg_extts(struct ice_pf *pf, bool ena, unsigned int chan, u32 gpio_pin,
1613 unsigned int extts_flags)
1614{
1615 u32 func, aux_reg, gpio_reg, irq_reg;
1616 struct ice_hw *hw = &pf->hw;
1617 u8 tmr_idx;
1618
1619 if (chan > (unsigned int)pf->ptp.info.n_ext_ts)
1620 return -EINVAL;
1621
1622 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
1623
1624 irq_reg = rd32(hw, PFINT_OICR_ENA);
1625
1626 if (ena) {
1627 /* Enable the interrupt */
1628 irq_reg |= PFINT_OICR_TSYN_EVNT_M;
1629 aux_reg = GLTSYN_AUX_IN_0_INT_ENA_M;
1630
1631#define GLTSYN_AUX_IN_0_EVNTLVL_RISING_EDGE BIT(0)
1632#define GLTSYN_AUX_IN_0_EVNTLVL_FALLING_EDGE BIT(1)
1633
1634 /* set event level to requested edge */
1635 if (extts_flags & PTP_FALLING_EDGE)
1636 aux_reg |= GLTSYN_AUX_IN_0_EVNTLVL_FALLING_EDGE;
1637 if (extts_flags & PTP_RISING_EDGE)
1638 aux_reg |= GLTSYN_AUX_IN_0_EVNTLVL_RISING_EDGE;
1639
1640 /* Write GPIO CTL reg.
1641 * 0x1 is input sampled by EVENT register(channel)
1642 * + num_in_channels * tmr_idx
1643 */
1644 func = 1 + chan + (tmr_idx * 3);
1645 gpio_reg = FIELD_PREP(GLGEN_GPIO_CTL_PIN_FUNC_M, func);
1646 pf->ptp.ext_ts_chan |= (1 << chan);
1647 } else {
1648 /* clear the values we set to reset defaults */
1649 aux_reg = 0;
1650 gpio_reg = 0;
1651 pf->ptp.ext_ts_chan &= ~(1 << chan);
1652 if (!pf->ptp.ext_ts_chan)
1653 irq_reg &= ~PFINT_OICR_TSYN_EVNT_M;
1654 }
1655
1656 wr32(hw, PFINT_OICR_ENA, irq_reg);
1657 wr32(hw, GLTSYN_AUX_IN(chan, tmr_idx), aux_reg);
1658 wr32(hw, GLGEN_GPIO_CTL(gpio_pin), gpio_reg);
1659
1660 return 0;
1661}
1662
1663/**
1664 * ice_ptp_cfg_clkout - Configure clock to generate periodic wave
1665 * @pf: Board private structure
1666 * @chan: GPIO channel (0-3)
1667 * @config: desired periodic clk configuration. NULL will disable channel
1668 * @store: If set to true the values will be stored
1669 *
1670 * Configure the internal clock generator modules to generate the clock wave of
1671 * specified period.
1672 */
1673static int ice_ptp_cfg_clkout(struct ice_pf *pf, unsigned int chan,
1674 struct ice_perout_channel *config, bool store)
1675{
1676 u64 current_time, period, start_time, phase;
1677 struct ice_hw *hw = &pf->hw;
1678 u32 func, val, gpio_pin;
1679 u8 tmr_idx;
1680
1681 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
1682
1683 /* 0. Reset mode & out_en in AUX_OUT */
1684 wr32(hw, GLTSYN_AUX_OUT(chan, tmr_idx), 0);
1685
1686 /* If we're disabling the output, clear out CLKO and TGT and keep
1687 * output level low
1688 */
1689 if (!config || !config->ena) {
1690 wr32(hw, GLTSYN_CLKO(chan, tmr_idx), 0);
1691 wr32(hw, GLTSYN_TGT_L(chan, tmr_idx), 0);
1692 wr32(hw, GLTSYN_TGT_H(chan, tmr_idx), 0);
1693
1694 val = GLGEN_GPIO_CTL_PIN_DIR_M;
1695 gpio_pin = pf->ptp.perout_channels[chan].gpio_pin;
1696 wr32(hw, GLGEN_GPIO_CTL(gpio_pin), val);
1697
1698 /* Store the value if requested */
1699 if (store)
1700 memset(&pf->ptp.perout_channels[chan], 0,
1701 sizeof(struct ice_perout_channel));
1702
1703 return 0;
1704 }
1705 period = config->period;
1706 start_time = config->start_time;
1707 div64_u64_rem(start_time, period, &phase);
1708 gpio_pin = config->gpio_pin;
1709
1710 /* 1. Write clkout with half of required period value */
1711 if (period & 0x1) {
1712 dev_err(ice_pf_to_dev(pf), "CLK Period must be an even value\n");
1713 goto err;
1714 }
1715
1716 period >>= 1;
1717
1718 /* For proper operation, the GLTSYN_CLKO must be larger than clock tick
1719 */
1720#define MIN_PULSE 3
1721 if (period <= MIN_PULSE || period > U32_MAX) {
1722 dev_err(ice_pf_to_dev(pf), "CLK Period must be > %d && < 2^33",
1723 MIN_PULSE * 2);
1724 goto err;
1725 }
1726
1727 wr32(hw, GLTSYN_CLKO(chan, tmr_idx), lower_32_bits(period));
1728
1729 /* Allow time for programming before start_time is hit */
1730 current_time = ice_ptp_read_src_clk_reg(pf, NULL);
1731
1732 /* if start time is in the past start the timer at the nearest second
1733 * maintaining phase
1734 */
1735 if (start_time < current_time)
1736 start_time = div64_u64(current_time + NSEC_PER_SEC - 1,
1737 NSEC_PER_SEC) * NSEC_PER_SEC + phase;
1738
1739 if (ice_is_e810(hw))
1740 start_time -= E810_OUT_PROP_DELAY_NS;
1741 else
1742 start_time -= ice_e82x_pps_delay(ice_e82x_time_ref(hw));
1743
1744 /* 2. Write TARGET time */
1745 wr32(hw, GLTSYN_TGT_L(chan, tmr_idx), lower_32_bits(start_time));
1746 wr32(hw, GLTSYN_TGT_H(chan, tmr_idx), upper_32_bits(start_time));
1747
1748 /* 3. Write AUX_OUT register */
1749 val = GLTSYN_AUX_OUT_0_OUT_ENA_M | GLTSYN_AUX_OUT_0_OUTMOD_M;
1750 wr32(hw, GLTSYN_AUX_OUT(chan, tmr_idx), val);
1751
1752 /* 4. write GPIO CTL reg */
1753 func = 8 + chan + (tmr_idx * 4);
1754 val = GLGEN_GPIO_CTL_PIN_DIR_M |
1755 FIELD_PREP(GLGEN_GPIO_CTL_PIN_FUNC_M, func);
1756 wr32(hw, GLGEN_GPIO_CTL(gpio_pin), val);
1757
1758 /* Store the value if requested */
1759 if (store) {
1760 memcpy(&pf->ptp.perout_channels[chan], config,
1761 sizeof(struct ice_perout_channel));
1762 pf->ptp.perout_channels[chan].start_time = phase;
1763 }
1764
1765 return 0;
1766err:
1767 dev_err(ice_pf_to_dev(pf), "PTP failed to cfg per_clk\n");
1768 return -EFAULT;
1769}
1770
1771/**
1772 * ice_ptp_disable_all_clkout - Disable all currently configured outputs
1773 * @pf: pointer to the PF structure
1774 *
1775 * Disable all currently configured clock outputs. This is necessary before
1776 * certain changes to the PTP hardware clock. Use ice_ptp_enable_all_clkout to
1777 * re-enable the clocks again.
1778 */
1779static void ice_ptp_disable_all_clkout(struct ice_pf *pf)
1780{
1781 uint i;
1782
1783 for (i = 0; i < pf->ptp.info.n_per_out; i++)
1784 if (pf->ptp.perout_channels[i].ena)
1785 ice_ptp_cfg_clkout(pf, i, NULL, false);
1786}
1787
1788/**
1789 * ice_ptp_enable_all_clkout - Enable all configured periodic clock outputs
1790 * @pf: pointer to the PF structure
1791 *
1792 * Enable all currently configured clock outputs. Use this after
1793 * ice_ptp_disable_all_clkout to reconfigure the output signals according to
1794 * their configuration.
1795 */
1796static void ice_ptp_enable_all_clkout(struct ice_pf *pf)
1797{
1798 uint i;
1799
1800 for (i = 0; i < pf->ptp.info.n_per_out; i++)
1801 if (pf->ptp.perout_channels[i].ena)
1802 ice_ptp_cfg_clkout(pf, i, &pf->ptp.perout_channels[i],
1803 false);
1804}
1805
1806/**
1807 * ice_ptp_gpio_enable_e810 - Enable/disable ancillary features of PHC
1808 * @info: the driver's PTP info structure
1809 * @rq: The requested feature to change
1810 * @on: Enable/disable flag
1811 */
1812static int
1813ice_ptp_gpio_enable_e810(struct ptp_clock_info *info,
1814 struct ptp_clock_request *rq, int on)
1815{
1816 struct ice_pf *pf = ptp_info_to_pf(info);
1817 struct ice_perout_channel clk_cfg = {0};
1818 bool sma_pres = false;
1819 unsigned int chan;
1820 u32 gpio_pin;
1821 int err;
1822
1823 if (ice_is_feature_supported(pf, ICE_F_SMA_CTRL))
1824 sma_pres = true;
1825
1826 switch (rq->type) {
1827 case PTP_CLK_REQ_PEROUT:
1828 chan = rq->perout.index;
1829 if (sma_pres) {
1830 if (chan == ice_pin_desc_e810t[SMA1].chan)
1831 clk_cfg.gpio_pin = GPIO_20;
1832 else if (chan == ice_pin_desc_e810t[SMA2].chan)
1833 clk_cfg.gpio_pin = GPIO_22;
1834 else
1835 return -1;
1836 } else if (ice_is_e810t(&pf->hw)) {
1837 if (chan == 0)
1838 clk_cfg.gpio_pin = GPIO_20;
1839 else
1840 clk_cfg.gpio_pin = GPIO_22;
1841 } else if (chan == PPS_CLK_GEN_CHAN) {
1842 clk_cfg.gpio_pin = PPS_PIN_INDEX;
1843 } else {
1844 clk_cfg.gpio_pin = chan;
1845 }
1846
1847 clk_cfg.period = ((rq->perout.period.sec * NSEC_PER_SEC) +
1848 rq->perout.period.nsec);
1849 clk_cfg.start_time = ((rq->perout.start.sec * NSEC_PER_SEC) +
1850 rq->perout.start.nsec);
1851 clk_cfg.ena = !!on;
1852
1853 err = ice_ptp_cfg_clkout(pf, chan, &clk_cfg, true);
1854 break;
1855 case PTP_CLK_REQ_EXTTS:
1856 chan = rq->extts.index;
1857 if (sma_pres) {
1858 if (chan < ice_pin_desc_e810t[SMA2].chan)
1859 gpio_pin = GPIO_21;
1860 else
1861 gpio_pin = GPIO_23;
1862 } else if (ice_is_e810t(&pf->hw)) {
1863 if (chan == 0)
1864 gpio_pin = GPIO_21;
1865 else
1866 gpio_pin = GPIO_23;
1867 } else {
1868 gpio_pin = chan;
1869 }
1870
1871 err = ice_ptp_cfg_extts(pf, !!on, chan, gpio_pin,
1872 rq->extts.flags);
1873 break;
1874 default:
1875 return -EOPNOTSUPP;
1876 }
1877
1878 return err;
1879}
1880
1881/**
1882 * ice_ptp_gpio_enable_e823 - Enable/disable ancillary features of PHC
1883 * @info: the driver's PTP info structure
1884 * @rq: The requested feature to change
1885 * @on: Enable/disable flag
1886 */
1887static int ice_ptp_gpio_enable_e823(struct ptp_clock_info *info,
1888 struct ptp_clock_request *rq, int on)
1889{
1890 struct ice_pf *pf = ptp_info_to_pf(info);
1891 struct ice_perout_channel clk_cfg = {0};
1892 int err;
1893
1894 switch (rq->type) {
1895 case PTP_CLK_REQ_PPS:
1896 clk_cfg.gpio_pin = PPS_PIN_INDEX;
1897 clk_cfg.period = NSEC_PER_SEC;
1898 clk_cfg.ena = !!on;
1899
1900 err = ice_ptp_cfg_clkout(pf, PPS_CLK_GEN_CHAN, &clk_cfg, true);
1901 break;
1902 case PTP_CLK_REQ_EXTTS:
1903 err = ice_ptp_cfg_extts(pf, !!on, rq->extts.index,
1904 TIME_SYNC_PIN_INDEX, rq->extts.flags);
1905 break;
1906 default:
1907 return -EOPNOTSUPP;
1908 }
1909
1910 return err;
1911}
1912
1913/**
1914 * ice_ptp_gettimex64 - Get the time of the clock
1915 * @info: the driver's PTP info structure
1916 * @ts: timespec64 structure to hold the current time value
1917 * @sts: Optional parameter for holding a pair of system timestamps from
1918 * the system clock. Will be ignored if NULL is given.
1919 *
1920 * Read the device clock and return the correct value on ns, after converting it
1921 * into a timespec struct.
1922 */
1923static int
1924ice_ptp_gettimex64(struct ptp_clock_info *info, struct timespec64 *ts,
1925 struct ptp_system_timestamp *sts)
1926{
1927 struct ice_pf *pf = ptp_info_to_pf(info);
1928 struct ice_hw *hw = &pf->hw;
1929
1930 if (!ice_ptp_lock(hw)) {
1931 dev_err(ice_pf_to_dev(pf), "PTP failed to get time\n");
1932 return -EBUSY;
1933 }
1934
1935 ice_ptp_read_time(pf, ts, sts);
1936 ice_ptp_unlock(hw);
1937
1938 return 0;
1939}
1940
1941/**
1942 * ice_ptp_settime64 - Set the time of the clock
1943 * @info: the driver's PTP info structure
1944 * @ts: timespec64 structure that holds the new time value
1945 *
1946 * Set the device clock to the user input value. The conversion from timespec
1947 * to ns happens in the write function.
1948 */
1949static int
1950ice_ptp_settime64(struct ptp_clock_info *info, const struct timespec64 *ts)
1951{
1952 struct ice_pf *pf = ptp_info_to_pf(info);
1953 struct timespec64 ts64 = *ts;
1954 struct ice_hw *hw = &pf->hw;
1955 int err;
1956
1957 /* For Vernier mode, we need to recalibrate after new settime
1958 * Start with disabling timestamp block
1959 */
1960 if (pf->ptp.port.link_up)
1961 ice_ptp_port_phy_stop(&pf->ptp.port);
1962
1963 if (!ice_ptp_lock(hw)) {
1964 err = -EBUSY;
1965 goto exit;
1966 }
1967
1968 /* Disable periodic outputs */
1969 ice_ptp_disable_all_clkout(pf);
1970
1971 err = ice_ptp_write_init(pf, &ts64);
1972 ice_ptp_unlock(hw);
1973
1974 if (!err)
1975 ice_ptp_reset_cached_phctime(pf);
1976
1977 /* Reenable periodic outputs */
1978 ice_ptp_enable_all_clkout(pf);
1979
1980 /* Recalibrate and re-enable timestamp blocks for E822/E823 */
1981 if (hw->phy_model == ICE_PHY_E82X)
1982 ice_ptp_restart_all_phy(pf);
1983exit:
1984 if (err) {
1985 dev_err(ice_pf_to_dev(pf), "PTP failed to set time %d\n", err);
1986 return err;
1987 }
1988
1989 return 0;
1990}
1991
1992/**
1993 * ice_ptp_adjtime_nonatomic - Do a non-atomic clock adjustment
1994 * @info: the driver's PTP info structure
1995 * @delta: Offset in nanoseconds to adjust the time by
1996 */
1997static int ice_ptp_adjtime_nonatomic(struct ptp_clock_info *info, s64 delta)
1998{
1999 struct timespec64 now, then;
2000 int ret;
2001
2002 then = ns_to_timespec64(delta);
2003 ret = ice_ptp_gettimex64(info, &now, NULL);
2004 if (ret)
2005 return ret;
2006 now = timespec64_add(now, then);
2007
2008 return ice_ptp_settime64(info, (const struct timespec64 *)&now);
2009}
2010
2011/**
2012 * ice_ptp_adjtime - Adjust the time of the clock by the indicated delta
2013 * @info: the driver's PTP info structure
2014 * @delta: Offset in nanoseconds to adjust the time by
2015 */
2016static int ice_ptp_adjtime(struct ptp_clock_info *info, s64 delta)
2017{
2018 struct ice_pf *pf = ptp_info_to_pf(info);
2019 struct ice_hw *hw = &pf->hw;
2020 struct device *dev;
2021 int err;
2022
2023 dev = ice_pf_to_dev(pf);
2024
2025 /* Hardware only supports atomic adjustments using signed 32-bit
2026 * integers. For any adjustment outside this range, perform
2027 * a non-atomic get->adjust->set flow.
2028 */
2029 if (delta > S32_MAX || delta < S32_MIN) {
2030 dev_dbg(dev, "delta = %lld, adjtime non-atomic\n", delta);
2031 return ice_ptp_adjtime_nonatomic(info, delta);
2032 }
2033
2034 if (!ice_ptp_lock(hw)) {
2035 dev_err(dev, "PTP failed to acquire semaphore in adjtime\n");
2036 return -EBUSY;
2037 }
2038
2039 /* Disable periodic outputs */
2040 ice_ptp_disable_all_clkout(pf);
2041
2042 err = ice_ptp_write_adj(pf, delta);
2043
2044 /* Reenable periodic outputs */
2045 ice_ptp_enable_all_clkout(pf);
2046
2047 ice_ptp_unlock(hw);
2048
2049 if (err) {
2050 dev_err(dev, "PTP failed to adjust time, err %d\n", err);
2051 return err;
2052 }
2053
2054 ice_ptp_reset_cached_phctime(pf);
2055
2056 return 0;
2057}
2058
2059#ifdef CONFIG_ICE_HWTS
2060/**
2061 * ice_ptp_get_syncdevicetime - Get the cross time stamp info
2062 * @device: Current device time
2063 * @system: System counter value read synchronously with device time
2064 * @ctx: Context provided by timekeeping code
2065 *
2066 * Read device and system (ART) clock simultaneously and return the corrected
2067 * clock values in ns.
2068 */
2069static int
2070ice_ptp_get_syncdevicetime(ktime_t *device,
2071 struct system_counterval_t *system,
2072 void *ctx)
2073{
2074 struct ice_pf *pf = (struct ice_pf *)ctx;
2075 struct ice_hw *hw = &pf->hw;
2076 u32 hh_lock, hh_art_ctl;
2077 int i;
2078
2079#define MAX_HH_HW_LOCK_TRIES 5
2080#define MAX_HH_CTL_LOCK_TRIES 100
2081
2082 for (i = 0; i < MAX_HH_HW_LOCK_TRIES; i++) {
2083 /* Get the HW lock */
2084 hh_lock = rd32(hw, PFHH_SEM + (PFTSYN_SEM_BYTES * hw->pf_id));
2085 if (hh_lock & PFHH_SEM_BUSY_M) {
2086 usleep_range(10000, 15000);
2087 continue;
2088 }
2089 break;
2090 }
2091 if (hh_lock & PFHH_SEM_BUSY_M) {
2092 dev_err(ice_pf_to_dev(pf), "PTP failed to get hh lock\n");
2093 return -EBUSY;
2094 }
2095
2096 /* Program cmd to master timer */
2097 ice_ptp_src_cmd(hw, ICE_PTP_READ_TIME);
2098
2099 /* Start the ART and device clock sync sequence */
2100 hh_art_ctl = rd32(hw, GLHH_ART_CTL);
2101 hh_art_ctl = hh_art_ctl | GLHH_ART_CTL_ACTIVE_M;
2102 wr32(hw, GLHH_ART_CTL, hh_art_ctl);
2103
2104 for (i = 0; i < MAX_HH_CTL_LOCK_TRIES; i++) {
2105 /* Wait for sync to complete */
2106 hh_art_ctl = rd32(hw, GLHH_ART_CTL);
2107 if (hh_art_ctl & GLHH_ART_CTL_ACTIVE_M) {
2108 udelay(1);
2109 continue;
2110 } else {
2111 u32 hh_ts_lo, hh_ts_hi, tmr_idx;
2112 u64 hh_ts;
2113
2114 tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc;
2115 /* Read ART time */
2116 hh_ts_lo = rd32(hw, GLHH_ART_TIME_L);
2117 hh_ts_hi = rd32(hw, GLHH_ART_TIME_H);
2118 hh_ts = ((u64)hh_ts_hi << 32) | hh_ts_lo;
2119 *system = convert_art_ns_to_tsc(hh_ts);
2120 /* Read Device source clock time */
2121 hh_ts_lo = rd32(hw, GLTSYN_HHTIME_L(tmr_idx));
2122 hh_ts_hi = rd32(hw, GLTSYN_HHTIME_H(tmr_idx));
2123 hh_ts = ((u64)hh_ts_hi << 32) | hh_ts_lo;
2124 *device = ns_to_ktime(hh_ts);
2125 break;
2126 }
2127 }
2128
2129 /* Clear the master timer */
2130 ice_ptp_src_cmd(hw, ICE_PTP_NOP);
2131
2132 /* Release HW lock */
2133 hh_lock = rd32(hw, PFHH_SEM + (PFTSYN_SEM_BYTES * hw->pf_id));
2134 hh_lock = hh_lock & ~PFHH_SEM_BUSY_M;
2135 wr32(hw, PFHH_SEM + (PFTSYN_SEM_BYTES * hw->pf_id), hh_lock);
2136
2137 if (i == MAX_HH_CTL_LOCK_TRIES)
2138 return -ETIMEDOUT;
2139
2140 return 0;
2141}
2142
2143/**
2144 * ice_ptp_getcrosststamp_e82x - Capture a device cross timestamp
2145 * @info: the driver's PTP info structure
2146 * @cts: The memory to fill the cross timestamp info
2147 *
2148 * Capture a cross timestamp between the ART and the device PTP hardware
2149 * clock. Fill the cross timestamp information and report it back to the
2150 * caller.
2151 *
2152 * This is only valid for E822 and E823 devices which have support for
2153 * generating the cross timestamp via PCIe PTM.
2154 *
2155 * In order to correctly correlate the ART timestamp back to the TSC time, the
2156 * CPU must have X86_FEATURE_TSC_KNOWN_FREQ.
2157 */
2158static int
2159ice_ptp_getcrosststamp_e82x(struct ptp_clock_info *info,
2160 struct system_device_crosststamp *cts)
2161{
2162 struct ice_pf *pf = ptp_info_to_pf(info);
2163
2164 return get_device_system_crosststamp(ice_ptp_get_syncdevicetime,
2165 pf, NULL, cts);
2166}
2167#endif /* CONFIG_ICE_HWTS */
2168
2169/**
2170 * ice_ptp_get_ts_config - ioctl interface to read the timestamping config
2171 * @pf: Board private structure
2172 * @ifr: ioctl data
2173 *
2174 * Copy the timestamping config to user buffer
2175 */
2176int ice_ptp_get_ts_config(struct ice_pf *pf, struct ifreq *ifr)
2177{
2178 struct hwtstamp_config *config;
2179
2180 if (pf->ptp.state != ICE_PTP_READY)
2181 return -EIO;
2182
2183 config = &pf->ptp.tstamp_config;
2184
2185 return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
2186 -EFAULT : 0;
2187}
2188
2189/**
2190 * ice_ptp_set_timestamp_mode - Setup driver for requested timestamp mode
2191 * @pf: Board private structure
2192 * @config: hwtstamp settings requested or saved
2193 */
2194static int
2195ice_ptp_set_timestamp_mode(struct ice_pf *pf, struct hwtstamp_config *config)
2196{
2197 switch (config->tx_type) {
2198 case HWTSTAMP_TX_OFF:
2199 pf->ptp.tstamp_config.tx_type = HWTSTAMP_TX_OFF;
2200 break;
2201 case HWTSTAMP_TX_ON:
2202 pf->ptp.tstamp_config.tx_type = HWTSTAMP_TX_ON;
2203 break;
2204 default:
2205 return -ERANGE;
2206 }
2207
2208 switch (config->rx_filter) {
2209 case HWTSTAMP_FILTER_NONE:
2210 pf->ptp.tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
2211 break;
2212 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2213 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2214 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2215 case HWTSTAMP_FILTER_PTP_V2_EVENT:
2216 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2217 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2218 case HWTSTAMP_FILTER_PTP_V2_SYNC:
2219 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2220 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2221 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2222 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2223 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2224 case HWTSTAMP_FILTER_NTP_ALL:
2225 case HWTSTAMP_FILTER_ALL:
2226 pf->ptp.tstamp_config.rx_filter = HWTSTAMP_FILTER_ALL;
2227 break;
2228 default:
2229 return -ERANGE;
2230 }
2231
2232 /* Immediately update the device timestamping mode */
2233 ice_ptp_restore_timestamp_mode(pf);
2234
2235 return 0;
2236}
2237
2238/**
2239 * ice_ptp_set_ts_config - ioctl interface to control the timestamping
2240 * @pf: Board private structure
2241 * @ifr: ioctl data
2242 *
2243 * Get the user config and store it
2244 */
2245int ice_ptp_set_ts_config(struct ice_pf *pf, struct ifreq *ifr)
2246{
2247 struct hwtstamp_config config;
2248 int err;
2249
2250 if (pf->ptp.state != ICE_PTP_READY)
2251 return -EAGAIN;
2252
2253 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2254 return -EFAULT;
2255
2256 err = ice_ptp_set_timestamp_mode(pf, &config);
2257 if (err)
2258 return err;
2259
2260 /* Return the actual configuration set */
2261 config = pf->ptp.tstamp_config;
2262
2263 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2264 -EFAULT : 0;
2265}
2266
2267/**
2268 * ice_ptp_get_rx_hwts - Get packet Rx timestamp in ns
2269 * @rx_desc: Receive descriptor
2270 * @pkt_ctx: Packet context to get the cached time
2271 *
2272 * The driver receives a notification in the receive descriptor with timestamp.
2273 */
2274u64 ice_ptp_get_rx_hwts(const union ice_32b_rx_flex_desc *rx_desc,
2275 const struct ice_pkt_ctx *pkt_ctx)
2276{
2277 u64 ts_ns, cached_time;
2278 u32 ts_high;
2279
2280 if (!(rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID))
2281 return 0;
2282
2283 cached_time = READ_ONCE(pkt_ctx->cached_phctime);
2284
2285 /* Do not report a timestamp if we don't have a cached PHC time */
2286 if (!cached_time)
2287 return 0;
2288
2289 /* Use ice_ptp_extend_32b_ts directly, using the ring-specific cached
2290 * PHC value, rather than accessing the PF. This also allows us to
2291 * simply pass the upper 32bits of nanoseconds directly. Calling
2292 * ice_ptp_extend_40b_ts is unnecessary as it would just discard these
2293 * bits itself.
2294 */
2295 ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high);
2296 ts_ns = ice_ptp_extend_32b_ts(cached_time, ts_high);
2297
2298 return ts_ns;
2299}
2300
2301/**
2302 * ice_ptp_disable_sma_pins_e810t - Disable E810-T SMA pins
2303 * @pf: pointer to the PF structure
2304 * @info: PTP clock info structure
2305 *
2306 * Disable the OS access to the SMA pins. Called to clear out the OS
2307 * indications of pin support when we fail to setup the E810-T SMA control
2308 * register.
2309 */
2310static void
2311ice_ptp_disable_sma_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info)
2312{
2313 struct device *dev = ice_pf_to_dev(pf);
2314
2315 dev_warn(dev, "Failed to configure E810-T SMA pin control\n");
2316
2317 info->enable = NULL;
2318 info->verify = NULL;
2319 info->n_pins = 0;
2320 info->n_ext_ts = 0;
2321 info->n_per_out = 0;
2322}
2323
2324/**
2325 * ice_ptp_setup_sma_pins_e810t - Setup the SMA pins
2326 * @pf: pointer to the PF structure
2327 * @info: PTP clock info structure
2328 *
2329 * Finish setting up the SMA pins by allocating pin_config, and setting it up
2330 * according to the current status of the SMA. On failure, disable all of the
2331 * extended SMA pin support.
2332 */
2333static void
2334ice_ptp_setup_sma_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info)
2335{
2336 struct device *dev = ice_pf_to_dev(pf);
2337 int err;
2338
2339 /* Allocate memory for kernel pins interface */
2340 info->pin_config = devm_kcalloc(dev, info->n_pins,
2341 sizeof(*info->pin_config), GFP_KERNEL);
2342 if (!info->pin_config) {
2343 ice_ptp_disable_sma_pins_e810t(pf, info);
2344 return;
2345 }
2346
2347 /* Read current SMA status */
2348 err = ice_get_sma_config_e810t(&pf->hw, info->pin_config);
2349 if (err)
2350 ice_ptp_disable_sma_pins_e810t(pf, info);
2351}
2352
2353/**
2354 * ice_ptp_setup_pins_e810 - Setup PTP pins in sysfs
2355 * @pf: pointer to the PF instance
2356 * @info: PTP clock capabilities
2357 */
2358static void
2359ice_ptp_setup_pins_e810(struct ice_pf *pf, struct ptp_clock_info *info)
2360{
2361 if (ice_is_feature_supported(pf, ICE_F_SMA_CTRL)) {
2362 info->n_ext_ts = N_EXT_TS_E810;
2363 info->n_per_out = N_PER_OUT_E810T;
2364 info->n_pins = NUM_PTP_PINS_E810T;
2365 info->verify = ice_verify_pin_e810t;
2366
2367 /* Complete setup of the SMA pins */
2368 ice_ptp_setup_sma_pins_e810t(pf, info);
2369 } else if (ice_is_e810t(&pf->hw)) {
2370 info->n_ext_ts = N_EXT_TS_NO_SMA_E810T;
2371 info->n_per_out = N_PER_OUT_NO_SMA_E810T;
2372 } else {
2373 info->n_per_out = N_PER_OUT_E810;
2374 info->n_ext_ts = N_EXT_TS_E810;
2375 }
2376}
2377
2378/**
2379 * ice_ptp_setup_pins_e823 - Setup PTP pins in sysfs
2380 * @pf: pointer to the PF instance
2381 * @info: PTP clock capabilities
2382 */
2383static void
2384ice_ptp_setup_pins_e823(struct ice_pf *pf, struct ptp_clock_info *info)
2385{
2386 info->pps = 1;
2387 info->n_per_out = 0;
2388 info->n_ext_ts = 1;
2389}
2390
2391/**
2392 * ice_ptp_set_funcs_e82x - Set specialized functions for E82x support
2393 * @pf: Board private structure
2394 * @info: PTP info to fill
2395 *
2396 * Assign functions to the PTP capabiltiies structure for E82x devices.
2397 * Functions which operate across all device families should be set directly
2398 * in ice_ptp_set_caps. Only add functions here which are distinct for E82x
2399 * devices.
2400 */
2401static void
2402ice_ptp_set_funcs_e82x(struct ice_pf *pf, struct ptp_clock_info *info)
2403{
2404#ifdef CONFIG_ICE_HWTS
2405 if (boot_cpu_has(X86_FEATURE_ART) &&
2406 boot_cpu_has(X86_FEATURE_TSC_KNOWN_FREQ))
2407 info->getcrosststamp = ice_ptp_getcrosststamp_e82x;
2408#endif /* CONFIG_ICE_HWTS */
2409}
2410
2411/**
2412 * ice_ptp_set_funcs_e810 - Set specialized functions for E810 support
2413 * @pf: Board private structure
2414 * @info: PTP info to fill
2415 *
2416 * Assign functions to the PTP capabiltiies structure for E810 devices.
2417 * Functions which operate across all device families should be set directly
2418 * in ice_ptp_set_caps. Only add functions here which are distinct for e810
2419 * devices.
2420 */
2421static void
2422ice_ptp_set_funcs_e810(struct ice_pf *pf, struct ptp_clock_info *info)
2423{
2424 info->enable = ice_ptp_gpio_enable_e810;
2425 ice_ptp_setup_pins_e810(pf, info);
2426}
2427
2428/**
2429 * ice_ptp_set_funcs_e823 - Set specialized functions for E823 support
2430 * @pf: Board private structure
2431 * @info: PTP info to fill
2432 *
2433 * Assign functions to the PTP capabiltiies structure for E823 devices.
2434 * Functions which operate across all device families should be set directly
2435 * in ice_ptp_set_caps. Only add functions here which are distinct for e823
2436 * devices.
2437 */
2438static void
2439ice_ptp_set_funcs_e823(struct ice_pf *pf, struct ptp_clock_info *info)
2440{
2441 ice_ptp_set_funcs_e82x(pf, info);
2442
2443 info->enable = ice_ptp_gpio_enable_e823;
2444 ice_ptp_setup_pins_e823(pf, info);
2445}
2446
2447/**
2448 * ice_ptp_set_caps - Set PTP capabilities
2449 * @pf: Board private structure
2450 */
2451static void ice_ptp_set_caps(struct ice_pf *pf)
2452{
2453 struct ptp_clock_info *info = &pf->ptp.info;
2454 struct device *dev = ice_pf_to_dev(pf);
2455
2456 snprintf(info->name, sizeof(info->name) - 1, "%s-%s-clk",
2457 dev_driver_string(dev), dev_name(dev));
2458 info->owner = THIS_MODULE;
2459 info->max_adj = 100000000;
2460 info->adjtime = ice_ptp_adjtime;
2461 info->adjfine = ice_ptp_adjfine;
2462 info->gettimex64 = ice_ptp_gettimex64;
2463 info->settime64 = ice_ptp_settime64;
2464
2465 if (ice_is_e810(&pf->hw))
2466 ice_ptp_set_funcs_e810(pf, info);
2467 else if (ice_is_e823(&pf->hw))
2468 ice_ptp_set_funcs_e823(pf, info);
2469 else
2470 ice_ptp_set_funcs_e82x(pf, info);
2471}
2472
2473/**
2474 * ice_ptp_create_clock - Create PTP clock device for userspace
2475 * @pf: Board private structure
2476 *
2477 * This function creates a new PTP clock device. It only creates one if we
2478 * don't already have one. Will return error if it can't create one, but success
2479 * if we already have a device. Should be used by ice_ptp_init to create clock
2480 * initially, and prevent global resets from creating new clock devices.
2481 */
2482static long ice_ptp_create_clock(struct ice_pf *pf)
2483{
2484 struct ptp_clock_info *info;
2485 struct device *dev;
2486
2487 /* No need to create a clock device if we already have one */
2488 if (pf->ptp.clock)
2489 return 0;
2490
2491 ice_ptp_set_caps(pf);
2492
2493 info = &pf->ptp.info;
2494 dev = ice_pf_to_dev(pf);
2495
2496 /* Attempt to register the clock before enabling the hardware. */
2497 pf->ptp.clock = ptp_clock_register(info, dev);
2498 if (IS_ERR(pf->ptp.clock)) {
2499 dev_err(ice_pf_to_dev(pf), "Failed to register PTP clock device");
2500 return PTR_ERR(pf->ptp.clock);
2501 }
2502
2503 return 0;
2504}
2505
2506/**
2507 * ice_ptp_request_ts - Request an available Tx timestamp index
2508 * @tx: the PTP Tx timestamp tracker to request from
2509 * @skb: the SKB to associate with this timestamp request
2510 */
2511s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb)
2512{
2513 unsigned long flags;
2514 u8 idx;
2515
2516 spin_lock_irqsave(&tx->lock, flags);
2517
2518 /* Check that this tracker is accepting new timestamp requests */
2519 if (!ice_ptp_is_tx_tracker_up(tx)) {
2520 spin_unlock_irqrestore(&tx->lock, flags);
2521 return -1;
2522 }
2523
2524 /* Find and set the first available index */
2525 idx = find_next_zero_bit(tx->in_use, tx->len,
2526 tx->last_ll_ts_idx_read + 1);
2527 if (idx == tx->len)
2528 idx = find_first_zero_bit(tx->in_use, tx->len);
2529
2530 if (idx < tx->len) {
2531 /* We got a valid index that no other thread could have set. Store
2532 * a reference to the skb and the start time to allow discarding old
2533 * requests.
2534 */
2535 set_bit(idx, tx->in_use);
2536 clear_bit(idx, tx->stale);
2537 tx->tstamps[idx].start = jiffies;
2538 tx->tstamps[idx].skb = skb_get(skb);
2539 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2540 ice_trace(tx_tstamp_request, skb, idx);
2541 }
2542
2543 spin_unlock_irqrestore(&tx->lock, flags);
2544
2545 /* return the appropriate PHY timestamp register index, -1 if no
2546 * indexes were available.
2547 */
2548 if (idx >= tx->len)
2549 return -1;
2550 else
2551 return idx + tx->offset;
2552}
2553
2554/**
2555 * ice_ptp_process_ts - Process the PTP Tx timestamps
2556 * @pf: Board private structure
2557 *
2558 * Returns: ICE_TX_TSTAMP_WORK_PENDING if there are any outstanding Tx
2559 * timestamps that need processing, and ICE_TX_TSTAMP_WORK_DONE otherwise.
2560 */
2561enum ice_tx_tstamp_work ice_ptp_process_ts(struct ice_pf *pf)
2562{
2563 switch (pf->ptp.tx_interrupt_mode) {
2564 case ICE_PTP_TX_INTERRUPT_NONE:
2565 /* This device has the clock owner handle timestamps for it */
2566 return ICE_TX_TSTAMP_WORK_DONE;
2567 case ICE_PTP_TX_INTERRUPT_SELF:
2568 /* This device handles its own timestamps */
2569 return ice_ptp_tx_tstamp(&pf->ptp.port.tx);
2570 case ICE_PTP_TX_INTERRUPT_ALL:
2571 /* This device handles timestamps for all ports */
2572 return ice_ptp_tx_tstamp_owner(pf);
2573 default:
2574 WARN_ONCE(1, "Unexpected Tx timestamp interrupt mode %u\n",
2575 pf->ptp.tx_interrupt_mode);
2576 return ICE_TX_TSTAMP_WORK_DONE;
2577 }
2578}
2579
2580/**
2581 * ice_ptp_maybe_trigger_tx_interrupt - Trigger Tx timstamp interrupt
2582 * @pf: Board private structure
2583 *
2584 * The device PHY issues Tx timestamp interrupts to the driver for processing
2585 * timestamp data from the PHY. It will not interrupt again until all
2586 * current timestamp data is read. In rare circumstances, it is possible that
2587 * the driver fails to read all outstanding data.
2588 *
2589 * To avoid getting permanently stuck, periodically check if the PHY has
2590 * outstanding timestamp data. If so, trigger an interrupt from software to
2591 * process this data.
2592 */
2593static void ice_ptp_maybe_trigger_tx_interrupt(struct ice_pf *pf)
2594{
2595 struct device *dev = ice_pf_to_dev(pf);
2596 struct ice_hw *hw = &pf->hw;
2597 bool trigger_oicr = false;
2598 unsigned int i;
2599
2600 if (ice_is_e810(hw))
2601 return;
2602
2603 if (!ice_pf_src_tmr_owned(pf))
2604 return;
2605
2606 for (i = 0; i < ICE_MAX_QUAD; i++) {
2607 u64 tstamp_ready;
2608 int err;
2609
2610 err = ice_get_phy_tx_tstamp_ready(&pf->hw, i, &tstamp_ready);
2611 if (!err && tstamp_ready) {
2612 trigger_oicr = true;
2613 break;
2614 }
2615 }
2616
2617 if (trigger_oicr) {
2618 /* Trigger a software interrupt, to ensure this data
2619 * gets processed.
2620 */
2621 dev_dbg(dev, "PTP periodic task detected waiting timestamps. Triggering Tx timestamp interrupt now.\n");
2622
2623 wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M);
2624 ice_flush(hw);
2625 }
2626}
2627
2628static void ice_ptp_periodic_work(struct kthread_work *work)
2629{
2630 struct ice_ptp *ptp = container_of(work, struct ice_ptp, work.work);
2631 struct ice_pf *pf = container_of(ptp, struct ice_pf, ptp);
2632 int err;
2633
2634 if (pf->ptp.state != ICE_PTP_READY)
2635 return;
2636
2637 err = ice_ptp_update_cached_phctime(pf);
2638
2639 ice_ptp_maybe_trigger_tx_interrupt(pf);
2640
2641 /* Run twice a second or reschedule if phc update failed */
2642 kthread_queue_delayed_work(ptp->kworker, &ptp->work,
2643 msecs_to_jiffies(err ? 10 : 500));
2644}
2645
2646/**
2647 * ice_ptp_prepare_for_reset - Prepare PTP for reset
2648 * @pf: Board private structure
2649 * @reset_type: the reset type being performed
2650 */
2651void ice_ptp_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
2652{
2653 struct ice_ptp *ptp = &pf->ptp;
2654 u8 src_tmr;
2655
2656 if (ptp->state != ICE_PTP_READY)
2657 return;
2658
2659 ptp->state = ICE_PTP_RESETTING;
2660
2661 /* Disable timestamping for both Tx and Rx */
2662 ice_ptp_disable_timestamp_mode(pf);
2663
2664 kthread_cancel_delayed_work_sync(&ptp->work);
2665
2666 if (reset_type == ICE_RESET_PFR)
2667 return;
2668
2669 ice_ptp_release_tx_tracker(pf, &pf->ptp.port.tx);
2670
2671 /* Disable periodic outputs */
2672 ice_ptp_disable_all_clkout(pf);
2673
2674 src_tmr = ice_get_ptp_src_clock_index(&pf->hw);
2675
2676 /* Disable source clock */
2677 wr32(&pf->hw, GLTSYN_ENA(src_tmr), (u32)~GLTSYN_ENA_TSYN_ENA_M);
2678
2679 /* Acquire PHC and system timer to restore after reset */
2680 ptp->reset_time = ktime_get_real_ns();
2681}
2682
2683/**
2684 * ice_ptp_rebuild_owner - Initialize PTP clock owner after reset
2685 * @pf: Board private structure
2686 *
2687 * Companion function for ice_ptp_rebuild() which handles tasks that only the
2688 * PTP clock owner instance should perform.
2689 */
2690static int ice_ptp_rebuild_owner(struct ice_pf *pf)
2691{
2692 struct ice_ptp *ptp = &pf->ptp;
2693 struct ice_hw *hw = &pf->hw;
2694 struct timespec64 ts;
2695 u64 time_diff;
2696 int err;
2697
2698 err = ice_ptp_init_phc(hw);
2699 if (err)
2700 return err;
2701
2702 /* Acquire the global hardware lock */
2703 if (!ice_ptp_lock(hw)) {
2704 err = -EBUSY;
2705 return err;
2706 }
2707
2708 /* Write the increment time value to PHY and LAN */
2709 err = ice_ptp_write_incval(hw, ice_base_incval(pf));
2710 if (err) {
2711 ice_ptp_unlock(hw);
2712 return err;
2713 }
2714
2715 /* Write the initial Time value to PHY and LAN using the cached PHC
2716 * time before the reset and time difference between stopping and
2717 * starting the clock.
2718 */
2719 if (ptp->cached_phc_time) {
2720 time_diff = ktime_get_real_ns() - ptp->reset_time;
2721 ts = ns_to_timespec64(ptp->cached_phc_time + time_diff);
2722 } else {
2723 ts = ktime_to_timespec64(ktime_get_real());
2724 }
2725 err = ice_ptp_write_init(pf, &ts);
2726 if (err) {
2727 ice_ptp_unlock(hw);
2728 return err;
2729 }
2730
2731 /* Release the global hardware lock */
2732 ice_ptp_unlock(hw);
2733
2734 /* Flush software tracking of any outstanding timestamps since we're
2735 * about to flush the PHY timestamp block.
2736 */
2737 ice_ptp_flush_all_tx_tracker(pf);
2738
2739 if (!ice_is_e810(hw)) {
2740 /* Enable quad interrupts */
2741 err = ice_ptp_cfg_phy_interrupt(pf, true, 1);
2742 if (err)
2743 return err;
2744
2745 ice_ptp_restart_all_phy(pf);
2746 }
2747
2748 return 0;
2749}
2750
2751/**
2752 * ice_ptp_rebuild - Initialize PTP hardware clock support after reset
2753 * @pf: Board private structure
2754 * @reset_type: the reset type being performed
2755 */
2756void ice_ptp_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
2757{
2758 struct ice_ptp *ptp = &pf->ptp;
2759 int err;
2760
2761 if (ptp->state == ICE_PTP_READY) {
2762 ice_ptp_prepare_for_reset(pf, reset_type);
2763 } else if (ptp->state != ICE_PTP_RESETTING) {
2764 err = -EINVAL;
2765 dev_err(ice_pf_to_dev(pf), "PTP was not initialized\n");
2766 goto err;
2767 }
2768
2769 if (ice_pf_src_tmr_owned(pf) && reset_type != ICE_RESET_PFR) {
2770 err = ice_ptp_rebuild_owner(pf);
2771 if (err)
2772 goto err;
2773 }
2774
2775 ptp->state = ICE_PTP_READY;
2776
2777 /* Start periodic work going */
2778 kthread_queue_delayed_work(ptp->kworker, &ptp->work, 0);
2779
2780 dev_info(ice_pf_to_dev(pf), "PTP reset successful\n");
2781 return;
2782
2783err:
2784 ptp->state = ICE_PTP_ERROR;
2785 dev_err(ice_pf_to_dev(pf), "PTP reset failed %d\n", err);
2786}
2787
2788/**
2789 * ice_ptp_aux_dev_to_aux_pf - Get auxiliary PF handle for the auxiliary device
2790 * @aux_dev: auxiliary device to get the auxiliary PF for
2791 */
2792static struct ice_pf *
2793ice_ptp_aux_dev_to_aux_pf(struct auxiliary_device *aux_dev)
2794{
2795 struct ice_ptp_port *aux_port;
2796 struct ice_ptp *aux_ptp;
2797
2798 aux_port = container_of(aux_dev, struct ice_ptp_port, aux_dev);
2799 aux_ptp = container_of(aux_port, struct ice_ptp, port);
2800
2801 return container_of(aux_ptp, struct ice_pf, ptp);
2802}
2803
2804/**
2805 * ice_ptp_aux_dev_to_owner_pf - Get PF handle for the auxiliary device
2806 * @aux_dev: auxiliary device to get the PF for
2807 */
2808static struct ice_pf *
2809ice_ptp_aux_dev_to_owner_pf(struct auxiliary_device *aux_dev)
2810{
2811 struct ice_ptp_port_owner *ports_owner;
2812 struct auxiliary_driver *aux_drv;
2813 struct ice_ptp *owner_ptp;
2814
2815 if (!aux_dev->dev.driver)
2816 return NULL;
2817
2818 aux_drv = to_auxiliary_drv(aux_dev->dev.driver);
2819 ports_owner = container_of(aux_drv, struct ice_ptp_port_owner,
2820 aux_driver);
2821 owner_ptp = container_of(ports_owner, struct ice_ptp, ports_owner);
2822 return container_of(owner_ptp, struct ice_pf, ptp);
2823}
2824
2825/**
2826 * ice_ptp_auxbus_probe - Probe auxiliary devices
2827 * @aux_dev: PF's auxiliary device
2828 * @id: Auxiliary device ID
2829 */
2830static int ice_ptp_auxbus_probe(struct auxiliary_device *aux_dev,
2831 const struct auxiliary_device_id *id)
2832{
2833 struct ice_pf *owner_pf = ice_ptp_aux_dev_to_owner_pf(aux_dev);
2834 struct ice_pf *aux_pf = ice_ptp_aux_dev_to_aux_pf(aux_dev);
2835
2836 if (WARN_ON(!owner_pf))
2837 return -ENODEV;
2838
2839 INIT_LIST_HEAD(&aux_pf->ptp.port.list_member);
2840 mutex_lock(&owner_pf->ptp.ports_owner.lock);
2841 list_add(&aux_pf->ptp.port.list_member,
2842 &owner_pf->ptp.ports_owner.ports);
2843 mutex_unlock(&owner_pf->ptp.ports_owner.lock);
2844
2845 return 0;
2846}
2847
2848/**
2849 * ice_ptp_auxbus_remove - Remove auxiliary devices from the bus
2850 * @aux_dev: PF's auxiliary device
2851 */
2852static void ice_ptp_auxbus_remove(struct auxiliary_device *aux_dev)
2853{
2854 struct ice_pf *owner_pf = ice_ptp_aux_dev_to_owner_pf(aux_dev);
2855 struct ice_pf *aux_pf = ice_ptp_aux_dev_to_aux_pf(aux_dev);
2856
2857 mutex_lock(&owner_pf->ptp.ports_owner.lock);
2858 list_del(&aux_pf->ptp.port.list_member);
2859 mutex_unlock(&owner_pf->ptp.ports_owner.lock);
2860}
2861
2862/**
2863 * ice_ptp_auxbus_shutdown
2864 * @aux_dev: PF's auxiliary device
2865 */
2866static void ice_ptp_auxbus_shutdown(struct auxiliary_device *aux_dev)
2867{
2868 /* Doing nothing here, but handle to auxbus driver must be satisfied */
2869}
2870
2871/**
2872 * ice_ptp_auxbus_suspend
2873 * @aux_dev: PF's auxiliary device
2874 * @state: power management state indicator
2875 */
2876static int
2877ice_ptp_auxbus_suspend(struct auxiliary_device *aux_dev, pm_message_t state)
2878{
2879 /* Doing nothing here, but handle to auxbus driver must be satisfied */
2880 return 0;
2881}
2882
2883/**
2884 * ice_ptp_auxbus_resume
2885 * @aux_dev: PF's auxiliary device
2886 */
2887static int ice_ptp_auxbus_resume(struct auxiliary_device *aux_dev)
2888{
2889 /* Doing nothing here, but handle to auxbus driver must be satisfied */
2890 return 0;
2891}
2892
2893/**
2894 * ice_ptp_auxbus_create_id_table - Create auxiliary device ID table
2895 * @pf: Board private structure
2896 * @name: auxiliary bus driver name
2897 */
2898static struct auxiliary_device_id *
2899ice_ptp_auxbus_create_id_table(struct ice_pf *pf, const char *name)
2900{
2901 struct auxiliary_device_id *ids;
2902
2903 /* Second id left empty to terminate the array */
2904 ids = devm_kcalloc(ice_pf_to_dev(pf), 2,
2905 sizeof(struct auxiliary_device_id), GFP_KERNEL);
2906 if (!ids)
2907 return NULL;
2908
2909 snprintf(ids[0].name, sizeof(ids[0].name), "ice.%s", name);
2910
2911 return ids;
2912}
2913
2914/**
2915 * ice_ptp_register_auxbus_driver - Register PTP auxiliary bus driver
2916 * @pf: Board private structure
2917 */
2918static int ice_ptp_register_auxbus_driver(struct ice_pf *pf)
2919{
2920 struct auxiliary_driver *aux_driver;
2921 struct ice_ptp *ptp;
2922 struct device *dev;
2923 char *name;
2924 int err;
2925
2926 ptp = &pf->ptp;
2927 dev = ice_pf_to_dev(pf);
2928 aux_driver = &ptp->ports_owner.aux_driver;
2929 INIT_LIST_HEAD(&ptp->ports_owner.ports);
2930 mutex_init(&ptp->ports_owner.lock);
2931 name = devm_kasprintf(dev, GFP_KERNEL, "ptp_aux_dev_%u_%u_clk%u",
2932 pf->pdev->bus->number, PCI_SLOT(pf->pdev->devfn),
2933 ice_get_ptp_src_clock_index(&pf->hw));
2934 if (!name)
2935 return -ENOMEM;
2936
2937 aux_driver->name = name;
2938 aux_driver->shutdown = ice_ptp_auxbus_shutdown;
2939 aux_driver->suspend = ice_ptp_auxbus_suspend;
2940 aux_driver->remove = ice_ptp_auxbus_remove;
2941 aux_driver->resume = ice_ptp_auxbus_resume;
2942 aux_driver->probe = ice_ptp_auxbus_probe;
2943 aux_driver->id_table = ice_ptp_auxbus_create_id_table(pf, name);
2944 if (!aux_driver->id_table)
2945 return -ENOMEM;
2946
2947 err = auxiliary_driver_register(aux_driver);
2948 if (err) {
2949 devm_kfree(dev, aux_driver->id_table);
2950 dev_err(dev, "Failed registering aux_driver, name <%s>\n",
2951 name);
2952 }
2953
2954 return err;
2955}
2956
2957/**
2958 * ice_ptp_unregister_auxbus_driver - Unregister PTP auxiliary bus driver
2959 * @pf: Board private structure
2960 */
2961static void ice_ptp_unregister_auxbus_driver(struct ice_pf *pf)
2962{
2963 struct auxiliary_driver *aux_driver = &pf->ptp.ports_owner.aux_driver;
2964
2965 auxiliary_driver_unregister(aux_driver);
2966 devm_kfree(ice_pf_to_dev(pf), aux_driver->id_table);
2967
2968 mutex_destroy(&pf->ptp.ports_owner.lock);
2969}
2970
2971/**
2972 * ice_ptp_clock_index - Get the PTP clock index for this device
2973 * @pf: Board private structure
2974 *
2975 * Returns: the PTP clock index associated with this PF, or -1 if no PTP clock
2976 * is associated.
2977 */
2978int ice_ptp_clock_index(struct ice_pf *pf)
2979{
2980 struct auxiliary_device *aux_dev;
2981 struct ice_pf *owner_pf;
2982 struct ptp_clock *clock;
2983
2984 aux_dev = &pf->ptp.port.aux_dev;
2985 owner_pf = ice_ptp_aux_dev_to_owner_pf(aux_dev);
2986 if (!owner_pf)
2987 return -1;
2988 clock = owner_pf->ptp.clock;
2989
2990 return clock ? ptp_clock_index(clock) : -1;
2991}
2992
2993/**
2994 * ice_ptp_init_owner - Initialize PTP_1588_CLOCK device
2995 * @pf: Board private structure
2996 *
2997 * Setup and initialize a PTP clock device that represents the device hardware
2998 * clock. Save the clock index for other functions connected to the same
2999 * hardware resource.
3000 */
3001static int ice_ptp_init_owner(struct ice_pf *pf)
3002{
3003 struct ice_hw *hw = &pf->hw;
3004 struct timespec64 ts;
3005 int err;
3006
3007 err = ice_ptp_init_phc(hw);
3008 if (err) {
3009 dev_err(ice_pf_to_dev(pf), "Failed to initialize PHC, err %d\n",
3010 err);
3011 return err;
3012 }
3013
3014 /* Acquire the global hardware lock */
3015 if (!ice_ptp_lock(hw)) {
3016 err = -EBUSY;
3017 goto err_exit;
3018 }
3019
3020 /* Write the increment time value to PHY and LAN */
3021 err = ice_ptp_write_incval(hw, ice_base_incval(pf));
3022 if (err) {
3023 ice_ptp_unlock(hw);
3024 goto err_exit;
3025 }
3026
3027 ts = ktime_to_timespec64(ktime_get_real());
3028 /* Write the initial Time value to PHY and LAN */
3029 err = ice_ptp_write_init(pf, &ts);
3030 if (err) {
3031 ice_ptp_unlock(hw);
3032 goto err_exit;
3033 }
3034
3035 /* Release the global hardware lock */
3036 ice_ptp_unlock(hw);
3037
3038 if (!ice_is_e810(hw)) {
3039 /* Enable quad interrupts */
3040 err = ice_ptp_cfg_phy_interrupt(pf, true, 1);
3041 if (err)
3042 goto err_exit;
3043 }
3044
3045 /* Ensure we have a clock device */
3046 err = ice_ptp_create_clock(pf);
3047 if (err)
3048 goto err_clk;
3049
3050 err = ice_ptp_register_auxbus_driver(pf);
3051 if (err) {
3052 dev_err(ice_pf_to_dev(pf), "Failed to register PTP auxbus driver");
3053 goto err_aux;
3054 }
3055
3056 return 0;
3057err_aux:
3058 ptp_clock_unregister(pf->ptp.clock);
3059err_clk:
3060 pf->ptp.clock = NULL;
3061err_exit:
3062 return err;
3063}
3064
3065/**
3066 * ice_ptp_init_work - Initialize PTP work threads
3067 * @pf: Board private structure
3068 * @ptp: PF PTP structure
3069 */
3070static int ice_ptp_init_work(struct ice_pf *pf, struct ice_ptp *ptp)
3071{
3072 struct kthread_worker *kworker;
3073
3074 /* Initialize work functions */
3075 kthread_init_delayed_work(&ptp->work, ice_ptp_periodic_work);
3076
3077 /* Allocate a kworker for handling work required for the ports
3078 * connected to the PTP hardware clock.
3079 */
3080 kworker = kthread_create_worker(0, "ice-ptp-%s",
3081 dev_name(ice_pf_to_dev(pf)));
3082 if (IS_ERR(kworker))
3083 return PTR_ERR(kworker);
3084
3085 ptp->kworker = kworker;
3086
3087 /* Start periodic work going */
3088 kthread_queue_delayed_work(ptp->kworker, &ptp->work, 0);
3089
3090 return 0;
3091}
3092
3093/**
3094 * ice_ptp_init_port - Initialize PTP port structure
3095 * @pf: Board private structure
3096 * @ptp_port: PTP port structure
3097 */
3098static int ice_ptp_init_port(struct ice_pf *pf, struct ice_ptp_port *ptp_port)
3099{
3100 struct ice_hw *hw = &pf->hw;
3101
3102 mutex_init(&ptp_port->ps_lock);
3103
3104 switch (hw->phy_model) {
3105 case ICE_PHY_E810:
3106 return ice_ptp_init_tx_e810(pf, &ptp_port->tx);
3107 case ICE_PHY_E82X:
3108 kthread_init_delayed_work(&ptp_port->ov_work,
3109 ice_ptp_wait_for_offsets);
3110
3111 return ice_ptp_init_tx_e82x(pf, &ptp_port->tx,
3112 ptp_port->port_num);
3113 default:
3114 return -ENODEV;
3115 }
3116}
3117
3118/**
3119 * ice_ptp_release_auxbus_device
3120 * @dev: device that utilizes the auxbus
3121 */
3122static void ice_ptp_release_auxbus_device(struct device *dev)
3123{
3124 /* Doing nothing here, but handle to auxbux device must be satisfied */
3125}
3126
3127/**
3128 * ice_ptp_create_auxbus_device - Create PTP auxiliary bus device
3129 * @pf: Board private structure
3130 */
3131static int ice_ptp_create_auxbus_device(struct ice_pf *pf)
3132{
3133 struct auxiliary_device *aux_dev;
3134 struct ice_ptp *ptp;
3135 struct device *dev;
3136 char *name;
3137 int err;
3138 u32 id;
3139
3140 ptp = &pf->ptp;
3141 id = ptp->port.port_num;
3142 dev = ice_pf_to_dev(pf);
3143
3144 aux_dev = &ptp->port.aux_dev;
3145
3146 name = devm_kasprintf(dev, GFP_KERNEL, "ptp_aux_dev_%u_%u_clk%u",
3147 pf->pdev->bus->number, PCI_SLOT(pf->pdev->devfn),
3148 ice_get_ptp_src_clock_index(&pf->hw));
3149 if (!name)
3150 return -ENOMEM;
3151
3152 aux_dev->name = name;
3153 aux_dev->id = id;
3154 aux_dev->dev.release = ice_ptp_release_auxbus_device;
3155 aux_dev->dev.parent = dev;
3156
3157 err = auxiliary_device_init(aux_dev);
3158 if (err)
3159 goto aux_err;
3160
3161 err = auxiliary_device_add(aux_dev);
3162 if (err) {
3163 auxiliary_device_uninit(aux_dev);
3164 goto aux_err;
3165 }
3166
3167 return 0;
3168aux_err:
3169 dev_err(dev, "Failed to create PTP auxiliary bus device <%s>\n", name);
3170 devm_kfree(dev, name);
3171 return err;
3172}
3173
3174/**
3175 * ice_ptp_remove_auxbus_device - Remove PTP auxiliary bus device
3176 * @pf: Board private structure
3177 */
3178static void ice_ptp_remove_auxbus_device(struct ice_pf *pf)
3179{
3180 struct auxiliary_device *aux_dev = &pf->ptp.port.aux_dev;
3181
3182 auxiliary_device_delete(aux_dev);
3183 auxiliary_device_uninit(aux_dev);
3184
3185 memset(aux_dev, 0, sizeof(*aux_dev));
3186}
3187
3188/**
3189 * ice_ptp_init_tx_interrupt_mode - Initialize device Tx interrupt mode
3190 * @pf: Board private structure
3191 *
3192 * Initialize the Tx timestamp interrupt mode for this device. For most device
3193 * types, each PF processes the interrupt and manages its own timestamps. For
3194 * E822-based devices, only the clock owner processes the timestamps. Other
3195 * PFs disable the interrupt and do not process their own timestamps.
3196 */
3197static void ice_ptp_init_tx_interrupt_mode(struct ice_pf *pf)
3198{
3199 switch (pf->hw.phy_model) {
3200 case ICE_PHY_E82X:
3201 /* E822 based PHY has the clock owner process the interrupt
3202 * for all ports.
3203 */
3204 if (ice_pf_src_tmr_owned(pf))
3205 pf->ptp.tx_interrupt_mode = ICE_PTP_TX_INTERRUPT_ALL;
3206 else
3207 pf->ptp.tx_interrupt_mode = ICE_PTP_TX_INTERRUPT_NONE;
3208 break;
3209 default:
3210 /* other PHY types handle their own Tx interrupt */
3211 pf->ptp.tx_interrupt_mode = ICE_PTP_TX_INTERRUPT_SELF;
3212 }
3213}
3214
3215/**
3216 * ice_ptp_init - Initialize PTP hardware clock support
3217 * @pf: Board private structure
3218 *
3219 * Set up the device for interacting with the PTP hardware clock for all
3220 * functions, both the function that owns the clock hardware, and the
3221 * functions connected to the clock hardware.
3222 *
3223 * The clock owner will allocate and register a ptp_clock with the
3224 * PTP_1588_CLOCK infrastructure. All functions allocate a kthread and work
3225 * items used for asynchronous work such as Tx timestamps and periodic work.
3226 */
3227void ice_ptp_init(struct ice_pf *pf)
3228{
3229 struct ice_ptp *ptp = &pf->ptp;
3230 struct ice_hw *hw = &pf->hw;
3231 int err;
3232
3233 ptp->state = ICE_PTP_INITIALIZING;
3234
3235 ice_ptp_init_phy_model(hw);
3236
3237 ice_ptp_init_tx_interrupt_mode(pf);
3238
3239 /* If this function owns the clock hardware, it must allocate and
3240 * configure the PTP clock device to represent it.
3241 */
3242 if (ice_pf_src_tmr_owned(pf)) {
3243 err = ice_ptp_init_owner(pf);
3244 if (err)
3245 goto err;
3246 }
3247
3248 ptp->port.port_num = hw->pf_id;
3249 err = ice_ptp_init_port(pf, &ptp->port);
3250 if (err)
3251 goto err;
3252
3253 /* Start the PHY timestamping block */
3254 ice_ptp_reset_phy_timestamping(pf);
3255
3256 /* Configure initial Tx interrupt settings */
3257 ice_ptp_cfg_tx_interrupt(pf);
3258
3259 err = ice_ptp_create_auxbus_device(pf);
3260 if (err)
3261 goto err;
3262
3263 ptp->state = ICE_PTP_READY;
3264
3265 err = ice_ptp_init_work(pf, ptp);
3266 if (err)
3267 goto err;
3268
3269 dev_info(ice_pf_to_dev(pf), "PTP init successful\n");
3270 return;
3271
3272err:
3273 /* If we registered a PTP clock, release it */
3274 if (pf->ptp.clock) {
3275 ptp_clock_unregister(ptp->clock);
3276 pf->ptp.clock = NULL;
3277 }
3278 ptp->state = ICE_PTP_ERROR;
3279 dev_err(ice_pf_to_dev(pf), "PTP failed %d\n", err);
3280}
3281
3282/**
3283 * ice_ptp_release - Disable the driver/HW support and unregister the clock
3284 * @pf: Board private structure
3285 *
3286 * This function handles the cleanup work required from the initialization by
3287 * clearing out the important information and unregistering the clock
3288 */
3289void ice_ptp_release(struct ice_pf *pf)
3290{
3291 if (pf->ptp.state != ICE_PTP_READY)
3292 return;
3293
3294 pf->ptp.state = ICE_PTP_UNINIT;
3295
3296 /* Disable timestamping for both Tx and Rx */
3297 ice_ptp_disable_timestamp_mode(pf);
3298
3299 ice_ptp_remove_auxbus_device(pf);
3300
3301 ice_ptp_release_tx_tracker(pf, &pf->ptp.port.tx);
3302
3303 kthread_cancel_delayed_work_sync(&pf->ptp.work);
3304
3305 ice_ptp_port_phy_stop(&pf->ptp.port);
3306 mutex_destroy(&pf->ptp.port.ps_lock);
3307 if (pf->ptp.kworker) {
3308 kthread_destroy_worker(pf->ptp.kworker);
3309 pf->ptp.kworker = NULL;
3310 }
3311
3312 if (ice_pf_src_tmr_owned(pf))
3313 ice_ptp_unregister_auxbus_driver(pf);
3314
3315 if (!pf->ptp.clock)
3316 return;
3317
3318 /* Disable periodic outputs */
3319 ice_ptp_disable_all_clkout(pf);
3320
3321 ptp_clock_unregister(pf->ptp.clock);
3322 pf->ptp.clock = NULL;
3323
3324 dev_info(ice_pf_to_dev(pf), "Removed PTP clock\n");
3325}
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2021, Intel Corporation. */
3
4#include "ice.h"
5#include "ice_lib.h"
6
7#define E810_OUT_PROP_DELAY_NS 1
8
9/**
10 * ice_set_tx_tstamp - Enable or disable Tx timestamping
11 * @pf: The PF pointer to search in
12 * @on: bool value for whether timestamps are enabled or disabled
13 */
14static void ice_set_tx_tstamp(struct ice_pf *pf, bool on)
15{
16 struct ice_vsi *vsi;
17 u32 val;
18 u16 i;
19
20 vsi = ice_get_main_vsi(pf);
21 if (!vsi)
22 return;
23
24 /* Set the timestamp enable flag for all the Tx rings */
25 ice_for_each_txq(vsi, i) {
26 if (!vsi->tx_rings[i])
27 continue;
28 vsi->tx_rings[i]->ptp_tx = on;
29 }
30
31 /* Configure the Tx timestamp interrupt */
32 val = rd32(&pf->hw, PFINT_OICR_ENA);
33 if (on)
34 val |= PFINT_OICR_TSYN_TX_M;
35 else
36 val &= ~PFINT_OICR_TSYN_TX_M;
37 wr32(&pf->hw, PFINT_OICR_ENA, val);
38}
39
40/**
41 * ice_set_rx_tstamp - Enable or disable Rx timestamping
42 * @pf: The PF pointer to search in
43 * @on: bool value for whether timestamps are enabled or disabled
44 */
45static void ice_set_rx_tstamp(struct ice_pf *pf, bool on)
46{
47 struct ice_vsi *vsi;
48 u16 i;
49
50 vsi = ice_get_main_vsi(pf);
51 if (!vsi)
52 return;
53
54 /* Set the timestamp flag for all the Rx rings */
55 ice_for_each_rxq(vsi, i) {
56 if (!vsi->rx_rings[i])
57 continue;
58 vsi->rx_rings[i]->ptp_rx = on;
59 }
60}
61
62/**
63 * ice_ptp_cfg_timestamp - Configure timestamp for init/deinit
64 * @pf: Board private structure
65 * @ena: bool value to enable or disable time stamp
66 *
67 * This function will configure timestamping during PTP initialization
68 * and deinitialization
69 */
70static void ice_ptp_cfg_timestamp(struct ice_pf *pf, bool ena)
71{
72 ice_set_tx_tstamp(pf, ena);
73 ice_set_rx_tstamp(pf, ena);
74
75 if (ena) {
76 pf->ptp.tstamp_config.rx_filter = HWTSTAMP_FILTER_ALL;
77 pf->ptp.tstamp_config.tx_type = HWTSTAMP_TX_ON;
78 } else {
79 pf->ptp.tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
80 pf->ptp.tstamp_config.tx_type = HWTSTAMP_TX_OFF;
81 }
82}
83
84/**
85 * ice_get_ptp_clock_index - Get the PTP clock index
86 * @pf: the PF pointer
87 *
88 * Determine the clock index of the PTP clock associated with this device. If
89 * this is the PF controlling the clock, just use the local access to the
90 * clock device pointer.
91 *
92 * Otherwise, read from the driver shared parameters to determine the clock
93 * index value.
94 *
95 * Returns: the index of the PTP clock associated with this device, or -1 if
96 * there is no associated clock.
97 */
98int ice_get_ptp_clock_index(struct ice_pf *pf)
99{
100 struct device *dev = ice_pf_to_dev(pf);
101 enum ice_aqc_driver_params param_idx;
102 struct ice_hw *hw = &pf->hw;
103 u8 tmr_idx;
104 u32 value;
105 int err;
106
107 /* Use the ptp_clock structure if we're the main PF */
108 if (pf->ptp.clock)
109 return ptp_clock_index(pf->ptp.clock);
110
111 tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc;
112 if (!tmr_idx)
113 param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR0;
114 else
115 param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR1;
116
117 err = ice_aq_get_driver_param(hw, param_idx, &value, NULL);
118 if (err) {
119 dev_err(dev, "Failed to read PTP clock index parameter, err %d aq_err %s\n",
120 err, ice_aq_str(hw->adminq.sq_last_status));
121 return -1;
122 }
123
124 /* The PTP clock index is an integer, and will be between 0 and
125 * INT_MAX. The highest bit of the driver shared parameter is used to
126 * indicate whether or not the currently stored clock index is valid.
127 */
128 if (!(value & PTP_SHARED_CLK_IDX_VALID))
129 return -1;
130
131 return value & ~PTP_SHARED_CLK_IDX_VALID;
132}
133
134/**
135 * ice_set_ptp_clock_index - Set the PTP clock index
136 * @pf: the PF pointer
137 *
138 * Set the PTP clock index for this device into the shared driver parameters,
139 * so that other PFs associated with this device can read it.
140 *
141 * If the PF is unable to store the clock index, it will log an error, but
142 * will continue operating PTP.
143 */
144static void ice_set_ptp_clock_index(struct ice_pf *pf)
145{
146 struct device *dev = ice_pf_to_dev(pf);
147 enum ice_aqc_driver_params param_idx;
148 struct ice_hw *hw = &pf->hw;
149 u8 tmr_idx;
150 u32 value;
151 int err;
152
153 if (!pf->ptp.clock)
154 return;
155
156 tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc;
157 if (!tmr_idx)
158 param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR0;
159 else
160 param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR1;
161
162 value = (u32)ptp_clock_index(pf->ptp.clock);
163 if (value > INT_MAX) {
164 dev_err(dev, "PTP Clock index is too large to store\n");
165 return;
166 }
167 value |= PTP_SHARED_CLK_IDX_VALID;
168
169 err = ice_aq_set_driver_param(hw, param_idx, value, NULL);
170 if (err) {
171 dev_err(dev, "Failed to set PTP clock index parameter, err %d aq_err %s\n",
172 err, ice_aq_str(hw->adminq.sq_last_status));
173 }
174}
175
176/**
177 * ice_clear_ptp_clock_index - Clear the PTP clock index
178 * @pf: the PF pointer
179 *
180 * Clear the PTP clock index for this device. Must be called when
181 * unregistering the PTP clock, in order to ensure other PFs stop reporting
182 * a clock object that no longer exists.
183 */
184static void ice_clear_ptp_clock_index(struct ice_pf *pf)
185{
186 struct device *dev = ice_pf_to_dev(pf);
187 enum ice_aqc_driver_params param_idx;
188 struct ice_hw *hw = &pf->hw;
189 u8 tmr_idx;
190 int err;
191
192 /* Do not clear the index if we don't own the timer */
193 if (!hw->func_caps.ts_func_info.src_tmr_owned)
194 return;
195
196 tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc;
197 if (!tmr_idx)
198 param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR0;
199 else
200 param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR1;
201
202 err = ice_aq_set_driver_param(hw, param_idx, 0, NULL);
203 if (err) {
204 dev_dbg(dev, "Failed to clear PTP clock index parameter, err %d aq_err %s\n",
205 err, ice_aq_str(hw->adminq.sq_last_status));
206 }
207}
208
209/**
210 * ice_ptp_read_src_clk_reg - Read the source clock register
211 * @pf: Board private structure
212 * @sts: Optional parameter for holding a pair of system timestamps from
213 * the system clock. Will be ignored if NULL is given.
214 */
215static u64
216ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts)
217{
218 struct ice_hw *hw = &pf->hw;
219 u32 hi, lo, lo2;
220 u8 tmr_idx;
221
222 tmr_idx = ice_get_ptp_src_clock_index(hw);
223 /* Read the system timestamp pre PHC read */
224 ptp_read_system_prets(sts);
225
226 lo = rd32(hw, GLTSYN_TIME_L(tmr_idx));
227
228 /* Read the system timestamp post PHC read */
229 ptp_read_system_postts(sts);
230
231 hi = rd32(hw, GLTSYN_TIME_H(tmr_idx));
232 lo2 = rd32(hw, GLTSYN_TIME_L(tmr_idx));
233
234 if (lo2 < lo) {
235 /* if TIME_L rolled over read TIME_L again and update
236 * system timestamps
237 */
238 ptp_read_system_prets(sts);
239 lo = rd32(hw, GLTSYN_TIME_L(tmr_idx));
240 ptp_read_system_postts(sts);
241 hi = rd32(hw, GLTSYN_TIME_H(tmr_idx));
242 }
243
244 return ((u64)hi << 32) | lo;
245}
246
247/**
248 * ice_ptp_update_cached_phctime - Update the cached PHC time values
249 * @pf: Board specific private structure
250 *
251 * This function updates the system time values which are cached in the PF
252 * structure and the Rx rings.
253 *
254 * This function must be called periodically to ensure that the cached value
255 * is never more than 2 seconds old. It must also be called whenever the PHC
256 * time has been changed.
257 */
258static void ice_ptp_update_cached_phctime(struct ice_pf *pf)
259{
260 u64 systime;
261 int i;
262
263 /* Read the current PHC time */
264 systime = ice_ptp_read_src_clk_reg(pf, NULL);
265
266 /* Update the cached PHC time stored in the PF structure */
267 WRITE_ONCE(pf->ptp.cached_phc_time, systime);
268
269 ice_for_each_vsi(pf, i) {
270 struct ice_vsi *vsi = pf->vsi[i];
271 int j;
272
273 if (!vsi)
274 continue;
275
276 if (vsi->type != ICE_VSI_PF)
277 continue;
278
279 ice_for_each_rxq(vsi, j) {
280 if (!vsi->rx_rings[j])
281 continue;
282 WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime);
283 }
284 }
285}
286
287/**
288 * ice_ptp_extend_32b_ts - Convert a 32b nanoseconds timestamp to 64b
289 * @cached_phc_time: recently cached copy of PHC time
290 * @in_tstamp: Ingress/egress 32b nanoseconds timestamp value
291 *
292 * Hardware captures timestamps which contain only 32 bits of nominal
293 * nanoseconds, as opposed to the 64bit timestamps that the stack expects.
294 * Note that the captured timestamp values may be 40 bits, but the lower
295 * 8 bits are sub-nanoseconds and generally discarded.
296 *
297 * Extend the 32bit nanosecond timestamp using the following algorithm and
298 * assumptions:
299 *
300 * 1) have a recently cached copy of the PHC time
301 * 2) assume that the in_tstamp was captured 2^31 nanoseconds (~2.1
302 * seconds) before or after the PHC time was captured.
303 * 3) calculate the delta between the cached time and the timestamp
304 * 4) if the delta is smaller than 2^31 nanoseconds, then the timestamp was
305 * captured after the PHC time. In this case, the full timestamp is just
306 * the cached PHC time plus the delta.
307 * 5) otherwise, if the delta is larger than 2^31 nanoseconds, then the
308 * timestamp was captured *before* the PHC time, i.e. because the PHC
309 * cache was updated after the timestamp was captured by hardware. In this
310 * case, the full timestamp is the cached time minus the inverse delta.
311 *
312 * This algorithm works even if the PHC time was updated after a Tx timestamp
313 * was requested, but before the Tx timestamp event was reported from
314 * hardware.
315 *
316 * This calculation primarily relies on keeping the cached PHC time up to
317 * date. If the timestamp was captured more than 2^31 nanoseconds after the
318 * PHC time, it is possible that the lower 32bits of PHC time have
319 * overflowed more than once, and we might generate an incorrect timestamp.
320 *
321 * This is prevented by (a) periodically updating the cached PHC time once
322 * a second, and (b) discarding any Tx timestamp packet if it has waited for
323 * a timestamp for more than one second.
324 */
325static u64 ice_ptp_extend_32b_ts(u64 cached_phc_time, u32 in_tstamp)
326{
327 u32 delta, phc_time_lo;
328 u64 ns;
329
330 /* Extract the lower 32 bits of the PHC time */
331 phc_time_lo = (u32)cached_phc_time;
332
333 /* Calculate the delta between the lower 32bits of the cached PHC
334 * time and the in_tstamp value
335 */
336 delta = (in_tstamp - phc_time_lo);
337
338 /* Do not assume that the in_tstamp is always more recent than the
339 * cached PHC time. If the delta is large, it indicates that the
340 * in_tstamp was taken in the past, and should be converted
341 * forward.
342 */
343 if (delta > (U32_MAX / 2)) {
344 /* reverse the delta calculation here */
345 delta = (phc_time_lo - in_tstamp);
346 ns = cached_phc_time - delta;
347 } else {
348 ns = cached_phc_time + delta;
349 }
350
351 return ns;
352}
353
354/**
355 * ice_ptp_extend_40b_ts - Convert a 40b timestamp to 64b nanoseconds
356 * @pf: Board private structure
357 * @in_tstamp: Ingress/egress 40b timestamp value
358 *
359 * The Tx and Rx timestamps are 40 bits wide, including 32 bits of nominal
360 * nanoseconds, 7 bits of sub-nanoseconds, and a valid bit.
361 *
362 * *--------------------------------------------------------------*
363 * | 32 bits of nanoseconds | 7 high bits of sub ns underflow | v |
364 * *--------------------------------------------------------------*
365 *
366 * The low bit is an indicator of whether the timestamp is valid. The next
367 * 7 bits are a capture of the upper 7 bits of the sub-nanosecond underflow,
368 * and the remaining 32 bits are the lower 32 bits of the PHC timer.
369 *
370 * It is assumed that the caller verifies the timestamp is valid prior to
371 * calling this function.
372 *
373 * Extract the 32bit nominal nanoseconds and extend them. Use the cached PHC
374 * time stored in the device private PTP structure as the basis for timestamp
375 * extension.
376 *
377 * See ice_ptp_extend_32b_ts for a detailed explanation of the extension
378 * algorithm.
379 */
380static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp)
381{
382 const u64 mask = GENMASK_ULL(31, 0);
383
384 return ice_ptp_extend_32b_ts(pf->ptp.cached_phc_time,
385 (in_tstamp >> 8) & mask);
386}
387
388/**
389 * ice_ptp_read_time - Read the time from the device
390 * @pf: Board private structure
391 * @ts: timespec structure to hold the current time value
392 * @sts: Optional parameter for holding a pair of system timestamps from
393 * the system clock. Will be ignored if NULL is given.
394 *
395 * This function reads the source clock registers and stores them in a timespec.
396 * However, since the registers are 64 bits of nanoseconds, we must convert the
397 * result to a timespec before we can return.
398 */
399static void
400ice_ptp_read_time(struct ice_pf *pf, struct timespec64 *ts,
401 struct ptp_system_timestamp *sts)
402{
403 u64 time_ns = ice_ptp_read_src_clk_reg(pf, sts);
404
405 *ts = ns_to_timespec64(time_ns);
406}
407
408/**
409 * ice_ptp_write_init - Set PHC time to provided value
410 * @pf: Board private structure
411 * @ts: timespec structure that holds the new time value
412 *
413 * Set the PHC time to the specified time provided in the timespec.
414 */
415static int ice_ptp_write_init(struct ice_pf *pf, struct timespec64 *ts)
416{
417 u64 ns = timespec64_to_ns(ts);
418 struct ice_hw *hw = &pf->hw;
419
420 return ice_ptp_init_time(hw, ns);
421}
422
423/**
424 * ice_ptp_write_adj - Adjust PHC clock time atomically
425 * @pf: Board private structure
426 * @adj: Adjustment in nanoseconds
427 *
428 * Perform an atomic adjustment of the PHC time by the specified number of
429 * nanoseconds.
430 */
431static int ice_ptp_write_adj(struct ice_pf *pf, s32 adj)
432{
433 struct ice_hw *hw = &pf->hw;
434
435 return ice_ptp_adj_clock(hw, adj);
436}
437
438/**
439 * ice_ptp_adjfine - Adjust clock increment rate
440 * @info: the driver's PTP info structure
441 * @scaled_ppm: Parts per million with 16-bit fractional field
442 *
443 * Adjust the frequency of the clock by the indicated scaled ppm from the
444 * base frequency.
445 */
446static int ice_ptp_adjfine(struct ptp_clock_info *info, long scaled_ppm)
447{
448 struct ice_pf *pf = ptp_info_to_pf(info);
449 u64 freq, divisor = 1000000ULL;
450 struct ice_hw *hw = &pf->hw;
451 s64 incval, diff;
452 int neg_adj = 0;
453 int err;
454
455 incval = ICE_PTP_NOMINAL_INCVAL_E810;
456
457 if (scaled_ppm < 0) {
458 neg_adj = 1;
459 scaled_ppm = -scaled_ppm;
460 }
461
462 while ((u64)scaled_ppm > div_u64(U64_MAX, incval)) {
463 /* handle overflow by scaling down the scaled_ppm and
464 * the divisor, losing some precision
465 */
466 scaled_ppm >>= 2;
467 divisor >>= 2;
468 }
469
470 freq = (incval * (u64)scaled_ppm) >> 16;
471 diff = div_u64(freq, divisor);
472
473 if (neg_adj)
474 incval -= diff;
475 else
476 incval += diff;
477
478 err = ice_ptp_write_incval_locked(hw, incval);
479 if (err) {
480 dev_err(ice_pf_to_dev(pf), "PTP failed to set incval, err %d\n",
481 err);
482 return -EIO;
483 }
484
485 return 0;
486}
487
488/**
489 * ice_ptp_extts_work - Workqueue task function
490 * @work: external timestamp work structure
491 *
492 * Service for PTP external clock event
493 */
494static void ice_ptp_extts_work(struct kthread_work *work)
495{
496 struct ice_ptp *ptp = container_of(work, struct ice_ptp, extts_work);
497 struct ice_pf *pf = container_of(ptp, struct ice_pf, ptp);
498 struct ptp_clock_event event;
499 struct ice_hw *hw = &pf->hw;
500 u8 chan, tmr_idx;
501 u32 hi, lo;
502
503 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
504 /* Event time is captured by one of the two matched registers
505 * GLTSYN_EVNT_L: 32 LSB of sampled time event
506 * GLTSYN_EVNT_H: 32 MSB of sampled time event
507 * Event is defined in GLTSYN_EVNT_0 register
508 */
509 for (chan = 0; chan < GLTSYN_EVNT_H_IDX_MAX; chan++) {
510 /* Check if channel is enabled */
511 if (pf->ptp.ext_ts_irq & (1 << chan)) {
512 lo = rd32(hw, GLTSYN_EVNT_L(chan, tmr_idx));
513 hi = rd32(hw, GLTSYN_EVNT_H(chan, tmr_idx));
514 event.timestamp = (((u64)hi) << 32) | lo;
515 event.type = PTP_CLOCK_EXTTS;
516 event.index = chan;
517
518 /* Fire event */
519 ptp_clock_event(pf->ptp.clock, &event);
520 pf->ptp.ext_ts_irq &= ~(1 << chan);
521 }
522 }
523}
524
525/**
526 * ice_ptp_cfg_extts - Configure EXTTS pin and channel
527 * @pf: Board private structure
528 * @ena: true to enable; false to disable
529 * @chan: GPIO channel (0-3)
530 * @gpio_pin: GPIO pin
531 * @extts_flags: request flags from the ptp_extts_request.flags
532 */
533static int
534ice_ptp_cfg_extts(struct ice_pf *pf, bool ena, unsigned int chan, u32 gpio_pin,
535 unsigned int extts_flags)
536{
537 u32 func, aux_reg, gpio_reg, irq_reg;
538 struct ice_hw *hw = &pf->hw;
539 u8 tmr_idx;
540
541 if (chan > (unsigned int)pf->ptp.info.n_ext_ts)
542 return -EINVAL;
543
544 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
545
546 irq_reg = rd32(hw, PFINT_OICR_ENA);
547
548 if (ena) {
549 /* Enable the interrupt */
550 irq_reg |= PFINT_OICR_TSYN_EVNT_M;
551 aux_reg = GLTSYN_AUX_IN_0_INT_ENA_M;
552
553#define GLTSYN_AUX_IN_0_EVNTLVL_RISING_EDGE BIT(0)
554#define GLTSYN_AUX_IN_0_EVNTLVL_FALLING_EDGE BIT(1)
555
556 /* set event level to requested edge */
557 if (extts_flags & PTP_FALLING_EDGE)
558 aux_reg |= GLTSYN_AUX_IN_0_EVNTLVL_FALLING_EDGE;
559 if (extts_flags & PTP_RISING_EDGE)
560 aux_reg |= GLTSYN_AUX_IN_0_EVNTLVL_RISING_EDGE;
561
562 /* Write GPIO CTL reg.
563 * 0x1 is input sampled by EVENT register(channel)
564 * + num_in_channels * tmr_idx
565 */
566 func = 1 + chan + (tmr_idx * 3);
567 gpio_reg = ((func << GLGEN_GPIO_CTL_PIN_FUNC_S) &
568 GLGEN_GPIO_CTL_PIN_FUNC_M);
569 pf->ptp.ext_ts_chan |= (1 << chan);
570 } else {
571 /* clear the values we set to reset defaults */
572 aux_reg = 0;
573 gpio_reg = 0;
574 pf->ptp.ext_ts_chan &= ~(1 << chan);
575 if (!pf->ptp.ext_ts_chan)
576 irq_reg &= ~PFINT_OICR_TSYN_EVNT_M;
577 }
578
579 wr32(hw, PFINT_OICR_ENA, irq_reg);
580 wr32(hw, GLTSYN_AUX_IN(chan, tmr_idx), aux_reg);
581 wr32(hw, GLGEN_GPIO_CTL(gpio_pin), gpio_reg);
582
583 return 0;
584}
585
586/**
587 * ice_ptp_cfg_clkout - Configure clock to generate periodic wave
588 * @pf: Board private structure
589 * @chan: GPIO channel (0-3)
590 * @config: desired periodic clk configuration. NULL will disable channel
591 * @store: If set to true the values will be stored
592 *
593 * Configure the internal clock generator modules to generate the clock wave of
594 * specified period.
595 */
596static int ice_ptp_cfg_clkout(struct ice_pf *pf, unsigned int chan,
597 struct ice_perout_channel *config, bool store)
598{
599 u64 current_time, period, start_time, phase;
600 struct ice_hw *hw = &pf->hw;
601 u32 func, val, gpio_pin;
602 u8 tmr_idx;
603
604 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
605
606 /* 0. Reset mode & out_en in AUX_OUT */
607 wr32(hw, GLTSYN_AUX_OUT(chan, tmr_idx), 0);
608
609 /* If we're disabling the output, clear out CLKO and TGT and keep
610 * output level low
611 */
612 if (!config || !config->ena) {
613 wr32(hw, GLTSYN_CLKO(chan, tmr_idx), 0);
614 wr32(hw, GLTSYN_TGT_L(chan, tmr_idx), 0);
615 wr32(hw, GLTSYN_TGT_H(chan, tmr_idx), 0);
616
617 val = GLGEN_GPIO_CTL_PIN_DIR_M;
618 gpio_pin = pf->ptp.perout_channels[chan].gpio_pin;
619 wr32(hw, GLGEN_GPIO_CTL(gpio_pin), val);
620
621 /* Store the value if requested */
622 if (store)
623 memset(&pf->ptp.perout_channels[chan], 0,
624 sizeof(struct ice_perout_channel));
625
626 return 0;
627 }
628 period = config->period;
629 start_time = config->start_time;
630 div64_u64_rem(start_time, period, &phase);
631 gpio_pin = config->gpio_pin;
632
633 /* 1. Write clkout with half of required period value */
634 if (period & 0x1) {
635 dev_err(ice_pf_to_dev(pf), "CLK Period must be an even value\n");
636 goto err;
637 }
638
639 period >>= 1;
640
641 /* For proper operation, the GLTSYN_CLKO must be larger than clock tick
642 */
643#define MIN_PULSE 3
644 if (period <= MIN_PULSE || period > U32_MAX) {
645 dev_err(ice_pf_to_dev(pf), "CLK Period must be > %d && < 2^33",
646 MIN_PULSE * 2);
647 goto err;
648 }
649
650 wr32(hw, GLTSYN_CLKO(chan, tmr_idx), lower_32_bits(period));
651
652 /* Allow time for programming before start_time is hit */
653 current_time = ice_ptp_read_src_clk_reg(pf, NULL);
654
655 /* if start time is in the past start the timer at the nearest second
656 * maintaining phase
657 */
658 if (start_time < current_time)
659 start_time = div64_u64(current_time + NSEC_PER_SEC - 1,
660 NSEC_PER_SEC) * NSEC_PER_SEC + phase;
661
662 start_time -= E810_OUT_PROP_DELAY_NS;
663
664 /* 2. Write TARGET time */
665 wr32(hw, GLTSYN_TGT_L(chan, tmr_idx), lower_32_bits(start_time));
666 wr32(hw, GLTSYN_TGT_H(chan, tmr_idx), upper_32_bits(start_time));
667
668 /* 3. Write AUX_OUT register */
669 val = GLTSYN_AUX_OUT_0_OUT_ENA_M | GLTSYN_AUX_OUT_0_OUTMOD_M;
670 wr32(hw, GLTSYN_AUX_OUT(chan, tmr_idx), val);
671
672 /* 4. write GPIO CTL reg */
673 func = 8 + chan + (tmr_idx * 4);
674 val = GLGEN_GPIO_CTL_PIN_DIR_M |
675 ((func << GLGEN_GPIO_CTL_PIN_FUNC_S) & GLGEN_GPIO_CTL_PIN_FUNC_M);
676 wr32(hw, GLGEN_GPIO_CTL(gpio_pin), val);
677
678 /* Store the value if requested */
679 if (store) {
680 memcpy(&pf->ptp.perout_channels[chan], config,
681 sizeof(struct ice_perout_channel));
682 pf->ptp.perout_channels[chan].start_time = phase;
683 }
684
685 return 0;
686err:
687 dev_err(ice_pf_to_dev(pf), "PTP failed to cfg per_clk\n");
688 return -EFAULT;
689}
690
691/**
692 * ice_ptp_disable_all_clkout - Disable all currently configured outputs
693 * @pf: pointer to the PF structure
694 *
695 * Disable all currently configured clock outputs. This is necessary before
696 * certain changes to the PTP hardware clock. Use ice_ptp_enable_all_clkout to
697 * re-enable the clocks again.
698 */
699static void ice_ptp_disable_all_clkout(struct ice_pf *pf)
700{
701 uint i;
702
703 for (i = 0; i < pf->ptp.info.n_per_out; i++)
704 if (pf->ptp.perout_channels[i].ena)
705 ice_ptp_cfg_clkout(pf, i, NULL, false);
706}
707
708/**
709 * ice_ptp_enable_all_clkout - Enable all configured periodic clock outputs
710 * @pf: pointer to the PF structure
711 *
712 * Enable all currently configured clock outputs. Use this after
713 * ice_ptp_disable_all_clkout to reconfigure the output signals according to
714 * their configuration.
715 */
716static void ice_ptp_enable_all_clkout(struct ice_pf *pf)
717{
718 uint i;
719
720 for (i = 0; i < pf->ptp.info.n_per_out; i++)
721 if (pf->ptp.perout_channels[i].ena)
722 ice_ptp_cfg_clkout(pf, i, &pf->ptp.perout_channels[i],
723 false);
724}
725
726/**
727 * ice_ptp_gpio_enable_e810 - Enable/disable ancillary features of PHC
728 * @info: the driver's PTP info structure
729 * @rq: The requested feature to change
730 * @on: Enable/disable flag
731 */
732static int
733ice_ptp_gpio_enable_e810(struct ptp_clock_info *info,
734 struct ptp_clock_request *rq, int on)
735{
736 struct ice_pf *pf = ptp_info_to_pf(info);
737 struct ice_perout_channel clk_cfg = {0};
738 unsigned int chan;
739 u32 gpio_pin;
740 int err;
741
742 switch (rq->type) {
743 case PTP_CLK_REQ_PEROUT:
744 chan = rq->perout.index;
745 if (chan == PPS_CLK_GEN_CHAN)
746 clk_cfg.gpio_pin = PPS_PIN_INDEX;
747 else
748 clk_cfg.gpio_pin = chan;
749
750 clk_cfg.period = ((rq->perout.period.sec * NSEC_PER_SEC) +
751 rq->perout.period.nsec);
752 clk_cfg.start_time = ((rq->perout.start.sec * NSEC_PER_SEC) +
753 rq->perout.start.nsec);
754 clk_cfg.ena = !!on;
755
756 err = ice_ptp_cfg_clkout(pf, chan, &clk_cfg, true);
757 break;
758 case PTP_CLK_REQ_EXTTS:
759 chan = rq->extts.index;
760 gpio_pin = chan;
761
762 err = ice_ptp_cfg_extts(pf, !!on, chan, gpio_pin,
763 rq->extts.flags);
764 break;
765 default:
766 return -EOPNOTSUPP;
767 }
768
769 return err;
770}
771
772/**
773 * ice_ptp_gettimex64 - Get the time of the clock
774 * @info: the driver's PTP info structure
775 * @ts: timespec64 structure to hold the current time value
776 * @sts: Optional parameter for holding a pair of system timestamps from
777 * the system clock. Will be ignored if NULL is given.
778 *
779 * Read the device clock and return the correct value on ns, after converting it
780 * into a timespec struct.
781 */
782static int
783ice_ptp_gettimex64(struct ptp_clock_info *info, struct timespec64 *ts,
784 struct ptp_system_timestamp *sts)
785{
786 struct ice_pf *pf = ptp_info_to_pf(info);
787 struct ice_hw *hw = &pf->hw;
788
789 if (!ice_ptp_lock(hw)) {
790 dev_err(ice_pf_to_dev(pf), "PTP failed to get time\n");
791 return -EBUSY;
792 }
793
794 ice_ptp_read_time(pf, ts, sts);
795 ice_ptp_unlock(hw);
796
797 return 0;
798}
799
800/**
801 * ice_ptp_settime64 - Set the time of the clock
802 * @info: the driver's PTP info structure
803 * @ts: timespec64 structure that holds the new time value
804 *
805 * Set the device clock to the user input value. The conversion from timespec
806 * to ns happens in the write function.
807 */
808static int
809ice_ptp_settime64(struct ptp_clock_info *info, const struct timespec64 *ts)
810{
811 struct ice_pf *pf = ptp_info_to_pf(info);
812 struct timespec64 ts64 = *ts;
813 struct ice_hw *hw = &pf->hw;
814 int err;
815
816 if (!ice_ptp_lock(hw)) {
817 err = -EBUSY;
818 goto exit;
819 }
820
821 /* Disable periodic outputs */
822 ice_ptp_disable_all_clkout(pf);
823
824 err = ice_ptp_write_init(pf, &ts64);
825 ice_ptp_unlock(hw);
826
827 if (!err)
828 ice_ptp_update_cached_phctime(pf);
829
830 /* Reenable periodic outputs */
831 ice_ptp_enable_all_clkout(pf);
832exit:
833 if (err) {
834 dev_err(ice_pf_to_dev(pf), "PTP failed to set time %d\n", err);
835 return err;
836 }
837
838 return 0;
839}
840
841/**
842 * ice_ptp_adjtime_nonatomic - Do a non-atomic clock adjustment
843 * @info: the driver's PTP info structure
844 * @delta: Offset in nanoseconds to adjust the time by
845 */
846static int ice_ptp_adjtime_nonatomic(struct ptp_clock_info *info, s64 delta)
847{
848 struct timespec64 now, then;
849
850 then = ns_to_timespec64(delta);
851 ice_ptp_gettimex64(info, &now, NULL);
852 now = timespec64_add(now, then);
853
854 return ice_ptp_settime64(info, (const struct timespec64 *)&now);
855}
856
857/**
858 * ice_ptp_adjtime - Adjust the time of the clock by the indicated delta
859 * @info: the driver's PTP info structure
860 * @delta: Offset in nanoseconds to adjust the time by
861 */
862static int ice_ptp_adjtime(struct ptp_clock_info *info, s64 delta)
863{
864 struct ice_pf *pf = ptp_info_to_pf(info);
865 struct ice_hw *hw = &pf->hw;
866 struct device *dev;
867 int err;
868
869 dev = ice_pf_to_dev(pf);
870
871 /* Hardware only supports atomic adjustments using signed 32-bit
872 * integers. For any adjustment outside this range, perform
873 * a non-atomic get->adjust->set flow.
874 */
875 if (delta > S32_MAX || delta < S32_MIN) {
876 dev_dbg(dev, "delta = %lld, adjtime non-atomic\n", delta);
877 return ice_ptp_adjtime_nonatomic(info, delta);
878 }
879
880 if (!ice_ptp_lock(hw)) {
881 dev_err(dev, "PTP failed to acquire semaphore in adjtime\n");
882 return -EBUSY;
883 }
884
885 /* Disable periodic outputs */
886 ice_ptp_disable_all_clkout(pf);
887
888 err = ice_ptp_write_adj(pf, delta);
889
890 /* Reenable periodic outputs */
891 ice_ptp_enable_all_clkout(pf);
892
893 ice_ptp_unlock(hw);
894
895 if (err) {
896 dev_err(dev, "PTP failed to adjust time, err %d\n", err);
897 return err;
898 }
899
900 ice_ptp_update_cached_phctime(pf);
901
902 return 0;
903}
904
905/**
906 * ice_ptp_get_ts_config - ioctl interface to read the timestamping config
907 * @pf: Board private structure
908 * @ifr: ioctl data
909 *
910 * Copy the timestamping config to user buffer
911 */
912int ice_ptp_get_ts_config(struct ice_pf *pf, struct ifreq *ifr)
913{
914 struct hwtstamp_config *config;
915
916 if (!test_bit(ICE_FLAG_PTP, pf->flags))
917 return -EIO;
918
919 config = &pf->ptp.tstamp_config;
920
921 return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
922 -EFAULT : 0;
923}
924
925/**
926 * ice_ptp_set_timestamp_mode - Setup driver for requested timestamp mode
927 * @pf: Board private structure
928 * @config: hwtstamp settings requested or saved
929 */
930static int
931ice_ptp_set_timestamp_mode(struct ice_pf *pf, struct hwtstamp_config *config)
932{
933 /* Reserved for future extensions. */
934 if (config->flags)
935 return -EINVAL;
936
937 switch (config->tx_type) {
938 case HWTSTAMP_TX_OFF:
939 ice_set_tx_tstamp(pf, false);
940 break;
941 case HWTSTAMP_TX_ON:
942 ice_set_tx_tstamp(pf, true);
943 break;
944 default:
945 return -ERANGE;
946 }
947
948 switch (config->rx_filter) {
949 case HWTSTAMP_FILTER_NONE:
950 ice_set_rx_tstamp(pf, false);
951 break;
952 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
953 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
954 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
955 case HWTSTAMP_FILTER_PTP_V2_EVENT:
956 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
957 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
958 case HWTSTAMP_FILTER_PTP_V2_SYNC:
959 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
960 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
961 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
962 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
963 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
964 case HWTSTAMP_FILTER_NTP_ALL:
965 case HWTSTAMP_FILTER_ALL:
966 config->rx_filter = HWTSTAMP_FILTER_ALL;
967 ice_set_rx_tstamp(pf, true);
968 break;
969 default:
970 return -ERANGE;
971 }
972
973 return 0;
974}
975
976/**
977 * ice_ptp_set_ts_config - ioctl interface to control the timestamping
978 * @pf: Board private structure
979 * @ifr: ioctl data
980 *
981 * Get the user config and store it
982 */
983int ice_ptp_set_ts_config(struct ice_pf *pf, struct ifreq *ifr)
984{
985 struct hwtstamp_config config;
986 int err;
987
988 if (!test_bit(ICE_FLAG_PTP, pf->flags))
989 return -EAGAIN;
990
991 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
992 return -EFAULT;
993
994 err = ice_ptp_set_timestamp_mode(pf, &config);
995 if (err)
996 return err;
997
998 /* Save these settings for future reference */
999 pf->ptp.tstamp_config = config;
1000
1001 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1002 -EFAULT : 0;
1003}
1004
1005/**
1006 * ice_ptp_rx_hwtstamp - Check for an Rx timestamp
1007 * @rx_ring: Ring to get the VSI info
1008 * @rx_desc: Receive descriptor
1009 * @skb: Particular skb to send timestamp with
1010 *
1011 * The driver receives a notification in the receive descriptor with timestamp.
1012 * The timestamp is in ns, so we must convert the result first.
1013 */
1014void
1015ice_ptp_rx_hwtstamp(struct ice_ring *rx_ring,
1016 union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb)
1017{
1018 u32 ts_high;
1019 u64 ts_ns;
1020
1021 /* Populate timesync data into skb */
1022 if (rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID) {
1023 struct skb_shared_hwtstamps *hwtstamps;
1024
1025 /* Use ice_ptp_extend_32b_ts directly, using the ring-specific
1026 * cached PHC value, rather than accessing the PF. This also
1027 * allows us to simply pass the upper 32bits of nanoseconds
1028 * directly. Calling ice_ptp_extend_40b_ts is unnecessary as
1029 * it would just discard these bits itself.
1030 */
1031 ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high);
1032 ts_ns = ice_ptp_extend_32b_ts(rx_ring->cached_phctime, ts_high);
1033
1034 hwtstamps = skb_hwtstamps(skb);
1035 memset(hwtstamps, 0, sizeof(*hwtstamps));
1036 hwtstamps->hwtstamp = ns_to_ktime(ts_ns);
1037 }
1038}
1039
1040/**
1041 * ice_ptp_setup_pins_e810 - Setup PTP pins in sysfs
1042 * @info: PTP clock capabilities
1043 */
1044static void ice_ptp_setup_pins_e810(struct ptp_clock_info *info)
1045{
1046 info->n_per_out = E810_N_PER_OUT;
1047 info->n_ext_ts = E810_N_EXT_TS;
1048}
1049
1050/**
1051 * ice_ptp_set_funcs_e810 - Set specialized functions for E810 support
1052 * @pf: Board private structure
1053 * @info: PTP info to fill
1054 *
1055 * Assign functions to the PTP capabiltiies structure for E810 devices.
1056 * Functions which operate across all device families should be set directly
1057 * in ice_ptp_set_caps. Only add functions here which are distinct for e810
1058 * devices.
1059 */
1060static void
1061ice_ptp_set_funcs_e810(struct ice_pf *pf, struct ptp_clock_info *info)
1062{
1063 info->enable = ice_ptp_gpio_enable_e810;
1064
1065 ice_ptp_setup_pins_e810(info);
1066}
1067
1068/**
1069 * ice_ptp_set_caps - Set PTP capabilities
1070 * @pf: Board private structure
1071 */
1072static void ice_ptp_set_caps(struct ice_pf *pf)
1073{
1074 struct ptp_clock_info *info = &pf->ptp.info;
1075 struct device *dev = ice_pf_to_dev(pf);
1076
1077 snprintf(info->name, sizeof(info->name) - 1, "%s-%s-clk",
1078 dev_driver_string(dev), dev_name(dev));
1079 info->owner = THIS_MODULE;
1080 info->max_adj = 999999999;
1081 info->adjtime = ice_ptp_adjtime;
1082 info->adjfine = ice_ptp_adjfine;
1083 info->gettimex64 = ice_ptp_gettimex64;
1084 info->settime64 = ice_ptp_settime64;
1085
1086 ice_ptp_set_funcs_e810(pf, info);
1087}
1088
1089/**
1090 * ice_ptp_create_clock - Create PTP clock device for userspace
1091 * @pf: Board private structure
1092 *
1093 * This function creates a new PTP clock device. It only creates one if we
1094 * don't already have one. Will return error if it can't create one, but success
1095 * if we already have a device. Should be used by ice_ptp_init to create clock
1096 * initially, and prevent global resets from creating new clock devices.
1097 */
1098static long ice_ptp_create_clock(struct ice_pf *pf)
1099{
1100 struct ptp_clock_info *info;
1101 struct ptp_clock *clock;
1102 struct device *dev;
1103
1104 /* No need to create a clock device if we already have one */
1105 if (pf->ptp.clock)
1106 return 0;
1107
1108 ice_ptp_set_caps(pf);
1109
1110 info = &pf->ptp.info;
1111 dev = ice_pf_to_dev(pf);
1112
1113 /* Allocate memory for kernel pins interface */
1114 if (info->n_pins) {
1115 info->pin_config = devm_kcalloc(dev, info->n_pins,
1116 sizeof(*info->pin_config),
1117 GFP_KERNEL);
1118 if (!info->pin_config) {
1119 info->n_pins = 0;
1120 return -ENOMEM;
1121 }
1122 }
1123
1124 /* Attempt to register the clock before enabling the hardware. */
1125 clock = ptp_clock_register(info, dev);
1126 if (IS_ERR(clock))
1127 return PTR_ERR(clock);
1128
1129 pf->ptp.clock = clock;
1130
1131 return 0;
1132}
1133
1134/**
1135 * ice_ptp_tx_tstamp_work - Process Tx timestamps for a port
1136 * @work: pointer to the kthread_work struct
1137 *
1138 * Process timestamps captured by the PHY associated with this port. To do
1139 * this, loop over each index with a waiting skb.
1140 *
1141 * If a given index has a valid timestamp, perform the following steps:
1142 *
1143 * 1) copy the timestamp out of the PHY register
1144 * 4) clear the timestamp valid bit in the PHY register
1145 * 5) unlock the index by clearing the associated in_use bit.
1146 * 2) extend the 40b timestamp value to get a 64bit timestamp
1147 * 3) send that timestamp to the stack
1148 *
1149 * After looping, if we still have waiting SKBs, then re-queue the work. This
1150 * may cause us effectively poll even when not strictly necessary. We do this
1151 * because it's possible a new timestamp was requested around the same time as
1152 * the interrupt. In some cases hardware might not interrupt us again when the
1153 * timestamp is captured.
1154 *
1155 * Note that we only take the tracking lock when clearing the bit and when
1156 * checking if we need to re-queue this task. The only place where bits can be
1157 * set is the hard xmit routine where an SKB has a request flag set. The only
1158 * places where we clear bits are this work function, or the periodic cleanup
1159 * thread. If the cleanup thread clears a bit we're processing we catch it
1160 * when we lock to clear the bit and then grab the SKB pointer. If a Tx thread
1161 * starts a new timestamp, we might not begin processing it right away but we
1162 * will notice it at the end when we re-queue the work item. If a Tx thread
1163 * starts a new timestamp just after this function exits without re-queuing,
1164 * the interrupt when the timestamp finishes should trigger. Avoiding holding
1165 * the lock for the entire function is important in order to ensure that Tx
1166 * threads do not get blocked while waiting for the lock.
1167 */
1168static void ice_ptp_tx_tstamp_work(struct kthread_work *work)
1169{
1170 struct ice_ptp_port *ptp_port;
1171 struct ice_ptp_tx *tx;
1172 struct ice_pf *pf;
1173 struct ice_hw *hw;
1174 u8 idx;
1175
1176 tx = container_of(work, struct ice_ptp_tx, work);
1177 if (!tx->init)
1178 return;
1179
1180 ptp_port = container_of(tx, struct ice_ptp_port, tx);
1181 pf = ptp_port_to_pf(ptp_port);
1182 hw = &pf->hw;
1183
1184 for_each_set_bit(idx, tx->in_use, tx->len) {
1185 struct skb_shared_hwtstamps shhwtstamps = {};
1186 u8 phy_idx = idx + tx->quad_offset;
1187 u64 raw_tstamp, tstamp;
1188 struct sk_buff *skb;
1189 int err;
1190
1191 err = ice_read_phy_tstamp(hw, tx->quad, phy_idx,
1192 &raw_tstamp);
1193 if (err)
1194 continue;
1195
1196 /* Check if the timestamp is valid */
1197 if (!(raw_tstamp & ICE_PTP_TS_VALID))
1198 continue;
1199
1200 /* clear the timestamp register, so that it won't show valid
1201 * again when re-used.
1202 */
1203 ice_clear_phy_tstamp(hw, tx->quad, phy_idx);
1204
1205 /* The timestamp is valid, so we'll go ahead and clear this
1206 * index and then send the timestamp up to the stack.
1207 */
1208 spin_lock(&tx->lock);
1209 clear_bit(idx, tx->in_use);
1210 skb = tx->tstamps[idx].skb;
1211 tx->tstamps[idx].skb = NULL;
1212 spin_unlock(&tx->lock);
1213
1214 /* it's (unlikely but) possible we raced with the cleanup
1215 * thread for discarding old timestamp requests.
1216 */
1217 if (!skb)
1218 continue;
1219
1220 /* Extend the timestamp using cached PHC time */
1221 tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp);
1222 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
1223
1224 skb_tstamp_tx(skb, &shhwtstamps);
1225 dev_kfree_skb_any(skb);
1226 }
1227
1228 /* Check if we still have work to do. If so, re-queue this task to
1229 * poll for remaining timestamps.
1230 */
1231 spin_lock(&tx->lock);
1232 if (!bitmap_empty(tx->in_use, tx->len))
1233 kthread_queue_work(pf->ptp.kworker, &tx->work);
1234 spin_unlock(&tx->lock);
1235}
1236
1237/**
1238 * ice_ptp_request_ts - Request an available Tx timestamp index
1239 * @tx: the PTP Tx timestamp tracker to request from
1240 * @skb: the SKB to associate with this timestamp request
1241 */
1242s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb)
1243{
1244 u8 idx;
1245
1246 /* Check if this tracker is initialized */
1247 if (!tx->init)
1248 return -1;
1249
1250 spin_lock(&tx->lock);
1251 /* Find and set the first available index */
1252 idx = find_first_zero_bit(tx->in_use, tx->len);
1253 if (idx < tx->len) {
1254 /* We got a valid index that no other thread could have set. Store
1255 * a reference to the skb and the start time to allow discarding old
1256 * requests.
1257 */
1258 set_bit(idx, tx->in_use);
1259 tx->tstamps[idx].start = jiffies;
1260 tx->tstamps[idx].skb = skb_get(skb);
1261 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1262 }
1263
1264 spin_unlock(&tx->lock);
1265
1266 /* return the appropriate PHY timestamp register index, -1 if no
1267 * indexes were available.
1268 */
1269 if (idx >= tx->len)
1270 return -1;
1271 else
1272 return idx + tx->quad_offset;
1273}
1274
1275/**
1276 * ice_ptp_process_ts - Spawn kthread work to handle timestamps
1277 * @pf: Board private structure
1278 *
1279 * Queue work required to process the PTP Tx timestamps outside of interrupt
1280 * context.
1281 */
1282void ice_ptp_process_ts(struct ice_pf *pf)
1283{
1284 if (pf->ptp.port.tx.init)
1285 kthread_queue_work(pf->ptp.kworker, &pf->ptp.port.tx.work);
1286}
1287
1288/**
1289 * ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps
1290 * @tx: Tx tracking structure to initialize
1291 *
1292 * Assumes that the length has already been initialized. Do not call directly,
1293 * use the ice_ptp_init_tx_e822 or ice_ptp_init_tx_e810 instead.
1294 */
1295static int
1296ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx)
1297{
1298 tx->tstamps = kcalloc(tx->len, sizeof(*tx->tstamps), GFP_KERNEL);
1299 if (!tx->tstamps)
1300 return -ENOMEM;
1301
1302 tx->in_use = bitmap_zalloc(tx->len, GFP_KERNEL);
1303 if (!tx->in_use) {
1304 kfree(tx->tstamps);
1305 tx->tstamps = NULL;
1306 return -ENOMEM;
1307 }
1308
1309 spin_lock_init(&tx->lock);
1310 kthread_init_work(&tx->work, ice_ptp_tx_tstamp_work);
1311
1312 tx->init = 1;
1313
1314 return 0;
1315}
1316
1317/**
1318 * ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker
1319 * @pf: Board private structure
1320 * @tx: the tracker to flush
1321 */
1322static void
1323ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
1324{
1325 u8 idx;
1326
1327 for (idx = 0; idx < tx->len; idx++) {
1328 u8 phy_idx = idx + tx->quad_offset;
1329
1330 spin_lock(&tx->lock);
1331 if (tx->tstamps[idx].skb) {
1332 dev_kfree_skb_any(tx->tstamps[idx].skb);
1333 tx->tstamps[idx].skb = NULL;
1334 }
1335 clear_bit(idx, tx->in_use);
1336 spin_unlock(&tx->lock);
1337
1338 /* Clear any potential residual timestamp in the PHY block */
1339 if (!pf->hw.reset_ongoing)
1340 ice_clear_phy_tstamp(&pf->hw, tx->quad, phy_idx);
1341 }
1342}
1343
1344/**
1345 * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker
1346 * @pf: Board private structure
1347 * @tx: Tx tracking structure to release
1348 *
1349 * Free memory associated with the Tx timestamp tracker.
1350 */
1351static void
1352ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
1353{
1354 tx->init = 0;
1355
1356 kthread_cancel_work_sync(&tx->work);
1357
1358 ice_ptp_flush_tx_tracker(pf, tx);
1359
1360 kfree(tx->tstamps);
1361 tx->tstamps = NULL;
1362
1363 kfree(tx->in_use);
1364 tx->in_use = NULL;
1365
1366 tx->len = 0;
1367}
1368
1369/**
1370 * ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps
1371 * @pf: Board private structure
1372 * @tx: the Tx tracking structure to initialize
1373 *
1374 * Initialize the Tx timestamp tracker for this PF. For E810 devices, each
1375 * port has its own block of timestamps, independent of the other ports.
1376 */
1377static int
1378ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)
1379{
1380 tx->quad = pf->hw.port_info->lport;
1381 tx->quad_offset = 0;
1382 tx->len = INDEX_PER_QUAD;
1383
1384 return ice_ptp_alloc_tx_tracker(tx);
1385}
1386
1387/**
1388 * ice_ptp_tx_tstamp_cleanup - Cleanup old timestamp requests that got dropped
1389 * @tx: PTP Tx tracker to clean up
1390 *
1391 * Loop through the Tx timestamp requests and see if any of them have been
1392 * waiting for a long time. Discard any SKBs that have been waiting for more
1393 * than 2 seconds. This is long enough to be reasonably sure that the
1394 * timestamp will never be captured. This might happen if the packet gets
1395 * discarded before it reaches the PHY timestamping block.
1396 */
1397static void ice_ptp_tx_tstamp_cleanup(struct ice_ptp_tx *tx)
1398{
1399 u8 idx;
1400
1401 if (!tx->init)
1402 return;
1403
1404 for_each_set_bit(idx, tx->in_use, tx->len) {
1405 struct sk_buff *skb;
1406
1407 /* Check if this SKB has been waiting for too long */
1408 if (time_is_after_jiffies(tx->tstamps[idx].start + 2 * HZ))
1409 continue;
1410
1411 spin_lock(&tx->lock);
1412 skb = tx->tstamps[idx].skb;
1413 tx->tstamps[idx].skb = NULL;
1414 clear_bit(idx, tx->in_use);
1415 spin_unlock(&tx->lock);
1416
1417 /* Free the SKB after we've cleared the bit */
1418 dev_kfree_skb_any(skb);
1419 }
1420}
1421
1422static void ice_ptp_periodic_work(struct kthread_work *work)
1423{
1424 struct ice_ptp *ptp = container_of(work, struct ice_ptp, work.work);
1425 struct ice_pf *pf = container_of(ptp, struct ice_pf, ptp);
1426
1427 if (!test_bit(ICE_FLAG_PTP, pf->flags))
1428 return;
1429
1430 ice_ptp_update_cached_phctime(pf);
1431
1432 ice_ptp_tx_tstamp_cleanup(&pf->ptp.port.tx);
1433
1434 /* Run twice a second */
1435 kthread_queue_delayed_work(ptp->kworker, &ptp->work,
1436 msecs_to_jiffies(500));
1437}
1438
1439/**
1440 * ice_ptp_init_owner - Initialize PTP_1588_CLOCK device
1441 * @pf: Board private structure
1442 *
1443 * Setup and initialize a PTP clock device that represents the device hardware
1444 * clock. Save the clock index for other functions connected to the same
1445 * hardware resource.
1446 */
1447static int ice_ptp_init_owner(struct ice_pf *pf)
1448{
1449 struct device *dev = ice_pf_to_dev(pf);
1450 struct ice_hw *hw = &pf->hw;
1451 struct timespec64 ts;
1452 u8 src_idx;
1453 int err;
1454
1455 wr32(hw, GLTSYN_SYNC_DLAY, 0);
1456
1457 /* Clear some HW residue and enable source clock */
1458 src_idx = hw->func_caps.ts_func_info.tmr_index_owned;
1459
1460 /* Enable source clocks */
1461 wr32(hw, GLTSYN_ENA(src_idx), GLTSYN_ENA_TSYN_ENA_M);
1462
1463 /* Enable PHY time sync */
1464 err = ice_ptp_init_phy_e810(hw);
1465 if (err)
1466 goto err_exit;
1467
1468 /* Clear event status indications for auxiliary pins */
1469 (void)rd32(hw, GLTSYN_STAT(src_idx));
1470
1471 /* Acquire the global hardware lock */
1472 if (!ice_ptp_lock(hw)) {
1473 err = -EBUSY;
1474 goto err_exit;
1475 }
1476
1477 /* Write the increment time value to PHY and LAN */
1478 err = ice_ptp_write_incval(hw, ICE_PTP_NOMINAL_INCVAL_E810);
1479 if (err) {
1480 ice_ptp_unlock(hw);
1481 goto err_exit;
1482 }
1483
1484 ts = ktime_to_timespec64(ktime_get_real());
1485 /* Write the initial Time value to PHY and LAN */
1486 err = ice_ptp_write_init(pf, &ts);
1487 if (err) {
1488 ice_ptp_unlock(hw);
1489 goto err_exit;
1490 }
1491
1492 /* Release the global hardware lock */
1493 ice_ptp_unlock(hw);
1494
1495 /* Ensure we have a clock device */
1496 err = ice_ptp_create_clock(pf);
1497 if (err)
1498 goto err_clk;
1499
1500 /* Store the PTP clock index for other PFs */
1501 ice_set_ptp_clock_index(pf);
1502
1503 return 0;
1504
1505err_clk:
1506 pf->ptp.clock = NULL;
1507err_exit:
1508 dev_err(dev, "PTP failed to register clock, err %d\n", err);
1509
1510 return err;
1511}
1512
1513/**
1514 * ice_ptp_init - Initialize the PTP support after device probe or reset
1515 * @pf: Board private structure
1516 *
1517 * This function sets device up for PTP support. The first time it is run, it
1518 * will create a clock device. It does not create a clock device if one
1519 * already exists. It also reconfigures the device after a reset.
1520 */
1521void ice_ptp_init(struct ice_pf *pf)
1522{
1523 struct device *dev = ice_pf_to_dev(pf);
1524 struct kthread_worker *kworker;
1525 struct ice_hw *hw = &pf->hw;
1526 int err;
1527
1528 /* PTP is currently only supported on E810 devices */
1529 if (!ice_is_e810(hw))
1530 return;
1531
1532 /* Check if this PF owns the source timer */
1533 if (hw->func_caps.ts_func_info.src_tmr_owned) {
1534 err = ice_ptp_init_owner(pf);
1535 if (err)
1536 return;
1537 }
1538
1539 /* Disable timestamping for both Tx and Rx */
1540 ice_ptp_cfg_timestamp(pf, false);
1541
1542 /* Initialize the PTP port Tx timestamp tracker */
1543 ice_ptp_init_tx_e810(pf, &pf->ptp.port.tx);
1544
1545 /* Initialize work functions */
1546 kthread_init_delayed_work(&pf->ptp.work, ice_ptp_periodic_work);
1547 kthread_init_work(&pf->ptp.extts_work, ice_ptp_extts_work);
1548
1549 /* Allocate a kworker for handling work required for the ports
1550 * connected to the PTP hardware clock.
1551 */
1552 kworker = kthread_create_worker(0, "ice-ptp-%s", dev_name(dev));
1553 if (IS_ERR(kworker)) {
1554 err = PTR_ERR(kworker);
1555 goto err_kworker;
1556 }
1557 pf->ptp.kworker = kworker;
1558
1559 set_bit(ICE_FLAG_PTP, pf->flags);
1560
1561 /* Start periodic work going */
1562 kthread_queue_delayed_work(pf->ptp.kworker, &pf->ptp.work, 0);
1563
1564 dev_info(dev, "PTP init successful\n");
1565 return;
1566
1567err_kworker:
1568 /* If we registered a PTP clock, release it */
1569 if (pf->ptp.clock) {
1570 ptp_clock_unregister(pf->ptp.clock);
1571 pf->ptp.clock = NULL;
1572 }
1573 dev_err(dev, "PTP failed %d\n", err);
1574}
1575
1576/**
1577 * ice_ptp_release - Disable the driver/HW support and unregister the clock
1578 * @pf: Board private structure
1579 *
1580 * This function handles the cleanup work required from the initialization by
1581 * clearing out the important information and unregistering the clock
1582 */
1583void ice_ptp_release(struct ice_pf *pf)
1584{
1585 /* Disable timestamping for both Tx and Rx */
1586 ice_ptp_cfg_timestamp(pf, false);
1587
1588 ice_ptp_release_tx_tracker(pf, &pf->ptp.port.tx);
1589
1590 clear_bit(ICE_FLAG_PTP, pf->flags);
1591
1592 kthread_cancel_delayed_work_sync(&pf->ptp.work);
1593
1594 if (pf->ptp.kworker) {
1595 kthread_destroy_worker(pf->ptp.kworker);
1596 pf->ptp.kworker = NULL;
1597 }
1598
1599 if (!pf->ptp.clock)
1600 return;
1601
1602 /* Disable periodic outputs */
1603 ice_ptp_disable_all_clkout(pf);
1604
1605 ice_clear_ptp_clock_index(pf);
1606 ptp_clock_unregister(pf->ptp.clock);
1607 pf->ptp.clock = NULL;
1608
1609 dev_info(ice_pf_to_dev(pf), "Removed PTP clock\n");
1610}