1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c)  2019 Intel Corporation */
3 
4 #include "igc.h"
5 
6 #include <linux/module.h>
7 #include <linux/device.h>
8 #include <linux/pci.h>
9 #include <linux/ptp_classify.h>
10 #include <linux/clocksource.h>
11 #include <linux/ktime.h>
12 
13 #define INCVALUE_MASK		0x7fffffff
14 #define ISGN			0x80000000
15 
16 #define IGC_SYSTIM_OVERFLOW_PERIOD	(HZ * 60 * 9)
17 #define IGC_PTP_TX_TIMEOUT		(HZ * 15)
18 
19 /* SYSTIM read access for I225 */
20 void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts)
21 {
22 	struct igc_hw *hw = &adapter->hw;
23 	u32 sec, nsec;
24 
25 	/* The timestamp is latched when SYSTIML is read. */
26 	nsec = rd32(IGC_SYSTIML);
27 	sec = rd32(IGC_SYSTIMH);
28 
29 	ts->tv_sec = sec;
30 	ts->tv_nsec = nsec;
31 }
32 
33 static void igc_ptp_write_i225(struct igc_adapter *adapter,
34 			       const struct timespec64 *ts)
35 {
36 	struct igc_hw *hw = &adapter->hw;
37 
38 	wr32(IGC_SYSTIML, ts->tv_nsec);
39 	wr32(IGC_SYSTIMH, ts->tv_sec);
40 }
41 
42 static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm)
43 {
44 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
45 					       ptp_caps);
46 	struct igc_hw *hw = &igc->hw;
47 	int neg_adj = 0;
48 	u64 rate;
49 	u32 inca;
50 
51 	if (scaled_ppm < 0) {
52 		neg_adj = 1;
53 		scaled_ppm = -scaled_ppm;
54 	}
55 	rate = scaled_ppm;
56 	rate <<= 14;
57 	rate = div_u64(rate, 78125);
58 
59 	inca = rate & INCVALUE_MASK;
60 	if (neg_adj)
61 		inca |= ISGN;
62 
63 	wr32(IGC_TIMINCA, inca);
64 
65 	return 0;
66 }
67 
68 static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta)
69 {
70 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
71 					       ptp_caps);
72 	struct timespec64 now, then = ns_to_timespec64(delta);
73 	unsigned long flags;
74 
75 	spin_lock_irqsave(&igc->tmreg_lock, flags);
76 
77 	igc_ptp_read(igc, &now);
78 	now = timespec64_add(now, then);
79 	igc_ptp_write_i225(igc, (const struct timespec64 *)&now);
80 
81 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
82 
83 	return 0;
84 }
85 
86 static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp,
87 				   struct timespec64 *ts,
88 				   struct ptp_system_timestamp *sts)
89 {
90 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
91 					       ptp_caps);
92 	struct igc_hw *hw = &igc->hw;
93 	unsigned long flags;
94 
95 	spin_lock_irqsave(&igc->tmreg_lock, flags);
96 
97 	ptp_read_system_prets(sts);
98 	ts->tv_nsec = rd32(IGC_SYSTIML);
99 	ts->tv_sec = rd32(IGC_SYSTIMH);
100 	ptp_read_system_postts(sts);
101 
102 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
103 
104 	return 0;
105 }
106 
107 static int igc_ptp_settime_i225(struct ptp_clock_info *ptp,
108 				const struct timespec64 *ts)
109 {
110 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
111 					       ptp_caps);
112 	unsigned long flags;
113 
114 	spin_lock_irqsave(&igc->tmreg_lock, flags);
115 
116 	igc_ptp_write_i225(igc, ts);
117 
118 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
119 
120 	return 0;
121 }
122 
123 static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp,
124 				       struct ptp_clock_request *rq, int on)
125 {
126 	return -EOPNOTSUPP;
127 }
128 
129 /**
130  * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp
131  * @adapter: board private structure
132  * @hwtstamps: timestamp structure to update
133  * @systim: unsigned 64bit system time value
134  *
135  * We need to convert the system time value stored in the RX/TXSTMP registers
136  * into a hwtstamp which can be used by the upper level timestamping functions.
137  **/
138 static void igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
139 				       struct skb_shared_hwtstamps *hwtstamps,
140 				       u64 systim)
141 {
142 	switch (adapter->hw.mac.type) {
143 	case igc_i225:
144 		memset(hwtstamps, 0, sizeof(*hwtstamps));
145 		/* Upper 32 bits contain s, lower 32 bits contain ns. */
146 		hwtstamps->hwtstamp = ktime_set(systim >> 32,
147 						systim & 0xFFFFFFFF);
148 		break;
149 	default:
150 		break;
151 	}
152 }
153 
154 /**
155  * igc_ptp_rx_pktstamp - Retrieve timestamp from Rx packet buffer
156  * @q_vector: Pointer to interrupt specific structure
157  * @va: Pointer to address containing Rx buffer
158  * @skb: Buffer containing timestamp and packet
159  *
160  * This function retrieves the timestamp saved in the beginning of packet
161  * buffer. While two timestamps are available, one in timer0 reference and the
162  * other in timer1 reference, this function considers only the timestamp in
163  * timer0 reference.
164  */
165 void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, __le32 *va,
166 			 struct sk_buff *skb)
167 {
168 	struct igc_adapter *adapter = q_vector->adapter;
169 	u64 regval;
170 	int adjust;
171 
172 	/* Timestamps are saved in little endian at the beginning of the packet
173 	 * buffer following the layout:
174 	 *
175 	 * DWORD: | 0              | 1              | 2              | 3              |
176 	 * Field: | Timer1 SYSTIML | Timer1 SYSTIMH | Timer0 SYSTIML | Timer0 SYSTIMH |
177 	 *
178 	 * SYSTIML holds the nanoseconds part while SYSTIMH holds the seconds
179 	 * part of the timestamp.
180 	 */
181 	regval = le32_to_cpu(va[2]);
182 	regval |= (u64)le32_to_cpu(va[3]) << 32;
183 	igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
184 
185 	/* Adjust timestamp for the RX latency based on link speed */
186 	switch (adapter->link_speed) {
187 	case SPEED_10:
188 		adjust = IGC_I225_RX_LATENCY_10;
189 		break;
190 	case SPEED_100:
191 		adjust = IGC_I225_RX_LATENCY_100;
192 		break;
193 	case SPEED_1000:
194 		adjust = IGC_I225_RX_LATENCY_1000;
195 		break;
196 	case SPEED_2500:
197 		adjust = IGC_I225_RX_LATENCY_2500;
198 		break;
199 	default:
200 		adjust = 0;
201 		netdev_warn_once(adapter->netdev, "Imprecise timestamp\n");
202 		break;
203 	}
204 	skb_hwtstamps(skb)->hwtstamp =
205 		ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
206 }
207 
208 static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter)
209 {
210 	struct igc_hw *hw = &adapter->hw;
211 	u32 val;
212 	int i;
213 
214 	wr32(IGC_TSYNCRXCTL, 0);
215 
216 	for (i = 0; i < adapter->num_rx_queues; i++) {
217 		val = rd32(IGC_SRRCTL(i));
218 		val &= ~IGC_SRRCTL_TIMESTAMP;
219 		wr32(IGC_SRRCTL(i), val);
220 	}
221 
222 	val = rd32(IGC_RXPBS);
223 	val &= ~IGC_RXPBS_CFG_TS_EN;
224 	wr32(IGC_RXPBS, val);
225 }
226 
227 static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
228 {
229 	struct igc_hw *hw = &adapter->hw;
230 	u32 val;
231 	int i;
232 
233 	val = rd32(IGC_RXPBS);
234 	val |= IGC_RXPBS_CFG_TS_EN;
235 	wr32(IGC_RXPBS, val);
236 
237 	for (i = 0; i < adapter->num_rx_queues; i++) {
238 		val = rd32(IGC_SRRCTL(i));
239 		/* FIXME: For now, only support retrieving RX timestamps from
240 		 * timer 0.
241 		 */
242 		val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) |
243 		       IGC_SRRCTL_TIMESTAMP;
244 		wr32(IGC_SRRCTL(i), val);
245 	}
246 
247 	val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL |
248 	      IGC_TSYNCRXCTL_RXSYNSIG;
249 	wr32(IGC_TSYNCRXCTL, val);
250 }
251 
252 static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
253 {
254 	struct igc_hw *hw = &adapter->hw;
255 
256 	wr32(IGC_TSYNCTXCTL, 0);
257 }
258 
259 static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter)
260 {
261 	struct igc_hw *hw = &adapter->hw;
262 
263 	wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG);
264 
265 	/* Read TXSTMP registers to discard any timestamp previously stored. */
266 	rd32(IGC_TXSTMPL);
267 	rd32(IGC_TXSTMPH);
268 }
269 
270 /**
271  * igc_ptp_set_timestamp_mode - setup hardware for timestamping
272  * @adapter: networking device structure
273  * @config: hwtstamp configuration
274  *
275  * Return: 0 in case of success, negative errno code otherwise.
276  */
277 static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
278 				      struct hwtstamp_config *config)
279 {
280 	/* reserved for future extensions */
281 	if (config->flags)
282 		return -EINVAL;
283 
284 	switch (config->tx_type) {
285 	case HWTSTAMP_TX_OFF:
286 		igc_ptp_disable_tx_timestamp(adapter);
287 		break;
288 	case HWTSTAMP_TX_ON:
289 		igc_ptp_enable_tx_timestamp(adapter);
290 		break;
291 	default:
292 		return -ERANGE;
293 	}
294 
295 	switch (config->rx_filter) {
296 	case HWTSTAMP_FILTER_NONE:
297 		igc_ptp_disable_rx_timestamp(adapter);
298 		break;
299 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
300 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
301 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
302 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
303 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
304 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
305 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
306 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
307 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
308 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
309 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
310 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
311 	case HWTSTAMP_FILTER_NTP_ALL:
312 	case HWTSTAMP_FILTER_ALL:
313 		igc_ptp_enable_rx_timestamp(adapter);
314 		config->rx_filter = HWTSTAMP_FILTER_ALL;
315 		break;
316 	default:
317 		return -ERANGE;
318 	}
319 
320 	return 0;
321 }
322 
323 static void igc_ptp_tx_timeout(struct igc_adapter *adapter)
324 {
325 	struct igc_hw *hw = &adapter->hw;
326 
327 	dev_kfree_skb_any(adapter->ptp_tx_skb);
328 	adapter->ptp_tx_skb = NULL;
329 	adapter->tx_hwtstamp_timeouts++;
330 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
331 	/* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */
332 	rd32(IGC_TXSTMPH);
333 	netdev_warn(adapter->netdev, "Tx timestamp timeout\n");
334 }
335 
336 void igc_ptp_tx_hang(struct igc_adapter *adapter)
337 {
338 	bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
339 					      IGC_PTP_TX_TIMEOUT);
340 
341 	if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
342 		return;
343 
344 	/* If we haven't received a timestamp within the timeout, it is
345 	 * reasonable to assume that it will never occur, so we can unlock the
346 	 * timestamp bit when this occurs.
347 	 */
348 	if (timeout) {
349 		cancel_work_sync(&adapter->ptp_tx_work);
350 		igc_ptp_tx_timeout(adapter);
351 	}
352 }
353 
354 /**
355  * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp
356  * @adapter: Board private structure
357  *
358  * If we were asked to do hardware stamping and such a time stamp is
359  * available, then it must have been for this skb here because we only
360  * allow only one such packet into the queue.
361  */
362 static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
363 {
364 	struct sk_buff *skb = adapter->ptp_tx_skb;
365 	struct skb_shared_hwtstamps shhwtstamps;
366 	struct igc_hw *hw = &adapter->hw;
367 	int adjust = 0;
368 	u64 regval;
369 
370 	if (WARN_ON_ONCE(!skb))
371 		return;
372 
373 	regval = rd32(IGC_TXSTMPL);
374 	regval |= (u64)rd32(IGC_TXSTMPH) << 32;
375 	igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
376 
377 	switch (adapter->link_speed) {
378 	case SPEED_10:
379 		adjust = IGC_I225_TX_LATENCY_10;
380 		break;
381 	case SPEED_100:
382 		adjust = IGC_I225_TX_LATENCY_100;
383 		break;
384 	case SPEED_1000:
385 		adjust = IGC_I225_TX_LATENCY_1000;
386 		break;
387 	case SPEED_2500:
388 		adjust = IGC_I225_TX_LATENCY_2500;
389 		break;
390 	}
391 
392 	shhwtstamps.hwtstamp =
393 		ktime_add_ns(shhwtstamps.hwtstamp, adjust);
394 
395 	/* Clear the lock early before calling skb_tstamp_tx so that
396 	 * applications are not woken up before the lock bit is clear. We use
397 	 * a copy of the skb pointer to ensure other threads can't change it
398 	 * while we're notifying the stack.
399 	 */
400 	adapter->ptp_tx_skb = NULL;
401 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
402 
403 	/* Notify the stack and free the skb after we've unlocked */
404 	skb_tstamp_tx(skb, &shhwtstamps);
405 	dev_kfree_skb_any(skb);
406 }
407 
408 /**
409  * igc_ptp_tx_work
410  * @work: pointer to work struct
411  *
412  * This work function polls the TSYNCTXCTL valid bit to determine when a
413  * timestamp has been taken for the current stored skb.
414  */
415 static void igc_ptp_tx_work(struct work_struct *work)
416 {
417 	struct igc_adapter *adapter = container_of(work, struct igc_adapter,
418 						   ptp_tx_work);
419 	struct igc_hw *hw = &adapter->hw;
420 	u32 tsynctxctl;
421 
422 	if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
423 		return;
424 
425 	tsynctxctl = rd32(IGC_TSYNCTXCTL);
426 	if (WARN_ON_ONCE(!(tsynctxctl & IGC_TSYNCTXCTL_TXTT_0)))
427 		return;
428 
429 	igc_ptp_tx_hwtstamp(adapter);
430 }
431 
432 /**
433  * igc_ptp_set_ts_config - set hardware time stamping config
434  * @netdev: network interface device structure
435  * @ifr: interface request data
436  *
437  **/
438 int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
439 {
440 	struct igc_adapter *adapter = netdev_priv(netdev);
441 	struct hwtstamp_config config;
442 	int err;
443 
444 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
445 		return -EFAULT;
446 
447 	err = igc_ptp_set_timestamp_mode(adapter, &config);
448 	if (err)
449 		return err;
450 
451 	/* save these settings for future reference */
452 	memcpy(&adapter->tstamp_config, &config,
453 	       sizeof(adapter->tstamp_config));
454 
455 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
456 		-EFAULT : 0;
457 }
458 
459 /**
460  * igc_ptp_get_ts_config - get hardware time stamping config
461  * @netdev: network interface device structure
462  * @ifr: interface request data
463  *
464  * Get the hwtstamp_config settings to return to the user. Rather than attempt
465  * to deconstruct the settings from the registers, just return a shadow copy
466  * of the last known settings.
467  **/
468 int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
469 {
470 	struct igc_adapter *adapter = netdev_priv(netdev);
471 	struct hwtstamp_config *config = &adapter->tstamp_config;
472 
473 	return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
474 		-EFAULT : 0;
475 }
476 
477 /**
478  * igc_ptp_init - Initialize PTP functionality
479  * @adapter: Board private structure
480  *
481  * This function is called at device probe to initialize the PTP
482  * functionality.
483  */
484 void igc_ptp_init(struct igc_adapter *adapter)
485 {
486 	struct net_device *netdev = adapter->netdev;
487 	struct igc_hw *hw = &adapter->hw;
488 
489 	switch (hw->mac.type) {
490 	case igc_i225:
491 		snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
492 		adapter->ptp_caps.owner = THIS_MODULE;
493 		adapter->ptp_caps.max_adj = 62499999;
494 		adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
495 		adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
496 		adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
497 		adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
498 		adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
499 		break;
500 	default:
501 		adapter->ptp_clock = NULL;
502 		return;
503 	}
504 
505 	spin_lock_init(&adapter->tmreg_lock);
506 	INIT_WORK(&adapter->ptp_tx_work, igc_ptp_tx_work);
507 
508 	adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
509 	adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
510 
511 	adapter->prev_ptp_time = ktime_to_timespec64(ktime_get_real());
512 	adapter->ptp_reset_start = ktime_get();
513 
514 	adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
515 						&adapter->pdev->dev);
516 	if (IS_ERR(adapter->ptp_clock)) {
517 		adapter->ptp_clock = NULL;
518 		netdev_err(netdev, "ptp_clock_register failed\n");
519 	} else if (adapter->ptp_clock) {
520 		netdev_info(netdev, "PHC added\n");
521 		adapter->ptp_flags |= IGC_PTP_ENABLED;
522 	}
523 }
524 
525 static void igc_ptp_time_save(struct igc_adapter *adapter)
526 {
527 	igc_ptp_read(adapter, &adapter->prev_ptp_time);
528 	adapter->ptp_reset_start = ktime_get();
529 }
530 
531 static void igc_ptp_time_restore(struct igc_adapter *adapter)
532 {
533 	struct timespec64 ts = adapter->prev_ptp_time;
534 	ktime_t delta;
535 
536 	delta = ktime_sub(ktime_get(), adapter->ptp_reset_start);
537 
538 	timespec64_add_ns(&ts, ktime_to_ns(delta));
539 
540 	igc_ptp_write_i225(adapter, &ts);
541 }
542 
543 /**
544  * igc_ptp_suspend - Disable PTP work items and prepare for suspend
545  * @adapter: Board private structure
546  *
547  * This function stops the overflow check work and PTP Tx timestamp work, and
548  * will prepare the device for OS suspend.
549  */
550 void igc_ptp_suspend(struct igc_adapter *adapter)
551 {
552 	if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
553 		return;
554 
555 	cancel_work_sync(&adapter->ptp_tx_work);
556 	dev_kfree_skb_any(adapter->ptp_tx_skb);
557 	adapter->ptp_tx_skb = NULL;
558 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
559 
560 	igc_ptp_time_save(adapter);
561 }
562 
563 /**
564  * igc_ptp_stop - Disable PTP device and stop the overflow check.
565  * @adapter: Board private structure.
566  *
567  * This function stops the PTP support and cancels the delayed work.
568  **/
569 void igc_ptp_stop(struct igc_adapter *adapter)
570 {
571 	igc_ptp_suspend(adapter);
572 
573 	if (adapter->ptp_clock) {
574 		ptp_clock_unregister(adapter->ptp_clock);
575 		netdev_info(adapter->netdev, "PHC removed\n");
576 		adapter->ptp_flags &= ~IGC_PTP_ENABLED;
577 	}
578 }
579 
580 /**
581  * igc_ptp_reset - Re-enable the adapter for PTP following a reset.
582  * @adapter: Board private structure.
583  *
584  * This function handles the reset work required to re-enable the PTP device.
585  **/
586 void igc_ptp_reset(struct igc_adapter *adapter)
587 {
588 	struct igc_hw *hw = &adapter->hw;
589 	unsigned long flags;
590 
591 	/* reset the tstamp_config */
592 	igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
593 
594 	spin_lock_irqsave(&adapter->tmreg_lock, flags);
595 
596 	switch (adapter->hw.mac.type) {
597 	case igc_i225:
598 		wr32(IGC_TSAUXC, 0x0);
599 		wr32(IGC_TSSDP, 0x0);
600 		wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS);
601 		wr32(IGC_IMS, IGC_IMS_TS);
602 		break;
603 	default:
604 		/* No work to do. */
605 		goto out;
606 	}
607 
608 	/* Re-initialize the timer. */
609 	if (hw->mac.type == igc_i225) {
610 		igc_ptp_time_restore(adapter);
611 	} else {
612 		timecounter_init(&adapter->tc, &adapter->cc,
613 				 ktime_to_ns(ktime_get_real()));
614 	}
615 out:
616 	spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
617 
618 	wrfl();
619 }
620