1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c)  2019 Intel Corporation */
3 
4 #include "igc.h"
5 
6 #include <linux/module.h>
7 #include <linux/device.h>
8 #include <linux/pci.h>
9 #include <linux/ptp_classify.h>
10 #include <linux/clocksource.h>
11 #include <linux/ktime.h>
12 
13 #define INCVALUE_MASK		0x7fffffff
14 #define ISGN			0x80000000
15 
16 #define IGC_SYSTIM_OVERFLOW_PERIOD	(HZ * 60 * 9)
17 #define IGC_PTP_TX_TIMEOUT		(HZ * 15)
18 
19 /* SYSTIM read access for I225 */
20 void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts)
21 {
22 	struct igc_hw *hw = &adapter->hw;
23 	u32 sec, nsec;
24 
25 	/* The timestamp is latched when SYSTIML is read. */
26 	nsec = rd32(IGC_SYSTIML);
27 	sec = rd32(IGC_SYSTIMH);
28 
29 	ts->tv_sec = sec;
30 	ts->tv_nsec = nsec;
31 }
32 
33 static void igc_ptp_write_i225(struct igc_adapter *adapter,
34 			       const struct timespec64 *ts)
35 {
36 	struct igc_hw *hw = &adapter->hw;
37 
38 	wr32(IGC_SYSTIML, ts->tv_nsec);
39 	wr32(IGC_SYSTIMH, ts->tv_sec);
40 }
41 
42 static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm)
43 {
44 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
45 					       ptp_caps);
46 	struct igc_hw *hw = &igc->hw;
47 	int neg_adj = 0;
48 	u64 rate;
49 	u32 inca;
50 
51 	if (scaled_ppm < 0) {
52 		neg_adj = 1;
53 		scaled_ppm = -scaled_ppm;
54 	}
55 	rate = scaled_ppm;
56 	rate <<= 14;
57 	rate = div_u64(rate, 78125);
58 
59 	inca = rate & INCVALUE_MASK;
60 	if (neg_adj)
61 		inca |= ISGN;
62 
63 	wr32(IGC_TIMINCA, inca);
64 
65 	return 0;
66 }
67 
68 static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta)
69 {
70 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
71 					       ptp_caps);
72 	struct timespec64 now, then = ns_to_timespec64(delta);
73 	unsigned long flags;
74 
75 	spin_lock_irqsave(&igc->tmreg_lock, flags);
76 
77 	igc_ptp_read(igc, &now);
78 	now = timespec64_add(now, then);
79 	igc_ptp_write_i225(igc, (const struct timespec64 *)&now);
80 
81 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
82 
83 	return 0;
84 }
85 
86 static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp,
87 				   struct timespec64 *ts,
88 				   struct ptp_system_timestamp *sts)
89 {
90 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
91 					       ptp_caps);
92 	struct igc_hw *hw = &igc->hw;
93 	unsigned long flags;
94 
95 	spin_lock_irqsave(&igc->tmreg_lock, flags);
96 
97 	ptp_read_system_prets(sts);
98 	ts->tv_nsec = rd32(IGC_SYSTIML);
99 	ts->tv_sec = rd32(IGC_SYSTIMH);
100 	ptp_read_system_postts(sts);
101 
102 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
103 
104 	return 0;
105 }
106 
107 static int igc_ptp_settime_i225(struct ptp_clock_info *ptp,
108 				const struct timespec64 *ts)
109 {
110 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
111 					       ptp_caps);
112 	unsigned long flags;
113 
114 	spin_lock_irqsave(&igc->tmreg_lock, flags);
115 
116 	igc_ptp_write_i225(igc, ts);
117 
118 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
119 
120 	return 0;
121 }
122 
123 static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp,
124 				       struct ptp_clock_request *rq, int on)
125 {
126 	return -EOPNOTSUPP;
127 }
128 
129 /**
130  * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp
131  * @adapter: board private structure
132  * @hwtstamps: timestamp structure to update
133  * @systim: unsigned 64bit system time value
134  *
135  * We need to convert the system time value stored in the RX/TXSTMP registers
136  * into a hwtstamp which can be used by the upper level timestamping functions.
137  **/
138 static void igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
139 				       struct skb_shared_hwtstamps *hwtstamps,
140 				       u64 systim)
141 {
142 	switch (adapter->hw.mac.type) {
143 	case igc_i225:
144 		memset(hwtstamps, 0, sizeof(*hwtstamps));
145 		/* Upper 32 bits contain s, lower 32 bits contain ns. */
146 		hwtstamps->hwtstamp = ktime_set(systim >> 32,
147 						systim & 0xFFFFFFFF);
148 		break;
149 	default:
150 		break;
151 	}
152 }
153 
154 /**
155  * igc_ptp_rx_pktstamp - retrieve Rx per packet timestamp
156  * @q_vector: Pointer to interrupt specific structure
157  * @va: Pointer to address containing Rx buffer
158  * @skb: Buffer containing timestamp and packet
159  *
160  * This function is meant to retrieve the first timestamp from the
161  * first buffer of an incoming frame. The value is stored in little
162  * endian format starting on byte 0. There's a second timestamp
163  * starting on byte 8.
164  **/
165 void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va,
166 			 struct sk_buff *skb)
167 {
168 	struct igc_adapter *adapter = q_vector->adapter;
169 	__le64 *regval = (__le64 *)va;
170 	int adjust = 0;
171 
172 	/* The timestamp is recorded in little endian format.
173 	 * DWORD: | 0          | 1           | 2          | 3
174 	 * Field: | Timer0 Low | Timer0 High | Timer1 Low | Timer1 High
175 	 */
176 	igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb),
177 				   le64_to_cpu(regval[0]));
178 
179 	/* adjust timestamp for the RX latency based on link speed */
180 	if (adapter->hw.mac.type == igc_i225) {
181 		switch (adapter->link_speed) {
182 		case SPEED_10:
183 			adjust = IGC_I225_RX_LATENCY_10;
184 			break;
185 		case SPEED_100:
186 			adjust = IGC_I225_RX_LATENCY_100;
187 			break;
188 		case SPEED_1000:
189 			adjust = IGC_I225_RX_LATENCY_1000;
190 			break;
191 		case SPEED_2500:
192 			adjust = IGC_I225_RX_LATENCY_2500;
193 			break;
194 		}
195 	}
196 	skb_hwtstamps(skb)->hwtstamp =
197 		ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
198 }
199 
200 static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter)
201 {
202 	struct igc_hw *hw = &adapter->hw;
203 	u32 val;
204 	int i;
205 
206 	wr32(IGC_TSYNCRXCTL, 0);
207 
208 	for (i = 0; i < adapter->num_rx_queues; i++) {
209 		val = rd32(IGC_SRRCTL(i));
210 		val &= ~IGC_SRRCTL_TIMESTAMP;
211 		wr32(IGC_SRRCTL(i), val);
212 	}
213 
214 	val = rd32(IGC_RXPBS);
215 	val &= ~IGC_RXPBS_CFG_TS_EN;
216 	wr32(IGC_RXPBS, val);
217 }
218 
219 static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
220 {
221 	struct igc_hw *hw = &adapter->hw;
222 	u32 val;
223 	int i;
224 
225 	val = rd32(IGC_RXPBS);
226 	val |= IGC_RXPBS_CFG_TS_EN;
227 	wr32(IGC_RXPBS, val);
228 
229 	for (i = 0; i < adapter->num_rx_queues; i++) {
230 		val = rd32(IGC_SRRCTL(i));
231 		/* FIXME: For now, only support retrieving RX timestamps from
232 		 * timer 0.
233 		 */
234 		val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) |
235 		       IGC_SRRCTL_TIMESTAMP;
236 		wr32(IGC_SRRCTL(i), val);
237 	}
238 
239 	val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL |
240 	      IGC_TSYNCRXCTL_RXSYNSIG;
241 	wr32(IGC_TSYNCRXCTL, val);
242 }
243 
244 static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
245 {
246 	struct igc_hw *hw = &adapter->hw;
247 
248 	wr32(IGC_TSYNCTXCTL, 0);
249 }
250 
251 static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter)
252 {
253 	struct igc_hw *hw = &adapter->hw;
254 
255 	wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG);
256 
257 	/* Read TXSTMP registers to discard any timestamp previously stored. */
258 	rd32(IGC_TXSTMPL);
259 	rd32(IGC_TXSTMPH);
260 }
261 
262 /**
263  * igc_ptp_set_timestamp_mode - setup hardware for timestamping
264  * @adapter: networking device structure
265  * @config: hwtstamp configuration
266  *
267  * Return: 0 in case of success, negative errno code otherwise.
268  */
269 static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
270 				      struct hwtstamp_config *config)
271 {
272 	/* reserved for future extensions */
273 	if (config->flags)
274 		return -EINVAL;
275 
276 	switch (config->tx_type) {
277 	case HWTSTAMP_TX_OFF:
278 		igc_ptp_disable_tx_timestamp(adapter);
279 		break;
280 	case HWTSTAMP_TX_ON:
281 		igc_ptp_enable_tx_timestamp(adapter);
282 		break;
283 	default:
284 		return -ERANGE;
285 	}
286 
287 	switch (config->rx_filter) {
288 	case HWTSTAMP_FILTER_NONE:
289 		igc_ptp_disable_rx_timestamp(adapter);
290 		break;
291 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
292 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
293 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
294 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
295 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
296 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
297 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
298 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
299 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
300 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
301 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
302 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
303 	case HWTSTAMP_FILTER_NTP_ALL:
304 	case HWTSTAMP_FILTER_ALL:
305 		igc_ptp_enable_rx_timestamp(adapter);
306 		config->rx_filter = HWTSTAMP_FILTER_ALL;
307 		break;
308 	default:
309 		return -ERANGE;
310 	}
311 
312 	return 0;
313 }
314 
315 static void igc_ptp_tx_timeout(struct igc_adapter *adapter)
316 {
317 	struct igc_hw *hw = &adapter->hw;
318 
319 	dev_kfree_skb_any(adapter->ptp_tx_skb);
320 	adapter->ptp_tx_skb = NULL;
321 	adapter->tx_hwtstamp_timeouts++;
322 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
323 	/* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */
324 	rd32(IGC_TXSTMPH);
325 	netdev_warn(adapter->netdev, "Tx timestamp timeout\n");
326 }
327 
328 void igc_ptp_tx_hang(struct igc_adapter *adapter)
329 {
330 	bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
331 					      IGC_PTP_TX_TIMEOUT);
332 
333 	if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
334 		return;
335 
336 	/* If we haven't received a timestamp within the timeout, it is
337 	 * reasonable to assume that it will never occur, so we can unlock the
338 	 * timestamp bit when this occurs.
339 	 */
340 	if (timeout) {
341 		cancel_work_sync(&adapter->ptp_tx_work);
342 		igc_ptp_tx_timeout(adapter);
343 	}
344 }
345 
346 /**
347  * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp
348  * @adapter: Board private structure
349  *
350  * If we were asked to do hardware stamping and such a time stamp is
351  * available, then it must have been for this skb here because we only
352  * allow only one such packet into the queue.
353  */
354 static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
355 {
356 	struct sk_buff *skb = adapter->ptp_tx_skb;
357 	struct skb_shared_hwtstamps shhwtstamps;
358 	struct igc_hw *hw = &adapter->hw;
359 	int adjust = 0;
360 	u64 regval;
361 
362 	if (WARN_ON_ONCE(!skb))
363 		return;
364 
365 	regval = rd32(IGC_TXSTMPL);
366 	regval |= (u64)rd32(IGC_TXSTMPH) << 32;
367 	igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
368 
369 	switch (adapter->link_speed) {
370 	case SPEED_10:
371 		adjust = IGC_I225_TX_LATENCY_10;
372 		break;
373 	case SPEED_100:
374 		adjust = IGC_I225_TX_LATENCY_100;
375 		break;
376 	case SPEED_1000:
377 		adjust = IGC_I225_TX_LATENCY_1000;
378 		break;
379 	case SPEED_2500:
380 		adjust = IGC_I225_TX_LATENCY_2500;
381 		break;
382 	}
383 
384 	shhwtstamps.hwtstamp =
385 		ktime_add_ns(shhwtstamps.hwtstamp, adjust);
386 
387 	/* Clear the lock early before calling skb_tstamp_tx so that
388 	 * applications are not woken up before the lock bit is clear. We use
389 	 * a copy of the skb pointer to ensure other threads can't change it
390 	 * while we're notifying the stack.
391 	 */
392 	adapter->ptp_tx_skb = NULL;
393 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
394 
395 	/* Notify the stack and free the skb after we've unlocked */
396 	skb_tstamp_tx(skb, &shhwtstamps);
397 	dev_kfree_skb_any(skb);
398 }
399 
400 /**
401  * igc_ptp_tx_work
402  * @work: pointer to work struct
403  *
404  * This work function polls the TSYNCTXCTL valid bit to determine when a
405  * timestamp has been taken for the current stored skb.
406  */
407 static void igc_ptp_tx_work(struct work_struct *work)
408 {
409 	struct igc_adapter *adapter = container_of(work, struct igc_adapter,
410 						   ptp_tx_work);
411 	struct igc_hw *hw = &adapter->hw;
412 	u32 tsynctxctl;
413 
414 	if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
415 		return;
416 
417 	tsynctxctl = rd32(IGC_TSYNCTXCTL);
418 	if (WARN_ON_ONCE(!(tsynctxctl & IGC_TSYNCTXCTL_TXTT_0)))
419 		return;
420 
421 	igc_ptp_tx_hwtstamp(adapter);
422 }
423 
424 /**
425  * igc_ptp_set_ts_config - set hardware time stamping config
426  * @netdev: network interface device structure
427  * @ifr: interface request data
428  *
429  **/
430 int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
431 {
432 	struct igc_adapter *adapter = netdev_priv(netdev);
433 	struct hwtstamp_config config;
434 	int err;
435 
436 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
437 		return -EFAULT;
438 
439 	err = igc_ptp_set_timestamp_mode(adapter, &config);
440 	if (err)
441 		return err;
442 
443 	/* save these settings for future reference */
444 	memcpy(&adapter->tstamp_config, &config,
445 	       sizeof(adapter->tstamp_config));
446 
447 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
448 		-EFAULT : 0;
449 }
450 
451 /**
452  * igc_ptp_get_ts_config - get hardware time stamping config
453  * @netdev: network interface device structure
454  * @ifr: interface request data
455  *
456  * Get the hwtstamp_config settings to return to the user. Rather than attempt
457  * to deconstruct the settings from the registers, just return a shadow copy
458  * of the last known settings.
459  **/
460 int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
461 {
462 	struct igc_adapter *adapter = netdev_priv(netdev);
463 	struct hwtstamp_config *config = &adapter->tstamp_config;
464 
465 	return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
466 		-EFAULT : 0;
467 }
468 
469 /**
470  * igc_ptp_init - Initialize PTP functionality
471  * @adapter: Board private structure
472  *
473  * This function is called at device probe to initialize the PTP
474  * functionality.
475  */
476 void igc_ptp_init(struct igc_adapter *adapter)
477 {
478 	struct net_device *netdev = adapter->netdev;
479 	struct igc_hw *hw = &adapter->hw;
480 
481 	switch (hw->mac.type) {
482 	case igc_i225:
483 		snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
484 		adapter->ptp_caps.owner = THIS_MODULE;
485 		adapter->ptp_caps.max_adj = 62499999;
486 		adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
487 		adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
488 		adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
489 		adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
490 		adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
491 		break;
492 	default:
493 		adapter->ptp_clock = NULL;
494 		return;
495 	}
496 
497 	spin_lock_init(&adapter->tmreg_lock);
498 	INIT_WORK(&adapter->ptp_tx_work, igc_ptp_tx_work);
499 
500 	adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
501 	adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
502 
503 	adapter->prev_ptp_time = ktime_to_timespec64(ktime_get_real());
504 	adapter->ptp_reset_start = ktime_get();
505 
506 	adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
507 						&adapter->pdev->dev);
508 	if (IS_ERR(adapter->ptp_clock)) {
509 		adapter->ptp_clock = NULL;
510 		netdev_err(netdev, "ptp_clock_register failed\n");
511 	} else if (adapter->ptp_clock) {
512 		netdev_info(netdev, "PHC added\n");
513 		adapter->ptp_flags |= IGC_PTP_ENABLED;
514 	}
515 }
516 
517 static void igc_ptp_time_save(struct igc_adapter *adapter)
518 {
519 	igc_ptp_read(adapter, &adapter->prev_ptp_time);
520 	adapter->ptp_reset_start = ktime_get();
521 }
522 
523 static void igc_ptp_time_restore(struct igc_adapter *adapter)
524 {
525 	struct timespec64 ts = adapter->prev_ptp_time;
526 	ktime_t delta;
527 
528 	delta = ktime_sub(ktime_get(), adapter->ptp_reset_start);
529 
530 	timespec64_add_ns(&ts, ktime_to_ns(delta));
531 
532 	igc_ptp_write_i225(adapter, &ts);
533 }
534 
535 /**
536  * igc_ptp_suspend - Disable PTP work items and prepare for suspend
537  * @adapter: Board private structure
538  *
539  * This function stops the overflow check work and PTP Tx timestamp work, and
540  * will prepare the device for OS suspend.
541  */
542 void igc_ptp_suspend(struct igc_adapter *adapter)
543 {
544 	if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
545 		return;
546 
547 	cancel_work_sync(&adapter->ptp_tx_work);
548 	dev_kfree_skb_any(adapter->ptp_tx_skb);
549 	adapter->ptp_tx_skb = NULL;
550 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
551 
552 	igc_ptp_time_save(adapter);
553 }
554 
555 /**
556  * igc_ptp_stop - Disable PTP device and stop the overflow check.
557  * @adapter: Board private structure.
558  *
559  * This function stops the PTP support and cancels the delayed work.
560  **/
561 void igc_ptp_stop(struct igc_adapter *adapter)
562 {
563 	igc_ptp_suspend(adapter);
564 
565 	if (adapter->ptp_clock) {
566 		ptp_clock_unregister(adapter->ptp_clock);
567 		netdev_info(adapter->netdev, "PHC removed\n");
568 		adapter->ptp_flags &= ~IGC_PTP_ENABLED;
569 	}
570 }
571 
572 /**
573  * igc_ptp_reset - Re-enable the adapter for PTP following a reset.
574  * @adapter: Board private structure.
575  *
576  * This function handles the reset work required to re-enable the PTP device.
577  **/
578 void igc_ptp_reset(struct igc_adapter *adapter)
579 {
580 	struct igc_hw *hw = &adapter->hw;
581 	unsigned long flags;
582 
583 	/* reset the tstamp_config */
584 	igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
585 
586 	spin_lock_irqsave(&adapter->tmreg_lock, flags);
587 
588 	switch (adapter->hw.mac.type) {
589 	case igc_i225:
590 		wr32(IGC_TSAUXC, 0x0);
591 		wr32(IGC_TSSDP, 0x0);
592 		wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS);
593 		wr32(IGC_IMS, IGC_IMS_TS);
594 		break;
595 	default:
596 		/* No work to do. */
597 		goto out;
598 	}
599 
600 	/* Re-initialize the timer. */
601 	if (hw->mac.type == igc_i225) {
602 		igc_ptp_time_restore(adapter);
603 	} else {
604 		timecounter_init(&adapter->tc, &adapter->cc,
605 				 ktime_to_ns(ktime_get_real()));
606 	}
607 out:
608 	spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
609 
610 	wrfl();
611 }
612