1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c)  2019 Intel Corporation */
3 
4 #include "igc.h"
5 
6 #include <linux/module.h>
7 #include <linux/device.h>
8 #include <linux/pci.h>
9 #include <linux/ptp_classify.h>
10 #include <linux/clocksource.h>
11 
12 #define INCVALUE_MASK		0x7fffffff
13 #define ISGN			0x80000000
14 
15 #define IGC_SYSTIM_OVERFLOW_PERIOD	(HZ * 60 * 9)
16 #define IGC_PTP_TX_TIMEOUT		(HZ * 15)
17 
18 /* SYSTIM read access for I225 */
19 static void igc_ptp_read_i225(struct igc_adapter *adapter,
20 			      struct timespec64 *ts)
21 {
22 	struct igc_hw *hw = &adapter->hw;
23 	u32 sec, nsec;
24 
25 	/* The timestamp latches on lowest register read. For I210/I211, the
26 	 * lowest register is SYSTIMR. Since we only need to provide nanosecond
27 	 * resolution, we can ignore it.
28 	 */
29 	rd32(IGC_SYSTIMR);
30 	nsec = rd32(IGC_SYSTIML);
31 	sec = rd32(IGC_SYSTIMH);
32 
33 	ts->tv_sec = sec;
34 	ts->tv_nsec = nsec;
35 }
36 
37 static void igc_ptp_write_i225(struct igc_adapter *adapter,
38 			       const struct timespec64 *ts)
39 {
40 	struct igc_hw *hw = &adapter->hw;
41 
42 	/* Writing the SYSTIMR register is not necessary as it only
43 	 * provides sub-nanosecond resolution.
44 	 */
45 	wr32(IGC_SYSTIML, ts->tv_nsec);
46 	wr32(IGC_SYSTIMH, ts->tv_sec);
47 }
48 
49 static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm)
50 {
51 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
52 					       ptp_caps);
53 	struct igc_hw *hw = &igc->hw;
54 	int neg_adj = 0;
55 	u64 rate;
56 	u32 inca;
57 
58 	if (scaled_ppm < 0) {
59 		neg_adj = 1;
60 		scaled_ppm = -scaled_ppm;
61 	}
62 	rate = scaled_ppm;
63 	rate <<= 14;
64 	rate = div_u64(rate, 78125);
65 
66 	inca = rate & INCVALUE_MASK;
67 	if (neg_adj)
68 		inca |= ISGN;
69 
70 	wr32(IGC_TIMINCA, inca);
71 
72 	return 0;
73 }
74 
75 static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta)
76 {
77 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
78 					       ptp_caps);
79 	struct timespec64 now, then = ns_to_timespec64(delta);
80 	unsigned long flags;
81 
82 	spin_lock_irqsave(&igc->tmreg_lock, flags);
83 
84 	igc_ptp_read_i225(igc, &now);
85 	now = timespec64_add(now, then);
86 	igc_ptp_write_i225(igc, (const struct timespec64 *)&now);
87 
88 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
89 
90 	return 0;
91 }
92 
93 static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp,
94 				   struct timespec64 *ts,
95 				   struct ptp_system_timestamp *sts)
96 {
97 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
98 					       ptp_caps);
99 	struct igc_hw *hw = &igc->hw;
100 	unsigned long flags;
101 
102 	spin_lock_irqsave(&igc->tmreg_lock, flags);
103 
104 	ptp_read_system_prets(sts);
105 	rd32(IGC_SYSTIMR);
106 	ptp_read_system_postts(sts);
107 	ts->tv_nsec = rd32(IGC_SYSTIML);
108 	ts->tv_sec = rd32(IGC_SYSTIMH);
109 
110 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
111 
112 	return 0;
113 }
114 
115 static int igc_ptp_settime_i225(struct ptp_clock_info *ptp,
116 				const struct timespec64 *ts)
117 {
118 	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
119 					       ptp_caps);
120 	unsigned long flags;
121 
122 	spin_lock_irqsave(&igc->tmreg_lock, flags);
123 
124 	igc_ptp_write_i225(igc, ts);
125 
126 	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
127 
128 	return 0;
129 }
130 
131 static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp,
132 				       struct ptp_clock_request *rq, int on)
133 {
134 	return -EOPNOTSUPP;
135 }
136 
137 /**
138  * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp
139  * @adapter: board private structure
140  * @hwtstamps: timestamp structure to update
141  * @systim: unsigned 64bit system time value
142  *
143  * We need to convert the system time value stored in the RX/TXSTMP registers
144  * into a hwtstamp which can be used by the upper level timestamping functions.
145  **/
146 static void igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
147 				       struct skb_shared_hwtstamps *hwtstamps,
148 				       u64 systim)
149 {
150 	switch (adapter->hw.mac.type) {
151 	case igc_i225:
152 		memset(hwtstamps, 0, sizeof(*hwtstamps));
153 		/* Upper 32 bits contain s, lower 32 bits contain ns. */
154 		hwtstamps->hwtstamp = ktime_set(systim >> 32,
155 						systim & 0xFFFFFFFF);
156 		break;
157 	default:
158 		break;
159 	}
160 }
161 
162 /**
163  * igc_ptp_rx_pktstamp - retrieve Rx per packet timestamp
164  * @q_vector: Pointer to interrupt specific structure
165  * @va: Pointer to address containing Rx buffer
166  * @skb: Buffer containing timestamp and packet
167  *
168  * This function is meant to retrieve the first timestamp from the
169  * first buffer of an incoming frame. The value is stored in little
170  * endian format starting on byte 0. There's a second timestamp
171  * starting on byte 8.
172  **/
173 void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va,
174 			 struct sk_buff *skb)
175 {
176 	struct igc_adapter *adapter = q_vector->adapter;
177 	__le64 *regval = (__le64 *)va;
178 	int adjust = 0;
179 
180 	/* The timestamp is recorded in little endian format.
181 	 * DWORD: | 0          | 1           | 2          | 3
182 	 * Field: | Timer0 Low | Timer0 High | Timer1 Low | Timer1 High
183 	 */
184 	igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb),
185 				   le64_to_cpu(regval[0]));
186 
187 	/* adjust timestamp for the RX latency based on link speed */
188 	if (adapter->hw.mac.type == igc_i225) {
189 		switch (adapter->link_speed) {
190 		case SPEED_10:
191 			adjust = IGC_I225_RX_LATENCY_10;
192 			break;
193 		case SPEED_100:
194 			adjust = IGC_I225_RX_LATENCY_100;
195 			break;
196 		case SPEED_1000:
197 			adjust = IGC_I225_RX_LATENCY_1000;
198 			break;
199 		case SPEED_2500:
200 			adjust = IGC_I225_RX_LATENCY_2500;
201 			break;
202 		}
203 	}
204 	skb_hwtstamps(skb)->hwtstamp =
205 		ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
206 }
207 
208 static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter)
209 {
210 	struct igc_hw *hw = &adapter->hw;
211 	u32 val;
212 	int i;
213 
214 	wr32(IGC_TSYNCRXCTL, 0);
215 
216 	for (i = 0; i < adapter->num_rx_queues; i++) {
217 		val = rd32(IGC_SRRCTL(i));
218 		val &= ~IGC_SRRCTL_TIMESTAMP;
219 		wr32(IGC_SRRCTL(i), val);
220 	}
221 
222 	val = rd32(IGC_RXPBS);
223 	val &= ~IGC_RXPBS_CFG_TS_EN;
224 	wr32(IGC_RXPBS, val);
225 }
226 
227 static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
228 {
229 	struct igc_hw *hw = &adapter->hw;
230 	u32 val;
231 	int i;
232 
233 	val = rd32(IGC_RXPBS);
234 	val |= IGC_RXPBS_CFG_TS_EN;
235 	wr32(IGC_RXPBS, val);
236 
237 	for (i = 0; i < adapter->num_rx_queues; i++) {
238 		val = rd32(IGC_SRRCTL(i));
239 		/* FIXME: For now, only support retrieving RX timestamps from
240 		 * timer 0.
241 		 */
242 		val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) |
243 		       IGC_SRRCTL_TIMESTAMP;
244 		wr32(IGC_SRRCTL(i), val);
245 	}
246 
247 	val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL |
248 	      IGC_TSYNCRXCTL_RXSYNSIG;
249 	wr32(IGC_TSYNCRXCTL, val);
250 }
251 
252 static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
253 {
254 	struct igc_hw *hw = &adapter->hw;
255 
256 	wr32(IGC_TSYNCTXCTL, 0);
257 }
258 
259 static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter)
260 {
261 	struct igc_hw *hw = &adapter->hw;
262 
263 	wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG);
264 
265 	/* Read TXSTMP registers to discard any timestamp previously stored. */
266 	rd32(IGC_TXSTMPL);
267 	rd32(IGC_TXSTMPH);
268 }
269 
270 /**
271  * igc_ptp_set_timestamp_mode - setup hardware for timestamping
272  * @adapter: networking device structure
273  * @config: hwtstamp configuration
274  *
275  * Return: 0 in case of success, negative errno code otherwise.
276  */
277 static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
278 				      struct hwtstamp_config *config)
279 {
280 	/* reserved for future extensions */
281 	if (config->flags)
282 		return -EINVAL;
283 
284 	switch (config->tx_type) {
285 	case HWTSTAMP_TX_OFF:
286 		igc_ptp_disable_tx_timestamp(adapter);
287 		break;
288 	case HWTSTAMP_TX_ON:
289 		igc_ptp_enable_tx_timestamp(adapter);
290 		break;
291 	default:
292 		return -ERANGE;
293 	}
294 
295 	switch (config->rx_filter) {
296 	case HWTSTAMP_FILTER_NONE:
297 		igc_ptp_disable_rx_timestamp(adapter);
298 		break;
299 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
300 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
301 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
302 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
303 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
304 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
305 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
306 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
307 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
308 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
309 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
310 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
311 	case HWTSTAMP_FILTER_NTP_ALL:
312 	case HWTSTAMP_FILTER_ALL:
313 		igc_ptp_enable_rx_timestamp(adapter);
314 		config->rx_filter = HWTSTAMP_FILTER_ALL;
315 		break;
316 	default:
317 		return -ERANGE;
318 	}
319 
320 	return 0;
321 }
322 
323 static void igc_ptp_tx_timeout(struct igc_adapter *adapter)
324 {
325 	struct igc_hw *hw = &adapter->hw;
326 
327 	dev_kfree_skb_any(adapter->ptp_tx_skb);
328 	adapter->ptp_tx_skb = NULL;
329 	adapter->tx_hwtstamp_timeouts++;
330 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
331 	/* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */
332 	rd32(IGC_TXSTMPH);
333 	netdev_warn(adapter->netdev, "Tx timestamp timeout\n");
334 }
335 
336 void igc_ptp_tx_hang(struct igc_adapter *adapter)
337 {
338 	bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
339 					      IGC_PTP_TX_TIMEOUT);
340 
341 	if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
342 		return;
343 
344 	/* If we haven't received a timestamp within the timeout, it is
345 	 * reasonable to assume that it will never occur, so we can unlock the
346 	 * timestamp bit when this occurs.
347 	 */
348 	if (timeout) {
349 		cancel_work_sync(&adapter->ptp_tx_work);
350 		igc_ptp_tx_timeout(adapter);
351 	}
352 }
353 
354 /**
355  * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp
356  * @adapter: Board private structure
357  *
358  * If we were asked to do hardware stamping and such a time stamp is
359  * available, then it must have been for this skb here because we only
360  * allow only one such packet into the queue.
361  */
362 static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
363 {
364 	struct sk_buff *skb = adapter->ptp_tx_skb;
365 	struct skb_shared_hwtstamps shhwtstamps;
366 	struct igc_hw *hw = &adapter->hw;
367 	u64 regval;
368 
369 	if (WARN_ON_ONCE(!skb))
370 		return;
371 
372 	regval = rd32(IGC_TXSTMPL);
373 	regval |= (u64)rd32(IGC_TXSTMPH) << 32;
374 	igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
375 
376 	/* Clear the lock early before calling skb_tstamp_tx so that
377 	 * applications are not woken up before the lock bit is clear. We use
378 	 * a copy of the skb pointer to ensure other threads can't change it
379 	 * while we're notifying the stack.
380 	 */
381 	adapter->ptp_tx_skb = NULL;
382 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
383 
384 	/* Notify the stack and free the skb after we've unlocked */
385 	skb_tstamp_tx(skb, &shhwtstamps);
386 	dev_kfree_skb_any(skb);
387 }
388 
389 /**
390  * igc_ptp_tx_work
391  * @work: pointer to work struct
392  *
393  * This work function polls the TSYNCTXCTL valid bit to determine when a
394  * timestamp has been taken for the current stored skb.
395  */
396 static void igc_ptp_tx_work(struct work_struct *work)
397 {
398 	struct igc_adapter *adapter = container_of(work, struct igc_adapter,
399 						   ptp_tx_work);
400 	struct igc_hw *hw = &adapter->hw;
401 	u32 tsynctxctl;
402 
403 	if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
404 		return;
405 
406 	if (time_is_before_jiffies(adapter->ptp_tx_start +
407 				   IGC_PTP_TX_TIMEOUT)) {
408 		igc_ptp_tx_timeout(adapter);
409 		return;
410 	}
411 
412 	tsynctxctl = rd32(IGC_TSYNCTXCTL);
413 	if (tsynctxctl & IGC_TSYNCTXCTL_VALID)
414 		igc_ptp_tx_hwtstamp(adapter);
415 	else
416 		/* reschedule to check later */
417 		schedule_work(&adapter->ptp_tx_work);
418 }
419 
420 /**
421  * igc_ptp_set_ts_config - set hardware time stamping config
422  * @netdev: network interface device structure
423  * @ifreq: interface request data
424  *
425  **/
426 int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
427 {
428 	struct igc_adapter *adapter = netdev_priv(netdev);
429 	struct hwtstamp_config config;
430 	int err;
431 
432 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
433 		return -EFAULT;
434 
435 	err = igc_ptp_set_timestamp_mode(adapter, &config);
436 	if (err)
437 		return err;
438 
439 	/* save these settings for future reference */
440 	memcpy(&adapter->tstamp_config, &config,
441 	       sizeof(adapter->tstamp_config));
442 
443 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
444 		-EFAULT : 0;
445 }
446 
447 /**
448  * igc_ptp_get_ts_config - get hardware time stamping config
449  * @netdev: network interface device structure
450  * @ifreq: interface request data
451  *
452  * Get the hwtstamp_config settings to return to the user. Rather than attempt
453  * to deconstruct the settings from the registers, just return a shadow copy
454  * of the last known settings.
455  **/
456 int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
457 {
458 	struct igc_adapter *adapter = netdev_priv(netdev);
459 	struct hwtstamp_config *config = &adapter->tstamp_config;
460 
461 	return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
462 		-EFAULT : 0;
463 }
464 
465 /**
466  * igc_ptp_init - Initialize PTP functionality
467  * @adapter: Board private structure
468  *
469  * This function is called at device probe to initialize the PTP
470  * functionality.
471  */
472 void igc_ptp_init(struct igc_adapter *adapter)
473 {
474 	struct net_device *netdev = adapter->netdev;
475 	struct igc_hw *hw = &adapter->hw;
476 
477 	switch (hw->mac.type) {
478 	case igc_i225:
479 		snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
480 		adapter->ptp_caps.owner = THIS_MODULE;
481 		adapter->ptp_caps.max_adj = 62499999;
482 		adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
483 		adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
484 		adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
485 		adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
486 		adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
487 		break;
488 	default:
489 		adapter->ptp_clock = NULL;
490 		return;
491 	}
492 
493 	spin_lock_init(&adapter->tmreg_lock);
494 	INIT_WORK(&adapter->ptp_tx_work, igc_ptp_tx_work);
495 
496 	adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
497 	adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
498 
499 	igc_ptp_reset(adapter);
500 
501 	adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
502 						&adapter->pdev->dev);
503 	if (IS_ERR(adapter->ptp_clock)) {
504 		adapter->ptp_clock = NULL;
505 		netdev_err(netdev, "ptp_clock_register failed\n");
506 	} else if (adapter->ptp_clock) {
507 		netdev_info(netdev, "PHC added\n");
508 		adapter->ptp_flags |= IGC_PTP_ENABLED;
509 	}
510 }
511 
512 /**
513  * igc_ptp_suspend - Disable PTP work items and prepare for suspend
514  * @adapter: Board private structure
515  *
516  * This function stops the overflow check work and PTP Tx timestamp work, and
517  * will prepare the device for OS suspend.
518  */
519 void igc_ptp_suspend(struct igc_adapter *adapter)
520 {
521 	if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
522 		return;
523 
524 	cancel_work_sync(&adapter->ptp_tx_work);
525 	dev_kfree_skb_any(adapter->ptp_tx_skb);
526 	adapter->ptp_tx_skb = NULL;
527 	clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
528 }
529 
530 /**
531  * igc_ptp_stop - Disable PTP device and stop the overflow check.
532  * @adapter: Board private structure.
533  *
534  * This function stops the PTP support and cancels the delayed work.
535  **/
536 void igc_ptp_stop(struct igc_adapter *adapter)
537 {
538 	igc_ptp_suspend(adapter);
539 
540 	if (adapter->ptp_clock) {
541 		ptp_clock_unregister(adapter->ptp_clock);
542 		netdev_info(adapter->netdev, "PHC removed\n");
543 		adapter->ptp_flags &= ~IGC_PTP_ENABLED;
544 	}
545 }
546 
547 /**
548  * igc_ptp_reset - Re-enable the adapter for PTP following a reset.
549  * @adapter: Board private structure.
550  *
551  * This function handles the reset work required to re-enable the PTP device.
552  **/
553 void igc_ptp_reset(struct igc_adapter *adapter)
554 {
555 	struct igc_hw *hw = &adapter->hw;
556 	unsigned long flags;
557 
558 	/* reset the tstamp_config */
559 	igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
560 
561 	spin_lock_irqsave(&adapter->tmreg_lock, flags);
562 
563 	switch (adapter->hw.mac.type) {
564 	case igc_i225:
565 		wr32(IGC_TSAUXC, 0x0);
566 		wr32(IGC_TSSDP, 0x0);
567 		wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS);
568 		wr32(IGC_IMS, IGC_IMS_TS);
569 		break;
570 	default:
571 		/* No work to do. */
572 		goto out;
573 	}
574 
575 	/* Re-initialize the timer. */
576 	if (hw->mac.type == igc_i225) {
577 		struct timespec64 ts64 = ktime_to_timespec64(ktime_get_real());
578 
579 		igc_ptp_write_i225(adapter, &ts64);
580 	} else {
581 		timecounter_init(&adapter->tc, &adapter->cc,
582 				 ktime_to_ns(ktime_get_real()));
583 	}
584 out:
585 	spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
586 
587 	wrfl();
588 }
589