1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell PTP driver
3  *
4  * Copyright (C) 2020 Marvell.
5  *
6  */
7 
8 #include <linux/bitfield.h>
9 #include <linux/device.h>
10 #include <linux/module.h>
11 #include <linux/pci.h>
12 #include <linux/hrtimer.h>
13 #include <linux/ktime.h>
14 
15 #include "ptp.h"
16 #include "mbox.h"
17 #include "rvu.h"
18 
19 #define DRV_NAME				"Marvell PTP Driver"
20 
21 #define PCI_DEVID_OCTEONTX2_PTP			0xA00C
22 #define PCI_SUBSYS_DEVID_OCTX2_98xx_PTP		0xB100
23 #define PCI_SUBSYS_DEVID_OCTX2_96XX_PTP		0xB200
24 #define PCI_SUBSYS_DEVID_OCTX2_95XX_PTP		0xB300
25 #define PCI_SUBSYS_DEVID_OCTX2_95XXN_PTP	0xB400
26 #define PCI_SUBSYS_DEVID_OCTX2_95MM_PTP		0xB500
27 #define PCI_SUBSYS_DEVID_OCTX2_95XXO_PTP	0xB600
28 #define PCI_DEVID_OCTEONTX2_RST			0xA085
29 #define PCI_DEVID_CN10K_PTP			0xA09E
30 #define PCI_SUBSYS_DEVID_CN10K_A_PTP		0xB900
31 #define PCI_SUBSYS_DEVID_CNF10K_A_PTP		0xBA00
32 #define PCI_SUBSYS_DEVID_CNF10K_B_PTP		0xBC00
33 
34 #define PCI_PTP_BAR_NO				0
35 
36 #define PTP_CLOCK_CFG				0xF00ULL
37 #define PTP_CLOCK_CFG_PTP_EN			BIT_ULL(0)
38 #define PTP_CLOCK_CFG_EXT_CLK_EN		BIT_ULL(1)
39 #define PTP_CLOCK_CFG_EXT_CLK_IN_MASK		GENMASK_ULL(7, 2)
40 #define PTP_CLOCK_CFG_TSTMP_EDGE		BIT_ULL(9)
41 #define PTP_CLOCK_CFG_TSTMP_EN			BIT_ULL(8)
42 #define PTP_CLOCK_CFG_TSTMP_IN_MASK		GENMASK_ULL(15, 10)
43 #define PTP_CLOCK_CFG_PPS_EN			BIT_ULL(30)
44 #define PTP_CLOCK_CFG_PPS_INV			BIT_ULL(31)
45 
46 #define PTP_PPS_HI_INCR				0xF60ULL
47 #define PTP_PPS_LO_INCR				0xF68ULL
48 #define PTP_PPS_THRESH_HI			0xF58ULL
49 
50 #define PTP_CLOCK_LO				0xF08ULL
51 #define PTP_CLOCK_HI				0xF10ULL
52 #define PTP_CLOCK_COMP				0xF18ULL
53 #define PTP_TIMESTAMP				0xF20ULL
54 #define PTP_CLOCK_SEC				0xFD0ULL
55 #define PTP_SEC_ROLLOVER			0xFD8ULL
56 
57 #define CYCLE_MULT				1000
58 
59 static struct ptp *first_ptp_block;
60 static const struct pci_device_id ptp_id_table[];
61 
62 static bool is_ptp_dev_cnf10kb(struct ptp *ptp)
63 {
64 	return (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_B_PTP) ? true : false;
65 }
66 
67 static bool is_ptp_dev_cn10k(struct ptp *ptp)
68 {
69 	return (ptp->pdev->device == PCI_DEVID_CN10K_PTP) ? true : false;
70 }
71 
72 static bool cn10k_ptp_errata(struct ptp *ptp)
73 {
74 	if (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_A_PTP ||
75 	    ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_A_PTP)
76 		return true;
77 	return false;
78 }
79 
80 static bool is_ptp_tsfmt_sec_nsec(struct ptp *ptp)
81 {
82 	if (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_A_PTP ||
83 	    ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_A_PTP)
84 		return true;
85 	return false;
86 }
87 
88 static enum hrtimer_restart ptp_reset_thresh(struct hrtimer *hrtimer)
89 {
90 	struct ptp *ptp = container_of(hrtimer, struct ptp, hrtimer);
91 	ktime_t curr_ts = ktime_get();
92 	ktime_t delta_ns, period_ns;
93 	u64 ptp_clock_hi;
94 
95 	/* calculate the elapsed time since last restart */
96 	delta_ns = ktime_to_ns(ktime_sub(curr_ts, ptp->last_ts));
97 
98 	/* if the ptp clock value has crossed 0.5 seconds,
99 	 * its too late to update pps threshold value, so
100 	 * update threshold after 1 second.
101 	 */
102 	ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI);
103 	if (ptp_clock_hi > 500000000) {
104 		period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - ptp_clock_hi));
105 	} else {
106 		writeq(500000000, ptp->reg_base + PTP_PPS_THRESH_HI);
107 		period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - delta_ns));
108 	}
109 
110 	hrtimer_forward_now(hrtimer, period_ns);
111 	ptp->last_ts = curr_ts;
112 
113 	return HRTIMER_RESTART;
114 }
115 
116 static void ptp_hrtimer_start(struct ptp *ptp, ktime_t start_ns)
117 {
118 	ktime_t period_ns;
119 
120 	period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - start_ns));
121 	hrtimer_start(&ptp->hrtimer, period_ns, HRTIMER_MODE_REL);
122 	ptp->last_ts = ktime_get();
123 }
124 
125 static u64 read_ptp_tstmp_sec_nsec(struct ptp *ptp)
126 {
127 	u64 sec, sec1, nsec;
128 	unsigned long flags;
129 
130 	spin_lock_irqsave(&ptp->ptp_lock, flags);
131 	sec = readq(ptp->reg_base + PTP_CLOCK_SEC) & 0xFFFFFFFFUL;
132 	nsec = readq(ptp->reg_base + PTP_CLOCK_HI);
133 	sec1 = readq(ptp->reg_base + PTP_CLOCK_SEC) & 0xFFFFFFFFUL;
134 	/* check nsec rollover */
135 	if (sec1 > sec) {
136 		nsec = readq(ptp->reg_base + PTP_CLOCK_HI);
137 		sec = sec1;
138 	}
139 	spin_unlock_irqrestore(&ptp->ptp_lock, flags);
140 
141 	return sec * NSEC_PER_SEC + nsec;
142 }
143 
144 static u64 read_ptp_tstmp_nsec(struct ptp *ptp)
145 {
146 	return readq(ptp->reg_base + PTP_CLOCK_HI);
147 }
148 
149 static u64 ptp_calc_adjusted_comp(u64 ptp_clock_freq)
150 {
151 	u64 comp, adj = 0, cycles_per_sec, ns_drift = 0;
152 	u32 ptp_clock_nsec, cycle_time;
153 	int cycle;
154 
155 	/* Errata:
156 	 * Issue #1: At the time of 1 sec rollover of the nano-second counter,
157 	 * the nano-second counter is set to 0. However, it should be set to
158 	 * (existing counter_value - 10^9).
159 	 *
160 	 * Issue #2: The nano-second counter rolls over at 0x3B9A_C9FF.
161 	 * It should roll over at 0x3B9A_CA00.
162 	 */
163 
164 	/* calculate ptp_clock_comp value */
165 	comp = ((u64)1000000000ULL << 32) / ptp_clock_freq;
166 	/* use CYCLE_MULT to avoid accuracy loss due to integer arithmetic */
167 	cycle_time = NSEC_PER_SEC * CYCLE_MULT / ptp_clock_freq;
168 	/* cycles per sec */
169 	cycles_per_sec = ptp_clock_freq;
170 
171 	/* check whether ptp nanosecond counter rolls over early */
172 	cycle = cycles_per_sec - 1;
173 	ptp_clock_nsec = (cycle * comp) >> 32;
174 	while (ptp_clock_nsec < NSEC_PER_SEC) {
175 		if (ptp_clock_nsec == 0x3B9AC9FF)
176 			goto calc_adj_comp;
177 		cycle++;
178 		ptp_clock_nsec = (cycle * comp) >> 32;
179 	}
180 	/* compute nanoseconds lost per second when nsec counter rolls over */
181 	ns_drift = ptp_clock_nsec - NSEC_PER_SEC;
182 	/* calculate ptp_clock_comp adjustment */
183 	if (ns_drift > 0) {
184 		adj = comp * ns_drift;
185 		adj = adj / 1000000000ULL;
186 	}
187 	/* speed up the ptp clock to account for nanoseconds lost */
188 	comp += adj;
189 	return comp;
190 
191 calc_adj_comp:
192 	/* slow down the ptp clock to not rollover early */
193 	adj = comp * cycle_time;
194 	adj = adj / 1000000000ULL;
195 	adj = adj / CYCLE_MULT;
196 	comp -= adj;
197 
198 	return comp;
199 }
200 
201 struct ptp *ptp_get(void)
202 {
203 	struct ptp *ptp = first_ptp_block;
204 
205 	/* Check PTP block is present in hardware */
206 	if (!pci_dev_present(ptp_id_table))
207 		return ERR_PTR(-ENODEV);
208 	/* Check driver is bound to PTP block */
209 	if (!ptp)
210 		ptp = ERR_PTR(-EPROBE_DEFER);
211 	else if (!IS_ERR(ptp))
212 		pci_dev_get(ptp->pdev);
213 
214 	return ptp;
215 }
216 
217 void ptp_put(struct ptp *ptp)
218 {
219 	if (!ptp)
220 		return;
221 
222 	pci_dev_put(ptp->pdev);
223 }
224 
225 static int ptp_adjfine(struct ptp *ptp, long scaled_ppm)
226 {
227 	bool neg_adj = false;
228 	u32 freq, freq_adj;
229 	u64 comp, adj;
230 	s64 ppb;
231 
232 	if (scaled_ppm < 0) {
233 		neg_adj = true;
234 		scaled_ppm = -scaled_ppm;
235 	}
236 
237 	/* The hardware adds the clock compensation value to the PTP clock
238 	 * on every coprocessor clock cycle. Typical convention is that it
239 	 * represent number of nanosecond betwen each cycle. In this
240 	 * convention compensation value is in 64 bit fixed-point
241 	 * representation where upper 32 bits are number of nanoseconds
242 	 * and lower is fractions of nanosecond.
243 	 * The scaled_ppm represent the ratio in "parts per million" by which
244 	 * the compensation value should be corrected.
245 	 * To calculate new compenstation value we use 64bit fixed point
246 	 * arithmetic on following formula
247 	 * comp = tbase + tbase * scaled_ppm / (1M * 2^16)
248 	 * where tbase is the basic compensation value calculated
249 	 * initialy in the probe function.
250 	 */
251 	/* convert scaled_ppm to ppb */
252 	ppb = 1 + scaled_ppm;
253 	ppb *= 125;
254 	ppb >>= 13;
255 
256 	if (cn10k_ptp_errata(ptp)) {
257 		/* calculate the new frequency based on ppb */
258 		freq_adj = (ptp->clock_rate * ppb) / 1000000000ULL;
259 		freq = neg_adj ? ptp->clock_rate + freq_adj : ptp->clock_rate - freq_adj;
260 		comp = ptp_calc_adjusted_comp(freq);
261 	} else {
262 		comp = ((u64)1000000000ull << 32) / ptp->clock_rate;
263 		adj = comp * ppb;
264 		adj = div_u64(adj, 1000000000ull);
265 		comp = neg_adj ? comp - adj : comp + adj;
266 	}
267 	writeq(comp, ptp->reg_base + PTP_CLOCK_COMP);
268 
269 	return 0;
270 }
271 
272 static int ptp_get_clock(struct ptp *ptp, u64 *clk)
273 {
274 	/* Return the current PTP clock */
275 	*clk = ptp->read_ptp_tstmp(ptp);
276 
277 	return 0;
278 }
279 
280 void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts)
281 {
282 	struct pci_dev *pdev;
283 	u64 clock_comp;
284 	u64 clock_cfg;
285 
286 	if (!ptp)
287 		return;
288 
289 	pdev = ptp->pdev;
290 
291 	if (!sclk) {
292 		dev_err(&pdev->dev, "PTP input clock cannot be zero\n");
293 		return;
294 	}
295 
296 	/* sclk is in MHz */
297 	ptp->clock_rate = sclk * 1000000;
298 
299 	/* Program the seconds rollover value to 1 second */
300 	if (is_ptp_dev_cnf10kb(ptp))
301 		writeq(0x3b9aca00, ptp->reg_base + PTP_SEC_ROLLOVER);
302 
303 	/* Enable PTP clock */
304 	clock_cfg = readq(ptp->reg_base + PTP_CLOCK_CFG);
305 
306 	if (ext_clk_freq) {
307 		ptp->clock_rate = ext_clk_freq;
308 		/* Set GPIO as PTP clock source */
309 		clock_cfg &= ~PTP_CLOCK_CFG_EXT_CLK_IN_MASK;
310 		clock_cfg |= PTP_CLOCK_CFG_EXT_CLK_EN;
311 	}
312 
313 	if (extts) {
314 		clock_cfg |= PTP_CLOCK_CFG_TSTMP_EDGE;
315 		/* Set GPIO as timestamping source */
316 		clock_cfg &= ~PTP_CLOCK_CFG_TSTMP_IN_MASK;
317 		clock_cfg |= PTP_CLOCK_CFG_TSTMP_EN;
318 	}
319 
320 	clock_cfg |= PTP_CLOCK_CFG_PTP_EN;
321 	clock_cfg |= PTP_CLOCK_CFG_PPS_EN | PTP_CLOCK_CFG_PPS_INV;
322 	writeq(clock_cfg, ptp->reg_base + PTP_CLOCK_CFG);
323 
324 	/* Set 50% duty cycle for 1Hz output */
325 	writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_HI_INCR);
326 	writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_LO_INCR);
327 	if (cn10k_ptp_errata(ptp)) {
328 		/* The ptp_clock_hi rollsover to zero once clock cycle before it
329 		 * reaches one second boundary. so, program the pps_lo_incr in
330 		 * such a way that the pps threshold value comparison at one
331 		 * second boundary will succeed and pps edge changes. After each
332 		 * one second boundary, the hrtimer handler will be invoked and
333 		 * reprograms the pps threshold value.
334 		 */
335 		ptp->clock_period = NSEC_PER_SEC / ptp->clock_rate;
336 		writeq((0x1dcd6500ULL - ptp->clock_period) << 32,
337 		       ptp->reg_base + PTP_PPS_LO_INCR);
338 	}
339 
340 	if (cn10k_ptp_errata(ptp))
341 		clock_comp = ptp_calc_adjusted_comp(ptp->clock_rate);
342 	else
343 		clock_comp = ((u64)1000000000ull << 32) / ptp->clock_rate;
344 
345 	/* Initial compensation value to start the nanosecs counter */
346 	writeq(clock_comp, ptp->reg_base + PTP_CLOCK_COMP);
347 }
348 
349 static int ptp_get_tstmp(struct ptp *ptp, u64 *clk)
350 {
351 	u64 timestamp;
352 
353 	if (is_ptp_dev_cn10k(ptp)) {
354 		timestamp = readq(ptp->reg_base + PTP_TIMESTAMP);
355 		*clk = (timestamp >> 32) * NSEC_PER_SEC + (timestamp & 0xFFFFFFFF);
356 	} else {
357 		*clk = readq(ptp->reg_base + PTP_TIMESTAMP);
358 	}
359 
360 	return 0;
361 }
362 
363 static int ptp_set_thresh(struct ptp *ptp, u64 thresh)
364 {
365 	if (!cn10k_ptp_errata(ptp))
366 		writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI);
367 
368 	return 0;
369 }
370 
371 static int ptp_extts_on(struct ptp *ptp, int on)
372 {
373 	u64 ptp_clock_hi;
374 
375 	if (cn10k_ptp_errata(ptp)) {
376 		if (on) {
377 			ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI);
378 			ptp_hrtimer_start(ptp, (ktime_t)ptp_clock_hi);
379 		} else {
380 			if (hrtimer_active(&ptp->hrtimer))
381 				hrtimer_cancel(&ptp->hrtimer);
382 		}
383 	}
384 
385 	return 0;
386 }
387 
388 static int ptp_probe(struct pci_dev *pdev,
389 		     const struct pci_device_id *ent)
390 {
391 	struct ptp *ptp;
392 	int err;
393 
394 	ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
395 	if (!ptp) {
396 		err = -ENOMEM;
397 		goto error;
398 	}
399 
400 	ptp->pdev = pdev;
401 
402 	err = pcim_enable_device(pdev);
403 	if (err)
404 		goto error_free;
405 
406 	err = pcim_iomap_regions(pdev, 1 << PCI_PTP_BAR_NO, pci_name(pdev));
407 	if (err)
408 		goto error_free;
409 
410 	ptp->reg_base = pcim_iomap_table(pdev)[PCI_PTP_BAR_NO];
411 
412 	pci_set_drvdata(pdev, ptp);
413 	if (!first_ptp_block)
414 		first_ptp_block = ptp;
415 
416 	spin_lock_init(&ptp->ptp_lock);
417 	if (is_ptp_tsfmt_sec_nsec(ptp))
418 		ptp->read_ptp_tstmp = &read_ptp_tstmp_sec_nsec;
419 	else
420 		ptp->read_ptp_tstmp = &read_ptp_tstmp_nsec;
421 
422 	if (cn10k_ptp_errata(ptp)) {
423 		hrtimer_init(&ptp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
424 		ptp->hrtimer.function = ptp_reset_thresh;
425 	}
426 
427 	return 0;
428 
429 error_free:
430 	kfree(ptp);
431 
432 error:
433 	/* For `ptp_get()` we need to differentiate between the case
434 	 * when the core has not tried to probe this device and the case when
435 	 * the probe failed.  In the later case we keep the error in
436 	 * `dev->driver_data`.
437 	 */
438 	pci_set_drvdata(pdev, ERR_PTR(err));
439 	if (!first_ptp_block)
440 		first_ptp_block = ERR_PTR(err);
441 
442 	return err;
443 }
444 
445 static void ptp_remove(struct pci_dev *pdev)
446 {
447 	struct ptp *ptp = pci_get_drvdata(pdev);
448 	u64 clock_cfg;
449 
450 	if (IS_ERR_OR_NULL(ptp))
451 		return;
452 
453 	if (cn10k_ptp_errata(ptp) && hrtimer_active(&ptp->hrtimer))
454 		hrtimer_cancel(&ptp->hrtimer);
455 
456 	/* Disable PTP clock */
457 	clock_cfg = readq(ptp->reg_base + PTP_CLOCK_CFG);
458 	clock_cfg &= ~PTP_CLOCK_CFG_PTP_EN;
459 	writeq(clock_cfg, ptp->reg_base + PTP_CLOCK_CFG);
460 	kfree(ptp);
461 }
462 
463 static const struct pci_device_id ptp_id_table[] = {
464 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_PTP,
465 			 PCI_VENDOR_ID_CAVIUM,
466 			 PCI_SUBSYS_DEVID_OCTX2_98xx_PTP) },
467 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_PTP,
468 			 PCI_VENDOR_ID_CAVIUM,
469 			 PCI_SUBSYS_DEVID_OCTX2_96XX_PTP) },
470 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_PTP,
471 			 PCI_VENDOR_ID_CAVIUM,
472 			 PCI_SUBSYS_DEVID_OCTX2_95XX_PTP) },
473 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_PTP,
474 			 PCI_VENDOR_ID_CAVIUM,
475 			 PCI_SUBSYS_DEVID_OCTX2_95XXN_PTP) },
476 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_PTP,
477 			 PCI_VENDOR_ID_CAVIUM,
478 			 PCI_SUBSYS_DEVID_OCTX2_95MM_PTP) },
479 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_PTP,
480 			 PCI_VENDOR_ID_CAVIUM,
481 			 PCI_SUBSYS_DEVID_OCTX2_95XXO_PTP) },
482 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_PTP) },
483 	{ 0, }
484 };
485 
486 struct pci_driver ptp_driver = {
487 	.name = DRV_NAME,
488 	.id_table = ptp_id_table,
489 	.probe = ptp_probe,
490 	.remove = ptp_remove,
491 };
492 
493 int rvu_mbox_handler_ptp_op(struct rvu *rvu, struct ptp_req *req,
494 			    struct ptp_rsp *rsp)
495 {
496 	int err = 0;
497 
498 	/* This function is the PTP mailbox handler invoked when
499 	 * called by AF consumers/netdev drivers via mailbox mechanism.
500 	 * It is used by netdev driver to get the PTP clock and to set
501 	 * frequency adjustments. Since mailbox can be called without
502 	 * notion of whether the driver is bound to ptp device below
503 	 * validation is needed as first step.
504 	 */
505 	if (!rvu->ptp)
506 		return -ENODEV;
507 
508 	switch (req->op) {
509 	case PTP_OP_ADJFINE:
510 		err = ptp_adjfine(rvu->ptp, req->scaled_ppm);
511 		break;
512 	case PTP_OP_GET_CLOCK:
513 		err = ptp_get_clock(rvu->ptp, &rsp->clk);
514 		break;
515 	case PTP_OP_GET_TSTMP:
516 		err = ptp_get_tstmp(rvu->ptp, &rsp->clk);
517 		break;
518 	case PTP_OP_SET_THRESH:
519 		err = ptp_set_thresh(rvu->ptp, req->thresh);
520 		break;
521 	case PTP_OP_EXTTS_ON:
522 		err = ptp_extts_on(rvu->ptp, req->extts_on);
523 		break;
524 	default:
525 		err = -EINVAL;
526 		break;
527 	}
528 
529 	return err;
530 }
531