1 // SPDX-License-Identifier: GPL-2.0+
2 #include <linux/clk.h>
3 #include <linux/clocksource.h>
4 #include <linux/clockchips.h>
5 #include <linux/cpuhotplug.h>
6 #include <linux/interrupt.h>
7 #include <linux/io.h>
8 #include <linux/iopoll.h>
9 #include <linux/err.h>
10 #include <linux/of.h>
11 #include <linux/of_address.h>
12 #include <linux/of_irq.h>
13 #include <linux/sched_clock.h>
14 
15 #include <linux/clk/clk-conf.h>
16 
17 #include <clocksource/timer-ti-dm.h>
18 #include <dt-bindings/bus/ti-sysc.h>
19 
20 /* For type1, set SYSC_OMAP2_CLOCKACTIVITY for fck off on idle, l4 clock on */
21 #define DMTIMER_TYPE1_ENABLE	((1 << 9) | (SYSC_IDLE_SMART << 3) | \
22 				 SYSC_OMAP2_ENAWAKEUP | SYSC_OMAP2_AUTOIDLE)
23 #define DMTIMER_TYPE1_DISABLE	(SYSC_OMAP2_SOFTRESET | SYSC_OMAP2_AUTOIDLE)
24 #define DMTIMER_TYPE2_ENABLE	(SYSC_IDLE_SMART_WKUP << 2)
25 #define DMTIMER_RESET_WAIT	100000
26 
27 #define DMTIMER_INST_DONT_CARE	~0U
28 
29 static int counter_32k;
30 static u32 clocksource;
31 static u32 clockevent;
32 
33 /*
34  * Subset of the timer registers we use. Note that the register offsets
35  * depend on the timer revision detected.
36  */
37 struct dmtimer_systimer {
38 	void __iomem *base;
39 	u8 sysc;
40 	u8 irq_stat;
41 	u8 irq_ena;
42 	u8 pend;
43 	u8 load;
44 	u8 counter;
45 	u8 ctrl;
46 	u8 wakeup;
47 	u8 ifctrl;
48 	struct clk *fck;
49 	struct clk *ick;
50 	unsigned long rate;
51 };
52 
53 struct dmtimer_clockevent {
54 	struct clock_event_device dev;
55 	struct dmtimer_systimer t;
56 	u32 period;
57 };
58 
59 struct dmtimer_clocksource {
60 	struct clocksource dev;
61 	struct dmtimer_systimer t;
62 	unsigned int loadval;
63 };
64 
65 /* Assumes v1 ip if bits [31:16] are zero */
66 static bool dmtimer_systimer_revision1(struct dmtimer_systimer *t)
67 {
68 	u32 tidr = readl_relaxed(t->base);
69 
70 	return !(tidr >> 16);
71 }
72 
73 static void dmtimer_systimer_enable(struct dmtimer_systimer *t)
74 {
75 	u32 val;
76 
77 	if (dmtimer_systimer_revision1(t))
78 		val = DMTIMER_TYPE1_ENABLE;
79 	else
80 		val = DMTIMER_TYPE2_ENABLE;
81 
82 	writel_relaxed(val, t->base + t->sysc);
83 }
84 
85 static void dmtimer_systimer_disable(struct dmtimer_systimer *t)
86 {
87 	if (!dmtimer_systimer_revision1(t))
88 		return;
89 
90 	writel_relaxed(DMTIMER_TYPE1_DISABLE, t->base + t->sysc);
91 }
92 
93 static int __init dmtimer_systimer_type1_reset(struct dmtimer_systimer *t)
94 {
95 	void __iomem *syss = t->base + OMAP_TIMER_V1_SYS_STAT_OFFSET;
96 	int ret;
97 	u32 l;
98 
99 	dmtimer_systimer_enable(t);
100 	writel_relaxed(BIT(1) | BIT(2), t->base + t->ifctrl);
101 	ret = readl_poll_timeout_atomic(syss, l, l & BIT(0), 100,
102 					DMTIMER_RESET_WAIT);
103 
104 	return ret;
105 }
106 
107 /* Note we must use io_base instead of func_base for type2 OCP regs */
108 static int __init dmtimer_systimer_type2_reset(struct dmtimer_systimer *t)
109 {
110 	void __iomem *sysc = t->base + t->sysc;
111 	u32 l;
112 
113 	dmtimer_systimer_enable(t);
114 	l = readl_relaxed(sysc);
115 	l |= BIT(0);
116 	writel_relaxed(l, sysc);
117 
118 	return readl_poll_timeout_atomic(sysc, l, !(l & BIT(0)), 100,
119 					 DMTIMER_RESET_WAIT);
120 }
121 
122 static int __init dmtimer_systimer_reset(struct dmtimer_systimer *t)
123 {
124 	int ret;
125 
126 	if (dmtimer_systimer_revision1(t))
127 		ret = dmtimer_systimer_type1_reset(t);
128 	else
129 		ret = dmtimer_systimer_type2_reset(t);
130 	if (ret < 0) {
131 		pr_err("%s failed with %i\n", __func__, ret);
132 
133 		return ret;
134 	}
135 
136 	return 0;
137 }
138 
139 static const struct of_device_id counter_match_table[] = {
140 	{ .compatible = "ti,omap-counter32k" },
141 	{ /* Sentinel */ },
142 };
143 
144 /*
145  * Check if the SoC als has a usable working 32 KiHz counter. The 32 KiHz
146  * counter is handled by timer-ti-32k, but we need to detect it as it
147  * affects the preferred dmtimer system timer configuration. There is
148  * typically no use for a dmtimer clocksource if the 32 KiHz counter is
149  * present, except on am437x as described below.
150  */
151 static void __init dmtimer_systimer_check_counter32k(void)
152 {
153 	struct device_node *np;
154 
155 	if (counter_32k)
156 		return;
157 
158 	np = of_find_matching_node(NULL, counter_match_table);
159 	if (!np) {
160 		counter_32k = -ENODEV;
161 
162 		return;
163 	}
164 
165 	if (of_device_is_available(np))
166 		counter_32k = 1;
167 	else
168 		counter_32k = -ENODEV;
169 
170 	of_node_put(np);
171 }
172 
173 static const struct of_device_id dmtimer_match_table[] = {
174 	{ .compatible = "ti,omap2420-timer", },
175 	{ .compatible = "ti,omap3430-timer", },
176 	{ .compatible = "ti,omap4430-timer", },
177 	{ .compatible = "ti,omap5430-timer", },
178 	{ .compatible = "ti,am335x-timer", },
179 	{ .compatible = "ti,am335x-timer-1ms", },
180 	{ .compatible = "ti,dm814-timer", },
181 	{ .compatible = "ti,dm816-timer", },
182 	{ /* Sentinel */ },
183 };
184 
185 /*
186  * Checks that system timers are configured to not reset and idle during
187  * the generic timer-ti-dm device driver probe. And that the system timer
188  * source clocks are properly configured. Also, let's not hog any DSP and
189  * PWM capable timers unnecessarily as system timers.
190  */
191 static bool __init dmtimer_is_preferred(struct device_node *np)
192 {
193 	if (!of_device_is_available(np))
194 		return false;
195 
196 	if (!of_property_read_bool(np->parent,
197 				   "ti,no-reset-on-init"))
198 		return false;
199 
200 	if (!of_property_read_bool(np->parent, "ti,no-idle"))
201 		return false;
202 
203 	/* Secure gptimer12 is always clocked with a fixed source */
204 	if (!of_property_read_bool(np, "ti,timer-secure")) {
205 		if (!of_property_read_bool(np, "assigned-clocks"))
206 			return false;
207 
208 		if (!of_property_read_bool(np, "assigned-clock-parents"))
209 			return false;
210 	}
211 
212 	if (of_property_read_bool(np, "ti,timer-dsp"))
213 		return false;
214 
215 	if (of_property_read_bool(np, "ti,timer-pwm"))
216 		return false;
217 
218 	return true;
219 }
220 
221 /*
222  * Finds the first available usable always-on timer, and assigns it to either
223  * clockevent or clocksource depending if the counter_32k is available on the
224  * SoC or not.
225  *
226  * Some omap3 boards with unreliable oscillator must not use the counter_32k
227  * or dmtimer1 with 32 KiHz source. Additionally, the boards with unreliable
228  * oscillator should really set counter_32k as disabled, and delete dmtimer1
229  * ti,always-on property, but let's not count on it. For these quirky cases,
230  * we prefer using the always-on secure dmtimer12 with the internal 32 KiHz
231  * clock as the clocksource, and any available dmtimer as clockevent.
232  *
233  * For am437x, we are using am335x style dmtimer clocksource. It is unclear
234  * if this quirk handling is really needed, but let's change it separately
235  * based on testing as it might cause side effects.
236  */
237 static void __init dmtimer_systimer_assign_alwon(void)
238 {
239 	struct device_node *np;
240 	u32 pa = 0;
241 	bool quirk_unreliable_oscillator = false;
242 
243 	/* Quirk unreliable 32 KiHz oscillator with incomplete dts */
244 	if (of_machine_is_compatible("ti,omap3-beagle-ab4") ||
245 	    of_machine_is_compatible("timll,omap3-devkit8000")) {
246 		quirk_unreliable_oscillator = true;
247 		counter_32k = -ENODEV;
248 	}
249 
250 	/* Quirk am437x using am335x style dmtimer clocksource */
251 	if (of_machine_is_compatible("ti,am43"))
252 		counter_32k = -ENODEV;
253 
254 	for_each_matching_node(np, dmtimer_match_table) {
255 		if (!dmtimer_is_preferred(np))
256 			continue;
257 
258 		if (of_property_read_bool(np, "ti,timer-alwon")) {
259 			const __be32 *addr;
260 
261 			addr = of_get_address(np, 0, NULL, NULL);
262 			pa = of_translate_address(np, addr);
263 			if (pa) {
264 				/* Quirky omap3 boards must use dmtimer12 */
265 				if (quirk_unreliable_oscillator &&
266 				    pa == 0x48318000)
267 					continue;
268 
269 				of_node_put(np);
270 				break;
271 			}
272 		}
273 	}
274 
275 	/* Usually no need for dmtimer clocksource if we have counter32 */
276 	if (counter_32k >= 0) {
277 		clockevent = pa;
278 		clocksource = 0;
279 	} else {
280 		clocksource = pa;
281 		clockevent = DMTIMER_INST_DONT_CARE;
282 	}
283 }
284 
285 /* Finds the first usable dmtimer, used for the don't care case */
286 static u32 __init dmtimer_systimer_find_first_available(void)
287 {
288 	struct device_node *np;
289 	const __be32 *addr;
290 	u32 pa = 0;
291 
292 	for_each_matching_node(np, dmtimer_match_table) {
293 		if (!dmtimer_is_preferred(np))
294 			continue;
295 
296 		addr = of_get_address(np, 0, NULL, NULL);
297 		pa = of_translate_address(np, addr);
298 		if (pa) {
299 			if (pa == clocksource || pa == clockevent) {
300 				pa = 0;
301 				continue;
302 			}
303 
304 			of_node_put(np);
305 			break;
306 		}
307 	}
308 
309 	return pa;
310 }
311 
312 /* Selects the best clocksource and clockevent to use */
313 static void __init dmtimer_systimer_select_best(void)
314 {
315 	dmtimer_systimer_check_counter32k();
316 	dmtimer_systimer_assign_alwon();
317 
318 	if (clockevent == DMTIMER_INST_DONT_CARE)
319 		clockevent = dmtimer_systimer_find_first_available();
320 
321 	pr_debug("%s: counter_32k: %i clocksource: %08x clockevent: %08x\n",
322 		 __func__, counter_32k, clocksource, clockevent);
323 }
324 
325 /* Interface clocks are only available on some SoCs variants */
326 static int __init dmtimer_systimer_init_clock(struct dmtimer_systimer *t,
327 					      struct device_node *np,
328 					      const char *name,
329 					      unsigned long *rate)
330 {
331 	struct clk *clock;
332 	unsigned long r;
333 	bool is_ick = false;
334 	int error;
335 
336 	is_ick = !strncmp(name, "ick", 3);
337 
338 	clock = of_clk_get_by_name(np, name);
339 	if ((PTR_ERR(clock) == -EINVAL) && is_ick)
340 		return 0;
341 	else if (IS_ERR(clock))
342 		return PTR_ERR(clock);
343 
344 	error = clk_prepare_enable(clock);
345 	if (error)
346 		return error;
347 
348 	r = clk_get_rate(clock);
349 	if (!r)
350 		return -ENODEV;
351 
352 	if (is_ick)
353 		t->ick = clock;
354 	else
355 		t->fck = clock;
356 
357 	*rate = r;
358 
359 	return 0;
360 }
361 
362 static int __init dmtimer_systimer_setup(struct device_node *np,
363 					 struct dmtimer_systimer *t)
364 {
365 	unsigned long rate;
366 	u8 regbase;
367 	int error;
368 
369 	if (!of_device_is_compatible(np->parent, "ti,sysc"))
370 		return -EINVAL;
371 
372 	t->base = of_iomap(np, 0);
373 	if (!t->base)
374 		return -ENXIO;
375 
376 	/*
377 	 * Enable optional assigned-clock-parents configured at the timer
378 	 * node level. For regular device drivers, this is done automatically
379 	 * by bus related code such as platform_drv_probe().
380 	 */
381 	error = of_clk_set_defaults(np, false);
382 	if (error < 0)
383 		pr_err("%s: clock source init failed: %i\n", __func__, error);
384 
385 	/* For ti-sysc, we have timer clocks at the parent module level */
386 	error = dmtimer_systimer_init_clock(t, np->parent, "fck", &rate);
387 	if (error)
388 		goto err_unmap;
389 
390 	t->rate = rate;
391 
392 	error = dmtimer_systimer_init_clock(t, np->parent, "ick", &rate);
393 	if (error)
394 		goto err_unmap;
395 
396 	if (dmtimer_systimer_revision1(t)) {
397 		t->irq_stat = OMAP_TIMER_V1_STAT_OFFSET;
398 		t->irq_ena = OMAP_TIMER_V1_INT_EN_OFFSET;
399 		t->pend = _OMAP_TIMER_WRITE_PEND_OFFSET;
400 		regbase = 0;
401 	} else {
402 		t->irq_stat = OMAP_TIMER_V2_IRQSTATUS;
403 		t->irq_ena = OMAP_TIMER_V2_IRQENABLE_SET;
404 		regbase = OMAP_TIMER_V2_FUNC_OFFSET;
405 		t->pend = regbase + _OMAP_TIMER_WRITE_PEND_OFFSET;
406 	}
407 
408 	t->sysc = OMAP_TIMER_OCP_CFG_OFFSET;
409 	t->load = regbase + _OMAP_TIMER_LOAD_OFFSET;
410 	t->counter = regbase + _OMAP_TIMER_COUNTER_OFFSET;
411 	t->ctrl = regbase + _OMAP_TIMER_CTRL_OFFSET;
412 	t->wakeup = regbase + _OMAP_TIMER_WAKEUP_EN_OFFSET;
413 	t->ifctrl = regbase + _OMAP_TIMER_IF_CTRL_OFFSET;
414 
415 	dmtimer_systimer_reset(t);
416 	dmtimer_systimer_enable(t);
417 	pr_debug("dmtimer rev %08x sysc %08x\n", readl_relaxed(t->base),
418 		 readl_relaxed(t->base + t->sysc));
419 
420 	return 0;
421 
422 err_unmap:
423 	iounmap(t->base);
424 
425 	return error;
426 }
427 
428 /* Clockevent */
429 static struct dmtimer_clockevent *
430 to_dmtimer_clockevent(struct clock_event_device *clockevent)
431 {
432 	return container_of(clockevent, struct dmtimer_clockevent, dev);
433 }
434 
435 static irqreturn_t dmtimer_clockevent_interrupt(int irq, void *data)
436 {
437 	struct dmtimer_clockevent *clkevt = data;
438 	struct dmtimer_systimer *t = &clkevt->t;
439 
440 	writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_stat);
441 	clkevt->dev.event_handler(&clkevt->dev);
442 
443 	return IRQ_HANDLED;
444 }
445 
446 static int dmtimer_set_next_event(unsigned long cycles,
447 				  struct clock_event_device *evt)
448 {
449 	struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt);
450 	struct dmtimer_systimer *t = &clkevt->t;
451 	void __iomem *pend = t->base + t->pend;
452 
453 	while (readl_relaxed(pend) & WP_TCRR)
454 		cpu_relax();
455 	writel_relaxed(0xffffffff - cycles, t->base + t->counter);
456 
457 	while (readl_relaxed(pend) & WP_TCLR)
458 		cpu_relax();
459 	writel_relaxed(OMAP_TIMER_CTRL_ST, t->base + t->ctrl);
460 
461 	return 0;
462 }
463 
464 static int dmtimer_clockevent_shutdown(struct clock_event_device *evt)
465 {
466 	struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt);
467 	struct dmtimer_systimer *t = &clkevt->t;
468 	void __iomem *ctrl = t->base + t->ctrl;
469 	u32 l;
470 
471 	l = readl_relaxed(ctrl);
472 	if (l & OMAP_TIMER_CTRL_ST) {
473 		l &= ~BIT(0);
474 		writel_relaxed(l, ctrl);
475 		/* Flush posted write */
476 		l = readl_relaxed(ctrl);
477 		/*  Wait for functional clock period x 3.5 */
478 		udelay(3500000 / t->rate + 1);
479 	}
480 	writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_stat);
481 
482 	return 0;
483 }
484 
485 static int dmtimer_set_periodic(struct clock_event_device *evt)
486 {
487 	struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt);
488 	struct dmtimer_systimer *t = &clkevt->t;
489 	void __iomem *pend = t->base + t->pend;
490 
491 	dmtimer_clockevent_shutdown(evt);
492 
493 	/* Looks like we need to first set the load value separately */
494 	while (readl_relaxed(pend) & WP_TLDR)
495 		cpu_relax();
496 	writel_relaxed(clkevt->period, t->base + t->load);
497 
498 	while (readl_relaxed(pend) & WP_TCRR)
499 		cpu_relax();
500 	writel_relaxed(clkevt->period, t->base + t->counter);
501 
502 	while (readl_relaxed(pend) & WP_TCLR)
503 		cpu_relax();
504 	writel_relaxed(OMAP_TIMER_CTRL_AR | OMAP_TIMER_CTRL_ST,
505 		       t->base + t->ctrl);
506 
507 	return 0;
508 }
509 
510 static void omap_clockevent_idle(struct clock_event_device *evt)
511 {
512 	struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt);
513 	struct dmtimer_systimer *t = &clkevt->t;
514 
515 	dmtimer_systimer_disable(t);
516 	clk_disable(t->fck);
517 }
518 
519 static void omap_clockevent_unidle(struct clock_event_device *evt)
520 {
521 	struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt);
522 	struct dmtimer_systimer *t = &clkevt->t;
523 	int error;
524 
525 	error = clk_enable(t->fck);
526 	if (error)
527 		pr_err("could not enable timer fck on resume: %i\n", error);
528 
529 	dmtimer_systimer_enable(t);
530 	writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena);
531 	writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup);
532 }
533 
534 static int __init dmtimer_clkevt_init_common(struct dmtimer_clockevent *clkevt,
535 					     struct device_node *np,
536 					     unsigned int features,
537 					     const struct cpumask *cpumask,
538 					     const char *name,
539 					     int rating)
540 {
541 	struct clock_event_device *dev;
542 	struct dmtimer_systimer *t;
543 	int error;
544 
545 	t = &clkevt->t;
546 	dev = &clkevt->dev;
547 
548 	/*
549 	 * We mostly use cpuidle_coupled with ARM local timers for runtime,
550 	 * so there's probably no use for CLOCK_EVT_FEAT_DYNIRQ here.
551 	 */
552 	dev->features = features;
553 	dev->rating = rating;
554 	dev->set_next_event = dmtimer_set_next_event;
555 	dev->set_state_shutdown = dmtimer_clockevent_shutdown;
556 	dev->set_state_periodic = dmtimer_set_periodic;
557 	dev->set_state_oneshot = dmtimer_clockevent_shutdown;
558 	dev->set_state_oneshot_stopped = dmtimer_clockevent_shutdown;
559 	dev->tick_resume = dmtimer_clockevent_shutdown;
560 	dev->cpumask = cpumask;
561 
562 	dev->irq = irq_of_parse_and_map(np, 0);
563 	if (!dev->irq)
564 		return -ENXIO;
565 
566 	error = dmtimer_systimer_setup(np, &clkevt->t);
567 	if (error)
568 		return error;
569 
570 	clkevt->period = 0xffffffff - DIV_ROUND_CLOSEST(t->rate, HZ);
571 
572 	/*
573 	 * For clock-event timers we never read the timer counter and
574 	 * so we are not impacted by errata i103 and i767. Therefore,
575 	 * we can safely ignore this errata for clock-event timers.
576 	 */
577 	writel_relaxed(OMAP_TIMER_CTRL_POSTED, t->base + t->ifctrl);
578 
579 	error = request_irq(dev->irq, dmtimer_clockevent_interrupt,
580 			    IRQF_TIMER, name, clkevt);
581 	if (error)
582 		goto err_out_unmap;
583 
584 	writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena);
585 	writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup);
586 
587 	pr_info("TI gptimer %s: %s%lu Hz at %pOF\n",
588 		name, of_find_property(np, "ti,timer-alwon", NULL) ?
589 		"always-on " : "", t->rate, np->parent);
590 
591 	return 0;
592 
593 err_out_unmap:
594 	iounmap(t->base);
595 
596 	return error;
597 }
598 
599 static int __init dmtimer_clockevent_init(struct device_node *np)
600 {
601 	struct dmtimer_clockevent *clkevt;
602 	int error;
603 
604 	clkevt = kzalloc(sizeof(*clkevt), GFP_KERNEL);
605 	if (!clkevt)
606 		return -ENOMEM;
607 
608 	error = dmtimer_clkevt_init_common(clkevt, np,
609 					   CLOCK_EVT_FEAT_PERIODIC |
610 					   CLOCK_EVT_FEAT_ONESHOT,
611 					   cpu_possible_mask, "clockevent",
612 					   300);
613 	if (error)
614 		goto err_out_free;
615 
616 	clockevents_config_and_register(&clkevt->dev, clkevt->t.rate,
617 					3, /* Timer internal resync latency */
618 					0xffffffff);
619 
620 	if (of_machine_is_compatible("ti,am33xx") ||
621 	    of_machine_is_compatible("ti,am43")) {
622 		clkevt->dev.suspend = omap_clockevent_idle;
623 		clkevt->dev.resume = omap_clockevent_unidle;
624 	}
625 
626 	return 0;
627 
628 err_out_free:
629 	kfree(clkevt);
630 
631 	return error;
632 }
633 
634 /* Dmtimer as percpu timer. See dra7 ARM architected timer wrap erratum i940 */
635 static DEFINE_PER_CPU(struct dmtimer_clockevent, dmtimer_percpu_timer);
636 
637 static int __init dmtimer_percpu_timer_init(struct device_node *np, int cpu)
638 {
639 	struct dmtimer_clockevent *clkevt;
640 	int error;
641 
642 	if (!cpu_possible(cpu))
643 		return -EINVAL;
644 
645 	if (!of_property_read_bool(np->parent, "ti,no-reset-on-init") ||
646 	    !of_property_read_bool(np->parent, "ti,no-idle"))
647 		pr_warn("Incomplete dtb for percpu dmtimer %pOF\n", np->parent);
648 
649 	clkevt = per_cpu_ptr(&dmtimer_percpu_timer, cpu);
650 
651 	error = dmtimer_clkevt_init_common(clkevt, np, CLOCK_EVT_FEAT_ONESHOT,
652 					   cpumask_of(cpu), "percpu-dmtimer",
653 					   500);
654 	if (error)
655 		return error;
656 
657 	return 0;
658 }
659 
660 /* See TRM for timer internal resynch latency */
661 static int omap_dmtimer_starting_cpu(unsigned int cpu)
662 {
663 	struct dmtimer_clockevent *clkevt = per_cpu_ptr(&dmtimer_percpu_timer, cpu);
664 	struct clock_event_device *dev = &clkevt->dev;
665 	struct dmtimer_systimer *t = &clkevt->t;
666 
667 	clockevents_config_and_register(dev, t->rate, 3, ULONG_MAX);
668 	irq_force_affinity(dev->irq, cpumask_of(cpu));
669 
670 	return 0;
671 }
672 
673 static int __init dmtimer_percpu_timer_startup(void)
674 {
675 	struct dmtimer_clockevent *clkevt = per_cpu_ptr(&dmtimer_percpu_timer, 0);
676 	struct dmtimer_systimer *t = &clkevt->t;
677 
678 	if (t->sysc) {
679 		cpuhp_setup_state(CPUHP_AP_TI_GP_TIMER_STARTING,
680 				  "clockevents/omap/gptimer:starting",
681 				  omap_dmtimer_starting_cpu, NULL);
682 	}
683 
684 	return 0;
685 }
686 subsys_initcall(dmtimer_percpu_timer_startup);
687 
688 static int __init dmtimer_percpu_quirk_init(struct device_node *np, u32 pa)
689 {
690 	struct device_node *arm_timer;
691 
692 	arm_timer = of_find_compatible_node(NULL, NULL, "arm,armv7-timer");
693 	if (of_device_is_available(arm_timer)) {
694 		pr_warn_once("ARM architected timer wrap issue i940 detected\n");
695 		return 0;
696 	}
697 
698 	if (pa == 0x48034000)		/* dra7 dmtimer3 */
699 		return dmtimer_percpu_timer_init(np, 0);
700 	else if (pa == 0x48036000)	/* dra7 dmtimer4 */
701 		return dmtimer_percpu_timer_init(np, 1);
702 
703 	return 0;
704 }
705 
706 /* Clocksource */
707 static struct dmtimer_clocksource *
708 to_dmtimer_clocksource(struct clocksource *cs)
709 {
710 	return container_of(cs, struct dmtimer_clocksource, dev);
711 }
712 
713 static u64 dmtimer_clocksource_read_cycles(struct clocksource *cs)
714 {
715 	struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs);
716 	struct dmtimer_systimer *t = &clksrc->t;
717 
718 	return (u64)readl_relaxed(t->base + t->counter);
719 }
720 
721 static void __iomem *dmtimer_sched_clock_counter;
722 
723 static u64 notrace dmtimer_read_sched_clock(void)
724 {
725 	return readl_relaxed(dmtimer_sched_clock_counter);
726 }
727 
728 static void dmtimer_clocksource_suspend(struct clocksource *cs)
729 {
730 	struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs);
731 	struct dmtimer_systimer *t = &clksrc->t;
732 
733 	clksrc->loadval = readl_relaxed(t->base + t->counter);
734 	dmtimer_systimer_disable(t);
735 	clk_disable(t->fck);
736 }
737 
738 static void dmtimer_clocksource_resume(struct clocksource *cs)
739 {
740 	struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs);
741 	struct dmtimer_systimer *t = &clksrc->t;
742 	int error;
743 
744 	error = clk_enable(t->fck);
745 	if (error)
746 		pr_err("could not enable timer fck on resume: %i\n", error);
747 
748 	dmtimer_systimer_enable(t);
749 	writel_relaxed(clksrc->loadval, t->base + t->counter);
750 	writel_relaxed(OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR,
751 		       t->base + t->ctrl);
752 }
753 
754 static int __init dmtimer_clocksource_init(struct device_node *np)
755 {
756 	struct dmtimer_clocksource *clksrc;
757 	struct dmtimer_systimer *t;
758 	struct clocksource *dev;
759 	int error;
760 
761 	clksrc = kzalloc(sizeof(*clksrc), GFP_KERNEL);
762 	if (!clksrc)
763 		return -ENOMEM;
764 
765 	dev = &clksrc->dev;
766 	t = &clksrc->t;
767 
768 	error = dmtimer_systimer_setup(np, t);
769 	if (error)
770 		goto err_out_free;
771 
772 	dev->name = "dmtimer";
773 	dev->rating = 300;
774 	dev->read = dmtimer_clocksource_read_cycles;
775 	dev->mask = CLOCKSOURCE_MASK(32);
776 	dev->flags = CLOCK_SOURCE_IS_CONTINUOUS;
777 
778 	/* Unlike for clockevent, legacy code sets suspend only for am4 */
779 	if (of_machine_is_compatible("ti,am43")) {
780 		dev->suspend = dmtimer_clocksource_suspend;
781 		dev->resume = dmtimer_clocksource_resume;
782 	}
783 
784 	writel_relaxed(0, t->base + t->counter);
785 	writel_relaxed(OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR,
786 		       t->base + t->ctrl);
787 
788 	pr_info("TI gptimer clocksource: %s%pOF\n",
789 		of_find_property(np, "ti,timer-alwon", NULL) ?
790 		"always-on " : "", np->parent);
791 
792 	if (!dmtimer_sched_clock_counter) {
793 		dmtimer_sched_clock_counter = t->base + t->counter;
794 		sched_clock_register(dmtimer_read_sched_clock, 32, t->rate);
795 	}
796 
797 	if (clocksource_register_hz(dev, t->rate))
798 		pr_err("Could not register clocksource %pOF\n", np);
799 
800 	return 0;
801 
802 err_out_free:
803 	kfree(clksrc);
804 
805 	return -ENODEV;
806 }
807 
808 /*
809  * To detect between a clocksource and clockevent, we assume the device tree
810  * has no interrupts configured for a clocksource timer.
811  */
812 static int __init dmtimer_systimer_init(struct device_node *np)
813 {
814 	const __be32 *addr;
815 	u32 pa;
816 
817 	/* One time init for the preferred timer configuration */
818 	if (!clocksource && !clockevent)
819 		dmtimer_systimer_select_best();
820 
821 	if (!clocksource && !clockevent) {
822 		pr_err("%s: unable to detect system timers, update dtb?\n",
823 		       __func__);
824 
825 		return -EINVAL;
826 	}
827 
828 	addr = of_get_address(np, 0, NULL, NULL);
829 	pa = of_translate_address(np, addr);
830 	if (!pa)
831 		return -EINVAL;
832 
833 	if (counter_32k <= 0 && clocksource == pa)
834 		return dmtimer_clocksource_init(np);
835 
836 	if (clockevent == pa)
837 		return dmtimer_clockevent_init(np);
838 
839 	if (of_machine_is_compatible("ti,dra7"))
840 		return dmtimer_percpu_quirk_init(np, pa);
841 
842 	return 0;
843 }
844 
845 TIMER_OF_DECLARE(systimer_omap2, "ti,omap2420-timer", dmtimer_systimer_init);
846 TIMER_OF_DECLARE(systimer_omap3, "ti,omap3430-timer", dmtimer_systimer_init);
847 TIMER_OF_DECLARE(systimer_omap4, "ti,omap4430-timer", dmtimer_systimer_init);
848 TIMER_OF_DECLARE(systimer_omap5, "ti,omap5430-timer", dmtimer_systimer_init);
849 TIMER_OF_DECLARE(systimer_am33x, "ti,am335x-timer", dmtimer_systimer_init);
850 TIMER_OF_DECLARE(systimer_am3ms, "ti,am335x-timer-1ms", dmtimer_systimer_init);
851 TIMER_OF_DECLARE(systimer_dm814, "ti,dm814-timer", dmtimer_systimer_init);
852 TIMER_OF_DECLARE(systimer_dm816, "ti,dm816-timer", dmtimer_systimer_init);
853