1 /*
2  *  linux/drivers/clocksource/arm_arch_timer.c
3  *
4  *  Copyright (C) 2011 ARM Ltd.
5  *  All Rights Reserved
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/device.h>
14 #include <linux/smp.h>
15 #include <linux/cpu.h>
16 #include <linux/cpu_pm.h>
17 #include <linux/clockchips.h>
18 #include <linux/clocksource.h>
19 #include <linux/interrupt.h>
20 #include <linux/of_irq.h>
21 #include <linux/of_address.h>
22 #include <linux/io.h>
23 #include <linux/slab.h>
24 #include <linux/sched_clock.h>
25 #include <linux/acpi.h>
26 
27 #include <asm/arch_timer.h>
28 #include <asm/virt.h>
29 
30 #include <clocksource/arm_arch_timer.h>
31 
32 #define CNTTIDR		0x08
33 #define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4))
34 
35 #define CNTACR(n)	(0x40 + ((n) * 4))
36 #define CNTACR_RPCT	BIT(0)
37 #define CNTACR_RVCT	BIT(1)
38 #define CNTACR_RFRQ	BIT(2)
39 #define CNTACR_RVOFF	BIT(3)
40 #define CNTACR_RWVT	BIT(4)
41 #define CNTACR_RWPT	BIT(5)
42 
43 #define CNTVCT_LO	0x08
44 #define CNTVCT_HI	0x0c
45 #define CNTFRQ		0x10
46 #define CNTP_TVAL	0x28
47 #define CNTP_CTL	0x2c
48 #define CNTV_TVAL	0x38
49 #define CNTV_CTL	0x3c
50 
51 #define ARCH_CP15_TIMER	BIT(0)
52 #define ARCH_MEM_TIMER	BIT(1)
53 static unsigned arch_timers_present __initdata;
54 
55 static void __iomem *arch_counter_base;
56 
57 struct arch_timer {
58 	void __iomem *base;
59 	struct clock_event_device evt;
60 };
61 
62 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
63 
64 static u32 arch_timer_rate;
65 
66 enum ppi_nr {
67 	PHYS_SECURE_PPI,
68 	PHYS_NONSECURE_PPI,
69 	VIRT_PPI,
70 	HYP_PPI,
71 	MAX_TIMER_PPI
72 };
73 
74 static int arch_timer_ppi[MAX_TIMER_PPI];
75 
76 static struct clock_event_device __percpu *arch_timer_evt;
77 
78 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
79 static bool arch_timer_c3stop;
80 static bool arch_timer_mem_use_virtual;
81 
82 /*
83  * Architected system timer support.
84  */
85 
86 static __always_inline
87 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
88 			  struct clock_event_device *clk)
89 {
90 	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
91 		struct arch_timer *timer = to_arch_timer(clk);
92 		switch (reg) {
93 		case ARCH_TIMER_REG_CTRL:
94 			writel_relaxed(val, timer->base + CNTP_CTL);
95 			break;
96 		case ARCH_TIMER_REG_TVAL:
97 			writel_relaxed(val, timer->base + CNTP_TVAL);
98 			break;
99 		}
100 	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
101 		struct arch_timer *timer = to_arch_timer(clk);
102 		switch (reg) {
103 		case ARCH_TIMER_REG_CTRL:
104 			writel_relaxed(val, timer->base + CNTV_CTL);
105 			break;
106 		case ARCH_TIMER_REG_TVAL:
107 			writel_relaxed(val, timer->base + CNTV_TVAL);
108 			break;
109 		}
110 	} else {
111 		arch_timer_reg_write_cp15(access, reg, val);
112 	}
113 }
114 
115 static __always_inline
116 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
117 			struct clock_event_device *clk)
118 {
119 	u32 val;
120 
121 	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
122 		struct arch_timer *timer = to_arch_timer(clk);
123 		switch (reg) {
124 		case ARCH_TIMER_REG_CTRL:
125 			val = readl_relaxed(timer->base + CNTP_CTL);
126 			break;
127 		case ARCH_TIMER_REG_TVAL:
128 			val = readl_relaxed(timer->base + CNTP_TVAL);
129 			break;
130 		}
131 	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
132 		struct arch_timer *timer = to_arch_timer(clk);
133 		switch (reg) {
134 		case ARCH_TIMER_REG_CTRL:
135 			val = readl_relaxed(timer->base + CNTV_CTL);
136 			break;
137 		case ARCH_TIMER_REG_TVAL:
138 			val = readl_relaxed(timer->base + CNTV_TVAL);
139 			break;
140 		}
141 	} else {
142 		val = arch_timer_reg_read_cp15(access, reg);
143 	}
144 
145 	return val;
146 }
147 
148 static __always_inline irqreturn_t timer_handler(const int access,
149 					struct clock_event_device *evt)
150 {
151 	unsigned long ctrl;
152 
153 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
154 	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
155 		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
156 		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
157 		evt->event_handler(evt);
158 		return IRQ_HANDLED;
159 	}
160 
161 	return IRQ_NONE;
162 }
163 
164 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
165 {
166 	struct clock_event_device *evt = dev_id;
167 
168 	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
169 }
170 
171 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
172 {
173 	struct clock_event_device *evt = dev_id;
174 
175 	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
176 }
177 
178 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
179 {
180 	struct clock_event_device *evt = dev_id;
181 
182 	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
183 }
184 
185 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
186 {
187 	struct clock_event_device *evt = dev_id;
188 
189 	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
190 }
191 
192 static __always_inline int timer_shutdown(const int access,
193 					  struct clock_event_device *clk)
194 {
195 	unsigned long ctrl;
196 
197 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
198 	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
199 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
200 
201 	return 0;
202 }
203 
204 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
205 {
206 	return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
207 }
208 
209 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
210 {
211 	return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
212 }
213 
214 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
215 {
216 	return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
217 }
218 
219 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
220 {
221 	return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
222 }
223 
224 static __always_inline void set_next_event(const int access, unsigned long evt,
225 					   struct clock_event_device *clk)
226 {
227 	unsigned long ctrl;
228 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
229 	ctrl |= ARCH_TIMER_CTRL_ENABLE;
230 	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
231 	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
232 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
233 }
234 
235 static int arch_timer_set_next_event_virt(unsigned long evt,
236 					  struct clock_event_device *clk)
237 {
238 	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
239 	return 0;
240 }
241 
242 static int arch_timer_set_next_event_phys(unsigned long evt,
243 					  struct clock_event_device *clk)
244 {
245 	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
246 	return 0;
247 }
248 
249 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
250 					      struct clock_event_device *clk)
251 {
252 	set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
253 	return 0;
254 }
255 
256 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
257 					      struct clock_event_device *clk)
258 {
259 	set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
260 	return 0;
261 }
262 
263 static void __arch_timer_setup(unsigned type,
264 			       struct clock_event_device *clk)
265 {
266 	clk->features = CLOCK_EVT_FEAT_ONESHOT;
267 
268 	if (type == ARCH_CP15_TIMER) {
269 		if (arch_timer_c3stop)
270 			clk->features |= CLOCK_EVT_FEAT_C3STOP;
271 		clk->name = "arch_sys_timer";
272 		clk->rating = 450;
273 		clk->cpumask = cpumask_of(smp_processor_id());
274 		clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
275 		switch (arch_timer_uses_ppi) {
276 		case VIRT_PPI:
277 			clk->set_state_shutdown = arch_timer_shutdown_virt;
278 			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
279 			clk->set_next_event = arch_timer_set_next_event_virt;
280 			break;
281 		case PHYS_SECURE_PPI:
282 		case PHYS_NONSECURE_PPI:
283 		case HYP_PPI:
284 			clk->set_state_shutdown = arch_timer_shutdown_phys;
285 			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
286 			clk->set_next_event = arch_timer_set_next_event_phys;
287 			break;
288 		default:
289 			BUG();
290 		}
291 	} else {
292 		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
293 		clk->name = "arch_mem_timer";
294 		clk->rating = 400;
295 		clk->cpumask = cpu_all_mask;
296 		if (arch_timer_mem_use_virtual) {
297 			clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
298 			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
299 			clk->set_next_event =
300 				arch_timer_set_next_event_virt_mem;
301 		} else {
302 			clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
303 			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
304 			clk->set_next_event =
305 				arch_timer_set_next_event_phys_mem;
306 		}
307 	}
308 
309 	clk->set_state_shutdown(clk);
310 
311 	clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
312 }
313 
314 static void arch_timer_evtstrm_enable(int divider)
315 {
316 	u32 cntkctl = arch_timer_get_cntkctl();
317 
318 	cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
319 	/* Set the divider and enable virtual event stream */
320 	cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
321 			| ARCH_TIMER_VIRT_EVT_EN;
322 	arch_timer_set_cntkctl(cntkctl);
323 	elf_hwcap |= HWCAP_EVTSTRM;
324 #ifdef CONFIG_COMPAT
325 	compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
326 #endif
327 }
328 
329 static void arch_timer_configure_evtstream(void)
330 {
331 	int evt_stream_div, pos;
332 
333 	/* Find the closest power of two to the divisor */
334 	evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
335 	pos = fls(evt_stream_div);
336 	if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
337 		pos--;
338 	/* enable event stream */
339 	arch_timer_evtstrm_enable(min(pos, 15));
340 }
341 
342 static void arch_counter_set_user_access(void)
343 {
344 	u32 cntkctl = arch_timer_get_cntkctl();
345 
346 	/* Disable user access to the timers and the physical counter */
347 	/* Also disable virtual event stream */
348 	cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
349 			| ARCH_TIMER_USR_VT_ACCESS_EN
350 			| ARCH_TIMER_VIRT_EVT_EN
351 			| ARCH_TIMER_USR_PCT_ACCESS_EN);
352 
353 	/* Enable user access to the virtual counter */
354 	cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
355 
356 	arch_timer_set_cntkctl(cntkctl);
357 }
358 
359 static bool arch_timer_has_nonsecure_ppi(void)
360 {
361 	return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
362 		arch_timer_ppi[PHYS_NONSECURE_PPI]);
363 }
364 
365 static int arch_timer_setup(struct clock_event_device *clk)
366 {
367 	__arch_timer_setup(ARCH_CP15_TIMER, clk);
368 
369 	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], 0);
370 
371 	if (arch_timer_has_nonsecure_ppi())
372 		enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
373 
374 	arch_counter_set_user_access();
375 	if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
376 		arch_timer_configure_evtstream();
377 
378 	return 0;
379 }
380 
381 static void
382 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
383 {
384 	/* Who has more than one independent system counter? */
385 	if (arch_timer_rate)
386 		return;
387 
388 	/*
389 	 * Try to determine the frequency from the device tree or CNTFRQ,
390 	 * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
391 	 */
392 	if (!acpi_disabled ||
393 	    of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
394 		if (cntbase)
395 			arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
396 		else
397 			arch_timer_rate = arch_timer_get_cntfrq();
398 	}
399 
400 	/* Check the timer frequency. */
401 	if (arch_timer_rate == 0)
402 		pr_warn("Architected timer frequency not available\n");
403 }
404 
405 static void arch_timer_banner(unsigned type)
406 {
407 	pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
408 		     type & ARCH_CP15_TIMER ? "cp15" : "",
409 		     type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
410 		     type & ARCH_MEM_TIMER ? "mmio" : "",
411 		     (unsigned long)arch_timer_rate / 1000000,
412 		     (unsigned long)(arch_timer_rate / 10000) % 100,
413 		     type & ARCH_CP15_TIMER ?
414 		     (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
415 			"",
416 		     type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
417 		     type & ARCH_MEM_TIMER ?
418 			arch_timer_mem_use_virtual ? "virt" : "phys" :
419 			"");
420 }
421 
422 u32 arch_timer_get_rate(void)
423 {
424 	return arch_timer_rate;
425 }
426 
427 static u64 arch_counter_get_cntvct_mem(void)
428 {
429 	u32 vct_lo, vct_hi, tmp_hi;
430 
431 	do {
432 		vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
433 		vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
434 		tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
435 	} while (vct_hi != tmp_hi);
436 
437 	return ((u64) vct_hi << 32) | vct_lo;
438 }
439 
440 /*
441  * Default to cp15 based access because arm64 uses this function for
442  * sched_clock() before DT is probed and the cp15 method is guaranteed
443  * to exist on arm64. arm doesn't use this before DT is probed so even
444  * if we don't have the cp15 accessors we won't have a problem.
445  */
446 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
447 
448 static cycle_t arch_counter_read(struct clocksource *cs)
449 {
450 	return arch_timer_read_counter();
451 }
452 
453 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
454 {
455 	return arch_timer_read_counter();
456 }
457 
458 static struct clocksource clocksource_counter = {
459 	.name	= "arch_sys_counter",
460 	.rating	= 400,
461 	.read	= arch_counter_read,
462 	.mask	= CLOCKSOURCE_MASK(56),
463 	.flags	= CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
464 };
465 
466 static struct cyclecounter cyclecounter = {
467 	.read	= arch_counter_read_cc,
468 	.mask	= CLOCKSOURCE_MASK(56),
469 };
470 
471 static struct arch_timer_kvm_info arch_timer_kvm_info;
472 
473 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
474 {
475 	return &arch_timer_kvm_info;
476 }
477 
478 static void __init arch_counter_register(unsigned type)
479 {
480 	u64 start_count;
481 
482 	/* Register the CP15 based counter if we have one */
483 	if (type & ARCH_CP15_TIMER) {
484 		if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
485 			arch_timer_read_counter = arch_counter_get_cntvct;
486 		else
487 			arch_timer_read_counter = arch_counter_get_cntpct;
488 	} else {
489 		arch_timer_read_counter = arch_counter_get_cntvct_mem;
490 
491 		/* If the clocksource name is "arch_sys_counter" the
492 		 * VDSO will attempt to read the CP15-based counter.
493 		 * Ensure this does not happen when CP15-based
494 		 * counter is not available.
495 		 */
496 		clocksource_counter.name = "arch_mem_counter";
497 	}
498 
499 	start_count = arch_timer_read_counter();
500 	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
501 	cyclecounter.mult = clocksource_counter.mult;
502 	cyclecounter.shift = clocksource_counter.shift;
503 	timecounter_init(&arch_timer_kvm_info.timecounter,
504 			 &cyclecounter, start_count);
505 
506 	/* 56 bits minimum, so we assume worst case rollover */
507 	sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
508 }
509 
510 static void arch_timer_stop(struct clock_event_device *clk)
511 {
512 	pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
513 		 clk->irq, smp_processor_id());
514 
515 	disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
516 	if (arch_timer_has_nonsecure_ppi())
517 		disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
518 
519 	clk->set_state_shutdown(clk);
520 }
521 
522 static int arch_timer_cpu_notify(struct notifier_block *self,
523 					   unsigned long action, void *hcpu)
524 {
525 	/*
526 	 * Grab cpu pointer in each case to avoid spurious
527 	 * preemptible warnings
528 	 */
529 	switch (action & ~CPU_TASKS_FROZEN) {
530 	case CPU_STARTING:
531 		arch_timer_setup(this_cpu_ptr(arch_timer_evt));
532 		break;
533 	case CPU_DYING:
534 		arch_timer_stop(this_cpu_ptr(arch_timer_evt));
535 		break;
536 	}
537 
538 	return NOTIFY_OK;
539 }
540 
541 static struct notifier_block arch_timer_cpu_nb = {
542 	.notifier_call = arch_timer_cpu_notify,
543 };
544 
545 #ifdef CONFIG_CPU_PM
546 static unsigned int saved_cntkctl;
547 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
548 				    unsigned long action, void *hcpu)
549 {
550 	if (action == CPU_PM_ENTER)
551 		saved_cntkctl = arch_timer_get_cntkctl();
552 	else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
553 		arch_timer_set_cntkctl(saved_cntkctl);
554 	return NOTIFY_OK;
555 }
556 
557 static struct notifier_block arch_timer_cpu_pm_notifier = {
558 	.notifier_call = arch_timer_cpu_pm_notify,
559 };
560 
561 static int __init arch_timer_cpu_pm_init(void)
562 {
563 	return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
564 }
565 #else
566 static int __init arch_timer_cpu_pm_init(void)
567 {
568 	return 0;
569 }
570 #endif
571 
572 static int __init arch_timer_register(void)
573 {
574 	int err;
575 	int ppi;
576 
577 	arch_timer_evt = alloc_percpu(struct clock_event_device);
578 	if (!arch_timer_evt) {
579 		err = -ENOMEM;
580 		goto out;
581 	}
582 
583 	ppi = arch_timer_ppi[arch_timer_uses_ppi];
584 	switch (arch_timer_uses_ppi) {
585 	case VIRT_PPI:
586 		err = request_percpu_irq(ppi, arch_timer_handler_virt,
587 					 "arch_timer", arch_timer_evt);
588 		break;
589 	case PHYS_SECURE_PPI:
590 	case PHYS_NONSECURE_PPI:
591 		err = request_percpu_irq(ppi, arch_timer_handler_phys,
592 					 "arch_timer", arch_timer_evt);
593 		if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
594 			ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
595 			err = request_percpu_irq(ppi, arch_timer_handler_phys,
596 						 "arch_timer", arch_timer_evt);
597 			if (err)
598 				free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
599 						arch_timer_evt);
600 		}
601 		break;
602 	case HYP_PPI:
603 		err = request_percpu_irq(ppi, arch_timer_handler_phys,
604 					 "arch_timer", arch_timer_evt);
605 		break;
606 	default:
607 		BUG();
608 	}
609 
610 	if (err) {
611 		pr_err("arch_timer: can't register interrupt %d (%d)\n",
612 		       ppi, err);
613 		goto out_free;
614 	}
615 
616 	err = register_cpu_notifier(&arch_timer_cpu_nb);
617 	if (err)
618 		goto out_free_irq;
619 
620 	err = arch_timer_cpu_pm_init();
621 	if (err)
622 		goto out_unreg_notify;
623 
624 	/* Immediately configure the timer on the boot CPU */
625 	arch_timer_setup(this_cpu_ptr(arch_timer_evt));
626 
627 	return 0;
628 
629 out_unreg_notify:
630 	unregister_cpu_notifier(&arch_timer_cpu_nb);
631 out_free_irq:
632 	free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
633 	if (arch_timer_has_nonsecure_ppi())
634 		free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
635 				arch_timer_evt);
636 
637 out_free:
638 	free_percpu(arch_timer_evt);
639 out:
640 	return err;
641 }
642 
643 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
644 {
645 	int ret;
646 	irq_handler_t func;
647 	struct arch_timer *t;
648 
649 	t = kzalloc(sizeof(*t), GFP_KERNEL);
650 	if (!t)
651 		return -ENOMEM;
652 
653 	t->base = base;
654 	t->evt.irq = irq;
655 	__arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
656 
657 	if (arch_timer_mem_use_virtual)
658 		func = arch_timer_handler_virt_mem;
659 	else
660 		func = arch_timer_handler_phys_mem;
661 
662 	ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
663 	if (ret) {
664 		pr_err("arch_timer: Failed to request mem timer irq\n");
665 		kfree(t);
666 	}
667 
668 	return ret;
669 }
670 
671 static const struct of_device_id arch_timer_of_match[] __initconst = {
672 	{ .compatible   = "arm,armv7-timer",    },
673 	{ .compatible   = "arm,armv8-timer",    },
674 	{},
675 };
676 
677 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
678 	{ .compatible   = "arm,armv7-timer-mem", },
679 	{},
680 };
681 
682 static bool __init
683 arch_timer_needs_probing(int type, const struct of_device_id *matches)
684 {
685 	struct device_node *dn;
686 	bool needs_probing = false;
687 
688 	dn = of_find_matching_node(NULL, matches);
689 	if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
690 		needs_probing = true;
691 	of_node_put(dn);
692 
693 	return needs_probing;
694 }
695 
696 static void __init arch_timer_common_init(void)
697 {
698 	unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
699 
700 	/* Wait until both nodes are probed if we have two timers */
701 	if ((arch_timers_present & mask) != mask) {
702 		if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
703 			return;
704 		if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
705 			return;
706 	}
707 
708 	arch_timer_banner(arch_timers_present);
709 	arch_counter_register(arch_timers_present);
710 	arch_timer_arch_init();
711 }
712 
713 static void __init arch_timer_init(void)
714 {
715 	/*
716 	 * If HYP mode is available, we know that the physical timer
717 	 * has been configured to be accessible from PL1. Use it, so
718 	 * that a guest can use the virtual timer instead.
719 	 *
720 	 * If no interrupt provided for virtual timer, we'll have to
721 	 * stick to the physical timer. It'd better be accessible...
722 	 *
723 	 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
724 	 * accesses to CNTP_*_EL1 registers are silently redirected to
725 	 * their CNTHP_*_EL2 counterparts, and use a different PPI
726 	 * number.
727 	 */
728 	if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
729 		bool has_ppi;
730 
731 		if (is_kernel_in_hyp_mode()) {
732 			arch_timer_uses_ppi = HYP_PPI;
733 			has_ppi = !!arch_timer_ppi[HYP_PPI];
734 		} else {
735 			arch_timer_uses_ppi = PHYS_SECURE_PPI;
736 			has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
737 				   !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
738 		}
739 
740 		if (!has_ppi) {
741 			pr_warn("arch_timer: No interrupt available, giving up\n");
742 			return;
743 		}
744 	}
745 
746 	arch_timer_register();
747 	arch_timer_common_init();
748 
749 	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
750 }
751 
752 static void __init arch_timer_of_init(struct device_node *np)
753 {
754 	int i;
755 
756 	if (arch_timers_present & ARCH_CP15_TIMER) {
757 		pr_warn("arch_timer: multiple nodes in dt, skipping\n");
758 		return;
759 	}
760 
761 	arch_timers_present |= ARCH_CP15_TIMER;
762 	for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
763 		arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
764 
765 	arch_timer_detect_rate(NULL, np);
766 
767 	arch_timer_c3stop = !of_property_read_bool(np, "always-on");
768 
769 	/*
770 	 * If we cannot rely on firmware initializing the timer registers then
771 	 * we should use the physical timers instead.
772 	 */
773 	if (IS_ENABLED(CONFIG_ARM) &&
774 	    of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
775 		arch_timer_uses_ppi = PHYS_SECURE_PPI;
776 
777 	arch_timer_init();
778 }
779 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
780 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
781 
782 static void __init arch_timer_mem_init(struct device_node *np)
783 {
784 	struct device_node *frame, *best_frame = NULL;
785 	void __iomem *cntctlbase, *base;
786 	unsigned int irq;
787 	u32 cnttidr;
788 
789 	arch_timers_present |= ARCH_MEM_TIMER;
790 	cntctlbase = of_iomap(np, 0);
791 	if (!cntctlbase) {
792 		pr_err("arch_timer: Can't find CNTCTLBase\n");
793 		return;
794 	}
795 
796 	cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
797 
798 	/*
799 	 * Try to find a virtual capable frame. Otherwise fall back to a
800 	 * physical capable frame.
801 	 */
802 	for_each_available_child_of_node(np, frame) {
803 		int n;
804 		u32 cntacr;
805 
806 		if (of_property_read_u32(frame, "frame-number", &n)) {
807 			pr_err("arch_timer: Missing frame-number\n");
808 			of_node_put(frame);
809 			goto out;
810 		}
811 
812 		/* Try enabling everything, and see what sticks */
813 		cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
814 			 CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
815 		writel_relaxed(cntacr, cntctlbase + CNTACR(n));
816 		cntacr = readl_relaxed(cntctlbase + CNTACR(n));
817 
818 		if ((cnttidr & CNTTIDR_VIRT(n)) &&
819 		    !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
820 			of_node_put(best_frame);
821 			best_frame = frame;
822 			arch_timer_mem_use_virtual = true;
823 			break;
824 		}
825 
826 		if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
827 			continue;
828 
829 		of_node_put(best_frame);
830 		best_frame = of_node_get(frame);
831 	}
832 
833 	base = arch_counter_base = of_iomap(best_frame, 0);
834 	if (!base) {
835 		pr_err("arch_timer: Can't map frame's registers\n");
836 		goto out;
837 	}
838 
839 	if (arch_timer_mem_use_virtual)
840 		irq = irq_of_parse_and_map(best_frame, 1);
841 	else
842 		irq = irq_of_parse_and_map(best_frame, 0);
843 
844 	if (!irq) {
845 		pr_err("arch_timer: Frame missing %s irq",
846 		       arch_timer_mem_use_virtual ? "virt" : "phys");
847 		goto out;
848 	}
849 
850 	arch_timer_detect_rate(base, np);
851 	arch_timer_mem_register(base, irq);
852 	arch_timer_common_init();
853 out:
854 	iounmap(cntctlbase);
855 	of_node_put(best_frame);
856 }
857 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
858 		       arch_timer_mem_init);
859 
860 #ifdef CONFIG_ACPI
861 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
862 {
863 	int trigger, polarity;
864 
865 	if (!interrupt)
866 		return 0;
867 
868 	trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
869 			: ACPI_LEVEL_SENSITIVE;
870 
871 	polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
872 			: ACPI_ACTIVE_HIGH;
873 
874 	return acpi_register_gsi(NULL, interrupt, trigger, polarity);
875 }
876 
877 /* Initialize per-processor generic timer */
878 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
879 {
880 	struct acpi_table_gtdt *gtdt;
881 
882 	if (arch_timers_present & ARCH_CP15_TIMER) {
883 		pr_warn("arch_timer: already initialized, skipping\n");
884 		return -EINVAL;
885 	}
886 
887 	gtdt = container_of(table, struct acpi_table_gtdt, header);
888 
889 	arch_timers_present |= ARCH_CP15_TIMER;
890 
891 	arch_timer_ppi[PHYS_SECURE_PPI] =
892 		map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
893 		gtdt->secure_el1_flags);
894 
895 	arch_timer_ppi[PHYS_NONSECURE_PPI] =
896 		map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
897 		gtdt->non_secure_el1_flags);
898 
899 	arch_timer_ppi[VIRT_PPI] =
900 		map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
901 		gtdt->virtual_timer_flags);
902 
903 	arch_timer_ppi[HYP_PPI] =
904 		map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
905 		gtdt->non_secure_el2_flags);
906 
907 	/* Get the frequency from CNTFRQ */
908 	arch_timer_detect_rate(NULL, NULL);
909 
910 	/* Always-on capability */
911 	arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
912 
913 	arch_timer_init();
914 	return 0;
915 }
916 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
917 #endif
918