1 /* 2 * linux/drivers/clocksource/arm_arch_timer.c 3 * 4 * Copyright (C) 2011 ARM Ltd. 5 * All Rights Reserved 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #define pr_fmt(fmt) "arm_arch_timer: " fmt 13 14 #include <linux/init.h> 15 #include <linux/kernel.h> 16 #include <linux/device.h> 17 #include <linux/smp.h> 18 #include <linux/cpu.h> 19 #include <linux/cpu_pm.h> 20 #include <linux/clockchips.h> 21 #include <linux/clocksource.h> 22 #include <linux/interrupt.h> 23 #include <linux/of_irq.h> 24 #include <linux/of_address.h> 25 #include <linux/io.h> 26 #include <linux/slab.h> 27 #include <linux/sched_clock.h> 28 #include <linux/acpi.h> 29 30 #include <asm/arch_timer.h> 31 #include <asm/virt.h> 32 33 #include <clocksource/arm_arch_timer.h> 34 35 #define CNTTIDR 0x08 36 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4)) 37 38 #define CNTACR(n) (0x40 + ((n) * 4)) 39 #define CNTACR_RPCT BIT(0) 40 #define CNTACR_RVCT BIT(1) 41 #define CNTACR_RFRQ BIT(2) 42 #define CNTACR_RVOFF BIT(3) 43 #define CNTACR_RWVT BIT(4) 44 #define CNTACR_RWPT BIT(5) 45 46 #define CNTVCT_LO 0x08 47 #define CNTVCT_HI 0x0c 48 #define CNTFRQ 0x10 49 #define CNTP_TVAL 0x28 50 #define CNTP_CTL 0x2c 51 #define CNTV_TVAL 0x38 52 #define CNTV_CTL 0x3c 53 54 #define ARCH_CP15_TIMER BIT(0) 55 #define ARCH_MEM_TIMER BIT(1) 56 static unsigned arch_timers_present __initdata; 57 58 static void __iomem *arch_counter_base; 59 60 struct arch_timer { 61 void __iomem *base; 62 struct clock_event_device evt; 63 }; 64 65 #define to_arch_timer(e) container_of(e, struct arch_timer, evt) 66 67 static u32 arch_timer_rate; 68 69 enum ppi_nr { 70 PHYS_SECURE_PPI, 71 PHYS_NONSECURE_PPI, 72 VIRT_PPI, 73 HYP_PPI, 74 MAX_TIMER_PPI 75 }; 76 77 static int arch_timer_ppi[MAX_TIMER_PPI]; 78 79 static struct clock_event_device __percpu *arch_timer_evt; 80 81 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI; 82 static bool arch_timer_c3stop; 83 static bool arch_timer_mem_use_virtual; 84 static bool arch_counter_suspend_stop; 85 86 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); 87 88 static int __init early_evtstrm_cfg(char *buf) 89 { 90 return strtobool(buf, &evtstrm_enable); 91 } 92 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); 93 94 /* 95 * Architected system timer support. 96 */ 97 98 #ifdef CONFIG_FSL_ERRATUM_A008585 99 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); 100 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); 101 102 static int fsl_a008585_enable = -1; 103 104 static int __init early_fsl_a008585_cfg(char *buf) 105 { 106 int ret; 107 bool val; 108 109 ret = strtobool(buf, &val); 110 if (ret) 111 return ret; 112 113 fsl_a008585_enable = val; 114 return 0; 115 } 116 early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg); 117 118 u32 __fsl_a008585_read_cntp_tval_el0(void) 119 { 120 return __fsl_a008585_read_reg(cntp_tval_el0); 121 } 122 123 u32 __fsl_a008585_read_cntv_tval_el0(void) 124 { 125 return __fsl_a008585_read_reg(cntv_tval_el0); 126 } 127 128 u64 __fsl_a008585_read_cntvct_el0(void) 129 { 130 return __fsl_a008585_read_reg(cntvct_el0); 131 } 132 EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0); 133 #endif /* CONFIG_FSL_ERRATUM_A008585 */ 134 135 static __always_inline 136 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, 137 struct clock_event_device *clk) 138 { 139 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { 140 struct arch_timer *timer = to_arch_timer(clk); 141 switch (reg) { 142 case ARCH_TIMER_REG_CTRL: 143 writel_relaxed(val, timer->base + CNTP_CTL); 144 break; 145 case ARCH_TIMER_REG_TVAL: 146 writel_relaxed(val, timer->base + CNTP_TVAL); 147 break; 148 } 149 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { 150 struct arch_timer *timer = to_arch_timer(clk); 151 switch (reg) { 152 case ARCH_TIMER_REG_CTRL: 153 writel_relaxed(val, timer->base + CNTV_CTL); 154 break; 155 case ARCH_TIMER_REG_TVAL: 156 writel_relaxed(val, timer->base + CNTV_TVAL); 157 break; 158 } 159 } else { 160 arch_timer_reg_write_cp15(access, reg, val); 161 } 162 } 163 164 static __always_inline 165 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, 166 struct clock_event_device *clk) 167 { 168 u32 val; 169 170 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { 171 struct arch_timer *timer = to_arch_timer(clk); 172 switch (reg) { 173 case ARCH_TIMER_REG_CTRL: 174 val = readl_relaxed(timer->base + CNTP_CTL); 175 break; 176 case ARCH_TIMER_REG_TVAL: 177 val = readl_relaxed(timer->base + CNTP_TVAL); 178 break; 179 } 180 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { 181 struct arch_timer *timer = to_arch_timer(clk); 182 switch (reg) { 183 case ARCH_TIMER_REG_CTRL: 184 val = readl_relaxed(timer->base + CNTV_CTL); 185 break; 186 case ARCH_TIMER_REG_TVAL: 187 val = readl_relaxed(timer->base + CNTV_TVAL); 188 break; 189 } 190 } else { 191 val = arch_timer_reg_read_cp15(access, reg); 192 } 193 194 return val; 195 } 196 197 static __always_inline irqreturn_t timer_handler(const int access, 198 struct clock_event_device *evt) 199 { 200 unsigned long ctrl; 201 202 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt); 203 if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { 204 ctrl |= ARCH_TIMER_CTRL_IT_MASK; 205 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt); 206 evt->event_handler(evt); 207 return IRQ_HANDLED; 208 } 209 210 return IRQ_NONE; 211 } 212 213 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id) 214 { 215 struct clock_event_device *evt = dev_id; 216 217 return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt); 218 } 219 220 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id) 221 { 222 struct clock_event_device *evt = dev_id; 223 224 return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt); 225 } 226 227 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id) 228 { 229 struct clock_event_device *evt = dev_id; 230 231 return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt); 232 } 233 234 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id) 235 { 236 struct clock_event_device *evt = dev_id; 237 238 return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt); 239 } 240 241 static __always_inline int timer_shutdown(const int access, 242 struct clock_event_device *clk) 243 { 244 unsigned long ctrl; 245 246 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 247 ctrl &= ~ARCH_TIMER_CTRL_ENABLE; 248 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 249 250 return 0; 251 } 252 253 static int arch_timer_shutdown_virt(struct clock_event_device *clk) 254 { 255 return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk); 256 } 257 258 static int arch_timer_shutdown_phys(struct clock_event_device *clk) 259 { 260 return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk); 261 } 262 263 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk) 264 { 265 return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk); 266 } 267 268 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk) 269 { 270 return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk); 271 } 272 273 static __always_inline void set_next_event(const int access, unsigned long evt, 274 struct clock_event_device *clk) 275 { 276 unsigned long ctrl; 277 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 278 ctrl |= ARCH_TIMER_CTRL_ENABLE; 279 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 280 arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk); 281 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 282 } 283 284 #ifdef CONFIG_FSL_ERRATUM_A008585 285 static __always_inline void fsl_a008585_set_next_event(const int access, 286 unsigned long evt, struct clock_event_device *clk) 287 { 288 unsigned long ctrl; 289 u64 cval = evt + arch_counter_get_cntvct(); 290 291 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 292 ctrl |= ARCH_TIMER_CTRL_ENABLE; 293 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 294 295 if (access == ARCH_TIMER_PHYS_ACCESS) 296 write_sysreg(cval, cntp_cval_el0); 297 else if (access == ARCH_TIMER_VIRT_ACCESS) 298 write_sysreg(cval, cntv_cval_el0); 299 300 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 301 } 302 303 static int fsl_a008585_set_next_event_virt(unsigned long evt, 304 struct clock_event_device *clk) 305 { 306 fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); 307 return 0; 308 } 309 310 static int fsl_a008585_set_next_event_phys(unsigned long evt, 311 struct clock_event_device *clk) 312 { 313 fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); 314 return 0; 315 } 316 #endif /* CONFIG_FSL_ERRATUM_A008585 */ 317 318 static int arch_timer_set_next_event_virt(unsigned long evt, 319 struct clock_event_device *clk) 320 { 321 set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); 322 return 0; 323 } 324 325 static int arch_timer_set_next_event_phys(unsigned long evt, 326 struct clock_event_device *clk) 327 { 328 set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); 329 return 0; 330 } 331 332 static int arch_timer_set_next_event_virt_mem(unsigned long evt, 333 struct clock_event_device *clk) 334 { 335 set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk); 336 return 0; 337 } 338 339 static int arch_timer_set_next_event_phys_mem(unsigned long evt, 340 struct clock_event_device *clk) 341 { 342 set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk); 343 return 0; 344 } 345 346 static void fsl_a008585_set_sne(struct clock_event_device *clk) 347 { 348 #ifdef CONFIG_FSL_ERRATUM_A008585 349 if (!static_branch_unlikely(&arch_timer_read_ool_enabled)) 350 return; 351 352 if (arch_timer_uses_ppi == VIRT_PPI) 353 clk->set_next_event = fsl_a008585_set_next_event_virt; 354 else 355 clk->set_next_event = fsl_a008585_set_next_event_phys; 356 #endif 357 } 358 359 static void __arch_timer_setup(unsigned type, 360 struct clock_event_device *clk) 361 { 362 clk->features = CLOCK_EVT_FEAT_ONESHOT; 363 364 if (type == ARCH_CP15_TIMER) { 365 if (arch_timer_c3stop) 366 clk->features |= CLOCK_EVT_FEAT_C3STOP; 367 clk->name = "arch_sys_timer"; 368 clk->rating = 450; 369 clk->cpumask = cpumask_of(smp_processor_id()); 370 clk->irq = arch_timer_ppi[arch_timer_uses_ppi]; 371 switch (arch_timer_uses_ppi) { 372 case VIRT_PPI: 373 clk->set_state_shutdown = arch_timer_shutdown_virt; 374 clk->set_state_oneshot_stopped = arch_timer_shutdown_virt; 375 clk->set_next_event = arch_timer_set_next_event_virt; 376 break; 377 case PHYS_SECURE_PPI: 378 case PHYS_NONSECURE_PPI: 379 case HYP_PPI: 380 clk->set_state_shutdown = arch_timer_shutdown_phys; 381 clk->set_state_oneshot_stopped = arch_timer_shutdown_phys; 382 clk->set_next_event = arch_timer_set_next_event_phys; 383 break; 384 default: 385 BUG(); 386 } 387 388 fsl_a008585_set_sne(clk); 389 } else { 390 clk->features |= CLOCK_EVT_FEAT_DYNIRQ; 391 clk->name = "arch_mem_timer"; 392 clk->rating = 400; 393 clk->cpumask = cpu_all_mask; 394 if (arch_timer_mem_use_virtual) { 395 clk->set_state_shutdown = arch_timer_shutdown_virt_mem; 396 clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem; 397 clk->set_next_event = 398 arch_timer_set_next_event_virt_mem; 399 } else { 400 clk->set_state_shutdown = arch_timer_shutdown_phys_mem; 401 clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem; 402 clk->set_next_event = 403 arch_timer_set_next_event_phys_mem; 404 } 405 } 406 407 clk->set_state_shutdown(clk); 408 409 clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); 410 } 411 412 static void arch_timer_evtstrm_enable(int divider) 413 { 414 u32 cntkctl = arch_timer_get_cntkctl(); 415 416 cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK; 417 /* Set the divider and enable virtual event stream */ 418 cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT) 419 | ARCH_TIMER_VIRT_EVT_EN; 420 arch_timer_set_cntkctl(cntkctl); 421 elf_hwcap |= HWCAP_EVTSTRM; 422 #ifdef CONFIG_COMPAT 423 compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM; 424 #endif 425 } 426 427 static void arch_timer_configure_evtstream(void) 428 { 429 int evt_stream_div, pos; 430 431 /* Find the closest power of two to the divisor */ 432 evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; 433 pos = fls(evt_stream_div); 434 if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) 435 pos--; 436 /* enable event stream */ 437 arch_timer_evtstrm_enable(min(pos, 15)); 438 } 439 440 static void arch_counter_set_user_access(void) 441 { 442 u32 cntkctl = arch_timer_get_cntkctl(); 443 444 /* Disable user access to the timers and the physical counter */ 445 /* Also disable virtual event stream */ 446 cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN 447 | ARCH_TIMER_USR_VT_ACCESS_EN 448 | ARCH_TIMER_VIRT_EVT_EN 449 | ARCH_TIMER_USR_PCT_ACCESS_EN); 450 451 /* Enable user access to the virtual counter */ 452 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; 453 454 arch_timer_set_cntkctl(cntkctl); 455 } 456 457 static bool arch_timer_has_nonsecure_ppi(void) 458 { 459 return (arch_timer_uses_ppi == PHYS_SECURE_PPI && 460 arch_timer_ppi[PHYS_NONSECURE_PPI]); 461 } 462 463 static u32 check_ppi_trigger(int irq) 464 { 465 u32 flags = irq_get_trigger_type(irq); 466 467 if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) { 468 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq); 469 pr_warn("WARNING: Please fix your firmware\n"); 470 flags = IRQF_TRIGGER_LOW; 471 } 472 473 return flags; 474 } 475 476 static int arch_timer_starting_cpu(unsigned int cpu) 477 { 478 struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); 479 u32 flags; 480 481 __arch_timer_setup(ARCH_CP15_TIMER, clk); 482 483 flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]); 484 enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags); 485 486 if (arch_timer_has_nonsecure_ppi()) { 487 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]); 488 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags); 489 } 490 491 arch_counter_set_user_access(); 492 if (evtstrm_enable) 493 arch_timer_configure_evtstream(); 494 495 return 0; 496 } 497 498 static void 499 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np) 500 { 501 /* Who has more than one independent system counter? */ 502 if (arch_timer_rate) 503 return; 504 505 /* 506 * Try to determine the frequency from the device tree or CNTFRQ, 507 * if ACPI is enabled, get the frequency from CNTFRQ ONLY. 508 */ 509 if (!acpi_disabled || 510 of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) { 511 if (cntbase) 512 arch_timer_rate = readl_relaxed(cntbase + CNTFRQ); 513 else 514 arch_timer_rate = arch_timer_get_cntfrq(); 515 } 516 517 /* Check the timer frequency. */ 518 if (arch_timer_rate == 0) 519 pr_warn("Architected timer frequency not available\n"); 520 } 521 522 static void arch_timer_banner(unsigned type) 523 { 524 pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n", 525 type & ARCH_CP15_TIMER ? "cp15" : "", 526 type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? " and " : "", 527 type & ARCH_MEM_TIMER ? "mmio" : "", 528 (unsigned long)arch_timer_rate / 1000000, 529 (unsigned long)(arch_timer_rate / 10000) % 100, 530 type & ARCH_CP15_TIMER ? 531 (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" : 532 "", 533 type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? "/" : "", 534 type & ARCH_MEM_TIMER ? 535 arch_timer_mem_use_virtual ? "virt" : "phys" : 536 ""); 537 } 538 539 u32 arch_timer_get_rate(void) 540 { 541 return arch_timer_rate; 542 } 543 544 static u64 arch_counter_get_cntvct_mem(void) 545 { 546 u32 vct_lo, vct_hi, tmp_hi; 547 548 do { 549 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); 550 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO); 551 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); 552 } while (vct_hi != tmp_hi); 553 554 return ((u64) vct_hi << 32) | vct_lo; 555 } 556 557 /* 558 * Default to cp15 based access because arm64 uses this function for 559 * sched_clock() before DT is probed and the cp15 method is guaranteed 560 * to exist on arm64. arm doesn't use this before DT is probed so even 561 * if we don't have the cp15 accessors we won't have a problem. 562 */ 563 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; 564 565 static u64 arch_counter_read(struct clocksource *cs) 566 { 567 return arch_timer_read_counter(); 568 } 569 570 static u64 arch_counter_read_cc(const struct cyclecounter *cc) 571 { 572 return arch_timer_read_counter(); 573 } 574 575 static struct clocksource clocksource_counter = { 576 .name = "arch_sys_counter", 577 .rating = 400, 578 .read = arch_counter_read, 579 .mask = CLOCKSOURCE_MASK(56), 580 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 581 }; 582 583 static struct cyclecounter cyclecounter = { 584 .read = arch_counter_read_cc, 585 .mask = CLOCKSOURCE_MASK(56), 586 }; 587 588 static struct arch_timer_kvm_info arch_timer_kvm_info; 589 590 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void) 591 { 592 return &arch_timer_kvm_info; 593 } 594 595 static void __init arch_counter_register(unsigned type) 596 { 597 u64 start_count; 598 599 /* Register the CP15 based counter if we have one */ 600 if (type & ARCH_CP15_TIMER) { 601 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI) 602 arch_timer_read_counter = arch_counter_get_cntvct; 603 else 604 arch_timer_read_counter = arch_counter_get_cntpct; 605 606 clocksource_counter.archdata.vdso_direct = true; 607 608 #ifdef CONFIG_FSL_ERRATUM_A008585 609 /* 610 * Don't use the vdso fastpath if errata require using 611 * the out-of-line counter accessor. 612 */ 613 if (static_branch_unlikely(&arch_timer_read_ool_enabled)) 614 clocksource_counter.archdata.vdso_direct = false; 615 #endif 616 } else { 617 arch_timer_read_counter = arch_counter_get_cntvct_mem; 618 } 619 620 if (!arch_counter_suspend_stop) 621 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP; 622 start_count = arch_timer_read_counter(); 623 clocksource_register_hz(&clocksource_counter, arch_timer_rate); 624 cyclecounter.mult = clocksource_counter.mult; 625 cyclecounter.shift = clocksource_counter.shift; 626 timecounter_init(&arch_timer_kvm_info.timecounter, 627 &cyclecounter, start_count); 628 629 /* 56 bits minimum, so we assume worst case rollover */ 630 sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate); 631 } 632 633 static void arch_timer_stop(struct clock_event_device *clk) 634 { 635 pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n", 636 clk->irq, smp_processor_id()); 637 638 disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]); 639 if (arch_timer_has_nonsecure_ppi()) 640 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]); 641 642 clk->set_state_shutdown(clk); 643 } 644 645 static int arch_timer_dying_cpu(unsigned int cpu) 646 { 647 struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); 648 649 arch_timer_stop(clk); 650 return 0; 651 } 652 653 #ifdef CONFIG_CPU_PM 654 static unsigned int saved_cntkctl; 655 static int arch_timer_cpu_pm_notify(struct notifier_block *self, 656 unsigned long action, void *hcpu) 657 { 658 if (action == CPU_PM_ENTER) 659 saved_cntkctl = arch_timer_get_cntkctl(); 660 else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) 661 arch_timer_set_cntkctl(saved_cntkctl); 662 return NOTIFY_OK; 663 } 664 665 static struct notifier_block arch_timer_cpu_pm_notifier = { 666 .notifier_call = arch_timer_cpu_pm_notify, 667 }; 668 669 static int __init arch_timer_cpu_pm_init(void) 670 { 671 return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier); 672 } 673 674 static void __init arch_timer_cpu_pm_deinit(void) 675 { 676 WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier)); 677 } 678 679 #else 680 static int __init arch_timer_cpu_pm_init(void) 681 { 682 return 0; 683 } 684 685 static void __init arch_timer_cpu_pm_deinit(void) 686 { 687 } 688 #endif 689 690 static int __init arch_timer_register(void) 691 { 692 int err; 693 int ppi; 694 695 arch_timer_evt = alloc_percpu(struct clock_event_device); 696 if (!arch_timer_evt) { 697 err = -ENOMEM; 698 goto out; 699 } 700 701 ppi = arch_timer_ppi[arch_timer_uses_ppi]; 702 switch (arch_timer_uses_ppi) { 703 case VIRT_PPI: 704 err = request_percpu_irq(ppi, arch_timer_handler_virt, 705 "arch_timer", arch_timer_evt); 706 break; 707 case PHYS_SECURE_PPI: 708 case PHYS_NONSECURE_PPI: 709 err = request_percpu_irq(ppi, arch_timer_handler_phys, 710 "arch_timer", arch_timer_evt); 711 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) { 712 ppi = arch_timer_ppi[PHYS_NONSECURE_PPI]; 713 err = request_percpu_irq(ppi, arch_timer_handler_phys, 714 "arch_timer", arch_timer_evt); 715 if (err) 716 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 717 arch_timer_evt); 718 } 719 break; 720 case HYP_PPI: 721 err = request_percpu_irq(ppi, arch_timer_handler_phys, 722 "arch_timer", arch_timer_evt); 723 break; 724 default: 725 BUG(); 726 } 727 728 if (err) { 729 pr_err("arch_timer: can't register interrupt %d (%d)\n", 730 ppi, err); 731 goto out_free; 732 } 733 734 err = arch_timer_cpu_pm_init(); 735 if (err) 736 goto out_unreg_notify; 737 738 739 /* Register and immediately configure the timer on the boot CPU */ 740 err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING, 741 "clockevents/arm/arch_timer:starting", 742 arch_timer_starting_cpu, arch_timer_dying_cpu); 743 if (err) 744 goto out_unreg_cpupm; 745 return 0; 746 747 out_unreg_cpupm: 748 arch_timer_cpu_pm_deinit(); 749 750 out_unreg_notify: 751 free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt); 752 if (arch_timer_has_nonsecure_ppi()) 753 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 754 arch_timer_evt); 755 756 out_free: 757 free_percpu(arch_timer_evt); 758 out: 759 return err; 760 } 761 762 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq) 763 { 764 int ret; 765 irq_handler_t func; 766 struct arch_timer *t; 767 768 t = kzalloc(sizeof(*t), GFP_KERNEL); 769 if (!t) 770 return -ENOMEM; 771 772 t->base = base; 773 t->evt.irq = irq; 774 __arch_timer_setup(ARCH_MEM_TIMER, &t->evt); 775 776 if (arch_timer_mem_use_virtual) 777 func = arch_timer_handler_virt_mem; 778 else 779 func = arch_timer_handler_phys_mem; 780 781 ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt); 782 if (ret) { 783 pr_err("arch_timer: Failed to request mem timer irq\n"); 784 kfree(t); 785 } 786 787 return ret; 788 } 789 790 static const struct of_device_id arch_timer_of_match[] __initconst = { 791 { .compatible = "arm,armv7-timer", }, 792 { .compatible = "arm,armv8-timer", }, 793 {}, 794 }; 795 796 static const struct of_device_id arch_timer_mem_of_match[] __initconst = { 797 { .compatible = "arm,armv7-timer-mem", }, 798 {}, 799 }; 800 801 static bool __init 802 arch_timer_needs_probing(int type, const struct of_device_id *matches) 803 { 804 struct device_node *dn; 805 bool needs_probing = false; 806 807 dn = of_find_matching_node(NULL, matches); 808 if (dn && of_device_is_available(dn) && !(arch_timers_present & type)) 809 needs_probing = true; 810 of_node_put(dn); 811 812 return needs_probing; 813 } 814 815 static int __init arch_timer_common_init(void) 816 { 817 unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER; 818 819 /* Wait until both nodes are probed if we have two timers */ 820 if ((arch_timers_present & mask) != mask) { 821 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match)) 822 return 0; 823 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match)) 824 return 0; 825 } 826 827 arch_timer_banner(arch_timers_present); 828 arch_counter_register(arch_timers_present); 829 return arch_timer_arch_init(); 830 } 831 832 static int __init arch_timer_init(void) 833 { 834 int ret; 835 /* 836 * If HYP mode is available, we know that the physical timer 837 * has been configured to be accessible from PL1. Use it, so 838 * that a guest can use the virtual timer instead. 839 * 840 * If no interrupt provided for virtual timer, we'll have to 841 * stick to the physical timer. It'd better be accessible... 842 * 843 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE 844 * accesses to CNTP_*_EL1 registers are silently redirected to 845 * their CNTHP_*_EL2 counterparts, and use a different PPI 846 * number. 847 */ 848 if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) { 849 bool has_ppi; 850 851 if (is_kernel_in_hyp_mode()) { 852 arch_timer_uses_ppi = HYP_PPI; 853 has_ppi = !!arch_timer_ppi[HYP_PPI]; 854 } else { 855 arch_timer_uses_ppi = PHYS_SECURE_PPI; 856 has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] || 857 !!arch_timer_ppi[PHYS_NONSECURE_PPI]); 858 } 859 860 if (!has_ppi) { 861 pr_warn("arch_timer: No interrupt available, giving up\n"); 862 return -EINVAL; 863 } 864 } 865 866 ret = arch_timer_register(); 867 if (ret) 868 return ret; 869 870 ret = arch_timer_common_init(); 871 if (ret) 872 return ret; 873 874 arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI]; 875 876 return 0; 877 } 878 879 static int __init arch_timer_of_init(struct device_node *np) 880 { 881 int i; 882 883 if (arch_timers_present & ARCH_CP15_TIMER) { 884 pr_warn("arch_timer: multiple nodes in dt, skipping\n"); 885 return 0; 886 } 887 888 arch_timers_present |= ARCH_CP15_TIMER; 889 for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++) 890 arch_timer_ppi[i] = irq_of_parse_and_map(np, i); 891 892 arch_timer_detect_rate(NULL, np); 893 894 arch_timer_c3stop = !of_property_read_bool(np, "always-on"); 895 896 #ifdef CONFIG_FSL_ERRATUM_A008585 897 if (fsl_a008585_enable < 0) 898 fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585"); 899 if (fsl_a008585_enable) { 900 static_branch_enable(&arch_timer_read_ool_enabled); 901 pr_info("Enabling workaround for FSL erratum A-008585\n"); 902 } 903 #endif 904 905 /* 906 * If we cannot rely on firmware initializing the timer registers then 907 * we should use the physical timers instead. 908 */ 909 if (IS_ENABLED(CONFIG_ARM) && 910 of_property_read_bool(np, "arm,cpu-registers-not-fw-configured")) 911 arch_timer_uses_ppi = PHYS_SECURE_PPI; 912 913 /* On some systems, the counter stops ticking when in suspend. */ 914 arch_counter_suspend_stop = of_property_read_bool(np, 915 "arm,no-tick-in-suspend"); 916 917 return arch_timer_init(); 918 } 919 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init); 920 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init); 921 922 static int __init arch_timer_mem_init(struct device_node *np) 923 { 924 struct device_node *frame, *best_frame = NULL; 925 void __iomem *cntctlbase, *base; 926 unsigned int irq, ret = -EINVAL; 927 u32 cnttidr; 928 929 arch_timers_present |= ARCH_MEM_TIMER; 930 cntctlbase = of_iomap(np, 0); 931 if (!cntctlbase) { 932 pr_err("arch_timer: Can't find CNTCTLBase\n"); 933 return -ENXIO; 934 } 935 936 cnttidr = readl_relaxed(cntctlbase + CNTTIDR); 937 938 /* 939 * Try to find a virtual capable frame. Otherwise fall back to a 940 * physical capable frame. 941 */ 942 for_each_available_child_of_node(np, frame) { 943 int n; 944 u32 cntacr; 945 946 if (of_property_read_u32(frame, "frame-number", &n)) { 947 pr_err("arch_timer: Missing frame-number\n"); 948 of_node_put(frame); 949 goto out; 950 } 951 952 /* Try enabling everything, and see what sticks */ 953 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT | 954 CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT; 955 writel_relaxed(cntacr, cntctlbase + CNTACR(n)); 956 cntacr = readl_relaxed(cntctlbase + CNTACR(n)); 957 958 if ((cnttidr & CNTTIDR_VIRT(n)) && 959 !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) { 960 of_node_put(best_frame); 961 best_frame = frame; 962 arch_timer_mem_use_virtual = true; 963 break; 964 } 965 966 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT)) 967 continue; 968 969 of_node_put(best_frame); 970 best_frame = of_node_get(frame); 971 } 972 973 ret= -ENXIO; 974 base = arch_counter_base = of_io_request_and_map(best_frame, 0, 975 "arch_mem_timer"); 976 if (IS_ERR(base)) { 977 pr_err("arch_timer: Can't map frame's registers\n"); 978 goto out; 979 } 980 981 if (arch_timer_mem_use_virtual) 982 irq = irq_of_parse_and_map(best_frame, 1); 983 else 984 irq = irq_of_parse_and_map(best_frame, 0); 985 986 ret = -EINVAL; 987 if (!irq) { 988 pr_err("arch_timer: Frame missing %s irq", 989 arch_timer_mem_use_virtual ? "virt" : "phys"); 990 goto out; 991 } 992 993 arch_timer_detect_rate(base, np); 994 ret = arch_timer_mem_register(base, irq); 995 if (ret) 996 goto out; 997 998 return arch_timer_common_init(); 999 out: 1000 iounmap(cntctlbase); 1001 of_node_put(best_frame); 1002 return ret; 1003 } 1004 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem", 1005 arch_timer_mem_init); 1006 1007 #ifdef CONFIG_ACPI 1008 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags) 1009 { 1010 int trigger, polarity; 1011 1012 if (!interrupt) 1013 return 0; 1014 1015 trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE 1016 : ACPI_LEVEL_SENSITIVE; 1017 1018 polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW 1019 : ACPI_ACTIVE_HIGH; 1020 1021 return acpi_register_gsi(NULL, interrupt, trigger, polarity); 1022 } 1023 1024 /* Initialize per-processor generic timer */ 1025 static int __init arch_timer_acpi_init(struct acpi_table_header *table) 1026 { 1027 struct acpi_table_gtdt *gtdt; 1028 1029 if (arch_timers_present & ARCH_CP15_TIMER) { 1030 pr_warn("arch_timer: already initialized, skipping\n"); 1031 return -EINVAL; 1032 } 1033 1034 gtdt = container_of(table, struct acpi_table_gtdt, header); 1035 1036 arch_timers_present |= ARCH_CP15_TIMER; 1037 1038 arch_timer_ppi[PHYS_SECURE_PPI] = 1039 map_generic_timer_interrupt(gtdt->secure_el1_interrupt, 1040 gtdt->secure_el1_flags); 1041 1042 arch_timer_ppi[PHYS_NONSECURE_PPI] = 1043 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt, 1044 gtdt->non_secure_el1_flags); 1045 1046 arch_timer_ppi[VIRT_PPI] = 1047 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt, 1048 gtdt->virtual_timer_flags); 1049 1050 arch_timer_ppi[HYP_PPI] = 1051 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt, 1052 gtdt->non_secure_el2_flags); 1053 1054 /* Get the frequency from CNTFRQ */ 1055 arch_timer_detect_rate(NULL, NULL); 1056 1057 /* Always-on capability */ 1058 arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON); 1059 1060 arch_timer_init(); 1061 return 0; 1062 } 1063 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init); 1064 #endif 1065