1 /* 2 * linux/drivers/clocksource/arm_arch_timer.c 3 * 4 * Copyright (C) 2011 ARM Ltd. 5 * All Rights Reserved 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #define pr_fmt(fmt) "arm_arch_timer: " fmt 13 14 #include <linux/init.h> 15 #include <linux/kernel.h> 16 #include <linux/device.h> 17 #include <linux/smp.h> 18 #include <linux/cpu.h> 19 #include <linux/cpu_pm.h> 20 #include <linux/clockchips.h> 21 #include <linux/clocksource.h> 22 #include <linux/interrupt.h> 23 #include <linux/of_irq.h> 24 #include <linux/of_address.h> 25 #include <linux/io.h> 26 #include <linux/slab.h> 27 #include <linux/sched/clock.h> 28 #include <linux/sched_clock.h> 29 #include <linux/acpi.h> 30 31 #include <asm/arch_timer.h> 32 #include <asm/virt.h> 33 34 #include <clocksource/arm_arch_timer.h> 35 36 #undef pr_fmt 37 #define pr_fmt(fmt) "arch_timer: " fmt 38 39 #define CNTTIDR 0x08 40 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4)) 41 42 #define CNTACR(n) (0x40 + ((n) * 4)) 43 #define CNTACR_RPCT BIT(0) 44 #define CNTACR_RVCT BIT(1) 45 #define CNTACR_RFRQ BIT(2) 46 #define CNTACR_RVOFF BIT(3) 47 #define CNTACR_RWVT BIT(4) 48 #define CNTACR_RWPT BIT(5) 49 50 #define CNTVCT_LO 0x08 51 #define CNTVCT_HI 0x0c 52 #define CNTFRQ 0x10 53 #define CNTP_TVAL 0x28 54 #define CNTP_CTL 0x2c 55 #define CNTV_TVAL 0x38 56 #define CNTV_CTL 0x3c 57 58 static unsigned arch_timers_present __initdata; 59 60 static void __iomem *arch_counter_base; 61 62 struct arch_timer { 63 void __iomem *base; 64 struct clock_event_device evt; 65 }; 66 67 #define to_arch_timer(e) container_of(e, struct arch_timer, evt) 68 69 static u32 arch_timer_rate; 70 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI]; 71 72 static struct clock_event_device __percpu *arch_timer_evt; 73 74 static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI; 75 static bool arch_timer_c3stop; 76 static bool arch_timer_mem_use_virtual; 77 static bool arch_counter_suspend_stop; 78 static bool vdso_default = true; 79 80 static cpumask_t evtstrm_available = CPU_MASK_NONE; 81 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); 82 83 static int __init early_evtstrm_cfg(char *buf) 84 { 85 return strtobool(buf, &evtstrm_enable); 86 } 87 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); 88 89 /* 90 * Architected system timer support. 91 */ 92 93 static __always_inline 94 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, 95 struct clock_event_device *clk) 96 { 97 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { 98 struct arch_timer *timer = to_arch_timer(clk); 99 switch (reg) { 100 case ARCH_TIMER_REG_CTRL: 101 writel_relaxed(val, timer->base + CNTP_CTL); 102 break; 103 case ARCH_TIMER_REG_TVAL: 104 writel_relaxed(val, timer->base + CNTP_TVAL); 105 break; 106 } 107 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { 108 struct arch_timer *timer = to_arch_timer(clk); 109 switch (reg) { 110 case ARCH_TIMER_REG_CTRL: 111 writel_relaxed(val, timer->base + CNTV_CTL); 112 break; 113 case ARCH_TIMER_REG_TVAL: 114 writel_relaxed(val, timer->base + CNTV_TVAL); 115 break; 116 } 117 } else { 118 arch_timer_reg_write_cp15(access, reg, val); 119 } 120 } 121 122 static __always_inline 123 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, 124 struct clock_event_device *clk) 125 { 126 u32 val; 127 128 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { 129 struct arch_timer *timer = to_arch_timer(clk); 130 switch (reg) { 131 case ARCH_TIMER_REG_CTRL: 132 val = readl_relaxed(timer->base + CNTP_CTL); 133 break; 134 case ARCH_TIMER_REG_TVAL: 135 val = readl_relaxed(timer->base + CNTP_TVAL); 136 break; 137 } 138 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { 139 struct arch_timer *timer = to_arch_timer(clk); 140 switch (reg) { 141 case ARCH_TIMER_REG_CTRL: 142 val = readl_relaxed(timer->base + CNTV_CTL); 143 break; 144 case ARCH_TIMER_REG_TVAL: 145 val = readl_relaxed(timer->base + CNTV_TVAL); 146 break; 147 } 148 } else { 149 val = arch_timer_reg_read_cp15(access, reg); 150 } 151 152 return val; 153 } 154 155 /* 156 * Default to cp15 based access because arm64 uses this function for 157 * sched_clock() before DT is probed and the cp15 method is guaranteed 158 * to exist on arm64. arm doesn't use this before DT is probed so even 159 * if we don't have the cp15 accessors we won't have a problem. 160 */ 161 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; 162 EXPORT_SYMBOL_GPL(arch_timer_read_counter); 163 164 static u64 arch_counter_read(struct clocksource *cs) 165 { 166 return arch_timer_read_counter(); 167 } 168 169 static u64 arch_counter_read_cc(const struct cyclecounter *cc) 170 { 171 return arch_timer_read_counter(); 172 } 173 174 static struct clocksource clocksource_counter = { 175 .name = "arch_sys_counter", 176 .rating = 400, 177 .read = arch_counter_read, 178 .mask = CLOCKSOURCE_MASK(56), 179 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 180 }; 181 182 static struct cyclecounter cyclecounter __ro_after_init = { 183 .read = arch_counter_read_cc, 184 .mask = CLOCKSOURCE_MASK(56), 185 }; 186 187 struct ate_acpi_oem_info { 188 char oem_id[ACPI_OEM_ID_SIZE + 1]; 189 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; 190 u32 oem_revision; 191 }; 192 193 #ifdef CONFIG_FSL_ERRATUM_A008585 194 /* 195 * The number of retries is an arbitrary value well beyond the highest number 196 * of iterations the loop has been observed to take. 197 */ 198 #define __fsl_a008585_read_reg(reg) ({ \ 199 u64 _old, _new; \ 200 int _retries = 200; \ 201 \ 202 do { \ 203 _old = read_sysreg(reg); \ 204 _new = read_sysreg(reg); \ 205 _retries--; \ 206 } while (unlikely(_old != _new) && _retries); \ 207 \ 208 WARN_ON_ONCE(!_retries); \ 209 _new; \ 210 }) 211 212 static u32 notrace fsl_a008585_read_cntp_tval_el0(void) 213 { 214 return __fsl_a008585_read_reg(cntp_tval_el0); 215 } 216 217 static u32 notrace fsl_a008585_read_cntv_tval_el0(void) 218 { 219 return __fsl_a008585_read_reg(cntv_tval_el0); 220 } 221 222 static u64 notrace fsl_a008585_read_cntpct_el0(void) 223 { 224 return __fsl_a008585_read_reg(cntpct_el0); 225 } 226 227 static u64 notrace fsl_a008585_read_cntvct_el0(void) 228 { 229 return __fsl_a008585_read_reg(cntvct_el0); 230 } 231 #endif 232 233 #ifdef CONFIG_HISILICON_ERRATUM_161010101 234 /* 235 * Verify whether the value of the second read is larger than the first by 236 * less than 32 is the only way to confirm the value is correct, so clear the 237 * lower 5 bits to check whether the difference is greater than 32 or not. 238 * Theoretically the erratum should not occur more than twice in succession 239 * when reading the system counter, but it is possible that some interrupts 240 * may lead to more than twice read errors, triggering the warning, so setting 241 * the number of retries far beyond the number of iterations the loop has been 242 * observed to take. 243 */ 244 #define __hisi_161010101_read_reg(reg) ({ \ 245 u64 _old, _new; \ 246 int _retries = 50; \ 247 \ 248 do { \ 249 _old = read_sysreg(reg); \ 250 _new = read_sysreg(reg); \ 251 _retries--; \ 252 } while (unlikely((_new - _old) >> 5) && _retries); \ 253 \ 254 WARN_ON_ONCE(!_retries); \ 255 _new; \ 256 }) 257 258 static u32 notrace hisi_161010101_read_cntp_tval_el0(void) 259 { 260 return __hisi_161010101_read_reg(cntp_tval_el0); 261 } 262 263 static u32 notrace hisi_161010101_read_cntv_tval_el0(void) 264 { 265 return __hisi_161010101_read_reg(cntv_tval_el0); 266 } 267 268 static u64 notrace hisi_161010101_read_cntpct_el0(void) 269 { 270 return __hisi_161010101_read_reg(cntpct_el0); 271 } 272 273 static u64 notrace hisi_161010101_read_cntvct_el0(void) 274 { 275 return __hisi_161010101_read_reg(cntvct_el0); 276 } 277 278 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = { 279 /* 280 * Note that trailing spaces are required to properly match 281 * the OEM table information. 282 */ 283 { 284 .oem_id = "HISI ", 285 .oem_table_id = "HIP05 ", 286 .oem_revision = 0, 287 }, 288 { 289 .oem_id = "HISI ", 290 .oem_table_id = "HIP06 ", 291 .oem_revision = 0, 292 }, 293 { 294 .oem_id = "HISI ", 295 .oem_table_id = "HIP07 ", 296 .oem_revision = 0, 297 }, 298 { /* Sentinel indicating the end of the OEM array */ }, 299 }; 300 #endif 301 302 #ifdef CONFIG_ARM64_ERRATUM_858921 303 static u64 notrace arm64_858921_read_cntpct_el0(void) 304 { 305 u64 old, new; 306 307 old = read_sysreg(cntpct_el0); 308 new = read_sysreg(cntpct_el0); 309 return (((old ^ new) >> 32) & 1) ? old : new; 310 } 311 312 static u64 notrace arm64_858921_read_cntvct_el0(void) 313 { 314 u64 old, new; 315 316 old = read_sysreg(cntvct_el0); 317 new = read_sysreg(cntvct_el0); 318 return (((old ^ new) >> 32) & 1) ? old : new; 319 } 320 #endif 321 322 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND 323 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround); 324 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); 325 326 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); 327 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); 328 329 static void erratum_set_next_event_tval_generic(const int access, unsigned long evt, 330 struct clock_event_device *clk) 331 { 332 unsigned long ctrl; 333 u64 cval; 334 335 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 336 ctrl |= ARCH_TIMER_CTRL_ENABLE; 337 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 338 339 if (access == ARCH_TIMER_PHYS_ACCESS) { 340 cval = evt + arch_counter_get_cntpct(); 341 write_sysreg(cval, cntp_cval_el0); 342 } else { 343 cval = evt + arch_counter_get_cntvct(); 344 write_sysreg(cval, cntv_cval_el0); 345 } 346 347 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 348 } 349 350 static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt, 351 struct clock_event_device *clk) 352 { 353 erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk); 354 return 0; 355 } 356 357 static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt, 358 struct clock_event_device *clk) 359 { 360 erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk); 361 return 0; 362 } 363 364 static const struct arch_timer_erratum_workaround ool_workarounds[] = { 365 #ifdef CONFIG_FSL_ERRATUM_A008585 366 { 367 .match_type = ate_match_dt, 368 .id = "fsl,erratum-a008585", 369 .desc = "Freescale erratum a005858", 370 .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0, 371 .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0, 372 .read_cntpct_el0 = fsl_a008585_read_cntpct_el0, 373 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0, 374 .set_next_event_phys = erratum_set_next_event_tval_phys, 375 .set_next_event_virt = erratum_set_next_event_tval_virt, 376 }, 377 #endif 378 #ifdef CONFIG_HISILICON_ERRATUM_161010101 379 { 380 .match_type = ate_match_dt, 381 .id = "hisilicon,erratum-161010101", 382 .desc = "HiSilicon erratum 161010101", 383 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0, 384 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0, 385 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0, 386 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0, 387 .set_next_event_phys = erratum_set_next_event_tval_phys, 388 .set_next_event_virt = erratum_set_next_event_tval_virt, 389 }, 390 { 391 .match_type = ate_match_acpi_oem_info, 392 .id = hisi_161010101_oem_info, 393 .desc = "HiSilicon erratum 161010101", 394 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0, 395 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0, 396 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0, 397 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0, 398 .set_next_event_phys = erratum_set_next_event_tval_phys, 399 .set_next_event_virt = erratum_set_next_event_tval_virt, 400 }, 401 #endif 402 #ifdef CONFIG_ARM64_ERRATUM_858921 403 { 404 .match_type = ate_match_local_cap_id, 405 .id = (void *)ARM64_WORKAROUND_858921, 406 .desc = "ARM erratum 858921", 407 .read_cntpct_el0 = arm64_858921_read_cntpct_el0, 408 .read_cntvct_el0 = arm64_858921_read_cntvct_el0, 409 }, 410 #endif 411 }; 412 413 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *, 414 const void *); 415 416 static 417 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa, 418 const void *arg) 419 { 420 const struct device_node *np = arg; 421 422 return of_property_read_bool(np, wa->id); 423 } 424 425 static 426 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa, 427 const void *arg) 428 { 429 return this_cpu_has_cap((uintptr_t)wa->id); 430 } 431 432 433 static 434 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa, 435 const void *arg) 436 { 437 static const struct ate_acpi_oem_info empty_oem_info = {}; 438 const struct ate_acpi_oem_info *info = wa->id; 439 const struct acpi_table_header *table = arg; 440 441 /* Iterate over the ACPI OEM info array, looking for a match */ 442 while (memcmp(info, &empty_oem_info, sizeof(*info))) { 443 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) && 444 !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) && 445 info->oem_revision == table->oem_revision) 446 return true; 447 448 info++; 449 } 450 451 return false; 452 } 453 454 static const struct arch_timer_erratum_workaround * 455 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, 456 ate_match_fn_t match_fn, 457 void *arg) 458 { 459 int i; 460 461 for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) { 462 if (ool_workarounds[i].match_type != type) 463 continue; 464 465 if (match_fn(&ool_workarounds[i], arg)) 466 return &ool_workarounds[i]; 467 } 468 469 return NULL; 470 } 471 472 static 473 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa, 474 bool local) 475 { 476 int i; 477 478 if (local) { 479 __this_cpu_write(timer_unstable_counter_workaround, wa); 480 } else { 481 for_each_possible_cpu(i) 482 per_cpu(timer_unstable_counter_workaround, i) = wa; 483 } 484 485 /* 486 * Use the locked version, as we're called from the CPU 487 * hotplug framework. Otherwise, we end-up in deadlock-land. 488 */ 489 static_branch_enable_cpuslocked(&arch_timer_read_ool_enabled); 490 491 /* 492 * Don't use the vdso fastpath if errata require using the 493 * out-of-line counter accessor. We may change our mind pretty 494 * late in the game (with a per-CPU erratum, for example), so 495 * change both the default value and the vdso itself. 496 */ 497 if (wa->read_cntvct_el0) { 498 clocksource_counter.archdata.vdso_direct = false; 499 vdso_default = false; 500 } 501 } 502 503 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type, 504 void *arg) 505 { 506 const struct arch_timer_erratum_workaround *wa; 507 ate_match_fn_t match_fn = NULL; 508 bool local = false; 509 510 switch (type) { 511 case ate_match_dt: 512 match_fn = arch_timer_check_dt_erratum; 513 break; 514 case ate_match_local_cap_id: 515 match_fn = arch_timer_check_local_cap_erratum; 516 local = true; 517 break; 518 case ate_match_acpi_oem_info: 519 match_fn = arch_timer_check_acpi_oem_erratum; 520 break; 521 default: 522 WARN_ON(1); 523 return; 524 } 525 526 wa = arch_timer_iterate_errata(type, match_fn, arg); 527 if (!wa) 528 return; 529 530 if (needs_unstable_timer_counter_workaround()) { 531 const struct arch_timer_erratum_workaround *__wa; 532 __wa = __this_cpu_read(timer_unstable_counter_workaround); 533 if (__wa && wa != __wa) 534 pr_warn("Can't enable workaround for %s (clashes with %s\n)", 535 wa->desc, __wa->desc); 536 537 if (__wa) 538 return; 539 } 540 541 arch_timer_enable_workaround(wa, local); 542 pr_info("Enabling %s workaround for %s\n", 543 local ? "local" : "global", wa->desc); 544 } 545 546 #define erratum_handler(fn, r, ...) \ 547 ({ \ 548 bool __val; \ 549 if (needs_unstable_timer_counter_workaround()) { \ 550 const struct arch_timer_erratum_workaround *__wa; \ 551 __wa = __this_cpu_read(timer_unstable_counter_workaround); \ 552 if (__wa && __wa->fn) { \ 553 r = __wa->fn(__VA_ARGS__); \ 554 __val = true; \ 555 } else { \ 556 __val = false; \ 557 } \ 558 } else { \ 559 __val = false; \ 560 } \ 561 __val; \ 562 }) 563 564 static bool arch_timer_this_cpu_has_cntvct_wa(void) 565 { 566 const struct arch_timer_erratum_workaround *wa; 567 568 wa = __this_cpu_read(timer_unstable_counter_workaround); 569 return wa && wa->read_cntvct_el0; 570 } 571 #else 572 #define arch_timer_check_ool_workaround(t,a) do { } while(0) 573 #define erratum_set_next_event_tval_virt(...) ({BUG(); 0;}) 574 #define erratum_set_next_event_tval_phys(...) ({BUG(); 0;}) 575 #define erratum_handler(fn, r, ...) ({false;}) 576 #define arch_timer_this_cpu_has_cntvct_wa() ({false;}) 577 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ 578 579 static __always_inline irqreturn_t timer_handler(const int access, 580 struct clock_event_device *evt) 581 { 582 unsigned long ctrl; 583 584 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt); 585 if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { 586 ctrl |= ARCH_TIMER_CTRL_IT_MASK; 587 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt); 588 evt->event_handler(evt); 589 return IRQ_HANDLED; 590 } 591 592 return IRQ_NONE; 593 } 594 595 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id) 596 { 597 struct clock_event_device *evt = dev_id; 598 599 return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt); 600 } 601 602 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id) 603 { 604 struct clock_event_device *evt = dev_id; 605 606 return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt); 607 } 608 609 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id) 610 { 611 struct clock_event_device *evt = dev_id; 612 613 return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt); 614 } 615 616 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id) 617 { 618 struct clock_event_device *evt = dev_id; 619 620 return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt); 621 } 622 623 static __always_inline int timer_shutdown(const int access, 624 struct clock_event_device *clk) 625 { 626 unsigned long ctrl; 627 628 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 629 ctrl &= ~ARCH_TIMER_CTRL_ENABLE; 630 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 631 632 return 0; 633 } 634 635 static int arch_timer_shutdown_virt(struct clock_event_device *clk) 636 { 637 return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk); 638 } 639 640 static int arch_timer_shutdown_phys(struct clock_event_device *clk) 641 { 642 return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk); 643 } 644 645 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk) 646 { 647 return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk); 648 } 649 650 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk) 651 { 652 return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk); 653 } 654 655 static __always_inline void set_next_event(const int access, unsigned long evt, 656 struct clock_event_device *clk) 657 { 658 unsigned long ctrl; 659 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 660 ctrl |= ARCH_TIMER_CTRL_ENABLE; 661 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 662 arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk); 663 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 664 } 665 666 static int arch_timer_set_next_event_virt(unsigned long evt, 667 struct clock_event_device *clk) 668 { 669 int ret; 670 671 if (erratum_handler(set_next_event_virt, ret, evt, clk)) 672 return ret; 673 674 set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); 675 return 0; 676 } 677 678 static int arch_timer_set_next_event_phys(unsigned long evt, 679 struct clock_event_device *clk) 680 { 681 int ret; 682 683 if (erratum_handler(set_next_event_phys, ret, evt, clk)) 684 return ret; 685 686 set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); 687 return 0; 688 } 689 690 static int arch_timer_set_next_event_virt_mem(unsigned long evt, 691 struct clock_event_device *clk) 692 { 693 set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk); 694 return 0; 695 } 696 697 static int arch_timer_set_next_event_phys_mem(unsigned long evt, 698 struct clock_event_device *clk) 699 { 700 set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk); 701 return 0; 702 } 703 704 static void __arch_timer_setup(unsigned type, 705 struct clock_event_device *clk) 706 { 707 clk->features = CLOCK_EVT_FEAT_ONESHOT; 708 709 if (type == ARCH_TIMER_TYPE_CP15) { 710 if (arch_timer_c3stop) 711 clk->features |= CLOCK_EVT_FEAT_C3STOP; 712 clk->name = "arch_sys_timer"; 713 clk->rating = 450; 714 clk->cpumask = cpumask_of(smp_processor_id()); 715 clk->irq = arch_timer_ppi[arch_timer_uses_ppi]; 716 switch (arch_timer_uses_ppi) { 717 case ARCH_TIMER_VIRT_PPI: 718 clk->set_state_shutdown = arch_timer_shutdown_virt; 719 clk->set_state_oneshot_stopped = arch_timer_shutdown_virt; 720 clk->set_next_event = arch_timer_set_next_event_virt; 721 break; 722 case ARCH_TIMER_PHYS_SECURE_PPI: 723 case ARCH_TIMER_PHYS_NONSECURE_PPI: 724 case ARCH_TIMER_HYP_PPI: 725 clk->set_state_shutdown = arch_timer_shutdown_phys; 726 clk->set_state_oneshot_stopped = arch_timer_shutdown_phys; 727 clk->set_next_event = arch_timer_set_next_event_phys; 728 break; 729 default: 730 BUG(); 731 } 732 733 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL); 734 } else { 735 clk->features |= CLOCK_EVT_FEAT_DYNIRQ; 736 clk->name = "arch_mem_timer"; 737 clk->rating = 400; 738 clk->cpumask = cpu_all_mask; 739 if (arch_timer_mem_use_virtual) { 740 clk->set_state_shutdown = arch_timer_shutdown_virt_mem; 741 clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem; 742 clk->set_next_event = 743 arch_timer_set_next_event_virt_mem; 744 } else { 745 clk->set_state_shutdown = arch_timer_shutdown_phys_mem; 746 clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem; 747 clk->set_next_event = 748 arch_timer_set_next_event_phys_mem; 749 } 750 } 751 752 clk->set_state_shutdown(clk); 753 754 clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); 755 } 756 757 static void arch_timer_evtstrm_enable(int divider) 758 { 759 u32 cntkctl = arch_timer_get_cntkctl(); 760 761 cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK; 762 /* Set the divider and enable virtual event stream */ 763 cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT) 764 | ARCH_TIMER_VIRT_EVT_EN; 765 arch_timer_set_cntkctl(cntkctl); 766 elf_hwcap |= HWCAP_EVTSTRM; 767 #ifdef CONFIG_COMPAT 768 compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM; 769 #endif 770 cpumask_set_cpu(smp_processor_id(), &evtstrm_available); 771 } 772 773 static void arch_timer_configure_evtstream(void) 774 { 775 int evt_stream_div, pos; 776 777 /* Find the closest power of two to the divisor */ 778 evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; 779 pos = fls(evt_stream_div); 780 if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) 781 pos--; 782 /* enable event stream */ 783 arch_timer_evtstrm_enable(min(pos, 15)); 784 } 785 786 static void arch_counter_set_user_access(void) 787 { 788 u32 cntkctl = arch_timer_get_cntkctl(); 789 790 /* Disable user access to the timers and both counters */ 791 /* Also disable virtual event stream */ 792 cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN 793 | ARCH_TIMER_USR_VT_ACCESS_EN 794 | ARCH_TIMER_USR_VCT_ACCESS_EN 795 | ARCH_TIMER_VIRT_EVT_EN 796 | ARCH_TIMER_USR_PCT_ACCESS_EN); 797 798 /* 799 * Enable user access to the virtual counter if it doesn't 800 * need to be workaround. The vdso may have been already 801 * disabled though. 802 */ 803 if (arch_timer_this_cpu_has_cntvct_wa()) 804 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id()); 805 else 806 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; 807 808 arch_timer_set_cntkctl(cntkctl); 809 } 810 811 static bool arch_timer_has_nonsecure_ppi(void) 812 { 813 return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI && 814 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]); 815 } 816 817 static u32 check_ppi_trigger(int irq) 818 { 819 u32 flags = irq_get_trigger_type(irq); 820 821 if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) { 822 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq); 823 pr_warn("WARNING: Please fix your firmware\n"); 824 flags = IRQF_TRIGGER_LOW; 825 } 826 827 return flags; 828 } 829 830 static int arch_timer_starting_cpu(unsigned int cpu) 831 { 832 struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); 833 u32 flags; 834 835 __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk); 836 837 flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]); 838 enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags); 839 840 if (arch_timer_has_nonsecure_ppi()) { 841 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]); 842 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI], 843 flags); 844 } 845 846 arch_counter_set_user_access(); 847 if (evtstrm_enable) 848 arch_timer_configure_evtstream(); 849 850 return 0; 851 } 852 853 /* 854 * For historical reasons, when probing with DT we use whichever (non-zero) 855 * rate was probed first, and don't verify that others match. If the first node 856 * probed has a clock-frequency property, this overrides the HW register. 857 */ 858 static void arch_timer_of_configure_rate(u32 rate, struct device_node *np) 859 { 860 /* Who has more than one independent system counter? */ 861 if (arch_timer_rate) 862 return; 863 864 if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) 865 arch_timer_rate = rate; 866 867 /* Check the timer frequency. */ 868 if (arch_timer_rate == 0) 869 pr_warn("frequency not available\n"); 870 } 871 872 static void arch_timer_banner(unsigned type) 873 { 874 pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n", 875 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "", 876 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? 877 " and " : "", 878 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "", 879 (unsigned long)arch_timer_rate / 1000000, 880 (unsigned long)(arch_timer_rate / 10000) % 100, 881 type & ARCH_TIMER_TYPE_CP15 ? 882 (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" : 883 "", 884 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "", 885 type & ARCH_TIMER_TYPE_MEM ? 886 arch_timer_mem_use_virtual ? "virt" : "phys" : 887 ""); 888 } 889 890 u32 arch_timer_get_rate(void) 891 { 892 return arch_timer_rate; 893 } 894 895 bool arch_timer_evtstrm_available(void) 896 { 897 /* 898 * We might get called from a preemptible context. This is fine 899 * because availability of the event stream should be always the same 900 * for a preemptible context and context where we might resume a task. 901 */ 902 return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available); 903 } 904 905 static u64 arch_counter_get_cntvct_mem(void) 906 { 907 u32 vct_lo, vct_hi, tmp_hi; 908 909 do { 910 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); 911 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO); 912 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); 913 } while (vct_hi != tmp_hi); 914 915 return ((u64) vct_hi << 32) | vct_lo; 916 } 917 918 static struct arch_timer_kvm_info arch_timer_kvm_info; 919 920 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void) 921 { 922 return &arch_timer_kvm_info; 923 } 924 925 static void __init arch_counter_register(unsigned type) 926 { 927 u64 start_count; 928 929 /* Register the CP15 based counter if we have one */ 930 if (type & ARCH_TIMER_TYPE_CP15) { 931 if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) || 932 arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) 933 arch_timer_read_counter = arch_counter_get_cntvct; 934 else 935 arch_timer_read_counter = arch_counter_get_cntpct; 936 937 clocksource_counter.archdata.vdso_direct = vdso_default; 938 } else { 939 arch_timer_read_counter = arch_counter_get_cntvct_mem; 940 } 941 942 if (!arch_counter_suspend_stop) 943 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP; 944 start_count = arch_timer_read_counter(); 945 clocksource_register_hz(&clocksource_counter, arch_timer_rate); 946 cyclecounter.mult = clocksource_counter.mult; 947 cyclecounter.shift = clocksource_counter.shift; 948 timecounter_init(&arch_timer_kvm_info.timecounter, 949 &cyclecounter, start_count); 950 951 /* 56 bits minimum, so we assume worst case rollover */ 952 sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate); 953 } 954 955 static void arch_timer_stop(struct clock_event_device *clk) 956 { 957 pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id()); 958 959 disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]); 960 if (arch_timer_has_nonsecure_ppi()) 961 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]); 962 963 clk->set_state_shutdown(clk); 964 } 965 966 static int arch_timer_dying_cpu(unsigned int cpu) 967 { 968 struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); 969 970 cpumask_clear_cpu(smp_processor_id(), &evtstrm_available); 971 972 arch_timer_stop(clk); 973 return 0; 974 } 975 976 #ifdef CONFIG_CPU_PM 977 static DEFINE_PER_CPU(unsigned long, saved_cntkctl); 978 static int arch_timer_cpu_pm_notify(struct notifier_block *self, 979 unsigned long action, void *hcpu) 980 { 981 if (action == CPU_PM_ENTER) { 982 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl()); 983 984 cpumask_clear_cpu(smp_processor_id(), &evtstrm_available); 985 } else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) { 986 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl)); 987 988 if (elf_hwcap & HWCAP_EVTSTRM) 989 cpumask_set_cpu(smp_processor_id(), &evtstrm_available); 990 } 991 return NOTIFY_OK; 992 } 993 994 static struct notifier_block arch_timer_cpu_pm_notifier = { 995 .notifier_call = arch_timer_cpu_pm_notify, 996 }; 997 998 static int __init arch_timer_cpu_pm_init(void) 999 { 1000 return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier); 1001 } 1002 1003 static void __init arch_timer_cpu_pm_deinit(void) 1004 { 1005 WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier)); 1006 } 1007 1008 #else 1009 static int __init arch_timer_cpu_pm_init(void) 1010 { 1011 return 0; 1012 } 1013 1014 static void __init arch_timer_cpu_pm_deinit(void) 1015 { 1016 } 1017 #endif 1018 1019 static int __init arch_timer_register(void) 1020 { 1021 int err; 1022 int ppi; 1023 1024 arch_timer_evt = alloc_percpu(struct clock_event_device); 1025 if (!arch_timer_evt) { 1026 err = -ENOMEM; 1027 goto out; 1028 } 1029 1030 ppi = arch_timer_ppi[arch_timer_uses_ppi]; 1031 switch (arch_timer_uses_ppi) { 1032 case ARCH_TIMER_VIRT_PPI: 1033 err = request_percpu_irq(ppi, arch_timer_handler_virt, 1034 "arch_timer", arch_timer_evt); 1035 break; 1036 case ARCH_TIMER_PHYS_SECURE_PPI: 1037 case ARCH_TIMER_PHYS_NONSECURE_PPI: 1038 err = request_percpu_irq(ppi, arch_timer_handler_phys, 1039 "arch_timer", arch_timer_evt); 1040 if (!err && arch_timer_has_nonsecure_ppi()) { 1041 ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]; 1042 err = request_percpu_irq(ppi, arch_timer_handler_phys, 1043 "arch_timer", arch_timer_evt); 1044 if (err) 1045 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI], 1046 arch_timer_evt); 1047 } 1048 break; 1049 case ARCH_TIMER_HYP_PPI: 1050 err = request_percpu_irq(ppi, arch_timer_handler_phys, 1051 "arch_timer", arch_timer_evt); 1052 break; 1053 default: 1054 BUG(); 1055 } 1056 1057 if (err) { 1058 pr_err("can't register interrupt %d (%d)\n", ppi, err); 1059 goto out_free; 1060 } 1061 1062 err = arch_timer_cpu_pm_init(); 1063 if (err) 1064 goto out_unreg_notify; 1065 1066 /* Register and immediately configure the timer on the boot CPU */ 1067 err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING, 1068 "clockevents/arm/arch_timer:starting", 1069 arch_timer_starting_cpu, arch_timer_dying_cpu); 1070 if (err) 1071 goto out_unreg_cpupm; 1072 return 0; 1073 1074 out_unreg_cpupm: 1075 arch_timer_cpu_pm_deinit(); 1076 1077 out_unreg_notify: 1078 free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt); 1079 if (arch_timer_has_nonsecure_ppi()) 1080 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI], 1081 arch_timer_evt); 1082 1083 out_free: 1084 free_percpu(arch_timer_evt); 1085 out: 1086 return err; 1087 } 1088 1089 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq) 1090 { 1091 int ret; 1092 irq_handler_t func; 1093 struct arch_timer *t; 1094 1095 t = kzalloc(sizeof(*t), GFP_KERNEL); 1096 if (!t) 1097 return -ENOMEM; 1098 1099 t->base = base; 1100 t->evt.irq = irq; 1101 __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt); 1102 1103 if (arch_timer_mem_use_virtual) 1104 func = arch_timer_handler_virt_mem; 1105 else 1106 func = arch_timer_handler_phys_mem; 1107 1108 ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt); 1109 if (ret) { 1110 pr_err("Failed to request mem timer irq\n"); 1111 kfree(t); 1112 } 1113 1114 return ret; 1115 } 1116 1117 static const struct of_device_id arch_timer_of_match[] __initconst = { 1118 { .compatible = "arm,armv7-timer", }, 1119 { .compatible = "arm,armv8-timer", }, 1120 {}, 1121 }; 1122 1123 static const struct of_device_id arch_timer_mem_of_match[] __initconst = { 1124 { .compatible = "arm,armv7-timer-mem", }, 1125 {}, 1126 }; 1127 1128 static bool __init arch_timer_needs_of_probing(void) 1129 { 1130 struct device_node *dn; 1131 bool needs_probing = false; 1132 unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM; 1133 1134 /* We have two timers, and both device-tree nodes are probed. */ 1135 if ((arch_timers_present & mask) == mask) 1136 return false; 1137 1138 /* 1139 * Only one type of timer is probed, 1140 * check if we have another type of timer node in device-tree. 1141 */ 1142 if (arch_timers_present & ARCH_TIMER_TYPE_CP15) 1143 dn = of_find_matching_node(NULL, arch_timer_mem_of_match); 1144 else 1145 dn = of_find_matching_node(NULL, arch_timer_of_match); 1146 1147 if (dn && of_device_is_available(dn)) 1148 needs_probing = true; 1149 1150 of_node_put(dn); 1151 1152 return needs_probing; 1153 } 1154 1155 static int __init arch_timer_common_init(void) 1156 { 1157 arch_timer_banner(arch_timers_present); 1158 arch_counter_register(arch_timers_present); 1159 return arch_timer_arch_init(); 1160 } 1161 1162 /** 1163 * arch_timer_select_ppi() - Select suitable PPI for the current system. 1164 * 1165 * If HYP mode is available, we know that the physical timer 1166 * has been configured to be accessible from PL1. Use it, so 1167 * that a guest can use the virtual timer instead. 1168 * 1169 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE 1170 * accesses to CNTP_*_EL1 registers are silently redirected to 1171 * their CNTHP_*_EL2 counterparts, and use a different PPI 1172 * number. 1173 * 1174 * If no interrupt provided for virtual timer, we'll have to 1175 * stick to the physical timer. It'd better be accessible... 1176 * For arm64 we never use the secure interrupt. 1177 * 1178 * Return: a suitable PPI type for the current system. 1179 */ 1180 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void) 1181 { 1182 if (is_kernel_in_hyp_mode()) 1183 return ARCH_TIMER_HYP_PPI; 1184 1185 if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI]) 1186 return ARCH_TIMER_VIRT_PPI; 1187 1188 if (IS_ENABLED(CONFIG_ARM64)) 1189 return ARCH_TIMER_PHYS_NONSECURE_PPI; 1190 1191 return ARCH_TIMER_PHYS_SECURE_PPI; 1192 } 1193 1194 static int __init arch_timer_of_init(struct device_node *np) 1195 { 1196 int i, ret; 1197 u32 rate; 1198 1199 if (arch_timers_present & ARCH_TIMER_TYPE_CP15) { 1200 pr_warn("multiple nodes in dt, skipping\n"); 1201 return 0; 1202 } 1203 1204 arch_timers_present |= ARCH_TIMER_TYPE_CP15; 1205 for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) 1206 arch_timer_ppi[i] = irq_of_parse_and_map(np, i); 1207 1208 arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI]; 1209 1210 rate = arch_timer_get_cntfrq(); 1211 arch_timer_of_configure_rate(rate, np); 1212 1213 arch_timer_c3stop = !of_property_read_bool(np, "always-on"); 1214 1215 /* Check for globally applicable workarounds */ 1216 arch_timer_check_ool_workaround(ate_match_dt, np); 1217 1218 /* 1219 * If we cannot rely on firmware initializing the timer registers then 1220 * we should use the physical timers instead. 1221 */ 1222 if (IS_ENABLED(CONFIG_ARM) && 1223 of_property_read_bool(np, "arm,cpu-registers-not-fw-configured")) 1224 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI; 1225 else 1226 arch_timer_uses_ppi = arch_timer_select_ppi(); 1227 1228 if (!arch_timer_ppi[arch_timer_uses_ppi]) { 1229 pr_err("No interrupt available, giving up\n"); 1230 return -EINVAL; 1231 } 1232 1233 /* On some systems, the counter stops ticking when in suspend. */ 1234 arch_counter_suspend_stop = of_property_read_bool(np, 1235 "arm,no-tick-in-suspend"); 1236 1237 ret = arch_timer_register(); 1238 if (ret) 1239 return ret; 1240 1241 if (arch_timer_needs_of_probing()) 1242 return 0; 1243 1244 return arch_timer_common_init(); 1245 } 1246 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init); 1247 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init); 1248 1249 static u32 __init 1250 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame) 1251 { 1252 void __iomem *base; 1253 u32 rate; 1254 1255 base = ioremap(frame->cntbase, frame->size); 1256 if (!base) { 1257 pr_err("Unable to map frame @ %pa\n", &frame->cntbase); 1258 return 0; 1259 } 1260 1261 rate = readl_relaxed(base + CNTFRQ); 1262 1263 iounmap(base); 1264 1265 return rate; 1266 } 1267 1268 static struct arch_timer_mem_frame * __init 1269 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem) 1270 { 1271 struct arch_timer_mem_frame *frame, *best_frame = NULL; 1272 void __iomem *cntctlbase; 1273 u32 cnttidr; 1274 int i; 1275 1276 cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size); 1277 if (!cntctlbase) { 1278 pr_err("Can't map CNTCTLBase @ %pa\n", 1279 &timer_mem->cntctlbase); 1280 return NULL; 1281 } 1282 1283 cnttidr = readl_relaxed(cntctlbase + CNTTIDR); 1284 1285 /* 1286 * Try to find a virtual capable frame. Otherwise fall back to a 1287 * physical capable frame. 1288 */ 1289 for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) { 1290 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT | 1291 CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT; 1292 1293 frame = &timer_mem->frame[i]; 1294 if (!frame->valid) 1295 continue; 1296 1297 /* Try enabling everything, and see what sticks */ 1298 writel_relaxed(cntacr, cntctlbase + CNTACR(i)); 1299 cntacr = readl_relaxed(cntctlbase + CNTACR(i)); 1300 1301 if ((cnttidr & CNTTIDR_VIRT(i)) && 1302 !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) { 1303 best_frame = frame; 1304 arch_timer_mem_use_virtual = true; 1305 break; 1306 } 1307 1308 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT)) 1309 continue; 1310 1311 best_frame = frame; 1312 } 1313 1314 iounmap(cntctlbase); 1315 1316 return best_frame; 1317 } 1318 1319 static int __init 1320 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame) 1321 { 1322 void __iomem *base; 1323 int ret, irq = 0; 1324 1325 if (arch_timer_mem_use_virtual) 1326 irq = frame->virt_irq; 1327 else 1328 irq = frame->phys_irq; 1329 1330 if (!irq) { 1331 pr_err("Frame missing %s irq.\n", 1332 arch_timer_mem_use_virtual ? "virt" : "phys"); 1333 return -EINVAL; 1334 } 1335 1336 if (!request_mem_region(frame->cntbase, frame->size, 1337 "arch_mem_timer")) 1338 return -EBUSY; 1339 1340 base = ioremap(frame->cntbase, frame->size); 1341 if (!base) { 1342 pr_err("Can't map frame's registers\n"); 1343 return -ENXIO; 1344 } 1345 1346 ret = arch_timer_mem_register(base, irq); 1347 if (ret) { 1348 iounmap(base); 1349 return ret; 1350 } 1351 1352 arch_counter_base = base; 1353 arch_timers_present |= ARCH_TIMER_TYPE_MEM; 1354 1355 return 0; 1356 } 1357 1358 static int __init arch_timer_mem_of_init(struct device_node *np) 1359 { 1360 struct arch_timer_mem *timer_mem; 1361 struct arch_timer_mem_frame *frame; 1362 struct device_node *frame_node; 1363 struct resource res; 1364 int ret = -EINVAL; 1365 u32 rate; 1366 1367 timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL); 1368 if (!timer_mem) 1369 return -ENOMEM; 1370 1371 if (of_address_to_resource(np, 0, &res)) 1372 goto out; 1373 timer_mem->cntctlbase = res.start; 1374 timer_mem->size = resource_size(&res); 1375 1376 for_each_available_child_of_node(np, frame_node) { 1377 u32 n; 1378 struct arch_timer_mem_frame *frame; 1379 1380 if (of_property_read_u32(frame_node, "frame-number", &n)) { 1381 pr_err(FW_BUG "Missing frame-number.\n"); 1382 of_node_put(frame_node); 1383 goto out; 1384 } 1385 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) { 1386 pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n", 1387 ARCH_TIMER_MEM_MAX_FRAMES - 1); 1388 of_node_put(frame_node); 1389 goto out; 1390 } 1391 frame = &timer_mem->frame[n]; 1392 1393 if (frame->valid) { 1394 pr_err(FW_BUG "Duplicated frame-number.\n"); 1395 of_node_put(frame_node); 1396 goto out; 1397 } 1398 1399 if (of_address_to_resource(frame_node, 0, &res)) { 1400 of_node_put(frame_node); 1401 goto out; 1402 } 1403 frame->cntbase = res.start; 1404 frame->size = resource_size(&res); 1405 1406 frame->virt_irq = irq_of_parse_and_map(frame_node, 1407 ARCH_TIMER_VIRT_SPI); 1408 frame->phys_irq = irq_of_parse_and_map(frame_node, 1409 ARCH_TIMER_PHYS_SPI); 1410 1411 frame->valid = true; 1412 } 1413 1414 frame = arch_timer_mem_find_best_frame(timer_mem); 1415 if (!frame) { 1416 pr_err("Unable to find a suitable frame in timer @ %pa\n", 1417 &timer_mem->cntctlbase); 1418 ret = -EINVAL; 1419 goto out; 1420 } 1421 1422 rate = arch_timer_mem_frame_get_cntfrq(frame); 1423 arch_timer_of_configure_rate(rate, np); 1424 1425 ret = arch_timer_mem_frame_register(frame); 1426 if (!ret && !arch_timer_needs_of_probing()) 1427 ret = arch_timer_common_init(); 1428 out: 1429 kfree(timer_mem); 1430 return ret; 1431 } 1432 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem", 1433 arch_timer_mem_of_init); 1434 1435 #ifdef CONFIG_ACPI_GTDT 1436 static int __init 1437 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem) 1438 { 1439 struct arch_timer_mem_frame *frame; 1440 u32 rate; 1441 int i; 1442 1443 for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) { 1444 frame = &timer_mem->frame[i]; 1445 1446 if (!frame->valid) 1447 continue; 1448 1449 rate = arch_timer_mem_frame_get_cntfrq(frame); 1450 if (rate == arch_timer_rate) 1451 continue; 1452 1453 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n", 1454 &frame->cntbase, 1455 (unsigned long)rate, (unsigned long)arch_timer_rate); 1456 1457 return -EINVAL; 1458 } 1459 1460 return 0; 1461 } 1462 1463 static int __init arch_timer_mem_acpi_init(int platform_timer_count) 1464 { 1465 struct arch_timer_mem *timers, *timer; 1466 struct arch_timer_mem_frame *frame, *best_frame = NULL; 1467 int timer_count, i, ret = 0; 1468 1469 timers = kcalloc(platform_timer_count, sizeof(*timers), 1470 GFP_KERNEL); 1471 if (!timers) 1472 return -ENOMEM; 1473 1474 ret = acpi_arch_timer_mem_init(timers, &timer_count); 1475 if (ret || !timer_count) 1476 goto out; 1477 1478 /* 1479 * While unlikely, it's theoretically possible that none of the frames 1480 * in a timer expose the combination of feature we want. 1481 */ 1482 for (i = 0; i < timer_count; i++) { 1483 timer = &timers[i]; 1484 1485 frame = arch_timer_mem_find_best_frame(timer); 1486 if (!best_frame) 1487 best_frame = frame; 1488 1489 ret = arch_timer_mem_verify_cntfrq(timer); 1490 if (ret) { 1491 pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n"); 1492 goto out; 1493 } 1494 1495 if (!best_frame) /* implies !frame */ 1496 /* 1497 * Only complain about missing suitable frames if we 1498 * haven't already found one in a previous iteration. 1499 */ 1500 pr_err("Unable to find a suitable frame in timer @ %pa\n", 1501 &timer->cntctlbase); 1502 } 1503 1504 if (best_frame) 1505 ret = arch_timer_mem_frame_register(best_frame); 1506 out: 1507 kfree(timers); 1508 return ret; 1509 } 1510 1511 /* Initialize per-processor generic timer and memory-mapped timer(if present) */ 1512 static int __init arch_timer_acpi_init(struct acpi_table_header *table) 1513 { 1514 int ret, platform_timer_count; 1515 1516 if (arch_timers_present & ARCH_TIMER_TYPE_CP15) { 1517 pr_warn("already initialized, skipping\n"); 1518 return -EINVAL; 1519 } 1520 1521 arch_timers_present |= ARCH_TIMER_TYPE_CP15; 1522 1523 ret = acpi_gtdt_init(table, &platform_timer_count); 1524 if (ret) { 1525 pr_err("Failed to init GTDT table.\n"); 1526 return ret; 1527 } 1528 1529 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] = 1530 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI); 1531 1532 arch_timer_ppi[ARCH_TIMER_VIRT_PPI] = 1533 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI); 1534 1535 arch_timer_ppi[ARCH_TIMER_HYP_PPI] = 1536 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI); 1537 1538 arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI]; 1539 1540 /* 1541 * When probing via ACPI, we have no mechanism to override the sysreg 1542 * CNTFRQ value. This *must* be correct. 1543 */ 1544 arch_timer_rate = arch_timer_get_cntfrq(); 1545 if (!arch_timer_rate) { 1546 pr_err(FW_BUG "frequency not available.\n"); 1547 return -EINVAL; 1548 } 1549 1550 arch_timer_uses_ppi = arch_timer_select_ppi(); 1551 if (!arch_timer_ppi[arch_timer_uses_ppi]) { 1552 pr_err("No interrupt available, giving up\n"); 1553 return -EINVAL; 1554 } 1555 1556 /* Always-on capability */ 1557 arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi); 1558 1559 /* Check for globally applicable workarounds */ 1560 arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table); 1561 1562 ret = arch_timer_register(); 1563 if (ret) 1564 return ret; 1565 1566 if (platform_timer_count && 1567 arch_timer_mem_acpi_init(platform_timer_count)) 1568 pr_err("Failed to initialize memory-mapped timer.\n"); 1569 1570 return arch_timer_common_init(); 1571 } 1572 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init); 1573 #endif 1574