1 /* 2 * linux/drivers/clocksource/arm_arch_timer.c 3 * 4 * Copyright (C) 2011 ARM Ltd. 5 * All Rights Reserved 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #define pr_fmt(fmt) "arm_arch_timer: " fmt 13 14 #include <linux/init.h> 15 #include <linux/kernel.h> 16 #include <linux/device.h> 17 #include <linux/smp.h> 18 #include <linux/cpu.h> 19 #include <linux/cpu_pm.h> 20 #include <linux/clockchips.h> 21 #include <linux/clocksource.h> 22 #include <linux/interrupt.h> 23 #include <linux/of_irq.h> 24 #include <linux/of_address.h> 25 #include <linux/io.h> 26 #include <linux/slab.h> 27 #include <linux/sched/clock.h> 28 #include <linux/sched_clock.h> 29 #include <linux/acpi.h> 30 31 #include <asm/arch_timer.h> 32 #include <asm/virt.h> 33 34 #include <clocksource/arm_arch_timer.h> 35 36 #undef pr_fmt 37 #define pr_fmt(fmt) "arch_timer: " fmt 38 39 #define CNTTIDR 0x08 40 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4)) 41 42 #define CNTACR(n) (0x40 + ((n) * 4)) 43 #define CNTACR_RPCT BIT(0) 44 #define CNTACR_RVCT BIT(1) 45 #define CNTACR_RFRQ BIT(2) 46 #define CNTACR_RVOFF BIT(3) 47 #define CNTACR_RWVT BIT(4) 48 #define CNTACR_RWPT BIT(5) 49 50 #define CNTVCT_LO 0x08 51 #define CNTVCT_HI 0x0c 52 #define CNTFRQ 0x10 53 #define CNTP_TVAL 0x28 54 #define CNTP_CTL 0x2c 55 #define CNTV_TVAL 0x38 56 #define CNTV_CTL 0x3c 57 58 static unsigned arch_timers_present __initdata; 59 60 static void __iomem *arch_counter_base; 61 62 struct arch_timer { 63 void __iomem *base; 64 struct clock_event_device evt; 65 }; 66 67 #define to_arch_timer(e) container_of(e, struct arch_timer, evt) 68 69 static u32 arch_timer_rate; 70 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI]; 71 72 static struct clock_event_device __percpu *arch_timer_evt; 73 74 static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI; 75 static bool arch_timer_c3stop; 76 static bool arch_timer_mem_use_virtual; 77 static bool arch_counter_suspend_stop; 78 static bool vdso_default = true; 79 80 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); 81 82 static int __init early_evtstrm_cfg(char *buf) 83 { 84 return strtobool(buf, &evtstrm_enable); 85 } 86 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); 87 88 /* 89 * Architected system timer support. 90 */ 91 92 static __always_inline 93 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, 94 struct clock_event_device *clk) 95 { 96 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { 97 struct arch_timer *timer = to_arch_timer(clk); 98 switch (reg) { 99 case ARCH_TIMER_REG_CTRL: 100 writel_relaxed(val, timer->base + CNTP_CTL); 101 break; 102 case ARCH_TIMER_REG_TVAL: 103 writel_relaxed(val, timer->base + CNTP_TVAL); 104 break; 105 } 106 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { 107 struct arch_timer *timer = to_arch_timer(clk); 108 switch (reg) { 109 case ARCH_TIMER_REG_CTRL: 110 writel_relaxed(val, timer->base + CNTV_CTL); 111 break; 112 case ARCH_TIMER_REG_TVAL: 113 writel_relaxed(val, timer->base + CNTV_TVAL); 114 break; 115 } 116 } else { 117 arch_timer_reg_write_cp15(access, reg, val); 118 } 119 } 120 121 static __always_inline 122 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, 123 struct clock_event_device *clk) 124 { 125 u32 val; 126 127 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { 128 struct arch_timer *timer = to_arch_timer(clk); 129 switch (reg) { 130 case ARCH_TIMER_REG_CTRL: 131 val = readl_relaxed(timer->base + CNTP_CTL); 132 break; 133 case ARCH_TIMER_REG_TVAL: 134 val = readl_relaxed(timer->base + CNTP_TVAL); 135 break; 136 } 137 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { 138 struct arch_timer *timer = to_arch_timer(clk); 139 switch (reg) { 140 case ARCH_TIMER_REG_CTRL: 141 val = readl_relaxed(timer->base + CNTV_CTL); 142 break; 143 case ARCH_TIMER_REG_TVAL: 144 val = readl_relaxed(timer->base + CNTV_TVAL); 145 break; 146 } 147 } else { 148 val = arch_timer_reg_read_cp15(access, reg); 149 } 150 151 return val; 152 } 153 154 /* 155 * Default to cp15 based access because arm64 uses this function for 156 * sched_clock() before DT is probed and the cp15 method is guaranteed 157 * to exist on arm64. arm doesn't use this before DT is probed so even 158 * if we don't have the cp15 accessors we won't have a problem. 159 */ 160 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; 161 162 static u64 arch_counter_read(struct clocksource *cs) 163 { 164 return arch_timer_read_counter(); 165 } 166 167 static u64 arch_counter_read_cc(const struct cyclecounter *cc) 168 { 169 return arch_timer_read_counter(); 170 } 171 172 static struct clocksource clocksource_counter = { 173 .name = "arch_sys_counter", 174 .rating = 400, 175 .read = arch_counter_read, 176 .mask = CLOCKSOURCE_MASK(56), 177 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 178 }; 179 180 static struct cyclecounter cyclecounter __ro_after_init = { 181 .read = arch_counter_read_cc, 182 .mask = CLOCKSOURCE_MASK(56), 183 }; 184 185 struct ate_acpi_oem_info { 186 char oem_id[ACPI_OEM_ID_SIZE + 1]; 187 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; 188 u32 oem_revision; 189 }; 190 191 #ifdef CONFIG_FSL_ERRATUM_A008585 192 /* 193 * The number of retries is an arbitrary value well beyond the highest number 194 * of iterations the loop has been observed to take. 195 */ 196 #define __fsl_a008585_read_reg(reg) ({ \ 197 u64 _old, _new; \ 198 int _retries = 200; \ 199 \ 200 do { \ 201 _old = read_sysreg(reg); \ 202 _new = read_sysreg(reg); \ 203 _retries--; \ 204 } while (unlikely(_old != _new) && _retries); \ 205 \ 206 WARN_ON_ONCE(!_retries); \ 207 _new; \ 208 }) 209 210 static u32 notrace fsl_a008585_read_cntp_tval_el0(void) 211 { 212 return __fsl_a008585_read_reg(cntp_tval_el0); 213 } 214 215 static u32 notrace fsl_a008585_read_cntv_tval_el0(void) 216 { 217 return __fsl_a008585_read_reg(cntv_tval_el0); 218 } 219 220 static u64 notrace fsl_a008585_read_cntvct_el0(void) 221 { 222 return __fsl_a008585_read_reg(cntvct_el0); 223 } 224 #endif 225 226 #ifdef CONFIG_HISILICON_ERRATUM_161010101 227 /* 228 * Verify whether the value of the second read is larger than the first by 229 * less than 32 is the only way to confirm the value is correct, so clear the 230 * lower 5 bits to check whether the difference is greater than 32 or not. 231 * Theoretically the erratum should not occur more than twice in succession 232 * when reading the system counter, but it is possible that some interrupts 233 * may lead to more than twice read errors, triggering the warning, so setting 234 * the number of retries far beyond the number of iterations the loop has been 235 * observed to take. 236 */ 237 #define __hisi_161010101_read_reg(reg) ({ \ 238 u64 _old, _new; \ 239 int _retries = 50; \ 240 \ 241 do { \ 242 _old = read_sysreg(reg); \ 243 _new = read_sysreg(reg); \ 244 _retries--; \ 245 } while (unlikely((_new - _old) >> 5) && _retries); \ 246 \ 247 WARN_ON_ONCE(!_retries); \ 248 _new; \ 249 }) 250 251 static u32 notrace hisi_161010101_read_cntp_tval_el0(void) 252 { 253 return __hisi_161010101_read_reg(cntp_tval_el0); 254 } 255 256 static u32 notrace hisi_161010101_read_cntv_tval_el0(void) 257 { 258 return __hisi_161010101_read_reg(cntv_tval_el0); 259 } 260 261 static u64 notrace hisi_161010101_read_cntvct_el0(void) 262 { 263 return __hisi_161010101_read_reg(cntvct_el0); 264 } 265 266 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = { 267 /* 268 * Note that trailing spaces are required to properly match 269 * the OEM table information. 270 */ 271 { 272 .oem_id = "HISI ", 273 .oem_table_id = "HIP05 ", 274 .oem_revision = 0, 275 }, 276 { 277 .oem_id = "HISI ", 278 .oem_table_id = "HIP06 ", 279 .oem_revision = 0, 280 }, 281 { 282 .oem_id = "HISI ", 283 .oem_table_id = "HIP07 ", 284 .oem_revision = 0, 285 }, 286 { /* Sentinel indicating the end of the OEM array */ }, 287 }; 288 #endif 289 290 #ifdef CONFIG_ARM64_ERRATUM_858921 291 static u64 notrace arm64_858921_read_cntvct_el0(void) 292 { 293 u64 old, new; 294 295 old = read_sysreg(cntvct_el0); 296 new = read_sysreg(cntvct_el0); 297 return (((old ^ new) >> 32) & 1) ? old : new; 298 } 299 #endif 300 301 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND 302 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, 303 timer_unstable_counter_workaround); 304 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); 305 306 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); 307 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); 308 309 static void erratum_set_next_event_tval_generic(const int access, unsigned long evt, 310 struct clock_event_device *clk) 311 { 312 unsigned long ctrl; 313 u64 cval = evt + arch_counter_get_cntvct(); 314 315 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 316 ctrl |= ARCH_TIMER_CTRL_ENABLE; 317 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 318 319 if (access == ARCH_TIMER_PHYS_ACCESS) 320 write_sysreg(cval, cntp_cval_el0); 321 else 322 write_sysreg(cval, cntv_cval_el0); 323 324 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 325 } 326 327 static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt, 328 struct clock_event_device *clk) 329 { 330 erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk); 331 return 0; 332 } 333 334 static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt, 335 struct clock_event_device *clk) 336 { 337 erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk); 338 return 0; 339 } 340 341 static const struct arch_timer_erratum_workaround ool_workarounds[] = { 342 #ifdef CONFIG_FSL_ERRATUM_A008585 343 { 344 .match_type = ate_match_dt, 345 .id = "fsl,erratum-a008585", 346 .desc = "Freescale erratum a005858", 347 .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0, 348 .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0, 349 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0, 350 .set_next_event_phys = erratum_set_next_event_tval_phys, 351 .set_next_event_virt = erratum_set_next_event_tval_virt, 352 }, 353 #endif 354 #ifdef CONFIG_HISILICON_ERRATUM_161010101 355 { 356 .match_type = ate_match_dt, 357 .id = "hisilicon,erratum-161010101", 358 .desc = "HiSilicon erratum 161010101", 359 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0, 360 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0, 361 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0, 362 .set_next_event_phys = erratum_set_next_event_tval_phys, 363 .set_next_event_virt = erratum_set_next_event_tval_virt, 364 }, 365 { 366 .match_type = ate_match_acpi_oem_info, 367 .id = hisi_161010101_oem_info, 368 .desc = "HiSilicon erratum 161010101", 369 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0, 370 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0, 371 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0, 372 .set_next_event_phys = erratum_set_next_event_tval_phys, 373 .set_next_event_virt = erratum_set_next_event_tval_virt, 374 }, 375 #endif 376 #ifdef CONFIG_ARM64_ERRATUM_858921 377 { 378 .match_type = ate_match_local_cap_id, 379 .id = (void *)ARM64_WORKAROUND_858921, 380 .desc = "ARM erratum 858921", 381 .read_cntvct_el0 = arm64_858921_read_cntvct_el0, 382 }, 383 #endif 384 }; 385 386 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *, 387 const void *); 388 389 static 390 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa, 391 const void *arg) 392 { 393 const struct device_node *np = arg; 394 395 return of_property_read_bool(np, wa->id); 396 } 397 398 static 399 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa, 400 const void *arg) 401 { 402 return this_cpu_has_cap((uintptr_t)wa->id); 403 } 404 405 406 static 407 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa, 408 const void *arg) 409 { 410 static const struct ate_acpi_oem_info empty_oem_info = {}; 411 const struct ate_acpi_oem_info *info = wa->id; 412 const struct acpi_table_header *table = arg; 413 414 /* Iterate over the ACPI OEM info array, looking for a match */ 415 while (memcmp(info, &empty_oem_info, sizeof(*info))) { 416 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) && 417 !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) && 418 info->oem_revision == table->oem_revision) 419 return true; 420 421 info++; 422 } 423 424 return false; 425 } 426 427 static const struct arch_timer_erratum_workaround * 428 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, 429 ate_match_fn_t match_fn, 430 void *arg) 431 { 432 int i; 433 434 for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) { 435 if (ool_workarounds[i].match_type != type) 436 continue; 437 438 if (match_fn(&ool_workarounds[i], arg)) 439 return &ool_workarounds[i]; 440 } 441 442 return NULL; 443 } 444 445 static 446 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa, 447 bool local) 448 { 449 int i; 450 451 if (local) { 452 __this_cpu_write(timer_unstable_counter_workaround, wa); 453 } else { 454 for_each_possible_cpu(i) 455 per_cpu(timer_unstable_counter_workaround, i) = wa; 456 } 457 458 /* 459 * Use the locked version, as we're called from the CPU 460 * hotplug framework. Otherwise, we end-up in deadlock-land. 461 */ 462 static_branch_enable_cpuslocked(&arch_timer_read_ool_enabled); 463 464 /* 465 * Don't use the vdso fastpath if errata require using the 466 * out-of-line counter accessor. We may change our mind pretty 467 * late in the game (with a per-CPU erratum, for example), so 468 * change both the default value and the vdso itself. 469 */ 470 if (wa->read_cntvct_el0) { 471 clocksource_counter.archdata.vdso_direct = false; 472 vdso_default = false; 473 } 474 } 475 476 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type, 477 void *arg) 478 { 479 const struct arch_timer_erratum_workaround *wa; 480 ate_match_fn_t match_fn = NULL; 481 bool local = false; 482 483 switch (type) { 484 case ate_match_dt: 485 match_fn = arch_timer_check_dt_erratum; 486 break; 487 case ate_match_local_cap_id: 488 match_fn = arch_timer_check_local_cap_erratum; 489 local = true; 490 break; 491 case ate_match_acpi_oem_info: 492 match_fn = arch_timer_check_acpi_oem_erratum; 493 break; 494 default: 495 WARN_ON(1); 496 return; 497 } 498 499 wa = arch_timer_iterate_errata(type, match_fn, arg); 500 if (!wa) 501 return; 502 503 if (needs_unstable_timer_counter_workaround()) { 504 const struct arch_timer_erratum_workaround *__wa; 505 __wa = __this_cpu_read(timer_unstable_counter_workaround); 506 if (__wa && wa != __wa) 507 pr_warn("Can't enable workaround for %s (clashes with %s\n)", 508 wa->desc, __wa->desc); 509 510 if (__wa) 511 return; 512 } 513 514 arch_timer_enable_workaround(wa, local); 515 pr_info("Enabling %s workaround for %s\n", 516 local ? "local" : "global", wa->desc); 517 } 518 519 #define erratum_handler(fn, r, ...) \ 520 ({ \ 521 bool __val; \ 522 if (needs_unstable_timer_counter_workaround()) { \ 523 const struct arch_timer_erratum_workaround *__wa; \ 524 __wa = __this_cpu_read(timer_unstable_counter_workaround); \ 525 if (__wa && __wa->fn) { \ 526 r = __wa->fn(__VA_ARGS__); \ 527 __val = true; \ 528 } else { \ 529 __val = false; \ 530 } \ 531 } else { \ 532 __val = false; \ 533 } \ 534 __val; \ 535 }) 536 537 static bool arch_timer_this_cpu_has_cntvct_wa(void) 538 { 539 const struct arch_timer_erratum_workaround *wa; 540 541 wa = __this_cpu_read(timer_unstable_counter_workaround); 542 return wa && wa->read_cntvct_el0; 543 } 544 #else 545 #define arch_timer_check_ool_workaround(t,a) do { } while(0) 546 #define erratum_set_next_event_tval_virt(...) ({BUG(); 0;}) 547 #define erratum_set_next_event_tval_phys(...) ({BUG(); 0;}) 548 #define erratum_handler(fn, r, ...) ({false;}) 549 #define arch_timer_this_cpu_has_cntvct_wa() ({false;}) 550 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ 551 552 static __always_inline irqreturn_t timer_handler(const int access, 553 struct clock_event_device *evt) 554 { 555 unsigned long ctrl; 556 557 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt); 558 if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { 559 ctrl |= ARCH_TIMER_CTRL_IT_MASK; 560 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt); 561 evt->event_handler(evt); 562 return IRQ_HANDLED; 563 } 564 565 return IRQ_NONE; 566 } 567 568 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id) 569 { 570 struct clock_event_device *evt = dev_id; 571 572 return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt); 573 } 574 575 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id) 576 { 577 struct clock_event_device *evt = dev_id; 578 579 return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt); 580 } 581 582 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id) 583 { 584 struct clock_event_device *evt = dev_id; 585 586 return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt); 587 } 588 589 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id) 590 { 591 struct clock_event_device *evt = dev_id; 592 593 return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt); 594 } 595 596 static __always_inline int timer_shutdown(const int access, 597 struct clock_event_device *clk) 598 { 599 unsigned long ctrl; 600 601 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 602 ctrl &= ~ARCH_TIMER_CTRL_ENABLE; 603 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 604 605 return 0; 606 } 607 608 static int arch_timer_shutdown_virt(struct clock_event_device *clk) 609 { 610 return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk); 611 } 612 613 static int arch_timer_shutdown_phys(struct clock_event_device *clk) 614 { 615 return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk); 616 } 617 618 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk) 619 { 620 return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk); 621 } 622 623 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk) 624 { 625 return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk); 626 } 627 628 static __always_inline void set_next_event(const int access, unsigned long evt, 629 struct clock_event_device *clk) 630 { 631 unsigned long ctrl; 632 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); 633 ctrl |= ARCH_TIMER_CTRL_ENABLE; 634 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 635 arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk); 636 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 637 } 638 639 static int arch_timer_set_next_event_virt(unsigned long evt, 640 struct clock_event_device *clk) 641 { 642 int ret; 643 644 if (erratum_handler(set_next_event_virt, ret, evt, clk)) 645 return ret; 646 647 set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); 648 return 0; 649 } 650 651 static int arch_timer_set_next_event_phys(unsigned long evt, 652 struct clock_event_device *clk) 653 { 654 int ret; 655 656 if (erratum_handler(set_next_event_phys, ret, evt, clk)) 657 return ret; 658 659 set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); 660 return 0; 661 } 662 663 static int arch_timer_set_next_event_virt_mem(unsigned long evt, 664 struct clock_event_device *clk) 665 { 666 set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk); 667 return 0; 668 } 669 670 static int arch_timer_set_next_event_phys_mem(unsigned long evt, 671 struct clock_event_device *clk) 672 { 673 set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk); 674 return 0; 675 } 676 677 static void __arch_timer_setup(unsigned type, 678 struct clock_event_device *clk) 679 { 680 clk->features = CLOCK_EVT_FEAT_ONESHOT; 681 682 if (type == ARCH_TIMER_TYPE_CP15) { 683 if (arch_timer_c3stop) 684 clk->features |= CLOCK_EVT_FEAT_C3STOP; 685 clk->name = "arch_sys_timer"; 686 clk->rating = 450; 687 clk->cpumask = cpumask_of(smp_processor_id()); 688 clk->irq = arch_timer_ppi[arch_timer_uses_ppi]; 689 switch (arch_timer_uses_ppi) { 690 case ARCH_TIMER_VIRT_PPI: 691 clk->set_state_shutdown = arch_timer_shutdown_virt; 692 clk->set_state_oneshot_stopped = arch_timer_shutdown_virt; 693 clk->set_next_event = arch_timer_set_next_event_virt; 694 break; 695 case ARCH_TIMER_PHYS_SECURE_PPI: 696 case ARCH_TIMER_PHYS_NONSECURE_PPI: 697 case ARCH_TIMER_HYP_PPI: 698 clk->set_state_shutdown = arch_timer_shutdown_phys; 699 clk->set_state_oneshot_stopped = arch_timer_shutdown_phys; 700 clk->set_next_event = arch_timer_set_next_event_phys; 701 break; 702 default: 703 BUG(); 704 } 705 706 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL); 707 } else { 708 clk->features |= CLOCK_EVT_FEAT_DYNIRQ; 709 clk->name = "arch_mem_timer"; 710 clk->rating = 400; 711 clk->cpumask = cpu_all_mask; 712 if (arch_timer_mem_use_virtual) { 713 clk->set_state_shutdown = arch_timer_shutdown_virt_mem; 714 clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem; 715 clk->set_next_event = 716 arch_timer_set_next_event_virt_mem; 717 } else { 718 clk->set_state_shutdown = arch_timer_shutdown_phys_mem; 719 clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem; 720 clk->set_next_event = 721 arch_timer_set_next_event_phys_mem; 722 } 723 } 724 725 clk->set_state_shutdown(clk); 726 727 clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); 728 } 729 730 static void arch_timer_evtstrm_enable(int divider) 731 { 732 u32 cntkctl = arch_timer_get_cntkctl(); 733 734 cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK; 735 /* Set the divider and enable virtual event stream */ 736 cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT) 737 | ARCH_TIMER_VIRT_EVT_EN; 738 arch_timer_set_cntkctl(cntkctl); 739 elf_hwcap |= HWCAP_EVTSTRM; 740 #ifdef CONFIG_COMPAT 741 compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM; 742 #endif 743 } 744 745 static void arch_timer_configure_evtstream(void) 746 { 747 int evt_stream_div, pos; 748 749 /* Find the closest power of two to the divisor */ 750 evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; 751 pos = fls(evt_stream_div); 752 if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) 753 pos--; 754 /* enable event stream */ 755 arch_timer_evtstrm_enable(min(pos, 15)); 756 } 757 758 static void arch_counter_set_user_access(void) 759 { 760 u32 cntkctl = arch_timer_get_cntkctl(); 761 762 /* Disable user access to the timers and both counters */ 763 /* Also disable virtual event stream */ 764 cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN 765 | ARCH_TIMER_USR_VT_ACCESS_EN 766 | ARCH_TIMER_USR_VCT_ACCESS_EN 767 | ARCH_TIMER_VIRT_EVT_EN 768 | ARCH_TIMER_USR_PCT_ACCESS_EN); 769 770 /* 771 * Enable user access to the virtual counter if it doesn't 772 * need to be workaround. The vdso may have been already 773 * disabled though. 774 */ 775 if (arch_timer_this_cpu_has_cntvct_wa()) 776 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id()); 777 else 778 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; 779 780 arch_timer_set_cntkctl(cntkctl); 781 } 782 783 static bool arch_timer_has_nonsecure_ppi(void) 784 { 785 return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI && 786 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]); 787 } 788 789 static u32 check_ppi_trigger(int irq) 790 { 791 u32 flags = irq_get_trigger_type(irq); 792 793 if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) { 794 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq); 795 pr_warn("WARNING: Please fix your firmware\n"); 796 flags = IRQF_TRIGGER_LOW; 797 } 798 799 return flags; 800 } 801 802 static int arch_timer_starting_cpu(unsigned int cpu) 803 { 804 struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); 805 u32 flags; 806 807 __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk); 808 809 flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]); 810 enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags); 811 812 if (arch_timer_has_nonsecure_ppi()) { 813 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]); 814 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI], 815 flags); 816 } 817 818 arch_counter_set_user_access(); 819 if (evtstrm_enable) 820 arch_timer_configure_evtstream(); 821 822 return 0; 823 } 824 825 /* 826 * For historical reasons, when probing with DT we use whichever (non-zero) 827 * rate was probed first, and don't verify that others match. If the first node 828 * probed has a clock-frequency property, this overrides the HW register. 829 */ 830 static void arch_timer_of_configure_rate(u32 rate, struct device_node *np) 831 { 832 /* Who has more than one independent system counter? */ 833 if (arch_timer_rate) 834 return; 835 836 if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) 837 arch_timer_rate = rate; 838 839 /* Check the timer frequency. */ 840 if (arch_timer_rate == 0) 841 pr_warn("frequency not available\n"); 842 } 843 844 static void arch_timer_banner(unsigned type) 845 { 846 pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n", 847 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "", 848 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? 849 " and " : "", 850 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "", 851 (unsigned long)arch_timer_rate / 1000000, 852 (unsigned long)(arch_timer_rate / 10000) % 100, 853 type & ARCH_TIMER_TYPE_CP15 ? 854 (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" : 855 "", 856 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "", 857 type & ARCH_TIMER_TYPE_MEM ? 858 arch_timer_mem_use_virtual ? "virt" : "phys" : 859 ""); 860 } 861 862 u32 arch_timer_get_rate(void) 863 { 864 return arch_timer_rate; 865 } 866 867 static u64 arch_counter_get_cntvct_mem(void) 868 { 869 u32 vct_lo, vct_hi, tmp_hi; 870 871 do { 872 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); 873 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO); 874 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); 875 } while (vct_hi != tmp_hi); 876 877 return ((u64) vct_hi << 32) | vct_lo; 878 } 879 880 static struct arch_timer_kvm_info arch_timer_kvm_info; 881 882 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void) 883 { 884 return &arch_timer_kvm_info; 885 } 886 887 static void __init arch_counter_register(unsigned type) 888 { 889 u64 start_count; 890 891 /* Register the CP15 based counter if we have one */ 892 if (type & ARCH_TIMER_TYPE_CP15) { 893 if (IS_ENABLED(CONFIG_ARM64) || 894 arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) 895 arch_timer_read_counter = arch_counter_get_cntvct; 896 else 897 arch_timer_read_counter = arch_counter_get_cntpct; 898 899 clocksource_counter.archdata.vdso_direct = vdso_default; 900 } else { 901 arch_timer_read_counter = arch_counter_get_cntvct_mem; 902 } 903 904 if (!arch_counter_suspend_stop) 905 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP; 906 start_count = arch_timer_read_counter(); 907 clocksource_register_hz(&clocksource_counter, arch_timer_rate); 908 cyclecounter.mult = clocksource_counter.mult; 909 cyclecounter.shift = clocksource_counter.shift; 910 timecounter_init(&arch_timer_kvm_info.timecounter, 911 &cyclecounter, start_count); 912 913 /* 56 bits minimum, so we assume worst case rollover */ 914 sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate); 915 } 916 917 static void arch_timer_stop(struct clock_event_device *clk) 918 { 919 pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id()); 920 921 disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]); 922 if (arch_timer_has_nonsecure_ppi()) 923 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]); 924 925 clk->set_state_shutdown(clk); 926 } 927 928 static int arch_timer_dying_cpu(unsigned int cpu) 929 { 930 struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); 931 932 arch_timer_stop(clk); 933 return 0; 934 } 935 936 #ifdef CONFIG_CPU_PM 937 static DEFINE_PER_CPU(unsigned long, saved_cntkctl); 938 static int arch_timer_cpu_pm_notify(struct notifier_block *self, 939 unsigned long action, void *hcpu) 940 { 941 if (action == CPU_PM_ENTER) 942 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl()); 943 else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) 944 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl)); 945 return NOTIFY_OK; 946 } 947 948 static struct notifier_block arch_timer_cpu_pm_notifier = { 949 .notifier_call = arch_timer_cpu_pm_notify, 950 }; 951 952 static int __init arch_timer_cpu_pm_init(void) 953 { 954 return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier); 955 } 956 957 static void __init arch_timer_cpu_pm_deinit(void) 958 { 959 WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier)); 960 } 961 962 #else 963 static int __init arch_timer_cpu_pm_init(void) 964 { 965 return 0; 966 } 967 968 static void __init arch_timer_cpu_pm_deinit(void) 969 { 970 } 971 #endif 972 973 static int __init arch_timer_register(void) 974 { 975 int err; 976 int ppi; 977 978 arch_timer_evt = alloc_percpu(struct clock_event_device); 979 if (!arch_timer_evt) { 980 err = -ENOMEM; 981 goto out; 982 } 983 984 ppi = arch_timer_ppi[arch_timer_uses_ppi]; 985 switch (arch_timer_uses_ppi) { 986 case ARCH_TIMER_VIRT_PPI: 987 err = request_percpu_irq(ppi, arch_timer_handler_virt, 988 "arch_timer", arch_timer_evt); 989 break; 990 case ARCH_TIMER_PHYS_SECURE_PPI: 991 case ARCH_TIMER_PHYS_NONSECURE_PPI: 992 err = request_percpu_irq(ppi, arch_timer_handler_phys, 993 "arch_timer", arch_timer_evt); 994 if (!err && arch_timer_has_nonsecure_ppi()) { 995 ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]; 996 err = request_percpu_irq(ppi, arch_timer_handler_phys, 997 "arch_timer", arch_timer_evt); 998 if (err) 999 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI], 1000 arch_timer_evt); 1001 } 1002 break; 1003 case ARCH_TIMER_HYP_PPI: 1004 err = request_percpu_irq(ppi, arch_timer_handler_phys, 1005 "arch_timer", arch_timer_evt); 1006 break; 1007 default: 1008 BUG(); 1009 } 1010 1011 if (err) { 1012 pr_err("can't register interrupt %d (%d)\n", ppi, err); 1013 goto out_free; 1014 } 1015 1016 err = arch_timer_cpu_pm_init(); 1017 if (err) 1018 goto out_unreg_notify; 1019 1020 1021 /* Register and immediately configure the timer on the boot CPU */ 1022 err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING, 1023 "clockevents/arm/arch_timer:starting", 1024 arch_timer_starting_cpu, arch_timer_dying_cpu); 1025 if (err) 1026 goto out_unreg_cpupm; 1027 return 0; 1028 1029 out_unreg_cpupm: 1030 arch_timer_cpu_pm_deinit(); 1031 1032 out_unreg_notify: 1033 free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt); 1034 if (arch_timer_has_nonsecure_ppi()) 1035 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI], 1036 arch_timer_evt); 1037 1038 out_free: 1039 free_percpu(arch_timer_evt); 1040 out: 1041 return err; 1042 } 1043 1044 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq) 1045 { 1046 int ret; 1047 irq_handler_t func; 1048 struct arch_timer *t; 1049 1050 t = kzalloc(sizeof(*t), GFP_KERNEL); 1051 if (!t) 1052 return -ENOMEM; 1053 1054 t->base = base; 1055 t->evt.irq = irq; 1056 __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt); 1057 1058 if (arch_timer_mem_use_virtual) 1059 func = arch_timer_handler_virt_mem; 1060 else 1061 func = arch_timer_handler_phys_mem; 1062 1063 ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt); 1064 if (ret) { 1065 pr_err("Failed to request mem timer irq\n"); 1066 kfree(t); 1067 } 1068 1069 return ret; 1070 } 1071 1072 static const struct of_device_id arch_timer_of_match[] __initconst = { 1073 { .compatible = "arm,armv7-timer", }, 1074 { .compatible = "arm,armv8-timer", }, 1075 {}, 1076 }; 1077 1078 static const struct of_device_id arch_timer_mem_of_match[] __initconst = { 1079 { .compatible = "arm,armv7-timer-mem", }, 1080 {}, 1081 }; 1082 1083 static bool __init arch_timer_needs_of_probing(void) 1084 { 1085 struct device_node *dn; 1086 bool needs_probing = false; 1087 unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM; 1088 1089 /* We have two timers, and both device-tree nodes are probed. */ 1090 if ((arch_timers_present & mask) == mask) 1091 return false; 1092 1093 /* 1094 * Only one type of timer is probed, 1095 * check if we have another type of timer node in device-tree. 1096 */ 1097 if (arch_timers_present & ARCH_TIMER_TYPE_CP15) 1098 dn = of_find_matching_node(NULL, arch_timer_mem_of_match); 1099 else 1100 dn = of_find_matching_node(NULL, arch_timer_of_match); 1101 1102 if (dn && of_device_is_available(dn)) 1103 needs_probing = true; 1104 1105 of_node_put(dn); 1106 1107 return needs_probing; 1108 } 1109 1110 static int __init arch_timer_common_init(void) 1111 { 1112 arch_timer_banner(arch_timers_present); 1113 arch_counter_register(arch_timers_present); 1114 return arch_timer_arch_init(); 1115 } 1116 1117 /** 1118 * arch_timer_select_ppi() - Select suitable PPI for the current system. 1119 * 1120 * If HYP mode is available, we know that the physical timer 1121 * has been configured to be accessible from PL1. Use it, so 1122 * that a guest can use the virtual timer instead. 1123 * 1124 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE 1125 * accesses to CNTP_*_EL1 registers are silently redirected to 1126 * their CNTHP_*_EL2 counterparts, and use a different PPI 1127 * number. 1128 * 1129 * If no interrupt provided for virtual timer, we'll have to 1130 * stick to the physical timer. It'd better be accessible... 1131 * For arm64 we never use the secure interrupt. 1132 * 1133 * Return: a suitable PPI type for the current system. 1134 */ 1135 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void) 1136 { 1137 if (is_kernel_in_hyp_mode()) 1138 return ARCH_TIMER_HYP_PPI; 1139 1140 if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI]) 1141 return ARCH_TIMER_VIRT_PPI; 1142 1143 if (IS_ENABLED(CONFIG_ARM64)) 1144 return ARCH_TIMER_PHYS_NONSECURE_PPI; 1145 1146 return ARCH_TIMER_PHYS_SECURE_PPI; 1147 } 1148 1149 static int __init arch_timer_of_init(struct device_node *np) 1150 { 1151 int i, ret; 1152 u32 rate; 1153 1154 if (arch_timers_present & ARCH_TIMER_TYPE_CP15) { 1155 pr_warn("multiple nodes in dt, skipping\n"); 1156 return 0; 1157 } 1158 1159 arch_timers_present |= ARCH_TIMER_TYPE_CP15; 1160 for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) 1161 arch_timer_ppi[i] = irq_of_parse_and_map(np, i); 1162 1163 arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI]; 1164 1165 rate = arch_timer_get_cntfrq(); 1166 arch_timer_of_configure_rate(rate, np); 1167 1168 arch_timer_c3stop = !of_property_read_bool(np, "always-on"); 1169 1170 /* Check for globally applicable workarounds */ 1171 arch_timer_check_ool_workaround(ate_match_dt, np); 1172 1173 /* 1174 * If we cannot rely on firmware initializing the timer registers then 1175 * we should use the physical timers instead. 1176 */ 1177 if (IS_ENABLED(CONFIG_ARM) && 1178 of_property_read_bool(np, "arm,cpu-registers-not-fw-configured")) 1179 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI; 1180 else 1181 arch_timer_uses_ppi = arch_timer_select_ppi(); 1182 1183 if (!arch_timer_ppi[arch_timer_uses_ppi]) { 1184 pr_err("No interrupt available, giving up\n"); 1185 return -EINVAL; 1186 } 1187 1188 /* On some systems, the counter stops ticking when in suspend. */ 1189 arch_counter_suspend_stop = of_property_read_bool(np, 1190 "arm,no-tick-in-suspend"); 1191 1192 ret = arch_timer_register(); 1193 if (ret) 1194 return ret; 1195 1196 if (arch_timer_needs_of_probing()) 1197 return 0; 1198 1199 return arch_timer_common_init(); 1200 } 1201 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init); 1202 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init); 1203 1204 static u32 __init 1205 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame) 1206 { 1207 void __iomem *base; 1208 u32 rate; 1209 1210 base = ioremap(frame->cntbase, frame->size); 1211 if (!base) { 1212 pr_err("Unable to map frame @ %pa\n", &frame->cntbase); 1213 return 0; 1214 } 1215 1216 rate = readl_relaxed(base + CNTFRQ); 1217 1218 iounmap(base); 1219 1220 return rate; 1221 } 1222 1223 static struct arch_timer_mem_frame * __init 1224 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem) 1225 { 1226 struct arch_timer_mem_frame *frame, *best_frame = NULL; 1227 void __iomem *cntctlbase; 1228 u32 cnttidr; 1229 int i; 1230 1231 cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size); 1232 if (!cntctlbase) { 1233 pr_err("Can't map CNTCTLBase @ %pa\n", 1234 &timer_mem->cntctlbase); 1235 return NULL; 1236 } 1237 1238 cnttidr = readl_relaxed(cntctlbase + CNTTIDR); 1239 1240 /* 1241 * Try to find a virtual capable frame. Otherwise fall back to a 1242 * physical capable frame. 1243 */ 1244 for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) { 1245 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT | 1246 CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT; 1247 1248 frame = &timer_mem->frame[i]; 1249 if (!frame->valid) 1250 continue; 1251 1252 /* Try enabling everything, and see what sticks */ 1253 writel_relaxed(cntacr, cntctlbase + CNTACR(i)); 1254 cntacr = readl_relaxed(cntctlbase + CNTACR(i)); 1255 1256 if ((cnttidr & CNTTIDR_VIRT(i)) && 1257 !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) { 1258 best_frame = frame; 1259 arch_timer_mem_use_virtual = true; 1260 break; 1261 } 1262 1263 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT)) 1264 continue; 1265 1266 best_frame = frame; 1267 } 1268 1269 iounmap(cntctlbase); 1270 1271 if (!best_frame) 1272 pr_err("Unable to find a suitable frame in timer @ %pa\n", 1273 &timer_mem->cntctlbase); 1274 1275 return best_frame; 1276 } 1277 1278 static int __init 1279 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame) 1280 { 1281 void __iomem *base; 1282 int ret, irq = 0; 1283 1284 if (arch_timer_mem_use_virtual) 1285 irq = frame->virt_irq; 1286 else 1287 irq = frame->phys_irq; 1288 1289 if (!irq) { 1290 pr_err("Frame missing %s irq.\n", 1291 arch_timer_mem_use_virtual ? "virt" : "phys"); 1292 return -EINVAL; 1293 } 1294 1295 if (!request_mem_region(frame->cntbase, frame->size, 1296 "arch_mem_timer")) 1297 return -EBUSY; 1298 1299 base = ioremap(frame->cntbase, frame->size); 1300 if (!base) { 1301 pr_err("Can't map frame's registers\n"); 1302 return -ENXIO; 1303 } 1304 1305 ret = arch_timer_mem_register(base, irq); 1306 if (ret) { 1307 iounmap(base); 1308 return ret; 1309 } 1310 1311 arch_counter_base = base; 1312 arch_timers_present |= ARCH_TIMER_TYPE_MEM; 1313 1314 return 0; 1315 } 1316 1317 static int __init arch_timer_mem_of_init(struct device_node *np) 1318 { 1319 struct arch_timer_mem *timer_mem; 1320 struct arch_timer_mem_frame *frame; 1321 struct device_node *frame_node; 1322 struct resource res; 1323 int ret = -EINVAL; 1324 u32 rate; 1325 1326 timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL); 1327 if (!timer_mem) 1328 return -ENOMEM; 1329 1330 if (of_address_to_resource(np, 0, &res)) 1331 goto out; 1332 timer_mem->cntctlbase = res.start; 1333 timer_mem->size = resource_size(&res); 1334 1335 for_each_available_child_of_node(np, frame_node) { 1336 u32 n; 1337 struct arch_timer_mem_frame *frame; 1338 1339 if (of_property_read_u32(frame_node, "frame-number", &n)) { 1340 pr_err(FW_BUG "Missing frame-number.\n"); 1341 of_node_put(frame_node); 1342 goto out; 1343 } 1344 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) { 1345 pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n", 1346 ARCH_TIMER_MEM_MAX_FRAMES - 1); 1347 of_node_put(frame_node); 1348 goto out; 1349 } 1350 frame = &timer_mem->frame[n]; 1351 1352 if (frame->valid) { 1353 pr_err(FW_BUG "Duplicated frame-number.\n"); 1354 of_node_put(frame_node); 1355 goto out; 1356 } 1357 1358 if (of_address_to_resource(frame_node, 0, &res)) { 1359 of_node_put(frame_node); 1360 goto out; 1361 } 1362 frame->cntbase = res.start; 1363 frame->size = resource_size(&res); 1364 1365 frame->virt_irq = irq_of_parse_and_map(frame_node, 1366 ARCH_TIMER_VIRT_SPI); 1367 frame->phys_irq = irq_of_parse_and_map(frame_node, 1368 ARCH_TIMER_PHYS_SPI); 1369 1370 frame->valid = true; 1371 } 1372 1373 frame = arch_timer_mem_find_best_frame(timer_mem); 1374 if (!frame) { 1375 ret = -EINVAL; 1376 goto out; 1377 } 1378 1379 rate = arch_timer_mem_frame_get_cntfrq(frame); 1380 arch_timer_of_configure_rate(rate, np); 1381 1382 ret = arch_timer_mem_frame_register(frame); 1383 if (!ret && !arch_timer_needs_of_probing()) 1384 ret = arch_timer_common_init(); 1385 out: 1386 kfree(timer_mem); 1387 return ret; 1388 } 1389 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem", 1390 arch_timer_mem_of_init); 1391 1392 #ifdef CONFIG_ACPI_GTDT 1393 static int __init 1394 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem) 1395 { 1396 struct arch_timer_mem_frame *frame; 1397 u32 rate; 1398 int i; 1399 1400 for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) { 1401 frame = &timer_mem->frame[i]; 1402 1403 if (!frame->valid) 1404 continue; 1405 1406 rate = arch_timer_mem_frame_get_cntfrq(frame); 1407 if (rate == arch_timer_rate) 1408 continue; 1409 1410 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n", 1411 &frame->cntbase, 1412 (unsigned long)rate, (unsigned long)arch_timer_rate); 1413 1414 return -EINVAL; 1415 } 1416 1417 return 0; 1418 } 1419 1420 static int __init arch_timer_mem_acpi_init(int platform_timer_count) 1421 { 1422 struct arch_timer_mem *timers, *timer; 1423 struct arch_timer_mem_frame *frame; 1424 int timer_count, i, ret = 0; 1425 1426 timers = kcalloc(platform_timer_count, sizeof(*timers), 1427 GFP_KERNEL); 1428 if (!timers) 1429 return -ENOMEM; 1430 1431 ret = acpi_arch_timer_mem_init(timers, &timer_count); 1432 if (ret || !timer_count) 1433 goto out; 1434 1435 for (i = 0; i < timer_count; i++) { 1436 ret = arch_timer_mem_verify_cntfrq(&timers[i]); 1437 if (ret) { 1438 pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n"); 1439 goto out; 1440 } 1441 } 1442 1443 /* 1444 * While unlikely, it's theoretically possible that none of the frames 1445 * in a timer expose the combination of feature we want. 1446 */ 1447 for (i = 0; i < timer_count; i++) { 1448 timer = &timers[i]; 1449 1450 frame = arch_timer_mem_find_best_frame(timer); 1451 if (frame) 1452 break; 1453 } 1454 1455 if (frame) 1456 ret = arch_timer_mem_frame_register(frame); 1457 out: 1458 kfree(timers); 1459 return ret; 1460 } 1461 1462 /* Initialize per-processor generic timer and memory-mapped timer(if present) */ 1463 static int __init arch_timer_acpi_init(struct acpi_table_header *table) 1464 { 1465 int ret, platform_timer_count; 1466 1467 if (arch_timers_present & ARCH_TIMER_TYPE_CP15) { 1468 pr_warn("already initialized, skipping\n"); 1469 return -EINVAL; 1470 } 1471 1472 arch_timers_present |= ARCH_TIMER_TYPE_CP15; 1473 1474 ret = acpi_gtdt_init(table, &platform_timer_count); 1475 if (ret) { 1476 pr_err("Failed to init GTDT table.\n"); 1477 return ret; 1478 } 1479 1480 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] = 1481 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI); 1482 1483 arch_timer_ppi[ARCH_TIMER_VIRT_PPI] = 1484 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI); 1485 1486 arch_timer_ppi[ARCH_TIMER_HYP_PPI] = 1487 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI); 1488 1489 arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI]; 1490 1491 /* 1492 * When probing via ACPI, we have no mechanism to override the sysreg 1493 * CNTFRQ value. This *must* be correct. 1494 */ 1495 arch_timer_rate = arch_timer_get_cntfrq(); 1496 if (!arch_timer_rate) { 1497 pr_err(FW_BUG "frequency not available.\n"); 1498 return -EINVAL; 1499 } 1500 1501 arch_timer_uses_ppi = arch_timer_select_ppi(); 1502 if (!arch_timer_ppi[arch_timer_uses_ppi]) { 1503 pr_err("No interrupt available, giving up\n"); 1504 return -EINVAL; 1505 } 1506 1507 /* Always-on capability */ 1508 arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi); 1509 1510 /* Check for globally applicable workarounds */ 1511 arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table); 1512 1513 ret = arch_timer_register(); 1514 if (ret) 1515 return ret; 1516 1517 if (platform_timer_count && 1518 arch_timer_mem_acpi_init(platform_timer_count)) 1519 pr_err("Failed to initialize memory-mapped timer.\n"); 1520 1521 return arch_timer_common_init(); 1522 } 1523 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init); 1524 #endif 1525