1 #include <linux/clocksource.h> 2 #include <linux/clockchips.h> 3 #include <linux/interrupt.h> 4 #include <linux/irq.h> 5 #include <linux/export.h> 6 #include <linux/delay.h> 7 #include <linux/errno.h> 8 #include <linux/i8253.h> 9 #include <linux/slab.h> 10 #include <linux/hpet.h> 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <linux/pm.h> 14 #include <linux/io.h> 15 16 #include <asm/cpufeature.h> 17 #include <asm/irqdomain.h> 18 #include <asm/fixmap.h> 19 #include <asm/hpet.h> 20 #include <asm/time.h> 21 22 #define HPET_MASK CLOCKSOURCE_MASK(32) 23 24 #define HPET_DEV_USED_BIT 2 25 #define HPET_DEV_USED (1 << HPET_DEV_USED_BIT) 26 #define HPET_DEV_VALID 0x8 27 #define HPET_DEV_FSB_CAP 0x1000 28 #define HPET_DEV_PERI_CAP 0x2000 29 30 #define HPET_MIN_CYCLES 128 31 #define HPET_MIN_PROG_DELTA (HPET_MIN_CYCLES + (HPET_MIN_CYCLES >> 1)) 32 33 /* 34 * HPET address is set in acpi/boot.c, when an ACPI entry exists 35 */ 36 unsigned long hpet_address; 37 u8 hpet_blockid; /* OS timer block num */ 38 bool hpet_msi_disable; 39 40 #ifdef CONFIG_PCI_MSI 41 static unsigned int hpet_num_timers; 42 #endif 43 static void __iomem *hpet_virt_address; 44 45 struct hpet_dev { 46 struct clock_event_device evt; 47 unsigned int num; 48 int cpu; 49 unsigned int irq; 50 unsigned int flags; 51 char name[10]; 52 }; 53 54 static inline struct hpet_dev *EVT_TO_HPET_DEV(struct clock_event_device *evtdev) 55 { 56 return container_of(evtdev, struct hpet_dev, evt); 57 } 58 59 inline unsigned int hpet_readl(unsigned int a) 60 { 61 return readl(hpet_virt_address + a); 62 } 63 64 static inline void hpet_writel(unsigned int d, unsigned int a) 65 { 66 writel(d, hpet_virt_address + a); 67 } 68 69 #ifdef CONFIG_X86_64 70 #include <asm/pgtable.h> 71 #endif 72 73 static inline void hpet_set_mapping(void) 74 { 75 hpet_virt_address = ioremap_nocache(hpet_address, HPET_MMAP_SIZE); 76 } 77 78 static inline void hpet_clear_mapping(void) 79 { 80 iounmap(hpet_virt_address); 81 hpet_virt_address = NULL; 82 } 83 84 /* 85 * HPET command line enable / disable 86 */ 87 bool boot_hpet_disable; 88 bool hpet_force_user; 89 static bool hpet_verbose; 90 91 static int __init hpet_setup(char *str) 92 { 93 while (str) { 94 char *next = strchr(str, ','); 95 96 if (next) 97 *next++ = 0; 98 if (!strncmp("disable", str, 7)) 99 boot_hpet_disable = true; 100 if (!strncmp("force", str, 5)) 101 hpet_force_user = true; 102 if (!strncmp("verbose", str, 7)) 103 hpet_verbose = true; 104 str = next; 105 } 106 return 1; 107 } 108 __setup("hpet=", hpet_setup); 109 110 static int __init disable_hpet(char *str) 111 { 112 boot_hpet_disable = true; 113 return 1; 114 } 115 __setup("nohpet", disable_hpet); 116 117 static inline int is_hpet_capable(void) 118 { 119 return !boot_hpet_disable && hpet_address; 120 } 121 122 /* 123 * HPET timer interrupt enable / disable 124 */ 125 static bool hpet_legacy_int_enabled; 126 127 /** 128 * is_hpet_enabled - check whether the hpet timer interrupt is enabled 129 */ 130 int is_hpet_enabled(void) 131 { 132 return is_hpet_capable() && hpet_legacy_int_enabled; 133 } 134 EXPORT_SYMBOL_GPL(is_hpet_enabled); 135 136 static void _hpet_print_config(const char *function, int line) 137 { 138 u32 i, timers, l, h; 139 printk(KERN_INFO "hpet: %s(%d):\n", function, line); 140 l = hpet_readl(HPET_ID); 141 h = hpet_readl(HPET_PERIOD); 142 timers = ((l & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT) + 1; 143 printk(KERN_INFO "hpet: ID: 0x%x, PERIOD: 0x%x\n", l, h); 144 l = hpet_readl(HPET_CFG); 145 h = hpet_readl(HPET_STATUS); 146 printk(KERN_INFO "hpet: CFG: 0x%x, STATUS: 0x%x\n", l, h); 147 l = hpet_readl(HPET_COUNTER); 148 h = hpet_readl(HPET_COUNTER+4); 149 printk(KERN_INFO "hpet: COUNTER_l: 0x%x, COUNTER_h: 0x%x\n", l, h); 150 151 for (i = 0; i < timers; i++) { 152 l = hpet_readl(HPET_Tn_CFG(i)); 153 h = hpet_readl(HPET_Tn_CFG(i)+4); 154 printk(KERN_INFO "hpet: T%d: CFG_l: 0x%x, CFG_h: 0x%x\n", 155 i, l, h); 156 l = hpet_readl(HPET_Tn_CMP(i)); 157 h = hpet_readl(HPET_Tn_CMP(i)+4); 158 printk(KERN_INFO "hpet: T%d: CMP_l: 0x%x, CMP_h: 0x%x\n", 159 i, l, h); 160 l = hpet_readl(HPET_Tn_ROUTE(i)); 161 h = hpet_readl(HPET_Tn_ROUTE(i)+4); 162 printk(KERN_INFO "hpet: T%d ROUTE_l: 0x%x, ROUTE_h: 0x%x\n", 163 i, l, h); 164 } 165 } 166 167 #define hpet_print_config() \ 168 do { \ 169 if (hpet_verbose) \ 170 _hpet_print_config(__func__, __LINE__); \ 171 } while (0) 172 173 /* 174 * When the hpet driver (/dev/hpet) is enabled, we need to reserve 175 * timer 0 and timer 1 in case of RTC emulation. 176 */ 177 #ifdef CONFIG_HPET 178 179 static void hpet_reserve_msi_timers(struct hpet_data *hd); 180 181 static void hpet_reserve_platform_timers(unsigned int id) 182 { 183 struct hpet __iomem *hpet = hpet_virt_address; 184 struct hpet_timer __iomem *timer = &hpet->hpet_timers[2]; 185 unsigned int nrtimers, i; 186 struct hpet_data hd; 187 188 nrtimers = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT) + 1; 189 190 memset(&hd, 0, sizeof(hd)); 191 hd.hd_phys_address = hpet_address; 192 hd.hd_address = hpet; 193 hd.hd_nirqs = nrtimers; 194 hpet_reserve_timer(&hd, 0); 195 196 #ifdef CONFIG_HPET_EMULATE_RTC 197 hpet_reserve_timer(&hd, 1); 198 #endif 199 200 /* 201 * NOTE that hd_irq[] reflects IOAPIC input pins (LEGACY_8254 202 * is wrong for i8259!) not the output IRQ. Many BIOS writers 203 * don't bother configuring *any* comparator interrupts. 204 */ 205 hd.hd_irq[0] = HPET_LEGACY_8254; 206 hd.hd_irq[1] = HPET_LEGACY_RTC; 207 208 for (i = 2; i < nrtimers; timer++, i++) { 209 hd.hd_irq[i] = (readl(&timer->hpet_config) & 210 Tn_INT_ROUTE_CNF_MASK) >> Tn_INT_ROUTE_CNF_SHIFT; 211 } 212 213 hpet_reserve_msi_timers(&hd); 214 215 hpet_alloc(&hd); 216 217 } 218 #else 219 static void hpet_reserve_platform_timers(unsigned int id) { } 220 #endif 221 222 /* 223 * Common hpet info 224 */ 225 static unsigned long hpet_freq; 226 227 static struct clock_event_device hpet_clockevent; 228 229 static void hpet_stop_counter(void) 230 { 231 u32 cfg = hpet_readl(HPET_CFG); 232 cfg &= ~HPET_CFG_ENABLE; 233 hpet_writel(cfg, HPET_CFG); 234 } 235 236 static void hpet_reset_counter(void) 237 { 238 hpet_writel(0, HPET_COUNTER); 239 hpet_writel(0, HPET_COUNTER + 4); 240 } 241 242 static void hpet_start_counter(void) 243 { 244 unsigned int cfg = hpet_readl(HPET_CFG); 245 cfg |= HPET_CFG_ENABLE; 246 hpet_writel(cfg, HPET_CFG); 247 } 248 249 static void hpet_restart_counter(void) 250 { 251 hpet_stop_counter(); 252 hpet_reset_counter(); 253 hpet_start_counter(); 254 } 255 256 static void hpet_resume_device(void) 257 { 258 force_hpet_resume(); 259 } 260 261 static void hpet_resume_counter(struct clocksource *cs) 262 { 263 hpet_resume_device(); 264 hpet_restart_counter(); 265 } 266 267 static void hpet_enable_legacy_int(void) 268 { 269 unsigned int cfg = hpet_readl(HPET_CFG); 270 271 cfg |= HPET_CFG_LEGACY; 272 hpet_writel(cfg, HPET_CFG); 273 hpet_legacy_int_enabled = true; 274 } 275 276 static void hpet_legacy_clockevent_register(void) 277 { 278 /* Start HPET legacy interrupts */ 279 hpet_enable_legacy_int(); 280 281 /* 282 * Start hpet with the boot cpu mask and make it 283 * global after the IO_APIC has been initialized. 284 */ 285 hpet_clockevent.cpumask = cpumask_of(boot_cpu_data.cpu_index); 286 clockevents_config_and_register(&hpet_clockevent, hpet_freq, 287 HPET_MIN_PROG_DELTA, 0x7FFFFFFF); 288 global_clock_event = &hpet_clockevent; 289 printk(KERN_DEBUG "hpet clockevent registered\n"); 290 } 291 292 static int hpet_set_periodic(struct clock_event_device *evt, int timer) 293 { 294 unsigned int cfg, cmp, now; 295 uint64_t delta; 296 297 hpet_stop_counter(); 298 delta = ((uint64_t)(NSEC_PER_SEC / HZ)) * evt->mult; 299 delta >>= evt->shift; 300 now = hpet_readl(HPET_COUNTER); 301 cmp = now + (unsigned int)delta; 302 cfg = hpet_readl(HPET_Tn_CFG(timer)); 303 cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC | HPET_TN_SETVAL | 304 HPET_TN_32BIT; 305 hpet_writel(cfg, HPET_Tn_CFG(timer)); 306 hpet_writel(cmp, HPET_Tn_CMP(timer)); 307 udelay(1); 308 /* 309 * HPET on AMD 81xx needs a second write (with HPET_TN_SETVAL 310 * cleared) to T0_CMP to set the period. The HPET_TN_SETVAL 311 * bit is automatically cleared after the first write. 312 * (See AMD-8111 HyperTransport I/O Hub Data Sheet, 313 * Publication # 24674) 314 */ 315 hpet_writel((unsigned int)delta, HPET_Tn_CMP(timer)); 316 hpet_start_counter(); 317 hpet_print_config(); 318 319 return 0; 320 } 321 322 static int hpet_set_oneshot(struct clock_event_device *evt, int timer) 323 { 324 unsigned int cfg; 325 326 cfg = hpet_readl(HPET_Tn_CFG(timer)); 327 cfg &= ~HPET_TN_PERIODIC; 328 cfg |= HPET_TN_ENABLE | HPET_TN_32BIT; 329 hpet_writel(cfg, HPET_Tn_CFG(timer)); 330 331 return 0; 332 } 333 334 static int hpet_shutdown(struct clock_event_device *evt, int timer) 335 { 336 unsigned int cfg; 337 338 cfg = hpet_readl(HPET_Tn_CFG(timer)); 339 cfg &= ~HPET_TN_ENABLE; 340 hpet_writel(cfg, HPET_Tn_CFG(timer)); 341 342 return 0; 343 } 344 345 static int hpet_resume(struct clock_event_device *evt) 346 { 347 hpet_enable_legacy_int(); 348 hpet_print_config(); 349 return 0; 350 } 351 352 static int hpet_next_event(unsigned long delta, 353 struct clock_event_device *evt, int timer) 354 { 355 u32 cnt; 356 s32 res; 357 358 cnt = hpet_readl(HPET_COUNTER); 359 cnt += (u32) delta; 360 hpet_writel(cnt, HPET_Tn_CMP(timer)); 361 362 /* 363 * HPETs are a complete disaster. The compare register is 364 * based on a equal comparison and neither provides a less 365 * than or equal functionality (which would require to take 366 * the wraparound into account) nor a simple count down event 367 * mode. Further the write to the comparator register is 368 * delayed internally up to two HPET clock cycles in certain 369 * chipsets (ATI, ICH9,10). Some newer AMD chipsets have even 370 * longer delays. We worked around that by reading back the 371 * compare register, but that required another workaround for 372 * ICH9,10 chips where the first readout after write can 373 * return the old stale value. We already had a minimum 374 * programming delta of 5us enforced, but a NMI or SMI hitting 375 * between the counter readout and the comparator write can 376 * move us behind that point easily. Now instead of reading 377 * the compare register back several times, we make the ETIME 378 * decision based on the following: Return ETIME if the 379 * counter value after the write is less than HPET_MIN_CYCLES 380 * away from the event or if the counter is already ahead of 381 * the event. The minimum programming delta for the generic 382 * clockevents code is set to 1.5 * HPET_MIN_CYCLES. 383 */ 384 res = (s32)(cnt - hpet_readl(HPET_COUNTER)); 385 386 return res < HPET_MIN_CYCLES ? -ETIME : 0; 387 } 388 389 static int hpet_legacy_shutdown(struct clock_event_device *evt) 390 { 391 return hpet_shutdown(evt, 0); 392 } 393 394 static int hpet_legacy_set_oneshot(struct clock_event_device *evt) 395 { 396 return hpet_set_oneshot(evt, 0); 397 } 398 399 static int hpet_legacy_set_periodic(struct clock_event_device *evt) 400 { 401 return hpet_set_periodic(evt, 0); 402 } 403 404 static int hpet_legacy_resume(struct clock_event_device *evt) 405 { 406 return hpet_resume(evt); 407 } 408 409 static int hpet_legacy_next_event(unsigned long delta, 410 struct clock_event_device *evt) 411 { 412 return hpet_next_event(delta, evt, 0); 413 } 414 415 /* 416 * The hpet clock event device 417 */ 418 static struct clock_event_device hpet_clockevent = { 419 .name = "hpet", 420 .features = CLOCK_EVT_FEAT_PERIODIC | 421 CLOCK_EVT_FEAT_ONESHOT, 422 .set_state_periodic = hpet_legacy_set_periodic, 423 .set_state_oneshot = hpet_legacy_set_oneshot, 424 .set_state_shutdown = hpet_legacy_shutdown, 425 .tick_resume = hpet_legacy_resume, 426 .set_next_event = hpet_legacy_next_event, 427 .irq = 0, 428 .rating = 50, 429 }; 430 431 /* 432 * HPET MSI Support 433 */ 434 #ifdef CONFIG_PCI_MSI 435 436 static DEFINE_PER_CPU(struct hpet_dev *, cpu_hpet_dev); 437 static struct hpet_dev *hpet_devs; 438 static struct irq_domain *hpet_domain; 439 440 void hpet_msi_unmask(struct irq_data *data) 441 { 442 struct hpet_dev *hdev = irq_data_get_irq_handler_data(data); 443 unsigned int cfg; 444 445 /* unmask it */ 446 cfg = hpet_readl(HPET_Tn_CFG(hdev->num)); 447 cfg |= HPET_TN_ENABLE | HPET_TN_FSB; 448 hpet_writel(cfg, HPET_Tn_CFG(hdev->num)); 449 } 450 451 void hpet_msi_mask(struct irq_data *data) 452 { 453 struct hpet_dev *hdev = irq_data_get_irq_handler_data(data); 454 unsigned int cfg; 455 456 /* mask it */ 457 cfg = hpet_readl(HPET_Tn_CFG(hdev->num)); 458 cfg &= ~(HPET_TN_ENABLE | HPET_TN_FSB); 459 hpet_writel(cfg, HPET_Tn_CFG(hdev->num)); 460 } 461 462 void hpet_msi_write(struct hpet_dev *hdev, struct msi_msg *msg) 463 { 464 hpet_writel(msg->data, HPET_Tn_ROUTE(hdev->num)); 465 hpet_writel(msg->address_lo, HPET_Tn_ROUTE(hdev->num) + 4); 466 } 467 468 void hpet_msi_read(struct hpet_dev *hdev, struct msi_msg *msg) 469 { 470 msg->data = hpet_readl(HPET_Tn_ROUTE(hdev->num)); 471 msg->address_lo = hpet_readl(HPET_Tn_ROUTE(hdev->num) + 4); 472 msg->address_hi = 0; 473 } 474 475 static int hpet_msi_shutdown(struct clock_event_device *evt) 476 { 477 struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); 478 479 return hpet_shutdown(evt, hdev->num); 480 } 481 482 static int hpet_msi_set_oneshot(struct clock_event_device *evt) 483 { 484 struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); 485 486 return hpet_set_oneshot(evt, hdev->num); 487 } 488 489 static int hpet_msi_set_periodic(struct clock_event_device *evt) 490 { 491 struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); 492 493 return hpet_set_periodic(evt, hdev->num); 494 } 495 496 static int hpet_msi_resume(struct clock_event_device *evt) 497 { 498 struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); 499 struct irq_data *data = irq_get_irq_data(hdev->irq); 500 struct msi_msg msg; 501 502 /* Restore the MSI msg and unmask the interrupt */ 503 irq_chip_compose_msi_msg(data, &msg); 504 hpet_msi_write(hdev, &msg); 505 hpet_msi_unmask(data); 506 return 0; 507 } 508 509 static int hpet_msi_next_event(unsigned long delta, 510 struct clock_event_device *evt) 511 { 512 struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); 513 return hpet_next_event(delta, evt, hdev->num); 514 } 515 516 static irqreturn_t hpet_interrupt_handler(int irq, void *data) 517 { 518 struct hpet_dev *dev = (struct hpet_dev *)data; 519 struct clock_event_device *hevt = &dev->evt; 520 521 if (!hevt->event_handler) { 522 printk(KERN_INFO "Spurious HPET timer interrupt on HPET timer %d\n", 523 dev->num); 524 return IRQ_HANDLED; 525 } 526 527 hevt->event_handler(hevt); 528 return IRQ_HANDLED; 529 } 530 531 static int hpet_setup_irq(struct hpet_dev *dev) 532 { 533 534 if (request_irq(dev->irq, hpet_interrupt_handler, 535 IRQF_TIMER | IRQF_NOBALANCING, 536 dev->name, dev)) 537 return -1; 538 539 disable_irq(dev->irq); 540 irq_set_affinity(dev->irq, cpumask_of(dev->cpu)); 541 enable_irq(dev->irq); 542 543 printk(KERN_DEBUG "hpet: %s irq %d for MSI\n", 544 dev->name, dev->irq); 545 546 return 0; 547 } 548 549 /* This should be called in specific @cpu */ 550 static void init_one_hpet_msi_clockevent(struct hpet_dev *hdev, int cpu) 551 { 552 struct clock_event_device *evt = &hdev->evt; 553 554 WARN_ON(cpu != smp_processor_id()); 555 if (!(hdev->flags & HPET_DEV_VALID)) 556 return; 557 558 hdev->cpu = cpu; 559 per_cpu(cpu_hpet_dev, cpu) = hdev; 560 evt->name = hdev->name; 561 hpet_setup_irq(hdev); 562 evt->irq = hdev->irq; 563 564 evt->rating = 110; 565 evt->features = CLOCK_EVT_FEAT_ONESHOT; 566 if (hdev->flags & HPET_DEV_PERI_CAP) { 567 evt->features |= CLOCK_EVT_FEAT_PERIODIC; 568 evt->set_state_periodic = hpet_msi_set_periodic; 569 } 570 571 evt->set_state_shutdown = hpet_msi_shutdown; 572 evt->set_state_oneshot = hpet_msi_set_oneshot; 573 evt->tick_resume = hpet_msi_resume; 574 evt->set_next_event = hpet_msi_next_event; 575 evt->cpumask = cpumask_of(hdev->cpu); 576 577 clockevents_config_and_register(evt, hpet_freq, HPET_MIN_PROG_DELTA, 578 0x7FFFFFFF); 579 } 580 581 #ifdef CONFIG_HPET 582 /* Reserve at least one timer for userspace (/dev/hpet) */ 583 #define RESERVE_TIMERS 1 584 #else 585 #define RESERVE_TIMERS 0 586 #endif 587 588 static void hpet_msi_capability_lookup(unsigned int start_timer) 589 { 590 unsigned int id; 591 unsigned int num_timers; 592 unsigned int num_timers_used = 0; 593 int i, irq; 594 595 if (hpet_msi_disable) 596 return; 597 598 if (boot_cpu_has(X86_FEATURE_ARAT)) 599 return; 600 id = hpet_readl(HPET_ID); 601 602 num_timers = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT); 603 num_timers++; /* Value read out starts from 0 */ 604 hpet_print_config(); 605 606 hpet_domain = hpet_create_irq_domain(hpet_blockid); 607 if (!hpet_domain) 608 return; 609 610 hpet_devs = kcalloc(num_timers, sizeof(struct hpet_dev), GFP_KERNEL); 611 if (!hpet_devs) 612 return; 613 614 hpet_num_timers = num_timers; 615 616 for (i = start_timer; i < num_timers - RESERVE_TIMERS; i++) { 617 struct hpet_dev *hdev = &hpet_devs[num_timers_used]; 618 unsigned int cfg = hpet_readl(HPET_Tn_CFG(i)); 619 620 /* Only consider HPET timer with MSI support */ 621 if (!(cfg & HPET_TN_FSB_CAP)) 622 continue; 623 624 hdev->flags = 0; 625 if (cfg & HPET_TN_PERIODIC_CAP) 626 hdev->flags |= HPET_DEV_PERI_CAP; 627 sprintf(hdev->name, "hpet%d", i); 628 hdev->num = i; 629 630 irq = hpet_assign_irq(hpet_domain, hdev, hdev->num); 631 if (irq <= 0) 632 continue; 633 634 hdev->irq = irq; 635 hdev->flags |= HPET_DEV_FSB_CAP; 636 hdev->flags |= HPET_DEV_VALID; 637 num_timers_used++; 638 if (num_timers_used == num_possible_cpus()) 639 break; 640 } 641 642 printk(KERN_INFO "HPET: %d timers in total, %d timers will be used for per-cpu timer\n", 643 num_timers, num_timers_used); 644 } 645 646 #ifdef CONFIG_HPET 647 static void hpet_reserve_msi_timers(struct hpet_data *hd) 648 { 649 int i; 650 651 if (!hpet_devs) 652 return; 653 654 for (i = 0; i < hpet_num_timers; i++) { 655 struct hpet_dev *hdev = &hpet_devs[i]; 656 657 if (!(hdev->flags & HPET_DEV_VALID)) 658 continue; 659 660 hd->hd_irq[hdev->num] = hdev->irq; 661 hpet_reserve_timer(hd, hdev->num); 662 } 663 } 664 #endif 665 666 static struct hpet_dev *hpet_get_unused_timer(void) 667 { 668 int i; 669 670 if (!hpet_devs) 671 return NULL; 672 673 for (i = 0; i < hpet_num_timers; i++) { 674 struct hpet_dev *hdev = &hpet_devs[i]; 675 676 if (!(hdev->flags & HPET_DEV_VALID)) 677 continue; 678 if (test_and_set_bit(HPET_DEV_USED_BIT, 679 (unsigned long *)&hdev->flags)) 680 continue; 681 return hdev; 682 } 683 return NULL; 684 } 685 686 struct hpet_work_struct { 687 struct delayed_work work; 688 struct completion complete; 689 }; 690 691 static void hpet_work(struct work_struct *w) 692 { 693 struct hpet_dev *hdev; 694 int cpu = smp_processor_id(); 695 struct hpet_work_struct *hpet_work; 696 697 hpet_work = container_of(w, struct hpet_work_struct, work.work); 698 699 hdev = hpet_get_unused_timer(); 700 if (hdev) 701 init_one_hpet_msi_clockevent(hdev, cpu); 702 703 complete(&hpet_work->complete); 704 } 705 706 static int hpet_cpuhp_online(unsigned int cpu) 707 { 708 struct hpet_work_struct work; 709 710 INIT_DELAYED_WORK_ONSTACK(&work.work, hpet_work); 711 init_completion(&work.complete); 712 /* FIXME: add schedule_work_on() */ 713 schedule_delayed_work_on(cpu, &work.work, 0); 714 wait_for_completion(&work.complete); 715 destroy_delayed_work_on_stack(&work.work); 716 return 0; 717 } 718 719 static int hpet_cpuhp_dead(unsigned int cpu) 720 { 721 struct hpet_dev *hdev = per_cpu(cpu_hpet_dev, cpu); 722 723 if (!hdev) 724 return 0; 725 free_irq(hdev->irq, hdev); 726 hdev->flags &= ~HPET_DEV_USED; 727 per_cpu(cpu_hpet_dev, cpu) = NULL; 728 return 0; 729 } 730 #else 731 732 static void hpet_msi_capability_lookup(unsigned int start_timer) 733 { 734 return; 735 } 736 737 #ifdef CONFIG_HPET 738 static void hpet_reserve_msi_timers(struct hpet_data *hd) 739 { 740 return; 741 } 742 #endif 743 744 #define hpet_cpuhp_online NULL 745 #define hpet_cpuhp_dead NULL 746 747 #endif 748 749 /* 750 * Clock source related code 751 */ 752 #if defined(CONFIG_SMP) && defined(CONFIG_64BIT) 753 /* 754 * Reading the HPET counter is a very slow operation. If a large number of 755 * CPUs are trying to access the HPET counter simultaneously, it can cause 756 * massive delay and slow down system performance dramatically. This may 757 * happen when HPET is the default clock source instead of TSC. For a 758 * really large system with hundreds of CPUs, the slowdown may be so 759 * severe that it may actually crash the system because of a NMI watchdog 760 * soft lockup, for example. 761 * 762 * If multiple CPUs are trying to access the HPET counter at the same time, 763 * we don't actually need to read the counter multiple times. Instead, the 764 * other CPUs can use the counter value read by the first CPU in the group. 765 * 766 * This special feature is only enabled on x86-64 systems. It is unlikely 767 * that 32-bit x86 systems will have enough CPUs to require this feature 768 * with its associated locking overhead. And we also need 64-bit atomic 769 * read. 770 * 771 * The lock and the hpet value are stored together and can be read in a 772 * single atomic 64-bit read. It is explicitly assumed that arch_spinlock_t 773 * is 32 bits in size. 774 */ 775 union hpet_lock { 776 struct { 777 arch_spinlock_t lock; 778 u32 value; 779 }; 780 u64 lockval; 781 }; 782 783 static union hpet_lock hpet __cacheline_aligned = { 784 { .lock = __ARCH_SPIN_LOCK_UNLOCKED, }, 785 }; 786 787 static u64 read_hpet(struct clocksource *cs) 788 { 789 unsigned long flags; 790 union hpet_lock old, new; 791 792 BUILD_BUG_ON(sizeof(union hpet_lock) != 8); 793 794 /* 795 * Read HPET directly if in NMI. 796 */ 797 if (in_nmi()) 798 return (u64)hpet_readl(HPET_COUNTER); 799 800 /* 801 * Read the current state of the lock and HPET value atomically. 802 */ 803 old.lockval = READ_ONCE(hpet.lockval); 804 805 if (arch_spin_is_locked(&old.lock)) 806 goto contended; 807 808 local_irq_save(flags); 809 if (arch_spin_trylock(&hpet.lock)) { 810 new.value = hpet_readl(HPET_COUNTER); 811 /* 812 * Use WRITE_ONCE() to prevent store tearing. 813 */ 814 WRITE_ONCE(hpet.value, new.value); 815 arch_spin_unlock(&hpet.lock); 816 local_irq_restore(flags); 817 return (u64)new.value; 818 } 819 local_irq_restore(flags); 820 821 contended: 822 /* 823 * Contended case 824 * -------------- 825 * Wait until the HPET value change or the lock is free to indicate 826 * its value is up-to-date. 827 * 828 * It is possible that old.value has already contained the latest 829 * HPET value while the lock holder was in the process of releasing 830 * the lock. Checking for lock state change will enable us to return 831 * the value immediately instead of waiting for the next HPET reader 832 * to come along. 833 */ 834 do { 835 cpu_relax(); 836 new.lockval = READ_ONCE(hpet.lockval); 837 } while ((new.value == old.value) && arch_spin_is_locked(&new.lock)); 838 839 return (u64)new.value; 840 } 841 #else 842 /* 843 * For UP or 32-bit. 844 */ 845 static u64 read_hpet(struct clocksource *cs) 846 { 847 return (u64)hpet_readl(HPET_COUNTER); 848 } 849 #endif 850 851 static struct clocksource clocksource_hpet = { 852 .name = "hpet", 853 .rating = 250, 854 .read = read_hpet, 855 .mask = HPET_MASK, 856 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 857 .resume = hpet_resume_counter, 858 }; 859 860 static int hpet_clocksource_register(void) 861 { 862 u64 start, now; 863 u64 t1; 864 865 /* Start the counter */ 866 hpet_restart_counter(); 867 868 /* Verify whether hpet counter works */ 869 t1 = hpet_readl(HPET_COUNTER); 870 start = rdtsc(); 871 872 /* 873 * We don't know the TSC frequency yet, but waiting for 874 * 200000 TSC cycles is safe: 875 * 4 GHz == 50us 876 * 1 GHz == 200us 877 */ 878 do { 879 rep_nop(); 880 now = rdtsc(); 881 } while ((now - start) < 200000UL); 882 883 if (t1 == hpet_readl(HPET_COUNTER)) { 884 printk(KERN_WARNING 885 "HPET counter not counting. HPET disabled\n"); 886 return -ENODEV; 887 } 888 889 clocksource_register_hz(&clocksource_hpet, (u32)hpet_freq); 890 return 0; 891 } 892 893 static u32 *hpet_boot_cfg; 894 895 /** 896 * hpet_enable - Try to setup the HPET timer. Returns 1 on success. 897 */ 898 int __init hpet_enable(void) 899 { 900 u32 hpet_period, cfg, id; 901 u64 freq; 902 unsigned int i, last; 903 904 if (!is_hpet_capable()) 905 return 0; 906 907 hpet_set_mapping(); 908 if (!hpet_virt_address) 909 return 0; 910 911 /* 912 * Read the period and check for a sane value: 913 */ 914 hpet_period = hpet_readl(HPET_PERIOD); 915 916 /* 917 * AMD SB700 based systems with spread spectrum enabled use a 918 * SMM based HPET emulation to provide proper frequency 919 * setting. The SMM code is initialized with the first HPET 920 * register access and takes some time to complete. During 921 * this time the config register reads 0xffffffff. We check 922 * for max. 1000 loops whether the config register reads a non 923 * 0xffffffff value to make sure that HPET is up and running 924 * before we go further. A counting loop is safe, as the HPET 925 * access takes thousands of CPU cycles. On non SB700 based 926 * machines this check is only done once and has no side 927 * effects. 928 */ 929 for (i = 0; hpet_readl(HPET_CFG) == 0xFFFFFFFF; i++) { 930 if (i == 1000) { 931 printk(KERN_WARNING 932 "HPET config register value = 0xFFFFFFFF. " 933 "Disabling HPET\n"); 934 goto out_nohpet; 935 } 936 } 937 938 if (hpet_period < HPET_MIN_PERIOD || hpet_period > HPET_MAX_PERIOD) 939 goto out_nohpet; 940 941 /* 942 * The period is a femto seconds value. Convert it to a 943 * frequency. 944 */ 945 freq = FSEC_PER_SEC; 946 do_div(freq, hpet_period); 947 hpet_freq = freq; 948 949 /* 950 * Read the HPET ID register to retrieve the IRQ routing 951 * information and the number of channels 952 */ 953 id = hpet_readl(HPET_ID); 954 hpet_print_config(); 955 956 last = (id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT; 957 958 #ifdef CONFIG_HPET_EMULATE_RTC 959 /* 960 * The legacy routing mode needs at least two channels, tick timer 961 * and the rtc emulation channel. 962 */ 963 if (!last) 964 goto out_nohpet; 965 #endif 966 967 cfg = hpet_readl(HPET_CFG); 968 hpet_boot_cfg = kmalloc_array(last + 2, sizeof(*hpet_boot_cfg), 969 GFP_KERNEL); 970 if (hpet_boot_cfg) 971 *hpet_boot_cfg = cfg; 972 else 973 pr_warn("HPET initial state will not be saved\n"); 974 cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY); 975 hpet_writel(cfg, HPET_CFG); 976 if (cfg) 977 pr_warn("Unrecognized bits %#x set in global cfg\n", cfg); 978 979 for (i = 0; i <= last; ++i) { 980 cfg = hpet_readl(HPET_Tn_CFG(i)); 981 if (hpet_boot_cfg) 982 hpet_boot_cfg[i + 1] = cfg; 983 cfg &= ~(HPET_TN_ENABLE | HPET_TN_LEVEL | HPET_TN_FSB); 984 hpet_writel(cfg, HPET_Tn_CFG(i)); 985 cfg &= ~(HPET_TN_PERIODIC | HPET_TN_PERIODIC_CAP 986 | HPET_TN_64BIT_CAP | HPET_TN_32BIT | HPET_TN_ROUTE 987 | HPET_TN_FSB | HPET_TN_FSB_CAP); 988 if (cfg) 989 pr_warn("Unrecognized bits %#x set in cfg#%u\n", 990 cfg, i); 991 } 992 hpet_print_config(); 993 994 if (hpet_clocksource_register()) 995 goto out_nohpet; 996 997 if (id & HPET_ID_LEGSUP) { 998 hpet_legacy_clockevent_register(); 999 return 1; 1000 } 1001 return 0; 1002 1003 out_nohpet: 1004 hpet_clear_mapping(); 1005 hpet_address = 0; 1006 return 0; 1007 } 1008 1009 /* 1010 * Needs to be late, as the reserve_timer code calls kalloc ! 1011 * 1012 * Not a problem on i386 as hpet_enable is called from late_time_init, 1013 * but on x86_64 it is necessary ! 1014 */ 1015 static __init int hpet_late_init(void) 1016 { 1017 int ret; 1018 1019 if (boot_hpet_disable) 1020 return -ENODEV; 1021 1022 if (!hpet_address) { 1023 if (!force_hpet_address) 1024 return -ENODEV; 1025 1026 hpet_address = force_hpet_address; 1027 hpet_enable(); 1028 } 1029 1030 if (!hpet_virt_address) 1031 return -ENODEV; 1032 1033 if (hpet_readl(HPET_ID) & HPET_ID_LEGSUP) 1034 hpet_msi_capability_lookup(2); 1035 else 1036 hpet_msi_capability_lookup(0); 1037 1038 hpet_reserve_platform_timers(hpet_readl(HPET_ID)); 1039 hpet_print_config(); 1040 1041 if (hpet_msi_disable) 1042 return 0; 1043 1044 if (boot_cpu_has(X86_FEATURE_ARAT)) 1045 return 0; 1046 1047 /* This notifier should be called after workqueue is ready */ 1048 ret = cpuhp_setup_state(CPUHP_AP_X86_HPET_ONLINE, "x86/hpet:online", 1049 hpet_cpuhp_online, NULL); 1050 if (ret) 1051 return ret; 1052 ret = cpuhp_setup_state(CPUHP_X86_HPET_DEAD, "x86/hpet:dead", NULL, 1053 hpet_cpuhp_dead); 1054 if (ret) 1055 goto err_cpuhp; 1056 return 0; 1057 1058 err_cpuhp: 1059 cpuhp_remove_state(CPUHP_AP_X86_HPET_ONLINE); 1060 return ret; 1061 } 1062 fs_initcall(hpet_late_init); 1063 1064 void hpet_disable(void) 1065 { 1066 if (is_hpet_capable() && hpet_virt_address) { 1067 unsigned int cfg = hpet_readl(HPET_CFG), id, last; 1068 1069 if (hpet_boot_cfg) 1070 cfg = *hpet_boot_cfg; 1071 else if (hpet_legacy_int_enabled) { 1072 cfg &= ~HPET_CFG_LEGACY; 1073 hpet_legacy_int_enabled = false; 1074 } 1075 cfg &= ~HPET_CFG_ENABLE; 1076 hpet_writel(cfg, HPET_CFG); 1077 1078 if (!hpet_boot_cfg) 1079 return; 1080 1081 id = hpet_readl(HPET_ID); 1082 last = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT); 1083 1084 for (id = 0; id <= last; ++id) 1085 hpet_writel(hpet_boot_cfg[id + 1], HPET_Tn_CFG(id)); 1086 1087 if (*hpet_boot_cfg & HPET_CFG_ENABLE) 1088 hpet_writel(*hpet_boot_cfg, HPET_CFG); 1089 } 1090 } 1091 1092 #ifdef CONFIG_HPET_EMULATE_RTC 1093 1094 /* HPET in LegacyReplacement Mode eats up RTC interrupt line. When, HPET 1095 * is enabled, we support RTC interrupt functionality in software. 1096 * RTC has 3 kinds of interrupts: 1097 * 1) Update Interrupt - generate an interrupt, every sec, when RTC clock 1098 * is updated 1099 * 2) Alarm Interrupt - generate an interrupt at a specific time of day 1100 * 3) Periodic Interrupt - generate periodic interrupt, with frequencies 1101 * 2Hz-8192Hz (2Hz-64Hz for non-root user) (all freqs in powers of 2) 1102 * (1) and (2) above are implemented using polling at a frequency of 1103 * 64 Hz. The exact frequency is a tradeoff between accuracy and interrupt 1104 * overhead. (DEFAULT_RTC_INT_FREQ) 1105 * For (3), we use interrupts at 64Hz or user specified periodic 1106 * frequency, whichever is higher. 1107 */ 1108 #include <linux/mc146818rtc.h> 1109 #include <linux/rtc.h> 1110 1111 #define DEFAULT_RTC_INT_FREQ 64 1112 #define DEFAULT_RTC_SHIFT 6 1113 #define RTC_NUM_INTS 1 1114 1115 static unsigned long hpet_rtc_flags; 1116 static int hpet_prev_update_sec; 1117 static struct rtc_time hpet_alarm_time; 1118 static unsigned long hpet_pie_count; 1119 static u32 hpet_t1_cmp; 1120 static u32 hpet_default_delta; 1121 static u32 hpet_pie_delta; 1122 static unsigned long hpet_pie_limit; 1123 1124 static rtc_irq_handler irq_handler; 1125 1126 /* 1127 * Check that the hpet counter c1 is ahead of the c2 1128 */ 1129 static inline int hpet_cnt_ahead(u32 c1, u32 c2) 1130 { 1131 return (s32)(c2 - c1) < 0; 1132 } 1133 1134 /* 1135 * Registers a IRQ handler. 1136 */ 1137 int hpet_register_irq_handler(rtc_irq_handler handler) 1138 { 1139 if (!is_hpet_enabled()) 1140 return -ENODEV; 1141 if (irq_handler) 1142 return -EBUSY; 1143 1144 irq_handler = handler; 1145 1146 return 0; 1147 } 1148 EXPORT_SYMBOL_GPL(hpet_register_irq_handler); 1149 1150 /* 1151 * Deregisters the IRQ handler registered with hpet_register_irq_handler() 1152 * and does cleanup. 1153 */ 1154 void hpet_unregister_irq_handler(rtc_irq_handler handler) 1155 { 1156 if (!is_hpet_enabled()) 1157 return; 1158 1159 irq_handler = NULL; 1160 hpet_rtc_flags = 0; 1161 } 1162 EXPORT_SYMBOL_GPL(hpet_unregister_irq_handler); 1163 1164 /* 1165 * Timer 1 for RTC emulation. We use one shot mode, as periodic mode 1166 * is not supported by all HPET implementations for timer 1. 1167 * 1168 * hpet_rtc_timer_init() is called when the rtc is initialized. 1169 */ 1170 int hpet_rtc_timer_init(void) 1171 { 1172 unsigned int cfg, cnt, delta; 1173 unsigned long flags; 1174 1175 if (!is_hpet_enabled()) 1176 return 0; 1177 1178 if (!hpet_default_delta) { 1179 uint64_t clc; 1180 1181 clc = (uint64_t) hpet_clockevent.mult * NSEC_PER_SEC; 1182 clc >>= hpet_clockevent.shift + DEFAULT_RTC_SHIFT; 1183 hpet_default_delta = clc; 1184 } 1185 1186 if (!(hpet_rtc_flags & RTC_PIE) || hpet_pie_limit) 1187 delta = hpet_default_delta; 1188 else 1189 delta = hpet_pie_delta; 1190 1191 local_irq_save(flags); 1192 1193 cnt = delta + hpet_readl(HPET_COUNTER); 1194 hpet_writel(cnt, HPET_T1_CMP); 1195 hpet_t1_cmp = cnt; 1196 1197 cfg = hpet_readl(HPET_T1_CFG); 1198 cfg &= ~HPET_TN_PERIODIC; 1199 cfg |= HPET_TN_ENABLE | HPET_TN_32BIT; 1200 hpet_writel(cfg, HPET_T1_CFG); 1201 1202 local_irq_restore(flags); 1203 1204 return 1; 1205 } 1206 EXPORT_SYMBOL_GPL(hpet_rtc_timer_init); 1207 1208 static void hpet_disable_rtc_channel(void) 1209 { 1210 u32 cfg = hpet_readl(HPET_T1_CFG); 1211 cfg &= ~HPET_TN_ENABLE; 1212 hpet_writel(cfg, HPET_T1_CFG); 1213 } 1214 1215 /* 1216 * The functions below are called from rtc driver. 1217 * Return 0 if HPET is not being used. 1218 * Otherwise do the necessary changes and return 1. 1219 */ 1220 int hpet_mask_rtc_irq_bit(unsigned long bit_mask) 1221 { 1222 if (!is_hpet_enabled()) 1223 return 0; 1224 1225 hpet_rtc_flags &= ~bit_mask; 1226 if (unlikely(!hpet_rtc_flags)) 1227 hpet_disable_rtc_channel(); 1228 1229 return 1; 1230 } 1231 EXPORT_SYMBOL_GPL(hpet_mask_rtc_irq_bit); 1232 1233 int hpet_set_rtc_irq_bit(unsigned long bit_mask) 1234 { 1235 unsigned long oldbits = hpet_rtc_flags; 1236 1237 if (!is_hpet_enabled()) 1238 return 0; 1239 1240 hpet_rtc_flags |= bit_mask; 1241 1242 if ((bit_mask & RTC_UIE) && !(oldbits & RTC_UIE)) 1243 hpet_prev_update_sec = -1; 1244 1245 if (!oldbits) 1246 hpet_rtc_timer_init(); 1247 1248 return 1; 1249 } 1250 EXPORT_SYMBOL_GPL(hpet_set_rtc_irq_bit); 1251 1252 int hpet_set_alarm_time(unsigned char hrs, unsigned char min, 1253 unsigned char sec) 1254 { 1255 if (!is_hpet_enabled()) 1256 return 0; 1257 1258 hpet_alarm_time.tm_hour = hrs; 1259 hpet_alarm_time.tm_min = min; 1260 hpet_alarm_time.tm_sec = sec; 1261 1262 return 1; 1263 } 1264 EXPORT_SYMBOL_GPL(hpet_set_alarm_time); 1265 1266 int hpet_set_periodic_freq(unsigned long freq) 1267 { 1268 uint64_t clc; 1269 1270 if (!is_hpet_enabled()) 1271 return 0; 1272 1273 if (freq <= DEFAULT_RTC_INT_FREQ) 1274 hpet_pie_limit = DEFAULT_RTC_INT_FREQ / freq; 1275 else { 1276 clc = (uint64_t) hpet_clockevent.mult * NSEC_PER_SEC; 1277 do_div(clc, freq); 1278 clc >>= hpet_clockevent.shift; 1279 hpet_pie_delta = clc; 1280 hpet_pie_limit = 0; 1281 } 1282 return 1; 1283 } 1284 EXPORT_SYMBOL_GPL(hpet_set_periodic_freq); 1285 1286 int hpet_rtc_dropped_irq(void) 1287 { 1288 return is_hpet_enabled(); 1289 } 1290 EXPORT_SYMBOL_GPL(hpet_rtc_dropped_irq); 1291 1292 static void hpet_rtc_timer_reinit(void) 1293 { 1294 unsigned int delta; 1295 int lost_ints = -1; 1296 1297 if (unlikely(!hpet_rtc_flags)) 1298 hpet_disable_rtc_channel(); 1299 1300 if (!(hpet_rtc_flags & RTC_PIE) || hpet_pie_limit) 1301 delta = hpet_default_delta; 1302 else 1303 delta = hpet_pie_delta; 1304 1305 /* 1306 * Increment the comparator value until we are ahead of the 1307 * current count. 1308 */ 1309 do { 1310 hpet_t1_cmp += delta; 1311 hpet_writel(hpet_t1_cmp, HPET_T1_CMP); 1312 lost_ints++; 1313 } while (!hpet_cnt_ahead(hpet_t1_cmp, hpet_readl(HPET_COUNTER))); 1314 1315 if (lost_ints) { 1316 if (hpet_rtc_flags & RTC_PIE) 1317 hpet_pie_count += lost_ints; 1318 if (printk_ratelimit()) 1319 printk(KERN_WARNING "hpet1: lost %d rtc interrupts\n", 1320 lost_ints); 1321 } 1322 } 1323 1324 irqreturn_t hpet_rtc_interrupt(int irq, void *dev_id) 1325 { 1326 struct rtc_time curr_time; 1327 unsigned long rtc_int_flag = 0; 1328 1329 hpet_rtc_timer_reinit(); 1330 memset(&curr_time, 0, sizeof(struct rtc_time)); 1331 1332 if (hpet_rtc_flags & (RTC_UIE | RTC_AIE)) 1333 mc146818_get_time(&curr_time); 1334 1335 if (hpet_rtc_flags & RTC_UIE && 1336 curr_time.tm_sec != hpet_prev_update_sec) { 1337 if (hpet_prev_update_sec >= 0) 1338 rtc_int_flag = RTC_UF; 1339 hpet_prev_update_sec = curr_time.tm_sec; 1340 } 1341 1342 if (hpet_rtc_flags & RTC_PIE && 1343 ++hpet_pie_count >= hpet_pie_limit) { 1344 rtc_int_flag |= RTC_PF; 1345 hpet_pie_count = 0; 1346 } 1347 1348 if (hpet_rtc_flags & RTC_AIE && 1349 (curr_time.tm_sec == hpet_alarm_time.tm_sec) && 1350 (curr_time.tm_min == hpet_alarm_time.tm_min) && 1351 (curr_time.tm_hour == hpet_alarm_time.tm_hour)) 1352 rtc_int_flag |= RTC_AF; 1353 1354 if (rtc_int_flag) { 1355 rtc_int_flag |= (RTC_IRQF | (RTC_NUM_INTS << 8)); 1356 if (irq_handler) 1357 irq_handler(rtc_int_flag, dev_id); 1358 } 1359 return IRQ_HANDLED; 1360 } 1361 EXPORT_SYMBOL_GPL(hpet_rtc_interrupt); 1362 #endif 1363