1 /* 2 * SGI NMI support routines 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 * 18 * Copyright (c) 2009-2013 Silicon Graphics, Inc. All Rights Reserved. 19 * Copyright (c) Mike Travis 20 */ 21 22 #include <linux/cpu.h> 23 #include <linux/delay.h> 24 #include <linux/kdb.h> 25 #include <linux/kexec.h> 26 #include <linux/kgdb.h> 27 #include <linux/module.h> 28 #include <linux/nmi.h> 29 #include <linux/sched.h> 30 #include <linux/slab.h> 31 #include <linux/clocksource.h> 32 33 #include <asm/apic.h> 34 #include <asm/current.h> 35 #include <asm/kdebug.h> 36 #include <asm/local64.h> 37 #include <asm/nmi.h> 38 #include <asm/traps.h> 39 #include <asm/uv/uv.h> 40 #include <asm/uv/uv_hub.h> 41 #include <asm/uv/uv_mmrs.h> 42 43 /* 44 * UV handler for NMI 45 * 46 * Handle system-wide NMI events generated by the global 'power nmi' command. 47 * 48 * Basic operation is to field the NMI interrupt on each cpu and wait 49 * until all cpus have arrived into the nmi handler. If some cpus do not 50 * make it into the handler, try and force them in with the IPI(NMI) signal. 51 * 52 * We also have to lessen UV Hub MMR accesses as much as possible as this 53 * disrupts the UV Hub's primary mission of directing NumaLink traffic and 54 * can cause system problems to occur. 55 * 56 * To do this we register our primary NMI notifier on the NMI_UNKNOWN 57 * chain. This reduces the number of false NMI calls when the perf 58 * tools are running which generate an enormous number of NMIs per 59 * second (~4M/s for 1024 cpu threads). Our secondary NMI handler is 60 * very short as it only checks that if it has been "pinged" with the 61 * IPI(NMI) signal as mentioned above, and does not read the UV Hub's MMR. 62 * 63 */ 64 65 static struct uv_hub_nmi_s **uv_hub_nmi_list; 66 67 DEFINE_PER_CPU(struct uv_cpu_nmi_s, uv_cpu_nmi); 68 EXPORT_PER_CPU_SYMBOL_GPL(uv_cpu_nmi); 69 70 static unsigned long nmi_mmr; 71 static unsigned long nmi_mmr_clear; 72 static unsigned long nmi_mmr_pending; 73 74 static atomic_t uv_in_nmi; 75 static atomic_t uv_nmi_cpu = ATOMIC_INIT(-1); 76 static atomic_t uv_nmi_cpus_in_nmi = ATOMIC_INIT(-1); 77 static atomic_t uv_nmi_slave_continue; 78 static cpumask_var_t uv_nmi_cpu_mask; 79 80 /* Values for uv_nmi_slave_continue */ 81 #define SLAVE_CLEAR 0 82 #define SLAVE_CONTINUE 1 83 #define SLAVE_EXIT 2 84 85 /* 86 * Default is all stack dumps go to the console and buffer. 87 * Lower level to send to log buffer only. 88 */ 89 static int uv_nmi_loglevel = CONSOLE_LOGLEVEL_DEFAULT; 90 module_param_named(dump_loglevel, uv_nmi_loglevel, int, 0644); 91 92 /* 93 * The following values show statistics on how perf events are affecting 94 * this system. 95 */ 96 static int param_get_local64(char *buffer, const struct kernel_param *kp) 97 { 98 return sprintf(buffer, "%lu\n", local64_read((local64_t *)kp->arg)); 99 } 100 101 static int param_set_local64(const char *val, const struct kernel_param *kp) 102 { 103 /* clear on any write */ 104 local64_set((local64_t *)kp->arg, 0); 105 return 0; 106 } 107 108 static const struct kernel_param_ops param_ops_local64 = { 109 .get = param_get_local64, 110 .set = param_set_local64, 111 }; 112 #define param_check_local64(name, p) __param_check(name, p, local64_t) 113 114 static local64_t uv_nmi_count; 115 module_param_named(nmi_count, uv_nmi_count, local64, 0644); 116 117 static local64_t uv_nmi_misses; 118 module_param_named(nmi_misses, uv_nmi_misses, local64, 0644); 119 120 static local64_t uv_nmi_ping_count; 121 module_param_named(ping_count, uv_nmi_ping_count, local64, 0644); 122 123 static local64_t uv_nmi_ping_misses; 124 module_param_named(ping_misses, uv_nmi_ping_misses, local64, 0644); 125 126 /* 127 * Following values allow tuning for large systems under heavy loading 128 */ 129 static int uv_nmi_initial_delay = 100; 130 module_param_named(initial_delay, uv_nmi_initial_delay, int, 0644); 131 132 static int uv_nmi_slave_delay = 100; 133 module_param_named(slave_delay, uv_nmi_slave_delay, int, 0644); 134 135 static int uv_nmi_loop_delay = 100; 136 module_param_named(loop_delay, uv_nmi_loop_delay, int, 0644); 137 138 static int uv_nmi_trigger_delay = 10000; 139 module_param_named(trigger_delay, uv_nmi_trigger_delay, int, 0644); 140 141 static int uv_nmi_wait_count = 100; 142 module_param_named(wait_count, uv_nmi_wait_count, int, 0644); 143 144 static int uv_nmi_retry_count = 500; 145 module_param_named(retry_count, uv_nmi_retry_count, int, 0644); 146 147 /* 148 * Valid NMI Actions: 149 * "dump" - dump process stack for each cpu 150 * "ips" - dump IP info for each cpu 151 * "kdump" - do crash dump 152 * "kdb" - enter KDB (default) 153 * "kgdb" - enter KGDB 154 */ 155 static char uv_nmi_action[8] = "kdb"; 156 module_param_string(action, uv_nmi_action, sizeof(uv_nmi_action), 0644); 157 158 static inline bool uv_nmi_action_is(const char *action) 159 { 160 return (strncmp(uv_nmi_action, action, strlen(action)) == 0); 161 } 162 163 /* Setup which NMI support is present in system */ 164 static void uv_nmi_setup_mmrs(void) 165 { 166 if (uv_read_local_mmr(UVH_NMI_MMRX_SUPPORTED)) { 167 uv_write_local_mmr(UVH_NMI_MMRX_REQ, 168 1UL << UVH_NMI_MMRX_REQ_SHIFT); 169 nmi_mmr = UVH_NMI_MMRX; 170 nmi_mmr_clear = UVH_NMI_MMRX_CLEAR; 171 nmi_mmr_pending = 1UL << UVH_NMI_MMRX_SHIFT; 172 pr_info("UV: SMI NMI support: %s\n", UVH_NMI_MMRX_TYPE); 173 } else { 174 nmi_mmr = UVH_NMI_MMR; 175 nmi_mmr_clear = UVH_NMI_MMR_CLEAR; 176 nmi_mmr_pending = 1UL << UVH_NMI_MMR_SHIFT; 177 pr_info("UV: SMI NMI support: %s\n", UVH_NMI_MMR_TYPE); 178 } 179 } 180 181 /* Read NMI MMR and check if NMI flag was set by BMC. */ 182 static inline int uv_nmi_test_mmr(struct uv_hub_nmi_s *hub_nmi) 183 { 184 hub_nmi->nmi_value = uv_read_local_mmr(nmi_mmr); 185 atomic_inc(&hub_nmi->read_mmr_count); 186 return !!(hub_nmi->nmi_value & nmi_mmr_pending); 187 } 188 189 static inline void uv_local_mmr_clear_nmi(void) 190 { 191 uv_write_local_mmr(nmi_mmr_clear, nmi_mmr_pending); 192 } 193 194 /* 195 * If first cpu in on this hub, set hub_nmi "in_nmi" and "owner" values and 196 * return true. If first cpu in on the system, set global "in_nmi" flag. 197 */ 198 static int uv_set_in_nmi(int cpu, struct uv_hub_nmi_s *hub_nmi) 199 { 200 int first = atomic_add_unless(&hub_nmi->in_nmi, 1, 1); 201 202 if (first) { 203 atomic_set(&hub_nmi->cpu_owner, cpu); 204 if (atomic_add_unless(&uv_in_nmi, 1, 1)) 205 atomic_set(&uv_nmi_cpu, cpu); 206 207 atomic_inc(&hub_nmi->nmi_count); 208 } 209 return first; 210 } 211 212 /* Check if this is a system NMI event */ 213 static int uv_check_nmi(struct uv_hub_nmi_s *hub_nmi) 214 { 215 int cpu = smp_processor_id(); 216 int nmi = 0; 217 218 local64_inc(&uv_nmi_count); 219 this_cpu_inc(uv_cpu_nmi.queries); 220 221 do { 222 nmi = atomic_read(&hub_nmi->in_nmi); 223 if (nmi) 224 break; 225 226 if (raw_spin_trylock(&hub_nmi->nmi_lock)) { 227 228 /* check hub MMR NMI flag */ 229 if (uv_nmi_test_mmr(hub_nmi)) { 230 uv_set_in_nmi(cpu, hub_nmi); 231 nmi = 1; 232 break; 233 } 234 235 /* MMR NMI flag is clear */ 236 raw_spin_unlock(&hub_nmi->nmi_lock); 237 238 } else { 239 /* wait a moment for the hub nmi locker to set flag */ 240 cpu_relax(); 241 udelay(uv_nmi_slave_delay); 242 243 /* re-check hub in_nmi flag */ 244 nmi = atomic_read(&hub_nmi->in_nmi); 245 if (nmi) 246 break; 247 } 248 249 /* check if this BMC missed setting the MMR NMI flag */ 250 if (!nmi) { 251 nmi = atomic_read(&uv_in_nmi); 252 if (nmi) 253 uv_set_in_nmi(cpu, hub_nmi); 254 } 255 256 } while (0); 257 258 if (!nmi) 259 local64_inc(&uv_nmi_misses); 260 261 return nmi; 262 } 263 264 /* Need to reset the NMI MMR register, but only once per hub. */ 265 static inline void uv_clear_nmi(int cpu) 266 { 267 struct uv_hub_nmi_s *hub_nmi = uv_hub_nmi; 268 269 if (cpu == atomic_read(&hub_nmi->cpu_owner)) { 270 atomic_set(&hub_nmi->cpu_owner, -1); 271 atomic_set(&hub_nmi->in_nmi, 0); 272 uv_local_mmr_clear_nmi(); 273 raw_spin_unlock(&hub_nmi->nmi_lock); 274 } 275 } 276 277 /* Ping non-responding cpus attemping to force them into the NMI handler */ 278 static void uv_nmi_nr_cpus_ping(void) 279 { 280 int cpu; 281 282 for_each_cpu(cpu, uv_nmi_cpu_mask) 283 uv_cpu_nmi_per(cpu).pinging = 1; 284 285 apic->send_IPI_mask(uv_nmi_cpu_mask, APIC_DM_NMI); 286 } 287 288 /* Clean up flags for cpus that ignored both NMI and ping */ 289 static void uv_nmi_cleanup_mask(void) 290 { 291 int cpu; 292 293 for_each_cpu(cpu, uv_nmi_cpu_mask) { 294 uv_cpu_nmi_per(cpu).pinging = 0; 295 uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_OUT; 296 cpumask_clear_cpu(cpu, uv_nmi_cpu_mask); 297 } 298 } 299 300 /* Loop waiting as cpus enter nmi handler */ 301 static int uv_nmi_wait_cpus(int first) 302 { 303 int i, j, k, n = num_online_cpus(); 304 int last_k = 0, waiting = 0; 305 306 if (first) { 307 cpumask_copy(uv_nmi_cpu_mask, cpu_online_mask); 308 k = 0; 309 } else { 310 k = n - cpumask_weight(uv_nmi_cpu_mask); 311 } 312 313 udelay(uv_nmi_initial_delay); 314 for (i = 0; i < uv_nmi_retry_count; i++) { 315 int loop_delay = uv_nmi_loop_delay; 316 317 for_each_cpu(j, uv_nmi_cpu_mask) { 318 if (uv_cpu_nmi_per(j).state) { 319 cpumask_clear_cpu(j, uv_nmi_cpu_mask); 320 if (++k >= n) 321 break; 322 } 323 } 324 if (k >= n) { /* all in? */ 325 k = n; 326 break; 327 } 328 if (last_k != k) { /* abort if no new cpus coming in */ 329 last_k = k; 330 waiting = 0; 331 } else if (++waiting > uv_nmi_wait_count) 332 break; 333 334 /* extend delay if waiting only for cpu 0 */ 335 if (waiting && (n - k) == 1 && 336 cpumask_test_cpu(0, uv_nmi_cpu_mask)) 337 loop_delay *= 100; 338 339 udelay(loop_delay); 340 } 341 atomic_set(&uv_nmi_cpus_in_nmi, k); 342 return n - k; 343 } 344 345 /* Wait until all slave cpus have entered UV NMI handler */ 346 static void uv_nmi_wait(int master) 347 { 348 /* indicate this cpu is in */ 349 this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_IN); 350 351 /* if not the first cpu in (the master), then we are a slave cpu */ 352 if (!master) 353 return; 354 355 do { 356 /* wait for all other cpus to gather here */ 357 if (!uv_nmi_wait_cpus(1)) 358 break; 359 360 /* if not all made it in, send IPI NMI to them */ 361 pr_alert("UV: Sending NMI IPI to %d non-responding CPUs: %*pbl\n", 362 cpumask_weight(uv_nmi_cpu_mask), 363 cpumask_pr_args(uv_nmi_cpu_mask)); 364 365 uv_nmi_nr_cpus_ping(); 366 367 /* if all cpus are in, then done */ 368 if (!uv_nmi_wait_cpus(0)) 369 break; 370 371 pr_alert("UV: %d CPUs not in NMI loop: %*pbl\n", 372 cpumask_weight(uv_nmi_cpu_mask), 373 cpumask_pr_args(uv_nmi_cpu_mask)); 374 } while (0); 375 376 pr_alert("UV: %d of %d CPUs in NMI\n", 377 atomic_read(&uv_nmi_cpus_in_nmi), num_online_cpus()); 378 } 379 380 /* Dump Instruction Pointer header */ 381 static void uv_nmi_dump_cpu_ip_hdr(void) 382 { 383 pr_info("\nUV: %4s %6s %-32s %s (Note: PID 0 not listed)\n", 384 "CPU", "PID", "COMMAND", "IP"); 385 } 386 387 /* Dump Instruction Pointer info */ 388 static void uv_nmi_dump_cpu_ip(int cpu, struct pt_regs *regs) 389 { 390 pr_info("UV: %4d %6d %-32.32s ", cpu, current->pid, current->comm); 391 printk_address(regs->ip); 392 } 393 394 /* 395 * Dump this CPU's state. If action was set to "kdump" and the crash_kexec 396 * failed, then we provide "dump" as an alternate action. Action "dump" now 397 * also includes the show "ips" (instruction pointers) action whereas the 398 * action "ips" only displays instruction pointers for the non-idle CPU's. 399 * This is an abbreviated form of the "ps" command. 400 */ 401 static void uv_nmi_dump_state_cpu(int cpu, struct pt_regs *regs) 402 { 403 const char *dots = " ................................. "; 404 405 if (cpu == 0) 406 uv_nmi_dump_cpu_ip_hdr(); 407 408 if (current->pid != 0 || !uv_nmi_action_is("ips")) 409 uv_nmi_dump_cpu_ip(cpu, regs); 410 411 if (uv_nmi_action_is("dump")) { 412 pr_info("UV:%sNMI process trace for CPU %d\n", dots, cpu); 413 show_regs(regs); 414 } 415 416 this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_DUMP_DONE); 417 } 418 419 /* Trigger a slave cpu to dump it's state */ 420 static void uv_nmi_trigger_dump(int cpu) 421 { 422 int retry = uv_nmi_trigger_delay; 423 424 if (uv_cpu_nmi_per(cpu).state != UV_NMI_STATE_IN) 425 return; 426 427 uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_DUMP; 428 do { 429 cpu_relax(); 430 udelay(10); 431 if (uv_cpu_nmi_per(cpu).state 432 != UV_NMI_STATE_DUMP) 433 return; 434 } while (--retry > 0); 435 436 pr_crit("UV: CPU %d stuck in process dump function\n", cpu); 437 uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_DUMP_DONE; 438 } 439 440 /* Wait until all cpus ready to exit */ 441 static void uv_nmi_sync_exit(int master) 442 { 443 atomic_dec(&uv_nmi_cpus_in_nmi); 444 if (master) { 445 while (atomic_read(&uv_nmi_cpus_in_nmi) > 0) 446 cpu_relax(); 447 atomic_set(&uv_nmi_slave_continue, SLAVE_CLEAR); 448 } else { 449 while (atomic_read(&uv_nmi_slave_continue)) 450 cpu_relax(); 451 } 452 } 453 454 /* Walk through cpu list and dump state of each */ 455 static void uv_nmi_dump_state(int cpu, struct pt_regs *regs, int master) 456 { 457 if (master) { 458 int tcpu; 459 int ignored = 0; 460 int saved_console_loglevel = console_loglevel; 461 462 pr_alert("UV: tracing %s for %d CPUs from CPU %d\n", 463 uv_nmi_action_is("ips") ? "IPs" : "processes", 464 atomic_read(&uv_nmi_cpus_in_nmi), cpu); 465 466 console_loglevel = uv_nmi_loglevel; 467 atomic_set(&uv_nmi_slave_continue, SLAVE_EXIT); 468 for_each_online_cpu(tcpu) { 469 if (cpumask_test_cpu(tcpu, uv_nmi_cpu_mask)) 470 ignored++; 471 else if (tcpu == cpu) 472 uv_nmi_dump_state_cpu(tcpu, regs); 473 else 474 uv_nmi_trigger_dump(tcpu); 475 } 476 if (ignored) 477 pr_alert("UV: %d CPUs ignored NMI\n", ignored); 478 479 console_loglevel = saved_console_loglevel; 480 pr_alert("UV: process trace complete\n"); 481 } else { 482 while (!atomic_read(&uv_nmi_slave_continue)) 483 cpu_relax(); 484 while (this_cpu_read(uv_cpu_nmi.state) != UV_NMI_STATE_DUMP) 485 cpu_relax(); 486 uv_nmi_dump_state_cpu(cpu, regs); 487 } 488 uv_nmi_sync_exit(master); 489 } 490 491 static void uv_nmi_touch_watchdogs(void) 492 { 493 touch_softlockup_watchdog_sync(); 494 clocksource_touch_watchdog(); 495 rcu_cpu_stall_reset(); 496 touch_nmi_watchdog(); 497 } 498 499 static atomic_t uv_nmi_kexec_failed; 500 501 #if defined(CONFIG_KEXEC_CORE) 502 static void uv_nmi_kdump(int cpu, int master, struct pt_regs *regs) 503 { 504 /* Call crash to dump system state */ 505 if (master) { 506 pr_emerg("UV: NMI executing crash_kexec on CPU%d\n", cpu); 507 crash_kexec(regs); 508 509 pr_emerg("UV: crash_kexec unexpectedly returned, "); 510 atomic_set(&uv_nmi_kexec_failed, 1); 511 if (!kexec_crash_image) { 512 pr_cont("crash kernel not loaded\n"); 513 return; 514 } 515 pr_cont("kexec busy, stalling cpus while waiting\n"); 516 } 517 518 /* If crash exec fails the slaves should return, otherwise stall */ 519 while (atomic_read(&uv_nmi_kexec_failed) == 0) 520 mdelay(10); 521 } 522 523 #else /* !CONFIG_KEXEC_CORE */ 524 static inline void uv_nmi_kdump(int cpu, int master, struct pt_regs *regs) 525 { 526 if (master) 527 pr_err("UV: NMI kdump: KEXEC not supported in this kernel\n"); 528 atomic_set(&uv_nmi_kexec_failed, 1); 529 } 530 #endif /* !CONFIG_KEXEC_CORE */ 531 532 #ifdef CONFIG_KGDB 533 #ifdef CONFIG_KGDB_KDB 534 static inline int uv_nmi_kdb_reason(void) 535 { 536 return KDB_REASON_SYSTEM_NMI; 537 } 538 #else /* !CONFIG_KGDB_KDB */ 539 static inline int uv_nmi_kdb_reason(void) 540 { 541 /* Insure user is expecting to attach gdb remote */ 542 if (uv_nmi_action_is("kgdb")) 543 return 0; 544 545 pr_err("UV: NMI error: KDB is not enabled in this kernel\n"); 546 return -1; 547 } 548 #endif /* CONFIG_KGDB_KDB */ 549 550 /* 551 * Call KGDB/KDB from NMI handler 552 * 553 * Note that if both KGDB and KDB are configured, then the action of 'kgdb' or 554 * 'kdb' has no affect on which is used. See the KGDB documention for further 555 * information. 556 */ 557 static void uv_call_kgdb_kdb(int cpu, struct pt_regs *regs, int master) 558 { 559 if (master) { 560 int reason = uv_nmi_kdb_reason(); 561 int ret; 562 563 if (reason < 0) 564 return; 565 566 /* call KGDB NMI handler as MASTER */ 567 ret = kgdb_nmicallin(cpu, X86_TRAP_NMI, regs, reason, 568 &uv_nmi_slave_continue); 569 if (ret) { 570 pr_alert("KGDB returned error, is kgdboc set?\n"); 571 atomic_set(&uv_nmi_slave_continue, SLAVE_EXIT); 572 } 573 } else { 574 /* wait for KGDB signal that it's ready for slaves to enter */ 575 int sig; 576 577 do { 578 cpu_relax(); 579 sig = atomic_read(&uv_nmi_slave_continue); 580 } while (!sig); 581 582 /* call KGDB as slave */ 583 if (sig == SLAVE_CONTINUE) 584 kgdb_nmicallback(cpu, regs); 585 } 586 uv_nmi_sync_exit(master); 587 } 588 589 #else /* !CONFIG_KGDB */ 590 static inline void uv_call_kgdb_kdb(int cpu, struct pt_regs *regs, int master) 591 { 592 pr_err("UV: NMI error: KGDB is not enabled in this kernel\n"); 593 } 594 #endif /* !CONFIG_KGDB */ 595 596 /* 597 * UV NMI handler 598 */ 599 int uv_handle_nmi(unsigned int reason, struct pt_regs *regs) 600 { 601 struct uv_hub_nmi_s *hub_nmi = uv_hub_nmi; 602 int cpu = smp_processor_id(); 603 int master = 0; 604 unsigned long flags; 605 606 local_irq_save(flags); 607 608 /* If not a UV System NMI, ignore */ 609 if (!this_cpu_read(uv_cpu_nmi.pinging) && !uv_check_nmi(hub_nmi)) { 610 local_irq_restore(flags); 611 return NMI_DONE; 612 } 613 614 /* Indicate we are the first CPU into the NMI handler */ 615 master = (atomic_read(&uv_nmi_cpu) == cpu); 616 617 /* If NMI action is "kdump", then attempt to do it */ 618 if (uv_nmi_action_is("kdump")) { 619 uv_nmi_kdump(cpu, master, regs); 620 621 /* Unexpected return, revert action to "dump" */ 622 if (master) 623 strncpy(uv_nmi_action, "dump", strlen(uv_nmi_action)); 624 } 625 626 /* Pause as all cpus enter the NMI handler */ 627 uv_nmi_wait(master); 628 629 /* Dump state of each cpu */ 630 if (uv_nmi_action_is("ips") || uv_nmi_action_is("dump")) 631 uv_nmi_dump_state(cpu, regs, master); 632 633 /* Call KGDB/KDB if enabled */ 634 else if (uv_nmi_action_is("kdb") || uv_nmi_action_is("kgdb")) 635 uv_call_kgdb_kdb(cpu, regs, master); 636 637 /* Clear per_cpu "in nmi" flag */ 638 this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_OUT); 639 640 /* Clear MMR NMI flag on each hub */ 641 uv_clear_nmi(cpu); 642 643 /* Clear global flags */ 644 if (master) { 645 if (cpumask_weight(uv_nmi_cpu_mask)) 646 uv_nmi_cleanup_mask(); 647 atomic_set(&uv_nmi_cpus_in_nmi, -1); 648 atomic_set(&uv_nmi_cpu, -1); 649 atomic_set(&uv_in_nmi, 0); 650 atomic_set(&uv_nmi_kexec_failed, 0); 651 } 652 653 uv_nmi_touch_watchdogs(); 654 local_irq_restore(flags); 655 656 return NMI_HANDLED; 657 } 658 659 /* 660 * NMI handler for pulling in CPUs when perf events are grabbing our NMI 661 */ 662 static int uv_handle_nmi_ping(unsigned int reason, struct pt_regs *regs) 663 { 664 int ret; 665 666 this_cpu_inc(uv_cpu_nmi.queries); 667 if (!this_cpu_read(uv_cpu_nmi.pinging)) { 668 local64_inc(&uv_nmi_ping_misses); 669 return NMI_DONE; 670 } 671 672 this_cpu_inc(uv_cpu_nmi.pings); 673 local64_inc(&uv_nmi_ping_count); 674 ret = uv_handle_nmi(reason, regs); 675 this_cpu_write(uv_cpu_nmi.pinging, 0); 676 return ret; 677 } 678 679 static void uv_register_nmi_notifier(void) 680 { 681 if (register_nmi_handler(NMI_UNKNOWN, uv_handle_nmi, 0, "uv")) 682 pr_warn("UV: NMI handler failed to register\n"); 683 684 if (register_nmi_handler(NMI_LOCAL, uv_handle_nmi_ping, 0, "uvping")) 685 pr_warn("UV: PING NMI handler failed to register\n"); 686 } 687 688 void uv_nmi_init(void) 689 { 690 unsigned int value; 691 692 /* 693 * Unmask NMI on all cpus 694 */ 695 value = apic_read(APIC_LVT1) | APIC_DM_NMI; 696 value &= ~APIC_LVT_MASKED; 697 apic_write(APIC_LVT1, value); 698 } 699 700 void uv_nmi_setup(void) 701 { 702 int size = sizeof(void *) * (1 << NODES_SHIFT); 703 int cpu, nid; 704 705 /* Setup hub nmi info */ 706 uv_nmi_setup_mmrs(); 707 uv_hub_nmi_list = kzalloc(size, GFP_KERNEL); 708 pr_info("UV: NMI hub list @ 0x%p (%d)\n", uv_hub_nmi_list, size); 709 BUG_ON(!uv_hub_nmi_list); 710 size = sizeof(struct uv_hub_nmi_s); 711 for_each_present_cpu(cpu) { 712 nid = cpu_to_node(cpu); 713 if (uv_hub_nmi_list[nid] == NULL) { 714 uv_hub_nmi_list[nid] = kzalloc_node(size, 715 GFP_KERNEL, nid); 716 BUG_ON(!uv_hub_nmi_list[nid]); 717 raw_spin_lock_init(&(uv_hub_nmi_list[nid]->nmi_lock)); 718 atomic_set(&uv_hub_nmi_list[nid]->cpu_owner, -1); 719 } 720 uv_hub_nmi_per(cpu) = uv_hub_nmi_list[nid]; 721 } 722 BUG_ON(!alloc_cpumask_var(&uv_nmi_cpu_mask, GFP_KERNEL)); 723 uv_register_nmi_notifier(); 724 } 725