1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 2000, 2001 Kanoj Sarcar 7 * Copyright (C) 2000, 2001 Ralf Baechle 8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc. 9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation 10 */ 11 #include <linux/cpu.h> 12 #include <linux/cpumask.h> 13 #include <linux/init.h> 14 #include <linux/interrupt.h> 15 #include <linux/seq_file.h> 16 #include <linux/smp.h> 17 #include <linux/threads.h> 18 #include <linux/export.h> 19 #include <linux/time.h> 20 #include <linux/tracepoint.h> 21 #include <linux/sched/hotplug.h> 22 #include <linux/sched/task_stack.h> 23 24 #include <asm/cpu.h> 25 #include <asm/idle.h> 26 #include <asm/loongson.h> 27 #include <asm/mmu_context.h> 28 #include <asm/numa.h> 29 #include <asm/processor.h> 30 #include <asm/setup.h> 31 #include <asm/time.h> 32 33 int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ 34 EXPORT_SYMBOL(__cpu_number_map); 35 36 int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ 37 EXPORT_SYMBOL(__cpu_logical_map); 38 39 /* Number of threads (siblings) per CPU core */ 40 int smp_num_siblings = 1; 41 EXPORT_SYMBOL(smp_num_siblings); 42 43 /* Representing the threads (siblings) of each logical CPU */ 44 cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; 45 EXPORT_SYMBOL(cpu_sibling_map); 46 47 /* Representing the core map of multi-core chips of each logical CPU */ 48 cpumask_t cpu_core_map[NR_CPUS] __read_mostly; 49 EXPORT_SYMBOL(cpu_core_map); 50 51 static DECLARE_COMPLETION(cpu_starting); 52 static DECLARE_COMPLETION(cpu_running); 53 54 /* 55 * A logcal cpu mask containing only one VPE per core to 56 * reduce the number of IPIs on large MT systems. 57 */ 58 cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; 59 EXPORT_SYMBOL(cpu_foreign_map); 60 61 /* representing cpus for which sibling maps can be computed */ 62 static cpumask_t cpu_sibling_setup_map; 63 64 /* representing cpus for which core maps can be computed */ 65 static cpumask_t cpu_core_setup_map; 66 67 struct secondary_data cpuboot_data; 68 static DEFINE_PER_CPU(int, cpu_state); 69 70 enum ipi_msg_type { 71 IPI_RESCHEDULE, 72 IPI_CALL_FUNCTION, 73 }; 74 75 static const char *ipi_types[NR_IPI] __tracepoint_string = { 76 [IPI_RESCHEDULE] = "Rescheduling interrupts", 77 [IPI_CALL_FUNCTION] = "Function call interrupts", 78 }; 79 80 void show_ipi_list(struct seq_file *p, int prec) 81 { 82 unsigned int cpu, i; 83 84 for (i = 0; i < NR_IPI; i++) { 85 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); 86 for_each_online_cpu(cpu) 87 seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).ipi_irqs[i]); 88 seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); 89 } 90 } 91 92 /* Send mailbox buffer via Mail_Send */ 93 static void csr_mail_send(uint64_t data, int cpu, int mailbox) 94 { 95 uint64_t val; 96 97 /* Send high 32 bits */ 98 val = IOCSR_MBUF_SEND_BLOCKING; 99 val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 100 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 101 val |= (data & IOCSR_MBUF_SEND_H32_MASK); 102 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 103 104 /* Send low 32 bits */ 105 val = IOCSR_MBUF_SEND_BLOCKING; 106 val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 107 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 108 val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); 109 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 110 }; 111 112 static u32 ipi_read_clear(int cpu) 113 { 114 u32 action; 115 116 /* Load the ipi register to figure out what we're supposed to do */ 117 action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); 118 /* Clear the ipi register to clear the interrupt */ 119 iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); 120 smp_mb(); 121 122 return action; 123 } 124 125 static void ipi_write_action(int cpu, u32 action) 126 { 127 unsigned int irq = 0; 128 129 while ((irq = ffs(action))) { 130 uint32_t val = IOCSR_IPI_SEND_BLOCKING; 131 132 val |= (irq - 1); 133 val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); 134 iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); 135 action &= ~BIT(irq - 1); 136 } 137 } 138 139 void loongson3_send_ipi_single(int cpu, unsigned int action) 140 { 141 ipi_write_action(cpu_logical_map(cpu), (u32)action); 142 } 143 144 void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action) 145 { 146 unsigned int i; 147 148 for_each_cpu(i, mask) 149 ipi_write_action(cpu_logical_map(i), (u32)action); 150 } 151 152 irqreturn_t loongson3_ipi_interrupt(int irq, void *dev) 153 { 154 unsigned int action; 155 unsigned int cpu = smp_processor_id(); 156 157 action = ipi_read_clear(cpu_logical_map(cpu)); 158 159 if (action & SMP_RESCHEDULE) { 160 scheduler_ipi(); 161 per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; 162 } 163 164 if (action & SMP_CALL_FUNCTION) { 165 generic_smp_call_function_interrupt(); 166 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; 167 } 168 169 return IRQ_HANDLED; 170 } 171 172 void __init loongson3_smp_setup(void) 173 { 174 cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; 175 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 176 177 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 178 pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); 179 } 180 181 void __init loongson3_prepare_cpus(unsigned int max_cpus) 182 { 183 int i = 0; 184 185 for (i = 0; i < loongson_sysconf.nr_cpus; i++) { 186 set_cpu_present(i, true); 187 csr_mail_send(0, __cpu_logical_map[i], 0); 188 } 189 190 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 191 } 192 193 /* 194 * Setup the PC, SP, and TP of a secondary processor and start it running! 195 */ 196 void loongson3_boot_secondary(int cpu, struct task_struct *idle) 197 { 198 unsigned long entry; 199 200 pr_info("Booting CPU#%d...\n", cpu); 201 202 entry = __pa_symbol((unsigned long)&smpboot_entry); 203 cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle); 204 cpuboot_data.thread_info = (unsigned long)task_thread_info(idle); 205 206 csr_mail_send(entry, cpu_logical_map(cpu), 0); 207 208 loongson3_send_ipi_single(cpu, SMP_BOOT_CPU); 209 } 210 211 /* 212 * SMP init and finish on secondary CPUs 213 */ 214 void loongson3_init_secondary(void) 215 { 216 unsigned int cpu = smp_processor_id(); 217 unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | 218 ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER; 219 220 change_csr_ecfg(ECFG0_IM, imask); 221 222 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 223 224 #ifdef CONFIG_NUMA 225 numa_add_cpu(cpu); 226 #endif 227 per_cpu(cpu_state, cpu) = CPU_ONLINE; 228 cpu_data[cpu].core = 229 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; 230 cpu_data[cpu].package = 231 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 232 } 233 234 void loongson3_smp_finish(void) 235 { 236 local_irq_enable(); 237 iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); 238 pr_info("CPU#%d finished\n", smp_processor_id()); 239 } 240 241 #ifdef CONFIG_HOTPLUG_CPU 242 243 static bool io_master(int cpu) 244 { 245 if (cpu == 0) 246 return true; 247 248 return false; 249 } 250 251 int loongson3_cpu_disable(void) 252 { 253 unsigned long flags; 254 unsigned int cpu = smp_processor_id(); 255 256 if (io_master(cpu)) 257 return -EBUSY; 258 259 #ifdef CONFIG_NUMA 260 numa_remove_cpu(cpu); 261 #endif 262 set_cpu_online(cpu, false); 263 calculate_cpu_foreign_map(); 264 local_irq_save(flags); 265 irq_migrate_all_off_this_cpu(); 266 clear_csr_ecfg(ECFG0_IM); 267 local_irq_restore(flags); 268 local_flush_tlb_all(); 269 270 return 0; 271 } 272 273 void loongson3_cpu_die(unsigned int cpu) 274 { 275 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 276 cpu_relax(); 277 278 mb(); 279 } 280 281 /* 282 * The target CPU should go to XKPRANGE (uncached area) and flush 283 * ICache/DCache/VCache before the control CPU can safely disable its clock. 284 */ 285 static void loongson3_play_dead(int *state_addr) 286 { 287 register int val; 288 register void *addr; 289 register void (*init_fn)(void); 290 291 __asm__ __volatile__( 292 " li.d %[addr], 0x8000000000000000\n" 293 "1: cacop 0x8, %[addr], 0 \n" /* flush ICache */ 294 " cacop 0x8, %[addr], 1 \n" 295 " cacop 0x8, %[addr], 2 \n" 296 " cacop 0x8, %[addr], 3 \n" 297 " cacop 0x9, %[addr], 0 \n" /* flush DCache */ 298 " cacop 0x9, %[addr], 1 \n" 299 " cacop 0x9, %[addr], 2 \n" 300 " cacop 0x9, %[addr], 3 \n" 301 " addi.w %[sets], %[sets], -1 \n" 302 " addi.d %[addr], %[addr], 0x40 \n" 303 " bnez %[sets], 1b \n" 304 " li.d %[addr], 0x8000000000000000\n" 305 "2: cacop 0xa, %[addr], 0 \n" /* flush VCache */ 306 " cacop 0xa, %[addr], 1 \n" 307 " cacop 0xa, %[addr], 2 \n" 308 " cacop 0xa, %[addr], 3 \n" 309 " cacop 0xa, %[addr], 4 \n" 310 " cacop 0xa, %[addr], 5 \n" 311 " cacop 0xa, %[addr], 6 \n" 312 " cacop 0xa, %[addr], 7 \n" 313 " cacop 0xa, %[addr], 8 \n" 314 " cacop 0xa, %[addr], 9 \n" 315 " cacop 0xa, %[addr], 10 \n" 316 " cacop 0xa, %[addr], 11 \n" 317 " cacop 0xa, %[addr], 12 \n" 318 " cacop 0xa, %[addr], 13 \n" 319 " cacop 0xa, %[addr], 14 \n" 320 " cacop 0xa, %[addr], 15 \n" 321 " addi.w %[vsets], %[vsets], -1 \n" 322 " addi.d %[addr], %[addr], 0x40 \n" 323 " bnez %[vsets], 2b \n" 324 " li.w %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ 325 " st.w %[val], %[state_addr], 0 \n" 326 " dbar 0 \n" 327 " cacop 0x11, %[state_addr], 0 \n" /* flush entry of *state_addr */ 328 : [addr] "=&r" (addr), [val] "=&r" (val) 329 : [state_addr] "r" (state_addr), 330 [sets] "r" (cpu_data[smp_processor_id()].dcache.sets), 331 [vsets] "r" (cpu_data[smp_processor_id()].vcache.sets)); 332 333 local_irq_enable(); 334 change_csr_ecfg(ECFG0_IM, ECFGF_IPI); 335 336 __asm__ __volatile__( 337 " idle 0 \n" 338 " li.w $t0, 0x1020 \n" 339 " iocsrrd.d %[init_fn], $t0 \n" /* Get init PC */ 340 : [init_fn] "=&r" (addr) 341 : /* No Input */ 342 : "a0"); 343 init_fn = __va(addr); 344 345 init_fn(); 346 unreachable(); 347 } 348 349 void play_dead(void) 350 { 351 int *state_addr; 352 unsigned int cpu = smp_processor_id(); 353 void (*play_dead_uncached)(int *s); 354 355 idle_task_exit(); 356 play_dead_uncached = (void *)TO_UNCACHE(__pa((unsigned long)loongson3_play_dead)); 357 state_addr = &per_cpu(cpu_state, cpu); 358 mb(); 359 play_dead_uncached(state_addr); 360 } 361 362 static int loongson3_enable_clock(unsigned int cpu) 363 { 364 uint64_t core_id = cpu_data[cpu].core; 365 uint64_t package_id = cpu_data[cpu].package; 366 367 LOONGSON_FREQCTRL(package_id) |= 1 << (core_id * 4 + 3); 368 369 return 0; 370 } 371 372 static int loongson3_disable_clock(unsigned int cpu) 373 { 374 uint64_t core_id = cpu_data[cpu].core; 375 uint64_t package_id = cpu_data[cpu].package; 376 377 LOONGSON_FREQCTRL(package_id) &= ~(1 << (core_id * 4 + 3)); 378 379 return 0; 380 } 381 382 static int register_loongson3_notifier(void) 383 { 384 return cpuhp_setup_state_nocalls(CPUHP_LOONGARCH_SOC_PREPARE, 385 "loongarch/loongson:prepare", 386 loongson3_enable_clock, 387 loongson3_disable_clock); 388 } 389 early_initcall(register_loongson3_notifier); 390 391 #endif 392 393 /* 394 * Power management 395 */ 396 #ifdef CONFIG_PM 397 398 static int loongson3_ipi_suspend(void) 399 { 400 return 0; 401 } 402 403 static void loongson3_ipi_resume(void) 404 { 405 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 406 } 407 408 static struct syscore_ops loongson3_ipi_syscore_ops = { 409 .resume = loongson3_ipi_resume, 410 .suspend = loongson3_ipi_suspend, 411 }; 412 413 /* 414 * Enable boot cpu ipi before enabling nonboot cpus 415 * during syscore_resume. 416 */ 417 static int __init ipi_pm_init(void) 418 { 419 register_syscore_ops(&loongson3_ipi_syscore_ops); 420 return 0; 421 } 422 423 core_initcall(ipi_pm_init); 424 #endif 425 426 static inline void set_cpu_sibling_map(int cpu) 427 { 428 int i; 429 430 cpumask_set_cpu(cpu, &cpu_sibling_setup_map); 431 432 if (smp_num_siblings <= 1) 433 cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]); 434 else { 435 for_each_cpu(i, &cpu_sibling_setup_map) { 436 if (cpus_are_siblings(cpu, i)) { 437 cpumask_set_cpu(i, &cpu_sibling_map[cpu]); 438 cpumask_set_cpu(cpu, &cpu_sibling_map[i]); 439 } 440 } 441 } 442 } 443 444 static inline void set_cpu_core_map(int cpu) 445 { 446 int i; 447 448 cpumask_set_cpu(cpu, &cpu_core_setup_map); 449 450 for_each_cpu(i, &cpu_core_setup_map) { 451 if (cpu_data[cpu].package == cpu_data[i].package) { 452 cpumask_set_cpu(i, &cpu_core_map[cpu]); 453 cpumask_set_cpu(cpu, &cpu_core_map[i]); 454 } 455 } 456 } 457 458 /* 459 * Calculate a new cpu_foreign_map mask whenever a 460 * new cpu appears or disappears. 461 */ 462 void calculate_cpu_foreign_map(void) 463 { 464 int i, k, core_present; 465 cpumask_t temp_foreign_map; 466 467 /* Re-calculate the mask */ 468 cpumask_clear(&temp_foreign_map); 469 for_each_online_cpu(i) { 470 core_present = 0; 471 for_each_cpu(k, &temp_foreign_map) 472 if (cpus_are_siblings(i, k)) 473 core_present = 1; 474 if (!core_present) 475 cpumask_set_cpu(i, &temp_foreign_map); 476 } 477 478 for_each_online_cpu(i) 479 cpumask_andnot(&cpu_foreign_map[i], 480 &temp_foreign_map, &cpu_sibling_map[i]); 481 } 482 483 /* Preload SMP state for boot cpu */ 484 void smp_prepare_boot_cpu(void) 485 { 486 unsigned int cpu, node, rr_node; 487 488 set_cpu_possible(0, true); 489 set_cpu_online(0, true); 490 set_my_cpu_offset(per_cpu_offset(0)); 491 492 rr_node = first_node(node_online_map); 493 for_each_possible_cpu(cpu) { 494 node = early_cpu_to_node(cpu); 495 496 /* 497 * The mapping between present cpus and nodes has been 498 * built during MADT and SRAT parsing. 499 * 500 * If possible cpus = present cpus here, early_cpu_to_node 501 * will return valid node. 502 * 503 * If possible cpus > present cpus here (e.g. some possible 504 * cpus will be added by cpu-hotplug later), for possible but 505 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE, 506 * and we just map them to online nodes in round-robin way. 507 * Once hotplugged, new correct mapping will be built for them. 508 */ 509 if (node != NUMA_NO_NODE) 510 set_cpu_numa_node(cpu, node); 511 else { 512 set_cpu_numa_node(cpu, rr_node); 513 rr_node = next_node_in(rr_node, node_online_map); 514 } 515 } 516 } 517 518 /* called from main before smp_init() */ 519 void __init smp_prepare_cpus(unsigned int max_cpus) 520 { 521 init_new_context(current, &init_mm); 522 current_thread_info()->cpu = 0; 523 loongson3_prepare_cpus(max_cpus); 524 set_cpu_sibling_map(0); 525 set_cpu_core_map(0); 526 calculate_cpu_foreign_map(); 527 #ifndef CONFIG_HOTPLUG_CPU 528 init_cpu_present(cpu_possible_mask); 529 #endif 530 } 531 532 int __cpu_up(unsigned int cpu, struct task_struct *tidle) 533 { 534 loongson3_boot_secondary(cpu, tidle); 535 536 /* Wait for CPU to start and be ready to sync counters */ 537 if (!wait_for_completion_timeout(&cpu_starting, 538 msecs_to_jiffies(5000))) { 539 pr_crit("CPU%u: failed to start\n", cpu); 540 return -EIO; 541 } 542 543 /* Wait for CPU to finish startup & mark itself online before return */ 544 wait_for_completion(&cpu_running); 545 546 return 0; 547 } 548 549 /* 550 * First C code run on the secondary CPUs after being started up by 551 * the master. 552 */ 553 asmlinkage void start_secondary(void) 554 { 555 unsigned int cpu; 556 557 sync_counter(); 558 cpu = smp_processor_id(); 559 set_my_cpu_offset(per_cpu_offset(cpu)); 560 561 cpu_probe(); 562 constant_clockevent_init(); 563 loongson3_init_secondary(); 564 565 set_cpu_sibling_map(cpu); 566 set_cpu_core_map(cpu); 567 568 notify_cpu_starting(cpu); 569 570 /* Notify boot CPU that we're starting */ 571 complete(&cpu_starting); 572 573 /* The CPU is running, now mark it online */ 574 set_cpu_online(cpu, true); 575 576 calculate_cpu_foreign_map(); 577 578 /* 579 * Notify boot CPU that we're up & online and it can safely return 580 * from __cpu_up() 581 */ 582 complete(&cpu_running); 583 584 /* 585 * irq will be enabled in loongson3_smp_finish(), enabling it too 586 * early is dangerous. 587 */ 588 WARN_ON_ONCE(!irqs_disabled()); 589 loongson3_smp_finish(); 590 591 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 592 } 593 594 void __init smp_cpus_done(unsigned int max_cpus) 595 { 596 } 597 598 static void stop_this_cpu(void *dummy) 599 { 600 set_cpu_online(smp_processor_id(), false); 601 calculate_cpu_foreign_map(); 602 local_irq_disable(); 603 while (true); 604 } 605 606 void smp_send_stop(void) 607 { 608 smp_call_function(stop_this_cpu, NULL, 0); 609 } 610 611 int setup_profiling_timer(unsigned int multiplier) 612 { 613 return 0; 614 } 615 616 static void flush_tlb_all_ipi(void *info) 617 { 618 local_flush_tlb_all(); 619 } 620 621 void flush_tlb_all(void) 622 { 623 on_each_cpu(flush_tlb_all_ipi, NULL, 1); 624 } 625 626 static void flush_tlb_mm_ipi(void *mm) 627 { 628 local_flush_tlb_mm((struct mm_struct *)mm); 629 } 630 631 void flush_tlb_mm(struct mm_struct *mm) 632 { 633 if (atomic_read(&mm->mm_users) == 0) 634 return; /* happens as a result of exit_mmap() */ 635 636 preempt_disable(); 637 638 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 639 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); 640 } else { 641 unsigned int cpu; 642 643 for_each_online_cpu(cpu) { 644 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 645 cpu_context(cpu, mm) = 0; 646 } 647 local_flush_tlb_mm(mm); 648 } 649 650 preempt_enable(); 651 } 652 653 struct flush_tlb_data { 654 struct vm_area_struct *vma; 655 unsigned long addr1; 656 unsigned long addr2; 657 }; 658 659 static void flush_tlb_range_ipi(void *info) 660 { 661 struct flush_tlb_data *fd = info; 662 663 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); 664 } 665 666 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 667 { 668 struct mm_struct *mm = vma->vm_mm; 669 670 preempt_disable(); 671 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 672 struct flush_tlb_data fd = { 673 .vma = vma, 674 .addr1 = start, 675 .addr2 = end, 676 }; 677 678 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); 679 } else { 680 unsigned int cpu; 681 682 for_each_online_cpu(cpu) { 683 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 684 cpu_context(cpu, mm) = 0; 685 } 686 local_flush_tlb_range(vma, start, end); 687 } 688 preempt_enable(); 689 } 690 691 static void flush_tlb_kernel_range_ipi(void *info) 692 { 693 struct flush_tlb_data *fd = info; 694 695 local_flush_tlb_kernel_range(fd->addr1, fd->addr2); 696 } 697 698 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 699 { 700 struct flush_tlb_data fd = { 701 .addr1 = start, 702 .addr2 = end, 703 }; 704 705 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); 706 } 707 708 static void flush_tlb_page_ipi(void *info) 709 { 710 struct flush_tlb_data *fd = info; 711 712 local_flush_tlb_page(fd->vma, fd->addr1); 713 } 714 715 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 716 { 717 preempt_disable(); 718 if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { 719 struct flush_tlb_data fd = { 720 .vma = vma, 721 .addr1 = page, 722 }; 723 724 on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); 725 } else { 726 unsigned int cpu; 727 728 for_each_online_cpu(cpu) { 729 if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) 730 cpu_context(cpu, vma->vm_mm) = 0; 731 } 732 local_flush_tlb_page(vma, page); 733 } 734 preempt_enable(); 735 } 736 EXPORT_SYMBOL(flush_tlb_page); 737 738 static void flush_tlb_one_ipi(void *info) 739 { 740 unsigned long vaddr = (unsigned long) info; 741 742 local_flush_tlb_one(vaddr); 743 } 744 745 void flush_tlb_one(unsigned long vaddr) 746 { 747 on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1); 748 } 749 EXPORT_SYMBOL(flush_tlb_one); 750