1 /* 2 * SMP initialisation and IPI support 3 * Based on arch/arm64/kernel/smp.c 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 * Copyright (C) 2015 Regents of the University of California 7 * Copyright (C) 2017 SiFive 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License 19 * along with this program. If not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include <linux/interrupt.h> 23 #include <linux/smp.h> 24 #include <linux/sched.h> 25 26 #include <asm/sbi.h> 27 #include <asm/tlbflush.h> 28 #include <asm/cacheflush.h> 29 30 /* A collection of single bit ipi messages. */ 31 static struct { 32 unsigned long bits ____cacheline_aligned; 33 } ipi_data[NR_CPUS] __cacheline_aligned; 34 35 enum ipi_message_type { 36 IPI_RESCHEDULE, 37 IPI_CALL_FUNC, 38 IPI_MAX 39 }; 40 41 42 /* Unsupported */ 43 int setup_profiling_timer(unsigned int multiplier) 44 { 45 return -EINVAL; 46 } 47 48 irqreturn_t handle_ipi(void) 49 { 50 unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; 51 52 /* Clear pending IPI */ 53 csr_clear(sip, SIE_SSIE); 54 55 while (true) { 56 unsigned long ops; 57 58 /* Order bit clearing and data access. */ 59 mb(); 60 61 ops = xchg(pending_ipis, 0); 62 if (ops == 0) 63 return IRQ_HANDLED; 64 65 if (ops & (1 << IPI_RESCHEDULE)) 66 scheduler_ipi(); 67 68 if (ops & (1 << IPI_CALL_FUNC)) 69 generic_smp_call_function_interrupt(); 70 71 BUG_ON((ops >> IPI_MAX) != 0); 72 73 /* Order data access and bit testing. */ 74 mb(); 75 } 76 77 return IRQ_HANDLED; 78 } 79 80 static void 81 send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation) 82 { 83 int i; 84 85 mb(); 86 for_each_cpu(i, to_whom) 87 set_bit(operation, &ipi_data[i].bits); 88 89 mb(); 90 sbi_send_ipi(cpumask_bits(to_whom)); 91 } 92 93 void arch_send_call_function_ipi_mask(struct cpumask *mask) 94 { 95 send_ipi_message(mask, IPI_CALL_FUNC); 96 } 97 98 void arch_send_call_function_single_ipi(int cpu) 99 { 100 send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); 101 } 102 103 static void ipi_stop(void *unused) 104 { 105 while (1) 106 wait_for_interrupt(); 107 } 108 109 void smp_send_stop(void) 110 { 111 on_each_cpu(ipi_stop, NULL, 1); 112 } 113 114 void smp_send_reschedule(int cpu) 115 { 116 send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); 117 } 118 119 /* 120 * Performs an icache flush for the given MM context. RISC-V has no direct 121 * mechanism for instruction cache shoot downs, so instead we send an IPI that 122 * informs the remote harts they need to flush their local instruction caches. 123 * To avoid pathologically slow behavior in a common case (a bunch of 124 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the 125 * IPIs for harts that are not currently executing a MM context and instead 126 * schedule a deferred local instruction cache flush to be performed before 127 * execution resumes on each hart. 128 */ 129 void flush_icache_mm(struct mm_struct *mm, bool local) 130 { 131 unsigned int cpu; 132 cpumask_t others, *mask; 133 134 preempt_disable(); 135 136 /* Mark every hart's icache as needing a flush for this MM. */ 137 mask = &mm->context.icache_stale_mask; 138 cpumask_setall(mask); 139 /* Flush this hart's I$ now, and mark it as flushed. */ 140 cpu = smp_processor_id(); 141 cpumask_clear_cpu(cpu, mask); 142 local_flush_icache_all(); 143 144 /* 145 * Flush the I$ of other harts concurrently executing, and mark them as 146 * flushed. 147 */ 148 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); 149 local |= cpumask_empty(&others); 150 if (mm != current->active_mm || !local) 151 sbi_remote_fence_i(others.bits); 152 else { 153 /* 154 * It's assumed that at least one strongly ordered operation is 155 * performed on this hart between setting a hart's cpumask bit 156 * and scheduling this MM context on that hart. Sending an SBI 157 * remote message will do this, but in the case where no 158 * messages are sent we still need to order this hart's writes 159 * with flush_icache_deferred(). 160 */ 161 smp_mb(); 162 } 163 164 preempt_enable(); 165 } 166