1 /* 2 * Copyright 2012 Michael Ellerman, IBM Corporation. 3 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License, version 2, as 7 * published by the Free Software Foundation. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/kvm_host.h> 12 #include <linux/err.h> 13 14 #include <asm/kvm_book3s.h> 15 #include <asm/kvm_ppc.h> 16 #include <asm/hvcall.h> 17 #include <asm/xics.h> 18 #include <asm/debug.h> 19 #include <asm/synch.h> 20 #include <asm/ppc-opcode.h> 21 22 #include "book3s_xics.h" 23 24 #define DEBUG_PASSUP 25 26 static inline void rm_writeb(unsigned long paddr, u8 val) 27 { 28 __asm__ __volatile__("sync; stbcix %0,0,%1" 29 : : "r" (val), "r" (paddr) : "memory"); 30 } 31 32 static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, 33 struct kvm_vcpu *this_vcpu) 34 { 35 struct kvmppc_icp *this_icp = this_vcpu->arch.icp; 36 unsigned long xics_phys; 37 int cpu; 38 39 /* Mark the target VCPU as having an interrupt pending */ 40 vcpu->stat.queue_intr++; 41 set_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions); 42 43 /* Kick self ? Just set MER and return */ 44 if (vcpu == this_vcpu) { 45 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_MER); 46 return; 47 } 48 49 /* Check if the core is loaded, if not, too hard */ 50 cpu = vcpu->cpu; 51 if (cpu < 0 || cpu >= nr_cpu_ids) { 52 this_icp->rm_action |= XICS_RM_KICK_VCPU; 53 this_icp->rm_kick_target = vcpu; 54 return; 55 } 56 /* In SMT cpu will always point to thread 0, we adjust it */ 57 cpu += vcpu->arch.ptid; 58 59 /* Not too hard, then poke the target */ 60 xics_phys = paca[cpu].kvm_hstate.xics_phys; 61 rm_writeb(xics_phys + XICS_MFRR, IPI_PRIORITY); 62 } 63 64 static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu) 65 { 66 /* Note: Only called on self ! */ 67 clear_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 68 &vcpu->arch.pending_exceptions); 69 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_MER); 70 } 71 72 static inline bool icp_rm_try_update(struct kvmppc_icp *icp, 73 union kvmppc_icp_state old, 74 union kvmppc_icp_state new) 75 { 76 struct kvm_vcpu *this_vcpu = local_paca->kvm_hstate.kvm_vcpu; 77 bool success; 78 79 /* Calculate new output value */ 80 new.out_ee = (new.xisr && (new.pending_pri < new.cppr)); 81 82 /* Attempt atomic update */ 83 success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw; 84 if (!success) 85 goto bail; 86 87 /* 88 * Check for output state update 89 * 90 * Note that this is racy since another processor could be updating 91 * the state already. This is why we never clear the interrupt output 92 * here, we only ever set it. The clear only happens prior to doing 93 * an update and only by the processor itself. Currently we do it 94 * in Accept (H_XIRR) and Up_Cppr (H_XPPR). 95 * 96 * We also do not try to figure out whether the EE state has changed, 97 * we unconditionally set it if the new state calls for it. The reason 98 * for that is that we opportunistically remove the pending interrupt 99 * flag when raising CPPR, so we need to set it back here if an 100 * interrupt is still pending. 101 */ 102 if (new.out_ee) 103 icp_rm_set_vcpu_irq(icp->vcpu, this_vcpu); 104 105 /* Expose the state change for debug purposes */ 106 this_vcpu->arch.icp->rm_dbgstate = new; 107 this_vcpu->arch.icp->rm_dbgtgt = icp->vcpu; 108 109 bail: 110 return success; 111 } 112 113 static inline int check_too_hard(struct kvmppc_xics *xics, 114 struct kvmppc_icp *icp) 115 { 116 return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS; 117 } 118 119 static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 120 u8 new_cppr) 121 { 122 union kvmppc_icp_state old_state, new_state; 123 bool resend; 124 125 /* 126 * This handles several related states in one operation: 127 * 128 * ICP State: Down_CPPR 129 * 130 * Load CPPR with new value and if the XISR is 0 131 * then check for resends: 132 * 133 * ICP State: Resend 134 * 135 * If MFRR is more favored than CPPR, check for IPIs 136 * and notify ICS of a potential resend. This is done 137 * asynchronously (when used in real mode, we will have 138 * to exit here). 139 * 140 * We do not handle the complete Check_IPI as documented 141 * here. In the PAPR, this state will be used for both 142 * Set_MFRR and Down_CPPR. However, we know that we aren't 143 * changing the MFRR state here so we don't need to handle 144 * the case of an MFRR causing a reject of a pending irq, 145 * this will have been handled when the MFRR was set in the 146 * first place. 147 * 148 * Thus we don't have to handle rejects, only resends. 149 * 150 * When implementing real mode for HV KVM, resend will lead to 151 * a H_TOO_HARD return and the whole transaction will be handled 152 * in virtual mode. 153 */ 154 do { 155 old_state = new_state = ACCESS_ONCE(icp->state); 156 157 /* Down_CPPR */ 158 new_state.cppr = new_cppr; 159 160 /* 161 * Cut down Resend / Check_IPI / IPI 162 * 163 * The logic is that we cannot have a pending interrupt 164 * trumped by an IPI at this point (see above), so we 165 * know that either the pending interrupt is already an 166 * IPI (in which case we don't care to override it) or 167 * it's either more favored than us or non existent 168 */ 169 if (new_state.mfrr < new_cppr && 170 new_state.mfrr <= new_state.pending_pri) { 171 new_state.pending_pri = new_state.mfrr; 172 new_state.xisr = XICS_IPI; 173 } 174 175 /* Latch/clear resend bit */ 176 resend = new_state.need_resend; 177 new_state.need_resend = 0; 178 179 } while (!icp_rm_try_update(icp, old_state, new_state)); 180 181 /* 182 * Now handle resend checks. Those are asynchronous to the ICP 183 * state update in HW (ie bus transactions) so we can handle them 184 * separately here as well. 185 */ 186 if (resend) 187 icp->rm_action |= XICS_RM_CHECK_RESEND; 188 } 189 190 191 unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu) 192 { 193 union kvmppc_icp_state old_state, new_state; 194 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 195 struct kvmppc_icp *icp = vcpu->arch.icp; 196 u32 xirr; 197 198 if (!xics || !xics->real_mode) 199 return H_TOO_HARD; 200 201 /* First clear the interrupt */ 202 icp_rm_clr_vcpu_irq(icp->vcpu); 203 204 /* 205 * ICP State: Accept_Interrupt 206 * 207 * Return the pending interrupt (if any) along with the 208 * current CPPR, then clear the XISR & set CPPR to the 209 * pending priority 210 */ 211 do { 212 old_state = new_state = ACCESS_ONCE(icp->state); 213 214 xirr = old_state.xisr | (((u32)old_state.cppr) << 24); 215 if (!old_state.xisr) 216 break; 217 new_state.cppr = new_state.pending_pri; 218 new_state.pending_pri = 0xff; 219 new_state.xisr = 0; 220 221 } while (!icp_rm_try_update(icp, old_state, new_state)); 222 223 /* Return the result in GPR4 */ 224 vcpu->arch.gpr[4] = xirr; 225 226 return check_too_hard(xics, icp); 227 } 228 229 int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, 230 unsigned long mfrr) 231 { 232 union kvmppc_icp_state old_state, new_state; 233 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 234 struct kvmppc_icp *icp, *this_icp = vcpu->arch.icp; 235 u32 reject; 236 bool resend; 237 bool local; 238 239 if (!xics || !xics->real_mode) 240 return H_TOO_HARD; 241 242 local = this_icp->server_num == server; 243 if (local) 244 icp = this_icp; 245 else 246 icp = kvmppc_xics_find_server(vcpu->kvm, server); 247 if (!icp) 248 return H_PARAMETER; 249 250 /* 251 * ICP state: Set_MFRR 252 * 253 * If the CPPR is more favored than the new MFRR, then 254 * nothing needs to be done as there can be no XISR to 255 * reject. 256 * 257 * If the CPPR is less favored, then we might be replacing 258 * an interrupt, and thus need to possibly reject it as in 259 * 260 * ICP state: Check_IPI 261 */ 262 do { 263 old_state = new_state = ACCESS_ONCE(icp->state); 264 265 /* Set_MFRR */ 266 new_state.mfrr = mfrr; 267 268 /* Check_IPI */ 269 reject = 0; 270 resend = false; 271 if (mfrr < new_state.cppr) { 272 /* Reject a pending interrupt if not an IPI */ 273 if (mfrr <= new_state.pending_pri) 274 reject = new_state.xisr; 275 new_state.pending_pri = mfrr; 276 new_state.xisr = XICS_IPI; 277 } 278 279 if (mfrr > old_state.mfrr && mfrr > new_state.cppr) { 280 resend = new_state.need_resend; 281 new_state.need_resend = 0; 282 } 283 } while (!icp_rm_try_update(icp, old_state, new_state)); 284 285 /* Pass rejects to virtual mode */ 286 if (reject && reject != XICS_IPI) { 287 this_icp->rm_action |= XICS_RM_REJECT; 288 this_icp->rm_reject = reject; 289 } 290 291 /* Pass resends to virtual mode */ 292 if (resend) 293 this_icp->rm_action |= XICS_RM_CHECK_RESEND; 294 295 return check_too_hard(xics, this_icp); 296 } 297 298 int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) 299 { 300 union kvmppc_icp_state old_state, new_state; 301 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 302 struct kvmppc_icp *icp = vcpu->arch.icp; 303 u32 reject; 304 305 if (!xics || !xics->real_mode) 306 return H_TOO_HARD; 307 308 /* 309 * ICP State: Set_CPPR 310 * 311 * We can safely compare the new value with the current 312 * value outside of the transaction as the CPPR is only 313 * ever changed by the processor on itself 314 */ 315 if (cppr > icp->state.cppr) { 316 icp_rm_down_cppr(xics, icp, cppr); 317 goto bail; 318 } else if (cppr == icp->state.cppr) 319 return H_SUCCESS; 320 321 /* 322 * ICP State: Up_CPPR 323 * 324 * The processor is raising its priority, this can result 325 * in a rejection of a pending interrupt: 326 * 327 * ICP State: Reject_Current 328 * 329 * We can remove EE from the current processor, the update 330 * transaction will set it again if needed 331 */ 332 icp_rm_clr_vcpu_irq(icp->vcpu); 333 334 do { 335 old_state = new_state = ACCESS_ONCE(icp->state); 336 337 reject = 0; 338 new_state.cppr = cppr; 339 340 if (cppr <= new_state.pending_pri) { 341 reject = new_state.xisr; 342 new_state.xisr = 0; 343 new_state.pending_pri = 0xff; 344 } 345 346 } while (!icp_rm_try_update(icp, old_state, new_state)); 347 348 /* Pass rejects to virtual mode */ 349 if (reject && reject != XICS_IPI) { 350 icp->rm_action |= XICS_RM_REJECT; 351 icp->rm_reject = reject; 352 } 353 bail: 354 return check_too_hard(xics, icp); 355 } 356 357 int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) 358 { 359 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 360 struct kvmppc_icp *icp = vcpu->arch.icp; 361 struct kvmppc_ics *ics; 362 struct ics_irq_state *state; 363 u32 irq = xirr & 0x00ffffff; 364 u16 src; 365 366 if (!xics || !xics->real_mode) 367 return H_TOO_HARD; 368 369 /* 370 * ICP State: EOI 371 * 372 * Note: If EOI is incorrectly used by SW to lower the CPPR 373 * value (ie more favored), we do not check for rejection of 374 * a pending interrupt, this is a SW error and PAPR sepcifies 375 * that we don't have to deal with it. 376 * 377 * The sending of an EOI to the ICS is handled after the 378 * CPPR update 379 * 380 * ICP State: Down_CPPR which we handle 381 * in a separate function as it's shared with H_CPPR. 382 */ 383 icp_rm_down_cppr(xics, icp, xirr >> 24); 384 385 /* IPIs have no EOI */ 386 if (irq == XICS_IPI) 387 goto bail; 388 /* 389 * EOI handling: If the interrupt is still asserted, we need to 390 * resend it. We can take a lockless "peek" at the ICS state here. 391 * 392 * "Message" interrupts will never have "asserted" set 393 */ 394 ics = kvmppc_xics_find_ics(xics, irq, &src); 395 if (!ics) 396 goto bail; 397 state = &ics->irq_state[src]; 398 399 /* Still asserted, resend it, we make it look like a reject */ 400 if (state->asserted) { 401 icp->rm_action |= XICS_RM_REJECT; 402 icp->rm_reject = irq; 403 } 404 405 if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) { 406 icp->rm_action |= XICS_RM_NOTIFY_EOI; 407 icp->rm_eoied_irq = irq; 408 } 409 bail: 410 return check_too_hard(xics, icp); 411 } 412