1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <linux/interrupt.h> 8 #include <linux/irq.h> 9 #include <linux/irqdomain.h> 10 #include <linux/msi.h> 11 #include <linux/sched.h> 12 13 #include <linux/irqchip/arm-gic-v4.h> 14 15 /* 16 * WARNING: The blurb below assumes that you understand the 17 * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets 18 * translated into GICv4 commands. So it effectively targets at most 19 * two individuals. You know who you are. 20 * 21 * The core GICv4 code is designed to *avoid* exposing too much of the 22 * core GIC code (that would in turn leak into the hypervisor code), 23 * and instead provide a hypervisor agnostic interface to the HW (of 24 * course, the astute reader will quickly realize that hypervisor 25 * agnostic actually means KVM-specific - what were you thinking?). 26 * 27 * In order to achieve a modicum of isolation, we try to hide most of 28 * the GICv4 "stuff" behind normal irqchip operations: 29 * 30 * - Any guest-visible VLPI is backed by a Linux interrupt (and a 31 * physical LPI which gets unmapped when the guest maps the 32 * VLPI). This allows the same DevID/EventID pair to be either 33 * mapped to the LPI (host) or the VLPI (guest). Note that this is 34 * exclusive, and you cannot have both. 35 * 36 * - Enabling/disabling a VLPI is done by issuing mask/unmask calls. 37 * 38 * - Guest INT/CLEAR commands are implemented through 39 * irq_set_irqchip_state(). 40 * 41 * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or 42 * issuing an INV after changing a priority) gets shoved into the 43 * irq_set_vcpu_affinity() method. While this is quite horrible 44 * (let's face it, this is the irqchip version of an ioctl), it 45 * confines the crap to a single location. And map/unmap really is 46 * about setting the affinity of a VLPI to a vcpu, so only INV is 47 * majorly out of place. So there. 48 * 49 * A number of commands are simply not provided by this interface, as 50 * they do not make direct sense. For example, MAPD is purely local to 51 * the virtual ITS (because it references a virtual device, and the 52 * physical ITS is still very much in charge of the physical 53 * device). Same goes for things like MAPC (the physical ITS deals 54 * with the actual vPE affinity, and not the braindead concept of 55 * collection). SYNC is not provided either, as each and every command 56 * is followed by a VSYNC. This could be relaxed in the future, should 57 * this be seen as a bottleneck (yes, this means *never*). 58 * 59 * But handling VLPIs is only one side of the job of the GICv4 60 * code. The other (darker) side is to take care of the doorbell 61 * interrupts which are delivered when a VLPI targeting a non-running 62 * vcpu is being made pending. 63 * 64 * The choice made here is that each vcpu (VPE in old northern GICv4 65 * dialect) gets a single doorbell LPI, no matter how many interrupts 66 * are targeting it. This has a nice property, which is that the 67 * interrupt becomes a handle for the VPE, and that the hypervisor 68 * code can manipulate it through the normal interrupt API: 69 * 70 * - VMs (or rather the VM abstraction that matters to the GIC) 71 * contain an irq domain where each interrupt maps to a VPE. In 72 * turn, this domain sits on top of the normal LPI allocator, and a 73 * specially crafted irq_chip implementation. 74 * 75 * - mask/unmask do what is expected on the doorbell interrupt. 76 * 77 * - irq_set_affinity is used to move a VPE from one redistributor to 78 * another. 79 * 80 * - irq_set_vcpu_affinity once again gets hijacked for the purpose of 81 * creating a new sub-API, namely scheduling/descheduling a VPE 82 * (which involves programming GICR_V{PROP,PEND}BASER) and 83 * performing INVALL operations. 84 */ 85 86 static struct irq_domain *gic_domain; 87 static const struct irq_domain_ops *vpe_domain_ops; 88 static const struct irq_domain_ops *sgi_domain_ops; 89 90 #ifdef CONFIG_ARM64 91 #include <asm/cpufeature.h> 92 93 bool gic_cpuif_has_vsgi(void) 94 { 95 unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); 96 97 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT); 98 99 return fld >= 0x3; 100 } 101 #else 102 bool gic_cpuif_has_vsgi(void) 103 { 104 return false; 105 } 106 #endif 107 108 static bool has_v4_1(void) 109 { 110 return !!sgi_domain_ops; 111 } 112 113 static bool has_v4_1_sgi(void) 114 { 115 return has_v4_1() && gic_cpuif_has_vsgi(); 116 } 117 118 static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) 119 { 120 char *name; 121 int sgi_base; 122 123 if (!has_v4_1_sgi()) 124 return 0; 125 126 name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current)); 127 if (!name) 128 goto err; 129 130 vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx); 131 if (!vpe->fwnode) 132 goto err; 133 134 kfree(name); 135 name = NULL; 136 137 vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16, 138 sgi_domain_ops, vpe); 139 if (!vpe->sgi_domain) 140 goto err; 141 142 sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16, 143 NUMA_NO_NODE, vpe, 144 false, NULL); 145 if (sgi_base <= 0) 146 goto err; 147 148 return 0; 149 150 err: 151 if (vpe->sgi_domain) 152 irq_domain_remove(vpe->sgi_domain); 153 if (vpe->fwnode) 154 irq_domain_free_fwnode(vpe->fwnode); 155 kfree(name); 156 return -ENOMEM; 157 } 158 159 int its_alloc_vcpu_irqs(struct its_vm *vm) 160 { 161 int vpe_base_irq, i; 162 163 vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe", 164 task_pid_nr(current)); 165 if (!vm->fwnode) 166 goto err; 167 168 vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes, 169 vm->fwnode, vpe_domain_ops, 170 vm); 171 if (!vm->domain) 172 goto err; 173 174 for (i = 0; i < vm->nr_vpes; i++) { 175 vm->vpes[i]->its_vm = vm; 176 vm->vpes[i]->idai = true; 177 } 178 179 vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes, 180 NUMA_NO_NODE, vm, 181 false, NULL); 182 if (vpe_base_irq <= 0) 183 goto err; 184 185 for (i = 0; i < vm->nr_vpes; i++) { 186 int ret; 187 vm->vpes[i]->irq = vpe_base_irq + i; 188 ret = its_alloc_vcpu_sgis(vm->vpes[i], i); 189 if (ret) 190 goto err; 191 } 192 193 return 0; 194 195 err: 196 if (vm->domain) 197 irq_domain_remove(vm->domain); 198 if (vm->fwnode) 199 irq_domain_free_fwnode(vm->fwnode); 200 201 return -ENOMEM; 202 } 203 204 static void its_free_sgi_irqs(struct its_vm *vm) 205 { 206 int i; 207 208 if (!has_v4_1_sgi()) 209 return; 210 211 for (i = 0; i < vm->nr_vpes; i++) { 212 unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0); 213 214 if (WARN_ON(!irq)) 215 continue; 216 217 irq_domain_free_irqs(irq, 16); 218 irq_domain_remove(vm->vpes[i]->sgi_domain); 219 irq_domain_free_fwnode(vm->vpes[i]->fwnode); 220 } 221 } 222 223 void its_free_vcpu_irqs(struct its_vm *vm) 224 { 225 its_free_sgi_irqs(vm); 226 irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes); 227 irq_domain_remove(vm->domain); 228 irq_domain_free_fwnode(vm->fwnode); 229 } 230 231 static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info) 232 { 233 return irq_set_vcpu_affinity(vpe->irq, info); 234 } 235 236 int its_make_vpe_non_resident(struct its_vpe *vpe, bool db) 237 { 238 struct irq_desc *desc = irq_to_desc(vpe->irq); 239 struct its_cmd_info info = { }; 240 int ret; 241 242 WARN_ON(preemptible()); 243 244 info.cmd_type = DESCHEDULE_VPE; 245 if (has_v4_1()) { 246 /* GICv4.1 can directly deal with doorbells */ 247 info.req_db = db; 248 } else { 249 /* Undo the nested disable_irq() calls... */ 250 while (db && irqd_irq_disabled(&desc->irq_data)) 251 enable_irq(vpe->irq); 252 } 253 254 ret = its_send_vpe_cmd(vpe, &info); 255 if (!ret) 256 vpe->resident = false; 257 258 vpe->ready = false; 259 260 return ret; 261 } 262 263 int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en) 264 { 265 struct its_cmd_info info = { }; 266 int ret; 267 268 WARN_ON(preemptible()); 269 270 info.cmd_type = SCHEDULE_VPE; 271 if (has_v4_1()) { 272 info.g0en = g0en; 273 info.g1en = g1en; 274 } else { 275 /* Disabled the doorbell, as we're about to enter the guest */ 276 disable_irq_nosync(vpe->irq); 277 } 278 279 ret = its_send_vpe_cmd(vpe, &info); 280 if (!ret) 281 vpe->resident = true; 282 283 return ret; 284 } 285 286 int its_commit_vpe(struct its_vpe *vpe) 287 { 288 struct its_cmd_info info = { 289 .cmd_type = COMMIT_VPE, 290 }; 291 int ret; 292 293 WARN_ON(preemptible()); 294 295 ret = its_send_vpe_cmd(vpe, &info); 296 if (!ret) 297 vpe->ready = true; 298 299 return ret; 300 } 301 302 303 int its_invall_vpe(struct its_vpe *vpe) 304 { 305 struct its_cmd_info info = { 306 .cmd_type = INVALL_VPE, 307 }; 308 309 return its_send_vpe_cmd(vpe, &info); 310 } 311 312 int its_map_vlpi(int irq, struct its_vlpi_map *map) 313 { 314 struct its_cmd_info info = { 315 .cmd_type = MAP_VLPI, 316 { 317 .map = map, 318 }, 319 }; 320 int ret; 321 322 /* 323 * The host will never see that interrupt firing again, so it 324 * is vital that we don't do any lazy masking. 325 */ 326 irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY); 327 328 ret = irq_set_vcpu_affinity(irq, &info); 329 if (ret) 330 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY); 331 332 return ret; 333 } 334 335 int its_get_vlpi(int irq, struct its_vlpi_map *map) 336 { 337 struct its_cmd_info info = { 338 .cmd_type = GET_VLPI, 339 { 340 .map = map, 341 }, 342 }; 343 344 return irq_set_vcpu_affinity(irq, &info); 345 } 346 347 int its_unmap_vlpi(int irq) 348 { 349 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY); 350 return irq_set_vcpu_affinity(irq, NULL); 351 } 352 353 int its_prop_update_vlpi(int irq, u8 config, bool inv) 354 { 355 struct its_cmd_info info = { 356 .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI, 357 { 358 .config = config, 359 }, 360 }; 361 362 return irq_set_vcpu_affinity(irq, &info); 363 } 364 365 int its_prop_update_vsgi(int irq, u8 priority, bool group) 366 { 367 struct its_cmd_info info = { 368 .cmd_type = PROP_UPDATE_VSGI, 369 { 370 .priority = priority, 371 .group = group, 372 }, 373 }; 374 375 return irq_set_vcpu_affinity(irq, &info); 376 } 377 378 int its_init_v4(struct irq_domain *domain, 379 const struct irq_domain_ops *vpe_ops, 380 const struct irq_domain_ops *sgi_ops) 381 { 382 if (domain) { 383 pr_info("ITS: Enabling GICv4 support\n"); 384 gic_domain = domain; 385 vpe_domain_ops = vpe_ops; 386 sgi_domain_ops = sgi_ops; 387 return 0; 388 } 389 390 pr_err("ITS: No GICv4 VPE domain allocated\n"); 391 return -ENODEV; 392 } 393