1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implementation of the IOMMU SVA API for the ARM SMMUv3 4 */ 5 6 #include <linux/mm.h> 7 #include <linux/mmu_context.h> 8 #include <linux/mmu_notifier.h> 9 #include <linux/sched/mm.h> 10 #include <linux/slab.h> 11 12 #include "arm-smmu-v3.h" 13 #include "../../iommu-sva.h" 14 #include "../../io-pgtable-arm.h" 15 16 struct arm_smmu_mmu_notifier { 17 struct mmu_notifier mn; 18 struct arm_smmu_ctx_desc *cd; 19 bool cleared; 20 refcount_t refs; 21 struct list_head list; 22 struct arm_smmu_domain *domain; 23 }; 24 25 #define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn) 26 27 struct arm_smmu_bond { 28 struct iommu_sva sva; 29 struct mm_struct *mm; 30 struct arm_smmu_mmu_notifier *smmu_mn; 31 struct list_head list; 32 refcount_t refs; 33 }; 34 35 #define sva_to_bond(handle) \ 36 container_of(handle, struct arm_smmu_bond, sva) 37 38 static DEFINE_MUTEX(sva_lock); 39 40 /* 41 * Check if the CPU ASID is available on the SMMU side. If a private context 42 * descriptor is using it, try to replace it. 43 */ 44 static struct arm_smmu_ctx_desc * 45 arm_smmu_share_asid(struct mm_struct *mm, u16 asid) 46 { 47 int ret; 48 u32 new_asid; 49 struct arm_smmu_ctx_desc *cd; 50 struct arm_smmu_device *smmu; 51 struct arm_smmu_domain *smmu_domain; 52 53 cd = xa_load(&arm_smmu_asid_xa, asid); 54 if (!cd) 55 return NULL; 56 57 if (cd->mm) { 58 if (WARN_ON(cd->mm != mm)) 59 return ERR_PTR(-EINVAL); 60 /* All devices bound to this mm use the same cd struct. */ 61 refcount_inc(&cd->refs); 62 return cd; 63 } 64 65 smmu_domain = container_of(cd, struct arm_smmu_domain, s1_cfg.cd); 66 smmu = smmu_domain->smmu; 67 68 ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd, 69 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 70 if (ret) 71 return ERR_PTR(-ENOSPC); 72 /* 73 * Race with unmap: TLB invalidations will start targeting the new ASID, 74 * which isn't assigned yet. We'll do an invalidate-all on the old ASID 75 * later, so it doesn't matter. 76 */ 77 cd->asid = new_asid; 78 /* 79 * Update ASID and invalidate CD in all associated masters. There will 80 * be some overlap between use of both ASIDs, until we invalidate the 81 * TLB. 82 */ 83 arm_smmu_write_ctx_desc(smmu_domain, 0, cd); 84 85 /* Invalidate TLB entries previously associated with that context */ 86 arm_smmu_tlb_inv_asid(smmu, asid); 87 88 xa_erase(&arm_smmu_asid_xa, asid); 89 return NULL; 90 } 91 92 static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) 93 { 94 u16 asid; 95 int err = 0; 96 u64 tcr, par, reg; 97 struct arm_smmu_ctx_desc *cd; 98 struct arm_smmu_ctx_desc *ret = NULL; 99 100 /* Don't free the mm until we release the ASID */ 101 mmgrab(mm); 102 103 asid = arm64_mm_context_get(mm); 104 if (!asid) { 105 err = -ESRCH; 106 goto out_drop_mm; 107 } 108 109 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 110 if (!cd) { 111 err = -ENOMEM; 112 goto out_put_context; 113 } 114 115 refcount_set(&cd->refs, 1); 116 117 mutex_lock(&arm_smmu_asid_lock); 118 ret = arm_smmu_share_asid(mm, asid); 119 if (ret) { 120 mutex_unlock(&arm_smmu_asid_lock); 121 goto out_free_cd; 122 } 123 124 err = xa_insert(&arm_smmu_asid_xa, asid, cd, GFP_KERNEL); 125 mutex_unlock(&arm_smmu_asid_lock); 126 127 if (err) 128 goto out_free_asid; 129 130 tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | 131 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | 132 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) | 133 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) | 134 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; 135 136 switch (PAGE_SIZE) { 137 case SZ_4K: 138 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K); 139 break; 140 case SZ_16K: 141 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K); 142 break; 143 case SZ_64K: 144 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K); 145 break; 146 default: 147 WARN_ON(1); 148 err = -EINVAL; 149 goto out_free_asid; 150 } 151 152 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 153 par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); 154 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par); 155 156 cd->ttbr = virt_to_phys(mm->pgd); 157 cd->tcr = tcr; 158 /* 159 * MAIR value is pretty much constant and global, so we can just get it 160 * from the current CPU register 161 */ 162 cd->mair = read_sysreg(mair_el1); 163 cd->asid = asid; 164 cd->mm = mm; 165 166 return cd; 167 168 out_free_asid: 169 arm_smmu_free_asid(cd); 170 out_free_cd: 171 kfree(cd); 172 out_put_context: 173 arm64_mm_context_put(mm); 174 out_drop_mm: 175 mmdrop(mm); 176 return err < 0 ? ERR_PTR(err) : ret; 177 } 178 179 static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) 180 { 181 if (arm_smmu_free_asid(cd)) { 182 /* Unpin ASID */ 183 arm64_mm_context_put(cd->mm); 184 mmdrop(cd->mm); 185 kfree(cd); 186 } 187 } 188 189 static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn, 190 struct mm_struct *mm, 191 unsigned long start, unsigned long end) 192 { 193 struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); 194 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 195 size_t size; 196 197 /* 198 * The mm_types defines vm_end as the first byte after the end address, 199 * different from IOMMU subsystem using the last address of an address 200 * range. So do a simple translation here by calculating size correctly. 201 */ 202 size = end - start; 203 204 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) 205 arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid, 206 PAGE_SIZE, false, smmu_domain); 207 arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start, size); 208 } 209 210 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 211 { 212 struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); 213 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 214 215 mutex_lock(&sva_lock); 216 if (smmu_mn->cleared) { 217 mutex_unlock(&sva_lock); 218 return; 219 } 220 221 /* 222 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, 223 * but disable translation. 224 */ 225 arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, &quiet_cd); 226 227 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid); 228 arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0); 229 230 smmu_mn->cleared = true; 231 mutex_unlock(&sva_lock); 232 } 233 234 static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) 235 { 236 kfree(mn_to_smmu(mn)); 237 } 238 239 static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { 240 .invalidate_range = arm_smmu_mm_invalidate_range, 241 .release = arm_smmu_mm_release, 242 .free_notifier = arm_smmu_mmu_notifier_free, 243 }; 244 245 /* Allocate or get existing MMU notifier for this {domain, mm} pair */ 246 static struct arm_smmu_mmu_notifier * 247 arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain, 248 struct mm_struct *mm) 249 { 250 int ret; 251 struct arm_smmu_ctx_desc *cd; 252 struct arm_smmu_mmu_notifier *smmu_mn; 253 254 list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) { 255 if (smmu_mn->mn.mm == mm) { 256 refcount_inc(&smmu_mn->refs); 257 return smmu_mn; 258 } 259 } 260 261 cd = arm_smmu_alloc_shared_cd(mm); 262 if (IS_ERR(cd)) 263 return ERR_CAST(cd); 264 265 smmu_mn = kzalloc(sizeof(*smmu_mn), GFP_KERNEL); 266 if (!smmu_mn) { 267 ret = -ENOMEM; 268 goto err_free_cd; 269 } 270 271 refcount_set(&smmu_mn->refs, 1); 272 smmu_mn->cd = cd; 273 smmu_mn->domain = smmu_domain; 274 smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops; 275 276 ret = mmu_notifier_register(&smmu_mn->mn, mm); 277 if (ret) { 278 kfree(smmu_mn); 279 goto err_free_cd; 280 } 281 282 ret = arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, cd); 283 if (ret) 284 goto err_put_notifier; 285 286 list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers); 287 return smmu_mn; 288 289 err_put_notifier: 290 /* Frees smmu_mn */ 291 mmu_notifier_put(&smmu_mn->mn); 292 err_free_cd: 293 arm_smmu_free_shared_cd(cd); 294 return ERR_PTR(ret); 295 } 296 297 static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn) 298 { 299 struct mm_struct *mm = smmu_mn->mn.mm; 300 struct arm_smmu_ctx_desc *cd = smmu_mn->cd; 301 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 302 303 if (!refcount_dec_and_test(&smmu_mn->refs)) 304 return; 305 306 list_del(&smmu_mn->list); 307 arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, NULL); 308 309 /* 310 * If we went through clear(), we've already invalidated, and no 311 * new TLB entry can have been formed. 312 */ 313 if (!smmu_mn->cleared) { 314 arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid); 315 arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0); 316 } 317 318 /* Frees smmu_mn */ 319 mmu_notifier_put(&smmu_mn->mn); 320 arm_smmu_free_shared_cd(cd); 321 } 322 323 static struct iommu_sva * 324 __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm) 325 { 326 int ret; 327 struct arm_smmu_bond *bond; 328 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 329 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 330 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 331 332 if (!master || !master->sva_enabled) 333 return ERR_PTR(-ENODEV); 334 335 /* If bind() was already called for this {dev, mm} pair, reuse it. */ 336 list_for_each_entry(bond, &master->bonds, list) { 337 if (bond->mm == mm) { 338 refcount_inc(&bond->refs); 339 return &bond->sva; 340 } 341 } 342 343 bond = kzalloc(sizeof(*bond), GFP_KERNEL); 344 if (!bond) 345 return ERR_PTR(-ENOMEM); 346 347 bond->mm = mm; 348 bond->sva.dev = dev; 349 refcount_set(&bond->refs, 1); 350 351 bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm); 352 if (IS_ERR(bond->smmu_mn)) { 353 ret = PTR_ERR(bond->smmu_mn); 354 goto err_free_bond; 355 } 356 357 list_add(&bond->list, &master->bonds); 358 return &bond->sva; 359 360 err_free_bond: 361 kfree(bond); 362 return ERR_PTR(ret); 363 } 364 365 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 366 { 367 unsigned long reg, fld; 368 unsigned long oas; 369 unsigned long asid_bits; 370 u32 feat_mask = ARM_SMMU_FEAT_COHERENCY; 371 372 if (vabits_actual == 52) 373 feat_mask |= ARM_SMMU_FEAT_VAX; 374 375 if ((smmu->features & feat_mask) != feat_mask) 376 return false; 377 378 if (!(smmu->pgsize_bitmap & PAGE_SIZE)) 379 return false; 380 381 /* 382 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're 383 * not even pretending to support AArch32 here. Abort if the MMU outputs 384 * addresses larger than what we support. 385 */ 386 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 387 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); 388 oas = id_aa64mmfr0_parange_to_phys_shift(fld); 389 if (smmu->oas < oas) 390 return false; 391 392 /* We can support bigger ASIDs than the CPU, but not smaller */ 393 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); 394 asid_bits = fld ? 16 : 8; 395 if (smmu->asid_bits < asid_bits) 396 return false; 397 398 /* 399 * See max_pinned_asids in arch/arm64/mm/context.c. The following is 400 * generally the maximum number of bindable processes. 401 */ 402 if (arm64_kernel_unmapped_at_el0()) 403 asid_bits--; 404 dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) - 405 num_possible_cpus() - 2); 406 407 return true; 408 } 409 410 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) 411 { 412 /* We're not keeping track of SIDs in fault events */ 413 if (master->num_streams != 1) 414 return false; 415 416 return master->stall_enabled; 417 } 418 419 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) 420 { 421 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 422 return false; 423 424 /* SSID support is mandatory for the moment */ 425 return master->ssid_bits; 426 } 427 428 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) 429 { 430 bool enabled; 431 432 mutex_lock(&sva_lock); 433 enabled = master->sva_enabled; 434 mutex_unlock(&sva_lock); 435 return enabled; 436 } 437 438 static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) 439 { 440 int ret; 441 struct device *dev = master->dev; 442 443 /* 444 * Drivers for devices supporting PRI or stall should enable IOPF first. 445 * Others have device-specific fault handlers and don't need IOPF. 446 */ 447 if (!arm_smmu_master_iopf_supported(master)) 448 return 0; 449 450 if (!master->iopf_enabled) 451 return -EINVAL; 452 453 ret = iopf_queue_add_device(master->smmu->evtq.iopf, dev); 454 if (ret) 455 return ret; 456 457 ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); 458 if (ret) { 459 iopf_queue_remove_device(master->smmu->evtq.iopf, dev); 460 return ret; 461 } 462 return 0; 463 } 464 465 static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) 466 { 467 struct device *dev = master->dev; 468 469 if (!master->iopf_enabled) 470 return; 471 472 iommu_unregister_device_fault_handler(dev); 473 iopf_queue_remove_device(master->smmu->evtq.iopf, dev); 474 } 475 476 int arm_smmu_master_enable_sva(struct arm_smmu_master *master) 477 { 478 int ret; 479 480 mutex_lock(&sva_lock); 481 ret = arm_smmu_master_sva_enable_iopf(master); 482 if (!ret) 483 master->sva_enabled = true; 484 mutex_unlock(&sva_lock); 485 486 return ret; 487 } 488 489 int arm_smmu_master_disable_sva(struct arm_smmu_master *master) 490 { 491 mutex_lock(&sva_lock); 492 if (!list_empty(&master->bonds)) { 493 dev_err(master->dev, "cannot disable SVA, device is bound\n"); 494 mutex_unlock(&sva_lock); 495 return -EBUSY; 496 } 497 arm_smmu_master_sva_disable_iopf(master); 498 master->sva_enabled = false; 499 mutex_unlock(&sva_lock); 500 501 return 0; 502 } 503 504 void arm_smmu_sva_notifier_synchronize(void) 505 { 506 /* 507 * Some MMU notifiers may still be waiting to be freed, using 508 * arm_smmu_mmu_notifier_free(). Wait for them. 509 */ 510 mmu_notifier_synchronize(); 511 } 512 513 void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 514 struct device *dev, ioasid_t id) 515 { 516 struct mm_struct *mm = domain->mm; 517 struct arm_smmu_bond *bond = NULL, *t; 518 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 519 520 mutex_lock(&sva_lock); 521 list_for_each_entry(t, &master->bonds, list) { 522 if (t->mm == mm) { 523 bond = t; 524 break; 525 } 526 } 527 528 if (!WARN_ON(!bond) && refcount_dec_and_test(&bond->refs)) { 529 list_del(&bond->list); 530 arm_smmu_mmu_notifier_put(bond->smmu_mn); 531 kfree(bond); 532 } 533 mutex_unlock(&sva_lock); 534 } 535 536 static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, 537 struct device *dev, ioasid_t id) 538 { 539 int ret = 0; 540 struct iommu_sva *handle; 541 struct mm_struct *mm = domain->mm; 542 543 mutex_lock(&sva_lock); 544 handle = __arm_smmu_sva_bind(dev, mm); 545 if (IS_ERR(handle)) 546 ret = PTR_ERR(handle); 547 mutex_unlock(&sva_lock); 548 549 return ret; 550 } 551 552 static void arm_smmu_sva_domain_free(struct iommu_domain *domain) 553 { 554 kfree(domain); 555 } 556 557 static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { 558 .set_dev_pasid = arm_smmu_sva_set_dev_pasid, 559 .free = arm_smmu_sva_domain_free 560 }; 561 562 struct iommu_domain *arm_smmu_sva_domain_alloc(void) 563 { 564 struct iommu_domain *domain; 565 566 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 567 if (!domain) 568 return NULL; 569 domain->ops = &arm_smmu_sva_domain_ops; 570 571 return domain; 572 } 573