1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * IOMMU API for ARM architected SMMUv3 implementations. 4 * 5 * Copyright (C) 2015 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 * 9 * This driver is powered by bad coffee and bombay mix. 10 */ 11 12 #include <linux/acpi.h> 13 #include <linux/acpi_iort.h> 14 #include <linux/bitops.h> 15 #include <linux/crash_dump.h> 16 #include <linux/delay.h> 17 #include <linux/err.h> 18 #include <linux/interrupt.h> 19 #include <linux/io-pgtable.h> 20 #include <linux/iopoll.h> 21 #include <linux/module.h> 22 #include <linux/msi.h> 23 #include <linux/of.h> 24 #include <linux/of_address.h> 25 #include <linux/of_platform.h> 26 #include <linux/pci.h> 27 #include <linux/pci-ats.h> 28 #include <linux/platform_device.h> 29 30 #include "arm-smmu-v3.h" 31 #include "../../dma-iommu.h" 32 #include "../../iommu-sva-lib.h" 33 34 static bool disable_bypass = true; 35 module_param(disable_bypass, bool, 0444); 36 MODULE_PARM_DESC(disable_bypass, 37 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 38 39 static bool disable_msipolling; 40 module_param(disable_msipolling, bool, 0444); 41 MODULE_PARM_DESC(disable_msipolling, 42 "Disable MSI-based polling for CMD_SYNC completion."); 43 44 enum arm_smmu_msi_index { 45 EVTQ_MSI_INDEX, 46 GERROR_MSI_INDEX, 47 PRIQ_MSI_INDEX, 48 ARM_SMMU_MAX_MSIS, 49 }; 50 51 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { 52 [EVTQ_MSI_INDEX] = { 53 ARM_SMMU_EVTQ_IRQ_CFG0, 54 ARM_SMMU_EVTQ_IRQ_CFG1, 55 ARM_SMMU_EVTQ_IRQ_CFG2, 56 }, 57 [GERROR_MSI_INDEX] = { 58 ARM_SMMU_GERROR_IRQ_CFG0, 59 ARM_SMMU_GERROR_IRQ_CFG1, 60 ARM_SMMU_GERROR_IRQ_CFG2, 61 }, 62 [PRIQ_MSI_INDEX] = { 63 ARM_SMMU_PRIQ_IRQ_CFG0, 64 ARM_SMMU_PRIQ_IRQ_CFG1, 65 ARM_SMMU_PRIQ_IRQ_CFG2, 66 }, 67 }; 68 69 struct arm_smmu_option_prop { 70 u32 opt; 71 const char *prop; 72 }; 73 74 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa); 75 DEFINE_MUTEX(arm_smmu_asid_lock); 76 77 /* 78 * Special value used by SVA when a process dies, to quiesce a CD without 79 * disabling it. 80 */ 81 struct arm_smmu_ctx_desc quiet_cd = { 0 }; 82 83 static struct arm_smmu_option_prop arm_smmu_options[] = { 84 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" }, 85 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"}, 86 { 0, NULL}, 87 }; 88 89 static void parse_driver_options(struct arm_smmu_device *smmu) 90 { 91 int i = 0; 92 93 do { 94 if (of_property_read_bool(smmu->dev->of_node, 95 arm_smmu_options[i].prop)) { 96 smmu->options |= arm_smmu_options[i].opt; 97 dev_notice(smmu->dev, "option %s\n", 98 arm_smmu_options[i].prop); 99 } 100 } while (arm_smmu_options[++i].opt); 101 } 102 103 /* Low-level queue manipulation functions */ 104 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n) 105 { 106 u32 space, prod, cons; 107 108 prod = Q_IDX(q, q->prod); 109 cons = Q_IDX(q, q->cons); 110 111 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons)) 112 space = (1 << q->max_n_shift) - (prod - cons); 113 else 114 space = cons - prod; 115 116 return space >= n; 117 } 118 119 static bool queue_full(struct arm_smmu_ll_queue *q) 120 { 121 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && 122 Q_WRP(q, q->prod) != Q_WRP(q, q->cons); 123 } 124 125 static bool queue_empty(struct arm_smmu_ll_queue *q) 126 { 127 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && 128 Q_WRP(q, q->prod) == Q_WRP(q, q->cons); 129 } 130 131 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod) 132 { 133 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) && 134 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) || 135 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) && 136 (Q_IDX(q, q->cons) <= Q_IDX(q, prod))); 137 } 138 139 static void queue_sync_cons_out(struct arm_smmu_queue *q) 140 { 141 /* 142 * Ensure that all CPU accesses (reads and writes) to the queue 143 * are complete before we update the cons pointer. 144 */ 145 __iomb(); 146 writel_relaxed(q->llq.cons, q->cons_reg); 147 } 148 149 static void queue_inc_cons(struct arm_smmu_ll_queue *q) 150 { 151 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1; 152 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons); 153 } 154 155 static int queue_sync_prod_in(struct arm_smmu_queue *q) 156 { 157 u32 prod; 158 int ret = 0; 159 160 /* 161 * We can't use the _relaxed() variant here, as we must prevent 162 * speculative reads of the queue before we have determined that 163 * prod has indeed moved. 164 */ 165 prod = readl(q->prod_reg); 166 167 if (Q_OVF(prod) != Q_OVF(q->llq.prod)) 168 ret = -EOVERFLOW; 169 170 q->llq.prod = prod; 171 return ret; 172 } 173 174 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n) 175 { 176 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 177 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 178 } 179 180 static void queue_poll_init(struct arm_smmu_device *smmu, 181 struct arm_smmu_queue_poll *qp) 182 { 183 qp->delay = 1; 184 qp->spin_cnt = 0; 185 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); 186 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); 187 } 188 189 static int queue_poll(struct arm_smmu_queue_poll *qp) 190 { 191 if (ktime_compare(ktime_get(), qp->timeout) > 0) 192 return -ETIMEDOUT; 193 194 if (qp->wfe) { 195 wfe(); 196 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) { 197 cpu_relax(); 198 } else { 199 udelay(qp->delay); 200 qp->delay *= 2; 201 qp->spin_cnt = 0; 202 } 203 204 return 0; 205 } 206 207 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords) 208 { 209 int i; 210 211 for (i = 0; i < n_dwords; ++i) 212 *dst++ = cpu_to_le64(*src++); 213 } 214 215 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords) 216 { 217 int i; 218 219 for (i = 0; i < n_dwords; ++i) 220 *dst++ = le64_to_cpu(*src++); 221 } 222 223 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent) 224 { 225 if (queue_empty(&q->llq)) 226 return -EAGAIN; 227 228 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords); 229 queue_inc_cons(&q->llq); 230 queue_sync_cons_out(q); 231 return 0; 232 } 233 234 /* High-level queue accessors */ 235 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) 236 { 237 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT); 238 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode); 239 240 switch (ent->opcode) { 241 case CMDQ_OP_TLBI_EL2_ALL: 242 case CMDQ_OP_TLBI_NSNH_ALL: 243 break; 244 case CMDQ_OP_PREFETCH_CFG: 245 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid); 246 break; 247 case CMDQ_OP_CFGI_CD: 248 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid); 249 fallthrough; 250 case CMDQ_OP_CFGI_STE: 251 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); 252 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf); 253 break; 254 case CMDQ_OP_CFGI_CD_ALL: 255 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); 256 break; 257 case CMDQ_OP_CFGI_ALL: 258 /* Cover the entire SID range */ 259 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); 260 break; 261 case CMDQ_OP_TLBI_NH_VA: 262 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 263 fallthrough; 264 case CMDQ_OP_TLBI_EL2_VA: 265 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); 266 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); 267 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); 268 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); 269 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); 270 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg); 271 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK; 272 break; 273 case CMDQ_OP_TLBI_S2_IPA: 274 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); 275 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); 276 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 277 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); 278 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); 279 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg); 280 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK; 281 break; 282 case CMDQ_OP_TLBI_NH_ASID: 283 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); 284 fallthrough; 285 case CMDQ_OP_TLBI_S12_VMALL: 286 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 287 break; 288 case CMDQ_OP_TLBI_EL2_ASID: 289 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); 290 break; 291 case CMDQ_OP_ATC_INV: 292 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 293 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global); 294 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid); 295 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid); 296 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size); 297 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK; 298 break; 299 case CMDQ_OP_PRI_RESP: 300 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 301 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid); 302 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid); 303 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid); 304 switch (ent->pri.resp) { 305 case PRI_RESP_DENY: 306 case PRI_RESP_FAIL: 307 case PRI_RESP_SUCC: 308 break; 309 default: 310 return -EINVAL; 311 } 312 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp); 313 break; 314 case CMDQ_OP_RESUME: 315 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid); 316 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp); 317 cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag); 318 break; 319 case CMDQ_OP_CMD_SYNC: 320 if (ent->sync.msiaddr) { 321 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ); 322 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK; 323 } else { 324 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV); 325 } 326 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH); 327 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); 328 break; 329 default: 330 return -ENOENT; 331 } 332 333 return 0; 334 } 335 336 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu) 337 { 338 return &smmu->cmdq; 339 } 340 341 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu, 342 struct arm_smmu_queue *q, u32 prod) 343 { 344 struct arm_smmu_cmdq_ent ent = { 345 .opcode = CMDQ_OP_CMD_SYNC, 346 }; 347 348 /* 349 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI 350 * payload, so the write will zero the entire command on that platform. 351 */ 352 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) { 353 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) * 354 q->ent_dwords * 8; 355 } 356 357 arm_smmu_cmdq_build_cmd(cmd, &ent); 358 } 359 360 static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, 361 struct arm_smmu_queue *q) 362 { 363 static const char * const cerror_str[] = { 364 [CMDQ_ERR_CERROR_NONE_IDX] = "No error", 365 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command", 366 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch", 367 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout", 368 }; 369 370 int i; 371 u64 cmd[CMDQ_ENT_DWORDS]; 372 u32 cons = readl_relaxed(q->cons_reg); 373 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons); 374 struct arm_smmu_cmdq_ent cmd_sync = { 375 .opcode = CMDQ_OP_CMD_SYNC, 376 }; 377 378 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons, 379 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown"); 380 381 switch (idx) { 382 case CMDQ_ERR_CERROR_ABT_IDX: 383 dev_err(smmu->dev, "retrying command fetch\n"); 384 return; 385 case CMDQ_ERR_CERROR_NONE_IDX: 386 return; 387 case CMDQ_ERR_CERROR_ATC_INV_IDX: 388 /* 389 * ATC Invalidation Completion timeout. CONS is still pointing 390 * at the CMD_SYNC. Attempt to complete other pending commands 391 * by repeating the CMD_SYNC, though we might well end up back 392 * here since the ATC invalidation may still be pending. 393 */ 394 return; 395 case CMDQ_ERR_CERROR_ILL_IDX: 396 default: 397 break; 398 } 399 400 /* 401 * We may have concurrent producers, so we need to be careful 402 * not to touch any of the shadow cmdq state. 403 */ 404 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords); 405 dev_err(smmu->dev, "skipping command in error state:\n"); 406 for (i = 0; i < ARRAY_SIZE(cmd); ++i) 407 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); 408 409 /* Convert the erroneous command into a CMD_SYNC */ 410 arm_smmu_cmdq_build_cmd(cmd, &cmd_sync); 411 412 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords); 413 } 414 415 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) 416 { 417 __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q); 418 } 419 420 /* 421 * Command queue locking. 422 * This is a form of bastardised rwlock with the following major changes: 423 * 424 * - The only LOCK routines are exclusive_trylock() and shared_lock(). 425 * Neither have barrier semantics, and instead provide only a control 426 * dependency. 427 * 428 * - The UNLOCK routines are supplemented with shared_tryunlock(), which 429 * fails if the caller appears to be the last lock holder (yes, this is 430 * racy). All successful UNLOCK routines have RELEASE semantics. 431 */ 432 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq) 433 { 434 int val; 435 436 /* 437 * We can try to avoid the cmpxchg() loop by simply incrementing the 438 * lock counter. When held in exclusive state, the lock counter is set 439 * to INT_MIN so these increments won't hurt as the value will remain 440 * negative. 441 */ 442 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0) 443 return; 444 445 do { 446 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0); 447 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val); 448 } 449 450 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq) 451 { 452 (void)atomic_dec_return_release(&cmdq->lock); 453 } 454 455 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq) 456 { 457 if (atomic_read(&cmdq->lock) == 1) 458 return false; 459 460 arm_smmu_cmdq_shared_unlock(cmdq); 461 return true; 462 } 463 464 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \ 465 ({ \ 466 bool __ret; \ 467 local_irq_save(flags); \ 468 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \ 469 if (!__ret) \ 470 local_irq_restore(flags); \ 471 __ret; \ 472 }) 473 474 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \ 475 ({ \ 476 atomic_set_release(&cmdq->lock, 0); \ 477 local_irq_restore(flags); \ 478 }) 479 480 481 /* 482 * Command queue insertion. 483 * This is made fiddly by our attempts to achieve some sort of scalability 484 * since there is one queue shared amongst all of the CPUs in the system. If 485 * you like mixed-size concurrency, dependency ordering and relaxed atomics, 486 * then you'll *love* this monstrosity. 487 * 488 * The basic idea is to split the queue up into ranges of commands that are 489 * owned by a given CPU; the owner may not have written all of the commands 490 * itself, but is responsible for advancing the hardware prod pointer when 491 * the time comes. The algorithm is roughly: 492 * 493 * 1. Allocate some space in the queue. At this point we also discover 494 * whether the head of the queue is currently owned by another CPU, 495 * or whether we are the owner. 496 * 497 * 2. Write our commands into our allocated slots in the queue. 498 * 499 * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map. 500 * 501 * 4. If we are an owner: 502 * a. Wait for the previous owner to finish. 503 * b. Mark the queue head as unowned, which tells us the range 504 * that we are responsible for publishing. 505 * c. Wait for all commands in our owned range to become valid. 506 * d. Advance the hardware prod pointer. 507 * e. Tell the next owner we've finished. 508 * 509 * 5. If we are inserting a CMD_SYNC (we may or may not have been an 510 * owner), then we need to stick around until it has completed: 511 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC 512 * to clear the first 4 bytes. 513 * b. Otherwise, we spin waiting for the hardware cons pointer to 514 * advance past our command. 515 * 516 * The devil is in the details, particularly the use of locking for handling 517 * SYNC completion and freeing up space in the queue before we think that it is 518 * full. 519 */ 520 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq, 521 u32 sprod, u32 eprod, bool set) 522 { 523 u32 swidx, sbidx, ewidx, ebidx; 524 struct arm_smmu_ll_queue llq = { 525 .max_n_shift = cmdq->q.llq.max_n_shift, 526 .prod = sprod, 527 }; 528 529 ewidx = BIT_WORD(Q_IDX(&llq, eprod)); 530 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG; 531 532 while (llq.prod != eprod) { 533 unsigned long mask; 534 atomic_long_t *ptr; 535 u32 limit = BITS_PER_LONG; 536 537 swidx = BIT_WORD(Q_IDX(&llq, llq.prod)); 538 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG; 539 540 ptr = &cmdq->valid_map[swidx]; 541 542 if ((swidx == ewidx) && (sbidx < ebidx)) 543 limit = ebidx; 544 545 mask = GENMASK(limit - 1, sbidx); 546 547 /* 548 * The valid bit is the inverse of the wrap bit. This means 549 * that a zero-initialised queue is invalid and, after marking 550 * all entries as valid, they become invalid again when we 551 * wrap. 552 */ 553 if (set) { 554 atomic_long_xor(mask, ptr); 555 } else { /* Poll */ 556 unsigned long valid; 557 558 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask; 559 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid); 560 } 561 562 llq.prod = queue_inc_prod_n(&llq, limit - sbidx); 563 } 564 } 565 566 /* Mark all entries in the range [sprod, eprod) as valid */ 567 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq, 568 u32 sprod, u32 eprod) 569 { 570 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true); 571 } 572 573 /* Wait for all entries in the range [sprod, eprod) to become valid */ 574 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq, 575 u32 sprod, u32 eprod) 576 { 577 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false); 578 } 579 580 /* Wait for the command queue to become non-full */ 581 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu, 582 struct arm_smmu_ll_queue *llq) 583 { 584 unsigned long flags; 585 struct arm_smmu_queue_poll qp; 586 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu); 587 int ret = 0; 588 589 /* 590 * Try to update our copy of cons by grabbing exclusive cmdq access. If 591 * that fails, spin until somebody else updates it for us. 592 */ 593 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) { 594 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg)); 595 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags); 596 llq->val = READ_ONCE(cmdq->q.llq.val); 597 return 0; 598 } 599 600 queue_poll_init(smmu, &qp); 601 do { 602 llq->val = READ_ONCE(cmdq->q.llq.val); 603 if (!queue_full(llq)) 604 break; 605 606 ret = queue_poll(&qp); 607 } while (!ret); 608 609 return ret; 610 } 611 612 /* 613 * Wait until the SMMU signals a CMD_SYNC completion MSI. 614 * Must be called with the cmdq lock held in some capacity. 615 */ 616 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu, 617 struct arm_smmu_ll_queue *llq) 618 { 619 int ret = 0; 620 struct arm_smmu_queue_poll qp; 621 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu); 622 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod)); 623 624 queue_poll_init(smmu, &qp); 625 626 /* 627 * The MSI won't generate an event, since it's being written back 628 * into the command queue. 629 */ 630 qp.wfe = false; 631 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp))); 632 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1); 633 return ret; 634 } 635 636 /* 637 * Wait until the SMMU cons index passes llq->prod. 638 * Must be called with the cmdq lock held in some capacity. 639 */ 640 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu, 641 struct arm_smmu_ll_queue *llq) 642 { 643 struct arm_smmu_queue_poll qp; 644 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu); 645 u32 prod = llq->prod; 646 int ret = 0; 647 648 queue_poll_init(smmu, &qp); 649 llq->val = READ_ONCE(cmdq->q.llq.val); 650 do { 651 if (queue_consumed(llq, prod)) 652 break; 653 654 ret = queue_poll(&qp); 655 656 /* 657 * This needs to be a readl() so that our subsequent call 658 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately. 659 * 660 * Specifically, we need to ensure that we observe all 661 * shared_lock()s by other CMD_SYNCs that share our owner, 662 * so that a failing call to tryunlock() means that we're 663 * the last one out and therefore we can safely advance 664 * cmdq->q.llq.cons. Roughly speaking: 665 * 666 * CPU 0 CPU1 CPU2 (us) 667 * 668 * if (sync) 669 * shared_lock(); 670 * 671 * dma_wmb(); 672 * set_valid_map(); 673 * 674 * if (owner) { 675 * poll_valid_map(); 676 * <control dependency> 677 * writel(prod_reg); 678 * 679 * readl(cons_reg); 680 * tryunlock(); 681 * 682 * Requires us to see CPU 0's shared_lock() acquisition. 683 */ 684 llq->cons = readl(cmdq->q.cons_reg); 685 } while (!ret); 686 687 return ret; 688 } 689 690 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu, 691 struct arm_smmu_ll_queue *llq) 692 { 693 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) 694 return __arm_smmu_cmdq_poll_until_msi(smmu, llq); 695 696 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq); 697 } 698 699 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds, 700 u32 prod, int n) 701 { 702 int i; 703 struct arm_smmu_ll_queue llq = { 704 .max_n_shift = cmdq->q.llq.max_n_shift, 705 .prod = prod, 706 }; 707 708 for (i = 0; i < n; ++i) { 709 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS]; 710 711 prod = queue_inc_prod_n(&llq, i); 712 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS); 713 } 714 } 715 716 /* 717 * This is the actual insertion function, and provides the following 718 * ordering guarantees to callers: 719 * 720 * - There is a dma_wmb() before publishing any commands to the queue. 721 * This can be relied upon to order prior writes to data structures 722 * in memory (such as a CD or an STE) before the command. 723 * 724 * - On completion of a CMD_SYNC, there is a control dependency. 725 * This can be relied upon to order subsequent writes to memory (e.g. 726 * freeing an IOVA) after completion of the CMD_SYNC. 727 * 728 * - Command insertion is totally ordered, so if two CPUs each race to 729 * insert their own list of commands then all of the commands from one 730 * CPU will appear before any of the commands from the other CPU. 731 */ 732 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, 733 u64 *cmds, int n, bool sync) 734 { 735 u64 cmd_sync[CMDQ_ENT_DWORDS]; 736 u32 prod; 737 unsigned long flags; 738 bool owner; 739 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu); 740 struct arm_smmu_ll_queue llq, head; 741 int ret = 0; 742 743 llq.max_n_shift = cmdq->q.llq.max_n_shift; 744 745 /* 1. Allocate some space in the queue */ 746 local_irq_save(flags); 747 llq.val = READ_ONCE(cmdq->q.llq.val); 748 do { 749 u64 old; 750 751 while (!queue_has_space(&llq, n + sync)) { 752 local_irq_restore(flags); 753 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq)) 754 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); 755 local_irq_save(flags); 756 } 757 758 head.cons = llq.cons; 759 head.prod = queue_inc_prod_n(&llq, n + sync) | 760 CMDQ_PROD_OWNED_FLAG; 761 762 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val); 763 if (old == llq.val) 764 break; 765 766 llq.val = old; 767 } while (1); 768 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG); 769 head.prod &= ~CMDQ_PROD_OWNED_FLAG; 770 llq.prod &= ~CMDQ_PROD_OWNED_FLAG; 771 772 /* 773 * 2. Write our commands into the queue 774 * Dependency ordering from the cmpxchg() loop above. 775 */ 776 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); 777 if (sync) { 778 prod = queue_inc_prod_n(&llq, n); 779 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod); 780 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS); 781 782 /* 783 * In order to determine completion of our CMD_SYNC, we must 784 * ensure that the queue can't wrap twice without us noticing. 785 * We achieve that by taking the cmdq lock as shared before 786 * marking our slot as valid. 787 */ 788 arm_smmu_cmdq_shared_lock(cmdq); 789 } 790 791 /* 3. Mark our slots as valid, ensuring commands are visible first */ 792 dma_wmb(); 793 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod); 794 795 /* 4. If we are the owner, take control of the SMMU hardware */ 796 if (owner) { 797 /* a. Wait for previous owner to finish */ 798 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod); 799 800 /* b. Stop gathering work by clearing the owned flag */ 801 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG, 802 &cmdq->q.llq.atomic.prod); 803 prod &= ~CMDQ_PROD_OWNED_FLAG; 804 805 /* 806 * c. Wait for any gathered work to be written to the queue. 807 * Note that we read our own entries so that we have the control 808 * dependency required by (d). 809 */ 810 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod); 811 812 /* 813 * d. Advance the hardware prod pointer 814 * Control dependency ordering from the entries becoming valid. 815 */ 816 writel_relaxed(prod, cmdq->q.prod_reg); 817 818 /* 819 * e. Tell the next owner we're done 820 * Make sure we've updated the hardware first, so that we don't 821 * race to update prod and potentially move it backwards. 822 */ 823 atomic_set_release(&cmdq->owner_prod, prod); 824 } 825 826 /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */ 827 if (sync) { 828 llq.prod = queue_inc_prod_n(&llq, n); 829 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq); 830 if (ret) { 831 dev_err_ratelimited(smmu->dev, 832 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n", 833 llq.prod, 834 readl_relaxed(cmdq->q.prod_reg), 835 readl_relaxed(cmdq->q.cons_reg)); 836 } 837 838 /* 839 * Try to unlock the cmdq lock. This will fail if we're the last 840 * reader, in which case we can safely update cmdq->q.llq.cons 841 */ 842 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) { 843 WRITE_ONCE(cmdq->q.llq.cons, llq.cons); 844 arm_smmu_cmdq_shared_unlock(cmdq); 845 } 846 } 847 848 local_irq_restore(flags); 849 return ret; 850 } 851 852 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, 853 struct arm_smmu_cmdq_ent *ent, 854 bool sync) 855 { 856 u64 cmd[CMDQ_ENT_DWORDS]; 857 858 if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) { 859 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", 860 ent->opcode); 861 return -EINVAL; 862 } 863 864 return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, sync); 865 } 866 867 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, 868 struct arm_smmu_cmdq_ent *ent) 869 { 870 return __arm_smmu_cmdq_issue_cmd(smmu, ent, false); 871 } 872 873 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu, 874 struct arm_smmu_cmdq_ent *ent) 875 { 876 return __arm_smmu_cmdq_issue_cmd(smmu, ent, true); 877 } 878 879 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, 880 struct arm_smmu_cmdq_batch *cmds, 881 struct arm_smmu_cmdq_ent *cmd) 882 { 883 int index; 884 885 if (cmds->num == CMDQ_BATCH_ENTRIES) { 886 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false); 887 cmds->num = 0; 888 } 889 890 index = cmds->num * CMDQ_ENT_DWORDS; 891 if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) { 892 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", 893 cmd->opcode); 894 return; 895 } 896 897 cmds->num++; 898 } 899 900 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, 901 struct arm_smmu_cmdq_batch *cmds) 902 { 903 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true); 904 } 905 906 static int arm_smmu_page_response(struct device *dev, 907 struct iommu_fault_event *unused, 908 struct iommu_page_response *resp) 909 { 910 struct arm_smmu_cmdq_ent cmd = {0}; 911 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 912 int sid = master->streams[0].id; 913 914 if (master->stall_enabled) { 915 cmd.opcode = CMDQ_OP_RESUME; 916 cmd.resume.sid = sid; 917 cmd.resume.stag = resp->grpid; 918 switch (resp->code) { 919 case IOMMU_PAGE_RESP_INVALID: 920 case IOMMU_PAGE_RESP_FAILURE: 921 cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; 922 break; 923 case IOMMU_PAGE_RESP_SUCCESS: 924 cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; 925 break; 926 default: 927 return -EINVAL; 928 } 929 } else { 930 return -ENODEV; 931 } 932 933 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); 934 /* 935 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP. 936 * RESUME consumption guarantees that the stalled transaction will be 937 * terminated... at some point in the future. PRI_RESP is fire and 938 * forget. 939 */ 940 941 return 0; 942 } 943 944 /* Context descriptor manipulation functions */ 945 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) 946 { 947 struct arm_smmu_cmdq_ent cmd = { 948 .opcode = smmu->features & ARM_SMMU_FEAT_E2H ? 949 CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID, 950 .tlbi.asid = asid, 951 }; 952 953 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 954 } 955 956 static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain, 957 int ssid, bool leaf) 958 { 959 size_t i; 960 unsigned long flags; 961 struct arm_smmu_master *master; 962 struct arm_smmu_cmdq_batch cmds; 963 struct arm_smmu_device *smmu = smmu_domain->smmu; 964 struct arm_smmu_cmdq_ent cmd = { 965 .opcode = CMDQ_OP_CFGI_CD, 966 .cfgi = { 967 .ssid = ssid, 968 .leaf = leaf, 969 }, 970 }; 971 972 cmds.num = 0; 973 974 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 975 list_for_each_entry(master, &smmu_domain->devices, domain_head) { 976 for (i = 0; i < master->num_streams; i++) { 977 cmd.cfgi.sid = master->streams[i].id; 978 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); 979 } 980 } 981 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 982 983 arm_smmu_cmdq_batch_submit(smmu, &cmds); 984 } 985 986 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu, 987 struct arm_smmu_l1_ctx_desc *l1_desc) 988 { 989 size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3); 990 991 l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, 992 &l1_desc->l2ptr_dma, GFP_KERNEL); 993 if (!l1_desc->l2ptr) { 994 dev_warn(smmu->dev, 995 "failed to allocate context descriptor table\n"); 996 return -ENOMEM; 997 } 998 return 0; 999 } 1000 1001 static void arm_smmu_write_cd_l1_desc(__le64 *dst, 1002 struct arm_smmu_l1_ctx_desc *l1_desc) 1003 { 1004 u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | 1005 CTXDESC_L1_DESC_V; 1006 1007 /* See comment in arm_smmu_write_ctx_desc() */ 1008 WRITE_ONCE(*dst, cpu_to_le64(val)); 1009 } 1010 1011 static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain, 1012 u32 ssid) 1013 { 1014 __le64 *l1ptr; 1015 unsigned int idx; 1016 struct arm_smmu_l1_ctx_desc *l1_desc; 1017 struct arm_smmu_device *smmu = smmu_domain->smmu; 1018 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg; 1019 1020 if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR) 1021 return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS; 1022 1023 idx = ssid >> CTXDESC_SPLIT; 1024 l1_desc = &cdcfg->l1_desc[idx]; 1025 if (!l1_desc->l2ptr) { 1026 if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) 1027 return NULL; 1028 1029 l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS; 1030 arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); 1031 /* An invalid L1CD can be cached */ 1032 arm_smmu_sync_cd(smmu_domain, ssid, false); 1033 } 1034 idx = ssid & (CTXDESC_L2_ENTRIES - 1); 1035 return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS; 1036 } 1037 1038 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid, 1039 struct arm_smmu_ctx_desc *cd) 1040 { 1041 /* 1042 * This function handles the following cases: 1043 * 1044 * (1) Install primary CD, for normal DMA traffic (SSID = 0). 1045 * (2) Install a secondary CD, for SID+SSID traffic. 1046 * (3) Update ASID of a CD. Atomically write the first 64 bits of the 1047 * CD, then invalidate the old entry and mappings. 1048 * (4) Quiesce the context without clearing the valid bit. Disable 1049 * translation, and ignore any translation fault. 1050 * (5) Remove a secondary CD. 1051 */ 1052 u64 val; 1053 bool cd_live; 1054 __le64 *cdptr; 1055 1056 if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax))) 1057 return -E2BIG; 1058 1059 cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid); 1060 if (!cdptr) 1061 return -ENOMEM; 1062 1063 val = le64_to_cpu(cdptr[0]); 1064 cd_live = !!(val & CTXDESC_CD_0_V); 1065 1066 if (!cd) { /* (5) */ 1067 val = 0; 1068 } else if (cd == &quiet_cd) { /* (4) */ 1069 val |= CTXDESC_CD_0_TCR_EPD0; 1070 } else if (cd_live) { /* (3) */ 1071 val &= ~CTXDESC_CD_0_ASID; 1072 val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid); 1073 /* 1074 * Until CD+TLB invalidation, both ASIDs may be used for tagging 1075 * this substream's traffic 1076 */ 1077 } else { /* (1) and (2) */ 1078 cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); 1079 cdptr[2] = 0; 1080 cdptr[3] = cpu_to_le64(cd->mair); 1081 1082 /* 1083 * STE is live, and the SMMU might read dwords of this CD in any 1084 * order. Ensure that it observes valid values before reading 1085 * V=1. 1086 */ 1087 arm_smmu_sync_cd(smmu_domain, ssid, true); 1088 1089 val = cd->tcr | 1090 #ifdef __BIG_ENDIAN 1091 CTXDESC_CD_0_ENDI | 1092 #endif 1093 CTXDESC_CD_0_R | CTXDESC_CD_0_A | 1094 (cd->mm ? 0 : CTXDESC_CD_0_ASET) | 1095 CTXDESC_CD_0_AA64 | 1096 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) | 1097 CTXDESC_CD_0_V; 1098 1099 if (smmu_domain->stall_enabled) 1100 val |= CTXDESC_CD_0_S; 1101 } 1102 1103 /* 1104 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3 1105 * "Configuration structures and configuration invalidation completion" 1106 * 1107 * The size of single-copy atomic reads made by the SMMU is 1108 * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single 1109 * field within an aligned 64-bit span of a structure can be altered 1110 * without first making the structure invalid. 1111 */ 1112 WRITE_ONCE(cdptr[0], cpu_to_le64(val)); 1113 arm_smmu_sync_cd(smmu_domain, ssid, true); 1114 return 0; 1115 } 1116 1117 static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain) 1118 { 1119 int ret; 1120 size_t l1size; 1121 size_t max_contexts; 1122 struct arm_smmu_device *smmu = smmu_domain->smmu; 1123 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; 1124 struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg; 1125 1126 max_contexts = 1 << cfg->s1cdmax; 1127 1128 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) || 1129 max_contexts <= CTXDESC_L2_ENTRIES) { 1130 cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR; 1131 cdcfg->num_l1_ents = max_contexts; 1132 1133 l1size = max_contexts * (CTXDESC_CD_DWORDS << 3); 1134 } else { 1135 cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2; 1136 cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts, 1137 CTXDESC_L2_ENTRIES); 1138 1139 cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents, 1140 sizeof(*cdcfg->l1_desc), 1141 GFP_KERNEL); 1142 if (!cdcfg->l1_desc) 1143 return -ENOMEM; 1144 1145 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3); 1146 } 1147 1148 cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma, 1149 GFP_KERNEL); 1150 if (!cdcfg->cdtab) { 1151 dev_warn(smmu->dev, "failed to allocate context descriptor\n"); 1152 ret = -ENOMEM; 1153 goto err_free_l1; 1154 } 1155 1156 return 0; 1157 1158 err_free_l1: 1159 if (cdcfg->l1_desc) { 1160 devm_kfree(smmu->dev, cdcfg->l1_desc); 1161 cdcfg->l1_desc = NULL; 1162 } 1163 return ret; 1164 } 1165 1166 static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain) 1167 { 1168 int i; 1169 size_t size, l1size; 1170 struct arm_smmu_device *smmu = smmu_domain->smmu; 1171 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg; 1172 1173 if (cdcfg->l1_desc) { 1174 size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3); 1175 1176 for (i = 0; i < cdcfg->num_l1_ents; i++) { 1177 if (!cdcfg->l1_desc[i].l2ptr) 1178 continue; 1179 1180 dmam_free_coherent(smmu->dev, size, 1181 cdcfg->l1_desc[i].l2ptr, 1182 cdcfg->l1_desc[i].l2ptr_dma); 1183 } 1184 devm_kfree(smmu->dev, cdcfg->l1_desc); 1185 cdcfg->l1_desc = NULL; 1186 1187 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3); 1188 } else { 1189 l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3); 1190 } 1191 1192 dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma); 1193 cdcfg->cdtab_dma = 0; 1194 cdcfg->cdtab = NULL; 1195 } 1196 1197 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd) 1198 { 1199 bool free; 1200 struct arm_smmu_ctx_desc *old_cd; 1201 1202 if (!cd->asid) 1203 return false; 1204 1205 free = refcount_dec_and_test(&cd->refs); 1206 if (free) { 1207 old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid); 1208 WARN_ON(old_cd != cd); 1209 } 1210 return free; 1211 } 1212 1213 /* Stream table manipulation functions */ 1214 static void 1215 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) 1216 { 1217 u64 val = 0; 1218 1219 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span); 1220 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; 1221 1222 /* See comment in arm_smmu_write_ctx_desc() */ 1223 WRITE_ONCE(*dst, cpu_to_le64(val)); 1224 } 1225 1226 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) 1227 { 1228 struct arm_smmu_cmdq_ent cmd = { 1229 .opcode = CMDQ_OP_CFGI_STE, 1230 .cfgi = { 1231 .sid = sid, 1232 .leaf = true, 1233 }, 1234 }; 1235 1236 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 1237 } 1238 1239 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, 1240 __le64 *dst) 1241 { 1242 /* 1243 * This is hideously complicated, but we only really care about 1244 * three cases at the moment: 1245 * 1246 * 1. Invalid (all zero) -> bypass/fault (init) 1247 * 2. Bypass/fault -> translation/bypass (attach) 1248 * 3. Translation/bypass -> bypass/fault (detach) 1249 * 1250 * Given that we can't update the STE atomically and the SMMU 1251 * doesn't read the thing in a defined order, that leaves us 1252 * with the following maintenance requirements: 1253 * 1254 * 1. Update Config, return (init time STEs aren't live) 1255 * 2. Write everything apart from dword 0, sync, write dword 0, sync 1256 * 3. Update Config, sync 1257 */ 1258 u64 val = le64_to_cpu(dst[0]); 1259 bool ste_live = false; 1260 struct arm_smmu_device *smmu = NULL; 1261 struct arm_smmu_s1_cfg *s1_cfg = NULL; 1262 struct arm_smmu_s2_cfg *s2_cfg = NULL; 1263 struct arm_smmu_domain *smmu_domain = NULL; 1264 struct arm_smmu_cmdq_ent prefetch_cmd = { 1265 .opcode = CMDQ_OP_PREFETCH_CFG, 1266 .prefetch = { 1267 .sid = sid, 1268 }, 1269 }; 1270 1271 if (master) { 1272 smmu_domain = master->domain; 1273 smmu = master->smmu; 1274 } 1275 1276 if (smmu_domain) { 1277 switch (smmu_domain->stage) { 1278 case ARM_SMMU_DOMAIN_S1: 1279 s1_cfg = &smmu_domain->s1_cfg; 1280 break; 1281 case ARM_SMMU_DOMAIN_S2: 1282 case ARM_SMMU_DOMAIN_NESTED: 1283 s2_cfg = &smmu_domain->s2_cfg; 1284 break; 1285 default: 1286 break; 1287 } 1288 } 1289 1290 if (val & STRTAB_STE_0_V) { 1291 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) { 1292 case STRTAB_STE_0_CFG_BYPASS: 1293 break; 1294 case STRTAB_STE_0_CFG_S1_TRANS: 1295 case STRTAB_STE_0_CFG_S2_TRANS: 1296 ste_live = true; 1297 break; 1298 case STRTAB_STE_0_CFG_ABORT: 1299 BUG_ON(!disable_bypass); 1300 break; 1301 default: 1302 BUG(); /* STE corruption */ 1303 } 1304 } 1305 1306 /* Nuke the existing STE_0 value, as we're going to rewrite it */ 1307 val = STRTAB_STE_0_V; 1308 1309 /* Bypass/fault */ 1310 if (!smmu_domain || !(s1_cfg || s2_cfg)) { 1311 if (!smmu_domain && disable_bypass) 1312 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); 1313 else 1314 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); 1315 1316 dst[0] = cpu_to_le64(val); 1317 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, 1318 STRTAB_STE_1_SHCFG_INCOMING)); 1319 dst[2] = 0; /* Nuke the VMID */ 1320 /* 1321 * The SMMU can perform negative caching, so we must sync 1322 * the STE regardless of whether the old value was live. 1323 */ 1324 if (smmu) 1325 arm_smmu_sync_ste_for_sid(smmu, sid); 1326 return; 1327 } 1328 1329 if (s1_cfg) { 1330 u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ? 1331 STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1; 1332 1333 BUG_ON(ste_live); 1334 dst[1] = cpu_to_le64( 1335 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | 1336 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | 1337 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | 1338 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | 1339 FIELD_PREP(STRTAB_STE_1_STRW, strw)); 1340 1341 if (smmu->features & ARM_SMMU_FEAT_STALLS && 1342 !master->stall_enabled) 1343 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); 1344 1345 val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) | 1346 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | 1347 FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) | 1348 FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt); 1349 } 1350 1351 if (s2_cfg) { 1352 BUG_ON(ste_live); 1353 dst[2] = cpu_to_le64( 1354 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | 1355 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) | 1356 #ifdef __BIG_ENDIAN 1357 STRTAB_STE_2_S2ENDI | 1358 #endif 1359 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | 1360 STRTAB_STE_2_S2R); 1361 1362 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); 1363 1364 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); 1365 } 1366 1367 if (master->ats_enabled) 1368 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS, 1369 STRTAB_STE_1_EATS_TRANS)); 1370 1371 arm_smmu_sync_ste_for_sid(smmu, sid); 1372 /* See comment in arm_smmu_write_ctx_desc() */ 1373 WRITE_ONCE(dst[0], cpu_to_le64(val)); 1374 arm_smmu_sync_ste_for_sid(smmu, sid); 1375 1376 /* It's likely that we'll want to use the new STE soon */ 1377 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) 1378 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); 1379 } 1380 1381 static void arm_smmu_init_bypass_stes(__le64 *strtab, unsigned int nent, bool force) 1382 { 1383 unsigned int i; 1384 u64 val = STRTAB_STE_0_V; 1385 1386 if (disable_bypass && !force) 1387 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); 1388 else 1389 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); 1390 1391 for (i = 0; i < nent; ++i) { 1392 strtab[0] = cpu_to_le64(val); 1393 strtab[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, 1394 STRTAB_STE_1_SHCFG_INCOMING)); 1395 strtab[2] = 0; 1396 strtab += STRTAB_STE_DWORDS; 1397 } 1398 } 1399 1400 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) 1401 { 1402 size_t size; 1403 void *strtab; 1404 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 1405 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; 1406 1407 if (desc->l2ptr) 1408 return 0; 1409 1410 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); 1411 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS]; 1412 1413 desc->span = STRTAB_SPLIT + 1; 1414 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma, 1415 GFP_KERNEL); 1416 if (!desc->l2ptr) { 1417 dev_err(smmu->dev, 1418 "failed to allocate l2 stream table for SID %u\n", 1419 sid); 1420 return -ENOMEM; 1421 } 1422 1423 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false); 1424 arm_smmu_write_strtab_l1_desc(strtab, desc); 1425 return 0; 1426 } 1427 1428 static struct arm_smmu_master * 1429 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) 1430 { 1431 struct rb_node *node; 1432 struct arm_smmu_stream *stream; 1433 1434 lockdep_assert_held(&smmu->streams_mutex); 1435 1436 node = smmu->streams.rb_node; 1437 while (node) { 1438 stream = rb_entry(node, struct arm_smmu_stream, node); 1439 if (stream->id < sid) 1440 node = node->rb_right; 1441 else if (stream->id > sid) 1442 node = node->rb_left; 1443 else 1444 return stream->master; 1445 } 1446 1447 return NULL; 1448 } 1449 1450 /* IRQ and event handlers */ 1451 static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) 1452 { 1453 int ret; 1454 u32 reason; 1455 u32 perm = 0; 1456 struct arm_smmu_master *master; 1457 bool ssid_valid = evt[0] & EVTQ_0_SSV; 1458 u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]); 1459 struct iommu_fault_event fault_evt = { }; 1460 struct iommu_fault *flt = &fault_evt.fault; 1461 1462 switch (FIELD_GET(EVTQ_0_ID, evt[0])) { 1463 case EVT_ID_TRANSLATION_FAULT: 1464 reason = IOMMU_FAULT_REASON_PTE_FETCH; 1465 break; 1466 case EVT_ID_ADDR_SIZE_FAULT: 1467 reason = IOMMU_FAULT_REASON_OOR_ADDRESS; 1468 break; 1469 case EVT_ID_ACCESS_FAULT: 1470 reason = IOMMU_FAULT_REASON_ACCESS; 1471 break; 1472 case EVT_ID_PERMISSION_FAULT: 1473 reason = IOMMU_FAULT_REASON_PERMISSION; 1474 break; 1475 default: 1476 return -EOPNOTSUPP; 1477 } 1478 1479 /* Stage-2 is always pinned at the moment */ 1480 if (evt[1] & EVTQ_1_S2) 1481 return -EFAULT; 1482 1483 if (evt[1] & EVTQ_1_RnW) 1484 perm |= IOMMU_FAULT_PERM_READ; 1485 else 1486 perm |= IOMMU_FAULT_PERM_WRITE; 1487 1488 if (evt[1] & EVTQ_1_InD) 1489 perm |= IOMMU_FAULT_PERM_EXEC; 1490 1491 if (evt[1] & EVTQ_1_PnU) 1492 perm |= IOMMU_FAULT_PERM_PRIV; 1493 1494 if (evt[1] & EVTQ_1_STALL) { 1495 flt->type = IOMMU_FAULT_PAGE_REQ; 1496 flt->prm = (struct iommu_fault_page_request) { 1497 .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE, 1498 .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]), 1499 .perm = perm, 1500 .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), 1501 }; 1502 1503 if (ssid_valid) { 1504 flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1505 flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); 1506 } 1507 } else { 1508 flt->type = IOMMU_FAULT_DMA_UNRECOV; 1509 flt->event = (struct iommu_fault_unrecoverable) { 1510 .reason = reason, 1511 .flags = IOMMU_FAULT_UNRECOV_ADDR_VALID, 1512 .perm = perm, 1513 .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), 1514 }; 1515 1516 if (ssid_valid) { 1517 flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID; 1518 flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); 1519 } 1520 } 1521 1522 mutex_lock(&smmu->streams_mutex); 1523 master = arm_smmu_find_master(smmu, sid); 1524 if (!master) { 1525 ret = -EINVAL; 1526 goto out_unlock; 1527 } 1528 1529 ret = iommu_report_device_fault(master->dev, &fault_evt); 1530 if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) { 1531 /* Nobody cared, abort the access */ 1532 struct iommu_page_response resp = { 1533 .pasid = flt->prm.pasid, 1534 .grpid = flt->prm.grpid, 1535 .code = IOMMU_PAGE_RESP_FAILURE, 1536 }; 1537 arm_smmu_page_response(master->dev, &fault_evt, &resp); 1538 } 1539 1540 out_unlock: 1541 mutex_unlock(&smmu->streams_mutex); 1542 return ret; 1543 } 1544 1545 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) 1546 { 1547 int i, ret; 1548 struct arm_smmu_device *smmu = dev; 1549 struct arm_smmu_queue *q = &smmu->evtq.q; 1550 struct arm_smmu_ll_queue *llq = &q->llq; 1551 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 1552 DEFAULT_RATELIMIT_BURST); 1553 u64 evt[EVTQ_ENT_DWORDS]; 1554 1555 do { 1556 while (!queue_remove_raw(q, evt)) { 1557 u8 id = FIELD_GET(EVTQ_0_ID, evt[0]); 1558 1559 ret = arm_smmu_handle_evt(smmu, evt); 1560 if (!ret || !__ratelimit(&rs)) 1561 continue; 1562 1563 dev_info(smmu->dev, "event 0x%02x received:\n", id); 1564 for (i = 0; i < ARRAY_SIZE(evt); ++i) 1565 dev_info(smmu->dev, "\t0x%016llx\n", 1566 (unsigned long long)evt[i]); 1567 1568 cond_resched(); 1569 } 1570 1571 /* 1572 * Not much we can do on overflow, so scream and pretend we're 1573 * trying harder. 1574 */ 1575 if (queue_sync_prod_in(q) == -EOVERFLOW) 1576 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n"); 1577 } while (!queue_empty(llq)); 1578 1579 /* Sync our overflow flag, as we believe we're up to speed */ 1580 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | 1581 Q_IDX(llq, llq->cons); 1582 return IRQ_HANDLED; 1583 } 1584 1585 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) 1586 { 1587 u32 sid, ssid; 1588 u16 grpid; 1589 bool ssv, last; 1590 1591 sid = FIELD_GET(PRIQ_0_SID, evt[0]); 1592 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]); 1593 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0; 1594 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]); 1595 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]); 1596 1597 dev_info(smmu->dev, "unexpected PRI request received:\n"); 1598 dev_info(smmu->dev, 1599 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n", 1600 sid, ssid, grpid, last ? "L" : "", 1601 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un", 1602 evt[0] & PRIQ_0_PERM_READ ? "R" : "", 1603 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "", 1604 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "", 1605 evt[1] & PRIQ_1_ADDR_MASK); 1606 1607 if (last) { 1608 struct arm_smmu_cmdq_ent cmd = { 1609 .opcode = CMDQ_OP_PRI_RESP, 1610 .substream_valid = ssv, 1611 .pri = { 1612 .sid = sid, 1613 .ssid = ssid, 1614 .grpid = grpid, 1615 .resp = PRI_RESP_DENY, 1616 }, 1617 }; 1618 1619 arm_smmu_cmdq_issue_cmd(smmu, &cmd); 1620 } 1621 } 1622 1623 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev) 1624 { 1625 struct arm_smmu_device *smmu = dev; 1626 struct arm_smmu_queue *q = &smmu->priq.q; 1627 struct arm_smmu_ll_queue *llq = &q->llq; 1628 u64 evt[PRIQ_ENT_DWORDS]; 1629 1630 do { 1631 while (!queue_remove_raw(q, evt)) 1632 arm_smmu_handle_ppr(smmu, evt); 1633 1634 if (queue_sync_prod_in(q) == -EOVERFLOW) 1635 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n"); 1636 } while (!queue_empty(llq)); 1637 1638 /* Sync our overflow flag, as we believe we're up to speed */ 1639 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | 1640 Q_IDX(llq, llq->cons); 1641 queue_sync_cons_out(q); 1642 return IRQ_HANDLED; 1643 } 1644 1645 static int arm_smmu_device_disable(struct arm_smmu_device *smmu); 1646 1647 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev) 1648 { 1649 u32 gerror, gerrorn, active; 1650 struct arm_smmu_device *smmu = dev; 1651 1652 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); 1653 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); 1654 1655 active = gerror ^ gerrorn; 1656 if (!(active & GERROR_ERR_MASK)) 1657 return IRQ_NONE; /* No errors pending */ 1658 1659 dev_warn(smmu->dev, 1660 "unexpected global error reported (0x%08x), this could be serious\n", 1661 active); 1662 1663 if (active & GERROR_SFM_ERR) { 1664 dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); 1665 arm_smmu_device_disable(smmu); 1666 } 1667 1668 if (active & GERROR_MSI_GERROR_ABT_ERR) 1669 dev_warn(smmu->dev, "GERROR MSI write aborted\n"); 1670 1671 if (active & GERROR_MSI_PRIQ_ABT_ERR) 1672 dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); 1673 1674 if (active & GERROR_MSI_EVTQ_ABT_ERR) 1675 dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); 1676 1677 if (active & GERROR_MSI_CMDQ_ABT_ERR) 1678 dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); 1679 1680 if (active & GERROR_PRIQ_ABT_ERR) 1681 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); 1682 1683 if (active & GERROR_EVTQ_ABT_ERR) 1684 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); 1685 1686 if (active & GERROR_CMDQ_ERR) 1687 arm_smmu_cmdq_skip_err(smmu); 1688 1689 writel(gerror, smmu->base + ARM_SMMU_GERRORN); 1690 return IRQ_HANDLED; 1691 } 1692 1693 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev) 1694 { 1695 struct arm_smmu_device *smmu = dev; 1696 1697 arm_smmu_evtq_thread(irq, dev); 1698 if (smmu->features & ARM_SMMU_FEAT_PRI) 1699 arm_smmu_priq_thread(irq, dev); 1700 1701 return IRQ_HANDLED; 1702 } 1703 1704 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) 1705 { 1706 arm_smmu_gerror_handler(irq, dev); 1707 return IRQ_WAKE_THREAD; 1708 } 1709 1710 static void 1711 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, 1712 struct arm_smmu_cmdq_ent *cmd) 1713 { 1714 size_t log2_span; 1715 size_t span_mask; 1716 /* ATC invalidates are always on 4096-bytes pages */ 1717 size_t inval_grain_shift = 12; 1718 unsigned long page_start, page_end; 1719 1720 /* 1721 * ATS and PASID: 1722 * 1723 * If substream_valid is clear, the PCIe TLP is sent without a PASID 1724 * prefix. In that case all ATC entries within the address range are 1725 * invalidated, including those that were requested with a PASID! There 1726 * is no way to invalidate only entries without PASID. 1727 * 1728 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID 1729 * traffic), translation requests without PASID create ATC entries 1730 * without PASID, which must be invalidated with substream_valid clear. 1731 * This has the unpleasant side-effect of invalidating all PASID-tagged 1732 * ATC entries within the address range. 1733 */ 1734 *cmd = (struct arm_smmu_cmdq_ent) { 1735 .opcode = CMDQ_OP_ATC_INV, 1736 .substream_valid = !!ssid, 1737 .atc.ssid = ssid, 1738 }; 1739 1740 if (!size) { 1741 cmd->atc.size = ATC_INV_SIZE_ALL; 1742 return; 1743 } 1744 1745 page_start = iova >> inval_grain_shift; 1746 page_end = (iova + size - 1) >> inval_grain_shift; 1747 1748 /* 1749 * In an ATS Invalidate Request, the address must be aligned on the 1750 * range size, which must be a power of two number of page sizes. We 1751 * thus have to choose between grossly over-invalidating the region, or 1752 * splitting the invalidation into multiple commands. For simplicity 1753 * we'll go with the first solution, but should refine it in the future 1754 * if multiple commands are shown to be more efficient. 1755 * 1756 * Find the smallest power of two that covers the range. The most 1757 * significant differing bit between the start and end addresses, 1758 * fls(start ^ end), indicates the required span. For example: 1759 * 1760 * We want to invalidate pages [8; 11]. This is already the ideal range: 1761 * x = 0b1000 ^ 0b1011 = 0b11 1762 * span = 1 << fls(x) = 4 1763 * 1764 * To invalidate pages [7; 10], we need to invalidate [0; 15]: 1765 * x = 0b0111 ^ 0b1010 = 0b1101 1766 * span = 1 << fls(x) = 16 1767 */ 1768 log2_span = fls_long(page_start ^ page_end); 1769 span_mask = (1ULL << log2_span) - 1; 1770 1771 page_start &= ~span_mask; 1772 1773 cmd->atc.addr = page_start << inval_grain_shift; 1774 cmd->atc.size = log2_span; 1775 } 1776 1777 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master) 1778 { 1779 int i; 1780 struct arm_smmu_cmdq_ent cmd; 1781 struct arm_smmu_cmdq_batch cmds; 1782 1783 arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd); 1784 1785 cmds.num = 0; 1786 for (i = 0; i < master->num_streams; i++) { 1787 cmd.atc.sid = master->streams[i].id; 1788 arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd); 1789 } 1790 1791 return arm_smmu_cmdq_batch_submit(master->smmu, &cmds); 1792 } 1793 1794 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, 1795 unsigned long iova, size_t size) 1796 { 1797 int i; 1798 unsigned long flags; 1799 struct arm_smmu_cmdq_ent cmd; 1800 struct arm_smmu_master *master; 1801 struct arm_smmu_cmdq_batch cmds; 1802 1803 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS)) 1804 return 0; 1805 1806 /* 1807 * Ensure that we've completed prior invalidation of the main TLBs 1808 * before we read 'nr_ats_masters' in case of a concurrent call to 1809 * arm_smmu_enable_ats(): 1810 * 1811 * // unmap() // arm_smmu_enable_ats() 1812 * TLBI+SYNC atomic_inc(&nr_ats_masters); 1813 * smp_mb(); [...] 1814 * atomic_read(&nr_ats_masters); pci_enable_ats() // writel() 1815 * 1816 * Ensures that we always see the incremented 'nr_ats_masters' count if 1817 * ATS was enabled at the PCI device before completion of the TLBI. 1818 */ 1819 smp_mb(); 1820 if (!atomic_read(&smmu_domain->nr_ats_masters)) 1821 return 0; 1822 1823 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd); 1824 1825 cmds.num = 0; 1826 1827 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 1828 list_for_each_entry(master, &smmu_domain->devices, domain_head) { 1829 if (!master->ats_enabled) 1830 continue; 1831 1832 for (i = 0; i < master->num_streams; i++) { 1833 cmd.atc.sid = master->streams[i].id; 1834 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd); 1835 } 1836 } 1837 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 1838 1839 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds); 1840 } 1841 1842 /* IO_PGTABLE API */ 1843 static void arm_smmu_tlb_inv_context(void *cookie) 1844 { 1845 struct arm_smmu_domain *smmu_domain = cookie; 1846 struct arm_smmu_device *smmu = smmu_domain->smmu; 1847 struct arm_smmu_cmdq_ent cmd; 1848 1849 /* 1850 * NOTE: when io-pgtable is in non-strict mode, we may get here with 1851 * PTEs previously cleared by unmaps on the current CPU not yet visible 1852 * to the SMMU. We are relying on the dma_wmb() implicit during cmd 1853 * insertion to guarantee those are observed before the TLBI. Do be 1854 * careful, 007. 1855 */ 1856 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 1857 arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid); 1858 } else { 1859 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL; 1860 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; 1861 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 1862 } 1863 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0); 1864 } 1865 1866 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, 1867 unsigned long iova, size_t size, 1868 size_t granule, 1869 struct arm_smmu_domain *smmu_domain) 1870 { 1871 struct arm_smmu_device *smmu = smmu_domain->smmu; 1872 unsigned long end = iova + size, num_pages = 0, tg = 0; 1873 size_t inv_range = granule; 1874 struct arm_smmu_cmdq_batch cmds; 1875 1876 if (!size) 1877 return; 1878 1879 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { 1880 /* Get the leaf page size */ 1881 tg = __ffs(smmu_domain->domain.pgsize_bitmap); 1882 1883 /* Convert page size of 12,14,16 (log2) to 1,2,3 */ 1884 cmd->tlbi.tg = (tg - 10) / 2; 1885 1886 /* Determine what level the granule is at */ 1887 cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); 1888 1889 num_pages = size >> tg; 1890 } 1891 1892 cmds.num = 0; 1893 1894 while (iova < end) { 1895 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { 1896 /* 1897 * On each iteration of the loop, the range is 5 bits 1898 * worth of the aligned size remaining. 1899 * The range in pages is: 1900 * 1901 * range = (num_pages & (0x1f << __ffs(num_pages))) 1902 */ 1903 unsigned long scale, num; 1904 1905 /* Determine the power of 2 multiple number of pages */ 1906 scale = __ffs(num_pages); 1907 cmd->tlbi.scale = scale; 1908 1909 /* Determine how many chunks of 2^scale size we have */ 1910 num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX; 1911 cmd->tlbi.num = num - 1; 1912 1913 /* range is num * 2^scale * pgsize */ 1914 inv_range = num << (scale + tg); 1915 1916 /* Clear out the lower order bits for the next iteration */ 1917 num_pages -= num << scale; 1918 } 1919 1920 cmd->tlbi.addr = iova; 1921 arm_smmu_cmdq_batch_add(smmu, &cmds, cmd); 1922 iova += inv_range; 1923 } 1924 arm_smmu_cmdq_batch_submit(smmu, &cmds); 1925 } 1926 1927 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size, 1928 size_t granule, bool leaf, 1929 struct arm_smmu_domain *smmu_domain) 1930 { 1931 struct arm_smmu_cmdq_ent cmd = { 1932 .tlbi = { 1933 .leaf = leaf, 1934 }, 1935 }; 1936 1937 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 1938 cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? 1939 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA; 1940 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; 1941 } else { 1942 cmd.opcode = CMDQ_OP_TLBI_S2_IPA; 1943 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; 1944 } 1945 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); 1946 1947 /* 1948 * Unfortunately, this can't be leaf-only since we may have 1949 * zapped an entire table. 1950 */ 1951 arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size); 1952 } 1953 1954 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, 1955 size_t granule, bool leaf, 1956 struct arm_smmu_domain *smmu_domain) 1957 { 1958 struct arm_smmu_cmdq_ent cmd = { 1959 .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? 1960 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA, 1961 .tlbi = { 1962 .asid = asid, 1963 .leaf = leaf, 1964 }, 1965 }; 1966 1967 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); 1968 } 1969 1970 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, 1971 unsigned long iova, size_t granule, 1972 void *cookie) 1973 { 1974 struct arm_smmu_domain *smmu_domain = cookie; 1975 struct iommu_domain *domain = &smmu_domain->domain; 1976 1977 iommu_iotlb_gather_add_page(domain, gather, iova, granule); 1978 } 1979 1980 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size, 1981 size_t granule, void *cookie) 1982 { 1983 arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie); 1984 } 1985 1986 static const struct iommu_flush_ops arm_smmu_flush_ops = { 1987 .tlb_flush_all = arm_smmu_tlb_inv_context, 1988 .tlb_flush_walk = arm_smmu_tlb_inv_walk, 1989 .tlb_add_page = arm_smmu_tlb_inv_page_nosync, 1990 }; 1991 1992 /* IOMMU API */ 1993 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) 1994 { 1995 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 1996 1997 switch (cap) { 1998 case IOMMU_CAP_CACHE_COHERENCY: 1999 /* Assume that a coherent TCU implies coherent TBUs */ 2000 return master->smmu->features & ARM_SMMU_FEAT_COHERENCY; 2001 case IOMMU_CAP_NOEXEC: 2002 return true; 2003 default: 2004 return false; 2005 } 2006 } 2007 2008 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) 2009 { 2010 struct arm_smmu_domain *smmu_domain; 2011 2012 if (type != IOMMU_DOMAIN_UNMANAGED && 2013 type != IOMMU_DOMAIN_DMA && 2014 type != IOMMU_DOMAIN_DMA_FQ && 2015 type != IOMMU_DOMAIN_IDENTITY) 2016 return NULL; 2017 2018 /* 2019 * Allocate the domain and initialise some of its data structures. 2020 * We can't really do anything meaningful until we've added a 2021 * master. 2022 */ 2023 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 2024 if (!smmu_domain) 2025 return NULL; 2026 2027 mutex_init(&smmu_domain->init_mutex); 2028 INIT_LIST_HEAD(&smmu_domain->devices); 2029 spin_lock_init(&smmu_domain->devices_lock); 2030 INIT_LIST_HEAD(&smmu_domain->mmu_notifiers); 2031 2032 return &smmu_domain->domain; 2033 } 2034 2035 static int arm_smmu_bitmap_alloc(unsigned long *map, int span) 2036 { 2037 int idx, size = 1 << span; 2038 2039 do { 2040 idx = find_first_zero_bit(map, size); 2041 if (idx == size) 2042 return -ENOSPC; 2043 } while (test_and_set_bit(idx, map)); 2044 2045 return idx; 2046 } 2047 2048 static void arm_smmu_bitmap_free(unsigned long *map, int idx) 2049 { 2050 clear_bit(idx, map); 2051 } 2052 2053 static void arm_smmu_domain_free(struct iommu_domain *domain) 2054 { 2055 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2056 struct arm_smmu_device *smmu = smmu_domain->smmu; 2057 2058 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 2059 2060 /* Free the CD and ASID, if we allocated them */ 2061 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 2062 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; 2063 2064 /* Prevent SVA from touching the CD while we're freeing it */ 2065 mutex_lock(&arm_smmu_asid_lock); 2066 if (cfg->cdcfg.cdtab) 2067 arm_smmu_free_cd_tables(smmu_domain); 2068 arm_smmu_free_asid(&cfg->cd); 2069 mutex_unlock(&arm_smmu_asid_lock); 2070 } else { 2071 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; 2072 if (cfg->vmid) 2073 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid); 2074 } 2075 2076 kfree(smmu_domain); 2077 } 2078 2079 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, 2080 struct arm_smmu_master *master, 2081 struct io_pgtable_cfg *pgtbl_cfg) 2082 { 2083 int ret; 2084 u32 asid; 2085 struct arm_smmu_device *smmu = smmu_domain->smmu; 2086 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; 2087 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr; 2088 2089 refcount_set(&cfg->cd.refs, 1); 2090 2091 /* Prevent SVA from modifying the ASID until it is written to the CD */ 2092 mutex_lock(&arm_smmu_asid_lock); 2093 ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd, 2094 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 2095 if (ret) 2096 goto out_unlock; 2097 2098 cfg->s1cdmax = master->ssid_bits; 2099 2100 smmu_domain->stall_enabled = master->stall_enabled; 2101 2102 ret = arm_smmu_alloc_cd_tables(smmu_domain); 2103 if (ret) 2104 goto out_free_asid; 2105 2106 cfg->cd.asid = (u16)asid; 2107 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 2108 cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) | 2109 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) | 2110 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) | 2111 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) | 2112 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) | 2113 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) | 2114 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; 2115 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair; 2116 2117 /* 2118 * Note that this will end up calling arm_smmu_sync_cd() before 2119 * the master has been added to the devices list for this domain. 2120 * This isn't an issue because the STE hasn't been installed yet. 2121 */ 2122 ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd); 2123 if (ret) 2124 goto out_free_cd_tables; 2125 2126 mutex_unlock(&arm_smmu_asid_lock); 2127 return 0; 2128 2129 out_free_cd_tables: 2130 arm_smmu_free_cd_tables(smmu_domain); 2131 out_free_asid: 2132 arm_smmu_free_asid(&cfg->cd); 2133 out_unlock: 2134 mutex_unlock(&arm_smmu_asid_lock); 2135 return ret; 2136 } 2137 2138 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, 2139 struct arm_smmu_master *master, 2140 struct io_pgtable_cfg *pgtbl_cfg) 2141 { 2142 int vmid; 2143 struct arm_smmu_device *smmu = smmu_domain->smmu; 2144 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; 2145 typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr; 2146 2147 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits); 2148 if (vmid < 0) 2149 return vmid; 2150 2151 vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr; 2152 cfg->vmid = (u16)vmid; 2153 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; 2154 cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) | 2155 FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) | 2156 FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) | 2157 FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) | 2158 FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) | 2159 FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) | 2160 FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps); 2161 return 0; 2162 } 2163 2164 static int arm_smmu_domain_finalise(struct iommu_domain *domain, 2165 struct arm_smmu_master *master) 2166 { 2167 int ret; 2168 unsigned long ias, oas; 2169 enum io_pgtable_fmt fmt; 2170 struct io_pgtable_cfg pgtbl_cfg; 2171 struct io_pgtable_ops *pgtbl_ops; 2172 int (*finalise_stage_fn)(struct arm_smmu_domain *, 2173 struct arm_smmu_master *, 2174 struct io_pgtable_cfg *); 2175 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2176 struct arm_smmu_device *smmu = smmu_domain->smmu; 2177 2178 if (domain->type == IOMMU_DOMAIN_IDENTITY) { 2179 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; 2180 return 0; 2181 } 2182 2183 /* Restrict the stage to what we can actually support */ 2184 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) 2185 smmu_domain->stage = ARM_SMMU_DOMAIN_S2; 2186 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 2187 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 2188 2189 switch (smmu_domain->stage) { 2190 case ARM_SMMU_DOMAIN_S1: 2191 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; 2192 ias = min_t(unsigned long, ias, VA_BITS); 2193 oas = smmu->ias; 2194 fmt = ARM_64_LPAE_S1; 2195 finalise_stage_fn = arm_smmu_domain_finalise_s1; 2196 break; 2197 case ARM_SMMU_DOMAIN_NESTED: 2198 case ARM_SMMU_DOMAIN_S2: 2199 ias = smmu->ias; 2200 oas = smmu->oas; 2201 fmt = ARM_64_LPAE_S2; 2202 finalise_stage_fn = arm_smmu_domain_finalise_s2; 2203 break; 2204 default: 2205 return -EINVAL; 2206 } 2207 2208 pgtbl_cfg = (struct io_pgtable_cfg) { 2209 .pgsize_bitmap = smmu->pgsize_bitmap, 2210 .ias = ias, 2211 .oas = oas, 2212 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, 2213 .tlb = &arm_smmu_flush_ops, 2214 .iommu_dev = smmu->dev, 2215 }; 2216 2217 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 2218 if (!pgtbl_ops) 2219 return -ENOMEM; 2220 2221 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 2222 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; 2223 domain->geometry.force_aperture = true; 2224 2225 ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg); 2226 if (ret < 0) { 2227 free_io_pgtable_ops(pgtbl_ops); 2228 return ret; 2229 } 2230 2231 smmu_domain->pgtbl_ops = pgtbl_ops; 2232 return 0; 2233 } 2234 2235 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) 2236 { 2237 __le64 *step; 2238 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 2239 2240 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { 2241 struct arm_smmu_strtab_l1_desc *l1_desc; 2242 int idx; 2243 2244 /* Two-level walk */ 2245 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS; 2246 l1_desc = &cfg->l1_desc[idx]; 2247 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS; 2248 step = &l1_desc->l2ptr[idx]; 2249 } else { 2250 /* Simple linear lookup */ 2251 step = &cfg->strtab[sid * STRTAB_STE_DWORDS]; 2252 } 2253 2254 return step; 2255 } 2256 2257 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master) 2258 { 2259 int i, j; 2260 struct arm_smmu_device *smmu = master->smmu; 2261 2262 for (i = 0; i < master->num_streams; ++i) { 2263 u32 sid = master->streams[i].id; 2264 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); 2265 2266 /* Bridged PCI devices may end up with duplicated IDs */ 2267 for (j = 0; j < i; j++) 2268 if (master->streams[j].id == sid) 2269 break; 2270 if (j < i) 2271 continue; 2272 2273 arm_smmu_write_strtab_ent(master, sid, step); 2274 } 2275 } 2276 2277 static bool arm_smmu_ats_supported(struct arm_smmu_master *master) 2278 { 2279 struct device *dev = master->dev; 2280 struct arm_smmu_device *smmu = master->smmu; 2281 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2282 2283 if (!(smmu->features & ARM_SMMU_FEAT_ATS)) 2284 return false; 2285 2286 if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS)) 2287 return false; 2288 2289 return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev)); 2290 } 2291 2292 static void arm_smmu_enable_ats(struct arm_smmu_master *master) 2293 { 2294 size_t stu; 2295 struct pci_dev *pdev; 2296 struct arm_smmu_device *smmu = master->smmu; 2297 struct arm_smmu_domain *smmu_domain = master->domain; 2298 2299 /* Don't enable ATS at the endpoint if it's not enabled in the STE */ 2300 if (!master->ats_enabled) 2301 return; 2302 2303 /* Smallest Translation Unit: log2 of the smallest supported granule */ 2304 stu = __ffs(smmu->pgsize_bitmap); 2305 pdev = to_pci_dev(master->dev); 2306 2307 atomic_inc(&smmu_domain->nr_ats_masters); 2308 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0); 2309 if (pci_enable_ats(pdev, stu)) 2310 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu); 2311 } 2312 2313 static void arm_smmu_disable_ats(struct arm_smmu_master *master) 2314 { 2315 struct arm_smmu_domain *smmu_domain = master->domain; 2316 2317 if (!master->ats_enabled) 2318 return; 2319 2320 pci_disable_ats(to_pci_dev(master->dev)); 2321 /* 2322 * Ensure ATS is disabled at the endpoint before we issue the 2323 * ATC invalidation via the SMMU. 2324 */ 2325 wmb(); 2326 arm_smmu_atc_inv_master(master); 2327 atomic_dec(&smmu_domain->nr_ats_masters); 2328 } 2329 2330 static int arm_smmu_enable_pasid(struct arm_smmu_master *master) 2331 { 2332 int ret; 2333 int features; 2334 int num_pasids; 2335 struct pci_dev *pdev; 2336 2337 if (!dev_is_pci(master->dev)) 2338 return -ENODEV; 2339 2340 pdev = to_pci_dev(master->dev); 2341 2342 features = pci_pasid_features(pdev); 2343 if (features < 0) 2344 return features; 2345 2346 num_pasids = pci_max_pasids(pdev); 2347 if (num_pasids <= 0) 2348 return num_pasids; 2349 2350 ret = pci_enable_pasid(pdev, features); 2351 if (ret) { 2352 dev_err(&pdev->dev, "Failed to enable PASID\n"); 2353 return ret; 2354 } 2355 2356 master->ssid_bits = min_t(u8, ilog2(num_pasids), 2357 master->smmu->ssid_bits); 2358 return 0; 2359 } 2360 2361 static void arm_smmu_disable_pasid(struct arm_smmu_master *master) 2362 { 2363 struct pci_dev *pdev; 2364 2365 if (!dev_is_pci(master->dev)) 2366 return; 2367 2368 pdev = to_pci_dev(master->dev); 2369 2370 if (!pdev->pasid_enabled) 2371 return; 2372 2373 master->ssid_bits = 0; 2374 pci_disable_pasid(pdev); 2375 } 2376 2377 static void arm_smmu_detach_dev(struct arm_smmu_master *master) 2378 { 2379 unsigned long flags; 2380 struct arm_smmu_domain *smmu_domain = master->domain; 2381 2382 if (!smmu_domain) 2383 return; 2384 2385 arm_smmu_disable_ats(master); 2386 2387 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2388 list_del(&master->domain_head); 2389 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 2390 2391 master->domain = NULL; 2392 master->ats_enabled = false; 2393 arm_smmu_install_ste_for_dev(master); 2394 } 2395 2396 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 2397 { 2398 int ret = 0; 2399 unsigned long flags; 2400 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2401 struct arm_smmu_device *smmu; 2402 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2403 struct arm_smmu_master *master; 2404 2405 if (!fwspec) 2406 return -ENOENT; 2407 2408 master = dev_iommu_priv_get(dev); 2409 smmu = master->smmu; 2410 2411 /* 2412 * Checking that SVA is disabled ensures that this device isn't bound to 2413 * any mm, and can be safely detached from its old domain. Bonds cannot 2414 * be removed concurrently since we're holding the group mutex. 2415 */ 2416 if (arm_smmu_master_sva_enabled(master)) { 2417 dev_err(dev, "cannot attach - SVA enabled\n"); 2418 return -EBUSY; 2419 } 2420 2421 arm_smmu_detach_dev(master); 2422 2423 mutex_lock(&smmu_domain->init_mutex); 2424 2425 if (!smmu_domain->smmu) { 2426 smmu_domain->smmu = smmu; 2427 ret = arm_smmu_domain_finalise(domain, master); 2428 if (ret) { 2429 smmu_domain->smmu = NULL; 2430 goto out_unlock; 2431 } 2432 } else if (smmu_domain->smmu != smmu) { 2433 dev_err(dev, 2434 "cannot attach to SMMU %s (upstream of %s)\n", 2435 dev_name(smmu_domain->smmu->dev), 2436 dev_name(smmu->dev)); 2437 ret = -ENXIO; 2438 goto out_unlock; 2439 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && 2440 master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) { 2441 dev_err(dev, 2442 "cannot attach to incompatible domain (%u SSID bits != %u)\n", 2443 smmu_domain->s1_cfg.s1cdmax, master->ssid_bits); 2444 ret = -EINVAL; 2445 goto out_unlock; 2446 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && 2447 smmu_domain->stall_enabled != master->stall_enabled) { 2448 dev_err(dev, "cannot attach to stall-%s domain\n", 2449 smmu_domain->stall_enabled ? "enabled" : "disabled"); 2450 ret = -EINVAL; 2451 goto out_unlock; 2452 } 2453 2454 master->domain = smmu_domain; 2455 2456 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS) 2457 master->ats_enabled = arm_smmu_ats_supported(master); 2458 2459 arm_smmu_install_ste_for_dev(master); 2460 2461 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2462 list_add(&master->domain_head, &smmu_domain->devices); 2463 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 2464 2465 arm_smmu_enable_ats(master); 2466 2467 out_unlock: 2468 mutex_unlock(&smmu_domain->init_mutex); 2469 return ret; 2470 } 2471 2472 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, 2473 phys_addr_t paddr, size_t pgsize, size_t pgcount, 2474 int prot, gfp_t gfp, size_t *mapped) 2475 { 2476 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 2477 2478 if (!ops) 2479 return -ENODEV; 2480 2481 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped); 2482 } 2483 2484 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, 2485 size_t pgsize, size_t pgcount, 2486 struct iommu_iotlb_gather *gather) 2487 { 2488 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2489 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 2490 2491 if (!ops) 2492 return 0; 2493 2494 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); 2495 } 2496 2497 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) 2498 { 2499 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2500 2501 if (smmu_domain->smmu) 2502 arm_smmu_tlb_inv_context(smmu_domain); 2503 } 2504 2505 static void arm_smmu_iotlb_sync(struct iommu_domain *domain, 2506 struct iommu_iotlb_gather *gather) 2507 { 2508 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2509 2510 if (!gather->pgsize) 2511 return; 2512 2513 arm_smmu_tlb_inv_range_domain(gather->start, 2514 gather->end - gather->start + 1, 2515 gather->pgsize, true, smmu_domain); 2516 } 2517 2518 static phys_addr_t 2519 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 2520 { 2521 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 2522 2523 if (!ops) 2524 return 0; 2525 2526 return ops->iova_to_phys(ops, iova); 2527 } 2528 2529 static struct platform_driver arm_smmu_driver; 2530 2531 static 2532 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) 2533 { 2534 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver, 2535 fwnode); 2536 put_device(dev); 2537 return dev ? dev_get_drvdata(dev) : NULL; 2538 } 2539 2540 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) 2541 { 2542 unsigned long limit = smmu->strtab_cfg.num_l1_ents; 2543 2544 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) 2545 limit *= 1UL << STRTAB_SPLIT; 2546 2547 return sid < limit; 2548 } 2549 2550 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid) 2551 { 2552 /* Check the SIDs are in range of the SMMU and our stream table */ 2553 if (!arm_smmu_sid_in_range(smmu, sid)) 2554 return -ERANGE; 2555 2556 /* Ensure l2 strtab is initialised */ 2557 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) 2558 return arm_smmu_init_l2_strtab(smmu, sid); 2559 2560 return 0; 2561 } 2562 2563 static int arm_smmu_insert_master(struct arm_smmu_device *smmu, 2564 struct arm_smmu_master *master) 2565 { 2566 int i; 2567 int ret = 0; 2568 struct arm_smmu_stream *new_stream, *cur_stream; 2569 struct rb_node **new_node, *parent_node = NULL; 2570 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); 2571 2572 master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams), 2573 GFP_KERNEL); 2574 if (!master->streams) 2575 return -ENOMEM; 2576 master->num_streams = fwspec->num_ids; 2577 2578 mutex_lock(&smmu->streams_mutex); 2579 for (i = 0; i < fwspec->num_ids; i++) { 2580 u32 sid = fwspec->ids[i]; 2581 2582 new_stream = &master->streams[i]; 2583 new_stream->id = sid; 2584 new_stream->master = master; 2585 2586 ret = arm_smmu_init_sid_strtab(smmu, sid); 2587 if (ret) 2588 break; 2589 2590 /* Insert into SID tree */ 2591 new_node = &(smmu->streams.rb_node); 2592 while (*new_node) { 2593 cur_stream = rb_entry(*new_node, struct arm_smmu_stream, 2594 node); 2595 parent_node = *new_node; 2596 if (cur_stream->id > new_stream->id) { 2597 new_node = &((*new_node)->rb_left); 2598 } else if (cur_stream->id < new_stream->id) { 2599 new_node = &((*new_node)->rb_right); 2600 } else { 2601 dev_warn(master->dev, 2602 "stream %u already in tree\n", 2603 cur_stream->id); 2604 ret = -EINVAL; 2605 break; 2606 } 2607 } 2608 if (ret) 2609 break; 2610 2611 rb_link_node(&new_stream->node, parent_node, new_node); 2612 rb_insert_color(&new_stream->node, &smmu->streams); 2613 } 2614 2615 if (ret) { 2616 for (i--; i >= 0; i--) 2617 rb_erase(&master->streams[i].node, &smmu->streams); 2618 kfree(master->streams); 2619 } 2620 mutex_unlock(&smmu->streams_mutex); 2621 2622 return ret; 2623 } 2624 2625 static void arm_smmu_remove_master(struct arm_smmu_master *master) 2626 { 2627 int i; 2628 struct arm_smmu_device *smmu = master->smmu; 2629 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); 2630 2631 if (!smmu || !master->streams) 2632 return; 2633 2634 mutex_lock(&smmu->streams_mutex); 2635 for (i = 0; i < fwspec->num_ids; i++) 2636 rb_erase(&master->streams[i].node, &smmu->streams); 2637 mutex_unlock(&smmu->streams_mutex); 2638 2639 kfree(master->streams); 2640 } 2641 2642 static struct iommu_ops arm_smmu_ops; 2643 2644 static struct iommu_device *arm_smmu_probe_device(struct device *dev) 2645 { 2646 int ret; 2647 struct arm_smmu_device *smmu; 2648 struct arm_smmu_master *master; 2649 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2650 2651 if (!fwspec || fwspec->ops != &arm_smmu_ops) 2652 return ERR_PTR(-ENODEV); 2653 2654 if (WARN_ON_ONCE(dev_iommu_priv_get(dev))) 2655 return ERR_PTR(-EBUSY); 2656 2657 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); 2658 if (!smmu) 2659 return ERR_PTR(-ENODEV); 2660 2661 master = kzalloc(sizeof(*master), GFP_KERNEL); 2662 if (!master) 2663 return ERR_PTR(-ENOMEM); 2664 2665 master->dev = dev; 2666 master->smmu = smmu; 2667 INIT_LIST_HEAD(&master->bonds); 2668 dev_iommu_priv_set(dev, master); 2669 2670 ret = arm_smmu_insert_master(smmu, master); 2671 if (ret) 2672 goto err_free_master; 2673 2674 device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits); 2675 master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits); 2676 2677 /* 2678 * Note that PASID must be enabled before, and disabled after ATS: 2679 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register 2680 * 2681 * Behavior is undefined if this bit is Set and the value of the PASID 2682 * Enable, Execute Requested Enable, or Privileged Mode Requested bits 2683 * are changed. 2684 */ 2685 arm_smmu_enable_pasid(master); 2686 2687 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB)) 2688 master->ssid_bits = min_t(u8, master->ssid_bits, 2689 CTXDESC_LINEAR_CDMAX); 2690 2691 if ((smmu->features & ARM_SMMU_FEAT_STALLS && 2692 device_property_read_bool(dev, "dma-can-stall")) || 2693 smmu->features & ARM_SMMU_FEAT_STALL_FORCE) 2694 master->stall_enabled = true; 2695 2696 return &smmu->iommu; 2697 2698 err_free_master: 2699 kfree(master); 2700 dev_iommu_priv_set(dev, NULL); 2701 return ERR_PTR(ret); 2702 } 2703 2704 static void arm_smmu_release_device(struct device *dev) 2705 { 2706 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2707 2708 if (WARN_ON(arm_smmu_master_sva_enabled(master))) 2709 iopf_queue_remove_device(master->smmu->evtq.iopf, dev); 2710 arm_smmu_detach_dev(master); 2711 arm_smmu_disable_pasid(master); 2712 arm_smmu_remove_master(master); 2713 kfree(master); 2714 } 2715 2716 static struct iommu_group *arm_smmu_device_group(struct device *dev) 2717 { 2718 struct iommu_group *group; 2719 2720 /* 2721 * We don't support devices sharing stream IDs other than PCI RID 2722 * aliases, since the necessary ID-to-device lookup becomes rather 2723 * impractical given a potential sparse 32-bit stream ID space. 2724 */ 2725 if (dev_is_pci(dev)) 2726 group = pci_device_group(dev); 2727 else 2728 group = generic_device_group(dev); 2729 2730 return group; 2731 } 2732 2733 static int arm_smmu_enable_nesting(struct iommu_domain *domain) 2734 { 2735 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2736 int ret = 0; 2737 2738 mutex_lock(&smmu_domain->init_mutex); 2739 if (smmu_domain->smmu) 2740 ret = -EPERM; 2741 else 2742 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; 2743 mutex_unlock(&smmu_domain->init_mutex); 2744 2745 return ret; 2746 } 2747 2748 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) 2749 { 2750 return iommu_fwspec_add_ids(dev, args->args, 1); 2751 } 2752 2753 static void arm_smmu_get_resv_regions(struct device *dev, 2754 struct list_head *head) 2755 { 2756 struct iommu_resv_region *region; 2757 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 2758 2759 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, 2760 prot, IOMMU_RESV_SW_MSI); 2761 if (!region) 2762 return; 2763 2764 list_add_tail(®ion->list, head); 2765 2766 iommu_dma_get_resv_regions(dev, head); 2767 } 2768 2769 static int arm_smmu_dev_enable_feature(struct device *dev, 2770 enum iommu_dev_features feat) 2771 { 2772 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2773 2774 if (!master) 2775 return -ENODEV; 2776 2777 switch (feat) { 2778 case IOMMU_DEV_FEAT_IOPF: 2779 if (!arm_smmu_master_iopf_supported(master)) 2780 return -EINVAL; 2781 if (master->iopf_enabled) 2782 return -EBUSY; 2783 master->iopf_enabled = true; 2784 return 0; 2785 case IOMMU_DEV_FEAT_SVA: 2786 if (!arm_smmu_master_sva_supported(master)) 2787 return -EINVAL; 2788 if (arm_smmu_master_sva_enabled(master)) 2789 return -EBUSY; 2790 return arm_smmu_master_enable_sva(master); 2791 default: 2792 return -EINVAL; 2793 } 2794 } 2795 2796 static int arm_smmu_dev_disable_feature(struct device *dev, 2797 enum iommu_dev_features feat) 2798 { 2799 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2800 2801 if (!master) 2802 return -EINVAL; 2803 2804 switch (feat) { 2805 case IOMMU_DEV_FEAT_IOPF: 2806 if (!master->iopf_enabled) 2807 return -EINVAL; 2808 if (master->sva_enabled) 2809 return -EBUSY; 2810 master->iopf_enabled = false; 2811 return 0; 2812 case IOMMU_DEV_FEAT_SVA: 2813 if (!arm_smmu_master_sva_enabled(master)) 2814 return -EINVAL; 2815 return arm_smmu_master_disable_sva(master); 2816 default: 2817 return -EINVAL; 2818 } 2819 } 2820 2821 /* 2822 * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the 2823 * PCIe link and save the data to memory by DMA. The hardware is restricted to 2824 * use identity mapping only. 2825 */ 2826 #define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \ 2827 (pdev)->device == 0xa12e) 2828 2829 static int arm_smmu_def_domain_type(struct device *dev) 2830 { 2831 if (dev_is_pci(dev)) { 2832 struct pci_dev *pdev = to_pci_dev(dev); 2833 2834 if (IS_HISI_PTT_DEVICE(pdev)) 2835 return IOMMU_DOMAIN_IDENTITY; 2836 } 2837 2838 return 0; 2839 } 2840 2841 static struct iommu_ops arm_smmu_ops = { 2842 .capable = arm_smmu_capable, 2843 .domain_alloc = arm_smmu_domain_alloc, 2844 .probe_device = arm_smmu_probe_device, 2845 .release_device = arm_smmu_release_device, 2846 .device_group = arm_smmu_device_group, 2847 .of_xlate = arm_smmu_of_xlate, 2848 .get_resv_regions = arm_smmu_get_resv_regions, 2849 .dev_enable_feat = arm_smmu_dev_enable_feature, 2850 .dev_disable_feat = arm_smmu_dev_disable_feature, 2851 .sva_bind = arm_smmu_sva_bind, 2852 .sva_unbind = arm_smmu_sva_unbind, 2853 .sva_get_pasid = arm_smmu_sva_get_pasid, 2854 .page_response = arm_smmu_page_response, 2855 .def_domain_type = arm_smmu_def_domain_type, 2856 .pgsize_bitmap = -1UL, /* Restricted during device attach */ 2857 .owner = THIS_MODULE, 2858 .default_domain_ops = &(const struct iommu_domain_ops) { 2859 .attach_dev = arm_smmu_attach_dev, 2860 .map_pages = arm_smmu_map_pages, 2861 .unmap_pages = arm_smmu_unmap_pages, 2862 .flush_iotlb_all = arm_smmu_flush_iotlb_all, 2863 .iotlb_sync = arm_smmu_iotlb_sync, 2864 .iova_to_phys = arm_smmu_iova_to_phys, 2865 .enable_nesting = arm_smmu_enable_nesting, 2866 .free = arm_smmu_domain_free, 2867 } 2868 }; 2869 2870 /* Probing and initialisation functions */ 2871 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, 2872 struct arm_smmu_queue *q, 2873 void __iomem *page, 2874 unsigned long prod_off, 2875 unsigned long cons_off, 2876 size_t dwords, const char *name) 2877 { 2878 size_t qsz; 2879 2880 do { 2881 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3; 2882 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, 2883 GFP_KERNEL); 2884 if (q->base || qsz < PAGE_SIZE) 2885 break; 2886 2887 q->llq.max_n_shift--; 2888 } while (1); 2889 2890 if (!q->base) { 2891 dev_err(smmu->dev, 2892 "failed to allocate queue (0x%zx bytes) for %s\n", 2893 qsz, name); 2894 return -ENOMEM; 2895 } 2896 2897 if (!WARN_ON(q->base_dma & (qsz - 1))) { 2898 dev_info(smmu->dev, "allocated %u entries for %s\n", 2899 1 << q->llq.max_n_shift, name); 2900 } 2901 2902 q->prod_reg = page + prod_off; 2903 q->cons_reg = page + cons_off; 2904 q->ent_dwords = dwords; 2905 2906 q->q_base = Q_BASE_RWA; 2907 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK; 2908 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift); 2909 2910 q->llq.prod = q->llq.cons = 0; 2911 return 0; 2912 } 2913 2914 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu) 2915 { 2916 struct arm_smmu_cmdq *cmdq = &smmu->cmdq; 2917 unsigned int nents = 1 << cmdq->q.llq.max_n_shift; 2918 2919 atomic_set(&cmdq->owner_prod, 0); 2920 atomic_set(&cmdq->lock, 0); 2921 2922 cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents, 2923 GFP_KERNEL); 2924 if (!cmdq->valid_map) 2925 return -ENOMEM; 2926 2927 return 0; 2928 } 2929 2930 static int arm_smmu_init_queues(struct arm_smmu_device *smmu) 2931 { 2932 int ret; 2933 2934 /* cmdq */ 2935 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base, 2936 ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS, 2937 CMDQ_ENT_DWORDS, "cmdq"); 2938 if (ret) 2939 return ret; 2940 2941 ret = arm_smmu_cmdq_init(smmu); 2942 if (ret) 2943 return ret; 2944 2945 /* evtq */ 2946 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1, 2947 ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS, 2948 EVTQ_ENT_DWORDS, "evtq"); 2949 if (ret) 2950 return ret; 2951 2952 if ((smmu->features & ARM_SMMU_FEAT_SVA) && 2953 (smmu->features & ARM_SMMU_FEAT_STALLS)) { 2954 smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev)); 2955 if (!smmu->evtq.iopf) 2956 return -ENOMEM; 2957 } 2958 2959 /* priq */ 2960 if (!(smmu->features & ARM_SMMU_FEAT_PRI)) 2961 return 0; 2962 2963 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1, 2964 ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS, 2965 PRIQ_ENT_DWORDS, "priq"); 2966 } 2967 2968 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) 2969 { 2970 unsigned int i; 2971 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 2972 void *strtab = smmu->strtab_cfg.strtab; 2973 2974 cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents, 2975 sizeof(*cfg->l1_desc), GFP_KERNEL); 2976 if (!cfg->l1_desc) 2977 return -ENOMEM; 2978 2979 for (i = 0; i < cfg->num_l1_ents; ++i) { 2980 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]); 2981 strtab += STRTAB_L1_DESC_DWORDS << 3; 2982 } 2983 2984 return 0; 2985 } 2986 2987 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) 2988 { 2989 void *strtab; 2990 u64 reg; 2991 u32 size, l1size; 2992 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 2993 2994 /* Calculate the L1 size, capped to the SIDSIZE. */ 2995 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); 2996 size = min(size, smmu->sid_bits - STRTAB_SPLIT); 2997 cfg->num_l1_ents = 1 << size; 2998 2999 size += STRTAB_SPLIT; 3000 if (size < smmu->sid_bits) 3001 dev_warn(smmu->dev, 3002 "2-level strtab only covers %u/%u bits of SID\n", 3003 size, smmu->sid_bits); 3004 3005 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3); 3006 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, 3007 GFP_KERNEL); 3008 if (!strtab) { 3009 dev_err(smmu->dev, 3010 "failed to allocate l1 stream table (%u bytes)\n", 3011 l1size); 3012 return -ENOMEM; 3013 } 3014 cfg->strtab = strtab; 3015 3016 /* Configure strtab_base_cfg for 2 levels */ 3017 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL); 3018 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size); 3019 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT); 3020 cfg->strtab_base_cfg = reg; 3021 3022 return arm_smmu_init_l1_strtab(smmu); 3023 } 3024 3025 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) 3026 { 3027 void *strtab; 3028 u64 reg; 3029 u32 size; 3030 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 3031 3032 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3); 3033 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma, 3034 GFP_KERNEL); 3035 if (!strtab) { 3036 dev_err(smmu->dev, 3037 "failed to allocate linear stream table (%u bytes)\n", 3038 size); 3039 return -ENOMEM; 3040 } 3041 cfg->strtab = strtab; 3042 cfg->num_l1_ents = 1 << smmu->sid_bits; 3043 3044 /* Configure strtab_base_cfg for a linear table covering all SIDs */ 3045 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR); 3046 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); 3047 cfg->strtab_base_cfg = reg; 3048 3049 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false); 3050 return 0; 3051 } 3052 3053 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) 3054 { 3055 u64 reg; 3056 int ret; 3057 3058 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) 3059 ret = arm_smmu_init_strtab_2lvl(smmu); 3060 else 3061 ret = arm_smmu_init_strtab_linear(smmu); 3062 3063 if (ret) 3064 return ret; 3065 3066 /* Set the strtab base address */ 3067 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK; 3068 reg |= STRTAB_BASE_RA; 3069 smmu->strtab_cfg.strtab_base = reg; 3070 3071 /* Allocate the first VMID for stage-2 bypass STEs */ 3072 set_bit(0, smmu->vmid_map); 3073 return 0; 3074 } 3075 3076 static int arm_smmu_init_structures(struct arm_smmu_device *smmu) 3077 { 3078 int ret; 3079 3080 mutex_init(&smmu->streams_mutex); 3081 smmu->streams = RB_ROOT; 3082 3083 ret = arm_smmu_init_queues(smmu); 3084 if (ret) 3085 return ret; 3086 3087 return arm_smmu_init_strtab(smmu); 3088 } 3089 3090 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, 3091 unsigned int reg_off, unsigned int ack_off) 3092 { 3093 u32 reg; 3094 3095 writel_relaxed(val, smmu->base + reg_off); 3096 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, 3097 1, ARM_SMMU_POLL_TIMEOUT_US); 3098 } 3099 3100 /* GBPA is "special" */ 3101 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr) 3102 { 3103 int ret; 3104 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA; 3105 3106 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), 3107 1, ARM_SMMU_POLL_TIMEOUT_US); 3108 if (ret) 3109 return ret; 3110 3111 reg &= ~clr; 3112 reg |= set; 3113 writel_relaxed(reg | GBPA_UPDATE, gbpa); 3114 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), 3115 1, ARM_SMMU_POLL_TIMEOUT_US); 3116 3117 if (ret) 3118 dev_err(smmu->dev, "GBPA not responding to update\n"); 3119 return ret; 3120 } 3121 3122 static void arm_smmu_free_msis(void *data) 3123 { 3124 struct device *dev = data; 3125 platform_msi_domain_free_irqs(dev); 3126 } 3127 3128 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) 3129 { 3130 phys_addr_t doorbell; 3131 struct device *dev = msi_desc_to_dev(desc); 3132 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 3133 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index]; 3134 3135 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; 3136 doorbell &= MSI_CFG0_ADDR_MASK; 3137 3138 writeq_relaxed(doorbell, smmu->base + cfg[0]); 3139 writel_relaxed(msg->data, smmu->base + cfg[1]); 3140 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); 3141 } 3142 3143 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) 3144 { 3145 int ret, nvec = ARM_SMMU_MAX_MSIS; 3146 struct device *dev = smmu->dev; 3147 3148 /* Clear the MSI address regs */ 3149 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0); 3150 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0); 3151 3152 if (smmu->features & ARM_SMMU_FEAT_PRI) 3153 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0); 3154 else 3155 nvec--; 3156 3157 if (!(smmu->features & ARM_SMMU_FEAT_MSI)) 3158 return; 3159 3160 if (!dev->msi.domain) { 3161 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n"); 3162 return; 3163 } 3164 3165 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */ 3166 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg); 3167 if (ret) { 3168 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n"); 3169 return; 3170 } 3171 3172 smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX); 3173 smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX); 3174 smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX); 3175 3176 /* Add callback to free MSIs on teardown */ 3177 devm_add_action(dev, arm_smmu_free_msis, dev); 3178 } 3179 3180 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) 3181 { 3182 int irq, ret; 3183 3184 arm_smmu_setup_msis(smmu); 3185 3186 /* Request interrupt lines */ 3187 irq = smmu->evtq.q.irq; 3188 if (irq) { 3189 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, 3190 arm_smmu_evtq_thread, 3191 IRQF_ONESHOT, 3192 "arm-smmu-v3-evtq", smmu); 3193 if (ret < 0) 3194 dev_warn(smmu->dev, "failed to enable evtq irq\n"); 3195 } else { 3196 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n"); 3197 } 3198 3199 irq = smmu->gerr_irq; 3200 if (irq) { 3201 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, 3202 0, "arm-smmu-v3-gerror", smmu); 3203 if (ret < 0) 3204 dev_warn(smmu->dev, "failed to enable gerror irq\n"); 3205 } else { 3206 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n"); 3207 } 3208 3209 if (smmu->features & ARM_SMMU_FEAT_PRI) { 3210 irq = smmu->priq.q.irq; 3211 if (irq) { 3212 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, 3213 arm_smmu_priq_thread, 3214 IRQF_ONESHOT, 3215 "arm-smmu-v3-priq", 3216 smmu); 3217 if (ret < 0) 3218 dev_warn(smmu->dev, 3219 "failed to enable priq irq\n"); 3220 } else { 3221 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n"); 3222 } 3223 } 3224 } 3225 3226 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) 3227 { 3228 int ret, irq; 3229 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; 3230 3231 /* Disable IRQs first */ 3232 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, 3233 ARM_SMMU_IRQ_CTRLACK); 3234 if (ret) { 3235 dev_err(smmu->dev, "failed to disable irqs\n"); 3236 return ret; 3237 } 3238 3239 irq = smmu->combined_irq; 3240 if (irq) { 3241 /* 3242 * Cavium ThunderX2 implementation doesn't support unique irq 3243 * lines. Use a single irq line for all the SMMUv3 interrupts. 3244 */ 3245 ret = devm_request_threaded_irq(smmu->dev, irq, 3246 arm_smmu_combined_irq_handler, 3247 arm_smmu_combined_irq_thread, 3248 IRQF_ONESHOT, 3249 "arm-smmu-v3-combined-irq", smmu); 3250 if (ret < 0) 3251 dev_warn(smmu->dev, "failed to enable combined irq\n"); 3252 } else 3253 arm_smmu_setup_unique_irqs(smmu); 3254 3255 if (smmu->features & ARM_SMMU_FEAT_PRI) 3256 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; 3257 3258 /* Enable interrupt generation on the SMMU */ 3259 ret = arm_smmu_write_reg_sync(smmu, irqen_flags, 3260 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK); 3261 if (ret) 3262 dev_warn(smmu->dev, "failed to enable irqs\n"); 3263 3264 return 0; 3265 } 3266 3267 static int arm_smmu_device_disable(struct arm_smmu_device *smmu) 3268 { 3269 int ret; 3270 3271 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); 3272 if (ret) 3273 dev_err(smmu->dev, "failed to clear cr0\n"); 3274 3275 return ret; 3276 } 3277 3278 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) 3279 { 3280 int ret; 3281 u32 reg, enables; 3282 struct arm_smmu_cmdq_ent cmd; 3283 3284 /* Clear CR0 and sync (disables SMMU and queue processing) */ 3285 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); 3286 if (reg & CR0_SMMUEN) { 3287 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); 3288 WARN_ON(is_kdump_kernel() && !disable_bypass); 3289 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); 3290 } 3291 3292 ret = arm_smmu_device_disable(smmu); 3293 if (ret) 3294 return ret; 3295 3296 /* CR1 (table and queue memory attributes) */ 3297 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) | 3298 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) | 3299 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) | 3300 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) | 3301 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) | 3302 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB); 3303 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); 3304 3305 /* CR2 (random crap) */ 3306 reg = CR2_PTM | CR2_RECINVSID; 3307 3308 if (smmu->features & ARM_SMMU_FEAT_E2H) 3309 reg |= CR2_E2H; 3310 3311 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2); 3312 3313 /* Stream table */ 3314 writeq_relaxed(smmu->strtab_cfg.strtab_base, 3315 smmu->base + ARM_SMMU_STRTAB_BASE); 3316 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg, 3317 smmu->base + ARM_SMMU_STRTAB_BASE_CFG); 3318 3319 /* Command queue */ 3320 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); 3321 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD); 3322 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS); 3323 3324 enables = CR0_CMDQEN; 3325 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3326 ARM_SMMU_CR0ACK); 3327 if (ret) { 3328 dev_err(smmu->dev, "failed to enable command queue\n"); 3329 return ret; 3330 } 3331 3332 /* Invalidate any cached configuration */ 3333 cmd.opcode = CMDQ_OP_CFGI_ALL; 3334 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 3335 3336 /* Invalidate any stale TLB entries */ 3337 if (smmu->features & ARM_SMMU_FEAT_HYP) { 3338 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL; 3339 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 3340 } 3341 3342 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL; 3343 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 3344 3345 /* Event queue */ 3346 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); 3347 writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD); 3348 writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS); 3349 3350 enables |= CR0_EVTQEN; 3351 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3352 ARM_SMMU_CR0ACK); 3353 if (ret) { 3354 dev_err(smmu->dev, "failed to enable event queue\n"); 3355 return ret; 3356 } 3357 3358 /* PRI queue */ 3359 if (smmu->features & ARM_SMMU_FEAT_PRI) { 3360 writeq_relaxed(smmu->priq.q.q_base, 3361 smmu->base + ARM_SMMU_PRIQ_BASE); 3362 writel_relaxed(smmu->priq.q.llq.prod, 3363 smmu->page1 + ARM_SMMU_PRIQ_PROD); 3364 writel_relaxed(smmu->priq.q.llq.cons, 3365 smmu->page1 + ARM_SMMU_PRIQ_CONS); 3366 3367 enables |= CR0_PRIQEN; 3368 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3369 ARM_SMMU_CR0ACK); 3370 if (ret) { 3371 dev_err(smmu->dev, "failed to enable PRI queue\n"); 3372 return ret; 3373 } 3374 } 3375 3376 if (smmu->features & ARM_SMMU_FEAT_ATS) { 3377 enables |= CR0_ATSCHK; 3378 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3379 ARM_SMMU_CR0ACK); 3380 if (ret) { 3381 dev_err(smmu->dev, "failed to enable ATS check\n"); 3382 return ret; 3383 } 3384 } 3385 3386 ret = arm_smmu_setup_irqs(smmu); 3387 if (ret) { 3388 dev_err(smmu->dev, "failed to setup irqs\n"); 3389 return ret; 3390 } 3391 3392 if (is_kdump_kernel()) 3393 enables &= ~(CR0_EVTQEN | CR0_PRIQEN); 3394 3395 /* Enable the SMMU interface, or ensure bypass */ 3396 if (!bypass || disable_bypass) { 3397 enables |= CR0_SMMUEN; 3398 } else { 3399 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT); 3400 if (ret) 3401 return ret; 3402 } 3403 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3404 ARM_SMMU_CR0ACK); 3405 if (ret) { 3406 dev_err(smmu->dev, "failed to enable SMMU interface\n"); 3407 return ret; 3408 } 3409 3410 return 0; 3411 } 3412 3413 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) 3414 { 3415 u32 reg; 3416 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY; 3417 3418 /* IDR0 */ 3419 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); 3420 3421 /* 2-level structures */ 3422 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL) 3423 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB; 3424 3425 if (reg & IDR0_CD2L) 3426 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB; 3427 3428 /* 3429 * Translation table endianness. 3430 * We currently require the same endianness as the CPU, but this 3431 * could be changed later by adding a new IO_PGTABLE_QUIRK. 3432 */ 3433 switch (FIELD_GET(IDR0_TTENDIAN, reg)) { 3434 case IDR0_TTENDIAN_MIXED: 3435 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE; 3436 break; 3437 #ifdef __BIG_ENDIAN 3438 case IDR0_TTENDIAN_BE: 3439 smmu->features |= ARM_SMMU_FEAT_TT_BE; 3440 break; 3441 #else 3442 case IDR0_TTENDIAN_LE: 3443 smmu->features |= ARM_SMMU_FEAT_TT_LE; 3444 break; 3445 #endif 3446 default: 3447 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n"); 3448 return -ENXIO; 3449 } 3450 3451 /* Boolean feature flags */ 3452 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI) 3453 smmu->features |= ARM_SMMU_FEAT_PRI; 3454 3455 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS) 3456 smmu->features |= ARM_SMMU_FEAT_ATS; 3457 3458 if (reg & IDR0_SEV) 3459 smmu->features |= ARM_SMMU_FEAT_SEV; 3460 3461 if (reg & IDR0_MSI) { 3462 smmu->features |= ARM_SMMU_FEAT_MSI; 3463 if (coherent && !disable_msipolling) 3464 smmu->options |= ARM_SMMU_OPT_MSIPOLL; 3465 } 3466 3467 if (reg & IDR0_HYP) { 3468 smmu->features |= ARM_SMMU_FEAT_HYP; 3469 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)) 3470 smmu->features |= ARM_SMMU_FEAT_E2H; 3471 } 3472 3473 /* 3474 * The coherency feature as set by FW is used in preference to the ID 3475 * register, but warn on mismatch. 3476 */ 3477 if (!!(reg & IDR0_COHACC) != coherent) 3478 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n", 3479 coherent ? "true" : "false"); 3480 3481 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) { 3482 case IDR0_STALL_MODEL_FORCE: 3483 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE; 3484 fallthrough; 3485 case IDR0_STALL_MODEL_STALL: 3486 smmu->features |= ARM_SMMU_FEAT_STALLS; 3487 } 3488 3489 if (reg & IDR0_S1P) 3490 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 3491 3492 if (reg & IDR0_S2P) 3493 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; 3494 3495 if (!(reg & (IDR0_S1P | IDR0_S2P))) { 3496 dev_err(smmu->dev, "no translation support!\n"); 3497 return -ENXIO; 3498 } 3499 3500 /* We only support the AArch64 table format at present */ 3501 switch (FIELD_GET(IDR0_TTF, reg)) { 3502 case IDR0_TTF_AARCH32_64: 3503 smmu->ias = 40; 3504 fallthrough; 3505 case IDR0_TTF_AARCH64: 3506 break; 3507 default: 3508 dev_err(smmu->dev, "AArch64 table format not supported!\n"); 3509 return -ENXIO; 3510 } 3511 3512 /* ASID/VMID sizes */ 3513 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8; 3514 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8; 3515 3516 /* IDR1 */ 3517 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1); 3518 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) { 3519 dev_err(smmu->dev, "embedded implementation not supported\n"); 3520 return -ENXIO; 3521 } 3522 3523 /* Queue sizes, capped to ensure natural alignment */ 3524 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, 3525 FIELD_GET(IDR1_CMDQS, reg)); 3526 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) { 3527 /* 3528 * We don't support splitting up batches, so one batch of 3529 * commands plus an extra sync needs to fit inside the command 3530 * queue. There's also no way we can handle the weird alignment 3531 * restrictions on the base pointer for a unit-length queue. 3532 */ 3533 dev_err(smmu->dev, "command queue size <= %d entries not supported\n", 3534 CMDQ_BATCH_ENTRIES); 3535 return -ENXIO; 3536 } 3537 3538 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT, 3539 FIELD_GET(IDR1_EVTQS, reg)); 3540 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT, 3541 FIELD_GET(IDR1_PRIQS, reg)); 3542 3543 /* SID/SSID sizes */ 3544 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg); 3545 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg); 3546 3547 /* 3548 * If the SMMU supports fewer bits than would fill a single L2 stream 3549 * table, use a linear table instead. 3550 */ 3551 if (smmu->sid_bits <= STRTAB_SPLIT) 3552 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; 3553 3554 /* IDR3 */ 3555 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3); 3556 if (FIELD_GET(IDR3_RIL, reg)) 3557 smmu->features |= ARM_SMMU_FEAT_RANGE_INV; 3558 3559 /* IDR5 */ 3560 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); 3561 3562 /* Maximum number of outstanding stalls */ 3563 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg); 3564 3565 /* Page sizes */ 3566 if (reg & IDR5_GRAN64K) 3567 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; 3568 if (reg & IDR5_GRAN16K) 3569 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; 3570 if (reg & IDR5_GRAN4K) 3571 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; 3572 3573 /* Input address size */ 3574 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT) 3575 smmu->features |= ARM_SMMU_FEAT_VAX; 3576 3577 /* Output address size */ 3578 switch (FIELD_GET(IDR5_OAS, reg)) { 3579 case IDR5_OAS_32_BIT: 3580 smmu->oas = 32; 3581 break; 3582 case IDR5_OAS_36_BIT: 3583 smmu->oas = 36; 3584 break; 3585 case IDR5_OAS_40_BIT: 3586 smmu->oas = 40; 3587 break; 3588 case IDR5_OAS_42_BIT: 3589 smmu->oas = 42; 3590 break; 3591 case IDR5_OAS_44_BIT: 3592 smmu->oas = 44; 3593 break; 3594 case IDR5_OAS_52_BIT: 3595 smmu->oas = 52; 3596 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */ 3597 break; 3598 default: 3599 dev_info(smmu->dev, 3600 "unknown output address size. Truncating to 48-bit\n"); 3601 fallthrough; 3602 case IDR5_OAS_48_BIT: 3603 smmu->oas = 48; 3604 } 3605 3606 if (arm_smmu_ops.pgsize_bitmap == -1UL) 3607 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; 3608 else 3609 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; 3610 3611 /* Set the DMA mask for our table walker */ 3612 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas))) 3613 dev_warn(smmu->dev, 3614 "failed to set DMA mask for table walker\n"); 3615 3616 smmu->ias = max(smmu->ias, smmu->oas); 3617 3618 if (arm_smmu_sva_supported(smmu)) 3619 smmu->features |= ARM_SMMU_FEAT_SVA; 3620 3621 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n", 3622 smmu->ias, smmu->oas, smmu->features); 3623 return 0; 3624 } 3625 3626 #ifdef CONFIG_ACPI 3627 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu) 3628 { 3629 switch (model) { 3630 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX: 3631 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY; 3632 break; 3633 case ACPI_IORT_SMMU_V3_HISILICON_HI161X: 3634 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH; 3635 break; 3636 } 3637 3638 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options); 3639 } 3640 3641 static int arm_smmu_device_acpi_probe(struct platform_device *pdev, 3642 struct arm_smmu_device *smmu) 3643 { 3644 struct acpi_iort_smmu_v3 *iort_smmu; 3645 struct device *dev = smmu->dev; 3646 struct acpi_iort_node *node; 3647 3648 node = *(struct acpi_iort_node **)dev_get_platdata(dev); 3649 3650 /* Retrieve SMMUv3 specific data */ 3651 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data; 3652 3653 acpi_smmu_get_options(iort_smmu->model, smmu); 3654 3655 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE) 3656 smmu->features |= ARM_SMMU_FEAT_COHERENCY; 3657 3658 return 0; 3659 } 3660 #else 3661 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev, 3662 struct arm_smmu_device *smmu) 3663 { 3664 return -ENODEV; 3665 } 3666 #endif 3667 3668 static int arm_smmu_device_dt_probe(struct platform_device *pdev, 3669 struct arm_smmu_device *smmu) 3670 { 3671 struct device *dev = &pdev->dev; 3672 u32 cells; 3673 int ret = -EINVAL; 3674 3675 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells)) 3676 dev_err(dev, "missing #iommu-cells property\n"); 3677 else if (cells != 1) 3678 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells); 3679 else 3680 ret = 0; 3681 3682 parse_driver_options(smmu); 3683 3684 if (of_dma_is_coherent(dev->of_node)) 3685 smmu->features |= ARM_SMMU_FEAT_COHERENCY; 3686 3687 return ret; 3688 } 3689 3690 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) 3691 { 3692 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY) 3693 return SZ_64K; 3694 else 3695 return SZ_128K; 3696 } 3697 3698 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start, 3699 resource_size_t size) 3700 { 3701 struct resource res = DEFINE_RES_MEM(start, size); 3702 3703 return devm_ioremap_resource(dev, &res); 3704 } 3705 3706 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) 3707 { 3708 struct list_head rmr_list; 3709 struct iommu_resv_region *e; 3710 3711 INIT_LIST_HEAD(&rmr_list); 3712 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 3713 3714 list_for_each_entry(e, &rmr_list, list) { 3715 __le64 *step; 3716 struct iommu_iort_rmr_data *rmr; 3717 int ret, i; 3718 3719 rmr = container_of(e, struct iommu_iort_rmr_data, rr); 3720 for (i = 0; i < rmr->num_sids; i++) { 3721 ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]); 3722 if (ret) { 3723 dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n", 3724 rmr->sids[i]); 3725 continue; 3726 } 3727 3728 step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]); 3729 arm_smmu_init_bypass_stes(step, 1, true); 3730 } 3731 } 3732 3733 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 3734 } 3735 3736 static int arm_smmu_device_probe(struct platform_device *pdev) 3737 { 3738 int irq, ret; 3739 struct resource *res; 3740 resource_size_t ioaddr; 3741 struct arm_smmu_device *smmu; 3742 struct device *dev = &pdev->dev; 3743 bool bypass; 3744 3745 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 3746 if (!smmu) 3747 return -ENOMEM; 3748 smmu->dev = dev; 3749 3750 if (dev->of_node) { 3751 ret = arm_smmu_device_dt_probe(pdev, smmu); 3752 } else { 3753 ret = arm_smmu_device_acpi_probe(pdev, smmu); 3754 if (ret == -ENODEV) 3755 return ret; 3756 } 3757 3758 /* Set bypass mode according to firmware probing result */ 3759 bypass = !!ret; 3760 3761 /* Base address */ 3762 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3763 if (!res) 3764 return -EINVAL; 3765 if (resource_size(res) < arm_smmu_resource_size(smmu)) { 3766 dev_err(dev, "MMIO region too small (%pr)\n", res); 3767 return -EINVAL; 3768 } 3769 ioaddr = res->start; 3770 3771 /* 3772 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain 3773 * the PMCG registers which are reserved by the PMU driver. 3774 */ 3775 smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ); 3776 if (IS_ERR(smmu->base)) 3777 return PTR_ERR(smmu->base); 3778 3779 if (arm_smmu_resource_size(smmu) > SZ_64K) { 3780 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K, 3781 ARM_SMMU_REG_SZ); 3782 if (IS_ERR(smmu->page1)) 3783 return PTR_ERR(smmu->page1); 3784 } else { 3785 smmu->page1 = smmu->base; 3786 } 3787 3788 /* Interrupt lines */ 3789 3790 irq = platform_get_irq_byname_optional(pdev, "combined"); 3791 if (irq > 0) 3792 smmu->combined_irq = irq; 3793 else { 3794 irq = platform_get_irq_byname_optional(pdev, "eventq"); 3795 if (irq > 0) 3796 smmu->evtq.q.irq = irq; 3797 3798 irq = platform_get_irq_byname_optional(pdev, "priq"); 3799 if (irq > 0) 3800 smmu->priq.q.irq = irq; 3801 3802 irq = platform_get_irq_byname_optional(pdev, "gerror"); 3803 if (irq > 0) 3804 smmu->gerr_irq = irq; 3805 } 3806 /* Probe the h/w */ 3807 ret = arm_smmu_device_hw_probe(smmu); 3808 if (ret) 3809 return ret; 3810 3811 /* Initialise in-memory data structures */ 3812 ret = arm_smmu_init_structures(smmu); 3813 if (ret) 3814 return ret; 3815 3816 /* Record our private device structure */ 3817 platform_set_drvdata(pdev, smmu); 3818 3819 /* Check for RMRs and install bypass STEs if any */ 3820 arm_smmu_rmr_install_bypass_ste(smmu); 3821 3822 /* Reset the device */ 3823 ret = arm_smmu_device_reset(smmu, bypass); 3824 if (ret) 3825 return ret; 3826 3827 /* And we're up. Go go go! */ 3828 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, 3829 "smmu3.%pa", &ioaddr); 3830 if (ret) 3831 return ret; 3832 3833 ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); 3834 if (ret) { 3835 dev_err(dev, "Failed to register iommu\n"); 3836 iommu_device_sysfs_remove(&smmu->iommu); 3837 return ret; 3838 } 3839 3840 return 0; 3841 } 3842 3843 static int arm_smmu_device_remove(struct platform_device *pdev) 3844 { 3845 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 3846 3847 iommu_device_unregister(&smmu->iommu); 3848 iommu_device_sysfs_remove(&smmu->iommu); 3849 arm_smmu_device_disable(smmu); 3850 iopf_queue_free(smmu->evtq.iopf); 3851 3852 return 0; 3853 } 3854 3855 static void arm_smmu_device_shutdown(struct platform_device *pdev) 3856 { 3857 arm_smmu_device_remove(pdev); 3858 } 3859 3860 static const struct of_device_id arm_smmu_of_match[] = { 3861 { .compatible = "arm,smmu-v3", }, 3862 { }, 3863 }; 3864 MODULE_DEVICE_TABLE(of, arm_smmu_of_match); 3865 3866 static void arm_smmu_driver_unregister(struct platform_driver *drv) 3867 { 3868 arm_smmu_sva_notifier_synchronize(); 3869 platform_driver_unregister(drv); 3870 } 3871 3872 static struct platform_driver arm_smmu_driver = { 3873 .driver = { 3874 .name = "arm-smmu-v3", 3875 .of_match_table = arm_smmu_of_match, 3876 .suppress_bind_attrs = true, 3877 }, 3878 .probe = arm_smmu_device_probe, 3879 .remove = arm_smmu_device_remove, 3880 .shutdown = arm_smmu_device_shutdown, 3881 }; 3882 module_driver(arm_smmu_driver, platform_driver_register, 3883 arm_smmu_driver_unregister); 3884 3885 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations"); 3886 MODULE_AUTHOR("Will Deacon <will@kernel.org>"); 3887 MODULE_ALIAS("platform:arm-smmu-v3"); 3888 MODULE_LICENSE("GPL v2"); 3889