1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * IOMMU API for ARM architected SMMUv3 implementations. 4 * 5 * Copyright (C) 2015 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 * 9 * This driver is powered by bad coffee and bombay mix. 10 */ 11 12 #include <linux/acpi.h> 13 #include <linux/acpi_iort.h> 14 #include <linux/bitops.h> 15 #include <linux/crash_dump.h> 16 #include <linux/delay.h> 17 #include <linux/err.h> 18 #include <linux/interrupt.h> 19 #include <linux/io-pgtable.h> 20 #include <linux/iopoll.h> 21 #include <linux/module.h> 22 #include <linux/msi.h> 23 #include <linux/of.h> 24 #include <linux/of_address.h> 25 #include <linux/of_platform.h> 26 #include <linux/pci.h> 27 #include <linux/pci-ats.h> 28 #include <linux/platform_device.h> 29 30 #include "arm-smmu-v3.h" 31 #include "../../dma-iommu.h" 32 #include "../../iommu-sva.h" 33 34 static bool disable_bypass = true; 35 module_param(disable_bypass, bool, 0444); 36 MODULE_PARM_DESC(disable_bypass, 37 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 38 39 static bool disable_msipolling; 40 module_param(disable_msipolling, bool, 0444); 41 MODULE_PARM_DESC(disable_msipolling, 42 "Disable MSI-based polling for CMD_SYNC completion."); 43 44 enum arm_smmu_msi_index { 45 EVTQ_MSI_INDEX, 46 GERROR_MSI_INDEX, 47 PRIQ_MSI_INDEX, 48 ARM_SMMU_MAX_MSIS, 49 }; 50 51 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { 52 [EVTQ_MSI_INDEX] = { 53 ARM_SMMU_EVTQ_IRQ_CFG0, 54 ARM_SMMU_EVTQ_IRQ_CFG1, 55 ARM_SMMU_EVTQ_IRQ_CFG2, 56 }, 57 [GERROR_MSI_INDEX] = { 58 ARM_SMMU_GERROR_IRQ_CFG0, 59 ARM_SMMU_GERROR_IRQ_CFG1, 60 ARM_SMMU_GERROR_IRQ_CFG2, 61 }, 62 [PRIQ_MSI_INDEX] = { 63 ARM_SMMU_PRIQ_IRQ_CFG0, 64 ARM_SMMU_PRIQ_IRQ_CFG1, 65 ARM_SMMU_PRIQ_IRQ_CFG2, 66 }, 67 }; 68 69 struct arm_smmu_option_prop { 70 u32 opt; 71 const char *prop; 72 }; 73 74 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa); 75 DEFINE_MUTEX(arm_smmu_asid_lock); 76 77 /* 78 * Special value used by SVA when a process dies, to quiesce a CD without 79 * disabling it. 80 */ 81 struct arm_smmu_ctx_desc quiet_cd = { 0 }; 82 83 static struct arm_smmu_option_prop arm_smmu_options[] = { 84 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" }, 85 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"}, 86 { 0, NULL}, 87 }; 88 89 static void parse_driver_options(struct arm_smmu_device *smmu) 90 { 91 int i = 0; 92 93 do { 94 if (of_property_read_bool(smmu->dev->of_node, 95 arm_smmu_options[i].prop)) { 96 smmu->options |= arm_smmu_options[i].opt; 97 dev_notice(smmu->dev, "option %s\n", 98 arm_smmu_options[i].prop); 99 } 100 } while (arm_smmu_options[++i].opt); 101 } 102 103 /* Low-level queue manipulation functions */ 104 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n) 105 { 106 u32 space, prod, cons; 107 108 prod = Q_IDX(q, q->prod); 109 cons = Q_IDX(q, q->cons); 110 111 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons)) 112 space = (1 << q->max_n_shift) - (prod - cons); 113 else 114 space = cons - prod; 115 116 return space >= n; 117 } 118 119 static bool queue_full(struct arm_smmu_ll_queue *q) 120 { 121 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && 122 Q_WRP(q, q->prod) != Q_WRP(q, q->cons); 123 } 124 125 static bool queue_empty(struct arm_smmu_ll_queue *q) 126 { 127 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && 128 Q_WRP(q, q->prod) == Q_WRP(q, q->cons); 129 } 130 131 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod) 132 { 133 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) && 134 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) || 135 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) && 136 (Q_IDX(q, q->cons) <= Q_IDX(q, prod))); 137 } 138 139 static void queue_sync_cons_out(struct arm_smmu_queue *q) 140 { 141 /* 142 * Ensure that all CPU accesses (reads and writes) to the queue 143 * are complete before we update the cons pointer. 144 */ 145 __iomb(); 146 writel_relaxed(q->llq.cons, q->cons_reg); 147 } 148 149 static void queue_inc_cons(struct arm_smmu_ll_queue *q) 150 { 151 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1; 152 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons); 153 } 154 155 static void queue_sync_cons_ovf(struct arm_smmu_queue *q) 156 { 157 struct arm_smmu_ll_queue *llq = &q->llq; 158 159 if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons))) 160 return; 161 162 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | 163 Q_IDX(llq, llq->cons); 164 queue_sync_cons_out(q); 165 } 166 167 static int queue_sync_prod_in(struct arm_smmu_queue *q) 168 { 169 u32 prod; 170 int ret = 0; 171 172 /* 173 * We can't use the _relaxed() variant here, as we must prevent 174 * speculative reads of the queue before we have determined that 175 * prod has indeed moved. 176 */ 177 prod = readl(q->prod_reg); 178 179 if (Q_OVF(prod) != Q_OVF(q->llq.prod)) 180 ret = -EOVERFLOW; 181 182 q->llq.prod = prod; 183 return ret; 184 } 185 186 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n) 187 { 188 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 189 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 190 } 191 192 static void queue_poll_init(struct arm_smmu_device *smmu, 193 struct arm_smmu_queue_poll *qp) 194 { 195 qp->delay = 1; 196 qp->spin_cnt = 0; 197 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); 198 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); 199 } 200 201 static int queue_poll(struct arm_smmu_queue_poll *qp) 202 { 203 if (ktime_compare(ktime_get(), qp->timeout) > 0) 204 return -ETIMEDOUT; 205 206 if (qp->wfe) { 207 wfe(); 208 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) { 209 cpu_relax(); 210 } else { 211 udelay(qp->delay); 212 qp->delay *= 2; 213 qp->spin_cnt = 0; 214 } 215 216 return 0; 217 } 218 219 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords) 220 { 221 int i; 222 223 for (i = 0; i < n_dwords; ++i) 224 *dst++ = cpu_to_le64(*src++); 225 } 226 227 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords) 228 { 229 int i; 230 231 for (i = 0; i < n_dwords; ++i) 232 *dst++ = le64_to_cpu(*src++); 233 } 234 235 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent) 236 { 237 if (queue_empty(&q->llq)) 238 return -EAGAIN; 239 240 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords); 241 queue_inc_cons(&q->llq); 242 queue_sync_cons_out(q); 243 return 0; 244 } 245 246 /* High-level queue accessors */ 247 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) 248 { 249 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT); 250 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode); 251 252 switch (ent->opcode) { 253 case CMDQ_OP_TLBI_EL2_ALL: 254 case CMDQ_OP_TLBI_NSNH_ALL: 255 break; 256 case CMDQ_OP_PREFETCH_CFG: 257 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid); 258 break; 259 case CMDQ_OP_CFGI_CD: 260 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid); 261 fallthrough; 262 case CMDQ_OP_CFGI_STE: 263 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); 264 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf); 265 break; 266 case CMDQ_OP_CFGI_CD_ALL: 267 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); 268 break; 269 case CMDQ_OP_CFGI_ALL: 270 /* Cover the entire SID range */ 271 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); 272 break; 273 case CMDQ_OP_TLBI_NH_VA: 274 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 275 fallthrough; 276 case CMDQ_OP_TLBI_EL2_VA: 277 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); 278 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); 279 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); 280 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); 281 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); 282 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg); 283 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK; 284 break; 285 case CMDQ_OP_TLBI_S2_IPA: 286 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); 287 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); 288 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 289 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); 290 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); 291 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg); 292 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK; 293 break; 294 case CMDQ_OP_TLBI_NH_ASID: 295 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); 296 fallthrough; 297 case CMDQ_OP_TLBI_S12_VMALL: 298 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 299 break; 300 case CMDQ_OP_TLBI_EL2_ASID: 301 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); 302 break; 303 case CMDQ_OP_ATC_INV: 304 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 305 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global); 306 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid); 307 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid); 308 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size); 309 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK; 310 break; 311 case CMDQ_OP_PRI_RESP: 312 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 313 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid); 314 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid); 315 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid); 316 switch (ent->pri.resp) { 317 case PRI_RESP_DENY: 318 case PRI_RESP_FAIL: 319 case PRI_RESP_SUCC: 320 break; 321 default: 322 return -EINVAL; 323 } 324 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp); 325 break; 326 case CMDQ_OP_RESUME: 327 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid); 328 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp); 329 cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag); 330 break; 331 case CMDQ_OP_CMD_SYNC: 332 if (ent->sync.msiaddr) { 333 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ); 334 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK; 335 } else { 336 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV); 337 } 338 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH); 339 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); 340 break; 341 default: 342 return -ENOENT; 343 } 344 345 return 0; 346 } 347 348 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu) 349 { 350 return &smmu->cmdq; 351 } 352 353 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu, 354 struct arm_smmu_queue *q, u32 prod) 355 { 356 struct arm_smmu_cmdq_ent ent = { 357 .opcode = CMDQ_OP_CMD_SYNC, 358 }; 359 360 /* 361 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI 362 * payload, so the write will zero the entire command on that platform. 363 */ 364 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) { 365 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) * 366 q->ent_dwords * 8; 367 } 368 369 arm_smmu_cmdq_build_cmd(cmd, &ent); 370 } 371 372 static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, 373 struct arm_smmu_queue *q) 374 { 375 static const char * const cerror_str[] = { 376 [CMDQ_ERR_CERROR_NONE_IDX] = "No error", 377 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command", 378 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch", 379 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout", 380 }; 381 382 int i; 383 u64 cmd[CMDQ_ENT_DWORDS]; 384 u32 cons = readl_relaxed(q->cons_reg); 385 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons); 386 struct arm_smmu_cmdq_ent cmd_sync = { 387 .opcode = CMDQ_OP_CMD_SYNC, 388 }; 389 390 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons, 391 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown"); 392 393 switch (idx) { 394 case CMDQ_ERR_CERROR_ABT_IDX: 395 dev_err(smmu->dev, "retrying command fetch\n"); 396 return; 397 case CMDQ_ERR_CERROR_NONE_IDX: 398 return; 399 case CMDQ_ERR_CERROR_ATC_INV_IDX: 400 /* 401 * ATC Invalidation Completion timeout. CONS is still pointing 402 * at the CMD_SYNC. Attempt to complete other pending commands 403 * by repeating the CMD_SYNC, though we might well end up back 404 * here since the ATC invalidation may still be pending. 405 */ 406 return; 407 case CMDQ_ERR_CERROR_ILL_IDX: 408 default: 409 break; 410 } 411 412 /* 413 * We may have concurrent producers, so we need to be careful 414 * not to touch any of the shadow cmdq state. 415 */ 416 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords); 417 dev_err(smmu->dev, "skipping command in error state:\n"); 418 for (i = 0; i < ARRAY_SIZE(cmd); ++i) 419 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); 420 421 /* Convert the erroneous command into a CMD_SYNC */ 422 arm_smmu_cmdq_build_cmd(cmd, &cmd_sync); 423 424 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords); 425 } 426 427 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) 428 { 429 __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q); 430 } 431 432 /* 433 * Command queue locking. 434 * This is a form of bastardised rwlock with the following major changes: 435 * 436 * - The only LOCK routines are exclusive_trylock() and shared_lock(). 437 * Neither have barrier semantics, and instead provide only a control 438 * dependency. 439 * 440 * - The UNLOCK routines are supplemented with shared_tryunlock(), which 441 * fails if the caller appears to be the last lock holder (yes, this is 442 * racy). All successful UNLOCK routines have RELEASE semantics. 443 */ 444 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq) 445 { 446 int val; 447 448 /* 449 * We can try to avoid the cmpxchg() loop by simply incrementing the 450 * lock counter. When held in exclusive state, the lock counter is set 451 * to INT_MIN so these increments won't hurt as the value will remain 452 * negative. 453 */ 454 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0) 455 return; 456 457 do { 458 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0); 459 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val); 460 } 461 462 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq) 463 { 464 (void)atomic_dec_return_release(&cmdq->lock); 465 } 466 467 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq) 468 { 469 if (atomic_read(&cmdq->lock) == 1) 470 return false; 471 472 arm_smmu_cmdq_shared_unlock(cmdq); 473 return true; 474 } 475 476 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \ 477 ({ \ 478 bool __ret; \ 479 local_irq_save(flags); \ 480 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \ 481 if (!__ret) \ 482 local_irq_restore(flags); \ 483 __ret; \ 484 }) 485 486 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \ 487 ({ \ 488 atomic_set_release(&cmdq->lock, 0); \ 489 local_irq_restore(flags); \ 490 }) 491 492 493 /* 494 * Command queue insertion. 495 * This is made fiddly by our attempts to achieve some sort of scalability 496 * since there is one queue shared amongst all of the CPUs in the system. If 497 * you like mixed-size concurrency, dependency ordering and relaxed atomics, 498 * then you'll *love* this monstrosity. 499 * 500 * The basic idea is to split the queue up into ranges of commands that are 501 * owned by a given CPU; the owner may not have written all of the commands 502 * itself, but is responsible for advancing the hardware prod pointer when 503 * the time comes. The algorithm is roughly: 504 * 505 * 1. Allocate some space in the queue. At this point we also discover 506 * whether the head of the queue is currently owned by another CPU, 507 * or whether we are the owner. 508 * 509 * 2. Write our commands into our allocated slots in the queue. 510 * 511 * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map. 512 * 513 * 4. If we are an owner: 514 * a. Wait for the previous owner to finish. 515 * b. Mark the queue head as unowned, which tells us the range 516 * that we are responsible for publishing. 517 * c. Wait for all commands in our owned range to become valid. 518 * d. Advance the hardware prod pointer. 519 * e. Tell the next owner we've finished. 520 * 521 * 5. If we are inserting a CMD_SYNC (we may or may not have been an 522 * owner), then we need to stick around until it has completed: 523 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC 524 * to clear the first 4 bytes. 525 * b. Otherwise, we spin waiting for the hardware cons pointer to 526 * advance past our command. 527 * 528 * The devil is in the details, particularly the use of locking for handling 529 * SYNC completion and freeing up space in the queue before we think that it is 530 * full. 531 */ 532 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq, 533 u32 sprod, u32 eprod, bool set) 534 { 535 u32 swidx, sbidx, ewidx, ebidx; 536 struct arm_smmu_ll_queue llq = { 537 .max_n_shift = cmdq->q.llq.max_n_shift, 538 .prod = sprod, 539 }; 540 541 ewidx = BIT_WORD(Q_IDX(&llq, eprod)); 542 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG; 543 544 while (llq.prod != eprod) { 545 unsigned long mask; 546 atomic_long_t *ptr; 547 u32 limit = BITS_PER_LONG; 548 549 swidx = BIT_WORD(Q_IDX(&llq, llq.prod)); 550 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG; 551 552 ptr = &cmdq->valid_map[swidx]; 553 554 if ((swidx == ewidx) && (sbidx < ebidx)) 555 limit = ebidx; 556 557 mask = GENMASK(limit - 1, sbidx); 558 559 /* 560 * The valid bit is the inverse of the wrap bit. This means 561 * that a zero-initialised queue is invalid and, after marking 562 * all entries as valid, they become invalid again when we 563 * wrap. 564 */ 565 if (set) { 566 atomic_long_xor(mask, ptr); 567 } else { /* Poll */ 568 unsigned long valid; 569 570 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask; 571 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid); 572 } 573 574 llq.prod = queue_inc_prod_n(&llq, limit - sbidx); 575 } 576 } 577 578 /* Mark all entries in the range [sprod, eprod) as valid */ 579 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq, 580 u32 sprod, u32 eprod) 581 { 582 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true); 583 } 584 585 /* Wait for all entries in the range [sprod, eprod) to become valid */ 586 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq, 587 u32 sprod, u32 eprod) 588 { 589 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false); 590 } 591 592 /* Wait for the command queue to become non-full */ 593 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu, 594 struct arm_smmu_ll_queue *llq) 595 { 596 unsigned long flags; 597 struct arm_smmu_queue_poll qp; 598 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu); 599 int ret = 0; 600 601 /* 602 * Try to update our copy of cons by grabbing exclusive cmdq access. If 603 * that fails, spin until somebody else updates it for us. 604 */ 605 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) { 606 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg)); 607 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags); 608 llq->val = READ_ONCE(cmdq->q.llq.val); 609 return 0; 610 } 611 612 queue_poll_init(smmu, &qp); 613 do { 614 llq->val = READ_ONCE(cmdq->q.llq.val); 615 if (!queue_full(llq)) 616 break; 617 618 ret = queue_poll(&qp); 619 } while (!ret); 620 621 return ret; 622 } 623 624 /* 625 * Wait until the SMMU signals a CMD_SYNC completion MSI. 626 * Must be called with the cmdq lock held in some capacity. 627 */ 628 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu, 629 struct arm_smmu_ll_queue *llq) 630 { 631 int ret = 0; 632 struct arm_smmu_queue_poll qp; 633 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu); 634 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod)); 635 636 queue_poll_init(smmu, &qp); 637 638 /* 639 * The MSI won't generate an event, since it's being written back 640 * into the command queue. 641 */ 642 qp.wfe = false; 643 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp))); 644 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1); 645 return ret; 646 } 647 648 /* 649 * Wait until the SMMU cons index passes llq->prod. 650 * Must be called with the cmdq lock held in some capacity. 651 */ 652 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu, 653 struct arm_smmu_ll_queue *llq) 654 { 655 struct arm_smmu_queue_poll qp; 656 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu); 657 u32 prod = llq->prod; 658 int ret = 0; 659 660 queue_poll_init(smmu, &qp); 661 llq->val = READ_ONCE(cmdq->q.llq.val); 662 do { 663 if (queue_consumed(llq, prod)) 664 break; 665 666 ret = queue_poll(&qp); 667 668 /* 669 * This needs to be a readl() so that our subsequent call 670 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately. 671 * 672 * Specifically, we need to ensure that we observe all 673 * shared_lock()s by other CMD_SYNCs that share our owner, 674 * so that a failing call to tryunlock() means that we're 675 * the last one out and therefore we can safely advance 676 * cmdq->q.llq.cons. Roughly speaking: 677 * 678 * CPU 0 CPU1 CPU2 (us) 679 * 680 * if (sync) 681 * shared_lock(); 682 * 683 * dma_wmb(); 684 * set_valid_map(); 685 * 686 * if (owner) { 687 * poll_valid_map(); 688 * <control dependency> 689 * writel(prod_reg); 690 * 691 * readl(cons_reg); 692 * tryunlock(); 693 * 694 * Requires us to see CPU 0's shared_lock() acquisition. 695 */ 696 llq->cons = readl(cmdq->q.cons_reg); 697 } while (!ret); 698 699 return ret; 700 } 701 702 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu, 703 struct arm_smmu_ll_queue *llq) 704 { 705 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) 706 return __arm_smmu_cmdq_poll_until_msi(smmu, llq); 707 708 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq); 709 } 710 711 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds, 712 u32 prod, int n) 713 { 714 int i; 715 struct arm_smmu_ll_queue llq = { 716 .max_n_shift = cmdq->q.llq.max_n_shift, 717 .prod = prod, 718 }; 719 720 for (i = 0; i < n; ++i) { 721 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS]; 722 723 prod = queue_inc_prod_n(&llq, i); 724 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS); 725 } 726 } 727 728 /* 729 * This is the actual insertion function, and provides the following 730 * ordering guarantees to callers: 731 * 732 * - There is a dma_wmb() before publishing any commands to the queue. 733 * This can be relied upon to order prior writes to data structures 734 * in memory (such as a CD or an STE) before the command. 735 * 736 * - On completion of a CMD_SYNC, there is a control dependency. 737 * This can be relied upon to order subsequent writes to memory (e.g. 738 * freeing an IOVA) after completion of the CMD_SYNC. 739 * 740 * - Command insertion is totally ordered, so if two CPUs each race to 741 * insert their own list of commands then all of the commands from one 742 * CPU will appear before any of the commands from the other CPU. 743 */ 744 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, 745 u64 *cmds, int n, bool sync) 746 { 747 u64 cmd_sync[CMDQ_ENT_DWORDS]; 748 u32 prod; 749 unsigned long flags; 750 bool owner; 751 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu); 752 struct arm_smmu_ll_queue llq, head; 753 int ret = 0; 754 755 llq.max_n_shift = cmdq->q.llq.max_n_shift; 756 757 /* 1. Allocate some space in the queue */ 758 local_irq_save(flags); 759 llq.val = READ_ONCE(cmdq->q.llq.val); 760 do { 761 u64 old; 762 763 while (!queue_has_space(&llq, n + sync)) { 764 local_irq_restore(flags); 765 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq)) 766 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); 767 local_irq_save(flags); 768 } 769 770 head.cons = llq.cons; 771 head.prod = queue_inc_prod_n(&llq, n + sync) | 772 CMDQ_PROD_OWNED_FLAG; 773 774 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val); 775 if (old == llq.val) 776 break; 777 778 llq.val = old; 779 } while (1); 780 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG); 781 head.prod &= ~CMDQ_PROD_OWNED_FLAG; 782 llq.prod &= ~CMDQ_PROD_OWNED_FLAG; 783 784 /* 785 * 2. Write our commands into the queue 786 * Dependency ordering from the cmpxchg() loop above. 787 */ 788 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); 789 if (sync) { 790 prod = queue_inc_prod_n(&llq, n); 791 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod); 792 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS); 793 794 /* 795 * In order to determine completion of our CMD_SYNC, we must 796 * ensure that the queue can't wrap twice without us noticing. 797 * We achieve that by taking the cmdq lock as shared before 798 * marking our slot as valid. 799 */ 800 arm_smmu_cmdq_shared_lock(cmdq); 801 } 802 803 /* 3. Mark our slots as valid, ensuring commands are visible first */ 804 dma_wmb(); 805 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod); 806 807 /* 4. If we are the owner, take control of the SMMU hardware */ 808 if (owner) { 809 /* a. Wait for previous owner to finish */ 810 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod); 811 812 /* b. Stop gathering work by clearing the owned flag */ 813 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG, 814 &cmdq->q.llq.atomic.prod); 815 prod &= ~CMDQ_PROD_OWNED_FLAG; 816 817 /* 818 * c. Wait for any gathered work to be written to the queue. 819 * Note that we read our own entries so that we have the control 820 * dependency required by (d). 821 */ 822 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod); 823 824 /* 825 * d. Advance the hardware prod pointer 826 * Control dependency ordering from the entries becoming valid. 827 */ 828 writel_relaxed(prod, cmdq->q.prod_reg); 829 830 /* 831 * e. Tell the next owner we're done 832 * Make sure we've updated the hardware first, so that we don't 833 * race to update prod and potentially move it backwards. 834 */ 835 atomic_set_release(&cmdq->owner_prod, prod); 836 } 837 838 /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */ 839 if (sync) { 840 llq.prod = queue_inc_prod_n(&llq, n); 841 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq); 842 if (ret) { 843 dev_err_ratelimited(smmu->dev, 844 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n", 845 llq.prod, 846 readl_relaxed(cmdq->q.prod_reg), 847 readl_relaxed(cmdq->q.cons_reg)); 848 } 849 850 /* 851 * Try to unlock the cmdq lock. This will fail if we're the last 852 * reader, in which case we can safely update cmdq->q.llq.cons 853 */ 854 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) { 855 WRITE_ONCE(cmdq->q.llq.cons, llq.cons); 856 arm_smmu_cmdq_shared_unlock(cmdq); 857 } 858 } 859 860 local_irq_restore(flags); 861 return ret; 862 } 863 864 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, 865 struct arm_smmu_cmdq_ent *ent, 866 bool sync) 867 { 868 u64 cmd[CMDQ_ENT_DWORDS]; 869 870 if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) { 871 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", 872 ent->opcode); 873 return -EINVAL; 874 } 875 876 return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, sync); 877 } 878 879 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, 880 struct arm_smmu_cmdq_ent *ent) 881 { 882 return __arm_smmu_cmdq_issue_cmd(smmu, ent, false); 883 } 884 885 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu, 886 struct arm_smmu_cmdq_ent *ent) 887 { 888 return __arm_smmu_cmdq_issue_cmd(smmu, ent, true); 889 } 890 891 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, 892 struct arm_smmu_cmdq_batch *cmds, 893 struct arm_smmu_cmdq_ent *cmd) 894 { 895 int index; 896 897 if (cmds->num == CMDQ_BATCH_ENTRIES) { 898 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false); 899 cmds->num = 0; 900 } 901 902 index = cmds->num * CMDQ_ENT_DWORDS; 903 if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) { 904 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", 905 cmd->opcode); 906 return; 907 } 908 909 cmds->num++; 910 } 911 912 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, 913 struct arm_smmu_cmdq_batch *cmds) 914 { 915 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true); 916 } 917 918 static int arm_smmu_page_response(struct device *dev, 919 struct iommu_fault_event *unused, 920 struct iommu_page_response *resp) 921 { 922 struct arm_smmu_cmdq_ent cmd = {0}; 923 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 924 int sid = master->streams[0].id; 925 926 if (master->stall_enabled) { 927 cmd.opcode = CMDQ_OP_RESUME; 928 cmd.resume.sid = sid; 929 cmd.resume.stag = resp->grpid; 930 switch (resp->code) { 931 case IOMMU_PAGE_RESP_INVALID: 932 case IOMMU_PAGE_RESP_FAILURE: 933 cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; 934 break; 935 case IOMMU_PAGE_RESP_SUCCESS: 936 cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; 937 break; 938 default: 939 return -EINVAL; 940 } 941 } else { 942 return -ENODEV; 943 } 944 945 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); 946 /* 947 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP. 948 * RESUME consumption guarantees that the stalled transaction will be 949 * terminated... at some point in the future. PRI_RESP is fire and 950 * forget. 951 */ 952 953 return 0; 954 } 955 956 /* Context descriptor manipulation functions */ 957 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) 958 { 959 struct arm_smmu_cmdq_ent cmd = { 960 .opcode = smmu->features & ARM_SMMU_FEAT_E2H ? 961 CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID, 962 .tlbi.asid = asid, 963 }; 964 965 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 966 } 967 968 static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain, 969 int ssid, bool leaf) 970 { 971 size_t i; 972 unsigned long flags; 973 struct arm_smmu_master *master; 974 struct arm_smmu_cmdq_batch cmds; 975 struct arm_smmu_device *smmu = smmu_domain->smmu; 976 struct arm_smmu_cmdq_ent cmd = { 977 .opcode = CMDQ_OP_CFGI_CD, 978 .cfgi = { 979 .ssid = ssid, 980 .leaf = leaf, 981 }, 982 }; 983 984 cmds.num = 0; 985 986 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 987 list_for_each_entry(master, &smmu_domain->devices, domain_head) { 988 for (i = 0; i < master->num_streams; i++) { 989 cmd.cfgi.sid = master->streams[i].id; 990 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); 991 } 992 } 993 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 994 995 arm_smmu_cmdq_batch_submit(smmu, &cmds); 996 } 997 998 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu, 999 struct arm_smmu_l1_ctx_desc *l1_desc) 1000 { 1001 size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3); 1002 1003 l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, 1004 &l1_desc->l2ptr_dma, GFP_KERNEL); 1005 if (!l1_desc->l2ptr) { 1006 dev_warn(smmu->dev, 1007 "failed to allocate context descriptor table\n"); 1008 return -ENOMEM; 1009 } 1010 return 0; 1011 } 1012 1013 static void arm_smmu_write_cd_l1_desc(__le64 *dst, 1014 struct arm_smmu_l1_ctx_desc *l1_desc) 1015 { 1016 u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | 1017 CTXDESC_L1_DESC_V; 1018 1019 /* See comment in arm_smmu_write_ctx_desc() */ 1020 WRITE_ONCE(*dst, cpu_to_le64(val)); 1021 } 1022 1023 static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain, 1024 u32 ssid) 1025 { 1026 __le64 *l1ptr; 1027 unsigned int idx; 1028 struct arm_smmu_l1_ctx_desc *l1_desc; 1029 struct arm_smmu_device *smmu = smmu_domain->smmu; 1030 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg; 1031 1032 if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR) 1033 return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS; 1034 1035 idx = ssid >> CTXDESC_SPLIT; 1036 l1_desc = &cdcfg->l1_desc[idx]; 1037 if (!l1_desc->l2ptr) { 1038 if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) 1039 return NULL; 1040 1041 l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS; 1042 arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); 1043 /* An invalid L1CD can be cached */ 1044 arm_smmu_sync_cd(smmu_domain, ssid, false); 1045 } 1046 idx = ssid & (CTXDESC_L2_ENTRIES - 1); 1047 return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS; 1048 } 1049 1050 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid, 1051 struct arm_smmu_ctx_desc *cd) 1052 { 1053 /* 1054 * This function handles the following cases: 1055 * 1056 * (1) Install primary CD, for normal DMA traffic (SSID = 0). 1057 * (2) Install a secondary CD, for SID+SSID traffic. 1058 * (3) Update ASID of a CD. Atomically write the first 64 bits of the 1059 * CD, then invalidate the old entry and mappings. 1060 * (4) Quiesce the context without clearing the valid bit. Disable 1061 * translation, and ignore any translation fault. 1062 * (5) Remove a secondary CD. 1063 */ 1064 u64 val; 1065 bool cd_live; 1066 __le64 *cdptr; 1067 1068 if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax))) 1069 return -E2BIG; 1070 1071 cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid); 1072 if (!cdptr) 1073 return -ENOMEM; 1074 1075 val = le64_to_cpu(cdptr[0]); 1076 cd_live = !!(val & CTXDESC_CD_0_V); 1077 1078 if (!cd) { /* (5) */ 1079 val = 0; 1080 } else if (cd == &quiet_cd) { /* (4) */ 1081 val |= CTXDESC_CD_0_TCR_EPD0; 1082 } else if (cd_live) { /* (3) */ 1083 val &= ~CTXDESC_CD_0_ASID; 1084 val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid); 1085 /* 1086 * Until CD+TLB invalidation, both ASIDs may be used for tagging 1087 * this substream's traffic 1088 */ 1089 } else { /* (1) and (2) */ 1090 cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); 1091 cdptr[2] = 0; 1092 cdptr[3] = cpu_to_le64(cd->mair); 1093 1094 /* 1095 * STE is live, and the SMMU might read dwords of this CD in any 1096 * order. Ensure that it observes valid values before reading 1097 * V=1. 1098 */ 1099 arm_smmu_sync_cd(smmu_domain, ssid, true); 1100 1101 val = cd->tcr | 1102 #ifdef __BIG_ENDIAN 1103 CTXDESC_CD_0_ENDI | 1104 #endif 1105 CTXDESC_CD_0_R | CTXDESC_CD_0_A | 1106 (cd->mm ? 0 : CTXDESC_CD_0_ASET) | 1107 CTXDESC_CD_0_AA64 | 1108 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) | 1109 CTXDESC_CD_0_V; 1110 1111 if (smmu_domain->stall_enabled) 1112 val |= CTXDESC_CD_0_S; 1113 } 1114 1115 /* 1116 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3 1117 * "Configuration structures and configuration invalidation completion" 1118 * 1119 * The size of single-copy atomic reads made by the SMMU is 1120 * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single 1121 * field within an aligned 64-bit span of a structure can be altered 1122 * without first making the structure invalid. 1123 */ 1124 WRITE_ONCE(cdptr[0], cpu_to_le64(val)); 1125 arm_smmu_sync_cd(smmu_domain, ssid, true); 1126 return 0; 1127 } 1128 1129 static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain) 1130 { 1131 int ret; 1132 size_t l1size; 1133 size_t max_contexts; 1134 struct arm_smmu_device *smmu = smmu_domain->smmu; 1135 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; 1136 struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg; 1137 1138 max_contexts = 1 << cfg->s1cdmax; 1139 1140 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) || 1141 max_contexts <= CTXDESC_L2_ENTRIES) { 1142 cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR; 1143 cdcfg->num_l1_ents = max_contexts; 1144 1145 l1size = max_contexts * (CTXDESC_CD_DWORDS << 3); 1146 } else { 1147 cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2; 1148 cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts, 1149 CTXDESC_L2_ENTRIES); 1150 1151 cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents, 1152 sizeof(*cdcfg->l1_desc), 1153 GFP_KERNEL); 1154 if (!cdcfg->l1_desc) 1155 return -ENOMEM; 1156 1157 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3); 1158 } 1159 1160 cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma, 1161 GFP_KERNEL); 1162 if (!cdcfg->cdtab) { 1163 dev_warn(smmu->dev, "failed to allocate context descriptor\n"); 1164 ret = -ENOMEM; 1165 goto err_free_l1; 1166 } 1167 1168 return 0; 1169 1170 err_free_l1: 1171 if (cdcfg->l1_desc) { 1172 devm_kfree(smmu->dev, cdcfg->l1_desc); 1173 cdcfg->l1_desc = NULL; 1174 } 1175 return ret; 1176 } 1177 1178 static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain) 1179 { 1180 int i; 1181 size_t size, l1size; 1182 struct arm_smmu_device *smmu = smmu_domain->smmu; 1183 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg; 1184 1185 if (cdcfg->l1_desc) { 1186 size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3); 1187 1188 for (i = 0; i < cdcfg->num_l1_ents; i++) { 1189 if (!cdcfg->l1_desc[i].l2ptr) 1190 continue; 1191 1192 dmam_free_coherent(smmu->dev, size, 1193 cdcfg->l1_desc[i].l2ptr, 1194 cdcfg->l1_desc[i].l2ptr_dma); 1195 } 1196 devm_kfree(smmu->dev, cdcfg->l1_desc); 1197 cdcfg->l1_desc = NULL; 1198 1199 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3); 1200 } else { 1201 l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3); 1202 } 1203 1204 dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma); 1205 cdcfg->cdtab_dma = 0; 1206 cdcfg->cdtab = NULL; 1207 } 1208 1209 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd) 1210 { 1211 bool free; 1212 struct arm_smmu_ctx_desc *old_cd; 1213 1214 if (!cd->asid) 1215 return false; 1216 1217 free = refcount_dec_and_test(&cd->refs); 1218 if (free) { 1219 old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid); 1220 WARN_ON(old_cd != cd); 1221 } 1222 return free; 1223 } 1224 1225 /* Stream table manipulation functions */ 1226 static void 1227 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) 1228 { 1229 u64 val = 0; 1230 1231 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span); 1232 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; 1233 1234 /* See comment in arm_smmu_write_ctx_desc() */ 1235 WRITE_ONCE(*dst, cpu_to_le64(val)); 1236 } 1237 1238 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) 1239 { 1240 struct arm_smmu_cmdq_ent cmd = { 1241 .opcode = CMDQ_OP_CFGI_STE, 1242 .cfgi = { 1243 .sid = sid, 1244 .leaf = true, 1245 }, 1246 }; 1247 1248 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 1249 } 1250 1251 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, 1252 __le64 *dst) 1253 { 1254 /* 1255 * This is hideously complicated, but we only really care about 1256 * three cases at the moment: 1257 * 1258 * 1. Invalid (all zero) -> bypass/fault (init) 1259 * 2. Bypass/fault -> translation/bypass (attach) 1260 * 3. Translation/bypass -> bypass/fault (detach) 1261 * 1262 * Given that we can't update the STE atomically and the SMMU 1263 * doesn't read the thing in a defined order, that leaves us 1264 * with the following maintenance requirements: 1265 * 1266 * 1. Update Config, return (init time STEs aren't live) 1267 * 2. Write everything apart from dword 0, sync, write dword 0, sync 1268 * 3. Update Config, sync 1269 */ 1270 u64 val = le64_to_cpu(dst[0]); 1271 bool ste_live = false; 1272 struct arm_smmu_device *smmu = NULL; 1273 struct arm_smmu_s1_cfg *s1_cfg = NULL; 1274 struct arm_smmu_s2_cfg *s2_cfg = NULL; 1275 struct arm_smmu_domain *smmu_domain = NULL; 1276 struct arm_smmu_cmdq_ent prefetch_cmd = { 1277 .opcode = CMDQ_OP_PREFETCH_CFG, 1278 .prefetch = { 1279 .sid = sid, 1280 }, 1281 }; 1282 1283 if (master) { 1284 smmu_domain = master->domain; 1285 smmu = master->smmu; 1286 } 1287 1288 if (smmu_domain) { 1289 switch (smmu_domain->stage) { 1290 case ARM_SMMU_DOMAIN_S1: 1291 s1_cfg = &smmu_domain->s1_cfg; 1292 break; 1293 case ARM_SMMU_DOMAIN_S2: 1294 case ARM_SMMU_DOMAIN_NESTED: 1295 s2_cfg = &smmu_domain->s2_cfg; 1296 break; 1297 default: 1298 break; 1299 } 1300 } 1301 1302 if (val & STRTAB_STE_0_V) { 1303 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) { 1304 case STRTAB_STE_0_CFG_BYPASS: 1305 break; 1306 case STRTAB_STE_0_CFG_S1_TRANS: 1307 case STRTAB_STE_0_CFG_S2_TRANS: 1308 ste_live = true; 1309 break; 1310 case STRTAB_STE_0_CFG_ABORT: 1311 BUG_ON(!disable_bypass); 1312 break; 1313 default: 1314 BUG(); /* STE corruption */ 1315 } 1316 } 1317 1318 /* Nuke the existing STE_0 value, as we're going to rewrite it */ 1319 val = STRTAB_STE_0_V; 1320 1321 /* Bypass/fault */ 1322 if (!smmu_domain || !(s1_cfg || s2_cfg)) { 1323 if (!smmu_domain && disable_bypass) 1324 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); 1325 else 1326 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); 1327 1328 dst[0] = cpu_to_le64(val); 1329 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, 1330 STRTAB_STE_1_SHCFG_INCOMING)); 1331 dst[2] = 0; /* Nuke the VMID */ 1332 /* 1333 * The SMMU can perform negative caching, so we must sync 1334 * the STE regardless of whether the old value was live. 1335 */ 1336 if (smmu) 1337 arm_smmu_sync_ste_for_sid(smmu, sid); 1338 return; 1339 } 1340 1341 if (s1_cfg) { 1342 u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ? 1343 STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1; 1344 1345 BUG_ON(ste_live); 1346 dst[1] = cpu_to_le64( 1347 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | 1348 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | 1349 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | 1350 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | 1351 FIELD_PREP(STRTAB_STE_1_STRW, strw)); 1352 1353 if (smmu->features & ARM_SMMU_FEAT_STALLS && 1354 !master->stall_enabled) 1355 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); 1356 1357 val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) | 1358 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | 1359 FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) | 1360 FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt); 1361 } 1362 1363 if (s2_cfg) { 1364 BUG_ON(ste_live); 1365 dst[2] = cpu_to_le64( 1366 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | 1367 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) | 1368 #ifdef __BIG_ENDIAN 1369 STRTAB_STE_2_S2ENDI | 1370 #endif 1371 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | 1372 STRTAB_STE_2_S2R); 1373 1374 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); 1375 1376 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); 1377 } 1378 1379 if (master->ats_enabled) 1380 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS, 1381 STRTAB_STE_1_EATS_TRANS)); 1382 1383 arm_smmu_sync_ste_for_sid(smmu, sid); 1384 /* See comment in arm_smmu_write_ctx_desc() */ 1385 WRITE_ONCE(dst[0], cpu_to_le64(val)); 1386 arm_smmu_sync_ste_for_sid(smmu, sid); 1387 1388 /* It's likely that we'll want to use the new STE soon */ 1389 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) 1390 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); 1391 } 1392 1393 static void arm_smmu_init_bypass_stes(__le64 *strtab, unsigned int nent, bool force) 1394 { 1395 unsigned int i; 1396 u64 val = STRTAB_STE_0_V; 1397 1398 if (disable_bypass && !force) 1399 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); 1400 else 1401 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); 1402 1403 for (i = 0; i < nent; ++i) { 1404 strtab[0] = cpu_to_le64(val); 1405 strtab[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, 1406 STRTAB_STE_1_SHCFG_INCOMING)); 1407 strtab[2] = 0; 1408 strtab += STRTAB_STE_DWORDS; 1409 } 1410 } 1411 1412 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) 1413 { 1414 size_t size; 1415 void *strtab; 1416 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 1417 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; 1418 1419 if (desc->l2ptr) 1420 return 0; 1421 1422 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); 1423 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS]; 1424 1425 desc->span = STRTAB_SPLIT + 1; 1426 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma, 1427 GFP_KERNEL); 1428 if (!desc->l2ptr) { 1429 dev_err(smmu->dev, 1430 "failed to allocate l2 stream table for SID %u\n", 1431 sid); 1432 return -ENOMEM; 1433 } 1434 1435 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false); 1436 arm_smmu_write_strtab_l1_desc(strtab, desc); 1437 return 0; 1438 } 1439 1440 static struct arm_smmu_master * 1441 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) 1442 { 1443 struct rb_node *node; 1444 struct arm_smmu_stream *stream; 1445 1446 lockdep_assert_held(&smmu->streams_mutex); 1447 1448 node = smmu->streams.rb_node; 1449 while (node) { 1450 stream = rb_entry(node, struct arm_smmu_stream, node); 1451 if (stream->id < sid) 1452 node = node->rb_right; 1453 else if (stream->id > sid) 1454 node = node->rb_left; 1455 else 1456 return stream->master; 1457 } 1458 1459 return NULL; 1460 } 1461 1462 /* IRQ and event handlers */ 1463 static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) 1464 { 1465 int ret; 1466 u32 reason; 1467 u32 perm = 0; 1468 struct arm_smmu_master *master; 1469 bool ssid_valid = evt[0] & EVTQ_0_SSV; 1470 u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]); 1471 struct iommu_fault_event fault_evt = { }; 1472 struct iommu_fault *flt = &fault_evt.fault; 1473 1474 switch (FIELD_GET(EVTQ_0_ID, evt[0])) { 1475 case EVT_ID_TRANSLATION_FAULT: 1476 reason = IOMMU_FAULT_REASON_PTE_FETCH; 1477 break; 1478 case EVT_ID_ADDR_SIZE_FAULT: 1479 reason = IOMMU_FAULT_REASON_OOR_ADDRESS; 1480 break; 1481 case EVT_ID_ACCESS_FAULT: 1482 reason = IOMMU_FAULT_REASON_ACCESS; 1483 break; 1484 case EVT_ID_PERMISSION_FAULT: 1485 reason = IOMMU_FAULT_REASON_PERMISSION; 1486 break; 1487 default: 1488 return -EOPNOTSUPP; 1489 } 1490 1491 /* Stage-2 is always pinned at the moment */ 1492 if (evt[1] & EVTQ_1_S2) 1493 return -EFAULT; 1494 1495 if (evt[1] & EVTQ_1_RnW) 1496 perm |= IOMMU_FAULT_PERM_READ; 1497 else 1498 perm |= IOMMU_FAULT_PERM_WRITE; 1499 1500 if (evt[1] & EVTQ_1_InD) 1501 perm |= IOMMU_FAULT_PERM_EXEC; 1502 1503 if (evt[1] & EVTQ_1_PnU) 1504 perm |= IOMMU_FAULT_PERM_PRIV; 1505 1506 if (evt[1] & EVTQ_1_STALL) { 1507 flt->type = IOMMU_FAULT_PAGE_REQ; 1508 flt->prm = (struct iommu_fault_page_request) { 1509 .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE, 1510 .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]), 1511 .perm = perm, 1512 .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), 1513 }; 1514 1515 if (ssid_valid) { 1516 flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1517 flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); 1518 } 1519 } else { 1520 flt->type = IOMMU_FAULT_DMA_UNRECOV; 1521 flt->event = (struct iommu_fault_unrecoverable) { 1522 .reason = reason, 1523 .flags = IOMMU_FAULT_UNRECOV_ADDR_VALID, 1524 .perm = perm, 1525 .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), 1526 }; 1527 1528 if (ssid_valid) { 1529 flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID; 1530 flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); 1531 } 1532 } 1533 1534 mutex_lock(&smmu->streams_mutex); 1535 master = arm_smmu_find_master(smmu, sid); 1536 if (!master) { 1537 ret = -EINVAL; 1538 goto out_unlock; 1539 } 1540 1541 ret = iommu_report_device_fault(master->dev, &fault_evt); 1542 if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) { 1543 /* Nobody cared, abort the access */ 1544 struct iommu_page_response resp = { 1545 .pasid = flt->prm.pasid, 1546 .grpid = flt->prm.grpid, 1547 .code = IOMMU_PAGE_RESP_FAILURE, 1548 }; 1549 arm_smmu_page_response(master->dev, &fault_evt, &resp); 1550 } 1551 1552 out_unlock: 1553 mutex_unlock(&smmu->streams_mutex); 1554 return ret; 1555 } 1556 1557 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) 1558 { 1559 int i, ret; 1560 struct arm_smmu_device *smmu = dev; 1561 struct arm_smmu_queue *q = &smmu->evtq.q; 1562 struct arm_smmu_ll_queue *llq = &q->llq; 1563 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 1564 DEFAULT_RATELIMIT_BURST); 1565 u64 evt[EVTQ_ENT_DWORDS]; 1566 1567 do { 1568 while (!queue_remove_raw(q, evt)) { 1569 u8 id = FIELD_GET(EVTQ_0_ID, evt[0]); 1570 1571 ret = arm_smmu_handle_evt(smmu, evt); 1572 if (!ret || !__ratelimit(&rs)) 1573 continue; 1574 1575 dev_info(smmu->dev, "event 0x%02x received:\n", id); 1576 for (i = 0; i < ARRAY_SIZE(evt); ++i) 1577 dev_info(smmu->dev, "\t0x%016llx\n", 1578 (unsigned long long)evt[i]); 1579 1580 cond_resched(); 1581 } 1582 1583 /* 1584 * Not much we can do on overflow, so scream and pretend we're 1585 * trying harder. 1586 */ 1587 if (queue_sync_prod_in(q) == -EOVERFLOW) 1588 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n"); 1589 } while (!queue_empty(llq)); 1590 1591 /* Sync our overflow flag, as we believe we're up to speed */ 1592 queue_sync_cons_ovf(q); 1593 return IRQ_HANDLED; 1594 } 1595 1596 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) 1597 { 1598 u32 sid, ssid; 1599 u16 grpid; 1600 bool ssv, last; 1601 1602 sid = FIELD_GET(PRIQ_0_SID, evt[0]); 1603 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]); 1604 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0; 1605 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]); 1606 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]); 1607 1608 dev_info(smmu->dev, "unexpected PRI request received:\n"); 1609 dev_info(smmu->dev, 1610 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n", 1611 sid, ssid, grpid, last ? "L" : "", 1612 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un", 1613 evt[0] & PRIQ_0_PERM_READ ? "R" : "", 1614 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "", 1615 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "", 1616 evt[1] & PRIQ_1_ADDR_MASK); 1617 1618 if (last) { 1619 struct arm_smmu_cmdq_ent cmd = { 1620 .opcode = CMDQ_OP_PRI_RESP, 1621 .substream_valid = ssv, 1622 .pri = { 1623 .sid = sid, 1624 .ssid = ssid, 1625 .grpid = grpid, 1626 .resp = PRI_RESP_DENY, 1627 }, 1628 }; 1629 1630 arm_smmu_cmdq_issue_cmd(smmu, &cmd); 1631 } 1632 } 1633 1634 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev) 1635 { 1636 struct arm_smmu_device *smmu = dev; 1637 struct arm_smmu_queue *q = &smmu->priq.q; 1638 struct arm_smmu_ll_queue *llq = &q->llq; 1639 u64 evt[PRIQ_ENT_DWORDS]; 1640 1641 do { 1642 while (!queue_remove_raw(q, evt)) 1643 arm_smmu_handle_ppr(smmu, evt); 1644 1645 if (queue_sync_prod_in(q) == -EOVERFLOW) 1646 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n"); 1647 } while (!queue_empty(llq)); 1648 1649 /* Sync our overflow flag, as we believe we're up to speed */ 1650 queue_sync_cons_ovf(q); 1651 return IRQ_HANDLED; 1652 } 1653 1654 static int arm_smmu_device_disable(struct arm_smmu_device *smmu); 1655 1656 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev) 1657 { 1658 u32 gerror, gerrorn, active; 1659 struct arm_smmu_device *smmu = dev; 1660 1661 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); 1662 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); 1663 1664 active = gerror ^ gerrorn; 1665 if (!(active & GERROR_ERR_MASK)) 1666 return IRQ_NONE; /* No errors pending */ 1667 1668 dev_warn(smmu->dev, 1669 "unexpected global error reported (0x%08x), this could be serious\n", 1670 active); 1671 1672 if (active & GERROR_SFM_ERR) { 1673 dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); 1674 arm_smmu_device_disable(smmu); 1675 } 1676 1677 if (active & GERROR_MSI_GERROR_ABT_ERR) 1678 dev_warn(smmu->dev, "GERROR MSI write aborted\n"); 1679 1680 if (active & GERROR_MSI_PRIQ_ABT_ERR) 1681 dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); 1682 1683 if (active & GERROR_MSI_EVTQ_ABT_ERR) 1684 dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); 1685 1686 if (active & GERROR_MSI_CMDQ_ABT_ERR) 1687 dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); 1688 1689 if (active & GERROR_PRIQ_ABT_ERR) 1690 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); 1691 1692 if (active & GERROR_EVTQ_ABT_ERR) 1693 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); 1694 1695 if (active & GERROR_CMDQ_ERR) 1696 arm_smmu_cmdq_skip_err(smmu); 1697 1698 writel(gerror, smmu->base + ARM_SMMU_GERRORN); 1699 return IRQ_HANDLED; 1700 } 1701 1702 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev) 1703 { 1704 struct arm_smmu_device *smmu = dev; 1705 1706 arm_smmu_evtq_thread(irq, dev); 1707 if (smmu->features & ARM_SMMU_FEAT_PRI) 1708 arm_smmu_priq_thread(irq, dev); 1709 1710 return IRQ_HANDLED; 1711 } 1712 1713 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) 1714 { 1715 arm_smmu_gerror_handler(irq, dev); 1716 return IRQ_WAKE_THREAD; 1717 } 1718 1719 static void 1720 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, 1721 struct arm_smmu_cmdq_ent *cmd) 1722 { 1723 size_t log2_span; 1724 size_t span_mask; 1725 /* ATC invalidates are always on 4096-bytes pages */ 1726 size_t inval_grain_shift = 12; 1727 unsigned long page_start, page_end; 1728 1729 /* 1730 * ATS and PASID: 1731 * 1732 * If substream_valid is clear, the PCIe TLP is sent without a PASID 1733 * prefix. In that case all ATC entries within the address range are 1734 * invalidated, including those that were requested with a PASID! There 1735 * is no way to invalidate only entries without PASID. 1736 * 1737 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID 1738 * traffic), translation requests without PASID create ATC entries 1739 * without PASID, which must be invalidated with substream_valid clear. 1740 * This has the unpleasant side-effect of invalidating all PASID-tagged 1741 * ATC entries within the address range. 1742 */ 1743 *cmd = (struct arm_smmu_cmdq_ent) { 1744 .opcode = CMDQ_OP_ATC_INV, 1745 .substream_valid = !!ssid, 1746 .atc.ssid = ssid, 1747 }; 1748 1749 if (!size) { 1750 cmd->atc.size = ATC_INV_SIZE_ALL; 1751 return; 1752 } 1753 1754 page_start = iova >> inval_grain_shift; 1755 page_end = (iova + size - 1) >> inval_grain_shift; 1756 1757 /* 1758 * In an ATS Invalidate Request, the address must be aligned on the 1759 * range size, which must be a power of two number of page sizes. We 1760 * thus have to choose between grossly over-invalidating the region, or 1761 * splitting the invalidation into multiple commands. For simplicity 1762 * we'll go with the first solution, but should refine it in the future 1763 * if multiple commands are shown to be more efficient. 1764 * 1765 * Find the smallest power of two that covers the range. The most 1766 * significant differing bit between the start and end addresses, 1767 * fls(start ^ end), indicates the required span. For example: 1768 * 1769 * We want to invalidate pages [8; 11]. This is already the ideal range: 1770 * x = 0b1000 ^ 0b1011 = 0b11 1771 * span = 1 << fls(x) = 4 1772 * 1773 * To invalidate pages [7; 10], we need to invalidate [0; 15]: 1774 * x = 0b0111 ^ 0b1010 = 0b1101 1775 * span = 1 << fls(x) = 16 1776 */ 1777 log2_span = fls_long(page_start ^ page_end); 1778 span_mask = (1ULL << log2_span) - 1; 1779 1780 page_start &= ~span_mask; 1781 1782 cmd->atc.addr = page_start << inval_grain_shift; 1783 cmd->atc.size = log2_span; 1784 } 1785 1786 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master) 1787 { 1788 int i; 1789 struct arm_smmu_cmdq_ent cmd; 1790 struct arm_smmu_cmdq_batch cmds; 1791 1792 arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd); 1793 1794 cmds.num = 0; 1795 for (i = 0; i < master->num_streams; i++) { 1796 cmd.atc.sid = master->streams[i].id; 1797 arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd); 1798 } 1799 1800 return arm_smmu_cmdq_batch_submit(master->smmu, &cmds); 1801 } 1802 1803 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, 1804 unsigned long iova, size_t size) 1805 { 1806 int i; 1807 unsigned long flags; 1808 struct arm_smmu_cmdq_ent cmd; 1809 struct arm_smmu_master *master; 1810 struct arm_smmu_cmdq_batch cmds; 1811 1812 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS)) 1813 return 0; 1814 1815 /* 1816 * Ensure that we've completed prior invalidation of the main TLBs 1817 * before we read 'nr_ats_masters' in case of a concurrent call to 1818 * arm_smmu_enable_ats(): 1819 * 1820 * // unmap() // arm_smmu_enable_ats() 1821 * TLBI+SYNC atomic_inc(&nr_ats_masters); 1822 * smp_mb(); [...] 1823 * atomic_read(&nr_ats_masters); pci_enable_ats() // writel() 1824 * 1825 * Ensures that we always see the incremented 'nr_ats_masters' count if 1826 * ATS was enabled at the PCI device before completion of the TLBI. 1827 */ 1828 smp_mb(); 1829 if (!atomic_read(&smmu_domain->nr_ats_masters)) 1830 return 0; 1831 1832 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd); 1833 1834 cmds.num = 0; 1835 1836 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 1837 list_for_each_entry(master, &smmu_domain->devices, domain_head) { 1838 if (!master->ats_enabled) 1839 continue; 1840 1841 for (i = 0; i < master->num_streams; i++) { 1842 cmd.atc.sid = master->streams[i].id; 1843 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd); 1844 } 1845 } 1846 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 1847 1848 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds); 1849 } 1850 1851 /* IO_PGTABLE API */ 1852 static void arm_smmu_tlb_inv_context(void *cookie) 1853 { 1854 struct arm_smmu_domain *smmu_domain = cookie; 1855 struct arm_smmu_device *smmu = smmu_domain->smmu; 1856 struct arm_smmu_cmdq_ent cmd; 1857 1858 /* 1859 * NOTE: when io-pgtable is in non-strict mode, we may get here with 1860 * PTEs previously cleared by unmaps on the current CPU not yet visible 1861 * to the SMMU. We are relying on the dma_wmb() implicit during cmd 1862 * insertion to guarantee those are observed before the TLBI. Do be 1863 * careful, 007. 1864 */ 1865 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 1866 arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid); 1867 } else { 1868 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL; 1869 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; 1870 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 1871 } 1872 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0); 1873 } 1874 1875 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, 1876 unsigned long iova, size_t size, 1877 size_t granule, 1878 struct arm_smmu_domain *smmu_domain) 1879 { 1880 struct arm_smmu_device *smmu = smmu_domain->smmu; 1881 unsigned long end = iova + size, num_pages = 0, tg = 0; 1882 size_t inv_range = granule; 1883 struct arm_smmu_cmdq_batch cmds; 1884 1885 if (!size) 1886 return; 1887 1888 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { 1889 /* Get the leaf page size */ 1890 tg = __ffs(smmu_domain->domain.pgsize_bitmap); 1891 1892 /* Convert page size of 12,14,16 (log2) to 1,2,3 */ 1893 cmd->tlbi.tg = (tg - 10) / 2; 1894 1895 /* Determine what level the granule is at */ 1896 cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); 1897 1898 num_pages = size >> tg; 1899 } 1900 1901 cmds.num = 0; 1902 1903 while (iova < end) { 1904 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { 1905 /* 1906 * On each iteration of the loop, the range is 5 bits 1907 * worth of the aligned size remaining. 1908 * The range in pages is: 1909 * 1910 * range = (num_pages & (0x1f << __ffs(num_pages))) 1911 */ 1912 unsigned long scale, num; 1913 1914 /* Determine the power of 2 multiple number of pages */ 1915 scale = __ffs(num_pages); 1916 cmd->tlbi.scale = scale; 1917 1918 /* Determine how many chunks of 2^scale size we have */ 1919 num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX; 1920 cmd->tlbi.num = num - 1; 1921 1922 /* range is num * 2^scale * pgsize */ 1923 inv_range = num << (scale + tg); 1924 1925 /* Clear out the lower order bits for the next iteration */ 1926 num_pages -= num << scale; 1927 } 1928 1929 cmd->tlbi.addr = iova; 1930 arm_smmu_cmdq_batch_add(smmu, &cmds, cmd); 1931 iova += inv_range; 1932 } 1933 arm_smmu_cmdq_batch_submit(smmu, &cmds); 1934 } 1935 1936 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size, 1937 size_t granule, bool leaf, 1938 struct arm_smmu_domain *smmu_domain) 1939 { 1940 struct arm_smmu_cmdq_ent cmd = { 1941 .tlbi = { 1942 .leaf = leaf, 1943 }, 1944 }; 1945 1946 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 1947 cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? 1948 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA; 1949 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; 1950 } else { 1951 cmd.opcode = CMDQ_OP_TLBI_S2_IPA; 1952 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; 1953 } 1954 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); 1955 1956 /* 1957 * Unfortunately, this can't be leaf-only since we may have 1958 * zapped an entire table. 1959 */ 1960 arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size); 1961 } 1962 1963 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, 1964 size_t granule, bool leaf, 1965 struct arm_smmu_domain *smmu_domain) 1966 { 1967 struct arm_smmu_cmdq_ent cmd = { 1968 .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? 1969 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA, 1970 .tlbi = { 1971 .asid = asid, 1972 .leaf = leaf, 1973 }, 1974 }; 1975 1976 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); 1977 } 1978 1979 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, 1980 unsigned long iova, size_t granule, 1981 void *cookie) 1982 { 1983 struct arm_smmu_domain *smmu_domain = cookie; 1984 struct iommu_domain *domain = &smmu_domain->domain; 1985 1986 iommu_iotlb_gather_add_page(domain, gather, iova, granule); 1987 } 1988 1989 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size, 1990 size_t granule, void *cookie) 1991 { 1992 arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie); 1993 } 1994 1995 static const struct iommu_flush_ops arm_smmu_flush_ops = { 1996 .tlb_flush_all = arm_smmu_tlb_inv_context, 1997 .tlb_flush_walk = arm_smmu_tlb_inv_walk, 1998 .tlb_add_page = arm_smmu_tlb_inv_page_nosync, 1999 }; 2000 2001 /* IOMMU API */ 2002 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) 2003 { 2004 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2005 2006 switch (cap) { 2007 case IOMMU_CAP_CACHE_COHERENCY: 2008 /* Assume that a coherent TCU implies coherent TBUs */ 2009 return master->smmu->features & ARM_SMMU_FEAT_COHERENCY; 2010 case IOMMU_CAP_NOEXEC: 2011 return true; 2012 default: 2013 return false; 2014 } 2015 } 2016 2017 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) 2018 { 2019 struct arm_smmu_domain *smmu_domain; 2020 2021 if (type == IOMMU_DOMAIN_SVA) 2022 return arm_smmu_sva_domain_alloc(); 2023 2024 if (type != IOMMU_DOMAIN_UNMANAGED && 2025 type != IOMMU_DOMAIN_DMA && 2026 type != IOMMU_DOMAIN_DMA_FQ && 2027 type != IOMMU_DOMAIN_IDENTITY) 2028 return NULL; 2029 2030 /* 2031 * Allocate the domain and initialise some of its data structures. 2032 * We can't really do anything meaningful until we've added a 2033 * master. 2034 */ 2035 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 2036 if (!smmu_domain) 2037 return NULL; 2038 2039 mutex_init(&smmu_domain->init_mutex); 2040 INIT_LIST_HEAD(&smmu_domain->devices); 2041 spin_lock_init(&smmu_domain->devices_lock); 2042 INIT_LIST_HEAD(&smmu_domain->mmu_notifiers); 2043 2044 return &smmu_domain->domain; 2045 } 2046 2047 static int arm_smmu_bitmap_alloc(unsigned long *map, int span) 2048 { 2049 int idx, size = 1 << span; 2050 2051 do { 2052 idx = find_first_zero_bit(map, size); 2053 if (idx == size) 2054 return -ENOSPC; 2055 } while (test_and_set_bit(idx, map)); 2056 2057 return idx; 2058 } 2059 2060 static void arm_smmu_bitmap_free(unsigned long *map, int idx) 2061 { 2062 clear_bit(idx, map); 2063 } 2064 2065 static void arm_smmu_domain_free(struct iommu_domain *domain) 2066 { 2067 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2068 struct arm_smmu_device *smmu = smmu_domain->smmu; 2069 2070 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 2071 2072 /* Free the CD and ASID, if we allocated them */ 2073 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 2074 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; 2075 2076 /* Prevent SVA from touching the CD while we're freeing it */ 2077 mutex_lock(&arm_smmu_asid_lock); 2078 if (cfg->cdcfg.cdtab) 2079 arm_smmu_free_cd_tables(smmu_domain); 2080 arm_smmu_free_asid(&cfg->cd); 2081 mutex_unlock(&arm_smmu_asid_lock); 2082 } else { 2083 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; 2084 if (cfg->vmid) 2085 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid); 2086 } 2087 2088 kfree(smmu_domain); 2089 } 2090 2091 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, 2092 struct arm_smmu_master *master, 2093 struct io_pgtable_cfg *pgtbl_cfg) 2094 { 2095 int ret; 2096 u32 asid; 2097 struct arm_smmu_device *smmu = smmu_domain->smmu; 2098 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; 2099 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr; 2100 2101 refcount_set(&cfg->cd.refs, 1); 2102 2103 /* Prevent SVA from modifying the ASID until it is written to the CD */ 2104 mutex_lock(&arm_smmu_asid_lock); 2105 ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd, 2106 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 2107 if (ret) 2108 goto out_unlock; 2109 2110 cfg->s1cdmax = master->ssid_bits; 2111 2112 smmu_domain->stall_enabled = master->stall_enabled; 2113 2114 ret = arm_smmu_alloc_cd_tables(smmu_domain); 2115 if (ret) 2116 goto out_free_asid; 2117 2118 cfg->cd.asid = (u16)asid; 2119 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 2120 cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) | 2121 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) | 2122 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) | 2123 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) | 2124 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) | 2125 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) | 2126 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; 2127 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair; 2128 2129 /* 2130 * Note that this will end up calling arm_smmu_sync_cd() before 2131 * the master has been added to the devices list for this domain. 2132 * This isn't an issue because the STE hasn't been installed yet. 2133 */ 2134 ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd); 2135 if (ret) 2136 goto out_free_cd_tables; 2137 2138 mutex_unlock(&arm_smmu_asid_lock); 2139 return 0; 2140 2141 out_free_cd_tables: 2142 arm_smmu_free_cd_tables(smmu_domain); 2143 out_free_asid: 2144 arm_smmu_free_asid(&cfg->cd); 2145 out_unlock: 2146 mutex_unlock(&arm_smmu_asid_lock); 2147 return ret; 2148 } 2149 2150 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, 2151 struct arm_smmu_master *master, 2152 struct io_pgtable_cfg *pgtbl_cfg) 2153 { 2154 int vmid; 2155 struct arm_smmu_device *smmu = smmu_domain->smmu; 2156 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; 2157 typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr; 2158 2159 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits); 2160 if (vmid < 0) 2161 return vmid; 2162 2163 vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr; 2164 cfg->vmid = (u16)vmid; 2165 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; 2166 cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) | 2167 FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) | 2168 FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) | 2169 FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) | 2170 FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) | 2171 FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) | 2172 FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps); 2173 return 0; 2174 } 2175 2176 static int arm_smmu_domain_finalise(struct iommu_domain *domain, 2177 struct arm_smmu_master *master) 2178 { 2179 int ret; 2180 unsigned long ias, oas; 2181 enum io_pgtable_fmt fmt; 2182 struct io_pgtable_cfg pgtbl_cfg; 2183 struct io_pgtable_ops *pgtbl_ops; 2184 int (*finalise_stage_fn)(struct arm_smmu_domain *, 2185 struct arm_smmu_master *, 2186 struct io_pgtable_cfg *); 2187 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2188 struct arm_smmu_device *smmu = smmu_domain->smmu; 2189 2190 if (domain->type == IOMMU_DOMAIN_IDENTITY) { 2191 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; 2192 return 0; 2193 } 2194 2195 /* Restrict the stage to what we can actually support */ 2196 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) 2197 smmu_domain->stage = ARM_SMMU_DOMAIN_S2; 2198 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 2199 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 2200 2201 switch (smmu_domain->stage) { 2202 case ARM_SMMU_DOMAIN_S1: 2203 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; 2204 ias = min_t(unsigned long, ias, VA_BITS); 2205 oas = smmu->ias; 2206 fmt = ARM_64_LPAE_S1; 2207 finalise_stage_fn = arm_smmu_domain_finalise_s1; 2208 break; 2209 case ARM_SMMU_DOMAIN_NESTED: 2210 case ARM_SMMU_DOMAIN_S2: 2211 ias = smmu->ias; 2212 oas = smmu->oas; 2213 fmt = ARM_64_LPAE_S2; 2214 finalise_stage_fn = arm_smmu_domain_finalise_s2; 2215 break; 2216 default: 2217 return -EINVAL; 2218 } 2219 2220 pgtbl_cfg = (struct io_pgtable_cfg) { 2221 .pgsize_bitmap = smmu->pgsize_bitmap, 2222 .ias = ias, 2223 .oas = oas, 2224 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, 2225 .tlb = &arm_smmu_flush_ops, 2226 .iommu_dev = smmu->dev, 2227 }; 2228 2229 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 2230 if (!pgtbl_ops) 2231 return -ENOMEM; 2232 2233 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 2234 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; 2235 domain->geometry.force_aperture = true; 2236 2237 ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg); 2238 if (ret < 0) { 2239 free_io_pgtable_ops(pgtbl_ops); 2240 return ret; 2241 } 2242 2243 smmu_domain->pgtbl_ops = pgtbl_ops; 2244 return 0; 2245 } 2246 2247 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) 2248 { 2249 __le64 *step; 2250 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 2251 2252 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { 2253 struct arm_smmu_strtab_l1_desc *l1_desc; 2254 int idx; 2255 2256 /* Two-level walk */ 2257 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS; 2258 l1_desc = &cfg->l1_desc[idx]; 2259 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS; 2260 step = &l1_desc->l2ptr[idx]; 2261 } else { 2262 /* Simple linear lookup */ 2263 step = &cfg->strtab[sid * STRTAB_STE_DWORDS]; 2264 } 2265 2266 return step; 2267 } 2268 2269 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master) 2270 { 2271 int i, j; 2272 struct arm_smmu_device *smmu = master->smmu; 2273 2274 for (i = 0; i < master->num_streams; ++i) { 2275 u32 sid = master->streams[i].id; 2276 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); 2277 2278 /* Bridged PCI devices may end up with duplicated IDs */ 2279 for (j = 0; j < i; j++) 2280 if (master->streams[j].id == sid) 2281 break; 2282 if (j < i) 2283 continue; 2284 2285 arm_smmu_write_strtab_ent(master, sid, step); 2286 } 2287 } 2288 2289 static bool arm_smmu_ats_supported(struct arm_smmu_master *master) 2290 { 2291 struct device *dev = master->dev; 2292 struct arm_smmu_device *smmu = master->smmu; 2293 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2294 2295 if (!(smmu->features & ARM_SMMU_FEAT_ATS)) 2296 return false; 2297 2298 if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS)) 2299 return false; 2300 2301 return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev)); 2302 } 2303 2304 static void arm_smmu_enable_ats(struct arm_smmu_master *master) 2305 { 2306 size_t stu; 2307 struct pci_dev *pdev; 2308 struct arm_smmu_device *smmu = master->smmu; 2309 struct arm_smmu_domain *smmu_domain = master->domain; 2310 2311 /* Don't enable ATS at the endpoint if it's not enabled in the STE */ 2312 if (!master->ats_enabled) 2313 return; 2314 2315 /* Smallest Translation Unit: log2 of the smallest supported granule */ 2316 stu = __ffs(smmu->pgsize_bitmap); 2317 pdev = to_pci_dev(master->dev); 2318 2319 atomic_inc(&smmu_domain->nr_ats_masters); 2320 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0); 2321 if (pci_enable_ats(pdev, stu)) 2322 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu); 2323 } 2324 2325 static void arm_smmu_disable_ats(struct arm_smmu_master *master) 2326 { 2327 struct arm_smmu_domain *smmu_domain = master->domain; 2328 2329 if (!master->ats_enabled) 2330 return; 2331 2332 pci_disable_ats(to_pci_dev(master->dev)); 2333 /* 2334 * Ensure ATS is disabled at the endpoint before we issue the 2335 * ATC invalidation via the SMMU. 2336 */ 2337 wmb(); 2338 arm_smmu_atc_inv_master(master); 2339 atomic_dec(&smmu_domain->nr_ats_masters); 2340 } 2341 2342 static int arm_smmu_enable_pasid(struct arm_smmu_master *master) 2343 { 2344 int ret; 2345 int features; 2346 int num_pasids; 2347 struct pci_dev *pdev; 2348 2349 if (!dev_is_pci(master->dev)) 2350 return -ENODEV; 2351 2352 pdev = to_pci_dev(master->dev); 2353 2354 features = pci_pasid_features(pdev); 2355 if (features < 0) 2356 return features; 2357 2358 num_pasids = pci_max_pasids(pdev); 2359 if (num_pasids <= 0) 2360 return num_pasids; 2361 2362 ret = pci_enable_pasid(pdev, features); 2363 if (ret) { 2364 dev_err(&pdev->dev, "Failed to enable PASID\n"); 2365 return ret; 2366 } 2367 2368 master->ssid_bits = min_t(u8, ilog2(num_pasids), 2369 master->smmu->ssid_bits); 2370 return 0; 2371 } 2372 2373 static void arm_smmu_disable_pasid(struct arm_smmu_master *master) 2374 { 2375 struct pci_dev *pdev; 2376 2377 if (!dev_is_pci(master->dev)) 2378 return; 2379 2380 pdev = to_pci_dev(master->dev); 2381 2382 if (!pdev->pasid_enabled) 2383 return; 2384 2385 master->ssid_bits = 0; 2386 pci_disable_pasid(pdev); 2387 } 2388 2389 static void arm_smmu_detach_dev(struct arm_smmu_master *master) 2390 { 2391 unsigned long flags; 2392 struct arm_smmu_domain *smmu_domain = master->domain; 2393 2394 if (!smmu_domain) 2395 return; 2396 2397 arm_smmu_disable_ats(master); 2398 2399 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2400 list_del(&master->domain_head); 2401 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 2402 2403 master->domain = NULL; 2404 master->ats_enabled = false; 2405 arm_smmu_install_ste_for_dev(master); 2406 } 2407 2408 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 2409 { 2410 int ret = 0; 2411 unsigned long flags; 2412 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2413 struct arm_smmu_device *smmu; 2414 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2415 struct arm_smmu_master *master; 2416 2417 if (!fwspec) 2418 return -ENOENT; 2419 2420 master = dev_iommu_priv_get(dev); 2421 smmu = master->smmu; 2422 2423 /* 2424 * Checking that SVA is disabled ensures that this device isn't bound to 2425 * any mm, and can be safely detached from its old domain. Bonds cannot 2426 * be removed concurrently since we're holding the group mutex. 2427 */ 2428 if (arm_smmu_master_sva_enabled(master)) { 2429 dev_err(dev, "cannot attach - SVA enabled\n"); 2430 return -EBUSY; 2431 } 2432 2433 arm_smmu_detach_dev(master); 2434 2435 mutex_lock(&smmu_domain->init_mutex); 2436 2437 if (!smmu_domain->smmu) { 2438 smmu_domain->smmu = smmu; 2439 ret = arm_smmu_domain_finalise(domain, master); 2440 if (ret) { 2441 smmu_domain->smmu = NULL; 2442 goto out_unlock; 2443 } 2444 } else if (smmu_domain->smmu != smmu) { 2445 ret = -EINVAL; 2446 goto out_unlock; 2447 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && 2448 master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) { 2449 ret = -EINVAL; 2450 goto out_unlock; 2451 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && 2452 smmu_domain->stall_enabled != master->stall_enabled) { 2453 ret = -EINVAL; 2454 goto out_unlock; 2455 } 2456 2457 master->domain = smmu_domain; 2458 2459 /* 2460 * The SMMU does not support enabling ATS with bypass. When the STE is 2461 * in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and 2462 * Translated transactions are denied as though ATS is disabled for the 2463 * stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and 2464 * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry). 2465 */ 2466 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS) 2467 master->ats_enabled = arm_smmu_ats_supported(master); 2468 2469 arm_smmu_install_ste_for_dev(master); 2470 2471 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2472 list_add(&master->domain_head, &smmu_domain->devices); 2473 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 2474 2475 arm_smmu_enable_ats(master); 2476 2477 out_unlock: 2478 mutex_unlock(&smmu_domain->init_mutex); 2479 return ret; 2480 } 2481 2482 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, 2483 phys_addr_t paddr, size_t pgsize, size_t pgcount, 2484 int prot, gfp_t gfp, size_t *mapped) 2485 { 2486 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 2487 2488 if (!ops) 2489 return -ENODEV; 2490 2491 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped); 2492 } 2493 2494 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, 2495 size_t pgsize, size_t pgcount, 2496 struct iommu_iotlb_gather *gather) 2497 { 2498 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2499 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 2500 2501 if (!ops) 2502 return 0; 2503 2504 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); 2505 } 2506 2507 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) 2508 { 2509 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2510 2511 if (smmu_domain->smmu) 2512 arm_smmu_tlb_inv_context(smmu_domain); 2513 } 2514 2515 static void arm_smmu_iotlb_sync(struct iommu_domain *domain, 2516 struct iommu_iotlb_gather *gather) 2517 { 2518 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2519 2520 if (!gather->pgsize) 2521 return; 2522 2523 arm_smmu_tlb_inv_range_domain(gather->start, 2524 gather->end - gather->start + 1, 2525 gather->pgsize, true, smmu_domain); 2526 } 2527 2528 static phys_addr_t 2529 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 2530 { 2531 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 2532 2533 if (!ops) 2534 return 0; 2535 2536 return ops->iova_to_phys(ops, iova); 2537 } 2538 2539 static struct platform_driver arm_smmu_driver; 2540 2541 static 2542 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) 2543 { 2544 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver, 2545 fwnode); 2546 put_device(dev); 2547 return dev ? dev_get_drvdata(dev) : NULL; 2548 } 2549 2550 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) 2551 { 2552 unsigned long limit = smmu->strtab_cfg.num_l1_ents; 2553 2554 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) 2555 limit *= 1UL << STRTAB_SPLIT; 2556 2557 return sid < limit; 2558 } 2559 2560 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid) 2561 { 2562 /* Check the SIDs are in range of the SMMU and our stream table */ 2563 if (!arm_smmu_sid_in_range(smmu, sid)) 2564 return -ERANGE; 2565 2566 /* Ensure l2 strtab is initialised */ 2567 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) 2568 return arm_smmu_init_l2_strtab(smmu, sid); 2569 2570 return 0; 2571 } 2572 2573 static int arm_smmu_insert_master(struct arm_smmu_device *smmu, 2574 struct arm_smmu_master *master) 2575 { 2576 int i; 2577 int ret = 0; 2578 struct arm_smmu_stream *new_stream, *cur_stream; 2579 struct rb_node **new_node, *parent_node = NULL; 2580 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); 2581 2582 master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams), 2583 GFP_KERNEL); 2584 if (!master->streams) 2585 return -ENOMEM; 2586 master->num_streams = fwspec->num_ids; 2587 2588 mutex_lock(&smmu->streams_mutex); 2589 for (i = 0; i < fwspec->num_ids; i++) { 2590 u32 sid = fwspec->ids[i]; 2591 2592 new_stream = &master->streams[i]; 2593 new_stream->id = sid; 2594 new_stream->master = master; 2595 2596 ret = arm_smmu_init_sid_strtab(smmu, sid); 2597 if (ret) 2598 break; 2599 2600 /* Insert into SID tree */ 2601 new_node = &(smmu->streams.rb_node); 2602 while (*new_node) { 2603 cur_stream = rb_entry(*new_node, struct arm_smmu_stream, 2604 node); 2605 parent_node = *new_node; 2606 if (cur_stream->id > new_stream->id) { 2607 new_node = &((*new_node)->rb_left); 2608 } else if (cur_stream->id < new_stream->id) { 2609 new_node = &((*new_node)->rb_right); 2610 } else { 2611 dev_warn(master->dev, 2612 "stream %u already in tree\n", 2613 cur_stream->id); 2614 ret = -EINVAL; 2615 break; 2616 } 2617 } 2618 if (ret) 2619 break; 2620 2621 rb_link_node(&new_stream->node, parent_node, new_node); 2622 rb_insert_color(&new_stream->node, &smmu->streams); 2623 } 2624 2625 if (ret) { 2626 for (i--; i >= 0; i--) 2627 rb_erase(&master->streams[i].node, &smmu->streams); 2628 kfree(master->streams); 2629 } 2630 mutex_unlock(&smmu->streams_mutex); 2631 2632 return ret; 2633 } 2634 2635 static void arm_smmu_remove_master(struct arm_smmu_master *master) 2636 { 2637 int i; 2638 struct arm_smmu_device *smmu = master->smmu; 2639 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); 2640 2641 if (!smmu || !master->streams) 2642 return; 2643 2644 mutex_lock(&smmu->streams_mutex); 2645 for (i = 0; i < fwspec->num_ids; i++) 2646 rb_erase(&master->streams[i].node, &smmu->streams); 2647 mutex_unlock(&smmu->streams_mutex); 2648 2649 kfree(master->streams); 2650 } 2651 2652 static struct iommu_ops arm_smmu_ops; 2653 2654 static struct iommu_device *arm_smmu_probe_device(struct device *dev) 2655 { 2656 int ret; 2657 struct arm_smmu_device *smmu; 2658 struct arm_smmu_master *master; 2659 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2660 2661 if (!fwspec || fwspec->ops != &arm_smmu_ops) 2662 return ERR_PTR(-ENODEV); 2663 2664 if (WARN_ON_ONCE(dev_iommu_priv_get(dev))) 2665 return ERR_PTR(-EBUSY); 2666 2667 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); 2668 if (!smmu) 2669 return ERR_PTR(-ENODEV); 2670 2671 master = kzalloc(sizeof(*master), GFP_KERNEL); 2672 if (!master) 2673 return ERR_PTR(-ENOMEM); 2674 2675 master->dev = dev; 2676 master->smmu = smmu; 2677 INIT_LIST_HEAD(&master->bonds); 2678 dev_iommu_priv_set(dev, master); 2679 2680 ret = arm_smmu_insert_master(smmu, master); 2681 if (ret) 2682 goto err_free_master; 2683 2684 device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits); 2685 master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits); 2686 2687 /* 2688 * Note that PASID must be enabled before, and disabled after ATS: 2689 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register 2690 * 2691 * Behavior is undefined if this bit is Set and the value of the PASID 2692 * Enable, Execute Requested Enable, or Privileged Mode Requested bits 2693 * are changed. 2694 */ 2695 arm_smmu_enable_pasid(master); 2696 2697 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB)) 2698 master->ssid_bits = min_t(u8, master->ssid_bits, 2699 CTXDESC_LINEAR_CDMAX); 2700 2701 if ((smmu->features & ARM_SMMU_FEAT_STALLS && 2702 device_property_read_bool(dev, "dma-can-stall")) || 2703 smmu->features & ARM_SMMU_FEAT_STALL_FORCE) 2704 master->stall_enabled = true; 2705 2706 return &smmu->iommu; 2707 2708 err_free_master: 2709 kfree(master); 2710 dev_iommu_priv_set(dev, NULL); 2711 return ERR_PTR(ret); 2712 } 2713 2714 static void arm_smmu_release_device(struct device *dev) 2715 { 2716 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2717 2718 if (WARN_ON(arm_smmu_master_sva_enabled(master))) 2719 iopf_queue_remove_device(master->smmu->evtq.iopf, dev); 2720 arm_smmu_detach_dev(master); 2721 arm_smmu_disable_pasid(master); 2722 arm_smmu_remove_master(master); 2723 kfree(master); 2724 } 2725 2726 static struct iommu_group *arm_smmu_device_group(struct device *dev) 2727 { 2728 struct iommu_group *group; 2729 2730 /* 2731 * We don't support devices sharing stream IDs other than PCI RID 2732 * aliases, since the necessary ID-to-device lookup becomes rather 2733 * impractical given a potential sparse 32-bit stream ID space. 2734 */ 2735 if (dev_is_pci(dev)) 2736 group = pci_device_group(dev); 2737 else 2738 group = generic_device_group(dev); 2739 2740 return group; 2741 } 2742 2743 static int arm_smmu_enable_nesting(struct iommu_domain *domain) 2744 { 2745 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2746 int ret = 0; 2747 2748 mutex_lock(&smmu_domain->init_mutex); 2749 if (smmu_domain->smmu) 2750 ret = -EPERM; 2751 else 2752 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; 2753 mutex_unlock(&smmu_domain->init_mutex); 2754 2755 return ret; 2756 } 2757 2758 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) 2759 { 2760 return iommu_fwspec_add_ids(dev, args->args, 1); 2761 } 2762 2763 static void arm_smmu_get_resv_regions(struct device *dev, 2764 struct list_head *head) 2765 { 2766 struct iommu_resv_region *region; 2767 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 2768 2769 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, 2770 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL); 2771 if (!region) 2772 return; 2773 2774 list_add_tail(®ion->list, head); 2775 2776 iommu_dma_get_resv_regions(dev, head); 2777 } 2778 2779 static int arm_smmu_dev_enable_feature(struct device *dev, 2780 enum iommu_dev_features feat) 2781 { 2782 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2783 2784 if (!master) 2785 return -ENODEV; 2786 2787 switch (feat) { 2788 case IOMMU_DEV_FEAT_IOPF: 2789 if (!arm_smmu_master_iopf_supported(master)) 2790 return -EINVAL; 2791 if (master->iopf_enabled) 2792 return -EBUSY; 2793 master->iopf_enabled = true; 2794 return 0; 2795 case IOMMU_DEV_FEAT_SVA: 2796 if (!arm_smmu_master_sva_supported(master)) 2797 return -EINVAL; 2798 if (arm_smmu_master_sva_enabled(master)) 2799 return -EBUSY; 2800 return arm_smmu_master_enable_sva(master); 2801 default: 2802 return -EINVAL; 2803 } 2804 } 2805 2806 static int arm_smmu_dev_disable_feature(struct device *dev, 2807 enum iommu_dev_features feat) 2808 { 2809 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2810 2811 if (!master) 2812 return -EINVAL; 2813 2814 switch (feat) { 2815 case IOMMU_DEV_FEAT_IOPF: 2816 if (!master->iopf_enabled) 2817 return -EINVAL; 2818 if (master->sva_enabled) 2819 return -EBUSY; 2820 master->iopf_enabled = false; 2821 return 0; 2822 case IOMMU_DEV_FEAT_SVA: 2823 if (!arm_smmu_master_sva_enabled(master)) 2824 return -EINVAL; 2825 return arm_smmu_master_disable_sva(master); 2826 default: 2827 return -EINVAL; 2828 } 2829 } 2830 2831 /* 2832 * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the 2833 * PCIe link and save the data to memory by DMA. The hardware is restricted to 2834 * use identity mapping only. 2835 */ 2836 #define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \ 2837 (pdev)->device == 0xa12e) 2838 2839 static int arm_smmu_def_domain_type(struct device *dev) 2840 { 2841 if (dev_is_pci(dev)) { 2842 struct pci_dev *pdev = to_pci_dev(dev); 2843 2844 if (IS_HISI_PTT_DEVICE(pdev)) 2845 return IOMMU_DOMAIN_IDENTITY; 2846 } 2847 2848 return 0; 2849 } 2850 2851 static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid) 2852 { 2853 struct iommu_domain *domain; 2854 2855 domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA); 2856 if (WARN_ON(IS_ERR(domain)) || !domain) 2857 return; 2858 2859 arm_smmu_sva_remove_dev_pasid(domain, dev, pasid); 2860 } 2861 2862 static struct iommu_ops arm_smmu_ops = { 2863 .capable = arm_smmu_capable, 2864 .domain_alloc = arm_smmu_domain_alloc, 2865 .probe_device = arm_smmu_probe_device, 2866 .release_device = arm_smmu_release_device, 2867 .device_group = arm_smmu_device_group, 2868 .of_xlate = arm_smmu_of_xlate, 2869 .get_resv_regions = arm_smmu_get_resv_regions, 2870 .remove_dev_pasid = arm_smmu_remove_dev_pasid, 2871 .dev_enable_feat = arm_smmu_dev_enable_feature, 2872 .dev_disable_feat = arm_smmu_dev_disable_feature, 2873 .page_response = arm_smmu_page_response, 2874 .def_domain_type = arm_smmu_def_domain_type, 2875 .pgsize_bitmap = -1UL, /* Restricted during device attach */ 2876 .owner = THIS_MODULE, 2877 .default_domain_ops = &(const struct iommu_domain_ops) { 2878 .attach_dev = arm_smmu_attach_dev, 2879 .map_pages = arm_smmu_map_pages, 2880 .unmap_pages = arm_smmu_unmap_pages, 2881 .flush_iotlb_all = arm_smmu_flush_iotlb_all, 2882 .iotlb_sync = arm_smmu_iotlb_sync, 2883 .iova_to_phys = arm_smmu_iova_to_phys, 2884 .enable_nesting = arm_smmu_enable_nesting, 2885 .free = arm_smmu_domain_free, 2886 } 2887 }; 2888 2889 /* Probing and initialisation functions */ 2890 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, 2891 struct arm_smmu_queue *q, 2892 void __iomem *page, 2893 unsigned long prod_off, 2894 unsigned long cons_off, 2895 size_t dwords, const char *name) 2896 { 2897 size_t qsz; 2898 2899 do { 2900 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3; 2901 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, 2902 GFP_KERNEL); 2903 if (q->base || qsz < PAGE_SIZE) 2904 break; 2905 2906 q->llq.max_n_shift--; 2907 } while (1); 2908 2909 if (!q->base) { 2910 dev_err(smmu->dev, 2911 "failed to allocate queue (0x%zx bytes) for %s\n", 2912 qsz, name); 2913 return -ENOMEM; 2914 } 2915 2916 if (!WARN_ON(q->base_dma & (qsz - 1))) { 2917 dev_info(smmu->dev, "allocated %u entries for %s\n", 2918 1 << q->llq.max_n_shift, name); 2919 } 2920 2921 q->prod_reg = page + prod_off; 2922 q->cons_reg = page + cons_off; 2923 q->ent_dwords = dwords; 2924 2925 q->q_base = Q_BASE_RWA; 2926 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK; 2927 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift); 2928 2929 q->llq.prod = q->llq.cons = 0; 2930 return 0; 2931 } 2932 2933 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu) 2934 { 2935 struct arm_smmu_cmdq *cmdq = &smmu->cmdq; 2936 unsigned int nents = 1 << cmdq->q.llq.max_n_shift; 2937 2938 atomic_set(&cmdq->owner_prod, 0); 2939 atomic_set(&cmdq->lock, 0); 2940 2941 cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents, 2942 GFP_KERNEL); 2943 if (!cmdq->valid_map) 2944 return -ENOMEM; 2945 2946 return 0; 2947 } 2948 2949 static int arm_smmu_init_queues(struct arm_smmu_device *smmu) 2950 { 2951 int ret; 2952 2953 /* cmdq */ 2954 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base, 2955 ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS, 2956 CMDQ_ENT_DWORDS, "cmdq"); 2957 if (ret) 2958 return ret; 2959 2960 ret = arm_smmu_cmdq_init(smmu); 2961 if (ret) 2962 return ret; 2963 2964 /* evtq */ 2965 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1, 2966 ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS, 2967 EVTQ_ENT_DWORDS, "evtq"); 2968 if (ret) 2969 return ret; 2970 2971 if ((smmu->features & ARM_SMMU_FEAT_SVA) && 2972 (smmu->features & ARM_SMMU_FEAT_STALLS)) { 2973 smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev)); 2974 if (!smmu->evtq.iopf) 2975 return -ENOMEM; 2976 } 2977 2978 /* priq */ 2979 if (!(smmu->features & ARM_SMMU_FEAT_PRI)) 2980 return 0; 2981 2982 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1, 2983 ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS, 2984 PRIQ_ENT_DWORDS, "priq"); 2985 } 2986 2987 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) 2988 { 2989 unsigned int i; 2990 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 2991 void *strtab = smmu->strtab_cfg.strtab; 2992 2993 cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents, 2994 sizeof(*cfg->l1_desc), GFP_KERNEL); 2995 if (!cfg->l1_desc) 2996 return -ENOMEM; 2997 2998 for (i = 0; i < cfg->num_l1_ents; ++i) { 2999 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]); 3000 strtab += STRTAB_L1_DESC_DWORDS << 3; 3001 } 3002 3003 return 0; 3004 } 3005 3006 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) 3007 { 3008 void *strtab; 3009 u64 reg; 3010 u32 size, l1size; 3011 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 3012 3013 /* Calculate the L1 size, capped to the SIDSIZE. */ 3014 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); 3015 size = min(size, smmu->sid_bits - STRTAB_SPLIT); 3016 cfg->num_l1_ents = 1 << size; 3017 3018 size += STRTAB_SPLIT; 3019 if (size < smmu->sid_bits) 3020 dev_warn(smmu->dev, 3021 "2-level strtab only covers %u/%u bits of SID\n", 3022 size, smmu->sid_bits); 3023 3024 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3); 3025 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, 3026 GFP_KERNEL); 3027 if (!strtab) { 3028 dev_err(smmu->dev, 3029 "failed to allocate l1 stream table (%u bytes)\n", 3030 l1size); 3031 return -ENOMEM; 3032 } 3033 cfg->strtab = strtab; 3034 3035 /* Configure strtab_base_cfg for 2 levels */ 3036 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL); 3037 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size); 3038 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT); 3039 cfg->strtab_base_cfg = reg; 3040 3041 return arm_smmu_init_l1_strtab(smmu); 3042 } 3043 3044 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) 3045 { 3046 void *strtab; 3047 u64 reg; 3048 u32 size; 3049 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 3050 3051 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3); 3052 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma, 3053 GFP_KERNEL); 3054 if (!strtab) { 3055 dev_err(smmu->dev, 3056 "failed to allocate linear stream table (%u bytes)\n", 3057 size); 3058 return -ENOMEM; 3059 } 3060 cfg->strtab = strtab; 3061 cfg->num_l1_ents = 1 << smmu->sid_bits; 3062 3063 /* Configure strtab_base_cfg for a linear table covering all SIDs */ 3064 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR); 3065 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); 3066 cfg->strtab_base_cfg = reg; 3067 3068 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false); 3069 return 0; 3070 } 3071 3072 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) 3073 { 3074 u64 reg; 3075 int ret; 3076 3077 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) 3078 ret = arm_smmu_init_strtab_2lvl(smmu); 3079 else 3080 ret = arm_smmu_init_strtab_linear(smmu); 3081 3082 if (ret) 3083 return ret; 3084 3085 /* Set the strtab base address */ 3086 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK; 3087 reg |= STRTAB_BASE_RA; 3088 smmu->strtab_cfg.strtab_base = reg; 3089 3090 /* Allocate the first VMID for stage-2 bypass STEs */ 3091 set_bit(0, smmu->vmid_map); 3092 return 0; 3093 } 3094 3095 static int arm_smmu_init_structures(struct arm_smmu_device *smmu) 3096 { 3097 int ret; 3098 3099 mutex_init(&smmu->streams_mutex); 3100 smmu->streams = RB_ROOT; 3101 3102 ret = arm_smmu_init_queues(smmu); 3103 if (ret) 3104 return ret; 3105 3106 return arm_smmu_init_strtab(smmu); 3107 } 3108 3109 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, 3110 unsigned int reg_off, unsigned int ack_off) 3111 { 3112 u32 reg; 3113 3114 writel_relaxed(val, smmu->base + reg_off); 3115 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, 3116 1, ARM_SMMU_POLL_TIMEOUT_US); 3117 } 3118 3119 /* GBPA is "special" */ 3120 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr) 3121 { 3122 int ret; 3123 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA; 3124 3125 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), 3126 1, ARM_SMMU_POLL_TIMEOUT_US); 3127 if (ret) 3128 return ret; 3129 3130 reg &= ~clr; 3131 reg |= set; 3132 writel_relaxed(reg | GBPA_UPDATE, gbpa); 3133 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), 3134 1, ARM_SMMU_POLL_TIMEOUT_US); 3135 3136 if (ret) 3137 dev_err(smmu->dev, "GBPA not responding to update\n"); 3138 return ret; 3139 } 3140 3141 static void arm_smmu_free_msis(void *data) 3142 { 3143 struct device *dev = data; 3144 platform_msi_domain_free_irqs(dev); 3145 } 3146 3147 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) 3148 { 3149 phys_addr_t doorbell; 3150 struct device *dev = msi_desc_to_dev(desc); 3151 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 3152 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index]; 3153 3154 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; 3155 doorbell &= MSI_CFG0_ADDR_MASK; 3156 3157 writeq_relaxed(doorbell, smmu->base + cfg[0]); 3158 writel_relaxed(msg->data, smmu->base + cfg[1]); 3159 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); 3160 } 3161 3162 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) 3163 { 3164 int ret, nvec = ARM_SMMU_MAX_MSIS; 3165 struct device *dev = smmu->dev; 3166 3167 /* Clear the MSI address regs */ 3168 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0); 3169 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0); 3170 3171 if (smmu->features & ARM_SMMU_FEAT_PRI) 3172 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0); 3173 else 3174 nvec--; 3175 3176 if (!(smmu->features & ARM_SMMU_FEAT_MSI)) 3177 return; 3178 3179 if (!dev->msi.domain) { 3180 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n"); 3181 return; 3182 } 3183 3184 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */ 3185 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg); 3186 if (ret) { 3187 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n"); 3188 return; 3189 } 3190 3191 smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX); 3192 smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX); 3193 smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX); 3194 3195 /* Add callback to free MSIs on teardown */ 3196 devm_add_action(dev, arm_smmu_free_msis, dev); 3197 } 3198 3199 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) 3200 { 3201 int irq, ret; 3202 3203 arm_smmu_setup_msis(smmu); 3204 3205 /* Request interrupt lines */ 3206 irq = smmu->evtq.q.irq; 3207 if (irq) { 3208 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, 3209 arm_smmu_evtq_thread, 3210 IRQF_ONESHOT, 3211 "arm-smmu-v3-evtq", smmu); 3212 if (ret < 0) 3213 dev_warn(smmu->dev, "failed to enable evtq irq\n"); 3214 } else { 3215 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n"); 3216 } 3217 3218 irq = smmu->gerr_irq; 3219 if (irq) { 3220 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, 3221 0, "arm-smmu-v3-gerror", smmu); 3222 if (ret < 0) 3223 dev_warn(smmu->dev, "failed to enable gerror irq\n"); 3224 } else { 3225 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n"); 3226 } 3227 3228 if (smmu->features & ARM_SMMU_FEAT_PRI) { 3229 irq = smmu->priq.q.irq; 3230 if (irq) { 3231 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, 3232 arm_smmu_priq_thread, 3233 IRQF_ONESHOT, 3234 "arm-smmu-v3-priq", 3235 smmu); 3236 if (ret < 0) 3237 dev_warn(smmu->dev, 3238 "failed to enable priq irq\n"); 3239 } else { 3240 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n"); 3241 } 3242 } 3243 } 3244 3245 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) 3246 { 3247 int ret, irq; 3248 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; 3249 3250 /* Disable IRQs first */ 3251 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, 3252 ARM_SMMU_IRQ_CTRLACK); 3253 if (ret) { 3254 dev_err(smmu->dev, "failed to disable irqs\n"); 3255 return ret; 3256 } 3257 3258 irq = smmu->combined_irq; 3259 if (irq) { 3260 /* 3261 * Cavium ThunderX2 implementation doesn't support unique irq 3262 * lines. Use a single irq line for all the SMMUv3 interrupts. 3263 */ 3264 ret = devm_request_threaded_irq(smmu->dev, irq, 3265 arm_smmu_combined_irq_handler, 3266 arm_smmu_combined_irq_thread, 3267 IRQF_ONESHOT, 3268 "arm-smmu-v3-combined-irq", smmu); 3269 if (ret < 0) 3270 dev_warn(smmu->dev, "failed to enable combined irq\n"); 3271 } else 3272 arm_smmu_setup_unique_irqs(smmu); 3273 3274 if (smmu->features & ARM_SMMU_FEAT_PRI) 3275 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; 3276 3277 /* Enable interrupt generation on the SMMU */ 3278 ret = arm_smmu_write_reg_sync(smmu, irqen_flags, 3279 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK); 3280 if (ret) 3281 dev_warn(smmu->dev, "failed to enable irqs\n"); 3282 3283 return 0; 3284 } 3285 3286 static int arm_smmu_device_disable(struct arm_smmu_device *smmu) 3287 { 3288 int ret; 3289 3290 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); 3291 if (ret) 3292 dev_err(smmu->dev, "failed to clear cr0\n"); 3293 3294 return ret; 3295 } 3296 3297 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) 3298 { 3299 int ret; 3300 u32 reg, enables; 3301 struct arm_smmu_cmdq_ent cmd; 3302 3303 /* Clear CR0 and sync (disables SMMU and queue processing) */ 3304 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); 3305 if (reg & CR0_SMMUEN) { 3306 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); 3307 WARN_ON(is_kdump_kernel() && !disable_bypass); 3308 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); 3309 } 3310 3311 ret = arm_smmu_device_disable(smmu); 3312 if (ret) 3313 return ret; 3314 3315 /* CR1 (table and queue memory attributes) */ 3316 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) | 3317 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) | 3318 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) | 3319 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) | 3320 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) | 3321 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB); 3322 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); 3323 3324 /* CR2 (random crap) */ 3325 reg = CR2_PTM | CR2_RECINVSID; 3326 3327 if (smmu->features & ARM_SMMU_FEAT_E2H) 3328 reg |= CR2_E2H; 3329 3330 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2); 3331 3332 /* Stream table */ 3333 writeq_relaxed(smmu->strtab_cfg.strtab_base, 3334 smmu->base + ARM_SMMU_STRTAB_BASE); 3335 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg, 3336 smmu->base + ARM_SMMU_STRTAB_BASE_CFG); 3337 3338 /* Command queue */ 3339 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); 3340 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD); 3341 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS); 3342 3343 enables = CR0_CMDQEN; 3344 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3345 ARM_SMMU_CR0ACK); 3346 if (ret) { 3347 dev_err(smmu->dev, "failed to enable command queue\n"); 3348 return ret; 3349 } 3350 3351 /* Invalidate any cached configuration */ 3352 cmd.opcode = CMDQ_OP_CFGI_ALL; 3353 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 3354 3355 /* Invalidate any stale TLB entries */ 3356 if (smmu->features & ARM_SMMU_FEAT_HYP) { 3357 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL; 3358 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 3359 } 3360 3361 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL; 3362 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 3363 3364 /* Event queue */ 3365 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); 3366 writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD); 3367 writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS); 3368 3369 enables |= CR0_EVTQEN; 3370 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3371 ARM_SMMU_CR0ACK); 3372 if (ret) { 3373 dev_err(smmu->dev, "failed to enable event queue\n"); 3374 return ret; 3375 } 3376 3377 /* PRI queue */ 3378 if (smmu->features & ARM_SMMU_FEAT_PRI) { 3379 writeq_relaxed(smmu->priq.q.q_base, 3380 smmu->base + ARM_SMMU_PRIQ_BASE); 3381 writel_relaxed(smmu->priq.q.llq.prod, 3382 smmu->page1 + ARM_SMMU_PRIQ_PROD); 3383 writel_relaxed(smmu->priq.q.llq.cons, 3384 smmu->page1 + ARM_SMMU_PRIQ_CONS); 3385 3386 enables |= CR0_PRIQEN; 3387 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3388 ARM_SMMU_CR0ACK); 3389 if (ret) { 3390 dev_err(smmu->dev, "failed to enable PRI queue\n"); 3391 return ret; 3392 } 3393 } 3394 3395 if (smmu->features & ARM_SMMU_FEAT_ATS) { 3396 enables |= CR0_ATSCHK; 3397 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3398 ARM_SMMU_CR0ACK); 3399 if (ret) { 3400 dev_err(smmu->dev, "failed to enable ATS check\n"); 3401 return ret; 3402 } 3403 } 3404 3405 ret = arm_smmu_setup_irqs(smmu); 3406 if (ret) { 3407 dev_err(smmu->dev, "failed to setup irqs\n"); 3408 return ret; 3409 } 3410 3411 if (is_kdump_kernel()) 3412 enables &= ~(CR0_EVTQEN | CR0_PRIQEN); 3413 3414 /* Enable the SMMU interface, or ensure bypass */ 3415 if (!bypass || disable_bypass) { 3416 enables |= CR0_SMMUEN; 3417 } else { 3418 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT); 3419 if (ret) 3420 return ret; 3421 } 3422 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 3423 ARM_SMMU_CR0ACK); 3424 if (ret) { 3425 dev_err(smmu->dev, "failed to enable SMMU interface\n"); 3426 return ret; 3427 } 3428 3429 return 0; 3430 } 3431 3432 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) 3433 { 3434 u32 reg; 3435 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY; 3436 3437 /* IDR0 */ 3438 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); 3439 3440 /* 2-level structures */ 3441 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL) 3442 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB; 3443 3444 if (reg & IDR0_CD2L) 3445 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB; 3446 3447 /* 3448 * Translation table endianness. 3449 * We currently require the same endianness as the CPU, but this 3450 * could be changed later by adding a new IO_PGTABLE_QUIRK. 3451 */ 3452 switch (FIELD_GET(IDR0_TTENDIAN, reg)) { 3453 case IDR0_TTENDIAN_MIXED: 3454 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE; 3455 break; 3456 #ifdef __BIG_ENDIAN 3457 case IDR0_TTENDIAN_BE: 3458 smmu->features |= ARM_SMMU_FEAT_TT_BE; 3459 break; 3460 #else 3461 case IDR0_TTENDIAN_LE: 3462 smmu->features |= ARM_SMMU_FEAT_TT_LE; 3463 break; 3464 #endif 3465 default: 3466 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n"); 3467 return -ENXIO; 3468 } 3469 3470 /* Boolean feature flags */ 3471 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI) 3472 smmu->features |= ARM_SMMU_FEAT_PRI; 3473 3474 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS) 3475 smmu->features |= ARM_SMMU_FEAT_ATS; 3476 3477 if (reg & IDR0_SEV) 3478 smmu->features |= ARM_SMMU_FEAT_SEV; 3479 3480 if (reg & IDR0_MSI) { 3481 smmu->features |= ARM_SMMU_FEAT_MSI; 3482 if (coherent && !disable_msipolling) 3483 smmu->options |= ARM_SMMU_OPT_MSIPOLL; 3484 } 3485 3486 if (reg & IDR0_HYP) { 3487 smmu->features |= ARM_SMMU_FEAT_HYP; 3488 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)) 3489 smmu->features |= ARM_SMMU_FEAT_E2H; 3490 } 3491 3492 /* 3493 * The coherency feature as set by FW is used in preference to the ID 3494 * register, but warn on mismatch. 3495 */ 3496 if (!!(reg & IDR0_COHACC) != coherent) 3497 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n", 3498 coherent ? "true" : "false"); 3499 3500 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) { 3501 case IDR0_STALL_MODEL_FORCE: 3502 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE; 3503 fallthrough; 3504 case IDR0_STALL_MODEL_STALL: 3505 smmu->features |= ARM_SMMU_FEAT_STALLS; 3506 } 3507 3508 if (reg & IDR0_S1P) 3509 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 3510 3511 if (reg & IDR0_S2P) 3512 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; 3513 3514 if (!(reg & (IDR0_S1P | IDR0_S2P))) { 3515 dev_err(smmu->dev, "no translation support!\n"); 3516 return -ENXIO; 3517 } 3518 3519 /* We only support the AArch64 table format at present */ 3520 switch (FIELD_GET(IDR0_TTF, reg)) { 3521 case IDR0_TTF_AARCH32_64: 3522 smmu->ias = 40; 3523 fallthrough; 3524 case IDR0_TTF_AARCH64: 3525 break; 3526 default: 3527 dev_err(smmu->dev, "AArch64 table format not supported!\n"); 3528 return -ENXIO; 3529 } 3530 3531 /* ASID/VMID sizes */ 3532 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8; 3533 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8; 3534 3535 /* IDR1 */ 3536 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1); 3537 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) { 3538 dev_err(smmu->dev, "embedded implementation not supported\n"); 3539 return -ENXIO; 3540 } 3541 3542 /* Queue sizes, capped to ensure natural alignment */ 3543 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, 3544 FIELD_GET(IDR1_CMDQS, reg)); 3545 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) { 3546 /* 3547 * We don't support splitting up batches, so one batch of 3548 * commands plus an extra sync needs to fit inside the command 3549 * queue. There's also no way we can handle the weird alignment 3550 * restrictions on the base pointer for a unit-length queue. 3551 */ 3552 dev_err(smmu->dev, "command queue size <= %d entries not supported\n", 3553 CMDQ_BATCH_ENTRIES); 3554 return -ENXIO; 3555 } 3556 3557 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT, 3558 FIELD_GET(IDR1_EVTQS, reg)); 3559 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT, 3560 FIELD_GET(IDR1_PRIQS, reg)); 3561 3562 /* SID/SSID sizes */ 3563 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg); 3564 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg); 3565 smmu->iommu.max_pasids = 1UL << smmu->ssid_bits; 3566 3567 /* 3568 * If the SMMU supports fewer bits than would fill a single L2 stream 3569 * table, use a linear table instead. 3570 */ 3571 if (smmu->sid_bits <= STRTAB_SPLIT) 3572 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; 3573 3574 /* IDR3 */ 3575 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3); 3576 if (FIELD_GET(IDR3_RIL, reg)) 3577 smmu->features |= ARM_SMMU_FEAT_RANGE_INV; 3578 3579 /* IDR5 */ 3580 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); 3581 3582 /* Maximum number of outstanding stalls */ 3583 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg); 3584 3585 /* Page sizes */ 3586 if (reg & IDR5_GRAN64K) 3587 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; 3588 if (reg & IDR5_GRAN16K) 3589 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; 3590 if (reg & IDR5_GRAN4K) 3591 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; 3592 3593 /* Input address size */ 3594 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT) 3595 smmu->features |= ARM_SMMU_FEAT_VAX; 3596 3597 /* Output address size */ 3598 switch (FIELD_GET(IDR5_OAS, reg)) { 3599 case IDR5_OAS_32_BIT: 3600 smmu->oas = 32; 3601 break; 3602 case IDR5_OAS_36_BIT: 3603 smmu->oas = 36; 3604 break; 3605 case IDR5_OAS_40_BIT: 3606 smmu->oas = 40; 3607 break; 3608 case IDR5_OAS_42_BIT: 3609 smmu->oas = 42; 3610 break; 3611 case IDR5_OAS_44_BIT: 3612 smmu->oas = 44; 3613 break; 3614 case IDR5_OAS_52_BIT: 3615 smmu->oas = 52; 3616 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */ 3617 break; 3618 default: 3619 dev_info(smmu->dev, 3620 "unknown output address size. Truncating to 48-bit\n"); 3621 fallthrough; 3622 case IDR5_OAS_48_BIT: 3623 smmu->oas = 48; 3624 } 3625 3626 if (arm_smmu_ops.pgsize_bitmap == -1UL) 3627 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; 3628 else 3629 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; 3630 3631 /* Set the DMA mask for our table walker */ 3632 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas))) 3633 dev_warn(smmu->dev, 3634 "failed to set DMA mask for table walker\n"); 3635 3636 smmu->ias = max(smmu->ias, smmu->oas); 3637 3638 if (arm_smmu_sva_supported(smmu)) 3639 smmu->features |= ARM_SMMU_FEAT_SVA; 3640 3641 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n", 3642 smmu->ias, smmu->oas, smmu->features); 3643 return 0; 3644 } 3645 3646 #ifdef CONFIG_ACPI 3647 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu) 3648 { 3649 switch (model) { 3650 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX: 3651 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY; 3652 break; 3653 case ACPI_IORT_SMMU_V3_HISILICON_HI161X: 3654 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH; 3655 break; 3656 } 3657 3658 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options); 3659 } 3660 3661 static int arm_smmu_device_acpi_probe(struct platform_device *pdev, 3662 struct arm_smmu_device *smmu) 3663 { 3664 struct acpi_iort_smmu_v3 *iort_smmu; 3665 struct device *dev = smmu->dev; 3666 struct acpi_iort_node *node; 3667 3668 node = *(struct acpi_iort_node **)dev_get_platdata(dev); 3669 3670 /* Retrieve SMMUv3 specific data */ 3671 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data; 3672 3673 acpi_smmu_get_options(iort_smmu->model, smmu); 3674 3675 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE) 3676 smmu->features |= ARM_SMMU_FEAT_COHERENCY; 3677 3678 return 0; 3679 } 3680 #else 3681 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev, 3682 struct arm_smmu_device *smmu) 3683 { 3684 return -ENODEV; 3685 } 3686 #endif 3687 3688 static int arm_smmu_device_dt_probe(struct platform_device *pdev, 3689 struct arm_smmu_device *smmu) 3690 { 3691 struct device *dev = &pdev->dev; 3692 u32 cells; 3693 int ret = -EINVAL; 3694 3695 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells)) 3696 dev_err(dev, "missing #iommu-cells property\n"); 3697 else if (cells != 1) 3698 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells); 3699 else 3700 ret = 0; 3701 3702 parse_driver_options(smmu); 3703 3704 if (of_dma_is_coherent(dev->of_node)) 3705 smmu->features |= ARM_SMMU_FEAT_COHERENCY; 3706 3707 return ret; 3708 } 3709 3710 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) 3711 { 3712 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY) 3713 return SZ_64K; 3714 else 3715 return SZ_128K; 3716 } 3717 3718 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start, 3719 resource_size_t size) 3720 { 3721 struct resource res = DEFINE_RES_MEM(start, size); 3722 3723 return devm_ioremap_resource(dev, &res); 3724 } 3725 3726 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) 3727 { 3728 struct list_head rmr_list; 3729 struct iommu_resv_region *e; 3730 3731 INIT_LIST_HEAD(&rmr_list); 3732 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 3733 3734 list_for_each_entry(e, &rmr_list, list) { 3735 __le64 *step; 3736 struct iommu_iort_rmr_data *rmr; 3737 int ret, i; 3738 3739 rmr = container_of(e, struct iommu_iort_rmr_data, rr); 3740 for (i = 0; i < rmr->num_sids; i++) { 3741 ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]); 3742 if (ret) { 3743 dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n", 3744 rmr->sids[i]); 3745 continue; 3746 } 3747 3748 step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]); 3749 arm_smmu_init_bypass_stes(step, 1, true); 3750 } 3751 } 3752 3753 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 3754 } 3755 3756 static int arm_smmu_device_probe(struct platform_device *pdev) 3757 { 3758 int irq, ret; 3759 struct resource *res; 3760 resource_size_t ioaddr; 3761 struct arm_smmu_device *smmu; 3762 struct device *dev = &pdev->dev; 3763 bool bypass; 3764 3765 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 3766 if (!smmu) 3767 return -ENOMEM; 3768 smmu->dev = dev; 3769 3770 if (dev->of_node) { 3771 ret = arm_smmu_device_dt_probe(pdev, smmu); 3772 } else { 3773 ret = arm_smmu_device_acpi_probe(pdev, smmu); 3774 if (ret == -ENODEV) 3775 return ret; 3776 } 3777 3778 /* Set bypass mode according to firmware probing result */ 3779 bypass = !!ret; 3780 3781 /* Base address */ 3782 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3783 if (!res) 3784 return -EINVAL; 3785 if (resource_size(res) < arm_smmu_resource_size(smmu)) { 3786 dev_err(dev, "MMIO region too small (%pr)\n", res); 3787 return -EINVAL; 3788 } 3789 ioaddr = res->start; 3790 3791 /* 3792 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain 3793 * the PMCG registers which are reserved by the PMU driver. 3794 */ 3795 smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ); 3796 if (IS_ERR(smmu->base)) 3797 return PTR_ERR(smmu->base); 3798 3799 if (arm_smmu_resource_size(smmu) > SZ_64K) { 3800 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K, 3801 ARM_SMMU_REG_SZ); 3802 if (IS_ERR(smmu->page1)) 3803 return PTR_ERR(smmu->page1); 3804 } else { 3805 smmu->page1 = smmu->base; 3806 } 3807 3808 /* Interrupt lines */ 3809 3810 irq = platform_get_irq_byname_optional(pdev, "combined"); 3811 if (irq > 0) 3812 smmu->combined_irq = irq; 3813 else { 3814 irq = platform_get_irq_byname_optional(pdev, "eventq"); 3815 if (irq > 0) 3816 smmu->evtq.q.irq = irq; 3817 3818 irq = platform_get_irq_byname_optional(pdev, "priq"); 3819 if (irq > 0) 3820 smmu->priq.q.irq = irq; 3821 3822 irq = platform_get_irq_byname_optional(pdev, "gerror"); 3823 if (irq > 0) 3824 smmu->gerr_irq = irq; 3825 } 3826 /* Probe the h/w */ 3827 ret = arm_smmu_device_hw_probe(smmu); 3828 if (ret) 3829 return ret; 3830 3831 /* Initialise in-memory data structures */ 3832 ret = arm_smmu_init_structures(smmu); 3833 if (ret) 3834 return ret; 3835 3836 /* Record our private device structure */ 3837 platform_set_drvdata(pdev, smmu); 3838 3839 /* Check for RMRs and install bypass STEs if any */ 3840 arm_smmu_rmr_install_bypass_ste(smmu); 3841 3842 /* Reset the device */ 3843 ret = arm_smmu_device_reset(smmu, bypass); 3844 if (ret) 3845 return ret; 3846 3847 /* And we're up. Go go go! */ 3848 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, 3849 "smmu3.%pa", &ioaddr); 3850 if (ret) 3851 return ret; 3852 3853 ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); 3854 if (ret) { 3855 dev_err(dev, "Failed to register iommu\n"); 3856 iommu_device_sysfs_remove(&smmu->iommu); 3857 return ret; 3858 } 3859 3860 return 0; 3861 } 3862 3863 static void arm_smmu_device_remove(struct platform_device *pdev) 3864 { 3865 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 3866 3867 iommu_device_unregister(&smmu->iommu); 3868 iommu_device_sysfs_remove(&smmu->iommu); 3869 arm_smmu_device_disable(smmu); 3870 iopf_queue_free(smmu->evtq.iopf); 3871 } 3872 3873 static void arm_smmu_device_shutdown(struct platform_device *pdev) 3874 { 3875 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 3876 3877 arm_smmu_device_disable(smmu); 3878 } 3879 3880 static const struct of_device_id arm_smmu_of_match[] = { 3881 { .compatible = "arm,smmu-v3", }, 3882 { }, 3883 }; 3884 MODULE_DEVICE_TABLE(of, arm_smmu_of_match); 3885 3886 static void arm_smmu_driver_unregister(struct platform_driver *drv) 3887 { 3888 arm_smmu_sva_notifier_synchronize(); 3889 platform_driver_unregister(drv); 3890 } 3891 3892 static struct platform_driver arm_smmu_driver = { 3893 .driver = { 3894 .name = "arm-smmu-v3", 3895 .of_match_table = arm_smmu_of_match, 3896 .suppress_bind_attrs = true, 3897 }, 3898 .probe = arm_smmu_device_probe, 3899 .remove_new = arm_smmu_device_remove, 3900 .shutdown = arm_smmu_device_shutdown, 3901 }; 3902 module_driver(arm_smmu_driver, platform_driver_register, 3903 arm_smmu_driver_unregister); 3904 3905 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations"); 3906 MODULE_AUTHOR("Will Deacon <will@kernel.org>"); 3907 MODULE_ALIAS("platform:arm-smmu-v3"); 3908 MODULE_LICENSE("GPL v2"); 3909