1 /* 2 * Copyright (C) 2014-2016 Broadcom Corporation 3 * Copyright (c) 2017 Red Hat, Inc. 4 * Written by Prem Mallappa, Eric Auger 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program; if not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include "qemu/osdep.h" 20 #include "hw/irq.h" 21 #include "hw/sysbus.h" 22 #include "migration/vmstate.h" 23 #include "hw/qdev-core.h" 24 #include "hw/pci/pci.h" 25 #include "exec/address-spaces.h" 26 #include "cpu.h" 27 #include "trace.h" 28 #include "qemu/log.h" 29 #include "qemu/error-report.h" 30 #include "qapi/error.h" 31 32 #include "hw/arm/smmuv3.h" 33 #include "smmuv3-internal.h" 34 35 /** 36 * smmuv3_trigger_irq - pulse @irq if enabled and update 37 * GERROR register in case of GERROR interrupt 38 * 39 * @irq: irq type 40 * @gerror_mask: mask of gerrors to toggle (relevant if @irq is GERROR) 41 */ 42 static void smmuv3_trigger_irq(SMMUv3State *s, SMMUIrq irq, 43 uint32_t gerror_mask) 44 { 45 46 bool pulse = false; 47 48 switch (irq) { 49 case SMMU_IRQ_EVTQ: 50 pulse = smmuv3_eventq_irq_enabled(s); 51 break; 52 case SMMU_IRQ_PRIQ: 53 qemu_log_mask(LOG_UNIMP, "PRI not yet supported\n"); 54 break; 55 case SMMU_IRQ_CMD_SYNC: 56 pulse = true; 57 break; 58 case SMMU_IRQ_GERROR: 59 { 60 uint32_t pending = s->gerror ^ s->gerrorn; 61 uint32_t new_gerrors = ~pending & gerror_mask; 62 63 if (!new_gerrors) { 64 /* only toggle non pending errors */ 65 return; 66 } 67 s->gerror ^= new_gerrors; 68 trace_smmuv3_write_gerror(new_gerrors, s->gerror); 69 70 pulse = smmuv3_gerror_irq_enabled(s); 71 break; 72 } 73 } 74 if (pulse) { 75 trace_smmuv3_trigger_irq(irq); 76 qemu_irq_pulse(s->irq[irq]); 77 } 78 } 79 80 static void smmuv3_write_gerrorn(SMMUv3State *s, uint32_t new_gerrorn) 81 { 82 uint32_t pending = s->gerror ^ s->gerrorn; 83 uint32_t toggled = s->gerrorn ^ new_gerrorn; 84 85 if (toggled & ~pending) { 86 qemu_log_mask(LOG_GUEST_ERROR, 87 "guest toggles non pending errors = 0x%x\n", 88 toggled & ~pending); 89 } 90 91 /* 92 * We do not raise any error in case guest toggles bits corresponding 93 * to not active IRQs (CONSTRAINED UNPREDICTABLE) 94 */ 95 s->gerrorn = new_gerrorn; 96 97 trace_smmuv3_write_gerrorn(toggled & pending, s->gerrorn); 98 } 99 100 static inline MemTxResult queue_read(SMMUQueue *q, void *data) 101 { 102 dma_addr_t addr = Q_CONS_ENTRY(q); 103 104 return dma_memory_read(&address_space_memory, addr, data, q->entry_size); 105 } 106 107 static MemTxResult queue_write(SMMUQueue *q, void *data) 108 { 109 dma_addr_t addr = Q_PROD_ENTRY(q); 110 MemTxResult ret; 111 112 ret = dma_memory_write(&address_space_memory, addr, data, q->entry_size); 113 if (ret != MEMTX_OK) { 114 return ret; 115 } 116 117 queue_prod_incr(q); 118 return MEMTX_OK; 119 } 120 121 static MemTxResult smmuv3_write_eventq(SMMUv3State *s, Evt *evt) 122 { 123 SMMUQueue *q = &s->eventq; 124 MemTxResult r; 125 126 if (!smmuv3_eventq_enabled(s)) { 127 return MEMTX_ERROR; 128 } 129 130 if (smmuv3_q_full(q)) { 131 return MEMTX_ERROR; 132 } 133 134 r = queue_write(q, evt); 135 if (r != MEMTX_OK) { 136 return r; 137 } 138 139 if (!smmuv3_q_empty(q)) { 140 smmuv3_trigger_irq(s, SMMU_IRQ_EVTQ, 0); 141 } 142 return MEMTX_OK; 143 } 144 145 void smmuv3_record_event(SMMUv3State *s, SMMUEventInfo *info) 146 { 147 Evt evt = {}; 148 MemTxResult r; 149 150 if (!smmuv3_eventq_enabled(s)) { 151 return; 152 } 153 154 EVT_SET_TYPE(&evt, info->type); 155 EVT_SET_SID(&evt, info->sid); 156 157 switch (info->type) { 158 case SMMU_EVT_NONE: 159 return; 160 case SMMU_EVT_F_UUT: 161 EVT_SET_SSID(&evt, info->u.f_uut.ssid); 162 EVT_SET_SSV(&evt, info->u.f_uut.ssv); 163 EVT_SET_ADDR(&evt, info->u.f_uut.addr); 164 EVT_SET_RNW(&evt, info->u.f_uut.rnw); 165 EVT_SET_PNU(&evt, info->u.f_uut.pnu); 166 EVT_SET_IND(&evt, info->u.f_uut.ind); 167 break; 168 case SMMU_EVT_C_BAD_STREAMID: 169 EVT_SET_SSID(&evt, info->u.c_bad_streamid.ssid); 170 EVT_SET_SSV(&evt, info->u.c_bad_streamid.ssv); 171 break; 172 case SMMU_EVT_F_STE_FETCH: 173 EVT_SET_SSID(&evt, info->u.f_ste_fetch.ssid); 174 EVT_SET_SSV(&evt, info->u.f_ste_fetch.ssv); 175 EVT_SET_ADDR2(&evt, info->u.f_ste_fetch.addr); 176 break; 177 case SMMU_EVT_C_BAD_STE: 178 EVT_SET_SSID(&evt, info->u.c_bad_ste.ssid); 179 EVT_SET_SSV(&evt, info->u.c_bad_ste.ssv); 180 break; 181 case SMMU_EVT_F_STREAM_DISABLED: 182 break; 183 case SMMU_EVT_F_TRANS_FORBIDDEN: 184 EVT_SET_ADDR(&evt, info->u.f_transl_forbidden.addr); 185 EVT_SET_RNW(&evt, info->u.f_transl_forbidden.rnw); 186 break; 187 case SMMU_EVT_C_BAD_SUBSTREAMID: 188 EVT_SET_SSID(&evt, info->u.c_bad_substream.ssid); 189 break; 190 case SMMU_EVT_F_CD_FETCH: 191 EVT_SET_SSID(&evt, info->u.f_cd_fetch.ssid); 192 EVT_SET_SSV(&evt, info->u.f_cd_fetch.ssv); 193 EVT_SET_ADDR(&evt, info->u.f_cd_fetch.addr); 194 break; 195 case SMMU_EVT_C_BAD_CD: 196 EVT_SET_SSID(&evt, info->u.c_bad_cd.ssid); 197 EVT_SET_SSV(&evt, info->u.c_bad_cd.ssv); 198 break; 199 case SMMU_EVT_F_WALK_EABT: 200 case SMMU_EVT_F_TRANSLATION: 201 case SMMU_EVT_F_ADDR_SIZE: 202 case SMMU_EVT_F_ACCESS: 203 case SMMU_EVT_F_PERMISSION: 204 EVT_SET_STALL(&evt, info->u.f_walk_eabt.stall); 205 EVT_SET_STAG(&evt, info->u.f_walk_eabt.stag); 206 EVT_SET_SSID(&evt, info->u.f_walk_eabt.ssid); 207 EVT_SET_SSV(&evt, info->u.f_walk_eabt.ssv); 208 EVT_SET_S2(&evt, info->u.f_walk_eabt.s2); 209 EVT_SET_ADDR(&evt, info->u.f_walk_eabt.addr); 210 EVT_SET_RNW(&evt, info->u.f_walk_eabt.rnw); 211 EVT_SET_PNU(&evt, info->u.f_walk_eabt.pnu); 212 EVT_SET_IND(&evt, info->u.f_walk_eabt.ind); 213 EVT_SET_CLASS(&evt, info->u.f_walk_eabt.class); 214 EVT_SET_ADDR2(&evt, info->u.f_walk_eabt.addr2); 215 break; 216 case SMMU_EVT_F_CFG_CONFLICT: 217 EVT_SET_SSID(&evt, info->u.f_cfg_conflict.ssid); 218 EVT_SET_SSV(&evt, info->u.f_cfg_conflict.ssv); 219 break; 220 /* rest is not implemented */ 221 case SMMU_EVT_F_BAD_ATS_TREQ: 222 case SMMU_EVT_F_TLB_CONFLICT: 223 case SMMU_EVT_E_PAGE_REQ: 224 default: 225 g_assert_not_reached(); 226 } 227 228 trace_smmuv3_record_event(smmu_event_string(info->type), info->sid); 229 r = smmuv3_write_eventq(s, &evt); 230 if (r != MEMTX_OK) { 231 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_EVENTQ_ABT_ERR_MASK); 232 } 233 info->recorded = true; 234 } 235 236 static void smmuv3_init_regs(SMMUv3State *s) 237 { 238 /** 239 * IDR0: stage1 only, AArch64 only, coherent access, 16b ASID, 240 * multi-level stream table 241 */ 242 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1); /* stage 1 supported */ 243 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTF, 2); /* AArch64 PTW only */ 244 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, COHACC, 1); /* IO coherent */ 245 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ASID16, 1); /* 16-bit ASID */ 246 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTENDIAN, 2); /* little endian */ 247 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STALL_MODEL, 1); /* No stall */ 248 /* terminated transaction will always be aborted/error returned */ 249 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TERM_MODEL, 1); 250 /* 2-level stream table supported */ 251 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STLEVEL, 1); 252 253 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SIDSIZE, SMMU_IDR1_SIDSIZE); 254 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, EVENTQS, SMMU_EVENTQS); 255 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, CMDQS, SMMU_CMDQS); 256 257 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1); 258 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1); 259 260 /* 4K and 64K granule support */ 261 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1); 262 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1); 263 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS); /* 44 bits */ 264 265 s->cmdq.base = deposit64(s->cmdq.base, 0, 5, SMMU_CMDQS); 266 s->cmdq.prod = 0; 267 s->cmdq.cons = 0; 268 s->cmdq.entry_size = sizeof(struct Cmd); 269 s->eventq.base = deposit64(s->eventq.base, 0, 5, SMMU_EVENTQS); 270 s->eventq.prod = 0; 271 s->eventq.cons = 0; 272 s->eventq.entry_size = sizeof(struct Evt); 273 274 s->features = 0; 275 s->sid_split = 0; 276 s->aidr = 0x1; 277 } 278 279 static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf, 280 SMMUEventInfo *event) 281 { 282 int ret; 283 284 trace_smmuv3_get_ste(addr); 285 /* TODO: guarantee 64-bit single-copy atomicity */ 286 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf)); 287 if (ret != MEMTX_OK) { 288 qemu_log_mask(LOG_GUEST_ERROR, 289 "Cannot fetch pte at address=0x%"PRIx64"\n", addr); 290 event->type = SMMU_EVT_F_STE_FETCH; 291 event->u.f_ste_fetch.addr = addr; 292 return -EINVAL; 293 } 294 return 0; 295 296 } 297 298 /* @ssid > 0 not supported yet */ 299 static int smmu_get_cd(SMMUv3State *s, STE *ste, uint32_t ssid, 300 CD *buf, SMMUEventInfo *event) 301 { 302 dma_addr_t addr = STE_CTXPTR(ste); 303 int ret; 304 305 trace_smmuv3_get_cd(addr); 306 /* TODO: guarantee 64-bit single-copy atomicity */ 307 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf)); 308 if (ret != MEMTX_OK) { 309 qemu_log_mask(LOG_GUEST_ERROR, 310 "Cannot fetch pte at address=0x%"PRIx64"\n", addr); 311 event->type = SMMU_EVT_F_CD_FETCH; 312 event->u.f_ste_fetch.addr = addr; 313 return -EINVAL; 314 } 315 return 0; 316 } 317 318 /* Returns < 0 in case of invalid STE, 0 otherwise */ 319 static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg, 320 STE *ste, SMMUEventInfo *event) 321 { 322 uint32_t config; 323 324 if (!STE_VALID(ste)) { 325 if (!event->inval_ste_allowed) { 326 qemu_log_mask(LOG_GUEST_ERROR, "invalid STE\n"); 327 } 328 goto bad_ste; 329 } 330 331 config = STE_CONFIG(ste); 332 333 if (STE_CFG_ABORT(config)) { 334 cfg->aborted = true; 335 return 0; 336 } 337 338 if (STE_CFG_BYPASS(config)) { 339 cfg->bypassed = true; 340 return 0; 341 } 342 343 if (STE_CFG_S2_ENABLED(config)) { 344 qemu_log_mask(LOG_UNIMP, "SMMUv3 does not support stage 2 yet\n"); 345 goto bad_ste; 346 } 347 348 if (STE_S1CDMAX(ste) != 0) { 349 qemu_log_mask(LOG_UNIMP, 350 "SMMUv3 does not support multiple context descriptors yet\n"); 351 goto bad_ste; 352 } 353 354 if (STE_S1STALLD(ste)) { 355 qemu_log_mask(LOG_UNIMP, 356 "SMMUv3 S1 stalling fault model not allowed yet\n"); 357 goto bad_ste; 358 } 359 return 0; 360 361 bad_ste: 362 event->type = SMMU_EVT_C_BAD_STE; 363 return -EINVAL; 364 } 365 366 /** 367 * smmu_find_ste - Return the stream table entry associated 368 * to the sid 369 * 370 * @s: smmuv3 handle 371 * @sid: stream ID 372 * @ste: returned stream table entry 373 * @event: handle to an event info 374 * 375 * Supports linear and 2-level stream table 376 * Return 0 on success, -EINVAL otherwise 377 */ 378 static int smmu_find_ste(SMMUv3State *s, uint32_t sid, STE *ste, 379 SMMUEventInfo *event) 380 { 381 dma_addr_t addr, strtab_base; 382 uint32_t log2size; 383 int strtab_size_shift; 384 int ret; 385 386 trace_smmuv3_find_ste(sid, s->features, s->sid_split); 387 log2size = FIELD_EX32(s->strtab_base_cfg, STRTAB_BASE_CFG, LOG2SIZE); 388 /* 389 * Check SID range against both guest-configured and implementation limits 390 */ 391 if (sid >= (1 << MIN(log2size, SMMU_IDR1_SIDSIZE))) { 392 event->type = SMMU_EVT_C_BAD_STREAMID; 393 return -EINVAL; 394 } 395 if (s->features & SMMU_FEATURE_2LVL_STE) { 396 int l1_ste_offset, l2_ste_offset, max_l2_ste, span; 397 dma_addr_t l1ptr, l2ptr; 398 STEDesc l1std; 399 400 /* 401 * Align strtab base address to table size. For this purpose, assume it 402 * is not bounded by SMMU_IDR1_SIDSIZE. 403 */ 404 strtab_size_shift = MAX(5, (int)log2size - s->sid_split - 1 + 3); 405 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK & 406 ~MAKE_64BIT_MASK(0, strtab_size_shift); 407 l1_ste_offset = sid >> s->sid_split; 408 l2_ste_offset = sid & ((1 << s->sid_split) - 1); 409 l1ptr = (dma_addr_t)(strtab_base + l1_ste_offset * sizeof(l1std)); 410 /* TODO: guarantee 64-bit single-copy atomicity */ 411 ret = dma_memory_read(&address_space_memory, l1ptr, &l1std, 412 sizeof(l1std)); 413 if (ret != MEMTX_OK) { 414 qemu_log_mask(LOG_GUEST_ERROR, 415 "Could not read L1PTR at 0X%"PRIx64"\n", l1ptr); 416 event->type = SMMU_EVT_F_STE_FETCH; 417 event->u.f_ste_fetch.addr = l1ptr; 418 return -EINVAL; 419 } 420 421 span = L1STD_SPAN(&l1std); 422 423 if (!span) { 424 /* l2ptr is not valid */ 425 if (!event->inval_ste_allowed) { 426 qemu_log_mask(LOG_GUEST_ERROR, 427 "invalid sid=%d (L1STD span=0)\n", sid); 428 } 429 event->type = SMMU_EVT_C_BAD_STREAMID; 430 return -EINVAL; 431 } 432 max_l2_ste = (1 << span) - 1; 433 l2ptr = l1std_l2ptr(&l1std); 434 trace_smmuv3_find_ste_2lvl(s->strtab_base, l1ptr, l1_ste_offset, 435 l2ptr, l2_ste_offset, max_l2_ste); 436 if (l2_ste_offset > max_l2_ste) { 437 qemu_log_mask(LOG_GUEST_ERROR, 438 "l2_ste_offset=%d > max_l2_ste=%d\n", 439 l2_ste_offset, max_l2_ste); 440 event->type = SMMU_EVT_C_BAD_STE; 441 return -EINVAL; 442 } 443 addr = l2ptr + l2_ste_offset * sizeof(*ste); 444 } else { 445 strtab_size_shift = log2size + 5; 446 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK & 447 ~MAKE_64BIT_MASK(0, strtab_size_shift); 448 addr = strtab_base + sid * sizeof(*ste); 449 } 450 451 if (smmu_get_ste(s, addr, ste, event)) { 452 return -EINVAL; 453 } 454 455 return 0; 456 } 457 458 static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event) 459 { 460 int ret = -EINVAL; 461 int i; 462 463 if (!CD_VALID(cd) || !CD_AARCH64(cd)) { 464 goto bad_cd; 465 } 466 if (!CD_A(cd)) { 467 goto bad_cd; /* SMMU_IDR0.TERM_MODEL == 1 */ 468 } 469 if (CD_S(cd)) { 470 goto bad_cd; /* !STE_SECURE && SMMU_IDR0.STALL_MODEL == 1 */ 471 } 472 if (CD_HA(cd) || CD_HD(cd)) { 473 goto bad_cd; /* HTTU = 0 */ 474 } 475 476 /* we support only those at the moment */ 477 cfg->aa64 = true; 478 cfg->stage = 1; 479 480 cfg->oas = oas2bits(CD_IPS(cd)); 481 cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas); 482 cfg->tbi = CD_TBI(cd); 483 cfg->asid = CD_ASID(cd); 484 485 trace_smmuv3_decode_cd(cfg->oas); 486 487 /* decode data dependent on TT */ 488 for (i = 0; i <= 1; i++) { 489 int tg, tsz; 490 SMMUTransTableInfo *tt = &cfg->tt[i]; 491 492 cfg->tt[i].disabled = CD_EPD(cd, i); 493 if (cfg->tt[i].disabled) { 494 continue; 495 } 496 497 tsz = CD_TSZ(cd, i); 498 if (tsz < 16 || tsz > 39) { 499 goto bad_cd; 500 } 501 502 tg = CD_TG(cd, i); 503 tt->granule_sz = tg2granule(tg, i); 504 if ((tt->granule_sz != 12 && tt->granule_sz != 16) || CD_ENDI(cd)) { 505 goto bad_cd; 506 } 507 508 tt->tsz = tsz; 509 tt->ttb = CD_TTB(cd, i); 510 if (tt->ttb & ~(MAKE_64BIT_MASK(0, cfg->oas))) { 511 goto bad_cd; 512 } 513 tt->had = CD_HAD(cd, i); 514 trace_smmuv3_decode_cd_tt(i, tt->tsz, tt->ttb, tt->granule_sz, tt->had); 515 } 516 517 event->record_trans_faults = CD_R(cd); 518 519 return 0; 520 521 bad_cd: 522 event->type = SMMU_EVT_C_BAD_CD; 523 return ret; 524 } 525 526 /** 527 * smmuv3_decode_config - Prepare the translation configuration 528 * for the @mr iommu region 529 * @mr: iommu memory region the translation config must be prepared for 530 * @cfg: output translation configuration which is populated through 531 * the different configuration decoding steps 532 * @event: must be zero'ed by the caller 533 * 534 * return < 0 in case of config decoding error (@event is filled 535 * accordingly). Return 0 otherwise. 536 */ 537 static int smmuv3_decode_config(IOMMUMemoryRegion *mr, SMMUTransCfg *cfg, 538 SMMUEventInfo *event) 539 { 540 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu); 541 uint32_t sid = smmu_get_sid(sdev); 542 SMMUv3State *s = sdev->smmu; 543 int ret; 544 STE ste; 545 CD cd; 546 547 ret = smmu_find_ste(s, sid, &ste, event); 548 if (ret) { 549 return ret; 550 } 551 552 ret = decode_ste(s, cfg, &ste, event); 553 if (ret) { 554 return ret; 555 } 556 557 if (cfg->aborted || cfg->bypassed) { 558 return 0; 559 } 560 561 ret = smmu_get_cd(s, &ste, 0 /* ssid */, &cd, event); 562 if (ret) { 563 return ret; 564 } 565 566 return decode_cd(cfg, &cd, event); 567 } 568 569 /** 570 * smmuv3_get_config - Look up for a cached copy of configuration data for 571 * @sdev and on cache miss performs a configuration structure decoding from 572 * guest RAM. 573 * 574 * @sdev: SMMUDevice handle 575 * @event: output event info 576 * 577 * The configuration cache contains data resulting from both STE and CD 578 * decoding under the form of an SMMUTransCfg struct. The hash table is indexed 579 * by the SMMUDevice handle. 580 */ 581 static SMMUTransCfg *smmuv3_get_config(SMMUDevice *sdev, SMMUEventInfo *event) 582 { 583 SMMUv3State *s = sdev->smmu; 584 SMMUState *bc = &s->smmu_state; 585 SMMUTransCfg *cfg; 586 587 cfg = g_hash_table_lookup(bc->configs, sdev); 588 if (cfg) { 589 sdev->cfg_cache_hits++; 590 trace_smmuv3_config_cache_hit(smmu_get_sid(sdev), 591 sdev->cfg_cache_hits, sdev->cfg_cache_misses, 592 100 * sdev->cfg_cache_hits / 593 (sdev->cfg_cache_hits + sdev->cfg_cache_misses)); 594 } else { 595 sdev->cfg_cache_misses++; 596 trace_smmuv3_config_cache_miss(smmu_get_sid(sdev), 597 sdev->cfg_cache_hits, sdev->cfg_cache_misses, 598 100 * sdev->cfg_cache_hits / 599 (sdev->cfg_cache_hits + sdev->cfg_cache_misses)); 600 cfg = g_new0(SMMUTransCfg, 1); 601 602 if (!smmuv3_decode_config(&sdev->iommu, cfg, event)) { 603 g_hash_table_insert(bc->configs, sdev, cfg); 604 } else { 605 g_free(cfg); 606 cfg = NULL; 607 } 608 } 609 return cfg; 610 } 611 612 static void smmuv3_flush_config(SMMUDevice *sdev) 613 { 614 SMMUv3State *s = sdev->smmu; 615 SMMUState *bc = &s->smmu_state; 616 617 trace_smmuv3_config_cache_inv(smmu_get_sid(sdev)); 618 g_hash_table_remove(bc->configs, sdev); 619 } 620 621 static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr, 622 IOMMUAccessFlags flag, int iommu_idx) 623 { 624 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu); 625 SMMUv3State *s = sdev->smmu; 626 uint32_t sid = smmu_get_sid(sdev); 627 SMMUEventInfo event = {.type = SMMU_EVT_NONE, 628 .sid = sid, 629 .inval_ste_allowed = false}; 630 SMMUPTWEventInfo ptw_info = {}; 631 SMMUTranslationStatus status; 632 SMMUState *bs = ARM_SMMU(s); 633 uint64_t page_mask, aligned_addr; 634 SMMUTLBEntry *cached_entry = NULL; 635 SMMUTransTableInfo *tt; 636 SMMUTransCfg *cfg = NULL; 637 IOMMUTLBEntry entry = { 638 .target_as = &address_space_memory, 639 .iova = addr, 640 .translated_addr = addr, 641 .addr_mask = ~(hwaddr)0, 642 .perm = IOMMU_NONE, 643 }; 644 645 qemu_mutex_lock(&s->mutex); 646 647 if (!smmu_enabled(s)) { 648 status = SMMU_TRANS_DISABLE; 649 goto epilogue; 650 } 651 652 cfg = smmuv3_get_config(sdev, &event); 653 if (!cfg) { 654 status = SMMU_TRANS_ERROR; 655 goto epilogue; 656 } 657 658 if (cfg->aborted) { 659 status = SMMU_TRANS_ABORT; 660 goto epilogue; 661 } 662 663 if (cfg->bypassed) { 664 status = SMMU_TRANS_BYPASS; 665 goto epilogue; 666 } 667 668 tt = select_tt(cfg, addr); 669 if (!tt) { 670 if (event.record_trans_faults) { 671 event.type = SMMU_EVT_F_TRANSLATION; 672 event.u.f_translation.addr = addr; 673 event.u.f_translation.rnw = flag & 0x1; 674 } 675 status = SMMU_TRANS_ERROR; 676 goto epilogue; 677 } 678 679 page_mask = (1ULL << (tt->granule_sz)) - 1; 680 aligned_addr = addr & ~page_mask; 681 682 cached_entry = smmu_iotlb_lookup(bs, cfg, tt, aligned_addr); 683 if (cached_entry) { 684 if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & IOMMU_WO)) { 685 status = SMMU_TRANS_ERROR; 686 if (event.record_trans_faults) { 687 event.type = SMMU_EVT_F_PERMISSION; 688 event.u.f_permission.addr = addr; 689 event.u.f_permission.rnw = flag & 0x1; 690 } 691 } else { 692 status = SMMU_TRANS_SUCCESS; 693 } 694 goto epilogue; 695 } 696 697 cached_entry = g_new0(SMMUTLBEntry, 1); 698 699 if (smmu_ptw(cfg, aligned_addr, flag, cached_entry, &ptw_info)) { 700 g_free(cached_entry); 701 switch (ptw_info.type) { 702 case SMMU_PTW_ERR_WALK_EABT: 703 event.type = SMMU_EVT_F_WALK_EABT; 704 event.u.f_walk_eabt.addr = addr; 705 event.u.f_walk_eabt.rnw = flag & 0x1; 706 event.u.f_walk_eabt.class = 0x1; 707 event.u.f_walk_eabt.addr2 = ptw_info.addr; 708 break; 709 case SMMU_PTW_ERR_TRANSLATION: 710 if (event.record_trans_faults) { 711 event.type = SMMU_EVT_F_TRANSLATION; 712 event.u.f_translation.addr = addr; 713 event.u.f_translation.rnw = flag & 0x1; 714 } 715 break; 716 case SMMU_PTW_ERR_ADDR_SIZE: 717 if (event.record_trans_faults) { 718 event.type = SMMU_EVT_F_ADDR_SIZE; 719 event.u.f_addr_size.addr = addr; 720 event.u.f_addr_size.rnw = flag & 0x1; 721 } 722 break; 723 case SMMU_PTW_ERR_ACCESS: 724 if (event.record_trans_faults) { 725 event.type = SMMU_EVT_F_ACCESS; 726 event.u.f_access.addr = addr; 727 event.u.f_access.rnw = flag & 0x1; 728 } 729 break; 730 case SMMU_PTW_ERR_PERMISSION: 731 if (event.record_trans_faults) { 732 event.type = SMMU_EVT_F_PERMISSION; 733 event.u.f_permission.addr = addr; 734 event.u.f_permission.rnw = flag & 0x1; 735 } 736 break; 737 default: 738 g_assert_not_reached(); 739 } 740 status = SMMU_TRANS_ERROR; 741 } else { 742 smmu_iotlb_insert(bs, cfg, cached_entry); 743 status = SMMU_TRANS_SUCCESS; 744 } 745 746 epilogue: 747 qemu_mutex_unlock(&s->mutex); 748 switch (status) { 749 case SMMU_TRANS_SUCCESS: 750 entry.perm = flag; 751 entry.translated_addr = cached_entry->entry.translated_addr + 752 (addr & cached_entry->entry.addr_mask); 753 entry.addr_mask = cached_entry->entry.addr_mask; 754 trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr, 755 entry.translated_addr, entry.perm); 756 break; 757 case SMMU_TRANS_DISABLE: 758 entry.perm = flag; 759 entry.addr_mask = ~TARGET_PAGE_MASK; 760 trace_smmuv3_translate_disable(mr->parent_obj.name, sid, addr, 761 entry.perm); 762 break; 763 case SMMU_TRANS_BYPASS: 764 entry.perm = flag; 765 entry.addr_mask = ~TARGET_PAGE_MASK; 766 trace_smmuv3_translate_bypass(mr->parent_obj.name, sid, addr, 767 entry.perm); 768 break; 769 case SMMU_TRANS_ABORT: 770 /* no event is recorded on abort */ 771 trace_smmuv3_translate_abort(mr->parent_obj.name, sid, addr, 772 entry.perm); 773 break; 774 case SMMU_TRANS_ERROR: 775 qemu_log_mask(LOG_GUEST_ERROR, 776 "%s translation failed for iova=0x%"PRIx64"(%s)\n", 777 mr->parent_obj.name, addr, smmu_event_string(event.type)); 778 smmuv3_record_event(s, &event); 779 break; 780 } 781 782 return entry; 783 } 784 785 /** 786 * smmuv3_notify_iova - call the notifier @n for a given 787 * @asid and @iova tuple. 788 * 789 * @mr: IOMMU mr region handle 790 * @n: notifier to be called 791 * @asid: address space ID or negative value if we don't care 792 * @iova: iova 793 * @tg: translation granule (if communicated through range invalidation) 794 * @num_pages: number of @granule sized pages (if tg != 0), otherwise 1 795 */ 796 static void smmuv3_notify_iova(IOMMUMemoryRegion *mr, 797 IOMMUNotifier *n, 798 int asid, dma_addr_t iova, 799 uint8_t tg, uint64_t num_pages) 800 { 801 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu); 802 IOMMUTLBEntry entry; 803 uint8_t granule = tg; 804 805 if (!tg) { 806 SMMUEventInfo event = {.inval_ste_allowed = true}; 807 SMMUTransCfg *cfg = smmuv3_get_config(sdev, &event); 808 SMMUTransTableInfo *tt; 809 810 if (!cfg) { 811 return; 812 } 813 814 if (asid >= 0 && cfg->asid != asid) { 815 return; 816 } 817 818 tt = select_tt(cfg, iova); 819 if (!tt) { 820 return; 821 } 822 granule = tt->granule_sz; 823 } 824 825 entry.target_as = &address_space_memory; 826 entry.iova = iova; 827 entry.addr_mask = num_pages * (1 << granule) - 1; 828 entry.perm = IOMMU_NONE; 829 830 memory_region_notify_one(n, &entry); 831 } 832 833 /* invalidate an asid/iova range tuple in all mr's */ 834 static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, dma_addr_t iova, 835 uint8_t tg, uint64_t num_pages) 836 { 837 SMMUDevice *sdev; 838 839 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { 840 IOMMUMemoryRegion *mr = &sdev->iommu; 841 IOMMUNotifier *n; 842 843 trace_smmuv3_inv_notifiers_iova(mr->parent_obj.name, asid, iova, 844 tg, num_pages); 845 846 IOMMU_NOTIFIER_FOREACH(n, mr) { 847 smmuv3_notify_iova(mr, n, asid, iova, tg, num_pages); 848 } 849 } 850 } 851 852 static void smmuv3_s1_range_inval(SMMUState *s, Cmd *cmd) 853 { 854 uint8_t scale = 0, num = 0, ttl = 0; 855 dma_addr_t addr = CMD_ADDR(cmd); 856 uint8_t type = CMD_TYPE(cmd); 857 uint16_t vmid = CMD_VMID(cmd); 858 bool leaf = CMD_LEAF(cmd); 859 uint8_t tg = CMD_TG(cmd); 860 hwaddr num_pages = 1; 861 int asid = -1; 862 863 if (tg) { 864 scale = CMD_SCALE(cmd); 865 num = CMD_NUM(cmd); 866 ttl = CMD_TTL(cmd); 867 num_pages = (num + 1) * (1 << (scale)); 868 } 869 870 if (type == SMMU_CMD_TLBI_NH_VA) { 871 asid = CMD_ASID(cmd); 872 } 873 trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, num_pages, ttl, leaf); 874 smmuv3_inv_notifiers_iova(s, asid, addr, tg, num_pages); 875 smmu_iotlb_inv_iova(s, asid, addr, tg, num_pages, ttl); 876 } 877 878 static int smmuv3_cmdq_consume(SMMUv3State *s) 879 { 880 SMMUState *bs = ARM_SMMU(s); 881 SMMUCmdError cmd_error = SMMU_CERROR_NONE; 882 SMMUQueue *q = &s->cmdq; 883 SMMUCommandType type = 0; 884 885 if (!smmuv3_cmdq_enabled(s)) { 886 return 0; 887 } 888 /* 889 * some commands depend on register values, typically CR0. In case those 890 * register values change while handling the command, spec says it 891 * is UNPREDICTABLE whether the command is interpreted under the new 892 * or old value. 893 */ 894 895 while (!smmuv3_q_empty(q)) { 896 uint32_t pending = s->gerror ^ s->gerrorn; 897 Cmd cmd; 898 899 trace_smmuv3_cmdq_consume(Q_PROD(q), Q_CONS(q), 900 Q_PROD_WRAP(q), Q_CONS_WRAP(q)); 901 902 if (FIELD_EX32(pending, GERROR, CMDQ_ERR)) { 903 break; 904 } 905 906 if (queue_read(q, &cmd) != MEMTX_OK) { 907 cmd_error = SMMU_CERROR_ABT; 908 break; 909 } 910 911 type = CMD_TYPE(&cmd); 912 913 trace_smmuv3_cmdq_opcode(smmu_cmd_string(type)); 914 915 qemu_mutex_lock(&s->mutex); 916 switch (type) { 917 case SMMU_CMD_SYNC: 918 if (CMD_SYNC_CS(&cmd) & CMD_SYNC_SIG_IRQ) { 919 smmuv3_trigger_irq(s, SMMU_IRQ_CMD_SYNC, 0); 920 } 921 break; 922 case SMMU_CMD_PREFETCH_CONFIG: 923 case SMMU_CMD_PREFETCH_ADDR: 924 break; 925 case SMMU_CMD_CFGI_STE: 926 { 927 uint32_t sid = CMD_SID(&cmd); 928 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid); 929 SMMUDevice *sdev; 930 931 if (CMD_SSEC(&cmd)) { 932 cmd_error = SMMU_CERROR_ILL; 933 break; 934 } 935 936 if (!mr) { 937 break; 938 } 939 940 trace_smmuv3_cmdq_cfgi_ste(sid); 941 sdev = container_of(mr, SMMUDevice, iommu); 942 smmuv3_flush_config(sdev); 943 944 break; 945 } 946 case SMMU_CMD_CFGI_STE_RANGE: /* same as SMMU_CMD_CFGI_ALL */ 947 { 948 uint32_t start = CMD_SID(&cmd), end, i; 949 uint8_t range = CMD_STE_RANGE(&cmd); 950 951 if (CMD_SSEC(&cmd)) { 952 cmd_error = SMMU_CERROR_ILL; 953 break; 954 } 955 956 end = start + (1 << (range + 1)) - 1; 957 trace_smmuv3_cmdq_cfgi_ste_range(start, end); 958 959 for (i = start; i <= end; i++) { 960 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, i); 961 SMMUDevice *sdev; 962 963 if (!mr) { 964 continue; 965 } 966 sdev = container_of(mr, SMMUDevice, iommu); 967 smmuv3_flush_config(sdev); 968 } 969 break; 970 } 971 case SMMU_CMD_CFGI_CD: 972 case SMMU_CMD_CFGI_CD_ALL: 973 { 974 uint32_t sid = CMD_SID(&cmd); 975 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid); 976 SMMUDevice *sdev; 977 978 if (CMD_SSEC(&cmd)) { 979 cmd_error = SMMU_CERROR_ILL; 980 break; 981 } 982 983 if (!mr) { 984 break; 985 } 986 987 trace_smmuv3_cmdq_cfgi_cd(sid); 988 sdev = container_of(mr, SMMUDevice, iommu); 989 smmuv3_flush_config(sdev); 990 break; 991 } 992 case SMMU_CMD_TLBI_NH_ASID: 993 { 994 uint16_t asid = CMD_ASID(&cmd); 995 996 trace_smmuv3_cmdq_tlbi_nh_asid(asid); 997 smmu_inv_notifiers_all(&s->smmu_state); 998 smmu_iotlb_inv_asid(bs, asid); 999 break; 1000 } 1001 case SMMU_CMD_TLBI_NH_ALL: 1002 case SMMU_CMD_TLBI_NSNH_ALL: 1003 trace_smmuv3_cmdq_tlbi_nh(); 1004 smmu_inv_notifiers_all(&s->smmu_state); 1005 smmu_iotlb_inv_all(bs); 1006 break; 1007 case SMMU_CMD_TLBI_NH_VAA: 1008 case SMMU_CMD_TLBI_NH_VA: 1009 smmuv3_s1_range_inval(bs, &cmd); 1010 break; 1011 case SMMU_CMD_TLBI_EL3_ALL: 1012 case SMMU_CMD_TLBI_EL3_VA: 1013 case SMMU_CMD_TLBI_EL2_ALL: 1014 case SMMU_CMD_TLBI_EL2_ASID: 1015 case SMMU_CMD_TLBI_EL2_VA: 1016 case SMMU_CMD_TLBI_EL2_VAA: 1017 case SMMU_CMD_TLBI_S12_VMALL: 1018 case SMMU_CMD_TLBI_S2_IPA: 1019 case SMMU_CMD_ATC_INV: 1020 case SMMU_CMD_PRI_RESP: 1021 case SMMU_CMD_RESUME: 1022 case SMMU_CMD_STALL_TERM: 1023 trace_smmuv3_unhandled_cmd(type); 1024 break; 1025 default: 1026 cmd_error = SMMU_CERROR_ILL; 1027 qemu_log_mask(LOG_GUEST_ERROR, 1028 "Illegal command type: %d\n", CMD_TYPE(&cmd)); 1029 break; 1030 } 1031 qemu_mutex_unlock(&s->mutex); 1032 if (cmd_error) { 1033 break; 1034 } 1035 /* 1036 * We only increment the cons index after the completion of 1037 * the command. We do that because the SYNC returns immediately 1038 * and does not check the completion of previous commands 1039 */ 1040 queue_cons_incr(q); 1041 } 1042 1043 if (cmd_error) { 1044 trace_smmuv3_cmdq_consume_error(smmu_cmd_string(type), cmd_error); 1045 smmu_write_cmdq_err(s, cmd_error); 1046 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_CMDQ_ERR_MASK); 1047 } 1048 1049 trace_smmuv3_cmdq_consume_out(Q_PROD(q), Q_CONS(q), 1050 Q_PROD_WRAP(q), Q_CONS_WRAP(q)); 1051 1052 return 0; 1053 } 1054 1055 static MemTxResult smmu_writell(SMMUv3State *s, hwaddr offset, 1056 uint64_t data, MemTxAttrs attrs) 1057 { 1058 switch (offset) { 1059 case A_GERROR_IRQ_CFG0: 1060 s->gerror_irq_cfg0 = data; 1061 return MEMTX_OK; 1062 case A_STRTAB_BASE: 1063 s->strtab_base = data; 1064 return MEMTX_OK; 1065 case A_CMDQ_BASE: 1066 s->cmdq.base = data; 1067 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5); 1068 if (s->cmdq.log2size > SMMU_CMDQS) { 1069 s->cmdq.log2size = SMMU_CMDQS; 1070 } 1071 return MEMTX_OK; 1072 case A_EVENTQ_BASE: 1073 s->eventq.base = data; 1074 s->eventq.log2size = extract64(s->eventq.base, 0, 5); 1075 if (s->eventq.log2size > SMMU_EVENTQS) { 1076 s->eventq.log2size = SMMU_EVENTQS; 1077 } 1078 return MEMTX_OK; 1079 case A_EVENTQ_IRQ_CFG0: 1080 s->eventq_irq_cfg0 = data; 1081 return MEMTX_OK; 1082 default: 1083 qemu_log_mask(LOG_UNIMP, 1084 "%s Unexpected 64-bit access to 0x%"PRIx64" (WI)\n", 1085 __func__, offset); 1086 return MEMTX_OK; 1087 } 1088 } 1089 1090 static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset, 1091 uint64_t data, MemTxAttrs attrs) 1092 { 1093 switch (offset) { 1094 case A_CR0: 1095 s->cr[0] = data; 1096 s->cr0ack = data & ~SMMU_CR0_RESERVED; 1097 /* in case the command queue has been enabled */ 1098 smmuv3_cmdq_consume(s); 1099 return MEMTX_OK; 1100 case A_CR1: 1101 s->cr[1] = data; 1102 return MEMTX_OK; 1103 case A_CR2: 1104 s->cr[2] = data; 1105 return MEMTX_OK; 1106 case A_IRQ_CTRL: 1107 s->irq_ctrl = data; 1108 return MEMTX_OK; 1109 case A_GERRORN: 1110 smmuv3_write_gerrorn(s, data); 1111 /* 1112 * By acknowledging the CMDQ_ERR, SW may notify cmds can 1113 * be processed again 1114 */ 1115 smmuv3_cmdq_consume(s); 1116 return MEMTX_OK; 1117 case A_GERROR_IRQ_CFG0: /* 64b */ 1118 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 0, 32, data); 1119 return MEMTX_OK; 1120 case A_GERROR_IRQ_CFG0 + 4: 1121 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 32, 32, data); 1122 return MEMTX_OK; 1123 case A_GERROR_IRQ_CFG1: 1124 s->gerror_irq_cfg1 = data; 1125 return MEMTX_OK; 1126 case A_GERROR_IRQ_CFG2: 1127 s->gerror_irq_cfg2 = data; 1128 return MEMTX_OK; 1129 case A_STRTAB_BASE: /* 64b */ 1130 s->strtab_base = deposit64(s->strtab_base, 0, 32, data); 1131 return MEMTX_OK; 1132 case A_STRTAB_BASE + 4: 1133 s->strtab_base = deposit64(s->strtab_base, 32, 32, data); 1134 return MEMTX_OK; 1135 case A_STRTAB_BASE_CFG: 1136 s->strtab_base_cfg = data; 1137 if (FIELD_EX32(data, STRTAB_BASE_CFG, FMT) == 1) { 1138 s->sid_split = FIELD_EX32(data, STRTAB_BASE_CFG, SPLIT); 1139 s->features |= SMMU_FEATURE_2LVL_STE; 1140 } 1141 return MEMTX_OK; 1142 case A_CMDQ_BASE: /* 64b */ 1143 s->cmdq.base = deposit64(s->cmdq.base, 0, 32, data); 1144 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5); 1145 if (s->cmdq.log2size > SMMU_CMDQS) { 1146 s->cmdq.log2size = SMMU_CMDQS; 1147 } 1148 return MEMTX_OK; 1149 case A_CMDQ_BASE + 4: /* 64b */ 1150 s->cmdq.base = deposit64(s->cmdq.base, 32, 32, data); 1151 return MEMTX_OK; 1152 case A_CMDQ_PROD: 1153 s->cmdq.prod = data; 1154 smmuv3_cmdq_consume(s); 1155 return MEMTX_OK; 1156 case A_CMDQ_CONS: 1157 s->cmdq.cons = data; 1158 return MEMTX_OK; 1159 case A_EVENTQ_BASE: /* 64b */ 1160 s->eventq.base = deposit64(s->eventq.base, 0, 32, data); 1161 s->eventq.log2size = extract64(s->eventq.base, 0, 5); 1162 if (s->eventq.log2size > SMMU_EVENTQS) { 1163 s->eventq.log2size = SMMU_EVENTQS; 1164 } 1165 return MEMTX_OK; 1166 case A_EVENTQ_BASE + 4: 1167 s->eventq.base = deposit64(s->eventq.base, 32, 32, data); 1168 return MEMTX_OK; 1169 case A_EVENTQ_PROD: 1170 s->eventq.prod = data; 1171 return MEMTX_OK; 1172 case A_EVENTQ_CONS: 1173 s->eventq.cons = data; 1174 return MEMTX_OK; 1175 case A_EVENTQ_IRQ_CFG0: /* 64b */ 1176 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 0, 32, data); 1177 return MEMTX_OK; 1178 case A_EVENTQ_IRQ_CFG0 + 4: 1179 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 32, 32, data); 1180 return MEMTX_OK; 1181 case A_EVENTQ_IRQ_CFG1: 1182 s->eventq_irq_cfg1 = data; 1183 return MEMTX_OK; 1184 case A_EVENTQ_IRQ_CFG2: 1185 s->eventq_irq_cfg2 = data; 1186 return MEMTX_OK; 1187 default: 1188 qemu_log_mask(LOG_UNIMP, 1189 "%s Unexpected 32-bit access to 0x%"PRIx64" (WI)\n", 1190 __func__, offset); 1191 return MEMTX_OK; 1192 } 1193 } 1194 1195 static MemTxResult smmu_write_mmio(void *opaque, hwaddr offset, uint64_t data, 1196 unsigned size, MemTxAttrs attrs) 1197 { 1198 SMMUState *sys = opaque; 1199 SMMUv3State *s = ARM_SMMUV3(sys); 1200 MemTxResult r; 1201 1202 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */ 1203 offset &= ~0x10000; 1204 1205 switch (size) { 1206 case 8: 1207 r = smmu_writell(s, offset, data, attrs); 1208 break; 1209 case 4: 1210 r = smmu_writel(s, offset, data, attrs); 1211 break; 1212 default: 1213 r = MEMTX_ERROR; 1214 break; 1215 } 1216 1217 trace_smmuv3_write_mmio(offset, data, size, r); 1218 return r; 1219 } 1220 1221 static MemTxResult smmu_readll(SMMUv3State *s, hwaddr offset, 1222 uint64_t *data, MemTxAttrs attrs) 1223 { 1224 switch (offset) { 1225 case A_GERROR_IRQ_CFG0: 1226 *data = s->gerror_irq_cfg0; 1227 return MEMTX_OK; 1228 case A_STRTAB_BASE: 1229 *data = s->strtab_base; 1230 return MEMTX_OK; 1231 case A_CMDQ_BASE: 1232 *data = s->cmdq.base; 1233 return MEMTX_OK; 1234 case A_EVENTQ_BASE: 1235 *data = s->eventq.base; 1236 return MEMTX_OK; 1237 default: 1238 *data = 0; 1239 qemu_log_mask(LOG_UNIMP, 1240 "%s Unexpected 64-bit access to 0x%"PRIx64" (RAZ)\n", 1241 __func__, offset); 1242 return MEMTX_OK; 1243 } 1244 } 1245 1246 static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset, 1247 uint64_t *data, MemTxAttrs attrs) 1248 { 1249 switch (offset) { 1250 case A_IDREGS ... A_IDREGS + 0x2f: 1251 *data = smmuv3_idreg(offset - A_IDREGS); 1252 return MEMTX_OK; 1253 case A_IDR0 ... A_IDR5: 1254 *data = s->idr[(offset - A_IDR0) / 4]; 1255 return MEMTX_OK; 1256 case A_IIDR: 1257 *data = s->iidr; 1258 return MEMTX_OK; 1259 case A_AIDR: 1260 *data = s->aidr; 1261 return MEMTX_OK; 1262 case A_CR0: 1263 *data = s->cr[0]; 1264 return MEMTX_OK; 1265 case A_CR0ACK: 1266 *data = s->cr0ack; 1267 return MEMTX_OK; 1268 case A_CR1: 1269 *data = s->cr[1]; 1270 return MEMTX_OK; 1271 case A_CR2: 1272 *data = s->cr[2]; 1273 return MEMTX_OK; 1274 case A_STATUSR: 1275 *data = s->statusr; 1276 return MEMTX_OK; 1277 case A_IRQ_CTRL: 1278 case A_IRQ_CTRL_ACK: 1279 *data = s->irq_ctrl; 1280 return MEMTX_OK; 1281 case A_GERROR: 1282 *data = s->gerror; 1283 return MEMTX_OK; 1284 case A_GERRORN: 1285 *data = s->gerrorn; 1286 return MEMTX_OK; 1287 case A_GERROR_IRQ_CFG0: /* 64b */ 1288 *data = extract64(s->gerror_irq_cfg0, 0, 32); 1289 return MEMTX_OK; 1290 case A_GERROR_IRQ_CFG0 + 4: 1291 *data = extract64(s->gerror_irq_cfg0, 32, 32); 1292 return MEMTX_OK; 1293 case A_GERROR_IRQ_CFG1: 1294 *data = s->gerror_irq_cfg1; 1295 return MEMTX_OK; 1296 case A_GERROR_IRQ_CFG2: 1297 *data = s->gerror_irq_cfg2; 1298 return MEMTX_OK; 1299 case A_STRTAB_BASE: /* 64b */ 1300 *data = extract64(s->strtab_base, 0, 32); 1301 return MEMTX_OK; 1302 case A_STRTAB_BASE + 4: /* 64b */ 1303 *data = extract64(s->strtab_base, 32, 32); 1304 return MEMTX_OK; 1305 case A_STRTAB_BASE_CFG: 1306 *data = s->strtab_base_cfg; 1307 return MEMTX_OK; 1308 case A_CMDQ_BASE: /* 64b */ 1309 *data = extract64(s->cmdq.base, 0, 32); 1310 return MEMTX_OK; 1311 case A_CMDQ_BASE + 4: 1312 *data = extract64(s->cmdq.base, 32, 32); 1313 return MEMTX_OK; 1314 case A_CMDQ_PROD: 1315 *data = s->cmdq.prod; 1316 return MEMTX_OK; 1317 case A_CMDQ_CONS: 1318 *data = s->cmdq.cons; 1319 return MEMTX_OK; 1320 case A_EVENTQ_BASE: /* 64b */ 1321 *data = extract64(s->eventq.base, 0, 32); 1322 return MEMTX_OK; 1323 case A_EVENTQ_BASE + 4: /* 64b */ 1324 *data = extract64(s->eventq.base, 32, 32); 1325 return MEMTX_OK; 1326 case A_EVENTQ_PROD: 1327 *data = s->eventq.prod; 1328 return MEMTX_OK; 1329 case A_EVENTQ_CONS: 1330 *data = s->eventq.cons; 1331 return MEMTX_OK; 1332 default: 1333 *data = 0; 1334 qemu_log_mask(LOG_UNIMP, 1335 "%s unhandled 32-bit access at 0x%"PRIx64" (RAZ)\n", 1336 __func__, offset); 1337 return MEMTX_OK; 1338 } 1339 } 1340 1341 static MemTxResult smmu_read_mmio(void *opaque, hwaddr offset, uint64_t *data, 1342 unsigned size, MemTxAttrs attrs) 1343 { 1344 SMMUState *sys = opaque; 1345 SMMUv3State *s = ARM_SMMUV3(sys); 1346 MemTxResult r; 1347 1348 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */ 1349 offset &= ~0x10000; 1350 1351 switch (size) { 1352 case 8: 1353 r = smmu_readll(s, offset, data, attrs); 1354 break; 1355 case 4: 1356 r = smmu_readl(s, offset, data, attrs); 1357 break; 1358 default: 1359 r = MEMTX_ERROR; 1360 break; 1361 } 1362 1363 trace_smmuv3_read_mmio(offset, *data, size, r); 1364 return r; 1365 } 1366 1367 static const MemoryRegionOps smmu_mem_ops = { 1368 .read_with_attrs = smmu_read_mmio, 1369 .write_with_attrs = smmu_write_mmio, 1370 .endianness = DEVICE_LITTLE_ENDIAN, 1371 .valid = { 1372 .min_access_size = 4, 1373 .max_access_size = 8, 1374 }, 1375 .impl = { 1376 .min_access_size = 4, 1377 .max_access_size = 8, 1378 }, 1379 }; 1380 1381 static void smmu_init_irq(SMMUv3State *s, SysBusDevice *dev) 1382 { 1383 int i; 1384 1385 for (i = 0; i < ARRAY_SIZE(s->irq); i++) { 1386 sysbus_init_irq(dev, &s->irq[i]); 1387 } 1388 } 1389 1390 static void smmu_reset(DeviceState *dev) 1391 { 1392 SMMUv3State *s = ARM_SMMUV3(dev); 1393 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s); 1394 1395 c->parent_reset(dev); 1396 1397 smmuv3_init_regs(s); 1398 } 1399 1400 static void smmu_realize(DeviceState *d, Error **errp) 1401 { 1402 SMMUState *sys = ARM_SMMU(d); 1403 SMMUv3State *s = ARM_SMMUV3(sys); 1404 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s); 1405 SysBusDevice *dev = SYS_BUS_DEVICE(d); 1406 Error *local_err = NULL; 1407 1408 c->parent_realize(d, &local_err); 1409 if (local_err) { 1410 error_propagate(errp, local_err); 1411 return; 1412 } 1413 1414 qemu_mutex_init(&s->mutex); 1415 1416 memory_region_init_io(&sys->iomem, OBJECT(s), 1417 &smmu_mem_ops, sys, TYPE_ARM_SMMUV3, 0x20000); 1418 1419 sys->mrtypename = TYPE_SMMUV3_IOMMU_MEMORY_REGION; 1420 1421 sysbus_init_mmio(dev, &sys->iomem); 1422 1423 smmu_init_irq(s, dev); 1424 } 1425 1426 static const VMStateDescription vmstate_smmuv3_queue = { 1427 .name = "smmuv3_queue", 1428 .version_id = 1, 1429 .minimum_version_id = 1, 1430 .fields = (VMStateField[]) { 1431 VMSTATE_UINT64(base, SMMUQueue), 1432 VMSTATE_UINT32(prod, SMMUQueue), 1433 VMSTATE_UINT32(cons, SMMUQueue), 1434 VMSTATE_UINT8(log2size, SMMUQueue), 1435 VMSTATE_END_OF_LIST(), 1436 }, 1437 }; 1438 1439 static const VMStateDescription vmstate_smmuv3 = { 1440 .name = "smmuv3", 1441 .version_id = 1, 1442 .minimum_version_id = 1, 1443 .fields = (VMStateField[]) { 1444 VMSTATE_UINT32(features, SMMUv3State), 1445 VMSTATE_UINT8(sid_size, SMMUv3State), 1446 VMSTATE_UINT8(sid_split, SMMUv3State), 1447 1448 VMSTATE_UINT32_ARRAY(cr, SMMUv3State, 3), 1449 VMSTATE_UINT32(cr0ack, SMMUv3State), 1450 VMSTATE_UINT32(statusr, SMMUv3State), 1451 VMSTATE_UINT32(irq_ctrl, SMMUv3State), 1452 VMSTATE_UINT32(gerror, SMMUv3State), 1453 VMSTATE_UINT32(gerrorn, SMMUv3State), 1454 VMSTATE_UINT64(gerror_irq_cfg0, SMMUv3State), 1455 VMSTATE_UINT32(gerror_irq_cfg1, SMMUv3State), 1456 VMSTATE_UINT32(gerror_irq_cfg2, SMMUv3State), 1457 VMSTATE_UINT64(strtab_base, SMMUv3State), 1458 VMSTATE_UINT32(strtab_base_cfg, SMMUv3State), 1459 VMSTATE_UINT64(eventq_irq_cfg0, SMMUv3State), 1460 VMSTATE_UINT32(eventq_irq_cfg1, SMMUv3State), 1461 VMSTATE_UINT32(eventq_irq_cfg2, SMMUv3State), 1462 1463 VMSTATE_STRUCT(cmdq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue), 1464 VMSTATE_STRUCT(eventq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue), 1465 1466 VMSTATE_END_OF_LIST(), 1467 }, 1468 }; 1469 1470 static void smmuv3_instance_init(Object *obj) 1471 { 1472 /* Nothing much to do here as of now */ 1473 } 1474 1475 static void smmuv3_class_init(ObjectClass *klass, void *data) 1476 { 1477 DeviceClass *dc = DEVICE_CLASS(klass); 1478 SMMUv3Class *c = ARM_SMMUV3_CLASS(klass); 1479 1480 dc->vmsd = &vmstate_smmuv3; 1481 device_class_set_parent_reset(dc, smmu_reset, &c->parent_reset); 1482 c->parent_realize = dc->realize; 1483 dc->realize = smmu_realize; 1484 } 1485 1486 static int smmuv3_notify_flag_changed(IOMMUMemoryRegion *iommu, 1487 IOMMUNotifierFlag old, 1488 IOMMUNotifierFlag new, 1489 Error **errp) 1490 { 1491 SMMUDevice *sdev = container_of(iommu, SMMUDevice, iommu); 1492 SMMUv3State *s3 = sdev->smmu; 1493 SMMUState *s = &(s3->smmu_state); 1494 1495 if (new & IOMMU_NOTIFIER_MAP) { 1496 error_setg(errp, 1497 "device %02x.%02x.%x requires iommu MAP notifier which is " 1498 "not currently supported", pci_bus_num(sdev->bus), 1499 PCI_SLOT(sdev->devfn), PCI_FUNC(sdev->devfn)); 1500 return -EINVAL; 1501 } 1502 1503 if (old == IOMMU_NOTIFIER_NONE) { 1504 trace_smmuv3_notify_flag_add(iommu->parent_obj.name); 1505 QLIST_INSERT_HEAD(&s->devices_with_notifiers, sdev, next); 1506 } else if (new == IOMMU_NOTIFIER_NONE) { 1507 trace_smmuv3_notify_flag_del(iommu->parent_obj.name); 1508 QLIST_REMOVE(sdev, next); 1509 } 1510 return 0; 1511 } 1512 1513 static void smmuv3_iommu_memory_region_class_init(ObjectClass *klass, 1514 void *data) 1515 { 1516 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 1517 1518 imrc->translate = smmuv3_translate; 1519 imrc->notify_flag_changed = smmuv3_notify_flag_changed; 1520 } 1521 1522 static const TypeInfo smmuv3_type_info = { 1523 .name = TYPE_ARM_SMMUV3, 1524 .parent = TYPE_ARM_SMMU, 1525 .instance_size = sizeof(SMMUv3State), 1526 .instance_init = smmuv3_instance_init, 1527 .class_size = sizeof(SMMUv3Class), 1528 .class_init = smmuv3_class_init, 1529 }; 1530 1531 static const TypeInfo smmuv3_iommu_memory_region_info = { 1532 .parent = TYPE_IOMMU_MEMORY_REGION, 1533 .name = TYPE_SMMUV3_IOMMU_MEMORY_REGION, 1534 .class_init = smmuv3_iommu_memory_region_class_init, 1535 }; 1536 1537 static void smmuv3_register_types(void) 1538 { 1539 type_register(&smmuv3_type_info); 1540 type_register(&smmuv3_iommu_memory_region_info); 1541 } 1542 1543 type_init(smmuv3_register_types) 1544 1545