1 /*
2 * Copyright (C) 2014-2016 Broadcom Corporation
3 * Copyright (c) 2017 Red Hat, Inc.
4 * Written by Prem Mallappa, Eric Auger
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, see <http://www.gnu.org/licenses/>.
17 */
18
19 #include "qemu/osdep.h"
20 #include "qemu/bitops.h"
21 #include "hw/irq.h"
22 #include "hw/sysbus.h"
23 #include "migration/vmstate.h"
24 #include "hw/qdev-properties.h"
25 #include "hw/qdev-core.h"
26 #include "hw/pci/pci.h"
27 #include "cpu.h"
28 #include "trace.h"
29 #include "qemu/log.h"
30 #include "qemu/error-report.h"
31 #include "qapi/error.h"
32
33 #include "hw/arm/smmuv3.h"
34 #include "smmuv3-internal.h"
35 #include "smmu-internal.h"
36
37 #define PTW_RECORD_FAULT(ptw_info, cfg) (((ptw_info).stage == SMMU_STAGE_1 && \
38 (cfg)->record_faults) || \
39 ((ptw_info).stage == SMMU_STAGE_2 && \
40 (cfg)->s2cfg.record_faults))
41
42 /**
43 * smmuv3_trigger_irq - pulse @irq if enabled and update
44 * GERROR register in case of GERROR interrupt
45 *
46 * @irq: irq type
47 * @gerror_mask: mask of gerrors to toggle (relevant if @irq is GERROR)
48 */
smmuv3_trigger_irq(SMMUv3State * s,SMMUIrq irq,uint32_t gerror_mask)49 static void smmuv3_trigger_irq(SMMUv3State *s, SMMUIrq irq,
50 uint32_t gerror_mask)
51 {
52
53 bool pulse = false;
54
55 switch (irq) {
56 case SMMU_IRQ_EVTQ:
57 pulse = smmuv3_eventq_irq_enabled(s);
58 break;
59 case SMMU_IRQ_PRIQ:
60 qemu_log_mask(LOG_UNIMP, "PRI not yet supported\n");
61 break;
62 case SMMU_IRQ_CMD_SYNC:
63 pulse = true;
64 break;
65 case SMMU_IRQ_GERROR:
66 {
67 uint32_t pending = s->gerror ^ s->gerrorn;
68 uint32_t new_gerrors = ~pending & gerror_mask;
69
70 if (!new_gerrors) {
71 /* only toggle non pending errors */
72 return;
73 }
74 s->gerror ^= new_gerrors;
75 trace_smmuv3_write_gerror(new_gerrors, s->gerror);
76
77 pulse = smmuv3_gerror_irq_enabled(s);
78 break;
79 }
80 }
81 if (pulse) {
82 trace_smmuv3_trigger_irq(irq);
83 qemu_irq_pulse(s->irq[irq]);
84 }
85 }
86
smmuv3_write_gerrorn(SMMUv3State * s,uint32_t new_gerrorn)87 static void smmuv3_write_gerrorn(SMMUv3State *s, uint32_t new_gerrorn)
88 {
89 uint32_t pending = s->gerror ^ s->gerrorn;
90 uint32_t toggled = s->gerrorn ^ new_gerrorn;
91
92 if (toggled & ~pending) {
93 qemu_log_mask(LOG_GUEST_ERROR,
94 "guest toggles non pending errors = 0x%x\n",
95 toggled & ~pending);
96 }
97
98 /*
99 * We do not raise any error in case guest toggles bits corresponding
100 * to not active IRQs (CONSTRAINED UNPREDICTABLE)
101 */
102 s->gerrorn = new_gerrorn;
103
104 trace_smmuv3_write_gerrorn(toggled & pending, s->gerrorn);
105 }
106
queue_read(SMMUQueue * q,Cmd * cmd)107 static inline MemTxResult queue_read(SMMUQueue *q, Cmd *cmd)
108 {
109 dma_addr_t addr = Q_CONS_ENTRY(q);
110 MemTxResult ret;
111 int i;
112
113 ret = dma_memory_read(&address_space_memory, addr, cmd, sizeof(Cmd),
114 MEMTXATTRS_UNSPECIFIED);
115 if (ret != MEMTX_OK) {
116 return ret;
117 }
118 for (i = 0; i < ARRAY_SIZE(cmd->word); i++) {
119 le32_to_cpus(&cmd->word[i]);
120 }
121 return ret;
122 }
123
queue_write(SMMUQueue * q,Evt * evt_in)124 static MemTxResult queue_write(SMMUQueue *q, Evt *evt_in)
125 {
126 dma_addr_t addr = Q_PROD_ENTRY(q);
127 MemTxResult ret;
128 Evt evt = *evt_in;
129 int i;
130
131 for (i = 0; i < ARRAY_SIZE(evt.word); i++) {
132 cpu_to_le32s(&evt.word[i]);
133 }
134 ret = dma_memory_write(&address_space_memory, addr, &evt, sizeof(Evt),
135 MEMTXATTRS_UNSPECIFIED);
136 if (ret != MEMTX_OK) {
137 return ret;
138 }
139
140 queue_prod_incr(q);
141 return MEMTX_OK;
142 }
143
smmuv3_write_eventq(SMMUv3State * s,Evt * evt)144 static MemTxResult smmuv3_write_eventq(SMMUv3State *s, Evt *evt)
145 {
146 SMMUQueue *q = &s->eventq;
147 MemTxResult r;
148
149 if (!smmuv3_eventq_enabled(s)) {
150 return MEMTX_ERROR;
151 }
152
153 if (smmuv3_q_full(q)) {
154 return MEMTX_ERROR;
155 }
156
157 r = queue_write(q, evt);
158 if (r != MEMTX_OK) {
159 return r;
160 }
161
162 if (!smmuv3_q_empty(q)) {
163 smmuv3_trigger_irq(s, SMMU_IRQ_EVTQ, 0);
164 }
165 return MEMTX_OK;
166 }
167
smmuv3_record_event(SMMUv3State * s,SMMUEventInfo * info)168 void smmuv3_record_event(SMMUv3State *s, SMMUEventInfo *info)
169 {
170 Evt evt = {};
171 MemTxResult r;
172
173 if (!smmuv3_eventq_enabled(s)) {
174 return;
175 }
176
177 EVT_SET_TYPE(&evt, info->type);
178 EVT_SET_SID(&evt, info->sid);
179
180 switch (info->type) {
181 case SMMU_EVT_NONE:
182 return;
183 case SMMU_EVT_F_UUT:
184 EVT_SET_SSID(&evt, info->u.f_uut.ssid);
185 EVT_SET_SSV(&evt, info->u.f_uut.ssv);
186 EVT_SET_ADDR(&evt, info->u.f_uut.addr);
187 EVT_SET_RNW(&evt, info->u.f_uut.rnw);
188 EVT_SET_PNU(&evt, info->u.f_uut.pnu);
189 EVT_SET_IND(&evt, info->u.f_uut.ind);
190 break;
191 case SMMU_EVT_C_BAD_STREAMID:
192 EVT_SET_SSID(&evt, info->u.c_bad_streamid.ssid);
193 EVT_SET_SSV(&evt, info->u.c_bad_streamid.ssv);
194 break;
195 case SMMU_EVT_F_STE_FETCH:
196 EVT_SET_SSID(&evt, info->u.f_ste_fetch.ssid);
197 EVT_SET_SSV(&evt, info->u.f_ste_fetch.ssv);
198 EVT_SET_ADDR2(&evt, info->u.f_ste_fetch.addr);
199 break;
200 case SMMU_EVT_C_BAD_STE:
201 EVT_SET_SSID(&evt, info->u.c_bad_ste.ssid);
202 EVT_SET_SSV(&evt, info->u.c_bad_ste.ssv);
203 break;
204 case SMMU_EVT_F_STREAM_DISABLED:
205 break;
206 case SMMU_EVT_F_TRANS_FORBIDDEN:
207 EVT_SET_ADDR(&evt, info->u.f_transl_forbidden.addr);
208 EVT_SET_RNW(&evt, info->u.f_transl_forbidden.rnw);
209 break;
210 case SMMU_EVT_C_BAD_SUBSTREAMID:
211 EVT_SET_SSID(&evt, info->u.c_bad_substream.ssid);
212 break;
213 case SMMU_EVT_F_CD_FETCH:
214 EVT_SET_SSID(&evt, info->u.f_cd_fetch.ssid);
215 EVT_SET_SSV(&evt, info->u.f_cd_fetch.ssv);
216 EVT_SET_ADDR(&evt, info->u.f_cd_fetch.addr);
217 break;
218 case SMMU_EVT_C_BAD_CD:
219 EVT_SET_SSID(&evt, info->u.c_bad_cd.ssid);
220 EVT_SET_SSV(&evt, info->u.c_bad_cd.ssv);
221 break;
222 case SMMU_EVT_F_WALK_EABT:
223 case SMMU_EVT_F_TRANSLATION:
224 case SMMU_EVT_F_ADDR_SIZE:
225 case SMMU_EVT_F_ACCESS:
226 case SMMU_EVT_F_PERMISSION:
227 EVT_SET_STALL(&evt, info->u.f_walk_eabt.stall);
228 EVT_SET_STAG(&evt, info->u.f_walk_eabt.stag);
229 EVT_SET_SSID(&evt, info->u.f_walk_eabt.ssid);
230 EVT_SET_SSV(&evt, info->u.f_walk_eabt.ssv);
231 EVT_SET_S2(&evt, info->u.f_walk_eabt.s2);
232 EVT_SET_ADDR(&evt, info->u.f_walk_eabt.addr);
233 EVT_SET_RNW(&evt, info->u.f_walk_eabt.rnw);
234 EVT_SET_PNU(&evt, info->u.f_walk_eabt.pnu);
235 EVT_SET_IND(&evt, info->u.f_walk_eabt.ind);
236 EVT_SET_CLASS(&evt, info->u.f_walk_eabt.class);
237 EVT_SET_ADDR2(&evt, info->u.f_walk_eabt.addr2);
238 break;
239 case SMMU_EVT_F_CFG_CONFLICT:
240 EVT_SET_SSID(&evt, info->u.f_cfg_conflict.ssid);
241 EVT_SET_SSV(&evt, info->u.f_cfg_conflict.ssv);
242 break;
243 /* rest is not implemented */
244 case SMMU_EVT_F_BAD_ATS_TREQ:
245 case SMMU_EVT_F_TLB_CONFLICT:
246 case SMMU_EVT_E_PAGE_REQ:
247 default:
248 g_assert_not_reached();
249 }
250
251 trace_smmuv3_record_event(smmu_event_string(info->type), info->sid);
252 r = smmuv3_write_eventq(s, &evt);
253 if (r != MEMTX_OK) {
254 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_EVENTQ_ABT_ERR_MASK);
255 }
256 info->recorded = true;
257 }
258
smmuv3_init_regs(SMMUv3State * s)259 static void smmuv3_init_regs(SMMUv3State *s)
260 {
261 /* Based on sys property, the stages supported in smmu will be advertised.*/
262 if (s->stage && !strcmp("2", s->stage)) {
263 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S2P, 1);
264 } else if (s->stage && !strcmp("nested", s->stage)) {
265 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1);
266 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S2P, 1);
267 } else {
268 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1);
269 }
270
271 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTF, 2); /* AArch64 PTW only */
272 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, COHACC, 1); /* IO coherent */
273 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ASID16, 1); /* 16-bit ASID */
274 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, VMID16, 1); /* 16-bit VMID */
275 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTENDIAN, 2); /* little endian */
276 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STALL_MODEL, 1); /* No stall */
277 /* terminated transaction will always be aborted/error returned */
278 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TERM_MODEL, 1);
279 /* 2-level stream table supported */
280 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STLEVEL, 1);
281
282 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SIDSIZE, SMMU_IDR1_SIDSIZE);
283 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, EVENTQS, SMMU_EVENTQS);
284 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, CMDQS, SMMU_CMDQS);
285
286 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1);
287 if (FIELD_EX32(s->idr[0], IDR0, S2P)) {
288 /* XNX is a stage-2-specific feature */
289 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, XNX, 1);
290 }
291 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1);
292 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, BBML, 2);
293
294 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS); /* 44 bits */
295 /* 4K, 16K and 64K granule support */
296 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1);
297 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN16K, 1);
298 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1);
299
300 s->cmdq.base = deposit64(s->cmdq.base, 0, 5, SMMU_CMDQS);
301 s->cmdq.prod = 0;
302 s->cmdq.cons = 0;
303 s->cmdq.entry_size = sizeof(struct Cmd);
304 s->eventq.base = deposit64(s->eventq.base, 0, 5, SMMU_EVENTQS);
305 s->eventq.prod = 0;
306 s->eventq.cons = 0;
307 s->eventq.entry_size = sizeof(struct Evt);
308
309 s->features = 0;
310 s->sid_split = 0;
311 s->aidr = 0x1;
312 s->cr[0] = 0;
313 s->cr0ack = 0;
314 s->irq_ctrl = 0;
315 s->gerror = 0;
316 s->gerrorn = 0;
317 s->statusr = 0;
318 s->gbpa = SMMU_GBPA_RESET_VAL;
319 }
320
smmu_get_ste(SMMUv3State * s,dma_addr_t addr,STE * buf,SMMUEventInfo * event)321 static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf,
322 SMMUEventInfo *event)
323 {
324 int ret, i;
325
326 trace_smmuv3_get_ste(addr);
327 /* TODO: guarantee 64-bit single-copy atomicity */
328 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
329 MEMTXATTRS_UNSPECIFIED);
330 if (ret != MEMTX_OK) {
331 qemu_log_mask(LOG_GUEST_ERROR,
332 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
333 event->type = SMMU_EVT_F_STE_FETCH;
334 event->u.f_ste_fetch.addr = addr;
335 return -EINVAL;
336 }
337 for (i = 0; i < ARRAY_SIZE(buf->word); i++) {
338 le32_to_cpus(&buf->word[i]);
339 }
340 return 0;
341
342 }
343
344 static SMMUTranslationStatus smmuv3_do_translate(SMMUv3State *s, hwaddr addr,
345 SMMUTransCfg *cfg,
346 SMMUEventInfo *event,
347 IOMMUAccessFlags flag,
348 SMMUTLBEntry **out_entry,
349 SMMUTranslationClass class);
350 /* @ssid > 0 not supported yet */
smmu_get_cd(SMMUv3State * s,STE * ste,SMMUTransCfg * cfg,uint32_t ssid,CD * buf,SMMUEventInfo * event)351 static int smmu_get_cd(SMMUv3State *s, STE *ste, SMMUTransCfg *cfg,
352 uint32_t ssid, CD *buf, SMMUEventInfo *event)
353 {
354 dma_addr_t addr = STE_CTXPTR(ste);
355 int ret, i;
356 SMMUTranslationStatus status;
357 SMMUTLBEntry *entry;
358
359 trace_smmuv3_get_cd(addr);
360
361 if (cfg->stage == SMMU_NESTED) {
362 status = smmuv3_do_translate(s, addr, cfg, event,
363 IOMMU_RO, &entry, SMMU_CLASS_CD);
364
365 /* Same PTW faults are reported but with CLASS = CD. */
366 if (status != SMMU_TRANS_SUCCESS) {
367 return -EINVAL;
368 }
369
370 addr = CACHED_ENTRY_TO_ADDR(entry, addr);
371 }
372
373 /* TODO: guarantee 64-bit single-copy atomicity */
374 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
375 MEMTXATTRS_UNSPECIFIED);
376 if (ret != MEMTX_OK) {
377 qemu_log_mask(LOG_GUEST_ERROR,
378 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
379 event->type = SMMU_EVT_F_CD_FETCH;
380 event->u.f_ste_fetch.addr = addr;
381 return -EINVAL;
382 }
383 for (i = 0; i < ARRAY_SIZE(buf->word); i++) {
384 le32_to_cpus(&buf->word[i]);
385 }
386 return 0;
387 }
388
389 /*
390 * Max valid value is 39 when SMMU_IDR3.STT == 0.
391 * In architectures after SMMUv3.0:
392 * - If STE.S2TG selects a 4KB or 16KB granule, the minimum valid value for this
393 * field is MAX(16, 64-IAS)
394 * - If STE.S2TG selects a 64KB granule, the minimum valid value for this field
395 * is (64-IAS).
396 * As we only support AA64, IAS = OAS.
397 */
s2t0sz_valid(SMMUTransCfg * cfg)398 static bool s2t0sz_valid(SMMUTransCfg *cfg)
399 {
400 if (cfg->s2cfg.tsz > 39) {
401 return false;
402 }
403
404 if (cfg->s2cfg.granule_sz == 16) {
405 return (cfg->s2cfg.tsz >= 64 - cfg->s2cfg.eff_ps);
406 }
407
408 return (cfg->s2cfg.tsz >= MAX(64 - cfg->s2cfg.eff_ps, 16));
409 }
410
411 /*
412 * Return true if s2 page table config is valid.
413 * This checks with the configured start level, ias_bits and granularity we can
414 * have a valid page table as described in ARM ARM D8.2 Translation process.
415 * The idea here is to see for the highest possible number of IPA bits, how
416 * many concatenated tables we would need, if it is more than 16, then this is
417 * not possible.
418 */
s2_pgtable_config_valid(uint8_t sl0,uint8_t t0sz,uint8_t gran)419 static bool s2_pgtable_config_valid(uint8_t sl0, uint8_t t0sz, uint8_t gran)
420 {
421 int level = get_start_level(sl0, gran);
422 uint64_t ipa_bits = 64 - t0sz;
423 uint64_t max_ipa = (1ULL << ipa_bits) - 1;
424 int nr_concat = pgd_concat_idx(level, gran, max_ipa) + 1;
425
426 return nr_concat <= VMSA_MAX_S2_CONCAT;
427 }
428
decode_ste_s2_cfg(SMMUv3State * s,SMMUTransCfg * cfg,STE * ste)429 static int decode_ste_s2_cfg(SMMUv3State *s, SMMUTransCfg *cfg,
430 STE *ste)
431 {
432 uint8_t oas = FIELD_EX32(s->idr[5], IDR5, OAS);
433
434 if (STE_S2AA64(ste) == 0x0) {
435 qemu_log_mask(LOG_UNIMP,
436 "SMMUv3 AArch32 tables not supported\n");
437 g_assert_not_reached();
438 }
439
440 switch (STE_S2TG(ste)) {
441 case 0x0: /* 4KB */
442 cfg->s2cfg.granule_sz = 12;
443 break;
444 case 0x1: /* 64KB */
445 cfg->s2cfg.granule_sz = 16;
446 break;
447 case 0x2: /* 16KB */
448 cfg->s2cfg.granule_sz = 14;
449 break;
450 default:
451 qemu_log_mask(LOG_GUEST_ERROR,
452 "SMMUv3 bad STE S2TG: %x\n", STE_S2TG(ste));
453 goto bad_ste;
454 }
455
456 cfg->s2cfg.vttb = STE_S2TTB(ste);
457
458 cfg->s2cfg.sl0 = STE_S2SL0(ste);
459 /* FEAT_TTST not supported. */
460 if (cfg->s2cfg.sl0 == 0x3) {
461 qemu_log_mask(LOG_UNIMP, "SMMUv3 S2SL0 = 0x3 has no meaning!\n");
462 goto bad_ste;
463 }
464
465 /* For AA64, The effective S2PS size is capped to the OAS. */
466 cfg->s2cfg.eff_ps = oas2bits(MIN(STE_S2PS(ste), oas));
467 /*
468 * For SMMUv3.1 and later, when OAS == IAS == 52, the stage 2 input
469 * range is further limited to 48 bits unless STE.S2TG indicates a
470 * 64KB granule.
471 */
472 if (cfg->s2cfg.granule_sz != 16) {
473 cfg->s2cfg.eff_ps = MIN(cfg->s2cfg.eff_ps, 48);
474 }
475 /*
476 * It is ILLEGAL for the address in S2TTB to be outside the range
477 * described by the effective S2PS value.
478 */
479 if (cfg->s2cfg.vttb & ~(MAKE_64BIT_MASK(0, cfg->s2cfg.eff_ps))) {
480 qemu_log_mask(LOG_GUEST_ERROR,
481 "SMMUv3 S2TTB too large 0x%" PRIx64
482 ", effective PS %d bits\n",
483 cfg->s2cfg.vttb, cfg->s2cfg.eff_ps);
484 goto bad_ste;
485 }
486
487 cfg->s2cfg.tsz = STE_S2T0SZ(ste);
488
489 if (!s2t0sz_valid(cfg)) {
490 qemu_log_mask(LOG_GUEST_ERROR, "SMMUv3 bad STE S2T0SZ = %d\n",
491 cfg->s2cfg.tsz);
492 goto bad_ste;
493 }
494
495 if (!s2_pgtable_config_valid(cfg->s2cfg.sl0, cfg->s2cfg.tsz,
496 cfg->s2cfg.granule_sz)) {
497 qemu_log_mask(LOG_GUEST_ERROR,
498 "SMMUv3 STE stage 2 config not valid!\n");
499 goto bad_ste;
500 }
501
502 /* Only LE supported(IDR0.TTENDIAN). */
503 if (STE_S2ENDI(ste)) {
504 qemu_log_mask(LOG_GUEST_ERROR,
505 "SMMUv3 STE_S2ENDI only supports LE!\n");
506 goto bad_ste;
507 }
508
509 cfg->s2cfg.affd = STE_S2AFFD(ste);
510
511 cfg->s2cfg.record_faults = STE_S2R(ste);
512 /* As stall is not supported. */
513 if (STE_S2S(ste)) {
514 qemu_log_mask(LOG_UNIMP, "SMMUv3 Stall not implemented!\n");
515 goto bad_ste;
516 }
517
518 return 0;
519
520 bad_ste:
521 return -EINVAL;
522 }
523
decode_ste_config(SMMUTransCfg * cfg,uint32_t config)524 static void decode_ste_config(SMMUTransCfg *cfg, uint32_t config)
525 {
526
527 if (STE_CFG_ABORT(config)) {
528 cfg->aborted = true;
529 return;
530 }
531 if (STE_CFG_BYPASS(config)) {
532 cfg->bypassed = true;
533 return;
534 }
535
536 if (STE_CFG_S1_ENABLED(config)) {
537 cfg->stage = SMMU_STAGE_1;
538 }
539
540 if (STE_CFG_S2_ENABLED(config)) {
541 cfg->stage |= SMMU_STAGE_2;
542 }
543 }
544
545 /* Returns < 0 in case of invalid STE, 0 otherwise */
decode_ste(SMMUv3State * s,SMMUTransCfg * cfg,STE * ste,SMMUEventInfo * event)546 static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg,
547 STE *ste, SMMUEventInfo *event)
548 {
549 uint32_t config;
550 uint8_t oas = FIELD_EX32(s->idr[5], IDR5, OAS);
551 int ret;
552
553 if (!STE_VALID(ste)) {
554 if (!event->inval_ste_allowed) {
555 qemu_log_mask(LOG_GUEST_ERROR, "invalid STE\n");
556 }
557 goto bad_ste;
558 }
559
560 config = STE_CONFIG(ste);
561
562 decode_ste_config(cfg, config);
563
564 if (cfg->aborted || cfg->bypassed) {
565 return 0;
566 }
567
568 /*
569 * If a stage is enabled in SW while not advertised, throw bad ste
570 * according to user manual(IHI0070E) "5.2 Stream Table Entry".
571 */
572 if (!STAGE1_SUPPORTED(s) && STE_CFG_S1_ENABLED(config)) {
573 qemu_log_mask(LOG_GUEST_ERROR, "SMMUv3 S1 used but not supported.\n");
574 goto bad_ste;
575 }
576 if (!STAGE2_SUPPORTED(s) && STE_CFG_S2_ENABLED(config)) {
577 qemu_log_mask(LOG_GUEST_ERROR, "SMMUv3 S2 used but not supported.\n");
578 goto bad_ste;
579 }
580
581 if (STAGE2_SUPPORTED(s)) {
582 /* VMID is considered even if s2 is disabled. */
583 cfg->s2cfg.vmid = STE_S2VMID(ste);
584 } else {
585 /* Default to -1 */
586 cfg->s2cfg.vmid = -1;
587 }
588
589 if (STE_CFG_S2_ENABLED(config)) {
590 /*
591 * Stage-1 OAS defaults to OAS even if not enabled as it would be used
592 * in input address check for stage-2.
593 */
594 cfg->oas = oas2bits(oas);
595 ret = decode_ste_s2_cfg(s, cfg, ste);
596 if (ret) {
597 goto bad_ste;
598 }
599 }
600
601 if (STE_S1CDMAX(ste) != 0) {
602 qemu_log_mask(LOG_UNIMP,
603 "SMMUv3 does not support multiple context descriptors yet\n");
604 goto bad_ste;
605 }
606
607 if (STE_S1STALLD(ste)) {
608 qemu_log_mask(LOG_UNIMP,
609 "SMMUv3 S1 stalling fault model not allowed yet\n");
610 goto bad_ste;
611 }
612 return 0;
613
614 bad_ste:
615 event->type = SMMU_EVT_C_BAD_STE;
616 return -EINVAL;
617 }
618
619 /**
620 * smmu_find_ste - Return the stream table entry associated
621 * to the sid
622 *
623 * @s: smmuv3 handle
624 * @sid: stream ID
625 * @ste: returned stream table entry
626 * @event: handle to an event info
627 *
628 * Supports linear and 2-level stream table
629 * Return 0 on success, -EINVAL otherwise
630 */
smmu_find_ste(SMMUv3State * s,uint32_t sid,STE * ste,SMMUEventInfo * event)631 static int smmu_find_ste(SMMUv3State *s, uint32_t sid, STE *ste,
632 SMMUEventInfo *event)
633 {
634 dma_addr_t addr, strtab_base;
635 uint32_t log2size;
636 int strtab_size_shift;
637 int ret;
638
639 trace_smmuv3_find_ste(sid, s->features, s->sid_split);
640 log2size = FIELD_EX32(s->strtab_base_cfg, STRTAB_BASE_CFG, LOG2SIZE);
641 /*
642 * Check SID range against both guest-configured and implementation limits
643 */
644 if (sid >= (1 << MIN(log2size, SMMU_IDR1_SIDSIZE))) {
645 event->type = SMMU_EVT_C_BAD_STREAMID;
646 return -EINVAL;
647 }
648 if (s->features & SMMU_FEATURE_2LVL_STE) {
649 int l1_ste_offset, l2_ste_offset, max_l2_ste, span, i;
650 dma_addr_t l1ptr, l2ptr;
651 STEDesc l1std;
652
653 /*
654 * Align strtab base address to table size. For this purpose, assume it
655 * is not bounded by SMMU_IDR1_SIDSIZE.
656 */
657 strtab_size_shift = MAX(5, (int)log2size - s->sid_split - 1 + 3);
658 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
659 ~MAKE_64BIT_MASK(0, strtab_size_shift);
660 l1_ste_offset = sid >> s->sid_split;
661 l2_ste_offset = sid & ((1 << s->sid_split) - 1);
662 l1ptr = (dma_addr_t)(strtab_base + l1_ste_offset * sizeof(l1std));
663 /* TODO: guarantee 64-bit single-copy atomicity */
664 ret = dma_memory_read(&address_space_memory, l1ptr, &l1std,
665 sizeof(l1std), MEMTXATTRS_UNSPECIFIED);
666 if (ret != MEMTX_OK) {
667 qemu_log_mask(LOG_GUEST_ERROR,
668 "Could not read L1PTR at 0X%"PRIx64"\n", l1ptr);
669 event->type = SMMU_EVT_F_STE_FETCH;
670 event->u.f_ste_fetch.addr = l1ptr;
671 return -EINVAL;
672 }
673 for (i = 0; i < ARRAY_SIZE(l1std.word); i++) {
674 le32_to_cpus(&l1std.word[i]);
675 }
676
677 span = L1STD_SPAN(&l1std);
678
679 if (!span) {
680 /* l2ptr is not valid */
681 if (!event->inval_ste_allowed) {
682 qemu_log_mask(LOG_GUEST_ERROR,
683 "invalid sid=%d (L1STD span=0)\n", sid);
684 }
685 event->type = SMMU_EVT_C_BAD_STREAMID;
686 return -EINVAL;
687 }
688 max_l2_ste = (1 << span) - 1;
689 l2ptr = l1std_l2ptr(&l1std);
690 trace_smmuv3_find_ste_2lvl(s->strtab_base, l1ptr, l1_ste_offset,
691 l2ptr, l2_ste_offset, max_l2_ste);
692 if (l2_ste_offset > max_l2_ste) {
693 qemu_log_mask(LOG_GUEST_ERROR,
694 "l2_ste_offset=%d > max_l2_ste=%d\n",
695 l2_ste_offset, max_l2_ste);
696 event->type = SMMU_EVT_C_BAD_STE;
697 return -EINVAL;
698 }
699 addr = l2ptr + l2_ste_offset * sizeof(*ste);
700 } else {
701 strtab_size_shift = log2size + 5;
702 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
703 ~MAKE_64BIT_MASK(0, strtab_size_shift);
704 addr = strtab_base + sid * sizeof(*ste);
705 }
706
707 if (smmu_get_ste(s, addr, ste, event)) {
708 return -EINVAL;
709 }
710
711 return 0;
712 }
713
decode_cd(SMMUv3State * s,SMMUTransCfg * cfg,CD * cd,SMMUEventInfo * event)714 static int decode_cd(SMMUv3State *s, SMMUTransCfg *cfg,
715 CD *cd, SMMUEventInfo *event)
716 {
717 int ret = -EINVAL;
718 int i;
719 SMMUTranslationStatus status;
720 SMMUTLBEntry *entry;
721 uint8_t oas = FIELD_EX32(s->idr[5], IDR5, OAS);
722
723 if (!CD_VALID(cd) || !CD_AARCH64(cd)) {
724 goto bad_cd;
725 }
726 if (!CD_A(cd)) {
727 goto bad_cd; /* SMMU_IDR0.TERM_MODEL == 1 */
728 }
729 if (CD_S(cd)) {
730 goto bad_cd; /* !STE_SECURE && SMMU_IDR0.STALL_MODEL == 1 */
731 }
732 if (CD_HA(cd) || CD_HD(cd)) {
733 goto bad_cd; /* HTTU = 0 */
734 }
735
736 /* we support only those at the moment */
737 cfg->aa64 = true;
738
739 cfg->oas = oas2bits(CD_IPS(cd));
740 cfg->oas = MIN(oas2bits(oas), cfg->oas);
741 cfg->tbi = CD_TBI(cd);
742 cfg->asid = CD_ASID(cd);
743 cfg->affd = CD_AFFD(cd);
744
745 trace_smmuv3_decode_cd(cfg->oas);
746
747 /* decode data dependent on TT */
748 for (i = 0; i <= 1; i++) {
749 int tg, tsz;
750 SMMUTransTableInfo *tt = &cfg->tt[i];
751
752 cfg->tt[i].disabled = CD_EPD(cd, i);
753 if (cfg->tt[i].disabled) {
754 continue;
755 }
756
757 tsz = CD_TSZ(cd, i);
758 if (tsz < 16 || tsz > 39) {
759 goto bad_cd;
760 }
761
762 tg = CD_TG(cd, i);
763 tt->granule_sz = tg2granule(tg, i);
764 if ((tt->granule_sz != 12 && tt->granule_sz != 14 &&
765 tt->granule_sz != 16) || CD_ENDI(cd)) {
766 goto bad_cd;
767 }
768
769 /*
770 * An address greater than 48 bits in size can only be output from a
771 * TTD when, in SMMUv3.1 and later, the effective IPS is 52 and a 64KB
772 * granule is in use for that translation table
773 */
774 if (tt->granule_sz != 16) {
775 cfg->oas = MIN(cfg->oas, 48);
776 }
777 tt->tsz = tsz;
778 tt->ttb = CD_TTB(cd, i);
779
780 if (tt->ttb & ~(MAKE_64BIT_MASK(0, cfg->oas))) {
781 goto bad_cd;
782 }
783
784 /* Translate the TTBx, from IPA to PA if nesting is enabled. */
785 if (cfg->stage == SMMU_NESTED) {
786 status = smmuv3_do_translate(s, tt->ttb, cfg, event, IOMMU_RO,
787 &entry, SMMU_CLASS_TT);
788 /*
789 * Same PTW faults are reported but with CLASS = TT.
790 * If TTBx is larger than the effective stage 1 output addres
791 * size, it reports C_BAD_CD, which is handled by the above case.
792 */
793 if (status != SMMU_TRANS_SUCCESS) {
794 return -EINVAL;
795 }
796 tt->ttb = CACHED_ENTRY_TO_ADDR(entry, tt->ttb);
797 }
798
799 tt->had = CD_HAD(cd, i);
800 trace_smmuv3_decode_cd_tt(i, tt->tsz, tt->ttb, tt->granule_sz, tt->had);
801 }
802
803 cfg->record_faults = CD_R(cd);
804
805 return 0;
806
807 bad_cd:
808 event->type = SMMU_EVT_C_BAD_CD;
809 return ret;
810 }
811
812 /**
813 * smmuv3_decode_config - Prepare the translation configuration
814 * for the @mr iommu region
815 * @mr: iommu memory region the translation config must be prepared for
816 * @cfg: output translation configuration which is populated through
817 * the different configuration decoding steps
818 * @event: must be zero'ed by the caller
819 *
820 * return < 0 in case of config decoding error (@event is filled
821 * accordingly). Return 0 otherwise.
822 */
smmuv3_decode_config(IOMMUMemoryRegion * mr,SMMUTransCfg * cfg,SMMUEventInfo * event)823 static int smmuv3_decode_config(IOMMUMemoryRegion *mr, SMMUTransCfg *cfg,
824 SMMUEventInfo *event)
825 {
826 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
827 uint32_t sid = smmu_get_sid(sdev);
828 SMMUv3State *s = sdev->smmu;
829 int ret;
830 STE ste;
831 CD cd;
832
833 /* ASID defaults to -1 (if s1 is not supported). */
834 cfg->asid = -1;
835
836 ret = smmu_find_ste(s, sid, &ste, event);
837 if (ret) {
838 return ret;
839 }
840
841 ret = decode_ste(s, cfg, &ste, event);
842 if (ret) {
843 return ret;
844 }
845
846 if (cfg->aborted || cfg->bypassed || (cfg->stage == SMMU_STAGE_2)) {
847 return 0;
848 }
849
850 ret = smmu_get_cd(s, &ste, cfg, 0 /* ssid */, &cd, event);
851 if (ret) {
852 return ret;
853 }
854
855 return decode_cd(s, cfg, &cd, event);
856 }
857
858 /**
859 * smmuv3_get_config - Look up for a cached copy of configuration data for
860 * @sdev and on cache miss performs a configuration structure decoding from
861 * guest RAM.
862 *
863 * @sdev: SMMUDevice handle
864 * @event: output event info
865 *
866 * The configuration cache contains data resulting from both STE and CD
867 * decoding under the form of an SMMUTransCfg struct. The hash table is indexed
868 * by the SMMUDevice handle.
869 */
smmuv3_get_config(SMMUDevice * sdev,SMMUEventInfo * event)870 static SMMUTransCfg *smmuv3_get_config(SMMUDevice *sdev, SMMUEventInfo *event)
871 {
872 SMMUv3State *s = sdev->smmu;
873 SMMUState *bc = &s->smmu_state;
874 SMMUTransCfg *cfg;
875
876 cfg = g_hash_table_lookup(bc->configs, sdev);
877 if (cfg) {
878 sdev->cfg_cache_hits++;
879 trace_smmuv3_config_cache_hit(smmu_get_sid(sdev),
880 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
881 100 * sdev->cfg_cache_hits /
882 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
883 } else {
884 sdev->cfg_cache_misses++;
885 trace_smmuv3_config_cache_miss(smmu_get_sid(sdev),
886 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
887 100 * sdev->cfg_cache_hits /
888 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
889 cfg = g_new0(SMMUTransCfg, 1);
890
891 if (!smmuv3_decode_config(&sdev->iommu, cfg, event)) {
892 g_hash_table_insert(bc->configs, sdev, cfg);
893 } else {
894 g_free(cfg);
895 cfg = NULL;
896 }
897 }
898 return cfg;
899 }
900
smmuv3_flush_config(SMMUDevice * sdev)901 static void smmuv3_flush_config(SMMUDevice *sdev)
902 {
903 SMMUv3State *s = sdev->smmu;
904 SMMUState *bc = &s->smmu_state;
905
906 trace_smmuv3_config_cache_inv(smmu_get_sid(sdev));
907 g_hash_table_remove(bc->configs, sdev);
908 }
909
910 /* Do translation with TLB lookup. */
smmuv3_do_translate(SMMUv3State * s,hwaddr addr,SMMUTransCfg * cfg,SMMUEventInfo * event,IOMMUAccessFlags flag,SMMUTLBEntry ** out_entry,SMMUTranslationClass class)911 static SMMUTranslationStatus smmuv3_do_translate(SMMUv3State *s, hwaddr addr,
912 SMMUTransCfg *cfg,
913 SMMUEventInfo *event,
914 IOMMUAccessFlags flag,
915 SMMUTLBEntry **out_entry,
916 SMMUTranslationClass class)
917 {
918 SMMUPTWEventInfo ptw_info = {};
919 SMMUState *bs = ARM_SMMU(s);
920 SMMUTLBEntry *cached_entry = NULL;
921 int asid, stage;
922 bool desc_s2_translation = class != SMMU_CLASS_IN;
923
924 /*
925 * The function uses the argument class to identify which stage is used:
926 * - CLASS = IN: Means an input translation, determine the stage from STE.
927 * - CLASS = CD: Means the addr is an IPA of the CD, and it would be
928 * translated using the stage-2.
929 * - CLASS = TT: Means the addr is an IPA of the stage-1 translation table
930 * and it would be translated using the stage-2.
931 * For the last 2 cases instead of having intrusive changes in the common
932 * logic, we modify the cfg to be a stage-2 translation only in case of
933 * nested, and then restore it after.
934 */
935 if (desc_s2_translation) {
936 asid = cfg->asid;
937 stage = cfg->stage;
938 cfg->asid = -1;
939 cfg->stage = SMMU_STAGE_2;
940 }
941
942 cached_entry = smmu_translate(bs, cfg, addr, flag, &ptw_info);
943
944 if (desc_s2_translation) {
945 cfg->asid = asid;
946 cfg->stage = stage;
947 }
948
949 if (!cached_entry) {
950 /* All faults from PTW has S2 field. */
951 event->u.f_walk_eabt.s2 = (ptw_info.stage == SMMU_STAGE_2);
952 /*
953 * Fault class is set as follows based on "class" input to
954 * the function and to "ptw_info" from "smmu_translate()"
955 * For stage-1:
956 * - EABT => CLASS_TT (hardcoded)
957 * - other events => CLASS_IN (input to function)
958 * For stage-2 => CLASS_IN (input to function)
959 * For nested, for all events:
960 * - CD fetch => CLASS_CD (input to function)
961 * - walking stage 1 translation table => CLASS_TT (from
962 * is_ipa_descriptor or input in case of TTBx)
963 * - s2 translation => CLASS_IN (input to function)
964 */
965 class = ptw_info.is_ipa_descriptor ? SMMU_CLASS_TT : class;
966 switch (ptw_info.type) {
967 case SMMU_PTW_ERR_WALK_EABT:
968 event->type = SMMU_EVT_F_WALK_EABT;
969 event->u.f_walk_eabt.rnw = flag & 0x1;
970 event->u.f_walk_eabt.class = (ptw_info.stage == SMMU_STAGE_2) ?
971 class : SMMU_CLASS_TT;
972 event->u.f_walk_eabt.addr2 = ptw_info.addr;
973 break;
974 case SMMU_PTW_ERR_TRANSLATION:
975 if (PTW_RECORD_FAULT(ptw_info, cfg)) {
976 event->type = SMMU_EVT_F_TRANSLATION;
977 event->u.f_translation.addr2 = ptw_info.addr;
978 event->u.f_translation.class = class;
979 event->u.f_translation.rnw = flag & 0x1;
980 }
981 break;
982 case SMMU_PTW_ERR_ADDR_SIZE:
983 if (PTW_RECORD_FAULT(ptw_info, cfg)) {
984 event->type = SMMU_EVT_F_ADDR_SIZE;
985 event->u.f_addr_size.addr2 = ptw_info.addr;
986 event->u.f_addr_size.class = class;
987 event->u.f_addr_size.rnw = flag & 0x1;
988 }
989 break;
990 case SMMU_PTW_ERR_ACCESS:
991 if (PTW_RECORD_FAULT(ptw_info, cfg)) {
992 event->type = SMMU_EVT_F_ACCESS;
993 event->u.f_access.addr2 = ptw_info.addr;
994 event->u.f_access.class = class;
995 event->u.f_access.rnw = flag & 0x1;
996 }
997 break;
998 case SMMU_PTW_ERR_PERMISSION:
999 if (PTW_RECORD_FAULT(ptw_info, cfg)) {
1000 event->type = SMMU_EVT_F_PERMISSION;
1001 event->u.f_permission.addr2 = ptw_info.addr;
1002 event->u.f_permission.class = class;
1003 event->u.f_permission.rnw = flag & 0x1;
1004 }
1005 break;
1006 default:
1007 g_assert_not_reached();
1008 }
1009 return SMMU_TRANS_ERROR;
1010 }
1011 *out_entry = cached_entry;
1012 return SMMU_TRANS_SUCCESS;
1013 }
1014
1015 /*
1016 * Sets the InputAddr for an SMMU_TRANS_ERROR, as it can't be
1017 * set from all contexts, as smmuv3_get_config() can return
1018 * translation faults in case of nested translation (for CD
1019 * and TTBx). But in that case the iova is not known.
1020 */
smmuv3_fixup_event(SMMUEventInfo * event,hwaddr iova)1021 static void smmuv3_fixup_event(SMMUEventInfo *event, hwaddr iova)
1022 {
1023 switch (event->type) {
1024 case SMMU_EVT_F_WALK_EABT:
1025 case SMMU_EVT_F_TRANSLATION:
1026 case SMMU_EVT_F_ADDR_SIZE:
1027 case SMMU_EVT_F_ACCESS:
1028 case SMMU_EVT_F_PERMISSION:
1029 event->u.f_walk_eabt.addr = iova;
1030 break;
1031 default:
1032 break;
1033 }
1034 }
1035
1036 /* Entry point to SMMU, does everything. */
smmuv3_translate(IOMMUMemoryRegion * mr,hwaddr addr,IOMMUAccessFlags flag,int iommu_idx)1037 static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
1038 IOMMUAccessFlags flag, int iommu_idx)
1039 {
1040 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
1041 SMMUv3State *s = sdev->smmu;
1042 uint32_t sid = smmu_get_sid(sdev);
1043 SMMUEventInfo event = {.type = SMMU_EVT_NONE,
1044 .sid = sid,
1045 .inval_ste_allowed = false};
1046 SMMUTranslationStatus status;
1047 SMMUTransCfg *cfg = NULL;
1048 IOMMUTLBEntry entry = {
1049 .target_as = &address_space_memory,
1050 .iova = addr,
1051 .translated_addr = addr,
1052 .addr_mask = ~(hwaddr)0,
1053 .perm = IOMMU_NONE,
1054 };
1055 SMMUTLBEntry *cached_entry = NULL;
1056
1057 qemu_mutex_lock(&s->mutex);
1058
1059 if (!smmu_enabled(s)) {
1060 if (FIELD_EX32(s->gbpa, GBPA, ABORT)) {
1061 status = SMMU_TRANS_ABORT;
1062 } else {
1063 status = SMMU_TRANS_DISABLE;
1064 }
1065 goto epilogue;
1066 }
1067
1068 cfg = smmuv3_get_config(sdev, &event);
1069 if (!cfg) {
1070 status = SMMU_TRANS_ERROR;
1071 goto epilogue;
1072 }
1073
1074 if (cfg->aborted) {
1075 status = SMMU_TRANS_ABORT;
1076 goto epilogue;
1077 }
1078
1079 if (cfg->bypassed) {
1080 status = SMMU_TRANS_BYPASS;
1081 goto epilogue;
1082 }
1083
1084 status = smmuv3_do_translate(s, addr, cfg, &event, flag,
1085 &cached_entry, SMMU_CLASS_IN);
1086
1087 epilogue:
1088 qemu_mutex_unlock(&s->mutex);
1089 switch (status) {
1090 case SMMU_TRANS_SUCCESS:
1091 entry.perm = cached_entry->entry.perm;
1092 entry.translated_addr = CACHED_ENTRY_TO_ADDR(cached_entry, addr);
1093 entry.addr_mask = cached_entry->entry.addr_mask;
1094 trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr,
1095 entry.translated_addr, entry.perm,
1096 cfg->stage);
1097 break;
1098 case SMMU_TRANS_DISABLE:
1099 entry.perm = flag;
1100 entry.addr_mask = ~TARGET_PAGE_MASK;
1101 trace_smmuv3_translate_disable(mr->parent_obj.name, sid, addr,
1102 entry.perm);
1103 break;
1104 case SMMU_TRANS_BYPASS:
1105 entry.perm = flag;
1106 entry.addr_mask = ~TARGET_PAGE_MASK;
1107 trace_smmuv3_translate_bypass(mr->parent_obj.name, sid, addr,
1108 entry.perm);
1109 break;
1110 case SMMU_TRANS_ABORT:
1111 /* no event is recorded on abort */
1112 trace_smmuv3_translate_abort(mr->parent_obj.name, sid, addr,
1113 entry.perm);
1114 break;
1115 case SMMU_TRANS_ERROR:
1116 smmuv3_fixup_event(&event, addr);
1117 qemu_log_mask(LOG_GUEST_ERROR,
1118 "%s translation failed for iova=0x%"PRIx64" (%s)\n",
1119 mr->parent_obj.name, addr, smmu_event_string(event.type));
1120 smmuv3_record_event(s, &event);
1121 break;
1122 }
1123
1124 return entry;
1125 }
1126
1127 /**
1128 * smmuv3_notify_iova - call the notifier @n for a given
1129 * @asid and @iova tuple.
1130 *
1131 * @mr: IOMMU mr region handle
1132 * @n: notifier to be called
1133 * @asid: address space ID or negative value if we don't care
1134 * @vmid: virtual machine ID or negative value if we don't care
1135 * @iova: iova
1136 * @tg: translation granule (if communicated through range invalidation)
1137 * @num_pages: number of @granule sized pages (if tg != 0), otherwise 1
1138 * @stage: Which stage(1 or 2) is used
1139 */
smmuv3_notify_iova(IOMMUMemoryRegion * mr,IOMMUNotifier * n,int asid,int vmid,dma_addr_t iova,uint8_t tg,uint64_t num_pages,int stage)1140 static void smmuv3_notify_iova(IOMMUMemoryRegion *mr,
1141 IOMMUNotifier *n,
1142 int asid, int vmid,
1143 dma_addr_t iova, uint8_t tg,
1144 uint64_t num_pages, int stage)
1145 {
1146 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
1147 SMMUEventInfo eventinfo = {.inval_ste_allowed = true};
1148 SMMUTransCfg *cfg = smmuv3_get_config(sdev, &eventinfo);
1149 IOMMUTLBEvent event;
1150 uint8_t granule;
1151
1152 if (!cfg) {
1153 return;
1154 }
1155
1156 /*
1157 * stage is passed from TLB invalidation commands which can be either
1158 * stage-1 or stage-2.
1159 * However, IOMMUTLBEvent only understands IOVA, for stage-1 or stage-2
1160 * SMMU instances we consider the input address as the IOVA, but when
1161 * nesting is used, we can't mix stage-1 and stage-2 addresses, so for
1162 * nesting only stage-1 is considered the IOVA and would be notified.
1163 */
1164 if ((stage == SMMU_STAGE_2) && (cfg->stage == SMMU_NESTED))
1165 return;
1166
1167 if (!tg) {
1168 SMMUTransTableInfo *tt;
1169
1170 if (asid >= 0 && cfg->asid != asid) {
1171 return;
1172 }
1173
1174 if (vmid >= 0 && cfg->s2cfg.vmid != vmid) {
1175 return;
1176 }
1177
1178 if (stage == SMMU_STAGE_1) {
1179 tt = select_tt(cfg, iova);
1180 if (!tt) {
1181 return;
1182 }
1183 granule = tt->granule_sz;
1184 } else {
1185 granule = cfg->s2cfg.granule_sz;
1186 }
1187
1188 } else {
1189 granule = tg * 2 + 10;
1190 }
1191
1192 event.type = IOMMU_NOTIFIER_UNMAP;
1193 event.entry.target_as = &address_space_memory;
1194 event.entry.iova = iova;
1195 event.entry.addr_mask = num_pages * (1 << granule) - 1;
1196 event.entry.perm = IOMMU_NONE;
1197
1198 memory_region_notify_iommu_one(n, &event);
1199 }
1200
1201 /* invalidate an asid/vmid/iova range tuple in all mr's */
smmuv3_inv_notifiers_iova(SMMUState * s,int asid,int vmid,dma_addr_t iova,uint8_t tg,uint64_t num_pages,int stage)1202 static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, int vmid,
1203 dma_addr_t iova, uint8_t tg,
1204 uint64_t num_pages, int stage)
1205 {
1206 SMMUDevice *sdev;
1207
1208 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
1209 IOMMUMemoryRegion *mr = &sdev->iommu;
1210 IOMMUNotifier *n;
1211
1212 trace_smmuv3_inv_notifiers_iova(mr->parent_obj.name, asid, vmid,
1213 iova, tg, num_pages, stage);
1214
1215 IOMMU_NOTIFIER_FOREACH(n, mr) {
1216 smmuv3_notify_iova(mr, n, asid, vmid, iova, tg, num_pages, stage);
1217 }
1218 }
1219 }
1220
smmuv3_range_inval(SMMUState * s,Cmd * cmd,SMMUStage stage)1221 static void smmuv3_range_inval(SMMUState *s, Cmd *cmd, SMMUStage stage)
1222 {
1223 dma_addr_t end, addr = CMD_ADDR(cmd);
1224 uint8_t type = CMD_TYPE(cmd);
1225 int vmid = -1;
1226 uint8_t scale = CMD_SCALE(cmd);
1227 uint8_t num = CMD_NUM(cmd);
1228 uint8_t ttl = CMD_TTL(cmd);
1229 bool leaf = CMD_LEAF(cmd);
1230 uint8_t tg = CMD_TG(cmd);
1231 uint64_t num_pages;
1232 uint8_t granule;
1233 int asid = -1;
1234 SMMUv3State *smmuv3 = ARM_SMMUV3(s);
1235
1236 /* Only consider VMID if stage-2 is supported. */
1237 if (STAGE2_SUPPORTED(smmuv3)) {
1238 vmid = CMD_VMID(cmd);
1239 }
1240
1241 if (type == SMMU_CMD_TLBI_NH_VA) {
1242 asid = CMD_ASID(cmd);
1243 }
1244
1245 if (!tg) {
1246 trace_smmuv3_range_inval(vmid, asid, addr, tg, 1, ttl, leaf, stage);
1247 smmuv3_inv_notifiers_iova(s, asid, vmid, addr, tg, 1, stage);
1248 if (stage == SMMU_STAGE_1) {
1249 smmu_iotlb_inv_iova(s, asid, vmid, addr, tg, 1, ttl);
1250 } else {
1251 smmu_iotlb_inv_ipa(s, vmid, addr, tg, 1, ttl);
1252 }
1253 return;
1254 }
1255
1256 /* RIL in use */
1257
1258 num_pages = (num + 1) * BIT_ULL(scale);
1259 granule = tg * 2 + 10;
1260
1261 /* Split invalidations into ^2 range invalidations */
1262 end = addr + (num_pages << granule) - 1;
1263
1264 while (addr != end + 1) {
1265 uint64_t mask = dma_aligned_pow2_mask(addr, end, 64);
1266
1267 num_pages = (mask + 1) >> granule;
1268 trace_smmuv3_range_inval(vmid, asid, addr, tg, num_pages,
1269 ttl, leaf, stage);
1270 smmuv3_inv_notifiers_iova(s, asid, vmid, addr, tg, num_pages, stage);
1271 if (stage == SMMU_STAGE_1) {
1272 smmu_iotlb_inv_iova(s, asid, vmid, addr, tg, num_pages, ttl);
1273 } else {
1274 smmu_iotlb_inv_ipa(s, vmid, addr, tg, num_pages, ttl);
1275 }
1276 addr += mask + 1;
1277 }
1278 }
1279
1280 static gboolean
smmuv3_invalidate_ste(gpointer key,gpointer value,gpointer user_data)1281 smmuv3_invalidate_ste(gpointer key, gpointer value, gpointer user_data)
1282 {
1283 SMMUDevice *sdev = (SMMUDevice *)key;
1284 uint32_t sid = smmu_get_sid(sdev);
1285 SMMUSIDRange *sid_range = (SMMUSIDRange *)user_data;
1286
1287 if (sid < sid_range->start || sid > sid_range->end) {
1288 return false;
1289 }
1290 trace_smmuv3_config_cache_inv(sid);
1291 return true;
1292 }
1293
smmuv3_cmdq_consume(SMMUv3State * s)1294 static int smmuv3_cmdq_consume(SMMUv3State *s)
1295 {
1296 SMMUState *bs = ARM_SMMU(s);
1297 SMMUCmdError cmd_error = SMMU_CERROR_NONE;
1298 SMMUQueue *q = &s->cmdq;
1299 SMMUCommandType type = 0;
1300
1301 if (!smmuv3_cmdq_enabled(s)) {
1302 return 0;
1303 }
1304 /*
1305 * some commands depend on register values, typically CR0. In case those
1306 * register values change while handling the command, spec says it
1307 * is UNPREDICTABLE whether the command is interpreted under the new
1308 * or old value.
1309 */
1310
1311 while (!smmuv3_q_empty(q)) {
1312 uint32_t pending = s->gerror ^ s->gerrorn;
1313 Cmd cmd;
1314
1315 trace_smmuv3_cmdq_consume(Q_PROD(q), Q_CONS(q),
1316 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
1317
1318 if (FIELD_EX32(pending, GERROR, CMDQ_ERR)) {
1319 break;
1320 }
1321
1322 if (queue_read(q, &cmd) != MEMTX_OK) {
1323 cmd_error = SMMU_CERROR_ABT;
1324 break;
1325 }
1326
1327 type = CMD_TYPE(&cmd);
1328
1329 trace_smmuv3_cmdq_opcode(smmu_cmd_string(type));
1330
1331 qemu_mutex_lock(&s->mutex);
1332 switch (type) {
1333 case SMMU_CMD_SYNC:
1334 if (CMD_SYNC_CS(&cmd) & CMD_SYNC_SIG_IRQ) {
1335 smmuv3_trigger_irq(s, SMMU_IRQ_CMD_SYNC, 0);
1336 }
1337 break;
1338 case SMMU_CMD_PREFETCH_CONFIG:
1339 case SMMU_CMD_PREFETCH_ADDR:
1340 break;
1341 case SMMU_CMD_CFGI_STE:
1342 {
1343 uint32_t sid = CMD_SID(&cmd);
1344 SMMUDevice *sdev = smmu_find_sdev(bs, sid);
1345
1346 if (CMD_SSEC(&cmd)) {
1347 cmd_error = SMMU_CERROR_ILL;
1348 break;
1349 }
1350
1351 if (!sdev) {
1352 break;
1353 }
1354
1355 trace_smmuv3_cmdq_cfgi_ste(sid);
1356 smmuv3_flush_config(sdev);
1357
1358 break;
1359 }
1360 case SMMU_CMD_CFGI_STE_RANGE: /* same as SMMU_CMD_CFGI_ALL */
1361 {
1362 uint32_t sid = CMD_SID(&cmd), mask;
1363 uint8_t range = CMD_STE_RANGE(&cmd);
1364 SMMUSIDRange sid_range;
1365
1366 if (CMD_SSEC(&cmd)) {
1367 cmd_error = SMMU_CERROR_ILL;
1368 break;
1369 }
1370
1371 mask = (1ULL << (range + 1)) - 1;
1372 sid_range.start = sid & ~mask;
1373 sid_range.end = sid_range.start + mask;
1374
1375 trace_smmuv3_cmdq_cfgi_ste_range(sid_range.start, sid_range.end);
1376 g_hash_table_foreach_remove(bs->configs, smmuv3_invalidate_ste,
1377 &sid_range);
1378 break;
1379 }
1380 case SMMU_CMD_CFGI_CD:
1381 case SMMU_CMD_CFGI_CD_ALL:
1382 {
1383 uint32_t sid = CMD_SID(&cmd);
1384 SMMUDevice *sdev = smmu_find_sdev(bs, sid);
1385
1386 if (CMD_SSEC(&cmd)) {
1387 cmd_error = SMMU_CERROR_ILL;
1388 break;
1389 }
1390
1391 if (!sdev) {
1392 break;
1393 }
1394
1395 trace_smmuv3_cmdq_cfgi_cd(sid);
1396 smmuv3_flush_config(sdev);
1397 break;
1398 }
1399 case SMMU_CMD_TLBI_NH_ASID:
1400 {
1401 int asid = CMD_ASID(&cmd);
1402 int vmid = -1;
1403
1404 if (!STAGE1_SUPPORTED(s)) {
1405 cmd_error = SMMU_CERROR_ILL;
1406 break;
1407 }
1408
1409 /*
1410 * VMID is only matched when stage 2 is supported, otherwise set it
1411 * to -1 as the value used for stage-1 only VMIDs.
1412 */
1413 if (STAGE2_SUPPORTED(s)) {
1414 vmid = CMD_VMID(&cmd);
1415 }
1416
1417 trace_smmuv3_cmdq_tlbi_nh_asid(asid);
1418 smmu_inv_notifiers_all(&s->smmu_state);
1419 smmu_iotlb_inv_asid_vmid(bs, asid, vmid);
1420 break;
1421 }
1422 case SMMU_CMD_TLBI_NH_ALL:
1423 {
1424 int vmid = -1;
1425
1426 if (!STAGE1_SUPPORTED(s)) {
1427 cmd_error = SMMU_CERROR_ILL;
1428 break;
1429 }
1430
1431 /*
1432 * If stage-2 is supported, invalidate for this VMID only, otherwise
1433 * invalidate the whole thing.
1434 */
1435 if (STAGE2_SUPPORTED(s)) {
1436 vmid = CMD_VMID(&cmd);
1437 trace_smmuv3_cmdq_tlbi_nh(vmid);
1438 smmu_iotlb_inv_vmid_s1(bs, vmid);
1439 break;
1440 }
1441 QEMU_FALLTHROUGH;
1442 }
1443 case SMMU_CMD_TLBI_NSNH_ALL:
1444 trace_smmuv3_cmdq_tlbi_nsnh();
1445 smmu_inv_notifiers_all(&s->smmu_state);
1446 smmu_iotlb_inv_all(bs);
1447 break;
1448 case SMMU_CMD_TLBI_NH_VAA:
1449 case SMMU_CMD_TLBI_NH_VA:
1450 if (!STAGE1_SUPPORTED(s)) {
1451 cmd_error = SMMU_CERROR_ILL;
1452 break;
1453 }
1454 smmuv3_range_inval(bs, &cmd, SMMU_STAGE_1);
1455 break;
1456 case SMMU_CMD_TLBI_S12_VMALL:
1457 {
1458 int vmid = CMD_VMID(&cmd);
1459
1460 if (!STAGE2_SUPPORTED(s)) {
1461 cmd_error = SMMU_CERROR_ILL;
1462 break;
1463 }
1464
1465 trace_smmuv3_cmdq_tlbi_s12_vmid(vmid);
1466 smmu_inv_notifiers_all(&s->smmu_state);
1467 smmu_iotlb_inv_vmid(bs, vmid);
1468 break;
1469 }
1470 case SMMU_CMD_TLBI_S2_IPA:
1471 if (!STAGE2_SUPPORTED(s)) {
1472 cmd_error = SMMU_CERROR_ILL;
1473 break;
1474 }
1475 /*
1476 * As currently only either s1 or s2 are supported
1477 * we can reuse same function for s2.
1478 */
1479 smmuv3_range_inval(bs, &cmd, SMMU_STAGE_2);
1480 break;
1481 case SMMU_CMD_TLBI_EL3_ALL:
1482 case SMMU_CMD_TLBI_EL3_VA:
1483 case SMMU_CMD_TLBI_EL2_ALL:
1484 case SMMU_CMD_TLBI_EL2_ASID:
1485 case SMMU_CMD_TLBI_EL2_VA:
1486 case SMMU_CMD_TLBI_EL2_VAA:
1487 case SMMU_CMD_ATC_INV:
1488 case SMMU_CMD_PRI_RESP:
1489 case SMMU_CMD_RESUME:
1490 case SMMU_CMD_STALL_TERM:
1491 trace_smmuv3_unhandled_cmd(type);
1492 break;
1493 default:
1494 cmd_error = SMMU_CERROR_ILL;
1495 break;
1496 }
1497 qemu_mutex_unlock(&s->mutex);
1498 if (cmd_error) {
1499 if (cmd_error == SMMU_CERROR_ILL) {
1500 qemu_log_mask(LOG_GUEST_ERROR,
1501 "Illegal command type: %d\n", CMD_TYPE(&cmd));
1502 }
1503 break;
1504 }
1505 /*
1506 * We only increment the cons index after the completion of
1507 * the command. We do that because the SYNC returns immediately
1508 * and does not check the completion of previous commands
1509 */
1510 queue_cons_incr(q);
1511 }
1512
1513 if (cmd_error) {
1514 trace_smmuv3_cmdq_consume_error(smmu_cmd_string(type), cmd_error);
1515 smmu_write_cmdq_err(s, cmd_error);
1516 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_CMDQ_ERR_MASK);
1517 }
1518
1519 trace_smmuv3_cmdq_consume_out(Q_PROD(q), Q_CONS(q),
1520 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
1521
1522 return 0;
1523 }
1524
smmu_writell(SMMUv3State * s,hwaddr offset,uint64_t data,MemTxAttrs attrs)1525 static MemTxResult smmu_writell(SMMUv3State *s, hwaddr offset,
1526 uint64_t data, MemTxAttrs attrs)
1527 {
1528 switch (offset) {
1529 case A_GERROR_IRQ_CFG0:
1530 s->gerror_irq_cfg0 = data;
1531 return MEMTX_OK;
1532 case A_STRTAB_BASE:
1533 s->strtab_base = data;
1534 return MEMTX_OK;
1535 case A_CMDQ_BASE:
1536 s->cmdq.base = data;
1537 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1538 if (s->cmdq.log2size > SMMU_CMDQS) {
1539 s->cmdq.log2size = SMMU_CMDQS;
1540 }
1541 return MEMTX_OK;
1542 case A_EVENTQ_BASE:
1543 s->eventq.base = data;
1544 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1545 if (s->eventq.log2size > SMMU_EVENTQS) {
1546 s->eventq.log2size = SMMU_EVENTQS;
1547 }
1548 return MEMTX_OK;
1549 case A_EVENTQ_IRQ_CFG0:
1550 s->eventq_irq_cfg0 = data;
1551 return MEMTX_OK;
1552 default:
1553 qemu_log_mask(LOG_UNIMP,
1554 "%s Unexpected 64-bit access to 0x%"PRIx64" (WI)\n",
1555 __func__, offset);
1556 return MEMTX_OK;
1557 }
1558 }
1559
smmu_writel(SMMUv3State * s,hwaddr offset,uint64_t data,MemTxAttrs attrs)1560 static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset,
1561 uint64_t data, MemTxAttrs attrs)
1562 {
1563 switch (offset) {
1564 case A_CR0:
1565 s->cr[0] = data;
1566 s->cr0ack = data & ~SMMU_CR0_RESERVED;
1567 /* in case the command queue has been enabled */
1568 smmuv3_cmdq_consume(s);
1569 return MEMTX_OK;
1570 case A_CR1:
1571 s->cr[1] = data;
1572 return MEMTX_OK;
1573 case A_CR2:
1574 s->cr[2] = data;
1575 return MEMTX_OK;
1576 case A_IRQ_CTRL:
1577 s->irq_ctrl = data;
1578 return MEMTX_OK;
1579 case A_GERRORN:
1580 smmuv3_write_gerrorn(s, data);
1581 /*
1582 * By acknowledging the CMDQ_ERR, SW may notify cmds can
1583 * be processed again
1584 */
1585 smmuv3_cmdq_consume(s);
1586 return MEMTX_OK;
1587 case A_GERROR_IRQ_CFG0: /* 64b */
1588 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 0, 32, data);
1589 return MEMTX_OK;
1590 case A_GERROR_IRQ_CFG0 + 4:
1591 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 32, 32, data);
1592 return MEMTX_OK;
1593 case A_GERROR_IRQ_CFG1:
1594 s->gerror_irq_cfg1 = data;
1595 return MEMTX_OK;
1596 case A_GERROR_IRQ_CFG2:
1597 s->gerror_irq_cfg2 = data;
1598 return MEMTX_OK;
1599 case A_GBPA:
1600 /*
1601 * If UPDATE is not set, the write is ignored. This is the only
1602 * permitted behavior in SMMUv3.2 and later.
1603 */
1604 if (data & R_GBPA_UPDATE_MASK) {
1605 /* Ignore update bit as write is synchronous. */
1606 s->gbpa = data & ~R_GBPA_UPDATE_MASK;
1607 }
1608 return MEMTX_OK;
1609 case A_STRTAB_BASE: /* 64b */
1610 s->strtab_base = deposit64(s->strtab_base, 0, 32, data);
1611 return MEMTX_OK;
1612 case A_STRTAB_BASE + 4:
1613 s->strtab_base = deposit64(s->strtab_base, 32, 32, data);
1614 return MEMTX_OK;
1615 case A_STRTAB_BASE_CFG:
1616 s->strtab_base_cfg = data;
1617 if (FIELD_EX32(data, STRTAB_BASE_CFG, FMT) == 1) {
1618 s->sid_split = FIELD_EX32(data, STRTAB_BASE_CFG, SPLIT);
1619 s->features |= SMMU_FEATURE_2LVL_STE;
1620 }
1621 return MEMTX_OK;
1622 case A_CMDQ_BASE: /* 64b */
1623 s->cmdq.base = deposit64(s->cmdq.base, 0, 32, data);
1624 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1625 if (s->cmdq.log2size > SMMU_CMDQS) {
1626 s->cmdq.log2size = SMMU_CMDQS;
1627 }
1628 return MEMTX_OK;
1629 case A_CMDQ_BASE + 4: /* 64b */
1630 s->cmdq.base = deposit64(s->cmdq.base, 32, 32, data);
1631 return MEMTX_OK;
1632 case A_CMDQ_PROD:
1633 s->cmdq.prod = data;
1634 smmuv3_cmdq_consume(s);
1635 return MEMTX_OK;
1636 case A_CMDQ_CONS:
1637 s->cmdq.cons = data;
1638 return MEMTX_OK;
1639 case A_EVENTQ_BASE: /* 64b */
1640 s->eventq.base = deposit64(s->eventq.base, 0, 32, data);
1641 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1642 if (s->eventq.log2size > SMMU_EVENTQS) {
1643 s->eventq.log2size = SMMU_EVENTQS;
1644 }
1645 return MEMTX_OK;
1646 case A_EVENTQ_BASE + 4:
1647 s->eventq.base = deposit64(s->eventq.base, 32, 32, data);
1648 return MEMTX_OK;
1649 case A_EVENTQ_PROD:
1650 s->eventq.prod = data;
1651 return MEMTX_OK;
1652 case A_EVENTQ_CONS:
1653 s->eventq.cons = data;
1654 return MEMTX_OK;
1655 case A_EVENTQ_IRQ_CFG0: /* 64b */
1656 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 0, 32, data);
1657 return MEMTX_OK;
1658 case A_EVENTQ_IRQ_CFG0 + 4:
1659 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 32, 32, data);
1660 return MEMTX_OK;
1661 case A_EVENTQ_IRQ_CFG1:
1662 s->eventq_irq_cfg1 = data;
1663 return MEMTX_OK;
1664 case A_EVENTQ_IRQ_CFG2:
1665 s->eventq_irq_cfg2 = data;
1666 return MEMTX_OK;
1667 default:
1668 qemu_log_mask(LOG_UNIMP,
1669 "%s Unexpected 32-bit access to 0x%"PRIx64" (WI)\n",
1670 __func__, offset);
1671 return MEMTX_OK;
1672 }
1673 }
1674
smmu_write_mmio(void * opaque,hwaddr offset,uint64_t data,unsigned size,MemTxAttrs attrs)1675 static MemTxResult smmu_write_mmio(void *opaque, hwaddr offset, uint64_t data,
1676 unsigned size, MemTxAttrs attrs)
1677 {
1678 SMMUState *sys = opaque;
1679 SMMUv3State *s = ARM_SMMUV3(sys);
1680 MemTxResult r;
1681
1682 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */
1683 offset &= ~0x10000;
1684
1685 switch (size) {
1686 case 8:
1687 r = smmu_writell(s, offset, data, attrs);
1688 break;
1689 case 4:
1690 r = smmu_writel(s, offset, data, attrs);
1691 break;
1692 default:
1693 r = MEMTX_ERROR;
1694 break;
1695 }
1696
1697 trace_smmuv3_write_mmio(offset, data, size, r);
1698 return r;
1699 }
1700
smmu_readll(SMMUv3State * s,hwaddr offset,uint64_t * data,MemTxAttrs attrs)1701 static MemTxResult smmu_readll(SMMUv3State *s, hwaddr offset,
1702 uint64_t *data, MemTxAttrs attrs)
1703 {
1704 switch (offset) {
1705 case A_GERROR_IRQ_CFG0:
1706 *data = s->gerror_irq_cfg0;
1707 return MEMTX_OK;
1708 case A_STRTAB_BASE:
1709 *data = s->strtab_base;
1710 return MEMTX_OK;
1711 case A_CMDQ_BASE:
1712 *data = s->cmdq.base;
1713 return MEMTX_OK;
1714 case A_EVENTQ_BASE:
1715 *data = s->eventq.base;
1716 return MEMTX_OK;
1717 default:
1718 *data = 0;
1719 qemu_log_mask(LOG_UNIMP,
1720 "%s Unexpected 64-bit access to 0x%"PRIx64" (RAZ)\n",
1721 __func__, offset);
1722 return MEMTX_OK;
1723 }
1724 }
1725
smmu_readl(SMMUv3State * s,hwaddr offset,uint64_t * data,MemTxAttrs attrs)1726 static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset,
1727 uint64_t *data, MemTxAttrs attrs)
1728 {
1729 switch (offset) {
1730 case A_IDREGS ... A_IDREGS + 0x2f:
1731 *data = smmuv3_idreg(offset - A_IDREGS);
1732 return MEMTX_OK;
1733 case A_IDR0 ... A_IDR5:
1734 *data = s->idr[(offset - A_IDR0) / 4];
1735 return MEMTX_OK;
1736 case A_IIDR:
1737 *data = s->iidr;
1738 return MEMTX_OK;
1739 case A_AIDR:
1740 *data = s->aidr;
1741 return MEMTX_OK;
1742 case A_CR0:
1743 *data = s->cr[0];
1744 return MEMTX_OK;
1745 case A_CR0ACK:
1746 *data = s->cr0ack;
1747 return MEMTX_OK;
1748 case A_CR1:
1749 *data = s->cr[1];
1750 return MEMTX_OK;
1751 case A_CR2:
1752 *data = s->cr[2];
1753 return MEMTX_OK;
1754 case A_STATUSR:
1755 *data = s->statusr;
1756 return MEMTX_OK;
1757 case A_GBPA:
1758 *data = s->gbpa;
1759 return MEMTX_OK;
1760 case A_IRQ_CTRL:
1761 case A_IRQ_CTRL_ACK:
1762 *data = s->irq_ctrl;
1763 return MEMTX_OK;
1764 case A_GERROR:
1765 *data = s->gerror;
1766 return MEMTX_OK;
1767 case A_GERRORN:
1768 *data = s->gerrorn;
1769 return MEMTX_OK;
1770 case A_GERROR_IRQ_CFG0: /* 64b */
1771 *data = extract64(s->gerror_irq_cfg0, 0, 32);
1772 return MEMTX_OK;
1773 case A_GERROR_IRQ_CFG0 + 4:
1774 *data = extract64(s->gerror_irq_cfg0, 32, 32);
1775 return MEMTX_OK;
1776 case A_GERROR_IRQ_CFG1:
1777 *data = s->gerror_irq_cfg1;
1778 return MEMTX_OK;
1779 case A_GERROR_IRQ_CFG2:
1780 *data = s->gerror_irq_cfg2;
1781 return MEMTX_OK;
1782 case A_STRTAB_BASE: /* 64b */
1783 *data = extract64(s->strtab_base, 0, 32);
1784 return MEMTX_OK;
1785 case A_STRTAB_BASE + 4: /* 64b */
1786 *data = extract64(s->strtab_base, 32, 32);
1787 return MEMTX_OK;
1788 case A_STRTAB_BASE_CFG:
1789 *data = s->strtab_base_cfg;
1790 return MEMTX_OK;
1791 case A_CMDQ_BASE: /* 64b */
1792 *data = extract64(s->cmdq.base, 0, 32);
1793 return MEMTX_OK;
1794 case A_CMDQ_BASE + 4:
1795 *data = extract64(s->cmdq.base, 32, 32);
1796 return MEMTX_OK;
1797 case A_CMDQ_PROD:
1798 *data = s->cmdq.prod;
1799 return MEMTX_OK;
1800 case A_CMDQ_CONS:
1801 *data = s->cmdq.cons;
1802 return MEMTX_OK;
1803 case A_EVENTQ_BASE: /* 64b */
1804 *data = extract64(s->eventq.base, 0, 32);
1805 return MEMTX_OK;
1806 case A_EVENTQ_BASE + 4: /* 64b */
1807 *data = extract64(s->eventq.base, 32, 32);
1808 return MEMTX_OK;
1809 case A_EVENTQ_PROD:
1810 *data = s->eventq.prod;
1811 return MEMTX_OK;
1812 case A_EVENTQ_CONS:
1813 *data = s->eventq.cons;
1814 return MEMTX_OK;
1815 default:
1816 *data = 0;
1817 qemu_log_mask(LOG_UNIMP,
1818 "%s unhandled 32-bit access at 0x%"PRIx64" (RAZ)\n",
1819 __func__, offset);
1820 return MEMTX_OK;
1821 }
1822 }
1823
smmu_read_mmio(void * opaque,hwaddr offset,uint64_t * data,unsigned size,MemTxAttrs attrs)1824 static MemTxResult smmu_read_mmio(void *opaque, hwaddr offset, uint64_t *data,
1825 unsigned size, MemTxAttrs attrs)
1826 {
1827 SMMUState *sys = opaque;
1828 SMMUv3State *s = ARM_SMMUV3(sys);
1829 MemTxResult r;
1830
1831 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */
1832 offset &= ~0x10000;
1833
1834 switch (size) {
1835 case 8:
1836 r = smmu_readll(s, offset, data, attrs);
1837 break;
1838 case 4:
1839 r = smmu_readl(s, offset, data, attrs);
1840 break;
1841 default:
1842 r = MEMTX_ERROR;
1843 break;
1844 }
1845
1846 trace_smmuv3_read_mmio(offset, *data, size, r);
1847 return r;
1848 }
1849
1850 static const MemoryRegionOps smmu_mem_ops = {
1851 .read_with_attrs = smmu_read_mmio,
1852 .write_with_attrs = smmu_write_mmio,
1853 .endianness = DEVICE_LITTLE_ENDIAN,
1854 .valid = {
1855 .min_access_size = 4,
1856 .max_access_size = 8,
1857 },
1858 .impl = {
1859 .min_access_size = 4,
1860 .max_access_size = 8,
1861 },
1862 };
1863
smmu_init_irq(SMMUv3State * s,SysBusDevice * dev)1864 static void smmu_init_irq(SMMUv3State *s, SysBusDevice *dev)
1865 {
1866 int i;
1867
1868 for (i = 0; i < ARRAY_SIZE(s->irq); i++) {
1869 sysbus_init_irq(dev, &s->irq[i]);
1870 }
1871 }
1872
smmu_reset_hold(Object * obj,ResetType type)1873 static void smmu_reset_hold(Object *obj, ResetType type)
1874 {
1875 SMMUv3State *s = ARM_SMMUV3(obj);
1876 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1877
1878 if (c->parent_phases.hold) {
1879 c->parent_phases.hold(obj, type);
1880 }
1881
1882 smmuv3_init_regs(s);
1883 }
1884
smmu_realize(DeviceState * d,Error ** errp)1885 static void smmu_realize(DeviceState *d, Error **errp)
1886 {
1887 SMMUState *sys = ARM_SMMU(d);
1888 SMMUv3State *s = ARM_SMMUV3(sys);
1889 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1890 SysBusDevice *dev = SYS_BUS_DEVICE(d);
1891 Error *local_err = NULL;
1892
1893 c->parent_realize(d, &local_err);
1894 if (local_err) {
1895 error_propagate(errp, local_err);
1896 return;
1897 }
1898
1899 qemu_mutex_init(&s->mutex);
1900
1901 memory_region_init_io(&sys->iomem, OBJECT(s),
1902 &smmu_mem_ops, sys, TYPE_ARM_SMMUV3, 0x20000);
1903
1904 sys->mrtypename = TYPE_SMMUV3_IOMMU_MEMORY_REGION;
1905
1906 sysbus_init_mmio(dev, &sys->iomem);
1907
1908 smmu_init_irq(s, dev);
1909 }
1910
1911 static const VMStateDescription vmstate_smmuv3_queue = {
1912 .name = "smmuv3_queue",
1913 .version_id = 1,
1914 .minimum_version_id = 1,
1915 .fields = (const VMStateField[]) {
1916 VMSTATE_UINT64(base, SMMUQueue),
1917 VMSTATE_UINT32(prod, SMMUQueue),
1918 VMSTATE_UINT32(cons, SMMUQueue),
1919 VMSTATE_UINT8(log2size, SMMUQueue),
1920 VMSTATE_END_OF_LIST(),
1921 },
1922 };
1923
smmuv3_gbpa_needed(void * opaque)1924 static bool smmuv3_gbpa_needed(void *opaque)
1925 {
1926 SMMUv3State *s = opaque;
1927
1928 /* Only migrate GBPA if it has different reset value. */
1929 return s->gbpa != SMMU_GBPA_RESET_VAL;
1930 }
1931
1932 static const VMStateDescription vmstate_gbpa = {
1933 .name = "smmuv3/gbpa",
1934 .version_id = 1,
1935 .minimum_version_id = 1,
1936 .needed = smmuv3_gbpa_needed,
1937 .fields = (const VMStateField[]) {
1938 VMSTATE_UINT32(gbpa, SMMUv3State),
1939 VMSTATE_END_OF_LIST()
1940 }
1941 };
1942
1943 static const VMStateDescription vmstate_smmuv3 = {
1944 .name = "smmuv3",
1945 .version_id = 1,
1946 .minimum_version_id = 1,
1947 .priority = MIG_PRI_IOMMU,
1948 .fields = (const VMStateField[]) {
1949 VMSTATE_UINT32(features, SMMUv3State),
1950 VMSTATE_UINT8(sid_size, SMMUv3State),
1951 VMSTATE_UINT8(sid_split, SMMUv3State),
1952
1953 VMSTATE_UINT32_ARRAY(cr, SMMUv3State, 3),
1954 VMSTATE_UINT32(cr0ack, SMMUv3State),
1955 VMSTATE_UINT32(statusr, SMMUv3State),
1956 VMSTATE_UINT32(irq_ctrl, SMMUv3State),
1957 VMSTATE_UINT32(gerror, SMMUv3State),
1958 VMSTATE_UINT32(gerrorn, SMMUv3State),
1959 VMSTATE_UINT64(gerror_irq_cfg0, SMMUv3State),
1960 VMSTATE_UINT32(gerror_irq_cfg1, SMMUv3State),
1961 VMSTATE_UINT32(gerror_irq_cfg2, SMMUv3State),
1962 VMSTATE_UINT64(strtab_base, SMMUv3State),
1963 VMSTATE_UINT32(strtab_base_cfg, SMMUv3State),
1964 VMSTATE_UINT64(eventq_irq_cfg0, SMMUv3State),
1965 VMSTATE_UINT32(eventq_irq_cfg1, SMMUv3State),
1966 VMSTATE_UINT32(eventq_irq_cfg2, SMMUv3State),
1967
1968 VMSTATE_STRUCT(cmdq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1969 VMSTATE_STRUCT(eventq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1970
1971 VMSTATE_END_OF_LIST(),
1972 },
1973 .subsections = (const VMStateDescription * const []) {
1974 &vmstate_gbpa,
1975 NULL
1976 }
1977 };
1978
1979 static Property smmuv3_properties[] = {
1980 /*
1981 * Stages of translation advertised.
1982 * "1": Stage 1
1983 * "2": Stage 2
1984 * "nested": Both stage 1 and stage 2
1985 * Defaults to stage 1
1986 */
1987 DEFINE_PROP_STRING("stage", SMMUv3State, stage),
1988 DEFINE_PROP_END_OF_LIST()
1989 };
1990
smmuv3_instance_init(Object * obj)1991 static void smmuv3_instance_init(Object *obj)
1992 {
1993 /* Nothing much to do here as of now */
1994 }
1995
smmuv3_class_init(ObjectClass * klass,void * data)1996 static void smmuv3_class_init(ObjectClass *klass, void *data)
1997 {
1998 DeviceClass *dc = DEVICE_CLASS(klass);
1999 ResettableClass *rc = RESETTABLE_CLASS(klass);
2000 SMMUv3Class *c = ARM_SMMUV3_CLASS(klass);
2001
2002 dc->vmsd = &vmstate_smmuv3;
2003 resettable_class_set_parent_phases(rc, NULL, smmu_reset_hold, NULL,
2004 &c->parent_phases);
2005 device_class_set_parent_realize(dc, smmu_realize,
2006 &c->parent_realize);
2007 device_class_set_props(dc, smmuv3_properties);
2008 }
2009
smmuv3_notify_flag_changed(IOMMUMemoryRegion * iommu,IOMMUNotifierFlag old,IOMMUNotifierFlag new,Error ** errp)2010 static int smmuv3_notify_flag_changed(IOMMUMemoryRegion *iommu,
2011 IOMMUNotifierFlag old,
2012 IOMMUNotifierFlag new,
2013 Error **errp)
2014 {
2015 SMMUDevice *sdev = container_of(iommu, SMMUDevice, iommu);
2016 SMMUv3State *s3 = sdev->smmu;
2017 SMMUState *s = &(s3->smmu_state);
2018
2019 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) {
2020 error_setg(errp, "SMMUv3 does not support dev-iotlb yet");
2021 return -EINVAL;
2022 }
2023
2024 if (new & IOMMU_NOTIFIER_MAP) {
2025 error_setg(errp,
2026 "device %02x.%02x.%x requires iommu MAP notifier which is "
2027 "not currently supported", pci_bus_num(sdev->bus),
2028 PCI_SLOT(sdev->devfn), PCI_FUNC(sdev->devfn));
2029 return -EINVAL;
2030 }
2031
2032 if (old == IOMMU_NOTIFIER_NONE) {
2033 trace_smmuv3_notify_flag_add(iommu->parent_obj.name);
2034 QLIST_INSERT_HEAD(&s->devices_with_notifiers, sdev, next);
2035 } else if (new == IOMMU_NOTIFIER_NONE) {
2036 trace_smmuv3_notify_flag_del(iommu->parent_obj.name);
2037 QLIST_REMOVE(sdev, next);
2038 }
2039 return 0;
2040 }
2041
smmuv3_iommu_memory_region_class_init(ObjectClass * klass,void * data)2042 static void smmuv3_iommu_memory_region_class_init(ObjectClass *klass,
2043 void *data)
2044 {
2045 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
2046
2047 imrc->translate = smmuv3_translate;
2048 imrc->notify_flag_changed = smmuv3_notify_flag_changed;
2049 }
2050
2051 static const TypeInfo smmuv3_type_info = {
2052 .name = TYPE_ARM_SMMUV3,
2053 .parent = TYPE_ARM_SMMU,
2054 .instance_size = sizeof(SMMUv3State),
2055 .instance_init = smmuv3_instance_init,
2056 .class_size = sizeof(SMMUv3Class),
2057 .class_init = smmuv3_class_init,
2058 };
2059
2060 static const TypeInfo smmuv3_iommu_memory_region_info = {
2061 .parent = TYPE_IOMMU_MEMORY_REGION,
2062 .name = TYPE_SMMUV3_IOMMU_MEMORY_REGION,
2063 .class_init = smmuv3_iommu_memory_region_class_init,
2064 };
2065
smmuv3_register_types(void)2066 static void smmuv3_register_types(void)
2067 {
2068 type_register(&smmuv3_type_info);
2069 type_register(&smmuv3_iommu_memory_region_info);
2070 }
2071
2072 type_init(smmuv3_register_types)
2073
2074