1085 static uint64_t set_step(CPUARMState *env, uint64_t toaddr,  in set_step()  argument
1091     setsize = MIN(setsize, page_limit(toaddr));  in set_step()
1093         uint64_t mtesize = mte_mops_probe(env, toaddr, setsize, *mtedesc);  in set_step()
1096             mte_check_fail(env, *mtedesc, toaddr, ra);  in set_step()
1105     toaddr = useronly_clean_ptr(toaddr);  in set_step()
1110     mem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, memidx);  in set_step()
1120         cpu_stb_mmuidx_ra(env, toaddr, data, memidx, ra);  in set_step()
1136 static uint64_t set_step_tags(CPUARMState *env, uint64_t toaddr,  in set_step_tags()  argument
1143     setsize = MIN(setsize, page_limit(toaddr));  in set_step_tags()
1145     cleanaddr = useronly_clean_ptr(toaddr);  in set_step_tags()
1164         cpu_st16_mmu(env, toaddr, int128_make128(repldata, repldata), oi16, ra);  in set_step_tags()
1165         mte_mops_set_tags(env, toaddr, 16, *mtedesc);  in set_step_tags()
1173     mte_mops_set_tags(env, toaddr, setsize, *mtedesc);  in set_step_tags()
1177 typedef uint64_t StepFn(CPUARMState *env, uint64_t toaddr,
1258     uint64_t toaddr = env->xregs[rd];  in do_setp()  local
1272         check_setg_alignment(env, toaddr, setsize, memidx, ra);  in do_setp()
1273     } else if (!mte_checks_needed(toaddr, mtedesc)) {  in do_setp()
1277     stagesetsize = MIN(setsize, page_limit(toaddr));  in do_setp()
1279         env->xregs[rd] = toaddr;  in do_setp()
1281         step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra);  in do_setp()
1282         toaddr += step;  in do_setp()
1287     env->xregs[rd] = toaddr + setsize;  in do_setp()
1317     uint64_t toaddr = env->xregs[rd] + env->xregs[rn];  in do_setm()  local
1343         check_setg_alignment(env, toaddr, setsize, memidx, ra);  in do_setm()
1344     } else if (!mte_checks_needed(toaddr, mtedesc)) {  in do_setm()
1351         step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra);  in do_setm()
1352         toaddr += step;  in do_setm()
1380     uint64_t toaddr = env->xregs[rd] + env->xregs[rn];  in do_sete()  local
1408         check_setg_alignment(env, toaddr, setsize, memidx, ra);  in do_sete()
1409     } else if (!mte_checks_needed(toaddr, mtedesc)) {  in do_sete()
1415         step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra);  in do_sete()
1416         toaddr += step;  in do_sete()
1444 static uint64_t copy_step(CPUARMState *env, uint64_t toaddr, uint64_t fromaddr,  in copy_step()  argument
1452     copysize = MIN(copysize, page_limit(toaddr));  in copy_step()
1468         uint64_t mtesize = mte_mops_probe(env, toaddr, copysize, *wdesc);  in copy_step()
1470             mte_check_fail(env, *wdesc, toaddr, ra);  in copy_step()
1477     toaddr = useronly_clean_ptr(toaddr);  in copy_step()
1480     wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx);  in copy_step()
1500             cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra);  in copy_step()
1516 static uint64_t copy_step_rev(CPUARMState *env, uint64_t toaddr,  in copy_step_rev()  argument
1525     copysize = MIN(copysize, page_limit_rev(toaddr));  in copy_step_rev()
1542         uint64_t mtesize = mte_mops_probe_rev(env, toaddr, copysize, *wdesc);  in copy_step_rev()
1544             mte_check_fail(env, *wdesc, toaddr, ra);  in copy_step_rev()
1551     toaddr = useronly_clean_ptr(toaddr);  in copy_step_rev()
1554     wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx);  in copy_step_rev()
1574             cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra);  in copy_step_rev()
1610     uint64_t toaddr = env->xregs[rd];  in do_cpyp()  local
1627         uint64_t ts = extract64(toaddr, 0, 56);  in do_cpyp()
1642     if (!mte_checks_needed(toaddr, wdesc)) {  in do_cpyp()
1647         stagecopysize = MIN(copysize, page_limit(toaddr));  in do_cpyp()
1650             env->xregs[rd] = toaddr;  in do_cpyp()
1653             step = copy_step(env, toaddr, fromaddr, stagecopysize,  in do_cpyp()
1655             toaddr += step;  in do_cpyp()
1661         env->xregs[rd] = toaddr + copysize;  in do_cpyp()
1670         toaddr += copysize - 1;  in do_cpyp()
1672         stagecopysize = MIN(copysize, page_limit_rev(toaddr));  in do_cpyp()
1676             step = copy_step_rev(env, toaddr, fromaddr, stagecopysize,  in do_cpyp()
1680             toaddr -= step;  in do_cpyp()
1721     uint64_t toaddr, fromaddr, copysize, step;  in do_cpym()  local
1737         toaddr = env->xregs[rd] + env->xregs[rn];  in do_cpym()
1743         toaddr = env->xregs[rd] + copysize - 1;  in do_cpym()
1750     if (!mte_checks_needed(toaddr, wdesc)) {  in do_cpym()
1759             step = copy_step(env, toaddr, fromaddr, copysize,  in do_cpym()
1761             toaddr += step;  in do_cpym()
1772             step = copy_step_rev(env, toaddr, fromaddr, copysize,  in do_cpym()
1774             toaddr -= step;  in do_cpym()
1808     uint64_t toaddr, fromaddr, copysize, step;  in do_cpye()  local
1824         toaddr = env->xregs[rd] + env->xregs[rn];  in do_cpye()
1830         toaddr = env->xregs[rd] + copysize - 1;  in do_cpye()
1837     if (!mte_checks_needed(toaddr, wdesc)) {  in do_cpye()
1850             step = copy_step(env, toaddr, fromaddr, copysize,  in do_cpye()
1852             toaddr += step;  in do_cpye()
1859             step = copy_step_rev(env, toaddr, fromaddr, copysize,  in do_cpye()
1861             toaddr -= step;  in do_cpye()