xref: /openbmc/qemu/target/s390x/tcg/mem_helper.c (revision 7a5951f6)
1 /*
2  *  S/390 memory access helper routines
3  *
4  *  Copyright (c) 2009 Ulrich Hecht
5  *  Copyright (c) 2009 Alexander Graf
6  *
7  * This library is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * This library is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19  */
20 
21 #include "qemu/osdep.h"
22 #include "qemu/log.h"
23 #include "cpu.h"
24 #include "s390x-internal.h"
25 #include "tcg_s390x.h"
26 #include "exec/helper-proto.h"
27 #include "exec/exec-all.h"
28 #include "exec/cpu_ldst.h"
29 #include "qemu/int128.h"
30 #include "qemu/atomic128.h"
31 #include "trace.h"
32 
33 #if !defined(CONFIG_USER_ONLY)
34 #include "hw/s390x/storage-keys.h"
35 #include "hw/boards.h"
36 #endif
37 
38 /*****************************************************************************/
39 /* Softmmu support */
40 
41 /* #define DEBUG_HELPER */
42 #ifdef DEBUG_HELPER
43 #define HELPER_LOG(x...) qemu_log(x)
44 #else
45 #define HELPER_LOG(x...)
46 #endif
47 
48 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key)
49 {
50     uint16_t pkm = env->cregs[3] >> 16;
51 
52     if (env->psw.mask & PSW_MASK_PSTATE) {
53         /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */
54         return pkm & (0x8000 >> psw_key);
55     }
56     return true;
57 }
58 
59 static bool is_destructive_overlap(CPUS390XState *env, uint64_t dest,
60                                    uint64_t src, uint32_t len)
61 {
62     if (!len || src == dest) {
63         return false;
64     }
65     /* Take care of wrapping at the end of address space. */
66     if (unlikely(wrap_address(env, src + len - 1) < src)) {
67         return dest > src || dest <= wrap_address(env, src + len - 1);
68     }
69     return dest > src && dest <= src + len - 1;
70 }
71 
72 /* Trigger a SPECIFICATION exception if an address or a length is not
73    naturally aligned.  */
74 static inline void check_alignment(CPUS390XState *env, uint64_t v,
75                                    int wordsize, uintptr_t ra)
76 {
77     if (v % wordsize) {
78         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
79     }
80 }
81 
82 /* Load a value from memory according to its size.  */
83 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr,
84                                            int wordsize, uintptr_t ra)
85 {
86     switch (wordsize) {
87     case 1:
88         return cpu_ldub_data_ra(env, addr, ra);
89     case 2:
90         return cpu_lduw_data_ra(env, addr, ra);
91     default:
92         abort();
93     }
94 }
95 
96 /* Store a to memory according to its size.  */
97 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr,
98                                       uint64_t value, int wordsize,
99                                       uintptr_t ra)
100 {
101     switch (wordsize) {
102     case 1:
103         cpu_stb_data_ra(env, addr, value, ra);
104         break;
105     case 2:
106         cpu_stw_data_ra(env, addr, value, ra);
107         break;
108     default:
109         abort();
110     }
111 }
112 
113 /* An access covers at most 4096 bytes and therefore at most two pages. */
114 typedef struct S390Access {
115     target_ulong vaddr1;
116     target_ulong vaddr2;
117     char *haddr1;
118     char *haddr2;
119     uint16_t size1;
120     uint16_t size2;
121     /*
122      * If we can't access the host page directly, we'll have to do I/O access
123      * via ld/st helpers. These are internal details, so we store the
124      * mmu idx to do the access here instead of passing it around in the
125      * helpers. Maybe, one day we can get rid of ld/st access - once we can
126      * handle TLB_NOTDIRTY differently. We don't expect these special accesses
127      * to trigger exceptions - only if we would have TLB_NOTDIRTY on LAP
128      * pages, we might trigger a new MMU translation - very unlikely that
129      * the mapping changes in between and we would trigger a fault.
130      */
131     int mmu_idx;
132 } S390Access;
133 
134 /*
135  * With nonfault=1, return the PGM_ exception that would have been injected
136  * into the guest; return 0 if no exception was detected.
137  *
138  * For !CONFIG_USER_ONLY, the TEC is stored stored to env->tlb_fill_tec.
139  * For CONFIG_USER_ONLY, the faulting address is stored to env->__excp_addr.
140  */
141 static int s390_probe_access(CPUArchState *env, target_ulong addr, int size,
142                              MMUAccessType access_type, int mmu_idx,
143                              bool nonfault, void **phost, uintptr_t ra)
144 {
145 #if defined(CONFIG_USER_ONLY)
146     return probe_access_flags(env, addr, access_type, mmu_idx,
147                               nonfault, phost, ra);
148 #else
149     int flags;
150 
151     env->tlb_fill_exc = 0;
152     flags = probe_access_flags(env, addr, access_type, mmu_idx, nonfault, phost,
153                                ra);
154     if (env->tlb_fill_exc) {
155         return env->tlb_fill_exc;
156     }
157 
158     if (unlikely(flags & TLB_WATCHPOINT)) {
159         /* S390 does not presently use transaction attributes. */
160         cpu_check_watchpoint(env_cpu(env), addr, size,
161                              MEMTXATTRS_UNSPECIFIED,
162                              (access_type == MMU_DATA_STORE
163                               ? BP_MEM_WRITE : BP_MEM_READ), ra);
164     }
165     return 0;
166 #endif
167 }
168 
169 static int access_prepare_nf(S390Access *access, CPUS390XState *env,
170                              bool nonfault, vaddr vaddr1, int size,
171                              MMUAccessType access_type,
172                              int mmu_idx, uintptr_t ra)
173 {
174     void *haddr1, *haddr2 = NULL;
175     int size1, size2, exc;
176     vaddr vaddr2 = 0;
177 
178     assert(size > 0 && size <= 4096);
179 
180     size1 = MIN(size, -(vaddr1 | TARGET_PAGE_MASK)),
181     size2 = size - size1;
182 
183     exc = s390_probe_access(env, vaddr1, size1, access_type, mmu_idx, nonfault,
184                             &haddr1, ra);
185     if (exc) {
186         return exc;
187     }
188     if (unlikely(size2)) {
189         /* The access crosses page boundaries. */
190         vaddr2 = wrap_address(env, vaddr1 + size1);
191         exc = s390_probe_access(env, vaddr2, size2, access_type, mmu_idx,
192                                 nonfault, &haddr2, ra);
193         if (exc) {
194             return exc;
195         }
196     }
197 
198     *access = (S390Access) {
199         .vaddr1 = vaddr1,
200         .vaddr2 = vaddr2,
201         .haddr1 = haddr1,
202         .haddr2 = haddr2,
203         .size1 = size1,
204         .size2 = size2,
205         .mmu_idx = mmu_idx
206     };
207     return 0;
208 }
209 
210 static S390Access access_prepare(CPUS390XState *env, vaddr vaddr, int size,
211                                  MMUAccessType access_type, int mmu_idx,
212                                  uintptr_t ra)
213 {
214     S390Access ret;
215     int exc = access_prepare_nf(&ret, env, false, vaddr, size,
216                                 access_type, mmu_idx, ra);
217     assert(!exc);
218     return ret;
219 }
220 
221 /* Helper to handle memset on a single page. */
222 static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr,
223                              uint8_t byte, uint16_t size, int mmu_idx,
224                              uintptr_t ra)
225 {
226 #ifdef CONFIG_USER_ONLY
227     g_assert(haddr);
228     memset(haddr, byte, size);
229 #else
230     MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
231     int i;
232 
233     if (likely(haddr)) {
234         memset(haddr, byte, size);
235     } else {
236         /*
237          * Do a single access and test if we can then get access to the
238          * page. This is especially relevant to speed up TLB_NOTDIRTY.
239          */
240         g_assert(size > 0);
241         cpu_stb_mmu(env, vaddr, byte, oi, ra);
242         haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
243         if (likely(haddr)) {
244             memset(haddr + 1, byte, size - 1);
245         } else {
246             for (i = 1; i < size; i++) {
247                 cpu_stb_mmu(env, vaddr + i, byte, oi, ra);
248             }
249         }
250     }
251 #endif
252 }
253 
254 static void access_memset(CPUS390XState *env, S390Access *desta,
255                           uint8_t byte, uintptr_t ra)
256 {
257 
258     do_access_memset(env, desta->vaddr1, desta->haddr1, byte, desta->size1,
259                      desta->mmu_idx, ra);
260     if (likely(!desta->size2)) {
261         return;
262     }
263     do_access_memset(env, desta->vaddr2, desta->haddr2, byte, desta->size2,
264                      desta->mmu_idx, ra);
265 }
266 
267 static uint8_t do_access_get_byte(CPUS390XState *env, vaddr vaddr, char **haddr,
268                                   int offset, int mmu_idx, uintptr_t ra)
269 {
270 #ifdef CONFIG_USER_ONLY
271     return ldub_p(*haddr + offset);
272 #else
273     MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
274     uint8_t byte;
275 
276     if (likely(*haddr)) {
277         return ldub_p(*haddr + offset);
278     }
279     /*
280      * Do a single access and test if we can then get access to the
281      * page. This is especially relevant to speed up TLB_NOTDIRTY.
282      */
283     byte = cpu_ldb_mmu(env, vaddr + offset, oi, ra);
284     *haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_LOAD, mmu_idx);
285     return byte;
286 #endif
287 }
288 
289 static uint8_t access_get_byte(CPUS390XState *env, S390Access *access,
290                                int offset, uintptr_t ra)
291 {
292     if (offset < access->size1) {
293         return do_access_get_byte(env, access->vaddr1, &access->haddr1,
294                                   offset, access->mmu_idx, ra);
295     }
296     return do_access_get_byte(env, access->vaddr2, &access->haddr2,
297                               offset - access->size1, access->mmu_idx, ra);
298 }
299 
300 static void do_access_set_byte(CPUS390XState *env, vaddr vaddr, char **haddr,
301                                int offset, uint8_t byte, int mmu_idx,
302                                uintptr_t ra)
303 {
304 #ifdef CONFIG_USER_ONLY
305     stb_p(*haddr + offset, byte);
306 #else
307     MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
308 
309     if (likely(*haddr)) {
310         stb_p(*haddr + offset, byte);
311         return;
312     }
313     /*
314      * Do a single access and test if we can then get access to the
315      * page. This is especially relevant to speed up TLB_NOTDIRTY.
316      */
317     cpu_stb_mmu(env, vaddr + offset, byte, oi, ra);
318     *haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
319 #endif
320 }
321 
322 static void access_set_byte(CPUS390XState *env, S390Access *access,
323                             int offset, uint8_t byte, uintptr_t ra)
324 {
325     if (offset < access->size1) {
326         do_access_set_byte(env, access->vaddr1, &access->haddr1, offset, byte,
327                            access->mmu_idx, ra);
328     } else {
329         do_access_set_byte(env, access->vaddr2, &access->haddr2,
330                            offset - access->size1, byte, access->mmu_idx, ra);
331     }
332 }
333 
334 /*
335  * Move data with the same semantics as memmove() in case ranges don't overlap
336  * or src > dest. Undefined behavior on destructive overlaps.
337  */
338 static void access_memmove(CPUS390XState *env, S390Access *desta,
339                            S390Access *srca, uintptr_t ra)
340 {
341     int diff;
342 
343     g_assert(desta->size1 + desta->size2 == srca->size1 + srca->size2);
344 
345     /* Fallback to slow access in case we don't have access to all host pages */
346     if (unlikely(!desta->haddr1 || (desta->size2 && !desta->haddr2) ||
347                  !srca->haddr1 || (srca->size2 && !srca->haddr2))) {
348         int i;
349 
350         for (i = 0; i < desta->size1 + desta->size2; i++) {
351             uint8_t byte = access_get_byte(env, srca, i, ra);
352 
353             access_set_byte(env, desta, i, byte, ra);
354         }
355         return;
356     }
357 
358     if (srca->size1 == desta->size1) {
359         memmove(desta->haddr1, srca->haddr1, srca->size1);
360         if (unlikely(srca->size2)) {
361             memmove(desta->haddr2, srca->haddr2, srca->size2);
362         }
363     } else if (srca->size1 < desta->size1) {
364         diff = desta->size1 - srca->size1;
365         memmove(desta->haddr1, srca->haddr1, srca->size1);
366         memmove(desta->haddr1 + srca->size1, srca->haddr2, diff);
367         if (likely(desta->size2)) {
368             memmove(desta->haddr2, srca->haddr2 + diff, desta->size2);
369         }
370     } else {
371         diff = srca->size1 - desta->size1;
372         memmove(desta->haddr1, srca->haddr1, desta->size1);
373         memmove(desta->haddr2, srca->haddr1 + desta->size1, diff);
374         if (likely(srca->size2)) {
375             memmove(desta->haddr2 + diff, srca->haddr2, srca->size2);
376         }
377     }
378 }
379 
380 static int mmu_idx_from_as(uint8_t as)
381 {
382     switch (as) {
383     case AS_PRIMARY:
384         return MMU_PRIMARY_IDX;
385     case AS_SECONDARY:
386         return MMU_SECONDARY_IDX;
387     case AS_HOME:
388         return MMU_HOME_IDX;
389     default:
390         /* FIXME AS_ACCREG */
391         g_assert_not_reached();
392     }
393 }
394 
395 /* and on array */
396 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest,
397                              uint64_t src, uintptr_t ra)
398 {
399     const int mmu_idx = cpu_mmu_index(env, false);
400     S390Access srca1, srca2, desta;
401     uint32_t i;
402     uint8_t c = 0;
403 
404     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
405                __func__, l, dest, src);
406 
407     /* NC always processes one more byte than specified - maximum is 256 */
408     l++;
409 
410     srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
411     srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
412     desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
413     for (i = 0; i < l; i++) {
414         const uint8_t x = access_get_byte(env, &srca1, i, ra) &
415                           access_get_byte(env, &srca2, i, ra);
416 
417         c |= x;
418         access_set_byte(env, &desta, i, x, ra);
419     }
420     return c != 0;
421 }
422 
423 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
424                     uint64_t src)
425 {
426     return do_helper_nc(env, l, dest, src, GETPC());
427 }
428 
429 /* xor on array */
430 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest,
431                              uint64_t src, uintptr_t ra)
432 {
433     const int mmu_idx = cpu_mmu_index(env, false);
434     S390Access srca1, srca2, desta;
435     uint32_t i;
436     uint8_t c = 0;
437 
438     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
439                __func__, l, dest, src);
440 
441     /* XC always processes one more byte than specified - maximum is 256 */
442     l++;
443 
444     srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
445     srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
446     desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
447 
448     /* xor with itself is the same as memset(0) */
449     if (src == dest) {
450         access_memset(env, &desta, 0, ra);
451         return 0;
452     }
453 
454     for (i = 0; i < l; i++) {
455         const uint8_t x = access_get_byte(env, &srca1, i, ra) ^
456                           access_get_byte(env, &srca2, i, ra);
457 
458         c |= x;
459         access_set_byte(env, &desta, i, x, ra);
460     }
461     return c != 0;
462 }
463 
464 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
465                     uint64_t src)
466 {
467     return do_helper_xc(env, l, dest, src, GETPC());
468 }
469 
470 /* or on array */
471 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest,
472                              uint64_t src, uintptr_t ra)
473 {
474     const int mmu_idx = cpu_mmu_index(env, false);
475     S390Access srca1, srca2, desta;
476     uint32_t i;
477     uint8_t c = 0;
478 
479     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
480                __func__, l, dest, src);
481 
482     /* OC always processes one more byte than specified - maximum is 256 */
483     l++;
484 
485     srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
486     srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
487     desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
488     for (i = 0; i < l; i++) {
489         const uint8_t x = access_get_byte(env, &srca1, i, ra) |
490                           access_get_byte(env, &srca2, i, ra);
491 
492         c |= x;
493         access_set_byte(env, &desta, i, x, ra);
494     }
495     return c != 0;
496 }
497 
498 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
499                     uint64_t src)
500 {
501     return do_helper_oc(env, l, dest, src, GETPC());
502 }
503 
504 /* memmove */
505 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest,
506                               uint64_t src, uintptr_t ra)
507 {
508     const int mmu_idx = cpu_mmu_index(env, false);
509     S390Access srca, desta;
510     uint32_t i;
511 
512     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
513                __func__, l, dest, src);
514 
515     /* MVC always copies one more byte than specified - maximum is 256 */
516     l++;
517 
518     srca = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
519     desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
520 
521     /*
522      * "When the operands overlap, the result is obtained as if the operands
523      * were processed one byte at a time". Only non-destructive overlaps
524      * behave like memmove().
525      */
526     if (dest == src + 1) {
527         access_memset(env, &desta, access_get_byte(env, &srca, 0, ra), ra);
528     } else if (!is_destructive_overlap(env, dest, src, l)) {
529         access_memmove(env, &desta, &srca, ra);
530     } else {
531         for (i = 0; i < l; i++) {
532             uint8_t byte = access_get_byte(env, &srca, i, ra);
533 
534             access_set_byte(env, &desta, i, byte, ra);
535         }
536     }
537 
538     return env->cc_op;
539 }
540 
541 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
542 {
543     do_helper_mvc(env, l, dest, src, GETPC());
544 }
545 
546 /* move right to left */
547 void HELPER(mvcrl)(CPUS390XState *env, uint64_t l, uint64_t dest, uint64_t src)
548 {
549     const int mmu_idx = cpu_mmu_index(env, false);
550     const uint64_t ra = GETPC();
551     S390Access srca, desta;
552     int32_t i;
553 
554     /* MVCRL always copies one more byte than specified - maximum is 256 */
555     l++;
556 
557     srca = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
558     desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
559 
560     for (i = l - 1; i >= 0; i--) {
561         uint8_t byte = access_get_byte(env, &srca, i, ra);
562         access_set_byte(env, &desta, i, byte, ra);
563     }
564 }
565 
566 /* move inverse  */
567 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
568 {
569     const int mmu_idx = cpu_mmu_index(env, false);
570     S390Access srca, desta;
571     uintptr_t ra = GETPC();
572     int i;
573 
574     /* MVCIN always copies one more byte than specified - maximum is 256 */
575     l++;
576 
577     src = wrap_address(env, src - l + 1);
578     srca = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
579     desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
580     for (i = 0; i < l; i++) {
581         const uint8_t x = access_get_byte(env, &srca, l - i - 1, ra);
582 
583         access_set_byte(env, &desta, i, x, ra);
584     }
585 }
586 
587 /* move numerics  */
588 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
589 {
590     const int mmu_idx = cpu_mmu_index(env, false);
591     S390Access srca1, srca2, desta;
592     uintptr_t ra = GETPC();
593     int i;
594 
595     /* MVN always copies one more byte than specified - maximum is 256 */
596     l++;
597 
598     srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
599     srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
600     desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
601     for (i = 0; i < l; i++) {
602         const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0x0f) |
603                           (access_get_byte(env, &srca2, i, ra) & 0xf0);
604 
605         access_set_byte(env, &desta, i, x, ra);
606     }
607 }
608 
609 /* move with offset  */
610 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
611 {
612     const int mmu_idx = cpu_mmu_index(env, false);
613     /* MVO always processes one more byte than specified - maximum is 16 */
614     const int len_dest = (l >> 4) + 1;
615     const int len_src = (l & 0xf) + 1;
616     uintptr_t ra = GETPC();
617     uint8_t byte_dest, byte_src;
618     S390Access srca, desta;
619     int i, j;
620 
621     srca = access_prepare(env, src, len_src, MMU_DATA_LOAD, mmu_idx, ra);
622     desta = access_prepare(env, dest, len_dest, MMU_DATA_STORE, mmu_idx, ra);
623 
624     /* Handle rightmost byte */
625     byte_dest = cpu_ldub_data_ra(env, dest + len_dest - 1, ra);
626     byte_src = access_get_byte(env, &srca, len_src - 1, ra);
627     byte_dest = (byte_dest & 0x0f) | (byte_src << 4);
628     access_set_byte(env, &desta, len_dest - 1, byte_dest, ra);
629 
630     /* Process remaining bytes from right to left */
631     for (i = len_dest - 2, j = len_src - 2; i >= 0; i--, j--) {
632         byte_dest = byte_src >> 4;
633         if (j >= 0) {
634             byte_src = access_get_byte(env, &srca, j, ra);
635         } else {
636             byte_src = 0;
637         }
638         byte_dest |= byte_src << 4;
639         access_set_byte(env, &desta, i, byte_dest, ra);
640     }
641 }
642 
643 /* move zones  */
644 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
645 {
646     const int mmu_idx = cpu_mmu_index(env, false);
647     S390Access srca1, srca2, desta;
648     uintptr_t ra = GETPC();
649     int i;
650 
651     /* MVZ always copies one more byte than specified - maximum is 256 */
652     l++;
653 
654     srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
655     srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
656     desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
657     for (i = 0; i < l; i++) {
658         const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0xf0) |
659                           (access_get_byte(env, &srca2, i, ra) & 0x0f);
660 
661         access_set_byte(env, &desta, i, x, ra);
662     }
663 }
664 
665 /* compare unsigned byte arrays */
666 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1,
667                               uint64_t s2, uintptr_t ra)
668 {
669     uint32_t i;
670     uint32_t cc = 0;
671 
672     HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
673                __func__, l, s1, s2);
674 
675     for (i = 0; i <= l; i++) {
676         uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra);
677         uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra);
678         HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
679         if (x < y) {
680             cc = 1;
681             break;
682         } else if (x > y) {
683             cc = 2;
684             break;
685         }
686     }
687 
688     HELPER_LOG("\n");
689     return cc;
690 }
691 
692 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
693 {
694     return do_helper_clc(env, l, s1, s2, GETPC());
695 }
696 
697 /* compare logical under mask */
698 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
699                      uint64_t addr)
700 {
701     uintptr_t ra = GETPC();
702     uint32_t cc = 0;
703 
704     HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
705                mask, addr);
706 
707     while (mask) {
708         if (mask & 8) {
709             uint8_t d = cpu_ldub_data_ra(env, addr, ra);
710             uint8_t r = extract32(r1, 24, 8);
711             HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
712                        addr);
713             if (r < d) {
714                 cc = 1;
715                 break;
716             } else if (r > d) {
717                 cc = 2;
718                 break;
719             }
720             addr++;
721         }
722         mask = (mask << 1) & 0xf;
723         r1 <<= 8;
724     }
725 
726     HELPER_LOG("\n");
727     return cc;
728 }
729 
730 static inline uint64_t get_address(CPUS390XState *env, int reg)
731 {
732     return wrap_address(env, env->regs[reg]);
733 }
734 
735 /*
736  * Store the address to the given register, zeroing out unused leftmost
737  * bits in bit positions 32-63 (24-bit and 31-bit mode only).
738  */
739 static inline void set_address_zero(CPUS390XState *env, int reg,
740                                     uint64_t address)
741 {
742     if (env->psw.mask & PSW_MASK_64) {
743         env->regs[reg] = address;
744     } else {
745         if (!(env->psw.mask & PSW_MASK_32)) {
746             address &= 0x00ffffff;
747         } else {
748             address &= 0x7fffffff;
749         }
750         env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
751     }
752 }
753 
754 static inline void set_address(CPUS390XState *env, int reg, uint64_t address)
755 {
756     if (env->psw.mask & PSW_MASK_64) {
757         /* 64-Bit mode */
758         env->regs[reg] = address;
759     } else {
760         if (!(env->psw.mask & PSW_MASK_32)) {
761             /* 24-Bit mode. According to the PoO it is implementation
762             dependent if bits 32-39 remain unchanged or are set to
763             zeros.  Choose the former so that the function can also be
764             used for TRT.  */
765             env->regs[reg] = deposit64(env->regs[reg], 0, 24, address);
766         } else {
767             /* 31-Bit mode. According to the PoO it is implementation
768             dependent if bit 32 remains unchanged or is set to zero.
769             Choose the latter so that the function can also be used for
770             TRT.  */
771             address &= 0x7fffffff;
772             env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
773         }
774     }
775 }
776 
777 static inline uint64_t wrap_length32(CPUS390XState *env, uint64_t length)
778 {
779     if (!(env->psw.mask & PSW_MASK_64)) {
780         return (uint32_t)length;
781     }
782     return length;
783 }
784 
785 static inline uint64_t wrap_length31(CPUS390XState *env, uint64_t length)
786 {
787     if (!(env->psw.mask & PSW_MASK_64)) {
788         /* 24-Bit and 31-Bit mode */
789         length &= 0x7fffffff;
790     }
791     return length;
792 }
793 
794 static inline uint64_t get_length(CPUS390XState *env, int reg)
795 {
796     return wrap_length31(env, env->regs[reg]);
797 }
798 
799 static inline void set_length(CPUS390XState *env, int reg, uint64_t length)
800 {
801     if (env->psw.mask & PSW_MASK_64) {
802         /* 64-Bit mode */
803         env->regs[reg] = length;
804     } else {
805         /* 24-Bit and 31-Bit mode */
806         env->regs[reg] = deposit64(env->regs[reg], 0, 32, length);
807     }
808 }
809 
810 /* search string (c is byte to search, r2 is string, r1 end of string) */
811 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
812 {
813     uintptr_t ra = GETPC();
814     uint64_t end, str;
815     uint32_t len;
816     uint8_t v, c = env->regs[0];
817 
818     /* Bits 32-55 must contain all 0.  */
819     if (env->regs[0] & 0xffffff00u) {
820         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
821     }
822 
823     str = get_address(env, r2);
824     end = get_address(env, r1);
825 
826     /* Lest we fail to service interrupts in a timely manner, limit the
827        amount of work we're willing to do.  For now, let's cap at 8k.  */
828     for (len = 0; len < 0x2000; ++len) {
829         if (str + len == end) {
830             /* Character not found.  R1 & R2 are unmodified.  */
831             env->cc_op = 2;
832             return;
833         }
834         v = cpu_ldub_data_ra(env, str + len, ra);
835         if (v == c) {
836             /* Character found.  Set R1 to the location; R2 is unmodified.  */
837             env->cc_op = 1;
838             set_address(env, r1, str + len);
839             return;
840         }
841     }
842 
843     /* CPU-determined bytes processed.  Advance R2 to next byte to process.  */
844     env->cc_op = 3;
845     set_address(env, r2, str + len);
846 }
847 
848 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2)
849 {
850     uintptr_t ra = GETPC();
851     uint32_t len;
852     uint16_t v, c = env->regs[0];
853     uint64_t end, str, adj_end;
854 
855     /* Bits 32-47 of R0 must be zero.  */
856     if (env->regs[0] & 0xffff0000u) {
857         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
858     }
859 
860     str = get_address(env, r2);
861     end = get_address(env, r1);
862 
863     /* If the LSB of the two addresses differ, use one extra byte.  */
864     adj_end = end + ((str ^ end) & 1);
865 
866     /* Lest we fail to service interrupts in a timely manner, limit the
867        amount of work we're willing to do.  For now, let's cap at 8k.  */
868     for (len = 0; len < 0x2000; len += 2) {
869         if (str + len == adj_end) {
870             /* End of input found.  */
871             env->cc_op = 2;
872             return;
873         }
874         v = cpu_lduw_data_ra(env, str + len, ra);
875         if (v == c) {
876             /* Character found.  Set R1 to the location; R2 is unmodified.  */
877             env->cc_op = 1;
878             set_address(env, r1, str + len);
879             return;
880         }
881     }
882 
883     /* CPU-determined bytes processed.  Advance R2 to next byte to process.  */
884     env->cc_op = 3;
885     set_address(env, r2, str + len);
886 }
887 
888 /* unsigned string compare (c is string terminator) */
889 uint64_t HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
890 {
891     uintptr_t ra = GETPC();
892     uint32_t len;
893 
894     c = c & 0xff;
895     s1 = wrap_address(env, s1);
896     s2 = wrap_address(env, s2);
897 
898     /* Lest we fail to service interrupts in a timely manner, limit the
899        amount of work we're willing to do.  For now, let's cap at 8k.  */
900     for (len = 0; len < 0x2000; ++len) {
901         uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra);
902         uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra);
903         if (v1 == v2) {
904             if (v1 == c) {
905                 /* Equal.  CC=0, and don't advance the registers.  */
906                 env->cc_op = 0;
907                 env->retxl = s2;
908                 return s1;
909             }
910         } else {
911             /* Unequal.  CC={1,2}, and advance the registers.  Note that
912                the terminator need not be zero, but the string that contains
913                the terminator is by definition "low".  */
914             env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
915             env->retxl = s2 + len;
916             return s1 + len;
917         }
918     }
919 
920     /* CPU-determined bytes equal; advance the registers.  */
921     env->cc_op = 3;
922     env->retxl = s2 + len;
923     return s1 + len;
924 }
925 
926 /* move page */
927 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint32_t r1, uint32_t r2)
928 {
929     const uint64_t src = get_address(env, r2) & TARGET_PAGE_MASK;
930     const uint64_t dst = get_address(env, r1) & TARGET_PAGE_MASK;
931     const int mmu_idx = cpu_mmu_index(env, false);
932     const bool f = extract64(r0, 11, 1);
933     const bool s = extract64(r0, 10, 1);
934     const bool cco = extract64(r0, 8, 1);
935     uintptr_t ra = GETPC();
936     S390Access srca, desta;
937     int exc;
938 
939     if ((f && s) || extract64(r0, 12, 4)) {
940         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC());
941     }
942 
943     /*
944      * We always manually handle exceptions such that we can properly store
945      * r1/r2 to the lowcore on page-translation exceptions.
946      *
947      * TODO: Access key handling
948      */
949     exc = access_prepare_nf(&srca, env, true, src, TARGET_PAGE_SIZE,
950                             MMU_DATA_LOAD, mmu_idx, ra);
951     if (exc) {
952         if (cco) {
953             return 2;
954         }
955         goto inject_exc;
956     }
957     exc = access_prepare_nf(&desta, env, true, dst, TARGET_PAGE_SIZE,
958                             MMU_DATA_STORE, mmu_idx, ra);
959     if (exc) {
960         if (cco && exc != PGM_PROTECTION) {
961             return 1;
962         }
963         goto inject_exc;
964     }
965     access_memmove(env, &desta, &srca, ra);
966     return 0; /* data moved */
967 inject_exc:
968 #if !defined(CONFIG_USER_ONLY)
969     if (exc != PGM_ADDRESSING) {
970         stq_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, trans_exc_code),
971                  env->tlb_fill_tec);
972     }
973     if (exc == PGM_PAGE_TRANS) {
974         stb_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, op_access_id),
975                  r1 << 4 | r2);
976     }
977 #endif
978     tcg_s390_program_interrupt(env, exc, ra);
979 }
980 
981 /* string copy */
982 uint32_t HELPER(mvst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
983 {
984     const int mmu_idx = cpu_mmu_index(env, false);
985     const uint64_t d = get_address(env, r1);
986     const uint64_t s = get_address(env, r2);
987     const uint8_t c = env->regs[0];
988     const int len = MIN(-(d | TARGET_PAGE_MASK), -(s | TARGET_PAGE_MASK));
989     S390Access srca, desta;
990     uintptr_t ra = GETPC();
991     int i;
992 
993     if (env->regs[0] & 0xffffff00ull) {
994         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
995     }
996 
997     /*
998      * Our access should not exceed single pages, as we must not report access
999      * exceptions exceeding the actually copied range (which we don't know at
1000      * this point). We might over-indicate watchpoints within the pages
1001      * (if we ever care, we have to limit processing to a single byte).
1002      */
1003     srca = access_prepare(env, s, len, MMU_DATA_LOAD, mmu_idx, ra);
1004     desta = access_prepare(env, d, len, MMU_DATA_STORE, mmu_idx, ra);
1005     for (i = 0; i < len; i++) {
1006         const uint8_t v = access_get_byte(env, &srca, i, ra);
1007 
1008         access_set_byte(env, &desta, i, v, ra);
1009         if (v == c) {
1010             set_address_zero(env, r1, d + i);
1011             return 1;
1012         }
1013     }
1014     set_address_zero(env, r1, d + len);
1015     set_address_zero(env, r2, s + len);
1016     return 3;
1017 }
1018 
1019 /* load access registers r1 to r3 from memory at a2 */
1020 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1021 {
1022     uintptr_t ra = GETPC();
1023     int i;
1024 
1025     if (a2 & 0x3) {
1026         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1027     }
1028 
1029     for (i = r1;; i = (i + 1) % 16) {
1030         env->aregs[i] = cpu_ldl_data_ra(env, a2, ra);
1031         a2 += 4;
1032 
1033         if (i == r3) {
1034             break;
1035         }
1036     }
1037 }
1038 
1039 /* store access registers r1 to r3 in memory at a2 */
1040 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1041 {
1042     uintptr_t ra = GETPC();
1043     int i;
1044 
1045     if (a2 & 0x3) {
1046         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1047     }
1048 
1049     for (i = r1;; i = (i + 1) % 16) {
1050         cpu_stl_data_ra(env, a2, env->aregs[i], ra);
1051         a2 += 4;
1052 
1053         if (i == r3) {
1054             break;
1055         }
1056     }
1057 }
1058 
1059 /* move long helper */
1060 static inline uint32_t do_mvcl(CPUS390XState *env,
1061                                uint64_t *dest, uint64_t *destlen,
1062                                uint64_t *src, uint64_t *srclen,
1063                                uint16_t pad, int wordsize, uintptr_t ra)
1064 {
1065     const int mmu_idx = cpu_mmu_index(env, false);
1066     int len = MIN(*destlen, -(*dest | TARGET_PAGE_MASK));
1067     S390Access srca, desta;
1068     int i, cc;
1069 
1070     if (*destlen == *srclen) {
1071         cc = 0;
1072     } else if (*destlen < *srclen) {
1073         cc = 1;
1074     } else {
1075         cc = 2;
1076     }
1077 
1078     if (!*destlen) {
1079         return cc;
1080     }
1081 
1082     /*
1083      * Only perform one type of type of operation (move/pad) at a time.
1084      * Stay within single pages.
1085      */
1086     if (*srclen) {
1087         /* Copy the src array */
1088         len = MIN(MIN(*srclen, -(*src | TARGET_PAGE_MASK)), len);
1089         *destlen -= len;
1090         *srclen -= len;
1091         srca = access_prepare(env, *src, len, MMU_DATA_LOAD, mmu_idx, ra);
1092         desta = access_prepare(env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1093         access_memmove(env, &desta, &srca, ra);
1094         *src = wrap_address(env, *src + len);
1095         *dest = wrap_address(env, *dest + len);
1096     } else if (wordsize == 1) {
1097         /* Pad the remaining area */
1098         *destlen -= len;
1099         desta = access_prepare(env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1100         access_memset(env, &desta, pad, ra);
1101         *dest = wrap_address(env, *dest + len);
1102     } else {
1103         desta = access_prepare(env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1104 
1105         /* The remaining length selects the padding byte. */
1106         for (i = 0; i < len; (*destlen)--, i++) {
1107             if (*destlen & 1) {
1108                 access_set_byte(env, &desta, i, pad, ra);
1109             } else {
1110                 access_set_byte(env, &desta, i, pad >> 8, ra);
1111             }
1112         }
1113         *dest = wrap_address(env, *dest + len);
1114     }
1115 
1116     return *destlen ? 3 : cc;
1117 }
1118 
1119 /* move long */
1120 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
1121 {
1122     const int mmu_idx = cpu_mmu_index(env, false);
1123     uintptr_t ra = GETPC();
1124     uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
1125     uint64_t dest = get_address(env, r1);
1126     uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
1127     uint64_t src = get_address(env, r2);
1128     uint8_t pad = env->regs[r2 + 1] >> 24;
1129     CPUState *cs = env_cpu(env);
1130     S390Access srca, desta;
1131     uint32_t cc, cur_len;
1132 
1133     if (is_destructive_overlap(env, dest, src, MIN(srclen, destlen))) {
1134         cc = 3;
1135     } else if (srclen == destlen) {
1136         cc = 0;
1137     } else if (destlen < srclen) {
1138         cc = 1;
1139     } else {
1140         cc = 2;
1141     }
1142 
1143     /* We might have to zero-out some bits even if there was no action. */
1144     if (unlikely(!destlen || cc == 3)) {
1145         set_address_zero(env, r2, src);
1146         set_address_zero(env, r1, dest);
1147         return cc;
1148     } else if (!srclen) {
1149         set_address_zero(env, r2, src);
1150     }
1151 
1152     /*
1153      * Only perform one type of type of operation (move/pad) in one step.
1154      * Stay within single pages.
1155      */
1156     while (destlen) {
1157         cur_len = MIN(destlen, -(dest | TARGET_PAGE_MASK));
1158         if (!srclen) {
1159             desta = access_prepare(env, dest, cur_len, MMU_DATA_STORE, mmu_idx,
1160                                    ra);
1161             access_memset(env, &desta, pad, ra);
1162         } else {
1163             cur_len = MIN(MIN(srclen, -(src | TARGET_PAGE_MASK)), cur_len);
1164 
1165             srca = access_prepare(env, src, cur_len, MMU_DATA_LOAD, mmu_idx,
1166                                   ra);
1167             desta = access_prepare(env, dest, cur_len, MMU_DATA_STORE, mmu_idx,
1168                                    ra);
1169             access_memmove(env, &desta, &srca, ra);
1170             src = wrap_address(env, src + cur_len);
1171             srclen -= cur_len;
1172             env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen);
1173             set_address_zero(env, r2, src);
1174         }
1175         dest = wrap_address(env, dest + cur_len);
1176         destlen -= cur_len;
1177         env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen);
1178         set_address_zero(env, r1, dest);
1179 
1180         /*
1181          * MVCL is interruptible. Return to the main loop if requested after
1182          * writing back all state to registers. If no interrupt will get
1183          * injected, we'll end up back in this handler and continue processing
1184          * the remaining parts.
1185          */
1186         if (destlen && unlikely(cpu_loop_exit_requested(cs))) {
1187             cpu_loop_exit_restore(cs, ra);
1188         }
1189     }
1190     return cc;
1191 }
1192 
1193 /* move long extended */
1194 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1195                        uint32_t r3)
1196 {
1197     uintptr_t ra = GETPC();
1198     uint64_t destlen = get_length(env, r1 + 1);
1199     uint64_t dest = get_address(env, r1);
1200     uint64_t srclen = get_length(env, r3 + 1);
1201     uint64_t src = get_address(env, r3);
1202     uint8_t pad = a2;
1203     uint32_t cc;
1204 
1205     cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra);
1206 
1207     set_length(env, r1 + 1, destlen);
1208     set_length(env, r3 + 1, srclen);
1209     set_address(env, r1, dest);
1210     set_address(env, r3, src);
1211 
1212     return cc;
1213 }
1214 
1215 /* move long unicode */
1216 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1217                        uint32_t r3)
1218 {
1219     uintptr_t ra = GETPC();
1220     uint64_t destlen = get_length(env, r1 + 1);
1221     uint64_t dest = get_address(env, r1);
1222     uint64_t srclen = get_length(env, r3 + 1);
1223     uint64_t src = get_address(env, r3);
1224     uint16_t pad = a2;
1225     uint32_t cc;
1226 
1227     cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra);
1228 
1229     set_length(env, r1 + 1, destlen);
1230     set_length(env, r3 + 1, srclen);
1231     set_address(env, r1, dest);
1232     set_address(env, r3, src);
1233 
1234     return cc;
1235 }
1236 
1237 /* compare logical long helper */
1238 static inline uint32_t do_clcl(CPUS390XState *env,
1239                                uint64_t *src1, uint64_t *src1len,
1240                                uint64_t *src3, uint64_t *src3len,
1241                                uint16_t pad, uint64_t limit,
1242                                int wordsize, uintptr_t ra)
1243 {
1244     uint64_t len = MAX(*src1len, *src3len);
1245     uint32_t cc = 0;
1246 
1247     check_alignment(env, *src1len | *src3len, wordsize, ra);
1248 
1249     if (!len) {
1250         return cc;
1251     }
1252 
1253     /* Lest we fail to service interrupts in a timely manner, limit the
1254        amount of work we're willing to do.  */
1255     if (len > limit) {
1256         len = limit;
1257         cc = 3;
1258     }
1259 
1260     for (; len; len -= wordsize) {
1261         uint16_t v1 = pad;
1262         uint16_t v3 = pad;
1263 
1264         if (*src1len) {
1265             v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra);
1266         }
1267         if (*src3len) {
1268             v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra);
1269         }
1270 
1271         if (v1 != v3) {
1272             cc = (v1 < v3) ? 1 : 2;
1273             break;
1274         }
1275 
1276         if (*src1len) {
1277             *src1 += wordsize;
1278             *src1len -= wordsize;
1279         }
1280         if (*src3len) {
1281             *src3 += wordsize;
1282             *src3len -= wordsize;
1283         }
1284     }
1285 
1286     return cc;
1287 }
1288 
1289 
1290 /* compare logical long */
1291 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
1292 {
1293     uintptr_t ra = GETPC();
1294     uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24);
1295     uint64_t src1 = get_address(env, r1);
1296     uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24);
1297     uint64_t src3 = get_address(env, r2);
1298     uint8_t pad = env->regs[r2 + 1] >> 24;
1299     uint32_t cc;
1300 
1301     cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra);
1302 
1303     env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len);
1304     env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len);
1305     set_address(env, r1, src1);
1306     set_address(env, r2, src3);
1307 
1308     return cc;
1309 }
1310 
1311 /* compare logical long extended memcompare insn with padding */
1312 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1313                        uint32_t r3)
1314 {
1315     uintptr_t ra = GETPC();
1316     uint64_t src1len = get_length(env, r1 + 1);
1317     uint64_t src1 = get_address(env, r1);
1318     uint64_t src3len = get_length(env, r3 + 1);
1319     uint64_t src3 = get_address(env, r3);
1320     uint8_t pad = a2;
1321     uint32_t cc;
1322 
1323     cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra);
1324 
1325     set_length(env, r1 + 1, src1len);
1326     set_length(env, r3 + 1, src3len);
1327     set_address(env, r1, src1);
1328     set_address(env, r3, src3);
1329 
1330     return cc;
1331 }
1332 
1333 /* compare logical long unicode memcompare insn with padding */
1334 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1335                        uint32_t r3)
1336 {
1337     uintptr_t ra = GETPC();
1338     uint64_t src1len = get_length(env, r1 + 1);
1339     uint64_t src1 = get_address(env, r1);
1340     uint64_t src3len = get_length(env, r3 + 1);
1341     uint64_t src3 = get_address(env, r3);
1342     uint16_t pad = a2;
1343     uint32_t cc = 0;
1344 
1345     cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra);
1346 
1347     set_length(env, r1 + 1, src1len);
1348     set_length(env, r3 + 1, src3len);
1349     set_address(env, r1, src1);
1350     set_address(env, r3, src3);
1351 
1352     return cc;
1353 }
1354 
1355 /* checksum */
1356 uint64_t HELPER(cksm)(CPUS390XState *env, uint64_t r1,
1357                       uint64_t src, uint64_t src_len)
1358 {
1359     uintptr_t ra = GETPC();
1360     uint64_t max_len, len;
1361     uint64_t cksm = (uint32_t)r1;
1362 
1363     /* Lest we fail to service interrupts in a timely manner, limit the
1364        amount of work we're willing to do.  For now, let's cap at 8k.  */
1365     max_len = (src_len > 0x2000 ? 0x2000 : src_len);
1366 
1367     /* Process full words as available.  */
1368     for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
1369         cksm += (uint32_t)cpu_ldl_data_ra(env, src, ra);
1370     }
1371 
1372     switch (max_len - len) {
1373     case 1:
1374         cksm += cpu_ldub_data_ra(env, src, ra) << 24;
1375         len += 1;
1376         break;
1377     case 2:
1378         cksm += cpu_lduw_data_ra(env, src, ra) << 16;
1379         len += 2;
1380         break;
1381     case 3:
1382         cksm += cpu_lduw_data_ra(env, src, ra) << 16;
1383         cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8;
1384         len += 3;
1385         break;
1386     }
1387 
1388     /* Fold the carry from the checksum.  Note that we can see carry-out
1389        during folding more than once (but probably not more than twice).  */
1390     while (cksm > 0xffffffffull) {
1391         cksm = (uint32_t)cksm + (cksm >> 32);
1392     }
1393 
1394     /* Indicate whether or not we've processed everything.  */
1395     env->cc_op = (len == src_len ? 0 : 3);
1396 
1397     /* Return both cksm and processed length.  */
1398     env->retxl = cksm;
1399     return len;
1400 }
1401 
1402 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src)
1403 {
1404     uintptr_t ra = GETPC();
1405     int len_dest = len >> 4;
1406     int len_src = len & 0xf;
1407     uint8_t b;
1408 
1409     dest += len_dest;
1410     src += len_src;
1411 
1412     /* last byte is special, it only flips the nibbles */
1413     b = cpu_ldub_data_ra(env, src, ra);
1414     cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1415     src--;
1416     len_src--;
1417 
1418     /* now pack every value */
1419     while (len_dest > 0) {
1420         b = 0;
1421 
1422         if (len_src >= 0) {
1423             b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1424             src--;
1425             len_src--;
1426         }
1427         if (len_src >= 0) {
1428             b |= cpu_ldub_data_ra(env, src, ra) << 4;
1429             src--;
1430             len_src--;
1431         }
1432 
1433         len_dest--;
1434         dest--;
1435         cpu_stb_data_ra(env, dest, b, ra);
1436     }
1437 }
1438 
1439 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src,
1440                            uint32_t srclen, int ssize, uintptr_t ra)
1441 {
1442     int i;
1443     /* The destination operand is always 16 bytes long.  */
1444     const int destlen = 16;
1445 
1446     /* The operands are processed from right to left.  */
1447     src += srclen - 1;
1448     dest += destlen - 1;
1449 
1450     for (i = 0; i < destlen; i++) {
1451         uint8_t b = 0;
1452 
1453         /* Start with a positive sign */
1454         if (i == 0) {
1455             b = 0xc;
1456         } else if (srclen > ssize) {
1457             b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1458             src -= ssize;
1459             srclen -= ssize;
1460         }
1461 
1462         if (srclen > ssize) {
1463             b |= cpu_ldub_data_ra(env, src, ra) << 4;
1464             src -= ssize;
1465             srclen -= ssize;
1466         }
1467 
1468         cpu_stb_data_ra(env, dest, b, ra);
1469         dest--;
1470     }
1471 }
1472 
1473 
1474 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src,
1475                  uint32_t srclen)
1476 {
1477     do_pkau(env, dest, src, srclen, 1, GETPC());
1478 }
1479 
1480 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src,
1481                  uint32_t srclen)
1482 {
1483     do_pkau(env, dest, src, srclen, 2, GETPC());
1484 }
1485 
1486 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
1487                   uint64_t src)
1488 {
1489     uintptr_t ra = GETPC();
1490     int len_dest = len >> 4;
1491     int len_src = len & 0xf;
1492     uint8_t b;
1493     int second_nibble = 0;
1494 
1495     dest += len_dest;
1496     src += len_src;
1497 
1498     /* last byte is special, it only flips the nibbles */
1499     b = cpu_ldub_data_ra(env, src, ra);
1500     cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1501     src--;
1502     len_src--;
1503 
1504     /* now pad every nibble with 0xf0 */
1505 
1506     while (len_dest > 0) {
1507         uint8_t cur_byte = 0;
1508 
1509         if (len_src > 0) {
1510             cur_byte = cpu_ldub_data_ra(env, src, ra);
1511         }
1512 
1513         len_dest--;
1514         dest--;
1515 
1516         /* only advance one nibble at a time */
1517         if (second_nibble) {
1518             cur_byte >>= 4;
1519             len_src--;
1520             src--;
1521         }
1522         second_nibble = !second_nibble;
1523 
1524         /* digit */
1525         cur_byte = (cur_byte & 0xf);
1526         /* zone bits */
1527         cur_byte |= 0xf0;
1528 
1529         cpu_stb_data_ra(env, dest, cur_byte, ra);
1530     }
1531 }
1532 
1533 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest,
1534                                  uint32_t destlen, int dsize, uint64_t src,
1535                                  uintptr_t ra)
1536 {
1537     int i;
1538     uint32_t cc;
1539     uint8_t b;
1540     /* The source operand is always 16 bytes long.  */
1541     const int srclen = 16;
1542 
1543     /* The operands are processed from right to left.  */
1544     src += srclen - 1;
1545     dest += destlen - dsize;
1546 
1547     /* Check for the sign.  */
1548     b = cpu_ldub_data_ra(env, src, ra);
1549     src--;
1550     switch (b & 0xf) {
1551     case 0xa:
1552     case 0xc:
1553     case 0xe ... 0xf:
1554         cc = 0;  /* plus */
1555         break;
1556     case 0xb:
1557     case 0xd:
1558         cc = 1;  /* minus */
1559         break;
1560     default:
1561     case 0x0 ... 0x9:
1562         cc = 3;  /* invalid */
1563         break;
1564     }
1565 
1566     /* Now pad every nibble with 0x30, advancing one nibble at a time. */
1567     for (i = 0; i < destlen; i += dsize) {
1568         if (i == (31 * dsize)) {
1569             /* If length is 32/64 bytes, the leftmost byte is 0. */
1570             b = 0;
1571         } else if (i % (2 * dsize)) {
1572             b = cpu_ldub_data_ra(env, src, ra);
1573             src--;
1574         } else {
1575             b >>= 4;
1576         }
1577         cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra);
1578         dest -= dsize;
1579     }
1580 
1581     return cc;
1582 }
1583 
1584 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1585                        uint64_t src)
1586 {
1587     return do_unpkau(env, dest, destlen, 1, src, GETPC());
1588 }
1589 
1590 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1591                        uint64_t src)
1592 {
1593     return do_unpkau(env, dest, destlen, 2, src, GETPC());
1594 }
1595 
1596 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen)
1597 {
1598     uintptr_t ra = GETPC();
1599     uint32_t cc = 0;
1600     int i;
1601 
1602     for (i = 0; i < destlen; i++) {
1603         uint8_t b = cpu_ldub_data_ra(env, dest + i, ra);
1604         /* digit */
1605         cc |= (b & 0xf0) > 0x90 ? 2 : 0;
1606 
1607         if (i == (destlen - 1)) {
1608             /* sign */
1609             cc |= (b & 0xf) < 0xa ? 1 : 0;
1610         } else {
1611             /* digit */
1612             cc |= (b & 0xf) > 0x9 ? 2 : 0;
1613         }
1614     }
1615 
1616     return cc;
1617 }
1618 
1619 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array,
1620                              uint64_t trans, uintptr_t ra)
1621 {
1622     uint32_t i;
1623 
1624     for (i = 0; i <= len; i++) {
1625         uint8_t byte = cpu_ldub_data_ra(env, array + i, ra);
1626         uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1627         cpu_stb_data_ra(env, array + i, new_byte, ra);
1628     }
1629 
1630     return env->cc_op;
1631 }
1632 
1633 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
1634                 uint64_t trans)
1635 {
1636     do_helper_tr(env, len, array, trans, GETPC());
1637 }
1638 
1639 uint64_t HELPER(tre)(CPUS390XState *env, uint64_t array,
1640                      uint64_t len, uint64_t trans)
1641 {
1642     uintptr_t ra = GETPC();
1643     uint8_t end = env->regs[0] & 0xff;
1644     uint64_t l = len;
1645     uint64_t i;
1646     uint32_t cc = 0;
1647 
1648     if (!(env->psw.mask & PSW_MASK_64)) {
1649         array &= 0x7fffffff;
1650         l = (uint32_t)l;
1651     }
1652 
1653     /* Lest we fail to service interrupts in a timely manner, limit the
1654        amount of work we're willing to do.  For now, let's cap at 8k.  */
1655     if (l > 0x2000) {
1656         l = 0x2000;
1657         cc = 3;
1658     }
1659 
1660     for (i = 0; i < l; i++) {
1661         uint8_t byte, new_byte;
1662 
1663         byte = cpu_ldub_data_ra(env, array + i, ra);
1664 
1665         if (byte == end) {
1666             cc = 1;
1667             break;
1668         }
1669 
1670         new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1671         cpu_stb_data_ra(env, array + i, new_byte, ra);
1672     }
1673 
1674     env->cc_op = cc;
1675     env->retxl = len - i;
1676     return array + i;
1677 }
1678 
1679 static inline uint32_t do_helper_trt(CPUS390XState *env, int len,
1680                                      uint64_t array, uint64_t trans,
1681                                      int inc, uintptr_t ra)
1682 {
1683     int i;
1684 
1685     for (i = 0; i <= len; i++) {
1686         uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra);
1687         uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra);
1688 
1689         if (sbyte != 0) {
1690             set_address(env, 1, array + i * inc);
1691             env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte);
1692             return (i == len) ? 2 : 1;
1693         }
1694     }
1695 
1696     return 0;
1697 }
1698 
1699 static uint32_t do_helper_trt_fwd(CPUS390XState *env, uint32_t len,
1700                                   uint64_t array, uint64_t trans,
1701                                   uintptr_t ra)
1702 {
1703     return do_helper_trt(env, len, array, trans, 1, ra);
1704 }
1705 
1706 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array,
1707                      uint64_t trans)
1708 {
1709     return do_helper_trt(env, len, array, trans, 1, GETPC());
1710 }
1711 
1712 static uint32_t do_helper_trt_bkwd(CPUS390XState *env, uint32_t len,
1713                                    uint64_t array, uint64_t trans,
1714                                    uintptr_t ra)
1715 {
1716     return do_helper_trt(env, len, array, trans, -1, ra);
1717 }
1718 
1719 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array,
1720                       uint64_t trans)
1721 {
1722     return do_helper_trt(env, len, array, trans, -1, GETPC());
1723 }
1724 
1725 /* Translate one/two to one/two */
1726 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2,
1727                       uint32_t tst, uint32_t sizes)
1728 {
1729     uintptr_t ra = GETPC();
1730     int dsize = (sizes & 1) ? 1 : 2;
1731     int ssize = (sizes & 2) ? 1 : 2;
1732     uint64_t tbl = get_address(env, 1);
1733     uint64_t dst = get_address(env, r1);
1734     uint64_t len = get_length(env, r1 + 1);
1735     uint64_t src = get_address(env, r2);
1736     uint32_t cc = 3;
1737     int i;
1738 
1739     /* The lower address bits of TBL are ignored.  For TROO, TROT, it's
1740        the low 3 bits (double-word aligned).  For TRTO, TRTT, it's either
1741        the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH).  */
1742     if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) {
1743         tbl &= -4096;
1744     } else {
1745         tbl &= -8;
1746     }
1747 
1748     check_alignment(env, len, ssize, ra);
1749 
1750     /* Lest we fail to service interrupts in a timely manner, */
1751     /* limit the amount of work we're willing to do.   */
1752     for (i = 0; i < 0x2000; i++) {
1753         uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra);
1754         uint64_t tble = tbl + (sval * dsize);
1755         uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra);
1756         if (dval == tst) {
1757             cc = 1;
1758             break;
1759         }
1760         cpu_stsize_data_ra(env, dst, dval, dsize, ra);
1761 
1762         len -= ssize;
1763         src += ssize;
1764         dst += dsize;
1765 
1766         if (len == 0) {
1767             cc = 0;
1768             break;
1769         }
1770     }
1771 
1772     set_address(env, r1, dst);
1773     set_length(env, r1 + 1, len);
1774     set_address(env, r2, src);
1775 
1776     return cc;
1777 }
1778 
1779 void HELPER(cdsg)(CPUS390XState *env, uint64_t addr,
1780                   uint32_t r1, uint32_t r3)
1781 {
1782     uintptr_t ra = GETPC();
1783     Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]);
1784     Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1785     Int128 oldv;
1786     uint64_t oldh, oldl;
1787     bool fail;
1788 
1789     check_alignment(env, addr, 16, ra);
1790 
1791     oldh = cpu_ldq_data_ra(env, addr + 0, ra);
1792     oldl = cpu_ldq_data_ra(env, addr + 8, ra);
1793 
1794     oldv = int128_make128(oldl, oldh);
1795     fail = !int128_eq(oldv, cmpv);
1796     if (fail) {
1797         newv = oldv;
1798     }
1799 
1800     cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra);
1801     cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra);
1802 
1803     env->cc_op = fail;
1804     env->regs[r1] = int128_gethi(oldv);
1805     env->regs[r1 + 1] = int128_getlo(oldv);
1806 }
1807 
1808 void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
1809                            uint32_t r1, uint32_t r3)
1810 {
1811     uintptr_t ra = GETPC();
1812     Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]);
1813     Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1814     int mem_idx;
1815     MemOpIdx oi;
1816     Int128 oldv;
1817     bool fail;
1818 
1819     assert(HAVE_CMPXCHG128);
1820 
1821     mem_idx = cpu_mmu_index(env, false);
1822     oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx);
1823     oldv = cpu_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
1824     fail = !int128_eq(oldv, cmpv);
1825 
1826     env->cc_op = fail;
1827     env->regs[r1] = int128_gethi(oldv);
1828     env->regs[r1 + 1] = int128_getlo(oldv);
1829 }
1830 
1831 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
1832                         uint64_t a2, bool parallel)
1833 {
1834     uint32_t mem_idx = cpu_mmu_index(env, false);
1835     uintptr_t ra = GETPC();
1836     uint32_t fc = extract32(env->regs[0], 0, 8);
1837     uint32_t sc = extract32(env->regs[0], 8, 8);
1838     uint64_t pl = get_address(env, 1) & -16;
1839     uint64_t svh, svl;
1840     uint32_t cc;
1841 
1842     /* Sanity check the function code and storage characteristic.  */
1843     if (fc > 1 || sc > 3) {
1844         if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) {
1845             goto spec_exception;
1846         }
1847         if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) {
1848             goto spec_exception;
1849         }
1850     }
1851 
1852     /* Sanity check the alignments.  */
1853     if (extract32(a1, 0, fc + 2) || extract32(a2, 0, sc)) {
1854         goto spec_exception;
1855     }
1856 
1857     /* Sanity check writability of the store address.  */
1858     probe_write(env, a2, 1 << sc, mem_idx, ra);
1859 
1860     /*
1861      * Note that the compare-and-swap is atomic, and the store is atomic,
1862      * but the complete operation is not.  Therefore we do not need to
1863      * assert serial context in order to implement this.  That said,
1864      * restart early if we can't support either operation that is supposed
1865      * to be atomic.
1866      */
1867     if (parallel) {
1868         uint32_t max = 2;
1869 #ifdef CONFIG_ATOMIC64
1870         max = 3;
1871 #endif
1872         if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) ||
1873             (HAVE_ATOMIC128  ? 0 : sc > max)) {
1874             cpu_loop_exit_atomic(env_cpu(env), ra);
1875         }
1876     }
1877 
1878     /* All loads happen before all stores.  For simplicity, load the entire
1879        store value area from the parameter list.  */
1880     svh = cpu_ldq_data_ra(env, pl + 16, ra);
1881     svl = cpu_ldq_data_ra(env, pl + 24, ra);
1882 
1883     switch (fc) {
1884     case 0:
1885         {
1886             uint32_t nv = cpu_ldl_data_ra(env, pl, ra);
1887             uint32_t cv = env->regs[r3];
1888             uint32_t ov;
1889 
1890             if (parallel) {
1891 #ifdef CONFIG_USER_ONLY
1892                 uint32_t *haddr = g2h(env_cpu(env), a1);
1893                 ov = qatomic_cmpxchg__nocheck(haddr, cv, nv);
1894 #else
1895                 MemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx);
1896                 ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra);
1897 #endif
1898             } else {
1899                 ov = cpu_ldl_data_ra(env, a1, ra);
1900                 cpu_stl_data_ra(env, a1, (ov == cv ? nv : ov), ra);
1901             }
1902             cc = (ov != cv);
1903             env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov);
1904         }
1905         break;
1906 
1907     case 1:
1908         {
1909             uint64_t nv = cpu_ldq_data_ra(env, pl, ra);
1910             uint64_t cv = env->regs[r3];
1911             uint64_t ov;
1912 
1913             if (parallel) {
1914 #ifdef CONFIG_ATOMIC64
1915                 MemOpIdx oi = make_memop_idx(MO_TEUQ | MO_ALIGN, mem_idx);
1916                 ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra);
1917 #else
1918                 /* Note that we asserted !parallel above.  */
1919                 g_assert_not_reached();
1920 #endif
1921             } else {
1922                 ov = cpu_ldq_data_ra(env, a1, ra);
1923                 cpu_stq_data_ra(env, a1, (ov == cv ? nv : ov), ra);
1924             }
1925             cc = (ov != cv);
1926             env->regs[r3] = ov;
1927         }
1928         break;
1929 
1930     case 2:
1931         {
1932             uint64_t nvh = cpu_ldq_data_ra(env, pl, ra);
1933             uint64_t nvl = cpu_ldq_data_ra(env, pl + 8, ra);
1934             Int128 nv = int128_make128(nvl, nvh);
1935             Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1936             Int128 ov;
1937 
1938             if (!parallel) {
1939                 uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra);
1940                 uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra);
1941 
1942                 ov = int128_make128(ol, oh);
1943                 cc = !int128_eq(ov, cv);
1944                 if (cc) {
1945                     nv = ov;
1946                 }
1947 
1948                 cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra);
1949                 cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra);
1950             } else if (HAVE_CMPXCHG128) {
1951                 MemOpIdx oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx);
1952                 ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
1953                 cc = !int128_eq(ov, cv);
1954             } else {
1955                 /* Note that we asserted !parallel above.  */
1956                 g_assert_not_reached();
1957             }
1958 
1959             env->regs[r3 + 0] = int128_gethi(ov);
1960             env->regs[r3 + 1] = int128_getlo(ov);
1961         }
1962         break;
1963 
1964     default:
1965         g_assert_not_reached();
1966     }
1967 
1968     /* Store only if the comparison succeeded.  Note that above we use a pair
1969        of 64-bit big-endian loads, so for sc < 3 we must extract the value
1970        from the most-significant bits of svh.  */
1971     if (cc == 0) {
1972         switch (sc) {
1973         case 0:
1974             cpu_stb_data_ra(env, a2, svh >> 56, ra);
1975             break;
1976         case 1:
1977             cpu_stw_data_ra(env, a2, svh >> 48, ra);
1978             break;
1979         case 2:
1980             cpu_stl_data_ra(env, a2, svh >> 32, ra);
1981             break;
1982         case 3:
1983             cpu_stq_data_ra(env, a2, svh, ra);
1984             break;
1985         case 4:
1986             if (!parallel) {
1987                 cpu_stq_data_ra(env, a2 + 0, svh, ra);
1988                 cpu_stq_data_ra(env, a2 + 8, svl, ra);
1989             } else if (HAVE_ATOMIC128) {
1990                 MemOpIdx oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx);
1991                 Int128 sv = int128_make128(svl, svh);
1992                 cpu_atomic_sto_be_mmu(env, a2, sv, oi, ra);
1993             } else {
1994                 /* Note that we asserted !parallel above.  */
1995                 g_assert_not_reached();
1996             }
1997             break;
1998         default:
1999             g_assert_not_reached();
2000         }
2001     }
2002 
2003     return cc;
2004 
2005  spec_exception:
2006     tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2007 }
2008 
2009 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2)
2010 {
2011     return do_csst(env, r3, a1, a2, false);
2012 }
2013 
2014 uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1,
2015                                uint64_t a2)
2016 {
2017     return do_csst(env, r3, a1, a2, true);
2018 }
2019 
2020 #if !defined(CONFIG_USER_ONLY)
2021 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2022 {
2023     uintptr_t ra = GETPC();
2024     bool PERchanged = false;
2025     uint64_t src = a2;
2026     uint32_t i;
2027 
2028     if (src & 0x7) {
2029         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2030     }
2031 
2032     for (i = r1;; i = (i + 1) % 16) {
2033         uint64_t val = cpu_ldq_data_ra(env, src, ra);
2034         if (env->cregs[i] != val && i >= 9 && i <= 11) {
2035             PERchanged = true;
2036         }
2037         env->cregs[i] = val;
2038         HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
2039                    i, src, val);
2040         src += sizeof(uint64_t);
2041 
2042         if (i == r3) {
2043             break;
2044         }
2045     }
2046 
2047     if (PERchanged && env->psw.mask & PSW_MASK_PER) {
2048         s390_cpu_recompute_watchpoints(env_cpu(env));
2049     }
2050 
2051     tlb_flush(env_cpu(env));
2052 }
2053 
2054 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2055 {
2056     uintptr_t ra = GETPC();
2057     bool PERchanged = false;
2058     uint64_t src = a2;
2059     uint32_t i;
2060 
2061     if (src & 0x3) {
2062         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2063     }
2064 
2065     for (i = r1;; i = (i + 1) % 16) {
2066         uint32_t val = cpu_ldl_data_ra(env, src, ra);
2067         if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) {
2068             PERchanged = true;
2069         }
2070         env->cregs[i] = deposit64(env->cregs[i], 0, 32, val);
2071         HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val);
2072         src += sizeof(uint32_t);
2073 
2074         if (i == r3) {
2075             break;
2076         }
2077     }
2078 
2079     if (PERchanged && env->psw.mask & PSW_MASK_PER) {
2080         s390_cpu_recompute_watchpoints(env_cpu(env));
2081     }
2082 
2083     tlb_flush(env_cpu(env));
2084 }
2085 
2086 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2087 {
2088     uintptr_t ra = GETPC();
2089     uint64_t dest = a2;
2090     uint32_t i;
2091 
2092     if (dest & 0x7) {
2093         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2094     }
2095 
2096     for (i = r1;; i = (i + 1) % 16) {
2097         cpu_stq_data_ra(env, dest, env->cregs[i], ra);
2098         dest += sizeof(uint64_t);
2099 
2100         if (i == r3) {
2101             break;
2102         }
2103     }
2104 }
2105 
2106 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2107 {
2108     uintptr_t ra = GETPC();
2109     uint64_t dest = a2;
2110     uint32_t i;
2111 
2112     if (dest & 0x3) {
2113         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2114     }
2115 
2116     for (i = r1;; i = (i + 1) % 16) {
2117         cpu_stl_data_ra(env, dest, env->cregs[i], ra);
2118         dest += sizeof(uint32_t);
2119 
2120         if (i == r3) {
2121             break;
2122         }
2123     }
2124 }
2125 
2126 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr)
2127 {
2128     uintptr_t ra = GETPC();
2129     int i;
2130 
2131     real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK;
2132 
2133     for (i = 0; i < TARGET_PAGE_SIZE; i += 8) {
2134         cpu_stq_mmuidx_ra(env, real_addr + i, 0, MMU_REAL_IDX, ra);
2135     }
2136 
2137     return 0;
2138 }
2139 
2140 uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2)
2141 {
2142     S390CPU *cpu = env_archcpu(env);
2143     CPUState *cs = env_cpu(env);
2144 
2145     /*
2146      * TODO: we currently don't handle all access protection types
2147      * (including access-list and key-controlled) as well as AR mode.
2148      */
2149     if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) {
2150         /* Fetching permitted; storing permitted */
2151         return 0;
2152     }
2153 
2154     if (env->int_pgm_code == PGM_PROTECTION) {
2155         /* retry if reading is possible */
2156         cs->exception_index = -1;
2157         if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) {
2158             /* Fetching permitted; storing not permitted */
2159             return 1;
2160         }
2161     }
2162 
2163     switch (env->int_pgm_code) {
2164     case PGM_PROTECTION:
2165         /* Fetching not permitted; storing not permitted */
2166         cs->exception_index = -1;
2167         return 2;
2168     case PGM_ADDRESSING:
2169     case PGM_TRANS_SPEC:
2170         /* exceptions forwarded to the guest */
2171         s390_cpu_virt_mem_handle_exc(cpu, GETPC());
2172         return 0;
2173     }
2174 
2175     /* Translation not available */
2176     cs->exception_index = -1;
2177     return 3;
2178 }
2179 
2180 /* insert storage key extended */
2181 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
2182 {
2183     static S390SKeysState *ss;
2184     static S390SKeysClass *skeyclass;
2185     uint64_t addr = wrap_address(env, r2);
2186     uint8_t key;
2187     int rc;
2188 
2189     addr = mmu_real2abs(env, addr);
2190     if (!mmu_absolute_addr_valid(addr, false)) {
2191         tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2192     }
2193 
2194     if (unlikely(!ss)) {
2195         ss = s390_get_skeys_device();
2196         skeyclass = S390_SKEYS_GET_CLASS(ss);
2197         if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2198             tlb_flush_all_cpus_synced(env_cpu(env));
2199         }
2200     }
2201 
2202     rc = skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2203     if (rc) {
2204         trace_get_skeys_nonzero(rc);
2205         return 0;
2206     }
2207     return key;
2208 }
2209 
2210 /* set storage key extended */
2211 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
2212 {
2213     static S390SKeysState *ss;
2214     static S390SKeysClass *skeyclass;
2215     uint64_t addr = wrap_address(env, r2);
2216     uint8_t key;
2217     int rc;
2218 
2219     addr = mmu_real2abs(env, addr);
2220     if (!mmu_absolute_addr_valid(addr, false)) {
2221         tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2222     }
2223 
2224     if (unlikely(!ss)) {
2225         ss = s390_get_skeys_device();
2226         skeyclass = S390_SKEYS_GET_CLASS(ss);
2227         if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2228             tlb_flush_all_cpus_synced(env_cpu(env));
2229         }
2230     }
2231 
2232     key = r1 & 0xfe;
2233     rc = skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2234     if (rc) {
2235         trace_set_skeys_nonzero(rc);
2236     }
2237    /*
2238     * As we can only flush by virtual address and not all the entries
2239     * that point to a physical address we have to flush the whole TLB.
2240     */
2241     tlb_flush_all_cpus_synced(env_cpu(env));
2242 }
2243 
2244 /* reset reference bit extended */
2245 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
2246 {
2247     uint64_t addr = wrap_address(env, r2);
2248     static S390SKeysState *ss;
2249     static S390SKeysClass *skeyclass;
2250     uint8_t re, key;
2251     int rc;
2252 
2253     addr = mmu_real2abs(env, addr);
2254     if (!mmu_absolute_addr_valid(addr, false)) {
2255         tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2256     }
2257 
2258     if (unlikely(!ss)) {
2259         ss = s390_get_skeys_device();
2260         skeyclass = S390_SKEYS_GET_CLASS(ss);
2261         if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2262             tlb_flush_all_cpus_synced(env_cpu(env));
2263         }
2264     }
2265 
2266     rc = skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2267     if (rc) {
2268         trace_get_skeys_nonzero(rc);
2269         return 0;
2270     }
2271 
2272     re = key & (SK_R | SK_C);
2273     key &= ~SK_R;
2274 
2275     rc = skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2276     if (rc) {
2277         trace_set_skeys_nonzero(rc);
2278         return 0;
2279     }
2280    /*
2281     * As we can only flush by virtual address and not all the entries
2282     * that point to a physical address we have to flush the whole TLB.
2283     */
2284     tlb_flush_all_cpus_synced(env_cpu(env));
2285 
2286     /*
2287      * cc
2288      *
2289      * 0  Reference bit zero; change bit zero
2290      * 1  Reference bit zero; change bit one
2291      * 2  Reference bit one; change bit zero
2292      * 3  Reference bit one; change bit one
2293      */
2294 
2295     return re >> 1;
2296 }
2297 
2298 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2,
2299                       uint64_t key)
2300 {
2301     const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2302     S390Access srca, desta;
2303     uintptr_t ra = GETPC();
2304     int cc = 0;
2305 
2306     HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
2307                __func__, l, a1, a2);
2308 
2309     if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) ||
2310         psw_as == AS_HOME || psw_as == AS_ACCREG) {
2311         s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2312     }
2313 
2314     if (!psw_key_valid(env, (key >> 4) & 0xf)) {
2315         s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2316     }
2317 
2318     l = wrap_length32(env, l);
2319     if (l > 256) {
2320         /* max 256 */
2321         l = 256;
2322         cc = 3;
2323     } else if (!l) {
2324         return cc;
2325     }
2326 
2327     srca = access_prepare(env, a2, l, MMU_DATA_LOAD, MMU_PRIMARY_IDX, ra);
2328     desta = access_prepare(env, a1, l, MMU_DATA_STORE, MMU_SECONDARY_IDX, ra);
2329     access_memmove(env, &desta, &srca, ra);
2330     return cc;
2331 }
2332 
2333 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2,
2334                       uint64_t key)
2335 {
2336     const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2337     S390Access srca, desta;
2338     uintptr_t ra = GETPC();
2339     int cc = 0;
2340 
2341     HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
2342                __func__, l, a1, a2);
2343 
2344     if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) ||
2345         psw_as == AS_HOME || psw_as == AS_ACCREG) {
2346         s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2347     }
2348 
2349     if (!psw_key_valid(env, (key >> 4) & 0xf)) {
2350         s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2351     }
2352 
2353     l = wrap_length32(env, l);
2354     if (l > 256) {
2355         /* max 256 */
2356         l = 256;
2357         cc = 3;
2358     } else if (!l) {
2359         return cc;
2360     }
2361 
2362     srca = access_prepare(env, a2, l, MMU_DATA_LOAD, MMU_SECONDARY_IDX, ra);
2363     desta = access_prepare(env, a1, l, MMU_DATA_STORE, MMU_PRIMARY_IDX, ra);
2364     access_memmove(env, &desta, &srca, ra);
2365     return cc;
2366 }
2367 
2368 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4)
2369 {
2370     CPUState *cs = env_cpu(env);
2371     const uintptr_t ra = GETPC();
2372     uint64_t table, entry, raddr;
2373     uint16_t entries, i, index = 0;
2374 
2375     if (r2 & 0xff000) {
2376         tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2377     }
2378 
2379     if (!(r2 & 0x800)) {
2380         /* invalidation-and-clearing operation */
2381         table = r1 & ASCE_ORIGIN;
2382         entries = (r2 & 0x7ff) + 1;
2383 
2384         switch (r1 & ASCE_TYPE_MASK) {
2385         case ASCE_TYPE_REGION1:
2386             index = (r2 >> 53) & 0x7ff;
2387             break;
2388         case ASCE_TYPE_REGION2:
2389             index = (r2 >> 42) & 0x7ff;
2390             break;
2391         case ASCE_TYPE_REGION3:
2392             index = (r2 >> 31) & 0x7ff;
2393             break;
2394         case ASCE_TYPE_SEGMENT:
2395             index = (r2 >> 20) & 0x7ff;
2396             break;
2397         }
2398         for (i = 0; i < entries; i++) {
2399             /* addresses are not wrapped in 24/31bit mode but table index is */
2400             raddr = table + ((index + i) & 0x7ff) * sizeof(entry);
2401             entry = cpu_ldq_mmuidx_ra(env, raddr, MMU_REAL_IDX, ra);
2402             if (!(entry & REGION_ENTRY_I)) {
2403                 /* we are allowed to not store if already invalid */
2404                 entry |= REGION_ENTRY_I;
2405                 cpu_stq_mmuidx_ra(env, raddr, entry, MMU_REAL_IDX, ra);
2406             }
2407         }
2408     }
2409 
2410     /* We simply flush the complete tlb, therefore we can ignore r3. */
2411     if (m4 & 1) {
2412         tlb_flush(cs);
2413     } else {
2414         tlb_flush_all_cpus_synced(cs);
2415     }
2416 }
2417 
2418 /* invalidate pte */
2419 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr,
2420                   uint32_t m4)
2421 {
2422     CPUState *cs = env_cpu(env);
2423     const uintptr_t ra = GETPC();
2424     uint64_t page = vaddr & TARGET_PAGE_MASK;
2425     uint64_t pte_addr, pte;
2426 
2427     /* Compute the page table entry address */
2428     pte_addr = (pto & SEGMENT_ENTRY_ORIGIN);
2429     pte_addr += VADDR_PAGE_TX(vaddr) * 8;
2430 
2431     /* Mark the page table entry as invalid */
2432     pte = cpu_ldq_mmuidx_ra(env, pte_addr, MMU_REAL_IDX, ra);
2433     pte |= PAGE_ENTRY_I;
2434     cpu_stq_mmuidx_ra(env, pte_addr, pte, MMU_REAL_IDX, ra);
2435 
2436     /* XXX we exploit the fact that Linux passes the exact virtual
2437        address here - it's not obliged to! */
2438     if (m4 & 1) {
2439         if (vaddr & ~VADDR_PAGE_TX_MASK) {
2440             tlb_flush_page(cs, page);
2441             /* XXX 31-bit hack */
2442             tlb_flush_page(cs, page ^ 0x80000000);
2443         } else {
2444             /* looks like we don't have a valid virtual address */
2445             tlb_flush(cs);
2446         }
2447     } else {
2448         if (vaddr & ~VADDR_PAGE_TX_MASK) {
2449             tlb_flush_page_all_cpus_synced(cs, page);
2450             /* XXX 31-bit hack */
2451             tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000);
2452         } else {
2453             /* looks like we don't have a valid virtual address */
2454             tlb_flush_all_cpus_synced(cs);
2455         }
2456     }
2457 }
2458 
2459 /* flush local tlb */
2460 void HELPER(ptlb)(CPUS390XState *env)
2461 {
2462     tlb_flush(env_cpu(env));
2463 }
2464 
2465 /* flush global tlb */
2466 void HELPER(purge)(CPUS390XState *env)
2467 {
2468     tlb_flush_all_cpus_synced(env_cpu(env));
2469 }
2470 
2471 /* load real address */
2472 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr)
2473 {
2474     uint64_t asc = env->psw.mask & PSW_MASK_ASC;
2475     uint64_t ret, tec;
2476     int flags, exc, cc;
2477 
2478     /* XXX incomplete - has more corner cases */
2479     if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
2480         tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, GETPC());
2481     }
2482 
2483     exc = mmu_translate(env, addr, MMU_S390_LRA, asc, &ret, &flags, &tec);
2484     if (exc) {
2485         cc = 3;
2486         ret = exc | 0x80000000;
2487     } else {
2488         cc = 0;
2489         ret |= addr & ~TARGET_PAGE_MASK;
2490     }
2491 
2492     env->cc_op = cc;
2493     return ret;
2494 }
2495 #endif
2496 
2497 /* load pair from quadword */
2498 uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr)
2499 {
2500     uintptr_t ra = GETPC();
2501     uint64_t hi, lo;
2502 
2503     check_alignment(env, addr, 16, ra);
2504     hi = cpu_ldq_data_ra(env, addr + 0, ra);
2505     lo = cpu_ldq_data_ra(env, addr + 8, ra);
2506 
2507     env->retxl = lo;
2508     return hi;
2509 }
2510 
2511 uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
2512 {
2513     uintptr_t ra = GETPC();
2514     uint64_t hi, lo;
2515     int mem_idx;
2516     MemOpIdx oi;
2517     Int128 v;
2518 
2519     assert(HAVE_ATOMIC128);
2520 
2521     mem_idx = cpu_mmu_index(env, false);
2522     oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx);
2523     v = cpu_atomic_ldo_be_mmu(env, addr, oi, ra);
2524     hi = int128_gethi(v);
2525     lo = int128_getlo(v);
2526 
2527     env->retxl = lo;
2528     return hi;
2529 }
2530 
2531 /* store pair to quadword */
2532 void HELPER(stpq)(CPUS390XState *env, uint64_t addr,
2533                   uint64_t low, uint64_t high)
2534 {
2535     uintptr_t ra = GETPC();
2536 
2537     check_alignment(env, addr, 16, ra);
2538     cpu_stq_data_ra(env, addr + 0, high, ra);
2539     cpu_stq_data_ra(env, addr + 8, low, ra);
2540 }
2541 
2542 void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
2543                            uint64_t low, uint64_t high)
2544 {
2545     uintptr_t ra = GETPC();
2546     int mem_idx;
2547     MemOpIdx oi;
2548     Int128 v;
2549 
2550     assert(HAVE_ATOMIC128);
2551 
2552     mem_idx = cpu_mmu_index(env, false);
2553     oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx);
2554     v = int128_make128(low, high);
2555     cpu_atomic_sto_be_mmu(env, addr, v, oi, ra);
2556 }
2557 
2558 /* Execute instruction.  This instruction executes an insn modified with
2559    the contents of r1.  It does not change the executed instruction in memory;
2560    it does not change the program counter.
2561 
2562    Perform this by recording the modified instruction in env->ex_value.
2563    This will be noticed by cpu_get_tb_cpu_state and thus tb translation.
2564 */
2565 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr)
2566 {
2567     uint64_t insn = cpu_lduw_code(env, addr);
2568     uint8_t opc = insn >> 8;
2569 
2570     /* Or in the contents of R1[56:63].  */
2571     insn |= r1 & 0xff;
2572 
2573     /* Load the rest of the instruction.  */
2574     insn <<= 48;
2575     switch (get_ilen(opc)) {
2576     case 2:
2577         break;
2578     case 4:
2579         insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32;
2580         break;
2581     case 6:
2582         insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16;
2583         break;
2584     default:
2585         g_assert_not_reached();
2586     }
2587 
2588     /* The very most common cases can be sped up by avoiding a new TB.  */
2589     if ((opc & 0xf0) == 0xd0) {
2590         typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t,
2591                                       uint64_t, uintptr_t);
2592         static const dx_helper dx[16] = {
2593             [0x0] = do_helper_trt_bkwd,
2594             [0x2] = do_helper_mvc,
2595             [0x4] = do_helper_nc,
2596             [0x5] = do_helper_clc,
2597             [0x6] = do_helper_oc,
2598             [0x7] = do_helper_xc,
2599             [0xc] = do_helper_tr,
2600             [0xd] = do_helper_trt_fwd,
2601         };
2602         dx_helper helper = dx[opc & 0xf];
2603 
2604         if (helper) {
2605             uint32_t l = extract64(insn, 48, 8);
2606             uint32_t b1 = extract64(insn, 44, 4);
2607             uint32_t d1 = extract64(insn, 32, 12);
2608             uint32_t b2 = extract64(insn, 28, 4);
2609             uint32_t d2 = extract64(insn, 16, 12);
2610             uint64_t a1 = wrap_address(env, (b1 ? env->regs[b1] : 0) + d1);
2611             uint64_t a2 = wrap_address(env, (b2 ? env->regs[b2] : 0) + d2);
2612 
2613             env->cc_op = helper(env, l, a1, a2, 0);
2614             env->psw.addr += ilen;
2615             return;
2616         }
2617     } else if (opc == 0x0a) {
2618         env->int_svc_code = extract64(insn, 48, 8);
2619         env->int_svc_ilen = ilen;
2620         helper_exception(env, EXCP_SVC);
2621         g_assert_not_reached();
2622     }
2623 
2624     /* Record the insn we want to execute as well as the ilen to use
2625        during the execution of the target insn.  This will also ensure
2626        that ex_value is non-zero, which flags that we are in a state
2627        that requires such execution.  */
2628     env->ex_value = insn | ilen;
2629 }
2630 
2631 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src,
2632                        uint64_t len)
2633 {
2634     const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY;
2635     const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2636     const uint64_t r0 = env->regs[0];
2637     const uintptr_t ra = GETPC();
2638     uint8_t dest_key, dest_as, dest_k, dest_a;
2639     uint8_t src_key, src_as, src_k, src_a;
2640     uint64_t val;
2641     int cc = 0;
2642 
2643     HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n",
2644                __func__, dest, src, len);
2645 
2646     if (!(env->psw.mask & PSW_MASK_DAT)) {
2647         tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2648     }
2649 
2650     /* OAC (operand access control) for the first operand -> dest */
2651     val = (r0 & 0xffff0000ULL) >> 16;
2652     dest_key = (val >> 12) & 0xf;
2653     dest_as = (val >> 6) & 0x3;
2654     dest_k = (val >> 1) & 0x1;
2655     dest_a = val & 0x1;
2656 
2657     /* OAC (operand access control) for the second operand -> src */
2658     val = (r0 & 0x0000ffffULL);
2659     src_key = (val >> 12) & 0xf;
2660     src_as = (val >> 6) & 0x3;
2661     src_k = (val >> 1) & 0x1;
2662     src_a = val & 0x1;
2663 
2664     if (!dest_k) {
2665         dest_key = psw_key;
2666     }
2667     if (!src_k) {
2668         src_key = psw_key;
2669     }
2670     if (!dest_a) {
2671         dest_as = psw_as;
2672     }
2673     if (!src_a) {
2674         src_as = psw_as;
2675     }
2676 
2677     if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) {
2678         tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2679     }
2680     if (!(env->cregs[0] & CR0_SECONDARY) &&
2681         (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) {
2682         tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2683     }
2684     if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) {
2685         tcg_s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2686     }
2687 
2688     len = wrap_length32(env, len);
2689     if (len > 4096) {
2690         cc = 3;
2691         len = 4096;
2692     }
2693 
2694     /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */
2695     if (src_as == AS_ACCREG || dest_as == AS_ACCREG ||
2696         (env->psw.mask & PSW_MASK_PSTATE)) {
2697         qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n",
2698                       __func__);
2699         tcg_s390_program_interrupt(env, PGM_ADDRESSING, ra);
2700     }
2701 
2702     /* FIXME: Access using correct keys and AR-mode */
2703     if (len) {
2704         S390Access srca = access_prepare(env, src, len, MMU_DATA_LOAD,
2705                                          mmu_idx_from_as(src_as), ra);
2706         S390Access desta = access_prepare(env, dest, len, MMU_DATA_STORE,
2707                                           mmu_idx_from_as(dest_as), ra);
2708 
2709         access_memmove(env, &desta, &srca, ra);
2710     }
2711 
2712     return cc;
2713 }
2714 
2715 /* Decode a Unicode character.  A return value < 0 indicates success, storing
2716    the UTF-32 result into OCHAR and the input length into OLEN.  A return
2717    value >= 0 indicates failure, and the CC value to be returned.  */
2718 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2719                                  uint64_t ilen, bool enh_check, uintptr_t ra,
2720                                  uint32_t *ochar, uint32_t *olen);
2721 
2722 /* Encode a Unicode character.  A return value < 0 indicates success, storing
2723    the bytes into ADDR and the output length into OLEN.  A return value >= 0
2724    indicates failure, and the CC value to be returned.  */
2725 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2726                                  uint64_t ilen, uintptr_t ra, uint32_t c,
2727                                  uint32_t *olen);
2728 
2729 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2730                        bool enh_check, uintptr_t ra,
2731                        uint32_t *ochar, uint32_t *olen)
2732 {
2733     uint8_t s0, s1, s2, s3;
2734     uint32_t c, l;
2735 
2736     if (ilen < 1) {
2737         return 0;
2738     }
2739     s0 = cpu_ldub_data_ra(env, addr, ra);
2740     if (s0 <= 0x7f) {
2741         /* one byte character */
2742         l = 1;
2743         c = s0;
2744     } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) {
2745         /* invalid character */
2746         return 2;
2747     } else if (s0 <= 0xdf) {
2748         /* two byte character */
2749         l = 2;
2750         if (ilen < 2) {
2751             return 0;
2752         }
2753         s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2754         c = s0 & 0x1f;
2755         c = (c << 6) | (s1 & 0x3f);
2756         if (enh_check && (s1 & 0xc0) != 0x80) {
2757             return 2;
2758         }
2759     } else if (s0 <= 0xef) {
2760         /* three byte character */
2761         l = 3;
2762         if (ilen < 3) {
2763             return 0;
2764         }
2765         s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2766         s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2767         c = s0 & 0x0f;
2768         c = (c << 6) | (s1 & 0x3f);
2769         c = (c << 6) | (s2 & 0x3f);
2770         /* Fold the byte-by-byte range descriptions in the PoO into
2771            tests against the complete value.  It disallows encodings
2772            that could be smaller, and the UTF-16 surrogates.  */
2773         if (enh_check
2774             && ((s1 & 0xc0) != 0x80
2775                 || (s2 & 0xc0) != 0x80
2776                 || c < 0x1000
2777                 || (c >= 0xd800 && c <= 0xdfff))) {
2778             return 2;
2779         }
2780     } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) {
2781         /* four byte character */
2782         l = 4;
2783         if (ilen < 4) {
2784             return 0;
2785         }
2786         s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2787         s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2788         s3 = cpu_ldub_data_ra(env, addr + 3, ra);
2789         c = s0 & 0x07;
2790         c = (c << 6) | (s1 & 0x3f);
2791         c = (c << 6) | (s2 & 0x3f);
2792         c = (c << 6) | (s3 & 0x3f);
2793         /* See above.  */
2794         if (enh_check
2795             && ((s1 & 0xc0) != 0x80
2796                 || (s2 & 0xc0) != 0x80
2797                 || (s3 & 0xc0) != 0x80
2798                 || c < 0x010000
2799                 || c > 0x10ffff)) {
2800             return 2;
2801         }
2802     } else {
2803         /* invalid character */
2804         return 2;
2805     }
2806 
2807     *ochar = c;
2808     *olen = l;
2809     return -1;
2810 }
2811 
2812 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2813                         bool enh_check, uintptr_t ra,
2814                         uint32_t *ochar, uint32_t *olen)
2815 {
2816     uint16_t s0, s1;
2817     uint32_t c, l;
2818 
2819     if (ilen < 2) {
2820         return 0;
2821     }
2822     s0 = cpu_lduw_data_ra(env, addr, ra);
2823     if ((s0 & 0xfc00) != 0xd800) {
2824         /* one word character */
2825         l = 2;
2826         c = s0;
2827     } else {
2828         /* two word character */
2829         l = 4;
2830         if (ilen < 4) {
2831             return 0;
2832         }
2833         s1 = cpu_lduw_data_ra(env, addr + 2, ra);
2834         c = extract32(s0, 6, 4) + 1;
2835         c = (c << 6) | (s0 & 0x3f);
2836         c = (c << 10) | (s1 & 0x3ff);
2837         if (enh_check && (s1 & 0xfc00) != 0xdc00) {
2838             /* invalid surrogate character */
2839             return 2;
2840         }
2841     }
2842 
2843     *ochar = c;
2844     *olen = l;
2845     return -1;
2846 }
2847 
2848 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2849                         bool enh_check, uintptr_t ra,
2850                         uint32_t *ochar, uint32_t *olen)
2851 {
2852     uint32_t c;
2853 
2854     if (ilen < 4) {
2855         return 0;
2856     }
2857     c = cpu_ldl_data_ra(env, addr, ra);
2858     if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) {
2859         /* invalid unicode character */
2860         return 2;
2861     }
2862 
2863     *ochar = c;
2864     *olen = 4;
2865     return -1;
2866 }
2867 
2868 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2869                        uintptr_t ra, uint32_t c, uint32_t *olen)
2870 {
2871     uint8_t d[4];
2872     uint32_t l, i;
2873 
2874     if (c <= 0x7f) {
2875         /* one byte character */
2876         l = 1;
2877         d[0] = c;
2878     } else if (c <= 0x7ff) {
2879         /* two byte character */
2880         l = 2;
2881         d[1] = 0x80 | extract32(c, 0, 6);
2882         d[0] = 0xc0 | extract32(c, 6, 5);
2883     } else if (c <= 0xffff) {
2884         /* three byte character */
2885         l = 3;
2886         d[2] = 0x80 | extract32(c, 0, 6);
2887         d[1] = 0x80 | extract32(c, 6, 6);
2888         d[0] = 0xe0 | extract32(c, 12, 4);
2889     } else {
2890         /* four byte character */
2891         l = 4;
2892         d[3] = 0x80 | extract32(c, 0, 6);
2893         d[2] = 0x80 | extract32(c, 6, 6);
2894         d[1] = 0x80 | extract32(c, 12, 6);
2895         d[0] = 0xf0 | extract32(c, 18, 3);
2896     }
2897 
2898     if (ilen < l) {
2899         return 1;
2900     }
2901     for (i = 0; i < l; ++i) {
2902         cpu_stb_data_ra(env, addr + i, d[i], ra);
2903     }
2904 
2905     *olen = l;
2906     return -1;
2907 }
2908 
2909 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2910                         uintptr_t ra, uint32_t c, uint32_t *olen)
2911 {
2912     uint16_t d0, d1;
2913 
2914     if (c <= 0xffff) {
2915         /* one word character */
2916         if (ilen < 2) {
2917             return 1;
2918         }
2919         cpu_stw_data_ra(env, addr, c, ra);
2920         *olen = 2;
2921     } else {
2922         /* two word character */
2923         if (ilen < 4) {
2924             return 1;
2925         }
2926         d1 = 0xdc00 | extract32(c, 0, 10);
2927         d0 = 0xd800 | extract32(c, 10, 6);
2928         d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1);
2929         cpu_stw_data_ra(env, addr + 0, d0, ra);
2930         cpu_stw_data_ra(env, addr + 2, d1, ra);
2931         *olen = 4;
2932     }
2933 
2934     return -1;
2935 }
2936 
2937 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2938                         uintptr_t ra, uint32_t c, uint32_t *olen)
2939 {
2940     if (ilen < 4) {
2941         return 1;
2942     }
2943     cpu_stl_data_ra(env, addr, c, ra);
2944     *olen = 4;
2945     return -1;
2946 }
2947 
2948 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1,
2949                                        uint32_t r2, uint32_t m3, uintptr_t ra,
2950                                        decode_unicode_fn decode,
2951                                        encode_unicode_fn encode)
2952 {
2953     uint64_t dst = get_address(env, r1);
2954     uint64_t dlen = get_length(env, r1 + 1);
2955     uint64_t src = get_address(env, r2);
2956     uint64_t slen = get_length(env, r2 + 1);
2957     bool enh_check = m3 & 1;
2958     int cc, i;
2959 
2960     /* Lest we fail to service interrupts in a timely manner, limit the
2961        amount of work we're willing to do.  For now, let's cap at 256.  */
2962     for (i = 0; i < 256; ++i) {
2963         uint32_t c, ilen, olen;
2964 
2965         cc = decode(env, src, slen, enh_check, ra, &c, &ilen);
2966         if (unlikely(cc >= 0)) {
2967             break;
2968         }
2969         cc = encode(env, dst, dlen, ra, c, &olen);
2970         if (unlikely(cc >= 0)) {
2971             break;
2972         }
2973 
2974         src += ilen;
2975         slen -= ilen;
2976         dst += olen;
2977         dlen -= olen;
2978         cc = 3;
2979     }
2980 
2981     set_address(env, r1, dst);
2982     set_length(env, r1 + 1, dlen);
2983     set_address(env, r2, src);
2984     set_length(env, r2 + 1, slen);
2985 
2986     return cc;
2987 }
2988 
2989 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2990 {
2991     return convert_unicode(env, r1, r2, m3, GETPC(),
2992                            decode_utf8, encode_utf16);
2993 }
2994 
2995 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2996 {
2997     return convert_unicode(env, r1, r2, m3, GETPC(),
2998                            decode_utf8, encode_utf32);
2999 }
3000 
3001 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
3002 {
3003     return convert_unicode(env, r1, r2, m3, GETPC(),
3004                            decode_utf16, encode_utf8);
3005 }
3006 
3007 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
3008 {
3009     return convert_unicode(env, r1, r2, m3, GETPC(),
3010                            decode_utf16, encode_utf32);
3011 }
3012 
3013 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
3014 {
3015     return convert_unicode(env, r1, r2, m3, GETPC(),
3016                            decode_utf32, encode_utf8);
3017 }
3018 
3019 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
3020 {
3021     return convert_unicode(env, r1, r2, m3, GETPC(),
3022                            decode_utf32, encode_utf16);
3023 }
3024 
3025 void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len,
3026                         uintptr_t ra)
3027 {
3028     /* test the actual access, not just any access to the page due to LAP */
3029     while (len) {
3030         const uint64_t pagelen = -(addr | TARGET_PAGE_MASK);
3031         const uint64_t curlen = MIN(pagelen, len);
3032 
3033         probe_write(env, addr, curlen, cpu_mmu_index(env, false), ra);
3034         addr = wrap_address(env, addr + curlen);
3035         len -= curlen;
3036     }
3037 }
3038 
3039 void HELPER(probe_write_access)(CPUS390XState *env, uint64_t addr, uint64_t len)
3040 {
3041     probe_write_access(env, addr, len, GETPC());
3042 }
3043