1 /*
2 * S/390 memory access helper routines
3 *
4 * Copyright (c) 2009 Ulrich Hecht
5 * Copyright (c) 2009 Alexander Graf
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/osdep.h"
22 #include "qemu/log.h"
23 #include "cpu.h"
24 #include "s390x-internal.h"
25 #include "tcg_s390x.h"
26 #include "exec/helper-proto.h"
27 #include "exec/cpu-common.h"
28 #include "exec/cputlb.h"
29 #include "exec/page-protection.h"
30 #include "accel/tcg/cpu-ldst.h"
31 #include "accel/tcg/probe.h"
32 #include "exec/target_page.h"
33 #include "exec/tlb-flags.h"
34 #include "accel/tcg/cpu-ops.h"
35 #include "accel/tcg/helper-retaddr.h"
36 #include "qemu/int128.h"
37 #include "qemu/atomic128.h"
38
39 #if defined(CONFIG_USER_ONLY)
40 #include "user/page-protection.h"
41 #else
42 #include "hw/s390x/storage-keys.h"
43 #include "hw/boards.h"
44 #endif
45
46 #ifdef CONFIG_USER_ONLY
47 # define user_or_likely(X) true
48 #else
49 # define user_or_likely(X) likely(X)
50 #endif
51
52 /*****************************************************************************/
53 /* Softmmu support */
54
55 /* #define DEBUG_HELPER */
56 #ifdef DEBUG_HELPER
57 #define HELPER_LOG(x...) qemu_log(x)
58 #else
59 #define HELPER_LOG(x...)
60 #endif
61
psw_key_valid(CPUS390XState * env,uint8_t psw_key)62 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key)
63 {
64 uint16_t pkm = env->cregs[3] >> 16;
65
66 if (env->psw.mask & PSW_MASK_PSTATE) {
67 /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */
68 return pkm & (0x8000 >> psw_key);
69 }
70 return true;
71 }
72
is_destructive_overlap(CPUS390XState * env,uint64_t dest,uint64_t src,uint32_t len)73 static bool is_destructive_overlap(CPUS390XState *env, uint64_t dest,
74 uint64_t src, uint32_t len)
75 {
76 if (!len || src == dest) {
77 return false;
78 }
79 /* Take care of wrapping at the end of address space. */
80 if (unlikely(wrap_address(env, src + len - 1) < src)) {
81 return dest > src || dest <= wrap_address(env, src + len - 1);
82 }
83 return dest > src && dest <= src + len - 1;
84 }
85
86 /* Trigger a SPECIFICATION exception if an address or a length is not
87 naturally aligned. */
check_alignment(CPUS390XState * env,uint64_t v,int wordsize,uintptr_t ra)88 static inline void check_alignment(CPUS390XState *env, uint64_t v,
89 int wordsize, uintptr_t ra)
90 {
91 if (v % wordsize) {
92 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
93 }
94 }
95
96 /* Load a value from memory according to its size. */
cpu_ldusize_data_ra(CPUS390XState * env,uint64_t addr,int wordsize,uintptr_t ra)97 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr,
98 int wordsize, uintptr_t ra)
99 {
100 switch (wordsize) {
101 case 1:
102 return cpu_ldub_data_ra(env, addr, ra);
103 case 2:
104 return cpu_lduw_data_ra(env, addr, ra);
105 default:
106 abort();
107 }
108 }
109
110 /* Store a to memory according to its size. */
cpu_stsize_data_ra(CPUS390XState * env,uint64_t addr,uint64_t value,int wordsize,uintptr_t ra)111 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr,
112 uint64_t value, int wordsize,
113 uintptr_t ra)
114 {
115 switch (wordsize) {
116 case 1:
117 cpu_stb_data_ra(env, addr, value, ra);
118 break;
119 case 2:
120 cpu_stw_data_ra(env, addr, value, ra);
121 break;
122 default:
123 abort();
124 }
125 }
126
127 /* An access covers at most 4096 bytes and therefore at most two pages. */
128 typedef struct S390Access {
129 vaddr vaddr1;
130 vaddr vaddr2;
131 void *haddr1;
132 void *haddr2;
133 uint16_t size1;
134 uint16_t size2;
135 /*
136 * If we can't access the host page directly, we'll have to do I/O access
137 * via ld/st helpers. These are internal details, so we store the
138 * mmu idx to do the access here instead of passing it around in the
139 * helpers.
140 */
141 int mmu_idx;
142 } S390Access;
143
144 /*
145 * With nonfault=1, return the PGM_ exception that would have been injected
146 * into the guest; return 0 if no exception was detected.
147 *
148 * For !CONFIG_USER_ONLY, the TEC is stored stored to env->tlb_fill_tec.
149 * For CONFIG_USER_ONLY, the faulting address is stored to env->__excp_addr.
150 */
s390_probe_access(CPUArchState * env,vaddr addr,int size,MMUAccessType access_type,int mmu_idx,bool nonfault,void ** phost,uintptr_t ra)151 static inline int s390_probe_access(CPUArchState *env, vaddr addr,
152 int size, MMUAccessType access_type,
153 int mmu_idx, bool nonfault,
154 void **phost, uintptr_t ra)
155 {
156 int flags = probe_access_flags(env, addr, size, access_type, mmu_idx,
157 nonfault, phost, ra);
158
159 if (unlikely(flags & TLB_INVALID_MASK)) {
160 #ifdef CONFIG_USER_ONLY
161 /* Address is in TEC in system mode; see s390_cpu_record_sigsegv. */
162 env->__excp_addr = addr & TARGET_PAGE_MASK;
163 return (page_get_flags(addr) & PAGE_VALID
164 ? PGM_PROTECTION : PGM_ADDRESSING);
165 #else
166 return env->tlb_fill_exc;
167 #endif
168 }
169
170 #ifndef CONFIG_USER_ONLY
171 if (unlikely(flags & TLB_WATCHPOINT)) {
172 /* S390 does not presently use transaction attributes. */
173 cpu_check_watchpoint(env_cpu(env), addr, size,
174 MEMTXATTRS_UNSPECIFIED,
175 (access_type == MMU_DATA_STORE
176 ? BP_MEM_WRITE : BP_MEM_READ), ra);
177 }
178 #endif
179
180 return 0;
181 }
182
access_prepare_nf(S390Access * access,CPUS390XState * env,bool nonfault,vaddr vaddr1,int size,MMUAccessType access_type,int mmu_idx,uintptr_t ra)183 static int access_prepare_nf(S390Access *access, CPUS390XState *env,
184 bool nonfault, vaddr vaddr1, int size,
185 MMUAccessType access_type,
186 int mmu_idx, uintptr_t ra)
187 {
188 int size1, size2, exc;
189
190 assert(size > 0 && size <= 4096);
191
192 size1 = MIN(size, -(vaddr1 | TARGET_PAGE_MASK)),
193 size2 = size - size1;
194
195 memset(access, 0, sizeof(*access));
196 access->vaddr1 = vaddr1;
197 access->size1 = size1;
198 access->size2 = size2;
199 access->mmu_idx = mmu_idx;
200
201 exc = s390_probe_access(env, vaddr1, size1, access_type, mmu_idx, nonfault,
202 &access->haddr1, ra);
203 if (unlikely(exc)) {
204 return exc;
205 }
206 if (unlikely(size2)) {
207 /* The access crosses page boundaries. */
208 vaddr vaddr2 = wrap_address(env, vaddr1 + size1);
209
210 access->vaddr2 = vaddr2;
211 exc = s390_probe_access(env, vaddr2, size2, access_type, mmu_idx,
212 nonfault, &access->haddr2, ra);
213 if (unlikely(exc)) {
214 return exc;
215 }
216 }
217 return 0;
218 }
219
access_prepare(S390Access * ret,CPUS390XState * env,vaddr vaddr,int size,MMUAccessType access_type,int mmu_idx,uintptr_t ra)220 static inline void access_prepare(S390Access *ret, CPUS390XState *env,
221 vaddr vaddr, int size,
222 MMUAccessType access_type, int mmu_idx,
223 uintptr_t ra)
224 {
225 int exc = access_prepare_nf(ret, env, false, vaddr, size,
226 access_type, mmu_idx, ra);
227 assert(!exc);
228 }
229
230 /* Helper to handle memset on a single page. */
do_access_memset(CPUS390XState * env,vaddr vaddr,char * haddr,uint8_t byte,uint16_t size,int mmu_idx,uintptr_t ra)231 static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr,
232 uint8_t byte, uint16_t size, int mmu_idx,
233 uintptr_t ra)
234 {
235 if (user_or_likely(haddr)) {
236 memset(haddr, byte, size);
237 } else {
238 MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
239 for (int i = 0; i < size; i++) {
240 cpu_stb_mmu(env, vaddr + i, byte, oi, ra);
241 }
242 }
243 }
244
access_memset(CPUS390XState * env,S390Access * desta,uint8_t byte,uintptr_t ra)245 static void access_memset(CPUS390XState *env, S390Access *desta,
246 uint8_t byte, uintptr_t ra)
247 {
248 set_helper_retaddr(ra);
249 do_access_memset(env, desta->vaddr1, desta->haddr1, byte, desta->size1,
250 desta->mmu_idx, ra);
251 if (unlikely(desta->size2)) {
252 do_access_memset(env, desta->vaddr2, desta->haddr2, byte,
253 desta->size2, desta->mmu_idx, ra);
254 }
255 clear_helper_retaddr();
256 }
257
access_get_byte(CPUS390XState * env,S390Access * access,int offset,uintptr_t ra)258 static uint8_t access_get_byte(CPUS390XState *env, S390Access *access,
259 int offset, uintptr_t ra)
260 {
261 vaddr vaddr = access->vaddr1;
262 void *haddr = access->haddr1;
263
264 if (unlikely(offset >= access->size1)) {
265 offset -= access->size1;
266 vaddr = access->vaddr2;
267 haddr = access->haddr2;
268 }
269
270 if (user_or_likely(haddr)) {
271 return ldub_p(haddr + offset);
272 } else {
273 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx);
274 return cpu_ldb_mmu(env, vaddr + offset, oi, ra);
275 }
276 }
277
access_set_byte(CPUS390XState * env,S390Access * access,int offset,uint8_t byte,uintptr_t ra)278 static void access_set_byte(CPUS390XState *env, S390Access *access,
279 int offset, uint8_t byte, uintptr_t ra)
280 {
281 vaddr vaddr = access->vaddr1;
282 void *haddr = access->haddr1;
283
284 if (unlikely(offset >= access->size1)) {
285 offset -= access->size1;
286 vaddr = access->vaddr2;
287 haddr = access->haddr2;
288 }
289
290 if (user_or_likely(haddr)) {
291 stb_p(haddr + offset, byte);
292 } else {
293 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx);
294 cpu_stb_mmu(env, vaddr + offset, byte, oi, ra);
295 }
296 }
297
298 /*
299 * Move data with the same semantics as memmove() in case ranges don't overlap
300 * or src > dest. Undefined behavior on destructive overlaps.
301 */
access_memmove(CPUS390XState * env,S390Access * desta,S390Access * srca,uintptr_t ra)302 static void access_memmove(CPUS390XState *env, S390Access *desta,
303 S390Access *srca, uintptr_t ra)
304 {
305 int len = desta->size1 + desta->size2;
306
307 assert(len == srca->size1 + srca->size2);
308
309 /* Fallback to slow access in case we don't have access to all host pages */
310 if (user_or_likely(desta->haddr1 &&
311 srca->haddr1 &&
312 (!desta->size2 || desta->haddr2) &&
313 (!srca->size2 || srca->haddr2))) {
314 int diff = desta->size1 - srca->size1;
315
316 if (likely(diff == 0)) {
317 memmove(desta->haddr1, srca->haddr1, srca->size1);
318 if (unlikely(srca->size2)) {
319 memmove(desta->haddr2, srca->haddr2, srca->size2);
320 }
321 } else if (diff > 0) {
322 memmove(desta->haddr1, srca->haddr1, srca->size1);
323 memmove(desta->haddr1 + srca->size1, srca->haddr2, diff);
324 if (likely(desta->size2)) {
325 memmove(desta->haddr2, srca->haddr2 + diff, desta->size2);
326 }
327 } else {
328 diff = -diff;
329 memmove(desta->haddr1, srca->haddr1, desta->size1);
330 memmove(desta->haddr2, srca->haddr1 + desta->size1, diff);
331 if (likely(srca->size2)) {
332 memmove(desta->haddr2 + diff, srca->haddr2, srca->size2);
333 }
334 }
335 } else {
336 for (int i = 0; i < len; i++) {
337 uint8_t byte = access_get_byte(env, srca, i, ra);
338 access_set_byte(env, desta, i, byte, ra);
339 }
340 }
341 }
342
mmu_idx_from_as(uint8_t as)343 static int mmu_idx_from_as(uint8_t as)
344 {
345 switch (as) {
346 case AS_PRIMARY:
347 return MMU_PRIMARY_IDX;
348 case AS_SECONDARY:
349 return MMU_SECONDARY_IDX;
350 case AS_HOME:
351 return MMU_HOME_IDX;
352 default:
353 /* FIXME AS_ACCREG */
354 g_assert_not_reached();
355 }
356 }
357
358 /* and on array */
do_helper_nc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)359 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest,
360 uint64_t src, uintptr_t ra)
361 {
362 const int mmu_idx = s390x_env_mmu_index(env, false);
363 S390Access srca1, srca2, desta;
364 uint32_t i;
365 uint8_t c = 0;
366
367 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
368 __func__, l, dest, src);
369
370 /* NC always processes one more byte than specified - maximum is 256 */
371 l++;
372
373 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
374 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
375 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
376 set_helper_retaddr(ra);
377
378 for (i = 0; i < l; i++) {
379 const uint8_t x = access_get_byte(env, &srca1, i, ra) &
380 access_get_byte(env, &srca2, i, ra);
381
382 c |= x;
383 access_set_byte(env, &desta, i, x, ra);
384 }
385
386 clear_helper_retaddr();
387 return c != 0;
388 }
389
HELPER(nc)390 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
391 uint64_t src)
392 {
393 return do_helper_nc(env, l, dest, src, GETPC());
394 }
395
396 /* xor on array */
do_helper_xc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)397 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest,
398 uint64_t src, uintptr_t ra)
399 {
400 const int mmu_idx = s390x_env_mmu_index(env, false);
401 S390Access srca1, srca2, desta;
402 uint32_t i;
403 uint8_t c = 0;
404
405 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
406 __func__, l, dest, src);
407
408 /* XC always processes one more byte than specified - maximum is 256 */
409 l++;
410
411 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
412 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
413 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
414
415 /* xor with itself is the same as memset(0) */
416 if (src == dest) {
417 access_memset(env, &desta, 0, ra);
418 return 0;
419 }
420
421 set_helper_retaddr(ra);
422 for (i = 0; i < l; i++) {
423 const uint8_t x = access_get_byte(env, &srca1, i, ra) ^
424 access_get_byte(env, &srca2, i, ra);
425
426 c |= x;
427 access_set_byte(env, &desta, i, x, ra);
428 }
429 clear_helper_retaddr();
430 return c != 0;
431 }
432
HELPER(xc)433 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
434 uint64_t src)
435 {
436 return do_helper_xc(env, l, dest, src, GETPC());
437 }
438
439 /* or on array */
do_helper_oc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)440 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest,
441 uint64_t src, uintptr_t ra)
442 {
443 const int mmu_idx = s390x_env_mmu_index(env, false);
444 S390Access srca1, srca2, desta;
445 uint32_t i;
446 uint8_t c = 0;
447
448 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
449 __func__, l, dest, src);
450
451 /* OC always processes one more byte than specified - maximum is 256 */
452 l++;
453
454 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
455 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
456 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
457 set_helper_retaddr(ra);
458
459 for (i = 0; i < l; i++) {
460 const uint8_t x = access_get_byte(env, &srca1, i, ra) |
461 access_get_byte(env, &srca2, i, ra);
462
463 c |= x;
464 access_set_byte(env, &desta, i, x, ra);
465 }
466
467 clear_helper_retaddr();
468 return c != 0;
469 }
470
HELPER(oc)471 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
472 uint64_t src)
473 {
474 return do_helper_oc(env, l, dest, src, GETPC());
475 }
476
477 /* memmove */
do_helper_mvc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)478 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest,
479 uint64_t src, uintptr_t ra)
480 {
481 const int mmu_idx = s390x_env_mmu_index(env, false);
482 S390Access srca, desta;
483 uint32_t i;
484
485 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
486 __func__, l, dest, src);
487
488 /* MVC always copies one more byte than specified - maximum is 256 */
489 l++;
490
491 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
492 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
493
494 /*
495 * "When the operands overlap, the result is obtained as if the operands
496 * were processed one byte at a time". Only non-destructive overlaps
497 * behave like memmove().
498 */
499 if (dest == src + 1) {
500 access_memset(env, &desta, access_get_byte(env, &srca, 0, ra), ra);
501 } else if (!is_destructive_overlap(env, dest, src, l)) {
502 access_memmove(env, &desta, &srca, ra);
503 } else {
504 set_helper_retaddr(ra);
505 for (i = 0; i < l; i++) {
506 uint8_t byte = access_get_byte(env, &srca, i, ra);
507
508 access_set_byte(env, &desta, i, byte, ra);
509 }
510 clear_helper_retaddr();
511 }
512
513 return env->cc_op;
514 }
515
HELPER(mvc)516 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
517 {
518 do_helper_mvc(env, l, dest, src, GETPC());
519 }
520
521 /* move right to left */
HELPER(mvcrl)522 void HELPER(mvcrl)(CPUS390XState *env, uint64_t l, uint64_t dest, uint64_t src)
523 {
524 const int mmu_idx = s390x_env_mmu_index(env, false);
525 const uint64_t ra = GETPC();
526 S390Access srca, desta;
527 int32_t i;
528
529 /* MVCRL always copies one more byte than specified - maximum is 256 */
530 l &= 0xff;
531 l++;
532
533 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
534 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
535
536 set_helper_retaddr(ra);
537 for (i = l - 1; i >= 0; i--) {
538 uint8_t byte = access_get_byte(env, &srca, i, ra);
539 access_set_byte(env, &desta, i, byte, ra);
540 }
541 clear_helper_retaddr();
542 }
543
544 /* move inverse */
HELPER(mvcin)545 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
546 {
547 const int mmu_idx = s390x_env_mmu_index(env, false);
548 S390Access srca, desta;
549 uintptr_t ra = GETPC();
550 int i;
551
552 /* MVCIN always copies one more byte than specified - maximum is 256 */
553 l++;
554
555 src = wrap_address(env, src - l + 1);
556 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
557 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
558
559 set_helper_retaddr(ra);
560 for (i = 0; i < l; i++) {
561 const uint8_t x = access_get_byte(env, &srca, l - i - 1, ra);
562 access_set_byte(env, &desta, i, x, ra);
563 }
564 clear_helper_retaddr();
565 }
566
567 /* move numerics */
HELPER(mvn)568 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
569 {
570 const int mmu_idx = s390x_env_mmu_index(env, false);
571 S390Access srca1, srca2, desta;
572 uintptr_t ra = GETPC();
573 int i;
574
575 /* MVN always copies one more byte than specified - maximum is 256 */
576 l++;
577
578 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
579 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
580 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
581
582 set_helper_retaddr(ra);
583 for (i = 0; i < l; i++) {
584 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0x0f) |
585 (access_get_byte(env, &srca2, i, ra) & 0xf0);
586
587 access_set_byte(env, &desta, i, x, ra);
588 }
589 clear_helper_retaddr();
590 }
591
592 /* move with offset */
HELPER(mvo)593 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
594 {
595 const int mmu_idx = s390x_env_mmu_index(env, false);
596 /* MVO always processes one more byte than specified - maximum is 16 */
597 const int len_dest = (l >> 4) + 1;
598 const int len_src = (l & 0xf) + 1;
599 uintptr_t ra = GETPC();
600 uint8_t byte_dest, byte_src;
601 S390Access srca, desta;
602 int i, j;
603
604 access_prepare(&srca, env, src, len_src, MMU_DATA_LOAD, mmu_idx, ra);
605 access_prepare(&desta, env, dest, len_dest, MMU_DATA_STORE, mmu_idx, ra);
606
607 /* Handle rightmost byte */
608 byte_dest = cpu_ldub_data_ra(env, dest + len_dest - 1, ra);
609
610 set_helper_retaddr(ra);
611 byte_src = access_get_byte(env, &srca, len_src - 1, ra);
612 byte_dest = (byte_dest & 0x0f) | (byte_src << 4);
613 access_set_byte(env, &desta, len_dest - 1, byte_dest, ra);
614
615 /* Process remaining bytes from right to left */
616 for (i = len_dest - 2, j = len_src - 2; i >= 0; i--, j--) {
617 byte_dest = byte_src >> 4;
618 if (j >= 0) {
619 byte_src = access_get_byte(env, &srca, j, ra);
620 } else {
621 byte_src = 0;
622 }
623 byte_dest |= byte_src << 4;
624 access_set_byte(env, &desta, i, byte_dest, ra);
625 }
626 clear_helper_retaddr();
627 }
628
629 /* move zones */
HELPER(mvz)630 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
631 {
632 const int mmu_idx = s390x_env_mmu_index(env, false);
633 S390Access srca1, srca2, desta;
634 uintptr_t ra = GETPC();
635 int i;
636
637 /* MVZ always copies one more byte than specified - maximum is 256 */
638 l++;
639
640 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
641 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
642 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
643
644 set_helper_retaddr(ra);
645 for (i = 0; i < l; i++) {
646 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0xf0) |
647 (access_get_byte(env, &srca2, i, ra) & 0x0f);
648
649 access_set_byte(env, &desta, i, x, ra);
650 }
651 clear_helper_retaddr();
652 }
653
654 /* compare unsigned byte arrays */
do_helper_clc(CPUS390XState * env,uint32_t l,uint64_t s1,uint64_t s2,uintptr_t ra)655 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1,
656 uint64_t s2, uintptr_t ra)
657 {
658 uint32_t i;
659 uint32_t cc = 0;
660
661 HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
662 __func__, l, s1, s2);
663
664 for (i = 0; i <= l; i++) {
665 uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra);
666 uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra);
667 HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
668 if (x < y) {
669 cc = 1;
670 break;
671 } else if (x > y) {
672 cc = 2;
673 break;
674 }
675 }
676
677 HELPER_LOG("\n");
678 return cc;
679 }
680
HELPER(clc)681 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
682 {
683 return do_helper_clc(env, l, s1, s2, GETPC());
684 }
685
686 /* compare logical under mask */
HELPER(clm)687 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
688 uint64_t addr)
689 {
690 uintptr_t ra = GETPC();
691 uint32_t cc = 0;
692
693 HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
694 mask, addr);
695
696 if (!mask) {
697 /* Recognize access exceptions for the first byte */
698 probe_read(env, addr, 1, s390x_env_mmu_index(env, false), ra);
699 }
700
701 while (mask) {
702 if (mask & 8) {
703 uint8_t d = cpu_ldub_data_ra(env, addr, ra);
704 uint8_t r = extract32(r1, 24, 8);
705 HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
706 addr);
707 if (r < d) {
708 cc = 1;
709 break;
710 } else if (r > d) {
711 cc = 2;
712 break;
713 }
714 addr++;
715 }
716 mask = (mask << 1) & 0xf;
717 r1 <<= 8;
718 }
719
720 HELPER_LOG("\n");
721 return cc;
722 }
723
get_address(CPUS390XState * env,int reg)724 static inline uint64_t get_address(CPUS390XState *env, int reg)
725 {
726 return wrap_address(env, env->regs[reg]);
727 }
728
729 /*
730 * Store the address to the given register, zeroing out unused leftmost
731 * bits in bit positions 32-63 (24-bit and 31-bit mode only).
732 */
set_address_zero(CPUS390XState * env,int reg,uint64_t address)733 static inline void set_address_zero(CPUS390XState *env, int reg,
734 uint64_t address)
735 {
736 if (env->psw.mask & PSW_MASK_64) {
737 env->regs[reg] = address;
738 } else {
739 if (!(env->psw.mask & PSW_MASK_32)) {
740 address &= 0x00ffffff;
741 } else {
742 address &= 0x7fffffff;
743 }
744 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
745 }
746 }
747
set_address(CPUS390XState * env,int reg,uint64_t address)748 static inline void set_address(CPUS390XState *env, int reg, uint64_t address)
749 {
750 if (env->psw.mask & PSW_MASK_64) {
751 /* 64-Bit mode */
752 env->regs[reg] = address;
753 } else {
754 if (!(env->psw.mask & PSW_MASK_32)) {
755 /* 24-Bit mode. According to the PoO it is implementation
756 dependent if bits 32-39 remain unchanged or are set to
757 zeros. Choose the former so that the function can also be
758 used for TRT. */
759 env->regs[reg] = deposit64(env->regs[reg], 0, 24, address);
760 } else {
761 /* 31-Bit mode. According to the PoO it is implementation
762 dependent if bit 32 remains unchanged or is set to zero.
763 Choose the latter so that the function can also be used for
764 TRT. */
765 address &= 0x7fffffff;
766 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
767 }
768 }
769 }
770
wrap_length32(CPUS390XState * env,uint64_t length)771 static inline uint64_t wrap_length32(CPUS390XState *env, uint64_t length)
772 {
773 if (!(env->psw.mask & PSW_MASK_64)) {
774 return (uint32_t)length;
775 }
776 return length;
777 }
778
wrap_length31(CPUS390XState * env,uint64_t length)779 static inline uint64_t wrap_length31(CPUS390XState *env, uint64_t length)
780 {
781 if (!(env->psw.mask & PSW_MASK_64)) {
782 /* 24-Bit and 31-Bit mode */
783 length &= 0x7fffffff;
784 }
785 return length;
786 }
787
get_length(CPUS390XState * env,int reg)788 static inline uint64_t get_length(CPUS390XState *env, int reg)
789 {
790 return wrap_length31(env, env->regs[reg]);
791 }
792
set_length(CPUS390XState * env,int reg,uint64_t length)793 static inline void set_length(CPUS390XState *env, int reg, uint64_t length)
794 {
795 if (env->psw.mask & PSW_MASK_64) {
796 /* 64-Bit mode */
797 env->regs[reg] = length;
798 } else {
799 /* 24-Bit and 31-Bit mode */
800 env->regs[reg] = deposit64(env->regs[reg], 0, 32, length);
801 }
802 }
803
804 /* search string (c is byte to search, r2 is string, r1 end of string) */
HELPER(srst)805 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
806 {
807 uintptr_t ra = GETPC();
808 uint64_t end, str;
809 uint32_t len;
810 uint8_t v, c = env->regs[0];
811
812 /* Bits 32-55 must contain all 0. */
813 if (env->regs[0] & 0xffffff00u) {
814 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
815 }
816
817 str = get_address(env, r2);
818 end = get_address(env, r1);
819
820 /* Lest we fail to service interrupts in a timely manner, limit the
821 amount of work we're willing to do. For now, let's cap at 8k. */
822 for (len = 0; len < 0x2000; ++len) {
823 if (str + len == end) {
824 /* Character not found. R1 & R2 are unmodified. */
825 env->cc_op = 2;
826 return;
827 }
828 v = cpu_ldub_data_ra(env, str + len, ra);
829 if (v == c) {
830 /* Character found. Set R1 to the location; R2 is unmodified. */
831 env->cc_op = 1;
832 set_address(env, r1, str + len);
833 return;
834 }
835 }
836
837 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
838 env->cc_op = 3;
839 set_address(env, r2, str + len);
840 }
841
HELPER(srstu)842 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2)
843 {
844 uintptr_t ra = GETPC();
845 uint32_t len;
846 uint16_t v, c = env->regs[0];
847 uint64_t end, str, adj_end;
848
849 /* Bits 32-47 of R0 must be zero. */
850 if (env->regs[0] & 0xffff0000u) {
851 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
852 }
853
854 str = get_address(env, r2);
855 end = get_address(env, r1);
856
857 /* If the LSB of the two addresses differ, use one extra byte. */
858 adj_end = end + ((str ^ end) & 1);
859
860 /* Lest we fail to service interrupts in a timely manner, limit the
861 amount of work we're willing to do. For now, let's cap at 8k. */
862 for (len = 0; len < 0x2000; len += 2) {
863 if (str + len == adj_end) {
864 /* End of input found. */
865 env->cc_op = 2;
866 return;
867 }
868 v = cpu_lduw_data_ra(env, str + len, ra);
869 if (v == c) {
870 /* Character found. Set R1 to the location; R2 is unmodified. */
871 env->cc_op = 1;
872 set_address(env, r1, str + len);
873 return;
874 }
875 }
876
877 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
878 env->cc_op = 3;
879 set_address(env, r2, str + len);
880 }
881
882 /* unsigned string compare (c is string terminator) */
HELPER(clst)883 Int128 HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
884 {
885 uintptr_t ra = GETPC();
886 uint32_t len;
887
888 c = c & 0xff;
889 s1 = wrap_address(env, s1);
890 s2 = wrap_address(env, s2);
891
892 /* Lest we fail to service interrupts in a timely manner, limit the
893 amount of work we're willing to do. For now, let's cap at 8k. */
894 for (len = 0; len < 0x2000; ++len) {
895 uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra);
896 uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra);
897 if (v1 == v2) {
898 if (v1 == c) {
899 /* Equal. CC=0, and don't advance the registers. */
900 env->cc_op = 0;
901 return int128_make128(s2, s1);
902 }
903 } else {
904 /* Unequal. CC={1,2}, and advance the registers. Note that
905 the terminator need not be zero, but the string that contains
906 the terminator is by definition "low". */
907 env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
908 return int128_make128(s2 + len, s1 + len);
909 }
910 }
911
912 /* CPU-determined bytes equal; advance the registers. */
913 env->cc_op = 3;
914 return int128_make128(s2 + len, s1 + len);
915 }
916
917 /* move page */
HELPER(mvpg)918 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint32_t r1, uint32_t r2)
919 {
920 const uint64_t src = get_address(env, r2) & TARGET_PAGE_MASK;
921 const uint64_t dst = get_address(env, r1) & TARGET_PAGE_MASK;
922 const int mmu_idx = s390x_env_mmu_index(env, false);
923 const bool f = extract64(r0, 11, 1);
924 const bool s = extract64(r0, 10, 1);
925 const bool cco = extract64(r0, 8, 1);
926 uintptr_t ra = GETPC();
927 S390Access srca, desta;
928 int exc;
929
930 if ((f && s) || extract64(r0, 12, 4)) {
931 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC());
932 }
933
934 /*
935 * We always manually handle exceptions such that we can properly store
936 * r1/r2 to the lowcore on page-translation exceptions.
937 *
938 * TODO: Access key handling
939 */
940 exc = access_prepare_nf(&srca, env, true, src, TARGET_PAGE_SIZE,
941 MMU_DATA_LOAD, mmu_idx, ra);
942 if (exc) {
943 if (cco) {
944 return 2;
945 }
946 goto inject_exc;
947 }
948 exc = access_prepare_nf(&desta, env, true, dst, TARGET_PAGE_SIZE,
949 MMU_DATA_STORE, mmu_idx, ra);
950 if (exc) {
951 if (cco && exc != PGM_PROTECTION) {
952 return 1;
953 }
954 goto inject_exc;
955 }
956 access_memmove(env, &desta, &srca, ra);
957 return 0; /* data moved */
958 inject_exc:
959 #if !defined(CONFIG_USER_ONLY)
960 if (exc != PGM_ADDRESSING) {
961 stq_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, trans_exc_code),
962 env->tlb_fill_tec);
963 }
964 if (exc == PGM_PAGE_TRANS) {
965 stb_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, op_access_id),
966 r1 << 4 | r2);
967 }
968 #endif
969 tcg_s390_program_interrupt(env, exc, ra);
970 }
971
972 /* string copy */
HELPER(mvst)973 uint32_t HELPER(mvst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
974 {
975 const int mmu_idx = s390x_env_mmu_index(env, false);
976 const uint64_t d = get_address(env, r1);
977 const uint64_t s = get_address(env, r2);
978 const uint8_t c = env->regs[0];
979 const int len = MIN(-(d | TARGET_PAGE_MASK), -(s | TARGET_PAGE_MASK));
980 S390Access srca, desta;
981 uintptr_t ra = GETPC();
982 int i;
983
984 if (env->regs[0] & 0xffffff00ull) {
985 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
986 }
987
988 /*
989 * Our access should not exceed single pages, as we must not report access
990 * exceptions exceeding the actually copied range (which we don't know at
991 * this point). We might over-indicate watchpoints within the pages
992 * (if we ever care, we have to limit processing to a single byte).
993 */
994 access_prepare(&srca, env, s, len, MMU_DATA_LOAD, mmu_idx, ra);
995 access_prepare(&desta, env, d, len, MMU_DATA_STORE, mmu_idx, ra);
996
997 set_helper_retaddr(ra);
998 for (i = 0; i < len; i++) {
999 const uint8_t v = access_get_byte(env, &srca, i, ra);
1000
1001 access_set_byte(env, &desta, i, v, ra);
1002 if (v == c) {
1003 clear_helper_retaddr();
1004 set_address_zero(env, r1, d + i);
1005 return 1;
1006 }
1007 }
1008 clear_helper_retaddr();
1009 set_address_zero(env, r1, d + len);
1010 set_address_zero(env, r2, s + len);
1011 return 3;
1012 }
1013
1014 /* load access registers r1 to r3 from memory at a2 */
HELPER(lam)1015 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1016 {
1017 uintptr_t ra = GETPC();
1018 int i;
1019
1020 if (a2 & 0x3) {
1021 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1022 }
1023
1024 for (i = r1;; i = (i + 1) % 16) {
1025 env->aregs[i] = cpu_ldl_data_ra(env, a2, ra);
1026 a2 += 4;
1027
1028 if (i == r3) {
1029 break;
1030 }
1031 }
1032 }
1033
1034 /* store access registers r1 to r3 in memory at a2 */
HELPER(stam)1035 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1036 {
1037 uintptr_t ra = GETPC();
1038 int i;
1039
1040 if (a2 & 0x3) {
1041 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1042 }
1043
1044 for (i = r1;; i = (i + 1) % 16) {
1045 cpu_stl_data_ra(env, a2, env->aregs[i], ra);
1046 a2 += 4;
1047
1048 if (i == r3) {
1049 break;
1050 }
1051 }
1052 }
1053
1054 /* move long helper */
do_mvcl(CPUS390XState * env,uint64_t * dest,uint64_t * destlen,uint64_t * src,uint64_t * srclen,uint16_t pad,int wordsize,uintptr_t ra)1055 static inline uint32_t do_mvcl(CPUS390XState *env,
1056 uint64_t *dest, uint64_t *destlen,
1057 uint64_t *src, uint64_t *srclen,
1058 uint16_t pad, int wordsize, uintptr_t ra)
1059 {
1060 const int mmu_idx = s390x_env_mmu_index(env, false);
1061 int len = MIN(*destlen, -(*dest | TARGET_PAGE_MASK));
1062 S390Access srca, desta;
1063 int i, cc;
1064
1065 if (*destlen == *srclen) {
1066 cc = 0;
1067 } else if (*destlen < *srclen) {
1068 cc = 1;
1069 } else {
1070 cc = 2;
1071 }
1072
1073 if (!*destlen) {
1074 return cc;
1075 }
1076
1077 /*
1078 * Only perform one type of type of operation (move/pad) at a time.
1079 * Stay within single pages.
1080 */
1081 if (*srclen) {
1082 /* Copy the src array */
1083 len = MIN(MIN(*srclen, -(*src | TARGET_PAGE_MASK)), len);
1084 *destlen -= len;
1085 *srclen -= len;
1086 access_prepare(&srca, env, *src, len, MMU_DATA_LOAD, mmu_idx, ra);
1087 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1088 access_memmove(env, &desta, &srca, ra);
1089 *src = wrap_address(env, *src + len);
1090 *dest = wrap_address(env, *dest + len);
1091 } else if (wordsize == 1) {
1092 /* Pad the remaining area */
1093 *destlen -= len;
1094 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1095 access_memset(env, &desta, pad, ra);
1096 *dest = wrap_address(env, *dest + len);
1097 } else {
1098 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1099 set_helper_retaddr(ra);
1100
1101 /* The remaining length selects the padding byte. */
1102 for (i = 0; i < len; (*destlen)--, i++) {
1103 if (*destlen & 1) {
1104 access_set_byte(env, &desta, i, pad, ra);
1105 } else {
1106 access_set_byte(env, &desta, i, pad >> 8, ra);
1107 }
1108 }
1109 clear_helper_retaddr();
1110 *dest = wrap_address(env, *dest + len);
1111 }
1112
1113 return *destlen ? 3 : cc;
1114 }
1115
1116 /* move long */
HELPER(mvcl)1117 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
1118 {
1119 const int mmu_idx = s390x_env_mmu_index(env, false);
1120 uintptr_t ra = GETPC();
1121 uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
1122 uint64_t dest = get_address(env, r1);
1123 uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
1124 uint64_t src = get_address(env, r2);
1125 uint8_t pad = env->regs[r2 + 1] >> 24;
1126 CPUState *cs = env_cpu(env);
1127 S390Access srca, desta;
1128 uint32_t cc, cur_len;
1129
1130 if (is_destructive_overlap(env, dest, src, MIN(srclen, destlen))) {
1131 cc = 3;
1132 } else if (srclen == destlen) {
1133 cc = 0;
1134 } else if (destlen < srclen) {
1135 cc = 1;
1136 } else {
1137 cc = 2;
1138 }
1139
1140 /* We might have to zero-out some bits even if there was no action. */
1141 if (unlikely(!destlen || cc == 3)) {
1142 set_address_zero(env, r2, src);
1143 set_address_zero(env, r1, dest);
1144 return cc;
1145 } else if (!srclen) {
1146 set_address_zero(env, r2, src);
1147 }
1148
1149 /*
1150 * Only perform one type of type of operation (move/pad) in one step.
1151 * Stay within single pages.
1152 */
1153 while (destlen) {
1154 cur_len = MIN(destlen, -(dest | TARGET_PAGE_MASK));
1155 if (!srclen) {
1156 access_prepare(&desta, env, dest, cur_len,
1157 MMU_DATA_STORE, mmu_idx, ra);
1158 access_memset(env, &desta, pad, ra);
1159 } else {
1160 cur_len = MIN(MIN(srclen, -(src | TARGET_PAGE_MASK)), cur_len);
1161
1162 access_prepare(&srca, env, src, cur_len,
1163 MMU_DATA_LOAD, mmu_idx, ra);
1164 access_prepare(&desta, env, dest, cur_len,
1165 MMU_DATA_STORE, mmu_idx, ra);
1166 access_memmove(env, &desta, &srca, ra);
1167 src = wrap_address(env, src + cur_len);
1168 srclen -= cur_len;
1169 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen);
1170 set_address_zero(env, r2, src);
1171 }
1172 dest = wrap_address(env, dest + cur_len);
1173 destlen -= cur_len;
1174 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen);
1175 set_address_zero(env, r1, dest);
1176
1177 /*
1178 * MVCL is interruptible. Return to the main loop if requested after
1179 * writing back all state to registers. If no interrupt will get
1180 * injected, we'll end up back in this handler and continue processing
1181 * the remaining parts.
1182 */
1183 if (destlen && unlikely(cpu_loop_exit_requested(cs))) {
1184 cpu_loop_exit_restore(cs, ra);
1185 }
1186 }
1187 return cc;
1188 }
1189
1190 /* move long extended */
HELPER(mvcle)1191 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1192 uint32_t r3)
1193 {
1194 uintptr_t ra = GETPC();
1195 uint64_t destlen = get_length(env, r1 + 1);
1196 uint64_t dest = get_address(env, r1);
1197 uint64_t srclen = get_length(env, r3 + 1);
1198 uint64_t src = get_address(env, r3);
1199 uint8_t pad = a2;
1200 uint32_t cc;
1201
1202 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra);
1203
1204 set_length(env, r1 + 1, destlen);
1205 set_length(env, r3 + 1, srclen);
1206 set_address(env, r1, dest);
1207 set_address(env, r3, src);
1208
1209 return cc;
1210 }
1211
1212 /* move long unicode */
HELPER(mvclu)1213 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1214 uint32_t r3)
1215 {
1216 uintptr_t ra = GETPC();
1217 uint64_t destlen = get_length(env, r1 + 1);
1218 uint64_t dest = get_address(env, r1);
1219 uint64_t srclen = get_length(env, r3 + 1);
1220 uint64_t src = get_address(env, r3);
1221 uint16_t pad = a2;
1222 uint32_t cc;
1223
1224 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra);
1225
1226 set_length(env, r1 + 1, destlen);
1227 set_length(env, r3 + 1, srclen);
1228 set_address(env, r1, dest);
1229 set_address(env, r3, src);
1230
1231 return cc;
1232 }
1233
1234 /* compare logical long helper */
do_clcl(CPUS390XState * env,uint64_t * src1,uint64_t * src1len,uint64_t * src3,uint64_t * src3len,uint16_t pad,uint64_t limit,int wordsize,uintptr_t ra)1235 static inline uint32_t do_clcl(CPUS390XState *env,
1236 uint64_t *src1, uint64_t *src1len,
1237 uint64_t *src3, uint64_t *src3len,
1238 uint16_t pad, uint64_t limit,
1239 int wordsize, uintptr_t ra)
1240 {
1241 uint64_t len = MAX(*src1len, *src3len);
1242 uint32_t cc = 0;
1243
1244 check_alignment(env, *src1len | *src3len, wordsize, ra);
1245
1246 if (!len) {
1247 return cc;
1248 }
1249
1250 /* Lest we fail to service interrupts in a timely manner, limit the
1251 amount of work we're willing to do. */
1252 if (len > limit) {
1253 len = limit;
1254 cc = 3;
1255 }
1256
1257 for (; len; len -= wordsize) {
1258 uint16_t v1 = pad;
1259 uint16_t v3 = pad;
1260
1261 if (*src1len) {
1262 v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra);
1263 }
1264 if (*src3len) {
1265 v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra);
1266 }
1267
1268 if (v1 != v3) {
1269 cc = (v1 < v3) ? 1 : 2;
1270 break;
1271 }
1272
1273 if (*src1len) {
1274 *src1 += wordsize;
1275 *src1len -= wordsize;
1276 }
1277 if (*src3len) {
1278 *src3 += wordsize;
1279 *src3len -= wordsize;
1280 }
1281 }
1282
1283 return cc;
1284 }
1285
1286
1287 /* compare logical long */
HELPER(clcl)1288 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
1289 {
1290 uintptr_t ra = GETPC();
1291 uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24);
1292 uint64_t src1 = get_address(env, r1);
1293 uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24);
1294 uint64_t src3 = get_address(env, r2);
1295 uint8_t pad = env->regs[r2 + 1] >> 24;
1296 uint32_t cc;
1297
1298 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra);
1299
1300 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len);
1301 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len);
1302 set_address(env, r1, src1);
1303 set_address(env, r2, src3);
1304
1305 return cc;
1306 }
1307
1308 /* compare logical long extended memcompare insn with padding */
HELPER(clcle)1309 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1310 uint32_t r3)
1311 {
1312 uintptr_t ra = GETPC();
1313 uint64_t src1len = get_length(env, r1 + 1);
1314 uint64_t src1 = get_address(env, r1);
1315 uint64_t src3len = get_length(env, r3 + 1);
1316 uint64_t src3 = get_address(env, r3);
1317 uint8_t pad = a2;
1318 uint32_t cc;
1319
1320 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra);
1321
1322 set_length(env, r1 + 1, src1len);
1323 set_length(env, r3 + 1, src3len);
1324 set_address(env, r1, src1);
1325 set_address(env, r3, src3);
1326
1327 return cc;
1328 }
1329
1330 /* compare logical long unicode memcompare insn with padding */
HELPER(clclu)1331 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1332 uint32_t r3)
1333 {
1334 uintptr_t ra = GETPC();
1335 uint64_t src1len = get_length(env, r1 + 1);
1336 uint64_t src1 = get_address(env, r1);
1337 uint64_t src3len = get_length(env, r3 + 1);
1338 uint64_t src3 = get_address(env, r3);
1339 uint16_t pad = a2;
1340 uint32_t cc = 0;
1341
1342 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra);
1343
1344 set_length(env, r1 + 1, src1len);
1345 set_length(env, r3 + 1, src3len);
1346 set_address(env, r1, src1);
1347 set_address(env, r3, src3);
1348
1349 return cc;
1350 }
1351
1352 /* checksum */
HELPER(cksm)1353 Int128 HELPER(cksm)(CPUS390XState *env, uint64_t r1,
1354 uint64_t src, uint64_t src_len)
1355 {
1356 uintptr_t ra = GETPC();
1357 uint64_t max_len, len;
1358 uint64_t cksm = (uint32_t)r1;
1359
1360 /* Lest we fail to service interrupts in a timely manner, limit the
1361 amount of work we're willing to do. For now, let's cap at 8k. */
1362 max_len = (src_len > 0x2000 ? 0x2000 : src_len);
1363
1364 /* Process full words as available. */
1365 for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
1366 cksm += (uint32_t)cpu_ldl_data_ra(env, src, ra);
1367 }
1368
1369 switch (max_len - len) {
1370 case 1:
1371 cksm += cpu_ldub_data_ra(env, src, ra) << 24;
1372 len += 1;
1373 break;
1374 case 2:
1375 cksm += cpu_lduw_data_ra(env, src, ra) << 16;
1376 len += 2;
1377 break;
1378 case 3:
1379 cksm += cpu_lduw_data_ra(env, src, ra) << 16;
1380 cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8;
1381 len += 3;
1382 break;
1383 }
1384
1385 /* Fold the carry from the checksum. Note that we can see carry-out
1386 during folding more than once (but probably not more than twice). */
1387 while (cksm > 0xffffffffull) {
1388 cksm = (uint32_t)cksm + (cksm >> 32);
1389 }
1390
1391 /* Indicate whether or not we've processed everything. */
1392 env->cc_op = (len == src_len ? 0 : 3);
1393
1394 /* Return both cksm and processed length. */
1395 return int128_make128(cksm, len);
1396 }
1397
HELPER(pack)1398 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src)
1399 {
1400 uintptr_t ra = GETPC();
1401 int len_dest = len >> 4;
1402 int len_src = len & 0xf;
1403 uint8_t b;
1404
1405 dest += len_dest;
1406 src += len_src;
1407
1408 /* last byte is special, it only flips the nibbles */
1409 b = cpu_ldub_data_ra(env, src, ra);
1410 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1411 src--;
1412 len_src--;
1413
1414 /* now pack every value */
1415 while (len_dest > 0) {
1416 b = 0;
1417
1418 if (len_src >= 0) {
1419 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1420 src--;
1421 len_src--;
1422 }
1423 if (len_src >= 0) {
1424 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1425 src--;
1426 len_src--;
1427 }
1428
1429 len_dest--;
1430 dest--;
1431 cpu_stb_data_ra(env, dest, b, ra);
1432 }
1433 }
1434
do_pkau(CPUS390XState * env,uint64_t dest,uint64_t src,uint32_t srclen,int ssize,uintptr_t ra)1435 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src,
1436 uint32_t srclen, int ssize, uintptr_t ra)
1437 {
1438 int i;
1439 /* The destination operand is always 16 bytes long. */
1440 const int destlen = 16;
1441
1442 /* The operands are processed from right to left. */
1443 src += srclen - 1;
1444 dest += destlen - 1;
1445
1446 for (i = 0; i < destlen; i++) {
1447 uint8_t b = 0;
1448
1449 /* Start with a positive sign */
1450 if (i == 0) {
1451 b = 0xc;
1452 } else if (srclen > ssize) {
1453 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1454 src -= ssize;
1455 srclen -= ssize;
1456 }
1457
1458 if (srclen > ssize) {
1459 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1460 src -= ssize;
1461 srclen -= ssize;
1462 }
1463
1464 cpu_stb_data_ra(env, dest, b, ra);
1465 dest--;
1466 }
1467 }
1468
1469
HELPER(pka)1470 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src,
1471 uint32_t srclen)
1472 {
1473 do_pkau(env, dest, src, srclen, 1, GETPC());
1474 }
1475
HELPER(pku)1476 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src,
1477 uint32_t srclen)
1478 {
1479 do_pkau(env, dest, src, srclen, 2, GETPC());
1480 }
1481
HELPER(unpk)1482 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
1483 uint64_t src)
1484 {
1485 uintptr_t ra = GETPC();
1486 int len_dest = len >> 4;
1487 int len_src = len & 0xf;
1488 uint8_t b;
1489 int second_nibble = 0;
1490
1491 dest += len_dest;
1492 src += len_src;
1493
1494 /* last byte is special, it only flips the nibbles */
1495 b = cpu_ldub_data_ra(env, src, ra);
1496 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1497 src--;
1498 len_src--;
1499
1500 /* now pad every nibble with 0xf0 */
1501
1502 while (len_dest > 0) {
1503 uint8_t cur_byte = 0;
1504
1505 if (len_src > 0) {
1506 cur_byte = cpu_ldub_data_ra(env, src, ra);
1507 }
1508
1509 len_dest--;
1510 dest--;
1511
1512 /* only advance one nibble at a time */
1513 if (second_nibble) {
1514 cur_byte >>= 4;
1515 len_src--;
1516 src--;
1517 }
1518 second_nibble = !second_nibble;
1519
1520 /* digit */
1521 cur_byte = (cur_byte & 0xf);
1522 /* zone bits */
1523 cur_byte |= 0xf0;
1524
1525 cpu_stb_data_ra(env, dest, cur_byte, ra);
1526 }
1527 }
1528
do_unpkau(CPUS390XState * env,uint64_t dest,uint32_t destlen,int dsize,uint64_t src,uintptr_t ra)1529 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest,
1530 uint32_t destlen, int dsize, uint64_t src,
1531 uintptr_t ra)
1532 {
1533 int i;
1534 uint32_t cc;
1535 uint8_t b;
1536 /* The source operand is always 16 bytes long. */
1537 const int srclen = 16;
1538
1539 /* The operands are processed from right to left. */
1540 src += srclen - 1;
1541 dest += destlen - dsize;
1542
1543 /* Check for the sign. */
1544 b = cpu_ldub_data_ra(env, src, ra);
1545 src--;
1546 switch (b & 0xf) {
1547 case 0xa:
1548 case 0xc:
1549 case 0xe ... 0xf:
1550 cc = 0; /* plus */
1551 break;
1552 case 0xb:
1553 case 0xd:
1554 cc = 1; /* minus */
1555 break;
1556 default:
1557 case 0x0 ... 0x9:
1558 cc = 3; /* invalid */
1559 break;
1560 }
1561
1562 /* Now pad every nibble with 0x30, advancing one nibble at a time. */
1563 for (i = 0; i < destlen; i += dsize) {
1564 if (i == (31 * dsize)) {
1565 /* If length is 32/64 bytes, the leftmost byte is 0. */
1566 b = 0;
1567 } else if (i % (2 * dsize)) {
1568 b = cpu_ldub_data_ra(env, src, ra);
1569 src--;
1570 } else {
1571 b >>= 4;
1572 }
1573 cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra);
1574 dest -= dsize;
1575 }
1576
1577 return cc;
1578 }
1579
HELPER(unpka)1580 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1581 uint64_t src)
1582 {
1583 return do_unpkau(env, dest, destlen, 1, src, GETPC());
1584 }
1585
HELPER(unpku)1586 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1587 uint64_t src)
1588 {
1589 return do_unpkau(env, dest, destlen, 2, src, GETPC());
1590 }
1591
HELPER(tp)1592 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen)
1593 {
1594 uintptr_t ra = GETPC();
1595 uint32_t cc = 0;
1596 int i;
1597
1598 for (i = 0; i < destlen; i++) {
1599 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra);
1600 /* digit */
1601 cc |= (b & 0xf0) > 0x90 ? 2 : 0;
1602
1603 if (i == (destlen - 1)) {
1604 /* sign */
1605 cc |= (b & 0xf) < 0xa ? 1 : 0;
1606 } else {
1607 /* digit */
1608 cc |= (b & 0xf) > 0x9 ? 2 : 0;
1609 }
1610 }
1611
1612 return cc;
1613 }
1614
do_helper_tr(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1615 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array,
1616 uint64_t trans, uintptr_t ra)
1617 {
1618 uint32_t i;
1619
1620 for (i = 0; i <= len; i++) {
1621 uint8_t byte = cpu_ldub_data_ra(env, array + i, ra);
1622 uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1623 cpu_stb_data_ra(env, array + i, new_byte, ra);
1624 }
1625
1626 return env->cc_op;
1627 }
1628
HELPER(tr)1629 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
1630 uint64_t trans)
1631 {
1632 do_helper_tr(env, len, array, trans, GETPC());
1633 }
1634
HELPER(tre)1635 Int128 HELPER(tre)(CPUS390XState *env, uint64_t array,
1636 uint64_t len, uint64_t trans)
1637 {
1638 uintptr_t ra = GETPC();
1639 uint8_t end = env->regs[0] & 0xff;
1640 uint64_t l = len;
1641 uint64_t i;
1642 uint32_t cc = 0;
1643
1644 if (!(env->psw.mask & PSW_MASK_64)) {
1645 array &= 0x7fffffff;
1646 l = (uint32_t)l;
1647 }
1648
1649 /* Lest we fail to service interrupts in a timely manner, limit the
1650 amount of work we're willing to do. For now, let's cap at 8k. */
1651 if (l > 0x2000) {
1652 l = 0x2000;
1653 cc = 3;
1654 }
1655
1656 for (i = 0; i < l; i++) {
1657 uint8_t byte, new_byte;
1658
1659 byte = cpu_ldub_data_ra(env, array + i, ra);
1660
1661 if (byte == end) {
1662 cc = 1;
1663 break;
1664 }
1665
1666 new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1667 cpu_stb_data_ra(env, array + i, new_byte, ra);
1668 }
1669
1670 env->cc_op = cc;
1671 return int128_make128(len - i, array + i);
1672 }
1673
do_helper_trt(CPUS390XState * env,int len,uint64_t array,uint64_t trans,int inc,uintptr_t ra)1674 static inline uint32_t do_helper_trt(CPUS390XState *env, int len,
1675 uint64_t array, uint64_t trans,
1676 int inc, uintptr_t ra)
1677 {
1678 int i;
1679
1680 for (i = 0; i <= len; i++) {
1681 uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra);
1682 uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra);
1683
1684 if (sbyte != 0) {
1685 set_address(env, 1, array + i * inc);
1686 env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte);
1687 return (i == len) ? 2 : 1;
1688 }
1689 }
1690
1691 return 0;
1692 }
1693
do_helper_trt_fwd(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1694 static uint32_t do_helper_trt_fwd(CPUS390XState *env, uint32_t len,
1695 uint64_t array, uint64_t trans,
1696 uintptr_t ra)
1697 {
1698 return do_helper_trt(env, len, array, trans, 1, ra);
1699 }
1700
HELPER(trt)1701 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array,
1702 uint64_t trans)
1703 {
1704 return do_helper_trt(env, len, array, trans, 1, GETPC());
1705 }
1706
do_helper_trt_bkwd(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1707 static uint32_t do_helper_trt_bkwd(CPUS390XState *env, uint32_t len,
1708 uint64_t array, uint64_t trans,
1709 uintptr_t ra)
1710 {
1711 return do_helper_trt(env, len, array, trans, -1, ra);
1712 }
1713
HELPER(trtr)1714 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array,
1715 uint64_t trans)
1716 {
1717 return do_helper_trt(env, len, array, trans, -1, GETPC());
1718 }
1719
1720 /* Translate one/two to one/two */
HELPER(trXX)1721 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2,
1722 uint32_t tst, uint32_t sizes)
1723 {
1724 uintptr_t ra = GETPC();
1725 int dsize = (sizes & 1) ? 1 : 2;
1726 int ssize = (sizes & 2) ? 1 : 2;
1727 uint64_t tbl = get_address(env, 1);
1728 uint64_t dst = get_address(env, r1);
1729 uint64_t len = get_length(env, r1 + 1);
1730 uint64_t src = get_address(env, r2);
1731 uint32_t cc = 3;
1732 int i;
1733
1734 /* The lower address bits of TBL are ignored. For TROO, TROT, it's
1735 the low 3 bits (double-word aligned). For TRTO, TRTT, it's either
1736 the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */
1737 if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) {
1738 tbl &= -4096;
1739 } else {
1740 tbl &= -8;
1741 }
1742
1743 check_alignment(env, len, ssize, ra);
1744
1745 /* Lest we fail to service interrupts in a timely manner, */
1746 /* limit the amount of work we're willing to do. */
1747 for (i = 0; i < 0x2000; i++) {
1748 uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra);
1749 uint64_t tble = tbl + (sval * dsize);
1750 uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra);
1751 if (dval == tst) {
1752 cc = 1;
1753 break;
1754 }
1755 cpu_stsize_data_ra(env, dst, dval, dsize, ra);
1756
1757 len -= ssize;
1758 src += ssize;
1759 dst += dsize;
1760
1761 if (len == 0) {
1762 cc = 0;
1763 break;
1764 }
1765 }
1766
1767 set_address(env, r1, dst);
1768 set_length(env, r1 + 1, len);
1769 set_address(env, r2, src);
1770
1771 return cc;
1772 }
1773
do_csst(CPUS390XState * env,uint32_t r3,uint64_t a1,uint64_t a2,bool parallel)1774 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
1775 uint64_t a2, bool parallel)
1776 {
1777 uint32_t mem_idx = s390x_env_mmu_index(env, false);
1778 MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, mem_idx);
1779 MemOpIdx oi8 = make_memop_idx(MO_TE | MO_64, mem_idx);
1780 MemOpIdx oi4 = make_memop_idx(MO_TE | MO_32, mem_idx);
1781 MemOpIdx oi2 = make_memop_idx(MO_TE | MO_16, mem_idx);
1782 MemOpIdx oi1 = make_memop_idx(MO_8, mem_idx);
1783 uintptr_t ra = GETPC();
1784 uint32_t fc = extract32(env->regs[0], 0, 8);
1785 uint32_t sc = extract32(env->regs[0], 8, 8);
1786 uint64_t pl = get_address(env, 1) & -16;
1787 uint64_t svh, svl;
1788 uint32_t cc;
1789
1790 /* Sanity check the function code and storage characteristic. */
1791 if (fc > 1 || sc > 3) {
1792 if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) {
1793 goto spec_exception;
1794 }
1795 if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) {
1796 goto spec_exception;
1797 }
1798 }
1799
1800 /* Sanity check the alignments. */
1801 if (extract32(a1, 0, fc + 2) || extract32(a2, 0, sc)) {
1802 goto spec_exception;
1803 }
1804
1805 /* Sanity check writability of the store address. */
1806 probe_write(env, a2, 1 << sc, mem_idx, ra);
1807
1808 /*
1809 * Note that the compare-and-swap is atomic, and the store is atomic,
1810 * but the complete operation is not. Therefore we do not need to
1811 * assert serial context in order to implement this. That said,
1812 * restart early if we can't support either operation that is supposed
1813 * to be atomic.
1814 */
1815 if (parallel) {
1816 uint32_t max = 2;
1817 #ifdef CONFIG_ATOMIC64
1818 max = 3;
1819 #endif
1820 if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) ||
1821 (HAVE_ATOMIC128_RW ? 0 : sc > max)) {
1822 cpu_loop_exit_atomic(env_cpu(env), ra);
1823 }
1824 }
1825
1826 /*
1827 * All loads happen before all stores. For simplicity, load the entire
1828 * store value area from the parameter list.
1829 */
1830 svh = cpu_ldq_mmu(env, pl + 16, oi8, ra);
1831 svl = cpu_ldq_mmu(env, pl + 24, oi8, ra);
1832
1833 switch (fc) {
1834 case 0:
1835 {
1836 uint32_t nv = cpu_ldl_mmu(env, pl, oi4, ra);
1837 uint32_t cv = env->regs[r3];
1838 uint32_t ov;
1839
1840 if (parallel) {
1841 ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi4, ra);
1842 } else {
1843 ov = cpu_ldl_mmu(env, a1, oi4, ra);
1844 cpu_stl_mmu(env, a1, (ov == cv ? nv : ov), oi4, ra);
1845 }
1846 cc = (ov != cv);
1847 env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov);
1848 }
1849 break;
1850
1851 case 1:
1852 {
1853 uint64_t nv = cpu_ldq_mmu(env, pl, oi8, ra);
1854 uint64_t cv = env->regs[r3];
1855 uint64_t ov;
1856
1857 if (parallel) {
1858 #ifdef CONFIG_ATOMIC64
1859 ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi8, ra);
1860 #else
1861 /* Note that we asserted !parallel above. */
1862 g_assert_not_reached();
1863 #endif
1864 } else {
1865 ov = cpu_ldq_mmu(env, a1, oi8, ra);
1866 cpu_stq_mmu(env, a1, (ov == cv ? nv : ov), oi8, ra);
1867 }
1868 cc = (ov != cv);
1869 env->regs[r3] = ov;
1870 }
1871 break;
1872
1873 case 2:
1874 {
1875 Int128 nv = cpu_ld16_mmu(env, pl, oi16, ra);
1876 Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1877 Int128 ov;
1878
1879 if (!parallel) {
1880 ov = cpu_ld16_mmu(env, a1, oi16, ra);
1881 cc = !int128_eq(ov, cv);
1882 if (cc) {
1883 nv = ov;
1884 }
1885 cpu_st16_mmu(env, a1, nv, oi16, ra);
1886 } else if (HAVE_CMPXCHG128) {
1887 ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi16, ra);
1888 cc = !int128_eq(ov, cv);
1889 } else {
1890 /* Note that we asserted !parallel above. */
1891 g_assert_not_reached();
1892 }
1893
1894 env->regs[r3 + 0] = int128_gethi(ov);
1895 env->regs[r3 + 1] = int128_getlo(ov);
1896 }
1897 break;
1898
1899 default:
1900 g_assert_not_reached();
1901 }
1902
1903 /* Store only if the comparison succeeded. Note that above we use a pair
1904 of 64-bit big-endian loads, so for sc < 3 we must extract the value
1905 from the most-significant bits of svh. */
1906 if (cc == 0) {
1907 switch (sc) {
1908 case 0:
1909 cpu_stb_mmu(env, a2, svh >> 56, oi1, ra);
1910 break;
1911 case 1:
1912 cpu_stw_mmu(env, a2, svh >> 48, oi2, ra);
1913 break;
1914 case 2:
1915 cpu_stl_mmu(env, a2, svh >> 32, oi4, ra);
1916 break;
1917 case 3:
1918 cpu_stq_mmu(env, a2, svh, oi8, ra);
1919 break;
1920 case 4:
1921 cpu_st16_mmu(env, a2, int128_make128(svl, svh), oi16, ra);
1922 break;
1923 default:
1924 g_assert_not_reached();
1925 }
1926 }
1927
1928 return cc;
1929
1930 spec_exception:
1931 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1932 }
1933
HELPER(csst)1934 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2)
1935 {
1936 return do_csst(env, r3, a1, a2, false);
1937 }
1938
HELPER(csst_parallel)1939 uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1,
1940 uint64_t a2)
1941 {
1942 return do_csst(env, r3, a1, a2, true);
1943 }
1944
1945 #if !defined(CONFIG_USER_ONLY)
HELPER(lctlg)1946 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1947 {
1948 uintptr_t ra = GETPC();
1949 bool PERchanged = false;
1950 uint64_t src = a2;
1951 uint32_t i;
1952
1953 if (src & 0x7) {
1954 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1955 }
1956
1957 for (i = r1;; i = (i + 1) % 16) {
1958 uint64_t val = cpu_ldq_data_ra(env, src, ra);
1959 if (env->cregs[i] != val && i >= 9 && i <= 11) {
1960 PERchanged = true;
1961 }
1962 if (i == 0 && !(env->cregs[i] & CR0_CKC_SC) && (val & CR0_CKC_SC)) {
1963 BQL_LOCK_GUARD();
1964 tcg_s390_tod_updated(env_cpu(env), RUN_ON_CPU_NULL);
1965 }
1966 env->cregs[i] = val;
1967 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
1968 i, src, val);
1969 src += sizeof(uint64_t);
1970
1971 if (i == r3) {
1972 break;
1973 }
1974 }
1975
1976 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
1977 s390_cpu_recompute_watchpoints(env_cpu(env));
1978 }
1979
1980 tlb_flush(env_cpu(env));
1981 }
1982
HELPER(lctl)1983 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1984 {
1985 uintptr_t ra = GETPC();
1986 bool PERchanged = false;
1987 uint64_t src = a2;
1988 uint32_t i;
1989
1990 if (src & 0x3) {
1991 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1992 }
1993
1994 for (i = r1;; i = (i + 1) % 16) {
1995 uint32_t val = cpu_ldl_data_ra(env, src, ra);
1996 uint64_t val64 = deposit64(env->cregs[i], 0, 32, val);
1997 if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) {
1998 PERchanged = true;
1999 }
2000 if (i == 0 && !(env->cregs[i] & CR0_CKC_SC) && (val64 & CR0_CKC_SC)) {
2001 BQL_LOCK_GUARD();
2002 tcg_s390_tod_updated(env_cpu(env), RUN_ON_CPU_NULL);
2003 }
2004 env->cregs[i] = val64;
2005 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val);
2006 src += sizeof(uint32_t);
2007
2008 if (i == r3) {
2009 break;
2010 }
2011 }
2012
2013 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
2014 s390_cpu_recompute_watchpoints(env_cpu(env));
2015 }
2016
2017 tlb_flush(env_cpu(env));
2018 }
2019
HELPER(stctg)2020 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2021 {
2022 uintptr_t ra = GETPC();
2023 uint64_t dest = a2;
2024 uint32_t i;
2025
2026 if (dest & 0x7) {
2027 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2028 }
2029
2030 for (i = r1;; i = (i + 1) % 16) {
2031 cpu_stq_data_ra(env, dest, env->cregs[i], ra);
2032 dest += sizeof(uint64_t);
2033
2034 if (i == r3) {
2035 break;
2036 }
2037 }
2038 }
2039
HELPER(stctl)2040 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2041 {
2042 uintptr_t ra = GETPC();
2043 uint64_t dest = a2;
2044 uint32_t i;
2045
2046 if (dest & 0x3) {
2047 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2048 }
2049
2050 for (i = r1;; i = (i + 1) % 16) {
2051 cpu_stl_data_ra(env, dest, env->cregs[i], ra);
2052 dest += sizeof(uint32_t);
2053
2054 if (i == r3) {
2055 break;
2056 }
2057 }
2058 }
2059
HELPER(testblock)2060 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr)
2061 {
2062 uintptr_t ra = GETPC();
2063 int i;
2064
2065 real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK;
2066
2067 for (i = 0; i < TARGET_PAGE_SIZE; i += 8) {
2068 cpu_stq_mmuidx_ra(env, real_addr + i, 0, MMU_REAL_IDX, ra);
2069 }
2070
2071 return 0;
2072 }
2073
HELPER(tprot)2074 uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2)
2075 {
2076 S390CPU *cpu = env_archcpu(env);
2077 CPUState *cs = env_cpu(env);
2078
2079 /*
2080 * TODO: we currently don't handle all access protection types
2081 * (including access-list and key-controlled) as well as AR mode.
2082 */
2083 if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) {
2084 /* Fetching permitted; storing permitted */
2085 return 0;
2086 }
2087
2088 if (env->int_pgm_code == PGM_PROTECTION) {
2089 /* retry if reading is possible */
2090 cs->exception_index = -1;
2091 if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) {
2092 /* Fetching permitted; storing not permitted */
2093 return 1;
2094 }
2095 }
2096
2097 switch (env->int_pgm_code) {
2098 case PGM_PROTECTION:
2099 /* Fetching not permitted; storing not permitted */
2100 cs->exception_index = -1;
2101 return 2;
2102 case PGM_ADDRESSING:
2103 case PGM_TRANS_SPEC:
2104 /* exceptions forwarded to the guest */
2105 s390_cpu_virt_mem_handle_exc(cpu, GETPC());
2106 return 0;
2107 }
2108
2109 /* Translation not available */
2110 cs->exception_index = -1;
2111 return 3;
2112 }
2113
2114 /* insert storage key extended */
HELPER(iske)2115 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
2116 {
2117 static S390SKeysState *ss;
2118 static S390SKeysClass *skeyclass;
2119 uint64_t addr = wrap_address(env, r2);
2120 uint8_t key;
2121 int rc;
2122
2123 addr = mmu_real2abs(env, addr);
2124 if (!mmu_absolute_addr_valid(addr, false)) {
2125 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2126 }
2127
2128 if (unlikely(!ss)) {
2129 ss = s390_get_skeys_device();
2130 skeyclass = S390_SKEYS_GET_CLASS(ss);
2131 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2132 tlb_flush_all_cpus_synced(env_cpu(env));
2133 }
2134 }
2135
2136 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2137 if (rc) {
2138 return 0;
2139 }
2140 return key;
2141 }
2142
2143 /* set storage key extended */
HELPER(sske)2144 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
2145 {
2146 static S390SKeysState *ss;
2147 static S390SKeysClass *skeyclass;
2148 uint64_t addr = wrap_address(env, r2);
2149 uint8_t key;
2150
2151 addr = mmu_real2abs(env, addr);
2152 if (!mmu_absolute_addr_valid(addr, false)) {
2153 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2154 }
2155
2156 if (unlikely(!ss)) {
2157 ss = s390_get_skeys_device();
2158 skeyclass = S390_SKEYS_GET_CLASS(ss);
2159 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2160 tlb_flush_all_cpus_synced(env_cpu(env));
2161 }
2162 }
2163
2164 key = r1 & 0xfe;
2165 s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2166 /*
2167 * As we can only flush by virtual address and not all the entries
2168 * that point to a physical address we have to flush the whole TLB.
2169 */
2170 tlb_flush_all_cpus_synced(env_cpu(env));
2171 }
2172
2173 /* reset reference bit extended */
HELPER(rrbe)2174 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
2175 {
2176 uint64_t addr = wrap_address(env, r2);
2177 static S390SKeysState *ss;
2178 static S390SKeysClass *skeyclass;
2179 uint8_t re, key;
2180 int rc;
2181
2182 addr = mmu_real2abs(env, addr);
2183 if (!mmu_absolute_addr_valid(addr, false)) {
2184 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2185 }
2186
2187 if (unlikely(!ss)) {
2188 ss = s390_get_skeys_device();
2189 skeyclass = S390_SKEYS_GET_CLASS(ss);
2190 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2191 tlb_flush_all_cpus_synced(env_cpu(env));
2192 }
2193 }
2194
2195 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2196 if (rc) {
2197 return 0;
2198 }
2199
2200 re = key & (SK_R | SK_C);
2201 key &= ~SK_R;
2202
2203 rc = s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2204 if (rc) {
2205 return 0;
2206 }
2207 /*
2208 * As we can only flush by virtual address and not all the entries
2209 * that point to a physical address we have to flush the whole TLB.
2210 */
2211 tlb_flush_all_cpus_synced(env_cpu(env));
2212
2213 /*
2214 * cc
2215 *
2216 * 0 Reference bit zero; change bit zero
2217 * 1 Reference bit zero; change bit one
2218 * 2 Reference bit one; change bit zero
2219 * 3 Reference bit one; change bit one
2220 */
2221
2222 return re >> 1;
2223 }
2224
HELPER(mvcs)2225 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2,
2226 uint64_t key)
2227 {
2228 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2229 S390Access srca, desta;
2230 uintptr_t ra = GETPC();
2231 int cc = 0;
2232
2233 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
2234 __func__, l, a1, a2);
2235
2236 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) ||
2237 psw_as == AS_HOME || psw_as == AS_ACCREG) {
2238 s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2239 }
2240
2241 if (!psw_key_valid(env, (key >> 4) & 0xf)) {
2242 s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2243 }
2244
2245 l = wrap_length32(env, l);
2246 if (l > 256) {
2247 /* max 256 */
2248 l = 256;
2249 cc = 3;
2250 } else if (!l) {
2251 return cc;
2252 }
2253
2254 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_PRIMARY_IDX, ra);
2255 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_SECONDARY_IDX, ra);
2256 access_memmove(env, &desta, &srca, ra);
2257 return cc;
2258 }
2259
HELPER(mvcp)2260 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2,
2261 uint64_t key)
2262 {
2263 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2264 S390Access srca, desta;
2265 uintptr_t ra = GETPC();
2266 int cc = 0;
2267
2268 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
2269 __func__, l, a1, a2);
2270
2271 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) ||
2272 psw_as == AS_HOME || psw_as == AS_ACCREG) {
2273 s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2274 }
2275
2276 if (!psw_key_valid(env, (key >> 4) & 0xf)) {
2277 s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2278 }
2279
2280 l = wrap_length32(env, l);
2281 if (l > 256) {
2282 /* max 256 */
2283 l = 256;
2284 cc = 3;
2285 } else if (!l) {
2286 return cc;
2287 }
2288 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_SECONDARY_IDX, ra);
2289 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_PRIMARY_IDX, ra);
2290 access_memmove(env, &desta, &srca, ra);
2291 return cc;
2292 }
2293
HELPER(idte)2294 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4)
2295 {
2296 CPUState *cs = env_cpu(env);
2297 const uintptr_t ra = GETPC();
2298 uint64_t table, entry, raddr;
2299 uint16_t entries, i, index = 0;
2300
2301 if (r2 & 0xff000) {
2302 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2303 }
2304
2305 if (!(r2 & 0x800)) {
2306 /* invalidation-and-clearing operation */
2307 table = r1 & ASCE_ORIGIN;
2308 entries = (r2 & 0x7ff) + 1;
2309
2310 switch (r1 & ASCE_TYPE_MASK) {
2311 case ASCE_TYPE_REGION1:
2312 index = (r2 >> 53) & 0x7ff;
2313 break;
2314 case ASCE_TYPE_REGION2:
2315 index = (r2 >> 42) & 0x7ff;
2316 break;
2317 case ASCE_TYPE_REGION3:
2318 index = (r2 >> 31) & 0x7ff;
2319 break;
2320 case ASCE_TYPE_SEGMENT:
2321 index = (r2 >> 20) & 0x7ff;
2322 break;
2323 }
2324 for (i = 0; i < entries; i++) {
2325 /* addresses are not wrapped in 24/31bit mode but table index is */
2326 raddr = table + ((index + i) & 0x7ff) * sizeof(entry);
2327 entry = cpu_ldq_mmuidx_ra(env, raddr, MMU_REAL_IDX, ra);
2328 if (!(entry & REGION_ENTRY_I)) {
2329 /* we are allowed to not store if already invalid */
2330 entry |= REGION_ENTRY_I;
2331 cpu_stq_mmuidx_ra(env, raddr, entry, MMU_REAL_IDX, ra);
2332 }
2333 }
2334 }
2335
2336 /* We simply flush the complete tlb, therefore we can ignore r3. */
2337 if (m4 & 1) {
2338 tlb_flush(cs);
2339 } else {
2340 tlb_flush_all_cpus_synced(cs);
2341 }
2342 }
2343
2344 /* invalidate pte */
HELPER(ipte)2345 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr,
2346 uint32_t m4)
2347 {
2348 CPUState *cs = env_cpu(env);
2349 const uintptr_t ra = GETPC();
2350 uint64_t page = vaddr & TARGET_PAGE_MASK;
2351 uint64_t pte_addr, pte;
2352
2353 /* Compute the page table entry address */
2354 pte_addr = (pto & SEGMENT_ENTRY_ORIGIN);
2355 pte_addr += VADDR_PAGE_TX(vaddr) * 8;
2356
2357 /* Mark the page table entry as invalid */
2358 pte = cpu_ldq_mmuidx_ra(env, pte_addr, MMU_REAL_IDX, ra);
2359 pte |= PAGE_ENTRY_I;
2360 cpu_stq_mmuidx_ra(env, pte_addr, pte, MMU_REAL_IDX, ra);
2361
2362 /* XXX we exploit the fact that Linux passes the exact virtual
2363 address here - it's not obliged to! */
2364 if (m4 & 1) {
2365 if (vaddr & ~VADDR_PAGE_TX_MASK) {
2366 tlb_flush_page(cs, page);
2367 /* XXX 31-bit hack */
2368 tlb_flush_page(cs, page ^ 0x80000000);
2369 } else {
2370 /* looks like we don't have a valid virtual address */
2371 tlb_flush(cs);
2372 }
2373 } else {
2374 if (vaddr & ~VADDR_PAGE_TX_MASK) {
2375 tlb_flush_page_all_cpus_synced(cs, page);
2376 /* XXX 31-bit hack */
2377 tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000);
2378 } else {
2379 /* looks like we don't have a valid virtual address */
2380 tlb_flush_all_cpus_synced(cs);
2381 }
2382 }
2383 }
2384
2385 /* flush local tlb */
HELPER(ptlb)2386 void HELPER(ptlb)(CPUS390XState *env)
2387 {
2388 tlb_flush(env_cpu(env));
2389 }
2390
2391 /* flush global tlb */
HELPER(purge)2392 void HELPER(purge)(CPUS390XState *env)
2393 {
2394 tlb_flush_all_cpus_synced(env_cpu(env));
2395 }
2396
2397 /* load real address */
HELPER(lra)2398 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t r1, uint64_t addr)
2399 {
2400 uint64_t asc = env->psw.mask & PSW_MASK_ASC;
2401 uint64_t ret, tec;
2402 int flags, exc, cc;
2403
2404 /* XXX incomplete - has more corner cases */
2405 if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
2406 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, GETPC());
2407 }
2408
2409 exc = mmu_translate(env, addr, MMU_S390_LRA, asc, &ret, &flags, &tec);
2410 if (exc) {
2411 cc = 3;
2412 ret = (r1 & 0xFFFFFFFF00000000ULL) | exc | 0x80000000;
2413 } else {
2414 cc = 0;
2415 ret |= addr & ~TARGET_PAGE_MASK;
2416 }
2417
2418 env->cc_op = cc;
2419 return ret;
2420 }
2421 #endif
2422
2423 /* Execute instruction. This instruction executes an insn modified with
2424 the contents of r1. It does not change the executed instruction in memory;
2425 it does not change the program counter.
2426
2427 Perform this by recording the modified instruction in env->ex_value.
2428 This will be noticed by cpu_get_tb_cpu_state and thus tb translation.
2429 */
HELPER(ex)2430 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr)
2431 {
2432 uint64_t insn;
2433 uint8_t opc;
2434
2435 /* EXECUTE targets must be at even addresses. */
2436 if (addr & 1) {
2437 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC());
2438 }
2439
2440 insn = cpu_lduw_code(env, addr);
2441 opc = insn >> 8;
2442
2443 /* Or in the contents of R1[56:63]. */
2444 insn |= r1 & 0xff;
2445
2446 /* Load the rest of the instruction. */
2447 insn <<= 48;
2448 switch (get_ilen(opc)) {
2449 case 2:
2450 break;
2451 case 4:
2452 insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32;
2453 break;
2454 case 6:
2455 insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16;
2456 break;
2457 default:
2458 g_assert_not_reached();
2459 }
2460
2461 /* The very most common cases can be sped up by avoiding a new TB. */
2462 if ((opc & 0xf0) == 0xd0) {
2463 typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t,
2464 uint64_t, uintptr_t);
2465 static const dx_helper dx[16] = {
2466 [0x0] = do_helper_trt_bkwd,
2467 [0x2] = do_helper_mvc,
2468 [0x4] = do_helper_nc,
2469 [0x5] = do_helper_clc,
2470 [0x6] = do_helper_oc,
2471 [0x7] = do_helper_xc,
2472 [0xc] = do_helper_tr,
2473 [0xd] = do_helper_trt_fwd,
2474 };
2475 dx_helper helper = dx[opc & 0xf];
2476
2477 if (helper) {
2478 uint32_t l = extract64(insn, 48, 8);
2479 uint32_t b1 = extract64(insn, 44, 4);
2480 uint32_t d1 = extract64(insn, 32, 12);
2481 uint32_t b2 = extract64(insn, 28, 4);
2482 uint32_t d2 = extract64(insn, 16, 12);
2483 uint64_t a1 = wrap_address(env, (b1 ? env->regs[b1] : 0) + d1);
2484 uint64_t a2 = wrap_address(env, (b2 ? env->regs[b2] : 0) + d2);
2485
2486 env->cc_op = helper(env, l, a1, a2, 0);
2487 env->psw.addr += ilen;
2488 return;
2489 }
2490 } else if (opc == 0x0a) {
2491 env->int_svc_code = extract64(insn, 48, 8);
2492 env->int_svc_ilen = ilen;
2493 helper_exception(env, EXCP_SVC);
2494 g_assert_not_reached();
2495 }
2496
2497 /* Record the insn we want to execute as well as the ilen to use
2498 during the execution of the target insn. This will also ensure
2499 that ex_value is non-zero, which flags that we are in a state
2500 that requires such execution. */
2501 env->ex_value = insn | ilen;
2502 env->ex_target = addr;
2503 }
2504
HELPER(mvcos)2505 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src,
2506 uint64_t len)
2507 {
2508 const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY;
2509 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2510 const uint64_t r0 = env->regs[0];
2511 const uintptr_t ra = GETPC();
2512 uint8_t dest_key, dest_as, dest_k, dest_a;
2513 uint8_t src_key, src_as, src_k, src_a;
2514 uint64_t val;
2515 int cc = 0;
2516
2517 HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n",
2518 __func__, dest, src, len);
2519
2520 if (!(env->psw.mask & PSW_MASK_DAT)) {
2521 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2522 }
2523
2524 /* OAC (operand access control) for the first operand -> dest */
2525 val = (r0 & 0xffff0000ULL) >> 16;
2526 dest_key = (val >> 12) & 0xf;
2527 dest_as = (val >> 6) & 0x3;
2528 dest_k = (val >> 1) & 0x1;
2529 dest_a = val & 0x1;
2530
2531 /* OAC (operand access control) for the second operand -> src */
2532 val = (r0 & 0x0000ffffULL);
2533 src_key = (val >> 12) & 0xf;
2534 src_as = (val >> 6) & 0x3;
2535 src_k = (val >> 1) & 0x1;
2536 src_a = val & 0x1;
2537
2538 if (!dest_k) {
2539 dest_key = psw_key;
2540 }
2541 if (!src_k) {
2542 src_key = psw_key;
2543 }
2544 if (!dest_a) {
2545 dest_as = psw_as;
2546 }
2547 if (!src_a) {
2548 src_as = psw_as;
2549 }
2550
2551 if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) {
2552 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2553 }
2554 if (!(env->cregs[0] & CR0_SECONDARY) &&
2555 (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) {
2556 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2557 }
2558 if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) {
2559 tcg_s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2560 }
2561
2562 len = wrap_length32(env, len);
2563 if (len > 4096) {
2564 cc = 3;
2565 len = 4096;
2566 }
2567
2568 /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */
2569 if (src_as == AS_ACCREG || dest_as == AS_ACCREG ||
2570 (env->psw.mask & PSW_MASK_PSTATE)) {
2571 qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n",
2572 __func__);
2573 tcg_s390_program_interrupt(env, PGM_ADDRESSING, ra);
2574 }
2575
2576 /* FIXME: Access using correct keys and AR-mode */
2577 if (len) {
2578 S390Access srca, desta;
2579
2580 access_prepare(&srca, env, src, len, MMU_DATA_LOAD,
2581 mmu_idx_from_as(src_as), ra);
2582 access_prepare(&desta, env, dest, len, MMU_DATA_STORE,
2583 mmu_idx_from_as(dest_as), ra);
2584
2585 access_memmove(env, &desta, &srca, ra);
2586 }
2587
2588 return cc;
2589 }
2590
2591 /* Decode a Unicode character. A return value < 0 indicates success, storing
2592 the UTF-32 result into OCHAR and the input length into OLEN. A return
2593 value >= 0 indicates failure, and the CC value to be returned. */
2594 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2595 uint64_t ilen, bool enh_check, uintptr_t ra,
2596 uint32_t *ochar, uint32_t *olen);
2597
2598 /* Encode a Unicode character. A return value < 0 indicates success, storing
2599 the bytes into ADDR and the output length into OLEN. A return value >= 0
2600 indicates failure, and the CC value to be returned. */
2601 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2602 uint64_t ilen, uintptr_t ra, uint32_t c,
2603 uint32_t *olen);
2604
decode_utf8(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2605 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2606 bool enh_check, uintptr_t ra,
2607 uint32_t *ochar, uint32_t *olen)
2608 {
2609 uint8_t s0, s1, s2, s3;
2610 uint32_t c, l;
2611
2612 if (ilen < 1) {
2613 return 0;
2614 }
2615 s0 = cpu_ldub_data_ra(env, addr, ra);
2616 if (s0 <= 0x7f) {
2617 /* one byte character */
2618 l = 1;
2619 c = s0;
2620 } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) {
2621 /* invalid character */
2622 return 2;
2623 } else if (s0 <= 0xdf) {
2624 /* two byte character */
2625 l = 2;
2626 if (ilen < 2) {
2627 return 0;
2628 }
2629 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2630 c = s0 & 0x1f;
2631 c = (c << 6) | (s1 & 0x3f);
2632 if (enh_check && (s1 & 0xc0) != 0x80) {
2633 return 2;
2634 }
2635 } else if (s0 <= 0xef) {
2636 /* three byte character */
2637 l = 3;
2638 if (ilen < 3) {
2639 return 0;
2640 }
2641 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2642 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2643 c = s0 & 0x0f;
2644 c = (c << 6) | (s1 & 0x3f);
2645 c = (c << 6) | (s2 & 0x3f);
2646 /* Fold the byte-by-byte range descriptions in the PoO into
2647 tests against the complete value. It disallows encodings
2648 that could be smaller, and the UTF-16 surrogates. */
2649 if (enh_check
2650 && ((s1 & 0xc0) != 0x80
2651 || (s2 & 0xc0) != 0x80
2652 || c < 0x1000
2653 || (c >= 0xd800 && c <= 0xdfff))) {
2654 return 2;
2655 }
2656 } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) {
2657 /* four byte character */
2658 l = 4;
2659 if (ilen < 4) {
2660 return 0;
2661 }
2662 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2663 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2664 s3 = cpu_ldub_data_ra(env, addr + 3, ra);
2665 c = s0 & 0x07;
2666 c = (c << 6) | (s1 & 0x3f);
2667 c = (c << 6) | (s2 & 0x3f);
2668 c = (c << 6) | (s3 & 0x3f);
2669 /* See above. */
2670 if (enh_check
2671 && ((s1 & 0xc0) != 0x80
2672 || (s2 & 0xc0) != 0x80
2673 || (s3 & 0xc0) != 0x80
2674 || c < 0x010000
2675 || c > 0x10ffff)) {
2676 return 2;
2677 }
2678 } else {
2679 /* invalid character */
2680 return 2;
2681 }
2682
2683 *ochar = c;
2684 *olen = l;
2685 return -1;
2686 }
2687
decode_utf16(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2688 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2689 bool enh_check, uintptr_t ra,
2690 uint32_t *ochar, uint32_t *olen)
2691 {
2692 uint16_t s0, s1;
2693 uint32_t c, l;
2694
2695 if (ilen < 2) {
2696 return 0;
2697 }
2698 s0 = cpu_lduw_data_ra(env, addr, ra);
2699 if ((s0 & 0xfc00) != 0xd800) {
2700 /* one word character */
2701 l = 2;
2702 c = s0;
2703 } else {
2704 /* two word character */
2705 l = 4;
2706 if (ilen < 4) {
2707 return 0;
2708 }
2709 s1 = cpu_lduw_data_ra(env, addr + 2, ra);
2710 c = extract32(s0, 6, 4) + 1;
2711 c = (c << 6) | (s0 & 0x3f);
2712 c = (c << 10) | (s1 & 0x3ff);
2713 if (enh_check && (s1 & 0xfc00) != 0xdc00) {
2714 /* invalid surrogate character */
2715 return 2;
2716 }
2717 }
2718
2719 *ochar = c;
2720 *olen = l;
2721 return -1;
2722 }
2723
decode_utf32(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2724 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2725 bool enh_check, uintptr_t ra,
2726 uint32_t *ochar, uint32_t *olen)
2727 {
2728 uint32_t c;
2729
2730 if (ilen < 4) {
2731 return 0;
2732 }
2733 c = cpu_ldl_data_ra(env, addr, ra);
2734 if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) {
2735 /* invalid unicode character */
2736 return 2;
2737 }
2738
2739 *ochar = c;
2740 *olen = 4;
2741 return -1;
2742 }
2743
encode_utf8(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2744 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2745 uintptr_t ra, uint32_t c, uint32_t *olen)
2746 {
2747 uint8_t d[4];
2748 uint32_t l, i;
2749
2750 if (c <= 0x7f) {
2751 /* one byte character */
2752 l = 1;
2753 d[0] = c;
2754 } else if (c <= 0x7ff) {
2755 /* two byte character */
2756 l = 2;
2757 d[1] = 0x80 | extract32(c, 0, 6);
2758 d[0] = 0xc0 | extract32(c, 6, 5);
2759 } else if (c <= 0xffff) {
2760 /* three byte character */
2761 l = 3;
2762 d[2] = 0x80 | extract32(c, 0, 6);
2763 d[1] = 0x80 | extract32(c, 6, 6);
2764 d[0] = 0xe0 | extract32(c, 12, 4);
2765 } else {
2766 /* four byte character */
2767 l = 4;
2768 d[3] = 0x80 | extract32(c, 0, 6);
2769 d[2] = 0x80 | extract32(c, 6, 6);
2770 d[1] = 0x80 | extract32(c, 12, 6);
2771 d[0] = 0xf0 | extract32(c, 18, 3);
2772 }
2773
2774 if (ilen < l) {
2775 return 1;
2776 }
2777 for (i = 0; i < l; ++i) {
2778 cpu_stb_data_ra(env, addr + i, d[i], ra);
2779 }
2780
2781 *olen = l;
2782 return -1;
2783 }
2784
encode_utf16(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2785 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2786 uintptr_t ra, uint32_t c, uint32_t *olen)
2787 {
2788 uint16_t d0, d1;
2789
2790 if (c <= 0xffff) {
2791 /* one word character */
2792 if (ilen < 2) {
2793 return 1;
2794 }
2795 cpu_stw_data_ra(env, addr, c, ra);
2796 *olen = 2;
2797 } else {
2798 /* two word character */
2799 if (ilen < 4) {
2800 return 1;
2801 }
2802 d1 = 0xdc00 | extract32(c, 0, 10);
2803 d0 = 0xd800 | extract32(c, 10, 6);
2804 d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1);
2805 cpu_stw_data_ra(env, addr + 0, d0, ra);
2806 cpu_stw_data_ra(env, addr + 2, d1, ra);
2807 *olen = 4;
2808 }
2809
2810 return -1;
2811 }
2812
encode_utf32(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2813 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2814 uintptr_t ra, uint32_t c, uint32_t *olen)
2815 {
2816 if (ilen < 4) {
2817 return 1;
2818 }
2819 cpu_stl_data_ra(env, addr, c, ra);
2820 *olen = 4;
2821 return -1;
2822 }
2823
convert_unicode(CPUS390XState * env,uint32_t r1,uint32_t r2,uint32_t m3,uintptr_t ra,decode_unicode_fn decode,encode_unicode_fn encode)2824 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1,
2825 uint32_t r2, uint32_t m3, uintptr_t ra,
2826 decode_unicode_fn decode,
2827 encode_unicode_fn encode)
2828 {
2829 uint64_t dst = get_address(env, r1);
2830 uint64_t dlen = get_length(env, r1 + 1);
2831 uint64_t src = get_address(env, r2);
2832 uint64_t slen = get_length(env, r2 + 1);
2833 bool enh_check = m3 & 1;
2834 int cc, i;
2835
2836 /* Lest we fail to service interrupts in a timely manner, limit the
2837 amount of work we're willing to do. For now, let's cap at 256. */
2838 for (i = 0; i < 256; ++i) {
2839 uint32_t c, ilen, olen;
2840
2841 cc = decode(env, src, slen, enh_check, ra, &c, &ilen);
2842 if (unlikely(cc >= 0)) {
2843 break;
2844 }
2845 cc = encode(env, dst, dlen, ra, c, &olen);
2846 if (unlikely(cc >= 0)) {
2847 break;
2848 }
2849
2850 src += ilen;
2851 slen -= ilen;
2852 dst += olen;
2853 dlen -= olen;
2854 cc = 3;
2855 }
2856
2857 set_address(env, r1, dst);
2858 set_length(env, r1 + 1, dlen);
2859 set_address(env, r2, src);
2860 set_length(env, r2 + 1, slen);
2861
2862 return cc;
2863 }
2864
HELPER(cu12)2865 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2866 {
2867 return convert_unicode(env, r1, r2, m3, GETPC(),
2868 decode_utf8, encode_utf16);
2869 }
2870
HELPER(cu14)2871 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2872 {
2873 return convert_unicode(env, r1, r2, m3, GETPC(),
2874 decode_utf8, encode_utf32);
2875 }
2876
HELPER(cu21)2877 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2878 {
2879 return convert_unicode(env, r1, r2, m3, GETPC(),
2880 decode_utf16, encode_utf8);
2881 }
2882
HELPER(cu24)2883 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2884 {
2885 return convert_unicode(env, r1, r2, m3, GETPC(),
2886 decode_utf16, encode_utf32);
2887 }
2888
HELPER(cu41)2889 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2890 {
2891 return convert_unicode(env, r1, r2, m3, GETPC(),
2892 decode_utf32, encode_utf8);
2893 }
2894
HELPER(cu42)2895 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2896 {
2897 return convert_unicode(env, r1, r2, m3, GETPC(),
2898 decode_utf32, encode_utf16);
2899 }
2900
probe_write_access(CPUS390XState * env,uint64_t addr,uint64_t len,uintptr_t ra)2901 void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len,
2902 uintptr_t ra)
2903 {
2904 const int mmu_idx = s390x_env_mmu_index(env, false);
2905
2906 /* test the actual access, not just any access to the page due to LAP */
2907 while (len) {
2908 const uint64_t pagelen = -(addr | TARGET_PAGE_MASK);
2909 const uint64_t curlen = MIN(pagelen, len);
2910
2911 probe_write(env, addr, curlen, mmu_idx, ra);
2912 addr = wrap_address(env, addr + curlen);
2913 len -= curlen;
2914 }
2915 }
2916
HELPER(probe_write_access)2917 void HELPER(probe_write_access)(CPUS390XState *env, uint64_t addr, uint64_t len)
2918 {
2919 probe_write_access(env, addr, len, GETPC());
2920 }
2921