1 /* 2 * S/390 memory access helper routines 3 * 4 * Copyright (c) 2009 Ulrich Hecht 5 * Copyright (c) 2009 Alexander Graf 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/osdep.h" 22 #include "qemu/log.h" 23 #include "cpu.h" 24 #include "s390x-internal.h" 25 #include "tcg_s390x.h" 26 #include "exec/helper-proto.h" 27 #include "exec/exec-all.h" 28 #include "exec/page-protection.h" 29 #include "exec/cpu_ldst.h" 30 #include "hw/core/tcg-cpu-ops.h" 31 #include "qemu/int128.h" 32 #include "qemu/atomic128.h" 33 34 #if !defined(CONFIG_USER_ONLY) 35 #include "hw/s390x/storage-keys.h" 36 #include "hw/boards.h" 37 #endif 38 39 #ifdef CONFIG_USER_ONLY 40 # define user_or_likely(X) true 41 #else 42 # define user_or_likely(X) likely(X) 43 #endif 44 45 /*****************************************************************************/ 46 /* Softmmu support */ 47 48 /* #define DEBUG_HELPER */ 49 #ifdef DEBUG_HELPER 50 #define HELPER_LOG(x...) qemu_log(x) 51 #else 52 #define HELPER_LOG(x...) 53 #endif 54 psw_key_valid(CPUS390XState * env,uint8_t psw_key)55 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key) 56 { 57 uint16_t pkm = env->cregs[3] >> 16; 58 59 if (env->psw.mask & PSW_MASK_PSTATE) { 60 /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */ 61 return pkm & (0x8000 >> psw_key); 62 } 63 return true; 64 } 65 is_destructive_overlap(CPUS390XState * env,uint64_t dest,uint64_t src,uint32_t len)66 static bool is_destructive_overlap(CPUS390XState *env, uint64_t dest, 67 uint64_t src, uint32_t len) 68 { 69 if (!len || src == dest) { 70 return false; 71 } 72 /* Take care of wrapping at the end of address space. */ 73 if (unlikely(wrap_address(env, src + len - 1) < src)) { 74 return dest > src || dest <= wrap_address(env, src + len - 1); 75 } 76 return dest > src && dest <= src + len - 1; 77 } 78 79 /* Trigger a SPECIFICATION exception if an address or a length is not 80 naturally aligned. */ check_alignment(CPUS390XState * env,uint64_t v,int wordsize,uintptr_t ra)81 static inline void check_alignment(CPUS390XState *env, uint64_t v, 82 int wordsize, uintptr_t ra) 83 { 84 if (v % wordsize) { 85 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 86 } 87 } 88 89 /* Load a value from memory according to its size. */ cpu_ldusize_data_ra(CPUS390XState * env,uint64_t addr,int wordsize,uintptr_t ra)90 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr, 91 int wordsize, uintptr_t ra) 92 { 93 switch (wordsize) { 94 case 1: 95 return cpu_ldub_data_ra(env, addr, ra); 96 case 2: 97 return cpu_lduw_data_ra(env, addr, ra); 98 default: 99 abort(); 100 } 101 } 102 103 /* Store a to memory according to its size. */ cpu_stsize_data_ra(CPUS390XState * env,uint64_t addr,uint64_t value,int wordsize,uintptr_t ra)104 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr, 105 uint64_t value, int wordsize, 106 uintptr_t ra) 107 { 108 switch (wordsize) { 109 case 1: 110 cpu_stb_data_ra(env, addr, value, ra); 111 break; 112 case 2: 113 cpu_stw_data_ra(env, addr, value, ra); 114 break; 115 default: 116 abort(); 117 } 118 } 119 120 /* An access covers at most 4096 bytes and therefore at most two pages. */ 121 typedef struct S390Access { 122 target_ulong vaddr1; 123 target_ulong vaddr2; 124 void *haddr1; 125 void *haddr2; 126 uint16_t size1; 127 uint16_t size2; 128 /* 129 * If we can't access the host page directly, we'll have to do I/O access 130 * via ld/st helpers. These are internal details, so we store the 131 * mmu idx to do the access here instead of passing it around in the 132 * helpers. 133 */ 134 int mmu_idx; 135 } S390Access; 136 137 /* 138 * With nonfault=1, return the PGM_ exception that would have been injected 139 * into the guest; return 0 if no exception was detected. 140 * 141 * For !CONFIG_USER_ONLY, the TEC is stored stored to env->tlb_fill_tec. 142 * For CONFIG_USER_ONLY, the faulting address is stored to env->__excp_addr. 143 */ s390_probe_access(CPUArchState * env,target_ulong addr,int size,MMUAccessType access_type,int mmu_idx,bool nonfault,void ** phost,uintptr_t ra)144 static inline int s390_probe_access(CPUArchState *env, target_ulong addr, 145 int size, MMUAccessType access_type, 146 int mmu_idx, bool nonfault, 147 void **phost, uintptr_t ra) 148 { 149 int flags = probe_access_flags(env, addr, size, access_type, mmu_idx, 150 nonfault, phost, ra); 151 152 if (unlikely(flags & TLB_INVALID_MASK)) { 153 #ifdef CONFIG_USER_ONLY 154 /* Address is in TEC in system mode; see s390_cpu_record_sigsegv. */ 155 env->__excp_addr = addr & TARGET_PAGE_MASK; 156 return (page_get_flags(addr) & PAGE_VALID 157 ? PGM_PROTECTION : PGM_ADDRESSING); 158 #else 159 return env->tlb_fill_exc; 160 #endif 161 } 162 163 #ifndef CONFIG_USER_ONLY 164 if (unlikely(flags & TLB_WATCHPOINT)) { 165 /* S390 does not presently use transaction attributes. */ 166 cpu_check_watchpoint(env_cpu(env), addr, size, 167 MEMTXATTRS_UNSPECIFIED, 168 (access_type == MMU_DATA_STORE 169 ? BP_MEM_WRITE : BP_MEM_READ), ra); 170 } 171 #endif 172 173 return 0; 174 } 175 access_prepare_nf(S390Access * access,CPUS390XState * env,bool nonfault,vaddr vaddr1,int size,MMUAccessType access_type,int mmu_idx,uintptr_t ra)176 static int access_prepare_nf(S390Access *access, CPUS390XState *env, 177 bool nonfault, vaddr vaddr1, int size, 178 MMUAccessType access_type, 179 int mmu_idx, uintptr_t ra) 180 { 181 int size1, size2, exc; 182 183 assert(size > 0 && size <= 4096); 184 185 size1 = MIN(size, -(vaddr1 | TARGET_PAGE_MASK)), 186 size2 = size - size1; 187 188 memset(access, 0, sizeof(*access)); 189 access->vaddr1 = vaddr1; 190 access->size1 = size1; 191 access->size2 = size2; 192 access->mmu_idx = mmu_idx; 193 194 exc = s390_probe_access(env, vaddr1, size1, access_type, mmu_idx, nonfault, 195 &access->haddr1, ra); 196 if (unlikely(exc)) { 197 return exc; 198 } 199 if (unlikely(size2)) { 200 /* The access crosses page boundaries. */ 201 vaddr vaddr2 = wrap_address(env, vaddr1 + size1); 202 203 access->vaddr2 = vaddr2; 204 exc = s390_probe_access(env, vaddr2, size2, access_type, mmu_idx, 205 nonfault, &access->haddr2, ra); 206 if (unlikely(exc)) { 207 return exc; 208 } 209 } 210 return 0; 211 } 212 access_prepare(S390Access * ret,CPUS390XState * env,vaddr vaddr,int size,MMUAccessType access_type,int mmu_idx,uintptr_t ra)213 static inline void access_prepare(S390Access *ret, CPUS390XState *env, 214 vaddr vaddr, int size, 215 MMUAccessType access_type, int mmu_idx, 216 uintptr_t ra) 217 { 218 int exc = access_prepare_nf(ret, env, false, vaddr, size, 219 access_type, mmu_idx, ra); 220 assert(!exc); 221 } 222 223 /* Helper to handle memset on a single page. */ do_access_memset(CPUS390XState * env,vaddr vaddr,char * haddr,uint8_t byte,uint16_t size,int mmu_idx,uintptr_t ra)224 static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr, 225 uint8_t byte, uint16_t size, int mmu_idx, 226 uintptr_t ra) 227 { 228 if (user_or_likely(haddr)) { 229 memset(haddr, byte, size); 230 } else { 231 MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); 232 for (int i = 0; i < size; i++) { 233 cpu_stb_mmu(env, vaddr + i, byte, oi, ra); 234 } 235 } 236 } 237 access_memset(CPUS390XState * env,S390Access * desta,uint8_t byte,uintptr_t ra)238 static void access_memset(CPUS390XState *env, S390Access *desta, 239 uint8_t byte, uintptr_t ra) 240 { 241 set_helper_retaddr(ra); 242 do_access_memset(env, desta->vaddr1, desta->haddr1, byte, desta->size1, 243 desta->mmu_idx, ra); 244 if (unlikely(desta->size2)) { 245 do_access_memset(env, desta->vaddr2, desta->haddr2, byte, 246 desta->size2, desta->mmu_idx, ra); 247 } 248 clear_helper_retaddr(); 249 } 250 access_get_byte(CPUS390XState * env,S390Access * access,int offset,uintptr_t ra)251 static uint8_t access_get_byte(CPUS390XState *env, S390Access *access, 252 int offset, uintptr_t ra) 253 { 254 target_ulong vaddr = access->vaddr1; 255 void *haddr = access->haddr1; 256 257 if (unlikely(offset >= access->size1)) { 258 offset -= access->size1; 259 vaddr = access->vaddr2; 260 haddr = access->haddr2; 261 } 262 263 if (user_or_likely(haddr)) { 264 return ldub_p(haddr + offset); 265 } else { 266 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx); 267 return cpu_ldb_mmu(env, vaddr + offset, oi, ra); 268 } 269 } 270 access_set_byte(CPUS390XState * env,S390Access * access,int offset,uint8_t byte,uintptr_t ra)271 static void access_set_byte(CPUS390XState *env, S390Access *access, 272 int offset, uint8_t byte, uintptr_t ra) 273 { 274 target_ulong vaddr = access->vaddr1; 275 void *haddr = access->haddr1; 276 277 if (unlikely(offset >= access->size1)) { 278 offset -= access->size1; 279 vaddr = access->vaddr2; 280 haddr = access->haddr2; 281 } 282 283 if (user_or_likely(haddr)) { 284 stb_p(haddr + offset, byte); 285 } else { 286 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx); 287 cpu_stb_mmu(env, vaddr + offset, byte, oi, ra); 288 } 289 } 290 291 /* 292 * Move data with the same semantics as memmove() in case ranges don't overlap 293 * or src > dest. Undefined behavior on destructive overlaps. 294 */ access_memmove(CPUS390XState * env,S390Access * desta,S390Access * srca,uintptr_t ra)295 static void access_memmove(CPUS390XState *env, S390Access *desta, 296 S390Access *srca, uintptr_t ra) 297 { 298 int len = desta->size1 + desta->size2; 299 300 assert(len == srca->size1 + srca->size2); 301 302 /* Fallback to slow access in case we don't have access to all host pages */ 303 if (user_or_likely(desta->haddr1 && 304 srca->haddr1 && 305 (!desta->size2 || desta->haddr2) && 306 (!srca->size2 || srca->haddr2))) { 307 int diff = desta->size1 - srca->size1; 308 309 if (likely(diff == 0)) { 310 memmove(desta->haddr1, srca->haddr1, srca->size1); 311 if (unlikely(srca->size2)) { 312 memmove(desta->haddr2, srca->haddr2, srca->size2); 313 } 314 } else if (diff > 0) { 315 memmove(desta->haddr1, srca->haddr1, srca->size1); 316 memmove(desta->haddr1 + srca->size1, srca->haddr2, diff); 317 if (likely(desta->size2)) { 318 memmove(desta->haddr2, srca->haddr2 + diff, desta->size2); 319 } 320 } else { 321 diff = -diff; 322 memmove(desta->haddr1, srca->haddr1, desta->size1); 323 memmove(desta->haddr2, srca->haddr1 + desta->size1, diff); 324 if (likely(srca->size2)) { 325 memmove(desta->haddr2 + diff, srca->haddr2, srca->size2); 326 } 327 } 328 } else { 329 for (int i = 0; i < len; i++) { 330 uint8_t byte = access_get_byte(env, srca, i, ra); 331 access_set_byte(env, desta, i, byte, ra); 332 } 333 } 334 } 335 mmu_idx_from_as(uint8_t as)336 static int mmu_idx_from_as(uint8_t as) 337 { 338 switch (as) { 339 case AS_PRIMARY: 340 return MMU_PRIMARY_IDX; 341 case AS_SECONDARY: 342 return MMU_SECONDARY_IDX; 343 case AS_HOME: 344 return MMU_HOME_IDX; 345 default: 346 /* FIXME AS_ACCREG */ 347 g_assert_not_reached(); 348 } 349 } 350 351 /* and on array */ do_helper_nc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)352 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest, 353 uint64_t src, uintptr_t ra) 354 { 355 const int mmu_idx = s390x_env_mmu_index(env, false); 356 S390Access srca1, srca2, desta; 357 uint32_t i; 358 uint8_t c = 0; 359 360 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 361 __func__, l, dest, src); 362 363 /* NC always processes one more byte than specified - maximum is 256 */ 364 l++; 365 366 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 367 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 368 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 369 set_helper_retaddr(ra); 370 371 for (i = 0; i < l; i++) { 372 const uint8_t x = access_get_byte(env, &srca1, i, ra) & 373 access_get_byte(env, &srca2, i, ra); 374 375 c |= x; 376 access_set_byte(env, &desta, i, x, ra); 377 } 378 379 clear_helper_retaddr(); 380 return c != 0; 381 } 382 HELPER(nc)383 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest, 384 uint64_t src) 385 { 386 return do_helper_nc(env, l, dest, src, GETPC()); 387 } 388 389 /* xor on array */ do_helper_xc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)390 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest, 391 uint64_t src, uintptr_t ra) 392 { 393 const int mmu_idx = s390x_env_mmu_index(env, false); 394 S390Access srca1, srca2, desta; 395 uint32_t i; 396 uint8_t c = 0; 397 398 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 399 __func__, l, dest, src); 400 401 /* XC always processes one more byte than specified - maximum is 256 */ 402 l++; 403 404 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 405 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 406 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 407 408 /* xor with itself is the same as memset(0) */ 409 if (src == dest) { 410 access_memset(env, &desta, 0, ra); 411 return 0; 412 } 413 414 set_helper_retaddr(ra); 415 for (i = 0; i < l; i++) { 416 const uint8_t x = access_get_byte(env, &srca1, i, ra) ^ 417 access_get_byte(env, &srca2, i, ra); 418 419 c |= x; 420 access_set_byte(env, &desta, i, x, ra); 421 } 422 clear_helper_retaddr(); 423 return c != 0; 424 } 425 HELPER(xc)426 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest, 427 uint64_t src) 428 { 429 return do_helper_xc(env, l, dest, src, GETPC()); 430 } 431 432 /* or on array */ do_helper_oc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)433 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest, 434 uint64_t src, uintptr_t ra) 435 { 436 const int mmu_idx = s390x_env_mmu_index(env, false); 437 S390Access srca1, srca2, desta; 438 uint32_t i; 439 uint8_t c = 0; 440 441 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 442 __func__, l, dest, src); 443 444 /* OC always processes one more byte than specified - maximum is 256 */ 445 l++; 446 447 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 448 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 449 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 450 set_helper_retaddr(ra); 451 452 for (i = 0; i < l; i++) { 453 const uint8_t x = access_get_byte(env, &srca1, i, ra) | 454 access_get_byte(env, &srca2, i, ra); 455 456 c |= x; 457 access_set_byte(env, &desta, i, x, ra); 458 } 459 460 clear_helper_retaddr(); 461 return c != 0; 462 } 463 HELPER(oc)464 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest, 465 uint64_t src) 466 { 467 return do_helper_oc(env, l, dest, src, GETPC()); 468 } 469 470 /* memmove */ do_helper_mvc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)471 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest, 472 uint64_t src, uintptr_t ra) 473 { 474 const int mmu_idx = s390x_env_mmu_index(env, false); 475 S390Access srca, desta; 476 uint32_t i; 477 478 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 479 __func__, l, dest, src); 480 481 /* MVC always copies one more byte than specified - maximum is 256 */ 482 l++; 483 484 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 485 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 486 487 /* 488 * "When the operands overlap, the result is obtained as if the operands 489 * were processed one byte at a time". Only non-destructive overlaps 490 * behave like memmove(). 491 */ 492 if (dest == src + 1) { 493 access_memset(env, &desta, access_get_byte(env, &srca, 0, ra), ra); 494 } else if (!is_destructive_overlap(env, dest, src, l)) { 495 access_memmove(env, &desta, &srca, ra); 496 } else { 497 set_helper_retaddr(ra); 498 for (i = 0; i < l; i++) { 499 uint8_t byte = access_get_byte(env, &srca, i, ra); 500 501 access_set_byte(env, &desta, i, byte, ra); 502 } 503 clear_helper_retaddr(); 504 } 505 506 return env->cc_op; 507 } 508 HELPER(mvc)509 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 510 { 511 do_helper_mvc(env, l, dest, src, GETPC()); 512 } 513 514 /* move right to left */ HELPER(mvcrl)515 void HELPER(mvcrl)(CPUS390XState *env, uint64_t l, uint64_t dest, uint64_t src) 516 { 517 const int mmu_idx = s390x_env_mmu_index(env, false); 518 const uint64_t ra = GETPC(); 519 S390Access srca, desta; 520 int32_t i; 521 522 /* MVCRL always copies one more byte than specified - maximum is 256 */ 523 l &= 0xff; 524 l++; 525 526 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 527 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 528 529 set_helper_retaddr(ra); 530 for (i = l - 1; i >= 0; i--) { 531 uint8_t byte = access_get_byte(env, &srca, i, ra); 532 access_set_byte(env, &desta, i, byte, ra); 533 } 534 clear_helper_retaddr(); 535 } 536 537 /* move inverse */ HELPER(mvcin)538 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 539 { 540 const int mmu_idx = s390x_env_mmu_index(env, false); 541 S390Access srca, desta; 542 uintptr_t ra = GETPC(); 543 int i; 544 545 /* MVCIN always copies one more byte than specified - maximum is 256 */ 546 l++; 547 548 src = wrap_address(env, src - l + 1); 549 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 550 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 551 552 set_helper_retaddr(ra); 553 for (i = 0; i < l; i++) { 554 const uint8_t x = access_get_byte(env, &srca, l - i - 1, ra); 555 access_set_byte(env, &desta, i, x, ra); 556 } 557 clear_helper_retaddr(); 558 } 559 560 /* move numerics */ HELPER(mvn)561 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 562 { 563 const int mmu_idx = s390x_env_mmu_index(env, false); 564 S390Access srca1, srca2, desta; 565 uintptr_t ra = GETPC(); 566 int i; 567 568 /* MVN always copies one more byte than specified - maximum is 256 */ 569 l++; 570 571 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 572 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 573 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 574 575 set_helper_retaddr(ra); 576 for (i = 0; i < l; i++) { 577 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0x0f) | 578 (access_get_byte(env, &srca2, i, ra) & 0xf0); 579 580 access_set_byte(env, &desta, i, x, ra); 581 } 582 clear_helper_retaddr(); 583 } 584 585 /* move with offset */ HELPER(mvo)586 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 587 { 588 const int mmu_idx = s390x_env_mmu_index(env, false); 589 /* MVO always processes one more byte than specified - maximum is 16 */ 590 const int len_dest = (l >> 4) + 1; 591 const int len_src = (l & 0xf) + 1; 592 uintptr_t ra = GETPC(); 593 uint8_t byte_dest, byte_src; 594 S390Access srca, desta; 595 int i, j; 596 597 access_prepare(&srca, env, src, len_src, MMU_DATA_LOAD, mmu_idx, ra); 598 access_prepare(&desta, env, dest, len_dest, MMU_DATA_STORE, mmu_idx, ra); 599 600 /* Handle rightmost byte */ 601 byte_dest = cpu_ldub_data_ra(env, dest + len_dest - 1, ra); 602 603 set_helper_retaddr(ra); 604 byte_src = access_get_byte(env, &srca, len_src - 1, ra); 605 byte_dest = (byte_dest & 0x0f) | (byte_src << 4); 606 access_set_byte(env, &desta, len_dest - 1, byte_dest, ra); 607 608 /* Process remaining bytes from right to left */ 609 for (i = len_dest - 2, j = len_src - 2; i >= 0; i--, j--) { 610 byte_dest = byte_src >> 4; 611 if (j >= 0) { 612 byte_src = access_get_byte(env, &srca, j, ra); 613 } else { 614 byte_src = 0; 615 } 616 byte_dest |= byte_src << 4; 617 access_set_byte(env, &desta, i, byte_dest, ra); 618 } 619 clear_helper_retaddr(); 620 } 621 622 /* move zones */ HELPER(mvz)623 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 624 { 625 const int mmu_idx = s390x_env_mmu_index(env, false); 626 S390Access srca1, srca2, desta; 627 uintptr_t ra = GETPC(); 628 int i; 629 630 /* MVZ always copies one more byte than specified - maximum is 256 */ 631 l++; 632 633 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 634 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 635 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 636 637 set_helper_retaddr(ra); 638 for (i = 0; i < l; i++) { 639 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0xf0) | 640 (access_get_byte(env, &srca2, i, ra) & 0x0f); 641 642 access_set_byte(env, &desta, i, x, ra); 643 } 644 clear_helper_retaddr(); 645 } 646 647 /* compare unsigned byte arrays */ do_helper_clc(CPUS390XState * env,uint32_t l,uint64_t s1,uint64_t s2,uintptr_t ra)648 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1, 649 uint64_t s2, uintptr_t ra) 650 { 651 uint32_t i; 652 uint32_t cc = 0; 653 654 HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n", 655 __func__, l, s1, s2); 656 657 for (i = 0; i <= l; i++) { 658 uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra); 659 uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra); 660 HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y); 661 if (x < y) { 662 cc = 1; 663 break; 664 } else if (x > y) { 665 cc = 2; 666 break; 667 } 668 } 669 670 HELPER_LOG("\n"); 671 return cc; 672 } 673 HELPER(clc)674 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2) 675 { 676 return do_helper_clc(env, l, s1, s2, GETPC()); 677 } 678 679 /* compare logical under mask */ HELPER(clm)680 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask, 681 uint64_t addr) 682 { 683 uintptr_t ra = GETPC(); 684 uint32_t cc = 0; 685 686 HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1, 687 mask, addr); 688 689 if (!mask) { 690 /* Recognize access exceptions for the first byte */ 691 probe_read(env, addr, 1, s390x_env_mmu_index(env, false), ra); 692 } 693 694 while (mask) { 695 if (mask & 8) { 696 uint8_t d = cpu_ldub_data_ra(env, addr, ra); 697 uint8_t r = extract32(r1, 24, 8); 698 HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d, 699 addr); 700 if (r < d) { 701 cc = 1; 702 break; 703 } else if (r > d) { 704 cc = 2; 705 break; 706 } 707 addr++; 708 } 709 mask = (mask << 1) & 0xf; 710 r1 <<= 8; 711 } 712 713 HELPER_LOG("\n"); 714 return cc; 715 } 716 get_address(CPUS390XState * env,int reg)717 static inline uint64_t get_address(CPUS390XState *env, int reg) 718 { 719 return wrap_address(env, env->regs[reg]); 720 } 721 722 /* 723 * Store the address to the given register, zeroing out unused leftmost 724 * bits in bit positions 32-63 (24-bit and 31-bit mode only). 725 */ set_address_zero(CPUS390XState * env,int reg,uint64_t address)726 static inline void set_address_zero(CPUS390XState *env, int reg, 727 uint64_t address) 728 { 729 if (env->psw.mask & PSW_MASK_64) { 730 env->regs[reg] = address; 731 } else { 732 if (!(env->psw.mask & PSW_MASK_32)) { 733 address &= 0x00ffffff; 734 } else { 735 address &= 0x7fffffff; 736 } 737 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address); 738 } 739 } 740 set_address(CPUS390XState * env,int reg,uint64_t address)741 static inline void set_address(CPUS390XState *env, int reg, uint64_t address) 742 { 743 if (env->psw.mask & PSW_MASK_64) { 744 /* 64-Bit mode */ 745 env->regs[reg] = address; 746 } else { 747 if (!(env->psw.mask & PSW_MASK_32)) { 748 /* 24-Bit mode. According to the PoO it is implementation 749 dependent if bits 32-39 remain unchanged or are set to 750 zeros. Choose the former so that the function can also be 751 used for TRT. */ 752 env->regs[reg] = deposit64(env->regs[reg], 0, 24, address); 753 } else { 754 /* 31-Bit mode. According to the PoO it is implementation 755 dependent if bit 32 remains unchanged or is set to zero. 756 Choose the latter so that the function can also be used for 757 TRT. */ 758 address &= 0x7fffffff; 759 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address); 760 } 761 } 762 } 763 wrap_length32(CPUS390XState * env,uint64_t length)764 static inline uint64_t wrap_length32(CPUS390XState *env, uint64_t length) 765 { 766 if (!(env->psw.mask & PSW_MASK_64)) { 767 return (uint32_t)length; 768 } 769 return length; 770 } 771 wrap_length31(CPUS390XState * env,uint64_t length)772 static inline uint64_t wrap_length31(CPUS390XState *env, uint64_t length) 773 { 774 if (!(env->psw.mask & PSW_MASK_64)) { 775 /* 24-Bit and 31-Bit mode */ 776 length &= 0x7fffffff; 777 } 778 return length; 779 } 780 get_length(CPUS390XState * env,int reg)781 static inline uint64_t get_length(CPUS390XState *env, int reg) 782 { 783 return wrap_length31(env, env->regs[reg]); 784 } 785 set_length(CPUS390XState * env,int reg,uint64_t length)786 static inline void set_length(CPUS390XState *env, int reg, uint64_t length) 787 { 788 if (env->psw.mask & PSW_MASK_64) { 789 /* 64-Bit mode */ 790 env->regs[reg] = length; 791 } else { 792 /* 24-Bit and 31-Bit mode */ 793 env->regs[reg] = deposit64(env->regs[reg], 0, 32, length); 794 } 795 } 796 797 /* search string (c is byte to search, r2 is string, r1 end of string) */ HELPER(srst)798 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2) 799 { 800 uintptr_t ra = GETPC(); 801 uint64_t end, str; 802 uint32_t len; 803 uint8_t v, c = env->regs[0]; 804 805 /* Bits 32-55 must contain all 0. */ 806 if (env->regs[0] & 0xffffff00u) { 807 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 808 } 809 810 str = get_address(env, r2); 811 end = get_address(env, r1); 812 813 /* Lest we fail to service interrupts in a timely manner, limit the 814 amount of work we're willing to do. For now, let's cap at 8k. */ 815 for (len = 0; len < 0x2000; ++len) { 816 if (str + len == end) { 817 /* Character not found. R1 & R2 are unmodified. */ 818 env->cc_op = 2; 819 return; 820 } 821 v = cpu_ldub_data_ra(env, str + len, ra); 822 if (v == c) { 823 /* Character found. Set R1 to the location; R2 is unmodified. */ 824 env->cc_op = 1; 825 set_address(env, r1, str + len); 826 return; 827 } 828 } 829 830 /* CPU-determined bytes processed. Advance R2 to next byte to process. */ 831 env->cc_op = 3; 832 set_address(env, r2, str + len); 833 } 834 HELPER(srstu)835 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2) 836 { 837 uintptr_t ra = GETPC(); 838 uint32_t len; 839 uint16_t v, c = env->regs[0]; 840 uint64_t end, str, adj_end; 841 842 /* Bits 32-47 of R0 must be zero. */ 843 if (env->regs[0] & 0xffff0000u) { 844 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 845 } 846 847 str = get_address(env, r2); 848 end = get_address(env, r1); 849 850 /* If the LSB of the two addresses differ, use one extra byte. */ 851 adj_end = end + ((str ^ end) & 1); 852 853 /* Lest we fail to service interrupts in a timely manner, limit the 854 amount of work we're willing to do. For now, let's cap at 8k. */ 855 for (len = 0; len < 0x2000; len += 2) { 856 if (str + len == adj_end) { 857 /* End of input found. */ 858 env->cc_op = 2; 859 return; 860 } 861 v = cpu_lduw_data_ra(env, str + len, ra); 862 if (v == c) { 863 /* Character found. Set R1 to the location; R2 is unmodified. */ 864 env->cc_op = 1; 865 set_address(env, r1, str + len); 866 return; 867 } 868 } 869 870 /* CPU-determined bytes processed. Advance R2 to next byte to process. */ 871 env->cc_op = 3; 872 set_address(env, r2, str + len); 873 } 874 875 /* unsigned string compare (c is string terminator) */ HELPER(clst)876 Int128 HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2) 877 { 878 uintptr_t ra = GETPC(); 879 uint32_t len; 880 881 c = c & 0xff; 882 s1 = wrap_address(env, s1); 883 s2 = wrap_address(env, s2); 884 885 /* Lest we fail to service interrupts in a timely manner, limit the 886 amount of work we're willing to do. For now, let's cap at 8k. */ 887 for (len = 0; len < 0x2000; ++len) { 888 uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra); 889 uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra); 890 if (v1 == v2) { 891 if (v1 == c) { 892 /* Equal. CC=0, and don't advance the registers. */ 893 env->cc_op = 0; 894 return int128_make128(s2, s1); 895 } 896 } else { 897 /* Unequal. CC={1,2}, and advance the registers. Note that 898 the terminator need not be zero, but the string that contains 899 the terminator is by definition "low". */ 900 env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2); 901 return int128_make128(s2 + len, s1 + len); 902 } 903 } 904 905 /* CPU-determined bytes equal; advance the registers. */ 906 env->cc_op = 3; 907 return int128_make128(s2 + len, s1 + len); 908 } 909 910 /* move page */ HELPER(mvpg)911 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint32_t r1, uint32_t r2) 912 { 913 const uint64_t src = get_address(env, r2) & TARGET_PAGE_MASK; 914 const uint64_t dst = get_address(env, r1) & TARGET_PAGE_MASK; 915 const int mmu_idx = s390x_env_mmu_index(env, false); 916 const bool f = extract64(r0, 11, 1); 917 const bool s = extract64(r0, 10, 1); 918 const bool cco = extract64(r0, 8, 1); 919 uintptr_t ra = GETPC(); 920 S390Access srca, desta; 921 int exc; 922 923 if ((f && s) || extract64(r0, 12, 4)) { 924 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC()); 925 } 926 927 /* 928 * We always manually handle exceptions such that we can properly store 929 * r1/r2 to the lowcore on page-translation exceptions. 930 * 931 * TODO: Access key handling 932 */ 933 exc = access_prepare_nf(&srca, env, true, src, TARGET_PAGE_SIZE, 934 MMU_DATA_LOAD, mmu_idx, ra); 935 if (exc) { 936 if (cco) { 937 return 2; 938 } 939 goto inject_exc; 940 } 941 exc = access_prepare_nf(&desta, env, true, dst, TARGET_PAGE_SIZE, 942 MMU_DATA_STORE, mmu_idx, ra); 943 if (exc) { 944 if (cco && exc != PGM_PROTECTION) { 945 return 1; 946 } 947 goto inject_exc; 948 } 949 access_memmove(env, &desta, &srca, ra); 950 return 0; /* data moved */ 951 inject_exc: 952 #if !defined(CONFIG_USER_ONLY) 953 if (exc != PGM_ADDRESSING) { 954 stq_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, trans_exc_code), 955 env->tlb_fill_tec); 956 } 957 if (exc == PGM_PAGE_TRANS) { 958 stb_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, op_access_id), 959 r1 << 4 | r2); 960 } 961 #endif 962 tcg_s390_program_interrupt(env, exc, ra); 963 } 964 965 /* string copy */ HELPER(mvst)966 uint32_t HELPER(mvst)(CPUS390XState *env, uint32_t r1, uint32_t r2) 967 { 968 const int mmu_idx = s390x_env_mmu_index(env, false); 969 const uint64_t d = get_address(env, r1); 970 const uint64_t s = get_address(env, r2); 971 const uint8_t c = env->regs[0]; 972 const int len = MIN(-(d | TARGET_PAGE_MASK), -(s | TARGET_PAGE_MASK)); 973 S390Access srca, desta; 974 uintptr_t ra = GETPC(); 975 int i; 976 977 if (env->regs[0] & 0xffffff00ull) { 978 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 979 } 980 981 /* 982 * Our access should not exceed single pages, as we must not report access 983 * exceptions exceeding the actually copied range (which we don't know at 984 * this point). We might over-indicate watchpoints within the pages 985 * (if we ever care, we have to limit processing to a single byte). 986 */ 987 access_prepare(&srca, env, s, len, MMU_DATA_LOAD, mmu_idx, ra); 988 access_prepare(&desta, env, d, len, MMU_DATA_STORE, mmu_idx, ra); 989 990 set_helper_retaddr(ra); 991 for (i = 0; i < len; i++) { 992 const uint8_t v = access_get_byte(env, &srca, i, ra); 993 994 access_set_byte(env, &desta, i, v, ra); 995 if (v == c) { 996 clear_helper_retaddr(); 997 set_address_zero(env, r1, d + i); 998 return 1; 999 } 1000 } 1001 clear_helper_retaddr(); 1002 set_address_zero(env, r1, d + len); 1003 set_address_zero(env, r2, s + len); 1004 return 3; 1005 } 1006 1007 /* load access registers r1 to r3 from memory at a2 */ HELPER(lam)1008 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1009 { 1010 uintptr_t ra = GETPC(); 1011 int i; 1012 1013 if (a2 & 0x3) { 1014 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1015 } 1016 1017 for (i = r1;; i = (i + 1) % 16) { 1018 env->aregs[i] = cpu_ldl_data_ra(env, a2, ra); 1019 a2 += 4; 1020 1021 if (i == r3) { 1022 break; 1023 } 1024 } 1025 } 1026 1027 /* store access registers r1 to r3 in memory at a2 */ HELPER(stam)1028 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1029 { 1030 uintptr_t ra = GETPC(); 1031 int i; 1032 1033 if (a2 & 0x3) { 1034 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1035 } 1036 1037 for (i = r1;; i = (i + 1) % 16) { 1038 cpu_stl_data_ra(env, a2, env->aregs[i], ra); 1039 a2 += 4; 1040 1041 if (i == r3) { 1042 break; 1043 } 1044 } 1045 } 1046 1047 /* move long helper */ do_mvcl(CPUS390XState * env,uint64_t * dest,uint64_t * destlen,uint64_t * src,uint64_t * srclen,uint16_t pad,int wordsize,uintptr_t ra)1048 static inline uint32_t do_mvcl(CPUS390XState *env, 1049 uint64_t *dest, uint64_t *destlen, 1050 uint64_t *src, uint64_t *srclen, 1051 uint16_t pad, int wordsize, uintptr_t ra) 1052 { 1053 const int mmu_idx = s390x_env_mmu_index(env, false); 1054 int len = MIN(*destlen, -(*dest | TARGET_PAGE_MASK)); 1055 S390Access srca, desta; 1056 int i, cc; 1057 1058 if (*destlen == *srclen) { 1059 cc = 0; 1060 } else if (*destlen < *srclen) { 1061 cc = 1; 1062 } else { 1063 cc = 2; 1064 } 1065 1066 if (!*destlen) { 1067 return cc; 1068 } 1069 1070 /* 1071 * Only perform one type of type of operation (move/pad) at a time. 1072 * Stay within single pages. 1073 */ 1074 if (*srclen) { 1075 /* Copy the src array */ 1076 len = MIN(MIN(*srclen, -(*src | TARGET_PAGE_MASK)), len); 1077 *destlen -= len; 1078 *srclen -= len; 1079 access_prepare(&srca, env, *src, len, MMU_DATA_LOAD, mmu_idx, ra); 1080 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1081 access_memmove(env, &desta, &srca, ra); 1082 *src = wrap_address(env, *src + len); 1083 *dest = wrap_address(env, *dest + len); 1084 } else if (wordsize == 1) { 1085 /* Pad the remaining area */ 1086 *destlen -= len; 1087 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1088 access_memset(env, &desta, pad, ra); 1089 *dest = wrap_address(env, *dest + len); 1090 } else { 1091 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1092 set_helper_retaddr(ra); 1093 1094 /* The remaining length selects the padding byte. */ 1095 for (i = 0; i < len; (*destlen)--, i++) { 1096 if (*destlen & 1) { 1097 access_set_byte(env, &desta, i, pad, ra); 1098 } else { 1099 access_set_byte(env, &desta, i, pad >> 8, ra); 1100 } 1101 } 1102 clear_helper_retaddr(); 1103 *dest = wrap_address(env, *dest + len); 1104 } 1105 1106 return *destlen ? 3 : cc; 1107 } 1108 1109 /* move long */ HELPER(mvcl)1110 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2) 1111 { 1112 const int mmu_idx = s390x_env_mmu_index(env, false); 1113 uintptr_t ra = GETPC(); 1114 uint64_t destlen = env->regs[r1 + 1] & 0xffffff; 1115 uint64_t dest = get_address(env, r1); 1116 uint64_t srclen = env->regs[r2 + 1] & 0xffffff; 1117 uint64_t src = get_address(env, r2); 1118 uint8_t pad = env->regs[r2 + 1] >> 24; 1119 CPUState *cs = env_cpu(env); 1120 S390Access srca, desta; 1121 uint32_t cc, cur_len; 1122 1123 if (is_destructive_overlap(env, dest, src, MIN(srclen, destlen))) { 1124 cc = 3; 1125 } else if (srclen == destlen) { 1126 cc = 0; 1127 } else if (destlen < srclen) { 1128 cc = 1; 1129 } else { 1130 cc = 2; 1131 } 1132 1133 /* We might have to zero-out some bits even if there was no action. */ 1134 if (unlikely(!destlen || cc == 3)) { 1135 set_address_zero(env, r2, src); 1136 set_address_zero(env, r1, dest); 1137 return cc; 1138 } else if (!srclen) { 1139 set_address_zero(env, r2, src); 1140 } 1141 1142 /* 1143 * Only perform one type of type of operation (move/pad) in one step. 1144 * Stay within single pages. 1145 */ 1146 while (destlen) { 1147 cur_len = MIN(destlen, -(dest | TARGET_PAGE_MASK)); 1148 if (!srclen) { 1149 access_prepare(&desta, env, dest, cur_len, 1150 MMU_DATA_STORE, mmu_idx, ra); 1151 access_memset(env, &desta, pad, ra); 1152 } else { 1153 cur_len = MIN(MIN(srclen, -(src | TARGET_PAGE_MASK)), cur_len); 1154 1155 access_prepare(&srca, env, src, cur_len, 1156 MMU_DATA_LOAD, mmu_idx, ra); 1157 access_prepare(&desta, env, dest, cur_len, 1158 MMU_DATA_STORE, mmu_idx, ra); 1159 access_memmove(env, &desta, &srca, ra); 1160 src = wrap_address(env, src + cur_len); 1161 srclen -= cur_len; 1162 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen); 1163 set_address_zero(env, r2, src); 1164 } 1165 dest = wrap_address(env, dest + cur_len); 1166 destlen -= cur_len; 1167 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen); 1168 set_address_zero(env, r1, dest); 1169 1170 /* 1171 * MVCL is interruptible. Return to the main loop if requested after 1172 * writing back all state to registers. If no interrupt will get 1173 * injected, we'll end up back in this handler and continue processing 1174 * the remaining parts. 1175 */ 1176 if (destlen && unlikely(cpu_loop_exit_requested(cs))) { 1177 cpu_loop_exit_restore(cs, ra); 1178 } 1179 } 1180 return cc; 1181 } 1182 1183 /* move long extended */ HELPER(mvcle)1184 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1185 uint32_t r3) 1186 { 1187 uintptr_t ra = GETPC(); 1188 uint64_t destlen = get_length(env, r1 + 1); 1189 uint64_t dest = get_address(env, r1); 1190 uint64_t srclen = get_length(env, r3 + 1); 1191 uint64_t src = get_address(env, r3); 1192 uint8_t pad = a2; 1193 uint32_t cc; 1194 1195 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra); 1196 1197 set_length(env, r1 + 1, destlen); 1198 set_length(env, r3 + 1, srclen); 1199 set_address(env, r1, dest); 1200 set_address(env, r3, src); 1201 1202 return cc; 1203 } 1204 1205 /* move long unicode */ HELPER(mvclu)1206 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1207 uint32_t r3) 1208 { 1209 uintptr_t ra = GETPC(); 1210 uint64_t destlen = get_length(env, r1 + 1); 1211 uint64_t dest = get_address(env, r1); 1212 uint64_t srclen = get_length(env, r3 + 1); 1213 uint64_t src = get_address(env, r3); 1214 uint16_t pad = a2; 1215 uint32_t cc; 1216 1217 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra); 1218 1219 set_length(env, r1 + 1, destlen); 1220 set_length(env, r3 + 1, srclen); 1221 set_address(env, r1, dest); 1222 set_address(env, r3, src); 1223 1224 return cc; 1225 } 1226 1227 /* compare logical long helper */ do_clcl(CPUS390XState * env,uint64_t * src1,uint64_t * src1len,uint64_t * src3,uint64_t * src3len,uint16_t pad,uint64_t limit,int wordsize,uintptr_t ra)1228 static inline uint32_t do_clcl(CPUS390XState *env, 1229 uint64_t *src1, uint64_t *src1len, 1230 uint64_t *src3, uint64_t *src3len, 1231 uint16_t pad, uint64_t limit, 1232 int wordsize, uintptr_t ra) 1233 { 1234 uint64_t len = MAX(*src1len, *src3len); 1235 uint32_t cc = 0; 1236 1237 check_alignment(env, *src1len | *src3len, wordsize, ra); 1238 1239 if (!len) { 1240 return cc; 1241 } 1242 1243 /* Lest we fail to service interrupts in a timely manner, limit the 1244 amount of work we're willing to do. */ 1245 if (len > limit) { 1246 len = limit; 1247 cc = 3; 1248 } 1249 1250 for (; len; len -= wordsize) { 1251 uint16_t v1 = pad; 1252 uint16_t v3 = pad; 1253 1254 if (*src1len) { 1255 v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra); 1256 } 1257 if (*src3len) { 1258 v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra); 1259 } 1260 1261 if (v1 != v3) { 1262 cc = (v1 < v3) ? 1 : 2; 1263 break; 1264 } 1265 1266 if (*src1len) { 1267 *src1 += wordsize; 1268 *src1len -= wordsize; 1269 } 1270 if (*src3len) { 1271 *src3 += wordsize; 1272 *src3len -= wordsize; 1273 } 1274 } 1275 1276 return cc; 1277 } 1278 1279 1280 /* compare logical long */ HELPER(clcl)1281 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2) 1282 { 1283 uintptr_t ra = GETPC(); 1284 uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24); 1285 uint64_t src1 = get_address(env, r1); 1286 uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24); 1287 uint64_t src3 = get_address(env, r2); 1288 uint8_t pad = env->regs[r2 + 1] >> 24; 1289 uint32_t cc; 1290 1291 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra); 1292 1293 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len); 1294 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len); 1295 set_address(env, r1, src1); 1296 set_address(env, r2, src3); 1297 1298 return cc; 1299 } 1300 1301 /* compare logical long extended memcompare insn with padding */ HELPER(clcle)1302 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1303 uint32_t r3) 1304 { 1305 uintptr_t ra = GETPC(); 1306 uint64_t src1len = get_length(env, r1 + 1); 1307 uint64_t src1 = get_address(env, r1); 1308 uint64_t src3len = get_length(env, r3 + 1); 1309 uint64_t src3 = get_address(env, r3); 1310 uint8_t pad = a2; 1311 uint32_t cc; 1312 1313 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra); 1314 1315 set_length(env, r1 + 1, src1len); 1316 set_length(env, r3 + 1, src3len); 1317 set_address(env, r1, src1); 1318 set_address(env, r3, src3); 1319 1320 return cc; 1321 } 1322 1323 /* compare logical long unicode memcompare insn with padding */ HELPER(clclu)1324 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1325 uint32_t r3) 1326 { 1327 uintptr_t ra = GETPC(); 1328 uint64_t src1len = get_length(env, r1 + 1); 1329 uint64_t src1 = get_address(env, r1); 1330 uint64_t src3len = get_length(env, r3 + 1); 1331 uint64_t src3 = get_address(env, r3); 1332 uint16_t pad = a2; 1333 uint32_t cc = 0; 1334 1335 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra); 1336 1337 set_length(env, r1 + 1, src1len); 1338 set_length(env, r3 + 1, src3len); 1339 set_address(env, r1, src1); 1340 set_address(env, r3, src3); 1341 1342 return cc; 1343 } 1344 1345 /* checksum */ HELPER(cksm)1346 Int128 HELPER(cksm)(CPUS390XState *env, uint64_t r1, 1347 uint64_t src, uint64_t src_len) 1348 { 1349 uintptr_t ra = GETPC(); 1350 uint64_t max_len, len; 1351 uint64_t cksm = (uint32_t)r1; 1352 1353 /* Lest we fail to service interrupts in a timely manner, limit the 1354 amount of work we're willing to do. For now, let's cap at 8k. */ 1355 max_len = (src_len > 0x2000 ? 0x2000 : src_len); 1356 1357 /* Process full words as available. */ 1358 for (len = 0; len + 4 <= max_len; len += 4, src += 4) { 1359 cksm += (uint32_t)cpu_ldl_data_ra(env, src, ra); 1360 } 1361 1362 switch (max_len - len) { 1363 case 1: 1364 cksm += cpu_ldub_data_ra(env, src, ra) << 24; 1365 len += 1; 1366 break; 1367 case 2: 1368 cksm += cpu_lduw_data_ra(env, src, ra) << 16; 1369 len += 2; 1370 break; 1371 case 3: 1372 cksm += cpu_lduw_data_ra(env, src, ra) << 16; 1373 cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8; 1374 len += 3; 1375 break; 1376 } 1377 1378 /* Fold the carry from the checksum. Note that we can see carry-out 1379 during folding more than once (but probably not more than twice). */ 1380 while (cksm > 0xffffffffull) { 1381 cksm = (uint32_t)cksm + (cksm >> 32); 1382 } 1383 1384 /* Indicate whether or not we've processed everything. */ 1385 env->cc_op = (len == src_len ? 0 : 3); 1386 1387 /* Return both cksm and processed length. */ 1388 return int128_make128(cksm, len); 1389 } 1390 HELPER(pack)1391 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src) 1392 { 1393 uintptr_t ra = GETPC(); 1394 int len_dest = len >> 4; 1395 int len_src = len & 0xf; 1396 uint8_t b; 1397 1398 dest += len_dest; 1399 src += len_src; 1400 1401 /* last byte is special, it only flips the nibbles */ 1402 b = cpu_ldub_data_ra(env, src, ra); 1403 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra); 1404 src--; 1405 len_src--; 1406 1407 /* now pack every value */ 1408 while (len_dest > 0) { 1409 b = 0; 1410 1411 if (len_src >= 0) { 1412 b = cpu_ldub_data_ra(env, src, ra) & 0x0f; 1413 src--; 1414 len_src--; 1415 } 1416 if (len_src >= 0) { 1417 b |= cpu_ldub_data_ra(env, src, ra) << 4; 1418 src--; 1419 len_src--; 1420 } 1421 1422 len_dest--; 1423 dest--; 1424 cpu_stb_data_ra(env, dest, b, ra); 1425 } 1426 } 1427 do_pkau(CPUS390XState * env,uint64_t dest,uint64_t src,uint32_t srclen,int ssize,uintptr_t ra)1428 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src, 1429 uint32_t srclen, int ssize, uintptr_t ra) 1430 { 1431 int i; 1432 /* The destination operand is always 16 bytes long. */ 1433 const int destlen = 16; 1434 1435 /* The operands are processed from right to left. */ 1436 src += srclen - 1; 1437 dest += destlen - 1; 1438 1439 for (i = 0; i < destlen; i++) { 1440 uint8_t b = 0; 1441 1442 /* Start with a positive sign */ 1443 if (i == 0) { 1444 b = 0xc; 1445 } else if (srclen > ssize) { 1446 b = cpu_ldub_data_ra(env, src, ra) & 0x0f; 1447 src -= ssize; 1448 srclen -= ssize; 1449 } 1450 1451 if (srclen > ssize) { 1452 b |= cpu_ldub_data_ra(env, src, ra) << 4; 1453 src -= ssize; 1454 srclen -= ssize; 1455 } 1456 1457 cpu_stb_data_ra(env, dest, b, ra); 1458 dest--; 1459 } 1460 } 1461 1462 HELPER(pka)1463 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src, 1464 uint32_t srclen) 1465 { 1466 do_pkau(env, dest, src, srclen, 1, GETPC()); 1467 } 1468 HELPER(pku)1469 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src, 1470 uint32_t srclen) 1471 { 1472 do_pkau(env, dest, src, srclen, 2, GETPC()); 1473 } 1474 HELPER(unpk)1475 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest, 1476 uint64_t src) 1477 { 1478 uintptr_t ra = GETPC(); 1479 int len_dest = len >> 4; 1480 int len_src = len & 0xf; 1481 uint8_t b; 1482 int second_nibble = 0; 1483 1484 dest += len_dest; 1485 src += len_src; 1486 1487 /* last byte is special, it only flips the nibbles */ 1488 b = cpu_ldub_data_ra(env, src, ra); 1489 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra); 1490 src--; 1491 len_src--; 1492 1493 /* now pad every nibble with 0xf0 */ 1494 1495 while (len_dest > 0) { 1496 uint8_t cur_byte = 0; 1497 1498 if (len_src > 0) { 1499 cur_byte = cpu_ldub_data_ra(env, src, ra); 1500 } 1501 1502 len_dest--; 1503 dest--; 1504 1505 /* only advance one nibble at a time */ 1506 if (second_nibble) { 1507 cur_byte >>= 4; 1508 len_src--; 1509 src--; 1510 } 1511 second_nibble = !second_nibble; 1512 1513 /* digit */ 1514 cur_byte = (cur_byte & 0xf); 1515 /* zone bits */ 1516 cur_byte |= 0xf0; 1517 1518 cpu_stb_data_ra(env, dest, cur_byte, ra); 1519 } 1520 } 1521 do_unpkau(CPUS390XState * env,uint64_t dest,uint32_t destlen,int dsize,uint64_t src,uintptr_t ra)1522 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest, 1523 uint32_t destlen, int dsize, uint64_t src, 1524 uintptr_t ra) 1525 { 1526 int i; 1527 uint32_t cc; 1528 uint8_t b; 1529 /* The source operand is always 16 bytes long. */ 1530 const int srclen = 16; 1531 1532 /* The operands are processed from right to left. */ 1533 src += srclen - 1; 1534 dest += destlen - dsize; 1535 1536 /* Check for the sign. */ 1537 b = cpu_ldub_data_ra(env, src, ra); 1538 src--; 1539 switch (b & 0xf) { 1540 case 0xa: 1541 case 0xc: 1542 case 0xe ... 0xf: 1543 cc = 0; /* plus */ 1544 break; 1545 case 0xb: 1546 case 0xd: 1547 cc = 1; /* minus */ 1548 break; 1549 default: 1550 case 0x0 ... 0x9: 1551 cc = 3; /* invalid */ 1552 break; 1553 } 1554 1555 /* Now pad every nibble with 0x30, advancing one nibble at a time. */ 1556 for (i = 0; i < destlen; i += dsize) { 1557 if (i == (31 * dsize)) { 1558 /* If length is 32/64 bytes, the leftmost byte is 0. */ 1559 b = 0; 1560 } else if (i % (2 * dsize)) { 1561 b = cpu_ldub_data_ra(env, src, ra); 1562 src--; 1563 } else { 1564 b >>= 4; 1565 } 1566 cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra); 1567 dest -= dsize; 1568 } 1569 1570 return cc; 1571 } 1572 HELPER(unpka)1573 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen, 1574 uint64_t src) 1575 { 1576 return do_unpkau(env, dest, destlen, 1, src, GETPC()); 1577 } 1578 HELPER(unpku)1579 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen, 1580 uint64_t src) 1581 { 1582 return do_unpkau(env, dest, destlen, 2, src, GETPC()); 1583 } 1584 HELPER(tp)1585 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen) 1586 { 1587 uintptr_t ra = GETPC(); 1588 uint32_t cc = 0; 1589 int i; 1590 1591 for (i = 0; i < destlen; i++) { 1592 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra); 1593 /* digit */ 1594 cc |= (b & 0xf0) > 0x90 ? 2 : 0; 1595 1596 if (i == (destlen - 1)) { 1597 /* sign */ 1598 cc |= (b & 0xf) < 0xa ? 1 : 0; 1599 } else { 1600 /* digit */ 1601 cc |= (b & 0xf) > 0x9 ? 2 : 0; 1602 } 1603 } 1604 1605 return cc; 1606 } 1607 do_helper_tr(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1608 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array, 1609 uint64_t trans, uintptr_t ra) 1610 { 1611 uint32_t i; 1612 1613 for (i = 0; i <= len; i++) { 1614 uint8_t byte = cpu_ldub_data_ra(env, array + i, ra); 1615 uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra); 1616 cpu_stb_data_ra(env, array + i, new_byte, ra); 1617 } 1618 1619 return env->cc_op; 1620 } 1621 HELPER(tr)1622 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array, 1623 uint64_t trans) 1624 { 1625 do_helper_tr(env, len, array, trans, GETPC()); 1626 } 1627 HELPER(tre)1628 Int128 HELPER(tre)(CPUS390XState *env, uint64_t array, 1629 uint64_t len, uint64_t trans) 1630 { 1631 uintptr_t ra = GETPC(); 1632 uint8_t end = env->regs[0] & 0xff; 1633 uint64_t l = len; 1634 uint64_t i; 1635 uint32_t cc = 0; 1636 1637 if (!(env->psw.mask & PSW_MASK_64)) { 1638 array &= 0x7fffffff; 1639 l = (uint32_t)l; 1640 } 1641 1642 /* Lest we fail to service interrupts in a timely manner, limit the 1643 amount of work we're willing to do. For now, let's cap at 8k. */ 1644 if (l > 0x2000) { 1645 l = 0x2000; 1646 cc = 3; 1647 } 1648 1649 for (i = 0; i < l; i++) { 1650 uint8_t byte, new_byte; 1651 1652 byte = cpu_ldub_data_ra(env, array + i, ra); 1653 1654 if (byte == end) { 1655 cc = 1; 1656 break; 1657 } 1658 1659 new_byte = cpu_ldub_data_ra(env, trans + byte, ra); 1660 cpu_stb_data_ra(env, array + i, new_byte, ra); 1661 } 1662 1663 env->cc_op = cc; 1664 return int128_make128(len - i, array + i); 1665 } 1666 do_helper_trt(CPUS390XState * env,int len,uint64_t array,uint64_t trans,int inc,uintptr_t ra)1667 static inline uint32_t do_helper_trt(CPUS390XState *env, int len, 1668 uint64_t array, uint64_t trans, 1669 int inc, uintptr_t ra) 1670 { 1671 int i; 1672 1673 for (i = 0; i <= len; i++) { 1674 uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra); 1675 uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra); 1676 1677 if (sbyte != 0) { 1678 set_address(env, 1, array + i * inc); 1679 env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte); 1680 return (i == len) ? 2 : 1; 1681 } 1682 } 1683 1684 return 0; 1685 } 1686 do_helper_trt_fwd(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1687 static uint32_t do_helper_trt_fwd(CPUS390XState *env, uint32_t len, 1688 uint64_t array, uint64_t trans, 1689 uintptr_t ra) 1690 { 1691 return do_helper_trt(env, len, array, trans, 1, ra); 1692 } 1693 HELPER(trt)1694 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array, 1695 uint64_t trans) 1696 { 1697 return do_helper_trt(env, len, array, trans, 1, GETPC()); 1698 } 1699 do_helper_trt_bkwd(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1700 static uint32_t do_helper_trt_bkwd(CPUS390XState *env, uint32_t len, 1701 uint64_t array, uint64_t trans, 1702 uintptr_t ra) 1703 { 1704 return do_helper_trt(env, len, array, trans, -1, ra); 1705 } 1706 HELPER(trtr)1707 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array, 1708 uint64_t trans) 1709 { 1710 return do_helper_trt(env, len, array, trans, -1, GETPC()); 1711 } 1712 1713 /* Translate one/two to one/two */ HELPER(trXX)1714 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2, 1715 uint32_t tst, uint32_t sizes) 1716 { 1717 uintptr_t ra = GETPC(); 1718 int dsize = (sizes & 1) ? 1 : 2; 1719 int ssize = (sizes & 2) ? 1 : 2; 1720 uint64_t tbl = get_address(env, 1); 1721 uint64_t dst = get_address(env, r1); 1722 uint64_t len = get_length(env, r1 + 1); 1723 uint64_t src = get_address(env, r2); 1724 uint32_t cc = 3; 1725 int i; 1726 1727 /* The lower address bits of TBL are ignored. For TROO, TROT, it's 1728 the low 3 bits (double-word aligned). For TRTO, TRTT, it's either 1729 the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */ 1730 if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) { 1731 tbl &= -4096; 1732 } else { 1733 tbl &= -8; 1734 } 1735 1736 check_alignment(env, len, ssize, ra); 1737 1738 /* Lest we fail to service interrupts in a timely manner, */ 1739 /* limit the amount of work we're willing to do. */ 1740 for (i = 0; i < 0x2000; i++) { 1741 uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra); 1742 uint64_t tble = tbl + (sval * dsize); 1743 uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra); 1744 if (dval == tst) { 1745 cc = 1; 1746 break; 1747 } 1748 cpu_stsize_data_ra(env, dst, dval, dsize, ra); 1749 1750 len -= ssize; 1751 src += ssize; 1752 dst += dsize; 1753 1754 if (len == 0) { 1755 cc = 0; 1756 break; 1757 } 1758 } 1759 1760 set_address(env, r1, dst); 1761 set_length(env, r1 + 1, len); 1762 set_address(env, r2, src); 1763 1764 return cc; 1765 } 1766 do_csst(CPUS390XState * env,uint32_t r3,uint64_t a1,uint64_t a2,bool parallel)1767 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, 1768 uint64_t a2, bool parallel) 1769 { 1770 uint32_t mem_idx = s390x_env_mmu_index(env, false); 1771 MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, mem_idx); 1772 MemOpIdx oi8 = make_memop_idx(MO_TE | MO_64, mem_idx); 1773 MemOpIdx oi4 = make_memop_idx(MO_TE | MO_32, mem_idx); 1774 MemOpIdx oi2 = make_memop_idx(MO_TE | MO_16, mem_idx); 1775 MemOpIdx oi1 = make_memop_idx(MO_8, mem_idx); 1776 uintptr_t ra = GETPC(); 1777 uint32_t fc = extract32(env->regs[0], 0, 8); 1778 uint32_t sc = extract32(env->regs[0], 8, 8); 1779 uint64_t pl = get_address(env, 1) & -16; 1780 uint64_t svh, svl; 1781 uint32_t cc; 1782 1783 /* Sanity check the function code and storage characteristic. */ 1784 if (fc > 1 || sc > 3) { 1785 if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) { 1786 goto spec_exception; 1787 } 1788 if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) { 1789 goto spec_exception; 1790 } 1791 } 1792 1793 /* Sanity check the alignments. */ 1794 if (extract32(a1, 0, fc + 2) || extract32(a2, 0, sc)) { 1795 goto spec_exception; 1796 } 1797 1798 /* Sanity check writability of the store address. */ 1799 probe_write(env, a2, 1 << sc, mem_idx, ra); 1800 1801 /* 1802 * Note that the compare-and-swap is atomic, and the store is atomic, 1803 * but the complete operation is not. Therefore we do not need to 1804 * assert serial context in order to implement this. That said, 1805 * restart early if we can't support either operation that is supposed 1806 * to be atomic. 1807 */ 1808 if (parallel) { 1809 uint32_t max = 2; 1810 #ifdef CONFIG_ATOMIC64 1811 max = 3; 1812 #endif 1813 if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) || 1814 (HAVE_ATOMIC128_RW ? 0 : sc > max)) { 1815 cpu_loop_exit_atomic(env_cpu(env), ra); 1816 } 1817 } 1818 1819 /* 1820 * All loads happen before all stores. For simplicity, load the entire 1821 * store value area from the parameter list. 1822 */ 1823 svh = cpu_ldq_mmu(env, pl + 16, oi8, ra); 1824 svl = cpu_ldq_mmu(env, pl + 24, oi8, ra); 1825 1826 switch (fc) { 1827 case 0: 1828 { 1829 uint32_t nv = cpu_ldl_mmu(env, pl, oi4, ra); 1830 uint32_t cv = env->regs[r3]; 1831 uint32_t ov; 1832 1833 if (parallel) { 1834 ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi4, ra); 1835 } else { 1836 ov = cpu_ldl_mmu(env, a1, oi4, ra); 1837 cpu_stl_mmu(env, a1, (ov == cv ? nv : ov), oi4, ra); 1838 } 1839 cc = (ov != cv); 1840 env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov); 1841 } 1842 break; 1843 1844 case 1: 1845 { 1846 uint64_t nv = cpu_ldq_mmu(env, pl, oi8, ra); 1847 uint64_t cv = env->regs[r3]; 1848 uint64_t ov; 1849 1850 if (parallel) { 1851 #ifdef CONFIG_ATOMIC64 1852 ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi8, ra); 1853 #else 1854 /* Note that we asserted !parallel above. */ 1855 g_assert_not_reached(); 1856 #endif 1857 } else { 1858 ov = cpu_ldq_mmu(env, a1, oi8, ra); 1859 cpu_stq_mmu(env, a1, (ov == cv ? nv : ov), oi8, ra); 1860 } 1861 cc = (ov != cv); 1862 env->regs[r3] = ov; 1863 } 1864 break; 1865 1866 case 2: 1867 { 1868 Int128 nv = cpu_ld16_mmu(env, pl, oi16, ra); 1869 Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]); 1870 Int128 ov; 1871 1872 if (!parallel) { 1873 ov = cpu_ld16_mmu(env, a1, oi16, ra); 1874 cc = !int128_eq(ov, cv); 1875 if (cc) { 1876 nv = ov; 1877 } 1878 cpu_st16_mmu(env, a1, nv, oi16, ra); 1879 } else if (HAVE_CMPXCHG128) { 1880 ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi16, ra); 1881 cc = !int128_eq(ov, cv); 1882 } else { 1883 /* Note that we asserted !parallel above. */ 1884 g_assert_not_reached(); 1885 } 1886 1887 env->regs[r3 + 0] = int128_gethi(ov); 1888 env->regs[r3 + 1] = int128_getlo(ov); 1889 } 1890 break; 1891 1892 default: 1893 g_assert_not_reached(); 1894 } 1895 1896 /* Store only if the comparison succeeded. Note that above we use a pair 1897 of 64-bit big-endian loads, so for sc < 3 we must extract the value 1898 from the most-significant bits of svh. */ 1899 if (cc == 0) { 1900 switch (sc) { 1901 case 0: 1902 cpu_stb_mmu(env, a2, svh >> 56, oi1, ra); 1903 break; 1904 case 1: 1905 cpu_stw_mmu(env, a2, svh >> 48, oi2, ra); 1906 break; 1907 case 2: 1908 cpu_stl_mmu(env, a2, svh >> 32, oi4, ra); 1909 break; 1910 case 3: 1911 cpu_stq_mmu(env, a2, svh, oi8, ra); 1912 break; 1913 case 4: 1914 cpu_st16_mmu(env, a2, int128_make128(svl, svh), oi16, ra); 1915 break; 1916 default: 1917 g_assert_not_reached(); 1918 } 1919 } 1920 1921 return cc; 1922 1923 spec_exception: 1924 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1925 } 1926 HELPER(csst)1927 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2) 1928 { 1929 return do_csst(env, r3, a1, a2, false); 1930 } 1931 HELPER(csst_parallel)1932 uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1, 1933 uint64_t a2) 1934 { 1935 return do_csst(env, r3, a1, a2, true); 1936 } 1937 1938 #if !defined(CONFIG_USER_ONLY) HELPER(lctlg)1939 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1940 { 1941 uintptr_t ra = GETPC(); 1942 bool PERchanged = false; 1943 uint64_t src = a2; 1944 uint32_t i; 1945 1946 if (src & 0x7) { 1947 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1948 } 1949 1950 for (i = r1;; i = (i + 1) % 16) { 1951 uint64_t val = cpu_ldq_data_ra(env, src, ra); 1952 if (env->cregs[i] != val && i >= 9 && i <= 11) { 1953 PERchanged = true; 1954 } 1955 env->cregs[i] = val; 1956 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n", 1957 i, src, val); 1958 src += sizeof(uint64_t); 1959 1960 if (i == r3) { 1961 break; 1962 } 1963 } 1964 1965 if (PERchanged && env->psw.mask & PSW_MASK_PER) { 1966 s390_cpu_recompute_watchpoints(env_cpu(env)); 1967 } 1968 1969 tlb_flush(env_cpu(env)); 1970 } 1971 HELPER(lctl)1972 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1973 { 1974 uintptr_t ra = GETPC(); 1975 bool PERchanged = false; 1976 uint64_t src = a2; 1977 uint32_t i; 1978 1979 if (src & 0x3) { 1980 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1981 } 1982 1983 for (i = r1;; i = (i + 1) % 16) { 1984 uint32_t val = cpu_ldl_data_ra(env, src, ra); 1985 if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) { 1986 PERchanged = true; 1987 } 1988 env->cregs[i] = deposit64(env->cregs[i], 0, 32, val); 1989 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val); 1990 src += sizeof(uint32_t); 1991 1992 if (i == r3) { 1993 break; 1994 } 1995 } 1996 1997 if (PERchanged && env->psw.mask & PSW_MASK_PER) { 1998 s390_cpu_recompute_watchpoints(env_cpu(env)); 1999 } 2000 2001 tlb_flush(env_cpu(env)); 2002 } 2003 HELPER(stctg)2004 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 2005 { 2006 uintptr_t ra = GETPC(); 2007 uint64_t dest = a2; 2008 uint32_t i; 2009 2010 if (dest & 0x7) { 2011 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2012 } 2013 2014 for (i = r1;; i = (i + 1) % 16) { 2015 cpu_stq_data_ra(env, dest, env->cregs[i], ra); 2016 dest += sizeof(uint64_t); 2017 2018 if (i == r3) { 2019 break; 2020 } 2021 } 2022 } 2023 HELPER(stctl)2024 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 2025 { 2026 uintptr_t ra = GETPC(); 2027 uint64_t dest = a2; 2028 uint32_t i; 2029 2030 if (dest & 0x3) { 2031 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2032 } 2033 2034 for (i = r1;; i = (i + 1) % 16) { 2035 cpu_stl_data_ra(env, dest, env->cregs[i], ra); 2036 dest += sizeof(uint32_t); 2037 2038 if (i == r3) { 2039 break; 2040 } 2041 } 2042 } 2043 HELPER(testblock)2044 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr) 2045 { 2046 uintptr_t ra = GETPC(); 2047 int i; 2048 2049 real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK; 2050 2051 for (i = 0; i < TARGET_PAGE_SIZE; i += 8) { 2052 cpu_stq_mmuidx_ra(env, real_addr + i, 0, MMU_REAL_IDX, ra); 2053 } 2054 2055 return 0; 2056 } 2057 HELPER(tprot)2058 uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2) 2059 { 2060 S390CPU *cpu = env_archcpu(env); 2061 CPUState *cs = env_cpu(env); 2062 2063 /* 2064 * TODO: we currently don't handle all access protection types 2065 * (including access-list and key-controlled) as well as AR mode. 2066 */ 2067 if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) { 2068 /* Fetching permitted; storing permitted */ 2069 return 0; 2070 } 2071 2072 if (env->int_pgm_code == PGM_PROTECTION) { 2073 /* retry if reading is possible */ 2074 cs->exception_index = -1; 2075 if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) { 2076 /* Fetching permitted; storing not permitted */ 2077 return 1; 2078 } 2079 } 2080 2081 switch (env->int_pgm_code) { 2082 case PGM_PROTECTION: 2083 /* Fetching not permitted; storing not permitted */ 2084 cs->exception_index = -1; 2085 return 2; 2086 case PGM_ADDRESSING: 2087 case PGM_TRANS_SPEC: 2088 /* exceptions forwarded to the guest */ 2089 s390_cpu_virt_mem_handle_exc(cpu, GETPC()); 2090 return 0; 2091 } 2092 2093 /* Translation not available */ 2094 cs->exception_index = -1; 2095 return 3; 2096 } 2097 2098 /* insert storage key extended */ HELPER(iske)2099 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2) 2100 { 2101 static S390SKeysState *ss; 2102 static S390SKeysClass *skeyclass; 2103 uint64_t addr = wrap_address(env, r2); 2104 uint8_t key; 2105 int rc; 2106 2107 addr = mmu_real2abs(env, addr); 2108 if (!mmu_absolute_addr_valid(addr, false)) { 2109 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2110 } 2111 2112 if (unlikely(!ss)) { 2113 ss = s390_get_skeys_device(); 2114 skeyclass = S390_SKEYS_GET_CLASS(ss); 2115 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2116 tlb_flush_all_cpus_synced(env_cpu(env)); 2117 } 2118 } 2119 2120 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2121 if (rc) { 2122 return 0; 2123 } 2124 return key; 2125 } 2126 2127 /* set storage key extended */ HELPER(sske)2128 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2) 2129 { 2130 static S390SKeysState *ss; 2131 static S390SKeysClass *skeyclass; 2132 uint64_t addr = wrap_address(env, r2); 2133 uint8_t key; 2134 2135 addr = mmu_real2abs(env, addr); 2136 if (!mmu_absolute_addr_valid(addr, false)) { 2137 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2138 } 2139 2140 if (unlikely(!ss)) { 2141 ss = s390_get_skeys_device(); 2142 skeyclass = S390_SKEYS_GET_CLASS(ss); 2143 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2144 tlb_flush_all_cpus_synced(env_cpu(env)); 2145 } 2146 } 2147 2148 key = r1 & 0xfe; 2149 s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2150 /* 2151 * As we can only flush by virtual address and not all the entries 2152 * that point to a physical address we have to flush the whole TLB. 2153 */ 2154 tlb_flush_all_cpus_synced(env_cpu(env)); 2155 } 2156 2157 /* reset reference bit extended */ HELPER(rrbe)2158 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2) 2159 { 2160 uint64_t addr = wrap_address(env, r2); 2161 static S390SKeysState *ss; 2162 static S390SKeysClass *skeyclass; 2163 uint8_t re, key; 2164 int rc; 2165 2166 addr = mmu_real2abs(env, addr); 2167 if (!mmu_absolute_addr_valid(addr, false)) { 2168 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2169 } 2170 2171 if (unlikely(!ss)) { 2172 ss = s390_get_skeys_device(); 2173 skeyclass = S390_SKEYS_GET_CLASS(ss); 2174 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2175 tlb_flush_all_cpus_synced(env_cpu(env)); 2176 } 2177 } 2178 2179 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2180 if (rc) { 2181 return 0; 2182 } 2183 2184 re = key & (SK_R | SK_C); 2185 key &= ~SK_R; 2186 2187 rc = s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2188 if (rc) { 2189 return 0; 2190 } 2191 /* 2192 * As we can only flush by virtual address and not all the entries 2193 * that point to a physical address we have to flush the whole TLB. 2194 */ 2195 tlb_flush_all_cpus_synced(env_cpu(env)); 2196 2197 /* 2198 * cc 2199 * 2200 * 0 Reference bit zero; change bit zero 2201 * 1 Reference bit zero; change bit one 2202 * 2 Reference bit one; change bit zero 2203 * 3 Reference bit one; change bit one 2204 */ 2205 2206 return re >> 1; 2207 } 2208 HELPER(mvcs)2209 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2, 2210 uint64_t key) 2211 { 2212 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2213 S390Access srca, desta; 2214 uintptr_t ra = GETPC(); 2215 int cc = 0; 2216 2217 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n", 2218 __func__, l, a1, a2); 2219 2220 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) || 2221 psw_as == AS_HOME || psw_as == AS_ACCREG) { 2222 s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2223 } 2224 2225 if (!psw_key_valid(env, (key >> 4) & 0xf)) { 2226 s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2227 } 2228 2229 l = wrap_length32(env, l); 2230 if (l > 256) { 2231 /* max 256 */ 2232 l = 256; 2233 cc = 3; 2234 } else if (!l) { 2235 return cc; 2236 } 2237 2238 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_PRIMARY_IDX, ra); 2239 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_SECONDARY_IDX, ra); 2240 access_memmove(env, &desta, &srca, ra); 2241 return cc; 2242 } 2243 HELPER(mvcp)2244 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2, 2245 uint64_t key) 2246 { 2247 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2248 S390Access srca, desta; 2249 uintptr_t ra = GETPC(); 2250 int cc = 0; 2251 2252 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n", 2253 __func__, l, a1, a2); 2254 2255 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) || 2256 psw_as == AS_HOME || psw_as == AS_ACCREG) { 2257 s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2258 } 2259 2260 if (!psw_key_valid(env, (key >> 4) & 0xf)) { 2261 s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2262 } 2263 2264 l = wrap_length32(env, l); 2265 if (l > 256) { 2266 /* max 256 */ 2267 l = 256; 2268 cc = 3; 2269 } else if (!l) { 2270 return cc; 2271 } 2272 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_SECONDARY_IDX, ra); 2273 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_PRIMARY_IDX, ra); 2274 access_memmove(env, &desta, &srca, ra); 2275 return cc; 2276 } 2277 HELPER(idte)2278 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4) 2279 { 2280 CPUState *cs = env_cpu(env); 2281 const uintptr_t ra = GETPC(); 2282 uint64_t table, entry, raddr; 2283 uint16_t entries, i, index = 0; 2284 2285 if (r2 & 0xff000) { 2286 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2287 } 2288 2289 if (!(r2 & 0x800)) { 2290 /* invalidation-and-clearing operation */ 2291 table = r1 & ASCE_ORIGIN; 2292 entries = (r2 & 0x7ff) + 1; 2293 2294 switch (r1 & ASCE_TYPE_MASK) { 2295 case ASCE_TYPE_REGION1: 2296 index = (r2 >> 53) & 0x7ff; 2297 break; 2298 case ASCE_TYPE_REGION2: 2299 index = (r2 >> 42) & 0x7ff; 2300 break; 2301 case ASCE_TYPE_REGION3: 2302 index = (r2 >> 31) & 0x7ff; 2303 break; 2304 case ASCE_TYPE_SEGMENT: 2305 index = (r2 >> 20) & 0x7ff; 2306 break; 2307 } 2308 for (i = 0; i < entries; i++) { 2309 /* addresses are not wrapped in 24/31bit mode but table index is */ 2310 raddr = table + ((index + i) & 0x7ff) * sizeof(entry); 2311 entry = cpu_ldq_mmuidx_ra(env, raddr, MMU_REAL_IDX, ra); 2312 if (!(entry & REGION_ENTRY_I)) { 2313 /* we are allowed to not store if already invalid */ 2314 entry |= REGION_ENTRY_I; 2315 cpu_stq_mmuidx_ra(env, raddr, entry, MMU_REAL_IDX, ra); 2316 } 2317 } 2318 } 2319 2320 /* We simply flush the complete tlb, therefore we can ignore r3. */ 2321 if (m4 & 1) { 2322 tlb_flush(cs); 2323 } else { 2324 tlb_flush_all_cpus_synced(cs); 2325 } 2326 } 2327 2328 /* invalidate pte */ HELPER(ipte)2329 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr, 2330 uint32_t m4) 2331 { 2332 CPUState *cs = env_cpu(env); 2333 const uintptr_t ra = GETPC(); 2334 uint64_t page = vaddr & TARGET_PAGE_MASK; 2335 uint64_t pte_addr, pte; 2336 2337 /* Compute the page table entry address */ 2338 pte_addr = (pto & SEGMENT_ENTRY_ORIGIN); 2339 pte_addr += VADDR_PAGE_TX(vaddr) * 8; 2340 2341 /* Mark the page table entry as invalid */ 2342 pte = cpu_ldq_mmuidx_ra(env, pte_addr, MMU_REAL_IDX, ra); 2343 pte |= PAGE_ENTRY_I; 2344 cpu_stq_mmuidx_ra(env, pte_addr, pte, MMU_REAL_IDX, ra); 2345 2346 /* XXX we exploit the fact that Linux passes the exact virtual 2347 address here - it's not obliged to! */ 2348 if (m4 & 1) { 2349 if (vaddr & ~VADDR_PAGE_TX_MASK) { 2350 tlb_flush_page(cs, page); 2351 /* XXX 31-bit hack */ 2352 tlb_flush_page(cs, page ^ 0x80000000); 2353 } else { 2354 /* looks like we don't have a valid virtual address */ 2355 tlb_flush(cs); 2356 } 2357 } else { 2358 if (vaddr & ~VADDR_PAGE_TX_MASK) { 2359 tlb_flush_page_all_cpus_synced(cs, page); 2360 /* XXX 31-bit hack */ 2361 tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000); 2362 } else { 2363 /* looks like we don't have a valid virtual address */ 2364 tlb_flush_all_cpus_synced(cs); 2365 } 2366 } 2367 } 2368 2369 /* flush local tlb */ HELPER(ptlb)2370 void HELPER(ptlb)(CPUS390XState *env) 2371 { 2372 tlb_flush(env_cpu(env)); 2373 } 2374 2375 /* flush global tlb */ HELPER(purge)2376 void HELPER(purge)(CPUS390XState *env) 2377 { 2378 tlb_flush_all_cpus_synced(env_cpu(env)); 2379 } 2380 2381 /* load real address */ HELPER(lra)2382 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t r1, uint64_t addr) 2383 { 2384 uint64_t asc = env->psw.mask & PSW_MASK_ASC; 2385 uint64_t ret, tec; 2386 int flags, exc, cc; 2387 2388 /* XXX incomplete - has more corner cases */ 2389 if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) { 2390 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, GETPC()); 2391 } 2392 2393 exc = mmu_translate(env, addr, MMU_S390_LRA, asc, &ret, &flags, &tec); 2394 if (exc) { 2395 cc = 3; 2396 ret = (r1 & 0xFFFFFFFF00000000ULL) | exc | 0x80000000; 2397 } else { 2398 cc = 0; 2399 ret |= addr & ~TARGET_PAGE_MASK; 2400 } 2401 2402 env->cc_op = cc; 2403 return ret; 2404 } 2405 #endif 2406 2407 /* Execute instruction. This instruction executes an insn modified with 2408 the contents of r1. It does not change the executed instruction in memory; 2409 it does not change the program counter. 2410 2411 Perform this by recording the modified instruction in env->ex_value. 2412 This will be noticed by cpu_get_tb_cpu_state and thus tb translation. 2413 */ HELPER(ex)2414 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr) 2415 { 2416 uint64_t insn; 2417 uint8_t opc; 2418 2419 /* EXECUTE targets must be at even addresses. */ 2420 if (addr & 1) { 2421 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC()); 2422 } 2423 2424 insn = cpu_lduw_code(env, addr); 2425 opc = insn >> 8; 2426 2427 /* Or in the contents of R1[56:63]. */ 2428 insn |= r1 & 0xff; 2429 2430 /* Load the rest of the instruction. */ 2431 insn <<= 48; 2432 switch (get_ilen(opc)) { 2433 case 2: 2434 break; 2435 case 4: 2436 insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32; 2437 break; 2438 case 6: 2439 insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16; 2440 break; 2441 default: 2442 g_assert_not_reached(); 2443 } 2444 2445 /* The very most common cases can be sped up by avoiding a new TB. */ 2446 if ((opc & 0xf0) == 0xd0) { 2447 typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t, 2448 uint64_t, uintptr_t); 2449 static const dx_helper dx[16] = { 2450 [0x0] = do_helper_trt_bkwd, 2451 [0x2] = do_helper_mvc, 2452 [0x4] = do_helper_nc, 2453 [0x5] = do_helper_clc, 2454 [0x6] = do_helper_oc, 2455 [0x7] = do_helper_xc, 2456 [0xc] = do_helper_tr, 2457 [0xd] = do_helper_trt_fwd, 2458 }; 2459 dx_helper helper = dx[opc & 0xf]; 2460 2461 if (helper) { 2462 uint32_t l = extract64(insn, 48, 8); 2463 uint32_t b1 = extract64(insn, 44, 4); 2464 uint32_t d1 = extract64(insn, 32, 12); 2465 uint32_t b2 = extract64(insn, 28, 4); 2466 uint32_t d2 = extract64(insn, 16, 12); 2467 uint64_t a1 = wrap_address(env, (b1 ? env->regs[b1] : 0) + d1); 2468 uint64_t a2 = wrap_address(env, (b2 ? env->regs[b2] : 0) + d2); 2469 2470 env->cc_op = helper(env, l, a1, a2, 0); 2471 env->psw.addr += ilen; 2472 return; 2473 } 2474 } else if (opc == 0x0a) { 2475 env->int_svc_code = extract64(insn, 48, 8); 2476 env->int_svc_ilen = ilen; 2477 helper_exception(env, EXCP_SVC); 2478 g_assert_not_reached(); 2479 } 2480 2481 /* Record the insn we want to execute as well as the ilen to use 2482 during the execution of the target insn. This will also ensure 2483 that ex_value is non-zero, which flags that we are in a state 2484 that requires such execution. */ 2485 env->ex_value = insn | ilen; 2486 env->ex_target = addr; 2487 } 2488 HELPER(mvcos)2489 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src, 2490 uint64_t len) 2491 { 2492 const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY; 2493 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2494 const uint64_t r0 = env->regs[0]; 2495 const uintptr_t ra = GETPC(); 2496 uint8_t dest_key, dest_as, dest_k, dest_a; 2497 uint8_t src_key, src_as, src_k, src_a; 2498 uint64_t val; 2499 int cc = 0; 2500 2501 HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n", 2502 __func__, dest, src, len); 2503 2504 if (!(env->psw.mask & PSW_MASK_DAT)) { 2505 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2506 } 2507 2508 /* OAC (operand access control) for the first operand -> dest */ 2509 val = (r0 & 0xffff0000ULL) >> 16; 2510 dest_key = (val >> 12) & 0xf; 2511 dest_as = (val >> 6) & 0x3; 2512 dest_k = (val >> 1) & 0x1; 2513 dest_a = val & 0x1; 2514 2515 /* OAC (operand access control) for the second operand -> src */ 2516 val = (r0 & 0x0000ffffULL); 2517 src_key = (val >> 12) & 0xf; 2518 src_as = (val >> 6) & 0x3; 2519 src_k = (val >> 1) & 0x1; 2520 src_a = val & 0x1; 2521 2522 if (!dest_k) { 2523 dest_key = psw_key; 2524 } 2525 if (!src_k) { 2526 src_key = psw_key; 2527 } 2528 if (!dest_a) { 2529 dest_as = psw_as; 2530 } 2531 if (!src_a) { 2532 src_as = psw_as; 2533 } 2534 2535 if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) { 2536 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2537 } 2538 if (!(env->cregs[0] & CR0_SECONDARY) && 2539 (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) { 2540 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2541 } 2542 if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) { 2543 tcg_s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2544 } 2545 2546 len = wrap_length32(env, len); 2547 if (len > 4096) { 2548 cc = 3; 2549 len = 4096; 2550 } 2551 2552 /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */ 2553 if (src_as == AS_ACCREG || dest_as == AS_ACCREG || 2554 (env->psw.mask & PSW_MASK_PSTATE)) { 2555 qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n", 2556 __func__); 2557 tcg_s390_program_interrupt(env, PGM_ADDRESSING, ra); 2558 } 2559 2560 /* FIXME: Access using correct keys and AR-mode */ 2561 if (len) { 2562 S390Access srca, desta; 2563 2564 access_prepare(&srca, env, src, len, MMU_DATA_LOAD, 2565 mmu_idx_from_as(src_as), ra); 2566 access_prepare(&desta, env, dest, len, MMU_DATA_STORE, 2567 mmu_idx_from_as(dest_as), ra); 2568 2569 access_memmove(env, &desta, &srca, ra); 2570 } 2571 2572 return cc; 2573 } 2574 2575 /* Decode a Unicode character. A return value < 0 indicates success, storing 2576 the UTF-32 result into OCHAR and the input length into OLEN. A return 2577 value >= 0 indicates failure, and the CC value to be returned. */ 2578 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr, 2579 uint64_t ilen, bool enh_check, uintptr_t ra, 2580 uint32_t *ochar, uint32_t *olen); 2581 2582 /* Encode a Unicode character. A return value < 0 indicates success, storing 2583 the bytes into ADDR and the output length into OLEN. A return value >= 0 2584 indicates failure, and the CC value to be returned. */ 2585 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr, 2586 uint64_t ilen, uintptr_t ra, uint32_t c, 2587 uint32_t *olen); 2588 decode_utf8(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2589 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2590 bool enh_check, uintptr_t ra, 2591 uint32_t *ochar, uint32_t *olen) 2592 { 2593 uint8_t s0, s1, s2, s3; 2594 uint32_t c, l; 2595 2596 if (ilen < 1) { 2597 return 0; 2598 } 2599 s0 = cpu_ldub_data_ra(env, addr, ra); 2600 if (s0 <= 0x7f) { 2601 /* one byte character */ 2602 l = 1; 2603 c = s0; 2604 } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) { 2605 /* invalid character */ 2606 return 2; 2607 } else if (s0 <= 0xdf) { 2608 /* two byte character */ 2609 l = 2; 2610 if (ilen < 2) { 2611 return 0; 2612 } 2613 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2614 c = s0 & 0x1f; 2615 c = (c << 6) | (s1 & 0x3f); 2616 if (enh_check && (s1 & 0xc0) != 0x80) { 2617 return 2; 2618 } 2619 } else if (s0 <= 0xef) { 2620 /* three byte character */ 2621 l = 3; 2622 if (ilen < 3) { 2623 return 0; 2624 } 2625 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2626 s2 = cpu_ldub_data_ra(env, addr + 2, ra); 2627 c = s0 & 0x0f; 2628 c = (c << 6) | (s1 & 0x3f); 2629 c = (c << 6) | (s2 & 0x3f); 2630 /* Fold the byte-by-byte range descriptions in the PoO into 2631 tests against the complete value. It disallows encodings 2632 that could be smaller, and the UTF-16 surrogates. */ 2633 if (enh_check 2634 && ((s1 & 0xc0) != 0x80 2635 || (s2 & 0xc0) != 0x80 2636 || c < 0x1000 2637 || (c >= 0xd800 && c <= 0xdfff))) { 2638 return 2; 2639 } 2640 } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) { 2641 /* four byte character */ 2642 l = 4; 2643 if (ilen < 4) { 2644 return 0; 2645 } 2646 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2647 s2 = cpu_ldub_data_ra(env, addr + 2, ra); 2648 s3 = cpu_ldub_data_ra(env, addr + 3, ra); 2649 c = s0 & 0x07; 2650 c = (c << 6) | (s1 & 0x3f); 2651 c = (c << 6) | (s2 & 0x3f); 2652 c = (c << 6) | (s3 & 0x3f); 2653 /* See above. */ 2654 if (enh_check 2655 && ((s1 & 0xc0) != 0x80 2656 || (s2 & 0xc0) != 0x80 2657 || (s3 & 0xc0) != 0x80 2658 || c < 0x010000 2659 || c > 0x10ffff)) { 2660 return 2; 2661 } 2662 } else { 2663 /* invalid character */ 2664 return 2; 2665 } 2666 2667 *ochar = c; 2668 *olen = l; 2669 return -1; 2670 } 2671 decode_utf16(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2672 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2673 bool enh_check, uintptr_t ra, 2674 uint32_t *ochar, uint32_t *olen) 2675 { 2676 uint16_t s0, s1; 2677 uint32_t c, l; 2678 2679 if (ilen < 2) { 2680 return 0; 2681 } 2682 s0 = cpu_lduw_data_ra(env, addr, ra); 2683 if ((s0 & 0xfc00) != 0xd800) { 2684 /* one word character */ 2685 l = 2; 2686 c = s0; 2687 } else { 2688 /* two word character */ 2689 l = 4; 2690 if (ilen < 4) { 2691 return 0; 2692 } 2693 s1 = cpu_lduw_data_ra(env, addr + 2, ra); 2694 c = extract32(s0, 6, 4) + 1; 2695 c = (c << 6) | (s0 & 0x3f); 2696 c = (c << 10) | (s1 & 0x3ff); 2697 if (enh_check && (s1 & 0xfc00) != 0xdc00) { 2698 /* invalid surrogate character */ 2699 return 2; 2700 } 2701 } 2702 2703 *ochar = c; 2704 *olen = l; 2705 return -1; 2706 } 2707 decode_utf32(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2708 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2709 bool enh_check, uintptr_t ra, 2710 uint32_t *ochar, uint32_t *olen) 2711 { 2712 uint32_t c; 2713 2714 if (ilen < 4) { 2715 return 0; 2716 } 2717 c = cpu_ldl_data_ra(env, addr, ra); 2718 if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) { 2719 /* invalid unicode character */ 2720 return 2; 2721 } 2722 2723 *ochar = c; 2724 *olen = 4; 2725 return -1; 2726 } 2727 encode_utf8(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2728 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2729 uintptr_t ra, uint32_t c, uint32_t *olen) 2730 { 2731 uint8_t d[4]; 2732 uint32_t l, i; 2733 2734 if (c <= 0x7f) { 2735 /* one byte character */ 2736 l = 1; 2737 d[0] = c; 2738 } else if (c <= 0x7ff) { 2739 /* two byte character */ 2740 l = 2; 2741 d[1] = 0x80 | extract32(c, 0, 6); 2742 d[0] = 0xc0 | extract32(c, 6, 5); 2743 } else if (c <= 0xffff) { 2744 /* three byte character */ 2745 l = 3; 2746 d[2] = 0x80 | extract32(c, 0, 6); 2747 d[1] = 0x80 | extract32(c, 6, 6); 2748 d[0] = 0xe0 | extract32(c, 12, 4); 2749 } else { 2750 /* four byte character */ 2751 l = 4; 2752 d[3] = 0x80 | extract32(c, 0, 6); 2753 d[2] = 0x80 | extract32(c, 6, 6); 2754 d[1] = 0x80 | extract32(c, 12, 6); 2755 d[0] = 0xf0 | extract32(c, 18, 3); 2756 } 2757 2758 if (ilen < l) { 2759 return 1; 2760 } 2761 for (i = 0; i < l; ++i) { 2762 cpu_stb_data_ra(env, addr + i, d[i], ra); 2763 } 2764 2765 *olen = l; 2766 return -1; 2767 } 2768 encode_utf16(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2769 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2770 uintptr_t ra, uint32_t c, uint32_t *olen) 2771 { 2772 uint16_t d0, d1; 2773 2774 if (c <= 0xffff) { 2775 /* one word character */ 2776 if (ilen < 2) { 2777 return 1; 2778 } 2779 cpu_stw_data_ra(env, addr, c, ra); 2780 *olen = 2; 2781 } else { 2782 /* two word character */ 2783 if (ilen < 4) { 2784 return 1; 2785 } 2786 d1 = 0xdc00 | extract32(c, 0, 10); 2787 d0 = 0xd800 | extract32(c, 10, 6); 2788 d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1); 2789 cpu_stw_data_ra(env, addr + 0, d0, ra); 2790 cpu_stw_data_ra(env, addr + 2, d1, ra); 2791 *olen = 4; 2792 } 2793 2794 return -1; 2795 } 2796 encode_utf32(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2797 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2798 uintptr_t ra, uint32_t c, uint32_t *olen) 2799 { 2800 if (ilen < 4) { 2801 return 1; 2802 } 2803 cpu_stl_data_ra(env, addr, c, ra); 2804 *olen = 4; 2805 return -1; 2806 } 2807 convert_unicode(CPUS390XState * env,uint32_t r1,uint32_t r2,uint32_t m3,uintptr_t ra,decode_unicode_fn decode,encode_unicode_fn encode)2808 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1, 2809 uint32_t r2, uint32_t m3, uintptr_t ra, 2810 decode_unicode_fn decode, 2811 encode_unicode_fn encode) 2812 { 2813 uint64_t dst = get_address(env, r1); 2814 uint64_t dlen = get_length(env, r1 + 1); 2815 uint64_t src = get_address(env, r2); 2816 uint64_t slen = get_length(env, r2 + 1); 2817 bool enh_check = m3 & 1; 2818 int cc, i; 2819 2820 /* Lest we fail to service interrupts in a timely manner, limit the 2821 amount of work we're willing to do. For now, let's cap at 256. */ 2822 for (i = 0; i < 256; ++i) { 2823 uint32_t c, ilen, olen; 2824 2825 cc = decode(env, src, slen, enh_check, ra, &c, &ilen); 2826 if (unlikely(cc >= 0)) { 2827 break; 2828 } 2829 cc = encode(env, dst, dlen, ra, c, &olen); 2830 if (unlikely(cc >= 0)) { 2831 break; 2832 } 2833 2834 src += ilen; 2835 slen -= ilen; 2836 dst += olen; 2837 dlen -= olen; 2838 cc = 3; 2839 } 2840 2841 set_address(env, r1, dst); 2842 set_length(env, r1 + 1, dlen); 2843 set_address(env, r2, src); 2844 set_length(env, r2 + 1, slen); 2845 2846 return cc; 2847 } 2848 HELPER(cu12)2849 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2850 { 2851 return convert_unicode(env, r1, r2, m3, GETPC(), 2852 decode_utf8, encode_utf16); 2853 } 2854 HELPER(cu14)2855 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2856 { 2857 return convert_unicode(env, r1, r2, m3, GETPC(), 2858 decode_utf8, encode_utf32); 2859 } 2860 HELPER(cu21)2861 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2862 { 2863 return convert_unicode(env, r1, r2, m3, GETPC(), 2864 decode_utf16, encode_utf8); 2865 } 2866 HELPER(cu24)2867 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2868 { 2869 return convert_unicode(env, r1, r2, m3, GETPC(), 2870 decode_utf16, encode_utf32); 2871 } 2872 HELPER(cu41)2873 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2874 { 2875 return convert_unicode(env, r1, r2, m3, GETPC(), 2876 decode_utf32, encode_utf8); 2877 } 2878 HELPER(cu42)2879 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2880 { 2881 return convert_unicode(env, r1, r2, m3, GETPC(), 2882 decode_utf32, encode_utf16); 2883 } 2884 probe_write_access(CPUS390XState * env,uint64_t addr,uint64_t len,uintptr_t ra)2885 void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len, 2886 uintptr_t ra) 2887 { 2888 const int mmu_idx = s390x_env_mmu_index(env, false); 2889 2890 /* test the actual access, not just any access to the page due to LAP */ 2891 while (len) { 2892 const uint64_t pagelen = -(addr | TARGET_PAGE_MASK); 2893 const uint64_t curlen = MIN(pagelen, len); 2894 2895 probe_write(env, addr, curlen, mmu_idx, ra); 2896 addr = wrap_address(env, addr + curlen); 2897 len -= curlen; 2898 } 2899 } 2900 HELPER(probe_write_access)2901 void HELPER(probe_write_access)(CPUS390XState *env, uint64_t addr, uint64_t len) 2902 { 2903 probe_write_access(env, addr, len, GETPC()); 2904 } 2905