1 /* 2 * S/390 memory access helper routines 3 * 4 * Copyright (c) 2009 Ulrich Hecht 5 * Copyright (c) 2009 Alexander Graf 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/osdep.h" 22 #include "qemu/log.h" 23 #include "cpu.h" 24 #include "s390x-internal.h" 25 #include "tcg_s390x.h" 26 #include "exec/helper-proto.h" 27 #include "exec/cpu-common.h" 28 #include "exec/cputlb.h" 29 #include "exec/page-protection.h" 30 #include "accel/tcg/cpu-ldst.h" 31 #include "accel/tcg/probe.h" 32 #include "exec/target_page.h" 33 #include "exec/tlb-flags.h" 34 #include "accel/tcg/cpu-ops.h" 35 #include "accel/tcg/helper-retaddr.h" 36 #include "qemu/int128.h" 37 #include "qemu/atomic128.h" 38 39 #if defined(CONFIG_USER_ONLY) 40 #include "user/page-protection.h" 41 #else 42 #include "hw/s390x/storage-keys.h" 43 #include "hw/boards.h" 44 #endif 45 46 #ifdef CONFIG_USER_ONLY 47 # define user_or_likely(X) true 48 #else 49 # define user_or_likely(X) likely(X) 50 #endif 51 52 /*****************************************************************************/ 53 /* Softmmu support */ 54 55 /* #define DEBUG_HELPER */ 56 #ifdef DEBUG_HELPER 57 #define HELPER_LOG(x...) qemu_log(x) 58 #else 59 #define HELPER_LOG(x...) 60 #endif 61 62 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key) 63 { 64 uint16_t pkm = env->cregs[3] >> 16; 65 66 if (env->psw.mask & PSW_MASK_PSTATE) { 67 /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */ 68 return pkm & (0x8000 >> psw_key); 69 } 70 return true; 71 } 72 73 static bool is_destructive_overlap(CPUS390XState *env, uint64_t dest, 74 uint64_t src, uint32_t len) 75 { 76 if (!len || src == dest) { 77 return false; 78 } 79 /* Take care of wrapping at the end of address space. */ 80 if (unlikely(wrap_address(env, src + len - 1) < src)) { 81 return dest > src || dest <= wrap_address(env, src + len - 1); 82 } 83 return dest > src && dest <= src + len - 1; 84 } 85 86 /* Trigger a SPECIFICATION exception if an address or a length is not 87 naturally aligned. */ 88 static inline void check_alignment(CPUS390XState *env, uint64_t v, 89 int wordsize, uintptr_t ra) 90 { 91 if (v % wordsize) { 92 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 93 } 94 } 95 96 /* Load a value from memory according to its size. */ 97 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr, 98 int wordsize, uintptr_t ra) 99 { 100 switch (wordsize) { 101 case 1: 102 return cpu_ldub_data_ra(env, addr, ra); 103 case 2: 104 return cpu_lduw_data_ra(env, addr, ra); 105 default: 106 abort(); 107 } 108 } 109 110 /* Store a to memory according to its size. */ 111 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr, 112 uint64_t value, int wordsize, 113 uintptr_t ra) 114 { 115 switch (wordsize) { 116 case 1: 117 cpu_stb_data_ra(env, addr, value, ra); 118 break; 119 case 2: 120 cpu_stw_data_ra(env, addr, value, ra); 121 break; 122 default: 123 abort(); 124 } 125 } 126 127 /* An access covers at most 4096 bytes and therefore at most two pages. */ 128 typedef struct S390Access { 129 vaddr vaddr1; 130 vaddr vaddr2; 131 void *haddr1; 132 void *haddr2; 133 uint16_t size1; 134 uint16_t size2; 135 /* 136 * If we can't access the host page directly, we'll have to do I/O access 137 * via ld/st helpers. These are internal details, so we store the 138 * mmu idx to do the access here instead of passing it around in the 139 * helpers. 140 */ 141 int mmu_idx; 142 } S390Access; 143 144 /* 145 * With nonfault=1, return the PGM_ exception that would have been injected 146 * into the guest; return 0 if no exception was detected. 147 * 148 * For !CONFIG_USER_ONLY, the TEC is stored stored to env->tlb_fill_tec. 149 * For CONFIG_USER_ONLY, the faulting address is stored to env->__excp_addr. 150 */ 151 static inline int s390_probe_access(CPUArchState *env, vaddr addr, 152 int size, MMUAccessType access_type, 153 int mmu_idx, bool nonfault, 154 void **phost, uintptr_t ra) 155 { 156 int flags = probe_access_flags(env, addr, size, access_type, mmu_idx, 157 nonfault, phost, ra); 158 159 if (unlikely(flags & TLB_INVALID_MASK)) { 160 #ifdef CONFIG_USER_ONLY 161 /* Address is in TEC in system mode; see s390_cpu_record_sigsegv. */ 162 env->__excp_addr = addr & TARGET_PAGE_MASK; 163 return (page_get_flags(addr) & PAGE_VALID 164 ? PGM_PROTECTION : PGM_ADDRESSING); 165 #else 166 return env->tlb_fill_exc; 167 #endif 168 } 169 170 #ifndef CONFIG_USER_ONLY 171 if (unlikely(flags & TLB_WATCHPOINT)) { 172 /* S390 does not presently use transaction attributes. */ 173 cpu_check_watchpoint(env_cpu(env), addr, size, 174 MEMTXATTRS_UNSPECIFIED, 175 (access_type == MMU_DATA_STORE 176 ? BP_MEM_WRITE : BP_MEM_READ), ra); 177 } 178 #endif 179 180 return 0; 181 } 182 183 static int access_prepare_nf(S390Access *access, CPUS390XState *env, 184 bool nonfault, vaddr vaddr1, int size, 185 MMUAccessType access_type, 186 int mmu_idx, uintptr_t ra) 187 { 188 int size1, size2, exc; 189 190 assert(size > 0 && size <= 4096); 191 192 size1 = MIN(size, -(vaddr1 | TARGET_PAGE_MASK)), 193 size2 = size - size1; 194 195 memset(access, 0, sizeof(*access)); 196 access->vaddr1 = vaddr1; 197 access->size1 = size1; 198 access->size2 = size2; 199 access->mmu_idx = mmu_idx; 200 201 exc = s390_probe_access(env, vaddr1, size1, access_type, mmu_idx, nonfault, 202 &access->haddr1, ra); 203 if (unlikely(exc)) { 204 return exc; 205 } 206 if (unlikely(size2)) { 207 /* The access crosses page boundaries. */ 208 vaddr vaddr2 = wrap_address(env, vaddr1 + size1); 209 210 access->vaddr2 = vaddr2; 211 exc = s390_probe_access(env, vaddr2, size2, access_type, mmu_idx, 212 nonfault, &access->haddr2, ra); 213 if (unlikely(exc)) { 214 return exc; 215 } 216 } 217 return 0; 218 } 219 220 static inline void access_prepare(S390Access *ret, CPUS390XState *env, 221 vaddr vaddr, int size, 222 MMUAccessType access_type, int mmu_idx, 223 uintptr_t ra) 224 { 225 int exc = access_prepare_nf(ret, env, false, vaddr, size, 226 access_type, mmu_idx, ra); 227 assert(!exc); 228 } 229 230 /* Helper to handle memset on a single page. */ 231 static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr, 232 uint8_t byte, uint16_t size, int mmu_idx, 233 uintptr_t ra) 234 { 235 if (user_or_likely(haddr)) { 236 memset(haddr, byte, size); 237 } else { 238 MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); 239 for (int i = 0; i < size; i++) { 240 cpu_stb_mmu(env, vaddr + i, byte, oi, ra); 241 } 242 } 243 } 244 245 static void access_memset(CPUS390XState *env, S390Access *desta, 246 uint8_t byte, uintptr_t ra) 247 { 248 set_helper_retaddr(ra); 249 do_access_memset(env, desta->vaddr1, desta->haddr1, byte, desta->size1, 250 desta->mmu_idx, ra); 251 if (unlikely(desta->size2)) { 252 do_access_memset(env, desta->vaddr2, desta->haddr2, byte, 253 desta->size2, desta->mmu_idx, ra); 254 } 255 clear_helper_retaddr(); 256 } 257 258 static uint8_t access_get_byte(CPUS390XState *env, S390Access *access, 259 int offset, uintptr_t ra) 260 { 261 vaddr vaddr = access->vaddr1; 262 void *haddr = access->haddr1; 263 264 if (unlikely(offset >= access->size1)) { 265 offset -= access->size1; 266 vaddr = access->vaddr2; 267 haddr = access->haddr2; 268 } 269 270 if (user_or_likely(haddr)) { 271 return ldub_p(haddr + offset); 272 } else { 273 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx); 274 return cpu_ldb_mmu(env, vaddr + offset, oi, ra); 275 } 276 } 277 278 static void access_set_byte(CPUS390XState *env, S390Access *access, 279 int offset, uint8_t byte, uintptr_t ra) 280 { 281 vaddr vaddr = access->vaddr1; 282 void *haddr = access->haddr1; 283 284 if (unlikely(offset >= access->size1)) { 285 offset -= access->size1; 286 vaddr = access->vaddr2; 287 haddr = access->haddr2; 288 } 289 290 if (user_or_likely(haddr)) { 291 stb_p(haddr + offset, byte); 292 } else { 293 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx); 294 cpu_stb_mmu(env, vaddr + offset, byte, oi, ra); 295 } 296 } 297 298 /* 299 * Move data with the same semantics as memmove() in case ranges don't overlap 300 * or src > dest. Undefined behavior on destructive overlaps. 301 */ 302 static void access_memmove(CPUS390XState *env, S390Access *desta, 303 S390Access *srca, uintptr_t ra) 304 { 305 int len = desta->size1 + desta->size2; 306 307 assert(len == srca->size1 + srca->size2); 308 309 /* Fallback to slow access in case we don't have access to all host pages */ 310 if (user_or_likely(desta->haddr1 && 311 srca->haddr1 && 312 (!desta->size2 || desta->haddr2) && 313 (!srca->size2 || srca->haddr2))) { 314 int diff = desta->size1 - srca->size1; 315 316 if (likely(diff == 0)) { 317 memmove(desta->haddr1, srca->haddr1, srca->size1); 318 if (unlikely(srca->size2)) { 319 memmove(desta->haddr2, srca->haddr2, srca->size2); 320 } 321 } else if (diff > 0) { 322 memmove(desta->haddr1, srca->haddr1, srca->size1); 323 memmove(desta->haddr1 + srca->size1, srca->haddr2, diff); 324 if (likely(desta->size2)) { 325 memmove(desta->haddr2, srca->haddr2 + diff, desta->size2); 326 } 327 } else { 328 diff = -diff; 329 memmove(desta->haddr1, srca->haddr1, desta->size1); 330 memmove(desta->haddr2, srca->haddr1 + desta->size1, diff); 331 if (likely(srca->size2)) { 332 memmove(desta->haddr2 + diff, srca->haddr2, srca->size2); 333 } 334 } 335 } else { 336 for (int i = 0; i < len; i++) { 337 uint8_t byte = access_get_byte(env, srca, i, ra); 338 access_set_byte(env, desta, i, byte, ra); 339 } 340 } 341 } 342 343 static int mmu_idx_from_as(uint8_t as) 344 { 345 switch (as) { 346 case AS_PRIMARY: 347 return MMU_PRIMARY_IDX; 348 case AS_SECONDARY: 349 return MMU_SECONDARY_IDX; 350 case AS_HOME: 351 return MMU_HOME_IDX; 352 default: 353 /* FIXME AS_ACCREG */ 354 g_assert_not_reached(); 355 } 356 } 357 358 /* and on array */ 359 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest, 360 uint64_t src, uintptr_t ra) 361 { 362 const int mmu_idx = s390x_env_mmu_index(env, false); 363 S390Access srca1, srca2, desta; 364 uint32_t i; 365 uint8_t c = 0; 366 367 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 368 __func__, l, dest, src); 369 370 /* NC always processes one more byte than specified - maximum is 256 */ 371 l++; 372 373 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 374 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 375 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 376 set_helper_retaddr(ra); 377 378 for (i = 0; i < l; i++) { 379 const uint8_t x = access_get_byte(env, &srca1, i, ra) & 380 access_get_byte(env, &srca2, i, ra); 381 382 c |= x; 383 access_set_byte(env, &desta, i, x, ra); 384 } 385 386 clear_helper_retaddr(); 387 return c != 0; 388 } 389 390 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest, 391 uint64_t src) 392 { 393 return do_helper_nc(env, l, dest, src, GETPC()); 394 } 395 396 /* xor on array */ 397 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest, 398 uint64_t src, uintptr_t ra) 399 { 400 const int mmu_idx = s390x_env_mmu_index(env, false); 401 S390Access srca1, srca2, desta; 402 uint32_t i; 403 uint8_t c = 0; 404 405 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 406 __func__, l, dest, src); 407 408 /* XC always processes one more byte than specified - maximum is 256 */ 409 l++; 410 411 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 412 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 413 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 414 415 /* xor with itself is the same as memset(0) */ 416 if (src == dest) { 417 access_memset(env, &desta, 0, ra); 418 return 0; 419 } 420 421 set_helper_retaddr(ra); 422 for (i = 0; i < l; i++) { 423 const uint8_t x = access_get_byte(env, &srca1, i, ra) ^ 424 access_get_byte(env, &srca2, i, ra); 425 426 c |= x; 427 access_set_byte(env, &desta, i, x, ra); 428 } 429 clear_helper_retaddr(); 430 return c != 0; 431 } 432 433 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest, 434 uint64_t src) 435 { 436 return do_helper_xc(env, l, dest, src, GETPC()); 437 } 438 439 /* or on array */ 440 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest, 441 uint64_t src, uintptr_t ra) 442 { 443 const int mmu_idx = s390x_env_mmu_index(env, false); 444 S390Access srca1, srca2, desta; 445 uint32_t i; 446 uint8_t c = 0; 447 448 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 449 __func__, l, dest, src); 450 451 /* OC always processes one more byte than specified - maximum is 256 */ 452 l++; 453 454 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 455 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 456 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 457 set_helper_retaddr(ra); 458 459 for (i = 0; i < l; i++) { 460 const uint8_t x = access_get_byte(env, &srca1, i, ra) | 461 access_get_byte(env, &srca2, i, ra); 462 463 c |= x; 464 access_set_byte(env, &desta, i, x, ra); 465 } 466 467 clear_helper_retaddr(); 468 return c != 0; 469 } 470 471 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest, 472 uint64_t src) 473 { 474 return do_helper_oc(env, l, dest, src, GETPC()); 475 } 476 477 /* memmove */ 478 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest, 479 uint64_t src, uintptr_t ra) 480 { 481 const int mmu_idx = s390x_env_mmu_index(env, false); 482 S390Access srca, desta; 483 uint32_t i; 484 485 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 486 __func__, l, dest, src); 487 488 /* MVC always copies one more byte than specified - maximum is 256 */ 489 l++; 490 491 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 492 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 493 494 /* 495 * "When the operands overlap, the result is obtained as if the operands 496 * were processed one byte at a time". Only non-destructive overlaps 497 * behave like memmove(). 498 */ 499 if (dest == src + 1) { 500 access_memset(env, &desta, access_get_byte(env, &srca, 0, ra), ra); 501 } else if (!is_destructive_overlap(env, dest, src, l)) { 502 access_memmove(env, &desta, &srca, ra); 503 } else { 504 set_helper_retaddr(ra); 505 for (i = 0; i < l; i++) { 506 uint8_t byte = access_get_byte(env, &srca, i, ra); 507 508 access_set_byte(env, &desta, i, byte, ra); 509 } 510 clear_helper_retaddr(); 511 } 512 513 return env->cc_op; 514 } 515 516 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 517 { 518 do_helper_mvc(env, l, dest, src, GETPC()); 519 } 520 521 /* move right to left */ 522 void HELPER(mvcrl)(CPUS390XState *env, uint64_t l, uint64_t dest, uint64_t src) 523 { 524 const int mmu_idx = s390x_env_mmu_index(env, false); 525 const uint64_t ra = GETPC(); 526 S390Access srca, desta; 527 int32_t i; 528 529 /* MVCRL always copies one more byte than specified - maximum is 256 */ 530 l &= 0xff; 531 l++; 532 533 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 534 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 535 536 set_helper_retaddr(ra); 537 for (i = l - 1; i >= 0; i--) { 538 uint8_t byte = access_get_byte(env, &srca, i, ra); 539 access_set_byte(env, &desta, i, byte, ra); 540 } 541 clear_helper_retaddr(); 542 } 543 544 /* move inverse */ 545 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 546 { 547 const int mmu_idx = s390x_env_mmu_index(env, false); 548 S390Access srca, desta; 549 uintptr_t ra = GETPC(); 550 int i; 551 552 /* MVCIN always copies one more byte than specified - maximum is 256 */ 553 l++; 554 555 src = wrap_address(env, src - l + 1); 556 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 557 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 558 559 set_helper_retaddr(ra); 560 for (i = 0; i < l; i++) { 561 const uint8_t x = access_get_byte(env, &srca, l - i - 1, ra); 562 access_set_byte(env, &desta, i, x, ra); 563 } 564 clear_helper_retaddr(); 565 } 566 567 /* move numerics */ 568 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 569 { 570 const int mmu_idx = s390x_env_mmu_index(env, false); 571 S390Access srca1, srca2, desta; 572 uintptr_t ra = GETPC(); 573 int i; 574 575 /* MVN always copies one more byte than specified - maximum is 256 */ 576 l++; 577 578 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 579 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 580 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 581 582 set_helper_retaddr(ra); 583 for (i = 0; i < l; i++) { 584 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0x0f) | 585 (access_get_byte(env, &srca2, i, ra) & 0xf0); 586 587 access_set_byte(env, &desta, i, x, ra); 588 } 589 clear_helper_retaddr(); 590 } 591 592 /* move with offset */ 593 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 594 { 595 const int mmu_idx = s390x_env_mmu_index(env, false); 596 /* MVO always processes one more byte than specified - maximum is 16 */ 597 const int len_dest = (l >> 4) + 1; 598 const int len_src = (l & 0xf) + 1; 599 uintptr_t ra = GETPC(); 600 uint8_t byte_dest, byte_src; 601 S390Access srca, desta; 602 int i, j; 603 604 access_prepare(&srca, env, src, len_src, MMU_DATA_LOAD, mmu_idx, ra); 605 access_prepare(&desta, env, dest, len_dest, MMU_DATA_STORE, mmu_idx, ra); 606 607 /* Handle rightmost byte */ 608 byte_dest = cpu_ldub_data_ra(env, dest + len_dest - 1, ra); 609 610 set_helper_retaddr(ra); 611 byte_src = access_get_byte(env, &srca, len_src - 1, ra); 612 byte_dest = (byte_dest & 0x0f) | (byte_src << 4); 613 access_set_byte(env, &desta, len_dest - 1, byte_dest, ra); 614 615 /* Process remaining bytes from right to left */ 616 for (i = len_dest - 2, j = len_src - 2; i >= 0; i--, j--) { 617 byte_dest = byte_src >> 4; 618 if (j >= 0) { 619 byte_src = access_get_byte(env, &srca, j, ra); 620 } else { 621 byte_src = 0; 622 } 623 byte_dest |= byte_src << 4; 624 access_set_byte(env, &desta, i, byte_dest, ra); 625 } 626 clear_helper_retaddr(); 627 } 628 629 /* move zones */ 630 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 631 { 632 const int mmu_idx = s390x_env_mmu_index(env, false); 633 S390Access srca1, srca2, desta; 634 uintptr_t ra = GETPC(); 635 int i; 636 637 /* MVZ always copies one more byte than specified - maximum is 256 */ 638 l++; 639 640 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 641 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 642 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 643 644 set_helper_retaddr(ra); 645 for (i = 0; i < l; i++) { 646 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0xf0) | 647 (access_get_byte(env, &srca2, i, ra) & 0x0f); 648 649 access_set_byte(env, &desta, i, x, ra); 650 } 651 clear_helper_retaddr(); 652 } 653 654 /* compare unsigned byte arrays */ 655 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1, 656 uint64_t s2, uintptr_t ra) 657 { 658 uint32_t i; 659 uint32_t cc = 0; 660 661 HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n", 662 __func__, l, s1, s2); 663 664 for (i = 0; i <= l; i++) { 665 uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra); 666 uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra); 667 HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y); 668 if (x < y) { 669 cc = 1; 670 break; 671 } else if (x > y) { 672 cc = 2; 673 break; 674 } 675 } 676 677 HELPER_LOG("\n"); 678 return cc; 679 } 680 681 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2) 682 { 683 return do_helper_clc(env, l, s1, s2, GETPC()); 684 } 685 686 /* compare logical under mask */ 687 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask, 688 uint64_t addr) 689 { 690 uintptr_t ra = GETPC(); 691 uint32_t cc = 0; 692 693 HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1, 694 mask, addr); 695 696 if (!mask) { 697 /* Recognize access exceptions for the first byte */ 698 probe_read(env, addr, 1, s390x_env_mmu_index(env, false), ra); 699 } 700 701 while (mask) { 702 if (mask & 8) { 703 uint8_t d = cpu_ldub_data_ra(env, addr, ra); 704 uint8_t r = extract32(r1, 24, 8); 705 HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d, 706 addr); 707 if (r < d) { 708 cc = 1; 709 break; 710 } else if (r > d) { 711 cc = 2; 712 break; 713 } 714 addr++; 715 } 716 mask = (mask << 1) & 0xf; 717 r1 <<= 8; 718 } 719 720 HELPER_LOG("\n"); 721 return cc; 722 } 723 724 static inline uint64_t get_address(CPUS390XState *env, int reg) 725 { 726 return wrap_address(env, env->regs[reg]); 727 } 728 729 /* 730 * Store the address to the given register, zeroing out unused leftmost 731 * bits in bit positions 32-63 (24-bit and 31-bit mode only). 732 */ 733 static inline void set_address_zero(CPUS390XState *env, int reg, 734 uint64_t address) 735 { 736 if (env->psw.mask & PSW_MASK_64) { 737 env->regs[reg] = address; 738 } else { 739 if (!(env->psw.mask & PSW_MASK_32)) { 740 address &= 0x00ffffff; 741 } else { 742 address &= 0x7fffffff; 743 } 744 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address); 745 } 746 } 747 748 static inline void set_address(CPUS390XState *env, int reg, uint64_t address) 749 { 750 if (env->psw.mask & PSW_MASK_64) { 751 /* 64-Bit mode */ 752 env->regs[reg] = address; 753 } else { 754 if (!(env->psw.mask & PSW_MASK_32)) { 755 /* 24-Bit mode. According to the PoO it is implementation 756 dependent if bits 32-39 remain unchanged or are set to 757 zeros. Choose the former so that the function can also be 758 used for TRT. */ 759 env->regs[reg] = deposit64(env->regs[reg], 0, 24, address); 760 } else { 761 /* 31-Bit mode. According to the PoO it is implementation 762 dependent if bit 32 remains unchanged or is set to zero. 763 Choose the latter so that the function can also be used for 764 TRT. */ 765 address &= 0x7fffffff; 766 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address); 767 } 768 } 769 } 770 771 static inline uint64_t wrap_length32(CPUS390XState *env, uint64_t length) 772 { 773 if (!(env->psw.mask & PSW_MASK_64)) { 774 return (uint32_t)length; 775 } 776 return length; 777 } 778 779 static inline uint64_t wrap_length31(CPUS390XState *env, uint64_t length) 780 { 781 if (!(env->psw.mask & PSW_MASK_64)) { 782 /* 24-Bit and 31-Bit mode */ 783 length &= 0x7fffffff; 784 } 785 return length; 786 } 787 788 static inline uint64_t get_length(CPUS390XState *env, int reg) 789 { 790 return wrap_length31(env, env->regs[reg]); 791 } 792 793 static inline void set_length(CPUS390XState *env, int reg, uint64_t length) 794 { 795 if (env->psw.mask & PSW_MASK_64) { 796 /* 64-Bit mode */ 797 env->regs[reg] = length; 798 } else { 799 /* 24-Bit and 31-Bit mode */ 800 env->regs[reg] = deposit64(env->regs[reg], 0, 32, length); 801 } 802 } 803 804 /* search string (c is byte to search, r2 is string, r1 end of string) */ 805 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2) 806 { 807 uintptr_t ra = GETPC(); 808 uint64_t end, str; 809 uint32_t len; 810 uint8_t v, c = env->regs[0]; 811 812 /* Bits 32-55 must contain all 0. */ 813 if (env->regs[0] & 0xffffff00u) { 814 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 815 } 816 817 str = get_address(env, r2); 818 end = get_address(env, r1); 819 820 /* Lest we fail to service interrupts in a timely manner, limit the 821 amount of work we're willing to do. For now, let's cap at 8k. */ 822 for (len = 0; len < 0x2000; ++len) { 823 if (str + len == end) { 824 /* Character not found. R1 & R2 are unmodified. */ 825 env->cc_op = 2; 826 return; 827 } 828 v = cpu_ldub_data_ra(env, str + len, ra); 829 if (v == c) { 830 /* Character found. Set R1 to the location; R2 is unmodified. */ 831 env->cc_op = 1; 832 set_address(env, r1, str + len); 833 return; 834 } 835 } 836 837 /* CPU-determined bytes processed. Advance R2 to next byte to process. */ 838 env->cc_op = 3; 839 set_address(env, r2, str + len); 840 } 841 842 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2) 843 { 844 uintptr_t ra = GETPC(); 845 uint32_t len; 846 uint16_t v, c = env->regs[0]; 847 uint64_t end, str, adj_end; 848 849 /* Bits 32-47 of R0 must be zero. */ 850 if (env->regs[0] & 0xffff0000u) { 851 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 852 } 853 854 str = get_address(env, r2); 855 end = get_address(env, r1); 856 857 /* If the LSB of the two addresses differ, use one extra byte. */ 858 adj_end = end + ((str ^ end) & 1); 859 860 /* Lest we fail to service interrupts in a timely manner, limit the 861 amount of work we're willing to do. For now, let's cap at 8k. */ 862 for (len = 0; len < 0x2000; len += 2) { 863 if (str + len == adj_end) { 864 /* End of input found. */ 865 env->cc_op = 2; 866 return; 867 } 868 v = cpu_lduw_data_ra(env, str + len, ra); 869 if (v == c) { 870 /* Character found. Set R1 to the location; R2 is unmodified. */ 871 env->cc_op = 1; 872 set_address(env, r1, str + len); 873 return; 874 } 875 } 876 877 /* CPU-determined bytes processed. Advance R2 to next byte to process. */ 878 env->cc_op = 3; 879 set_address(env, r2, str + len); 880 } 881 882 /* unsigned string compare (c is string terminator) */ 883 Int128 HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2) 884 { 885 uintptr_t ra = GETPC(); 886 uint32_t len; 887 888 c = c & 0xff; 889 s1 = wrap_address(env, s1); 890 s2 = wrap_address(env, s2); 891 892 /* Lest we fail to service interrupts in a timely manner, limit the 893 amount of work we're willing to do. For now, let's cap at 8k. */ 894 for (len = 0; len < 0x2000; ++len) { 895 uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra); 896 uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra); 897 if (v1 == v2) { 898 if (v1 == c) { 899 /* Equal. CC=0, and don't advance the registers. */ 900 env->cc_op = 0; 901 return int128_make128(s2, s1); 902 } 903 } else { 904 /* Unequal. CC={1,2}, and advance the registers. Note that 905 the terminator need not be zero, but the string that contains 906 the terminator is by definition "low". */ 907 env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2); 908 return int128_make128(s2 + len, s1 + len); 909 } 910 } 911 912 /* CPU-determined bytes equal; advance the registers. */ 913 env->cc_op = 3; 914 return int128_make128(s2 + len, s1 + len); 915 } 916 917 /* move page */ 918 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint32_t r1, uint32_t r2) 919 { 920 const uint64_t src = get_address(env, r2) & TARGET_PAGE_MASK; 921 const uint64_t dst = get_address(env, r1) & TARGET_PAGE_MASK; 922 const int mmu_idx = s390x_env_mmu_index(env, false); 923 const bool f = extract64(r0, 11, 1); 924 const bool s = extract64(r0, 10, 1); 925 const bool cco = extract64(r0, 8, 1); 926 uintptr_t ra = GETPC(); 927 S390Access srca, desta; 928 int exc; 929 930 if ((f && s) || extract64(r0, 12, 4)) { 931 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC()); 932 } 933 934 /* 935 * We always manually handle exceptions such that we can properly store 936 * r1/r2 to the lowcore on page-translation exceptions. 937 * 938 * TODO: Access key handling 939 */ 940 exc = access_prepare_nf(&srca, env, true, src, TARGET_PAGE_SIZE, 941 MMU_DATA_LOAD, mmu_idx, ra); 942 if (exc) { 943 if (cco) { 944 return 2; 945 } 946 goto inject_exc; 947 } 948 exc = access_prepare_nf(&desta, env, true, dst, TARGET_PAGE_SIZE, 949 MMU_DATA_STORE, mmu_idx, ra); 950 if (exc) { 951 if (cco && exc != PGM_PROTECTION) { 952 return 1; 953 } 954 goto inject_exc; 955 } 956 access_memmove(env, &desta, &srca, ra); 957 return 0; /* data moved */ 958 inject_exc: 959 #if !defined(CONFIG_USER_ONLY) 960 if (exc != PGM_ADDRESSING) { 961 stq_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, trans_exc_code), 962 env->tlb_fill_tec); 963 } 964 if (exc == PGM_PAGE_TRANS) { 965 stb_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, op_access_id), 966 r1 << 4 | r2); 967 } 968 #endif 969 tcg_s390_program_interrupt(env, exc, ra); 970 } 971 972 /* string copy */ 973 uint32_t HELPER(mvst)(CPUS390XState *env, uint32_t r1, uint32_t r2) 974 { 975 const int mmu_idx = s390x_env_mmu_index(env, false); 976 const uint64_t d = get_address(env, r1); 977 const uint64_t s = get_address(env, r2); 978 const uint8_t c = env->regs[0]; 979 const int len = MIN(-(d | TARGET_PAGE_MASK), -(s | TARGET_PAGE_MASK)); 980 S390Access srca, desta; 981 uintptr_t ra = GETPC(); 982 int i; 983 984 if (env->regs[0] & 0xffffff00ull) { 985 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 986 } 987 988 /* 989 * Our access should not exceed single pages, as we must not report access 990 * exceptions exceeding the actually copied range (which we don't know at 991 * this point). We might over-indicate watchpoints within the pages 992 * (if we ever care, we have to limit processing to a single byte). 993 */ 994 access_prepare(&srca, env, s, len, MMU_DATA_LOAD, mmu_idx, ra); 995 access_prepare(&desta, env, d, len, MMU_DATA_STORE, mmu_idx, ra); 996 997 set_helper_retaddr(ra); 998 for (i = 0; i < len; i++) { 999 const uint8_t v = access_get_byte(env, &srca, i, ra); 1000 1001 access_set_byte(env, &desta, i, v, ra); 1002 if (v == c) { 1003 clear_helper_retaddr(); 1004 set_address_zero(env, r1, d + i); 1005 return 1; 1006 } 1007 } 1008 clear_helper_retaddr(); 1009 set_address_zero(env, r1, d + len); 1010 set_address_zero(env, r2, s + len); 1011 return 3; 1012 } 1013 1014 /* load access registers r1 to r3 from memory at a2 */ 1015 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1016 { 1017 uintptr_t ra = GETPC(); 1018 int i; 1019 1020 if (a2 & 0x3) { 1021 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1022 } 1023 1024 for (i = r1;; i = (i + 1) % 16) { 1025 env->aregs[i] = cpu_ldl_data_ra(env, a2, ra); 1026 a2 += 4; 1027 1028 if (i == r3) { 1029 break; 1030 } 1031 } 1032 } 1033 1034 /* store access registers r1 to r3 in memory at a2 */ 1035 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1036 { 1037 uintptr_t ra = GETPC(); 1038 int i; 1039 1040 if (a2 & 0x3) { 1041 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1042 } 1043 1044 for (i = r1;; i = (i + 1) % 16) { 1045 cpu_stl_data_ra(env, a2, env->aregs[i], ra); 1046 a2 += 4; 1047 1048 if (i == r3) { 1049 break; 1050 } 1051 } 1052 } 1053 1054 /* move long helper */ 1055 static inline uint32_t do_mvcl(CPUS390XState *env, 1056 uint64_t *dest, uint64_t *destlen, 1057 uint64_t *src, uint64_t *srclen, 1058 uint16_t pad, int wordsize, uintptr_t ra) 1059 { 1060 const int mmu_idx = s390x_env_mmu_index(env, false); 1061 int len = MIN(*destlen, -(*dest | TARGET_PAGE_MASK)); 1062 S390Access srca, desta; 1063 int i, cc; 1064 1065 if (*destlen == *srclen) { 1066 cc = 0; 1067 } else if (*destlen < *srclen) { 1068 cc = 1; 1069 } else { 1070 cc = 2; 1071 } 1072 1073 if (!*destlen) { 1074 return cc; 1075 } 1076 1077 /* 1078 * Only perform one type of type of operation (move/pad) at a time. 1079 * Stay within single pages. 1080 */ 1081 if (*srclen) { 1082 /* Copy the src array */ 1083 len = MIN(MIN(*srclen, -(*src | TARGET_PAGE_MASK)), len); 1084 *destlen -= len; 1085 *srclen -= len; 1086 access_prepare(&srca, env, *src, len, MMU_DATA_LOAD, mmu_idx, ra); 1087 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1088 access_memmove(env, &desta, &srca, ra); 1089 *src = wrap_address(env, *src + len); 1090 *dest = wrap_address(env, *dest + len); 1091 } else if (wordsize == 1) { 1092 /* Pad the remaining area */ 1093 *destlen -= len; 1094 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1095 access_memset(env, &desta, pad, ra); 1096 *dest = wrap_address(env, *dest + len); 1097 } else { 1098 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1099 set_helper_retaddr(ra); 1100 1101 /* The remaining length selects the padding byte. */ 1102 for (i = 0; i < len; (*destlen)--, i++) { 1103 if (*destlen & 1) { 1104 access_set_byte(env, &desta, i, pad, ra); 1105 } else { 1106 access_set_byte(env, &desta, i, pad >> 8, ra); 1107 } 1108 } 1109 clear_helper_retaddr(); 1110 *dest = wrap_address(env, *dest + len); 1111 } 1112 1113 return *destlen ? 3 : cc; 1114 } 1115 1116 /* move long */ 1117 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2) 1118 { 1119 const int mmu_idx = s390x_env_mmu_index(env, false); 1120 uintptr_t ra = GETPC(); 1121 uint64_t destlen = env->regs[r1 + 1] & 0xffffff; 1122 uint64_t dest = get_address(env, r1); 1123 uint64_t srclen = env->regs[r2 + 1] & 0xffffff; 1124 uint64_t src = get_address(env, r2); 1125 uint8_t pad = env->regs[r2 + 1] >> 24; 1126 CPUState *cs = env_cpu(env); 1127 S390Access srca, desta; 1128 uint32_t cc, cur_len; 1129 1130 if (is_destructive_overlap(env, dest, src, MIN(srclen, destlen))) { 1131 cc = 3; 1132 } else if (srclen == destlen) { 1133 cc = 0; 1134 } else if (destlen < srclen) { 1135 cc = 1; 1136 } else { 1137 cc = 2; 1138 } 1139 1140 /* We might have to zero-out some bits even if there was no action. */ 1141 if (unlikely(!destlen || cc == 3)) { 1142 set_address_zero(env, r2, src); 1143 set_address_zero(env, r1, dest); 1144 return cc; 1145 } else if (!srclen) { 1146 set_address_zero(env, r2, src); 1147 } 1148 1149 /* 1150 * Only perform one type of type of operation (move/pad) in one step. 1151 * Stay within single pages. 1152 */ 1153 while (destlen) { 1154 cur_len = MIN(destlen, -(dest | TARGET_PAGE_MASK)); 1155 if (!srclen) { 1156 access_prepare(&desta, env, dest, cur_len, 1157 MMU_DATA_STORE, mmu_idx, ra); 1158 access_memset(env, &desta, pad, ra); 1159 } else { 1160 cur_len = MIN(MIN(srclen, -(src | TARGET_PAGE_MASK)), cur_len); 1161 1162 access_prepare(&srca, env, src, cur_len, 1163 MMU_DATA_LOAD, mmu_idx, ra); 1164 access_prepare(&desta, env, dest, cur_len, 1165 MMU_DATA_STORE, mmu_idx, ra); 1166 access_memmove(env, &desta, &srca, ra); 1167 src = wrap_address(env, src + cur_len); 1168 srclen -= cur_len; 1169 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen); 1170 set_address_zero(env, r2, src); 1171 } 1172 dest = wrap_address(env, dest + cur_len); 1173 destlen -= cur_len; 1174 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen); 1175 set_address_zero(env, r1, dest); 1176 1177 /* 1178 * MVCL is interruptible. Return to the main loop if requested after 1179 * writing back all state to registers. If no interrupt will get 1180 * injected, we'll end up back in this handler and continue processing 1181 * the remaining parts. 1182 */ 1183 if (destlen && unlikely(cpu_loop_exit_requested(cs))) { 1184 cpu_loop_exit_restore(cs, ra); 1185 } 1186 } 1187 return cc; 1188 } 1189 1190 /* move long extended */ 1191 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1192 uint32_t r3) 1193 { 1194 uintptr_t ra = GETPC(); 1195 uint64_t destlen = get_length(env, r1 + 1); 1196 uint64_t dest = get_address(env, r1); 1197 uint64_t srclen = get_length(env, r3 + 1); 1198 uint64_t src = get_address(env, r3); 1199 uint8_t pad = a2; 1200 uint32_t cc; 1201 1202 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra); 1203 1204 set_length(env, r1 + 1, destlen); 1205 set_length(env, r3 + 1, srclen); 1206 set_address(env, r1, dest); 1207 set_address(env, r3, src); 1208 1209 return cc; 1210 } 1211 1212 /* move long unicode */ 1213 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1214 uint32_t r3) 1215 { 1216 uintptr_t ra = GETPC(); 1217 uint64_t destlen = get_length(env, r1 + 1); 1218 uint64_t dest = get_address(env, r1); 1219 uint64_t srclen = get_length(env, r3 + 1); 1220 uint64_t src = get_address(env, r3); 1221 uint16_t pad = a2; 1222 uint32_t cc; 1223 1224 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra); 1225 1226 set_length(env, r1 + 1, destlen); 1227 set_length(env, r3 + 1, srclen); 1228 set_address(env, r1, dest); 1229 set_address(env, r3, src); 1230 1231 return cc; 1232 } 1233 1234 /* compare logical long helper */ 1235 static inline uint32_t do_clcl(CPUS390XState *env, 1236 uint64_t *src1, uint64_t *src1len, 1237 uint64_t *src3, uint64_t *src3len, 1238 uint16_t pad, uint64_t limit, 1239 int wordsize, uintptr_t ra) 1240 { 1241 uint64_t len = MAX(*src1len, *src3len); 1242 uint32_t cc = 0; 1243 1244 check_alignment(env, *src1len | *src3len, wordsize, ra); 1245 1246 if (!len) { 1247 return cc; 1248 } 1249 1250 /* Lest we fail to service interrupts in a timely manner, limit the 1251 amount of work we're willing to do. */ 1252 if (len > limit) { 1253 len = limit; 1254 cc = 3; 1255 } 1256 1257 for (; len; len -= wordsize) { 1258 uint16_t v1 = pad; 1259 uint16_t v3 = pad; 1260 1261 if (*src1len) { 1262 v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra); 1263 } 1264 if (*src3len) { 1265 v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra); 1266 } 1267 1268 if (v1 != v3) { 1269 cc = (v1 < v3) ? 1 : 2; 1270 break; 1271 } 1272 1273 if (*src1len) { 1274 *src1 += wordsize; 1275 *src1len -= wordsize; 1276 } 1277 if (*src3len) { 1278 *src3 += wordsize; 1279 *src3len -= wordsize; 1280 } 1281 } 1282 1283 return cc; 1284 } 1285 1286 1287 /* compare logical long */ 1288 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2) 1289 { 1290 uintptr_t ra = GETPC(); 1291 uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24); 1292 uint64_t src1 = get_address(env, r1); 1293 uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24); 1294 uint64_t src3 = get_address(env, r2); 1295 uint8_t pad = env->regs[r2 + 1] >> 24; 1296 uint32_t cc; 1297 1298 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra); 1299 1300 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len); 1301 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len); 1302 set_address(env, r1, src1); 1303 set_address(env, r2, src3); 1304 1305 return cc; 1306 } 1307 1308 /* compare logical long extended memcompare insn with padding */ 1309 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1310 uint32_t r3) 1311 { 1312 uintptr_t ra = GETPC(); 1313 uint64_t src1len = get_length(env, r1 + 1); 1314 uint64_t src1 = get_address(env, r1); 1315 uint64_t src3len = get_length(env, r3 + 1); 1316 uint64_t src3 = get_address(env, r3); 1317 uint8_t pad = a2; 1318 uint32_t cc; 1319 1320 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra); 1321 1322 set_length(env, r1 + 1, src1len); 1323 set_length(env, r3 + 1, src3len); 1324 set_address(env, r1, src1); 1325 set_address(env, r3, src3); 1326 1327 return cc; 1328 } 1329 1330 /* compare logical long unicode memcompare insn with padding */ 1331 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1332 uint32_t r3) 1333 { 1334 uintptr_t ra = GETPC(); 1335 uint64_t src1len = get_length(env, r1 + 1); 1336 uint64_t src1 = get_address(env, r1); 1337 uint64_t src3len = get_length(env, r3 + 1); 1338 uint64_t src3 = get_address(env, r3); 1339 uint16_t pad = a2; 1340 uint32_t cc = 0; 1341 1342 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra); 1343 1344 set_length(env, r1 + 1, src1len); 1345 set_length(env, r3 + 1, src3len); 1346 set_address(env, r1, src1); 1347 set_address(env, r3, src3); 1348 1349 return cc; 1350 } 1351 1352 /* checksum */ 1353 Int128 HELPER(cksm)(CPUS390XState *env, uint64_t r1, 1354 uint64_t src, uint64_t src_len) 1355 { 1356 uintptr_t ra = GETPC(); 1357 uint64_t max_len, len; 1358 uint64_t cksm = (uint32_t)r1; 1359 1360 /* Lest we fail to service interrupts in a timely manner, limit the 1361 amount of work we're willing to do. For now, let's cap at 8k. */ 1362 max_len = (src_len > 0x2000 ? 0x2000 : src_len); 1363 1364 /* Process full words as available. */ 1365 for (len = 0; len + 4 <= max_len; len += 4, src += 4) { 1366 cksm += (uint32_t)cpu_ldl_data_ra(env, src, ra); 1367 } 1368 1369 switch (max_len - len) { 1370 case 1: 1371 cksm += cpu_ldub_data_ra(env, src, ra) << 24; 1372 len += 1; 1373 break; 1374 case 2: 1375 cksm += cpu_lduw_data_ra(env, src, ra) << 16; 1376 len += 2; 1377 break; 1378 case 3: 1379 cksm += cpu_lduw_data_ra(env, src, ra) << 16; 1380 cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8; 1381 len += 3; 1382 break; 1383 } 1384 1385 /* Fold the carry from the checksum. Note that we can see carry-out 1386 during folding more than once (but probably not more than twice). */ 1387 while (cksm > 0xffffffffull) { 1388 cksm = (uint32_t)cksm + (cksm >> 32); 1389 } 1390 1391 /* Indicate whether or not we've processed everything. */ 1392 env->cc_op = (len == src_len ? 0 : 3); 1393 1394 /* Return both cksm and processed length. */ 1395 return int128_make128(cksm, len); 1396 } 1397 1398 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src) 1399 { 1400 uintptr_t ra = GETPC(); 1401 int len_dest = len >> 4; 1402 int len_src = len & 0xf; 1403 uint8_t b; 1404 1405 dest += len_dest; 1406 src += len_src; 1407 1408 /* last byte is special, it only flips the nibbles */ 1409 b = cpu_ldub_data_ra(env, src, ra); 1410 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra); 1411 src--; 1412 len_src--; 1413 1414 /* now pack every value */ 1415 while (len_dest > 0) { 1416 b = 0; 1417 1418 if (len_src >= 0) { 1419 b = cpu_ldub_data_ra(env, src, ra) & 0x0f; 1420 src--; 1421 len_src--; 1422 } 1423 if (len_src >= 0) { 1424 b |= cpu_ldub_data_ra(env, src, ra) << 4; 1425 src--; 1426 len_src--; 1427 } 1428 1429 len_dest--; 1430 dest--; 1431 cpu_stb_data_ra(env, dest, b, ra); 1432 } 1433 } 1434 1435 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src, 1436 uint32_t srclen, int ssize, uintptr_t ra) 1437 { 1438 int i; 1439 /* The destination operand is always 16 bytes long. */ 1440 const int destlen = 16; 1441 1442 /* The operands are processed from right to left. */ 1443 src += srclen - 1; 1444 dest += destlen - 1; 1445 1446 for (i = 0; i < destlen; i++) { 1447 uint8_t b = 0; 1448 1449 /* Start with a positive sign */ 1450 if (i == 0) { 1451 b = 0xc; 1452 } else if (srclen > ssize) { 1453 b = cpu_ldub_data_ra(env, src, ra) & 0x0f; 1454 src -= ssize; 1455 srclen -= ssize; 1456 } 1457 1458 if (srclen > ssize) { 1459 b |= cpu_ldub_data_ra(env, src, ra) << 4; 1460 src -= ssize; 1461 srclen -= ssize; 1462 } 1463 1464 cpu_stb_data_ra(env, dest, b, ra); 1465 dest--; 1466 } 1467 } 1468 1469 1470 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src, 1471 uint32_t srclen) 1472 { 1473 do_pkau(env, dest, src, srclen, 1, GETPC()); 1474 } 1475 1476 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src, 1477 uint32_t srclen) 1478 { 1479 do_pkau(env, dest, src, srclen, 2, GETPC()); 1480 } 1481 1482 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest, 1483 uint64_t src) 1484 { 1485 uintptr_t ra = GETPC(); 1486 int len_dest = len >> 4; 1487 int len_src = len & 0xf; 1488 uint8_t b; 1489 int second_nibble = 0; 1490 1491 dest += len_dest; 1492 src += len_src; 1493 1494 /* last byte is special, it only flips the nibbles */ 1495 b = cpu_ldub_data_ra(env, src, ra); 1496 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra); 1497 src--; 1498 len_src--; 1499 1500 /* now pad every nibble with 0xf0 */ 1501 1502 while (len_dest > 0) { 1503 uint8_t cur_byte = 0; 1504 1505 if (len_src > 0) { 1506 cur_byte = cpu_ldub_data_ra(env, src, ra); 1507 } 1508 1509 len_dest--; 1510 dest--; 1511 1512 /* only advance one nibble at a time */ 1513 if (second_nibble) { 1514 cur_byte >>= 4; 1515 len_src--; 1516 src--; 1517 } 1518 second_nibble = !second_nibble; 1519 1520 /* digit */ 1521 cur_byte = (cur_byte & 0xf); 1522 /* zone bits */ 1523 cur_byte |= 0xf0; 1524 1525 cpu_stb_data_ra(env, dest, cur_byte, ra); 1526 } 1527 } 1528 1529 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest, 1530 uint32_t destlen, int dsize, uint64_t src, 1531 uintptr_t ra) 1532 { 1533 int i; 1534 uint32_t cc; 1535 uint8_t b; 1536 /* The source operand is always 16 bytes long. */ 1537 const int srclen = 16; 1538 1539 /* The operands are processed from right to left. */ 1540 src += srclen - 1; 1541 dest += destlen - dsize; 1542 1543 /* Check for the sign. */ 1544 b = cpu_ldub_data_ra(env, src, ra); 1545 src--; 1546 switch (b & 0xf) { 1547 case 0xa: 1548 case 0xc: 1549 case 0xe ... 0xf: 1550 cc = 0; /* plus */ 1551 break; 1552 case 0xb: 1553 case 0xd: 1554 cc = 1; /* minus */ 1555 break; 1556 default: 1557 case 0x0 ... 0x9: 1558 cc = 3; /* invalid */ 1559 break; 1560 } 1561 1562 /* Now pad every nibble with 0x30, advancing one nibble at a time. */ 1563 for (i = 0; i < destlen; i += dsize) { 1564 if (i == (31 * dsize)) { 1565 /* If length is 32/64 bytes, the leftmost byte is 0. */ 1566 b = 0; 1567 } else if (i % (2 * dsize)) { 1568 b = cpu_ldub_data_ra(env, src, ra); 1569 src--; 1570 } else { 1571 b >>= 4; 1572 } 1573 cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra); 1574 dest -= dsize; 1575 } 1576 1577 return cc; 1578 } 1579 1580 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen, 1581 uint64_t src) 1582 { 1583 return do_unpkau(env, dest, destlen, 1, src, GETPC()); 1584 } 1585 1586 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen, 1587 uint64_t src) 1588 { 1589 return do_unpkau(env, dest, destlen, 2, src, GETPC()); 1590 } 1591 1592 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen) 1593 { 1594 uintptr_t ra = GETPC(); 1595 uint32_t cc = 0; 1596 int i; 1597 1598 for (i = 0; i < destlen; i++) { 1599 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra); 1600 /* digit */ 1601 cc |= (b & 0xf0) > 0x90 ? 2 : 0; 1602 1603 if (i == (destlen - 1)) { 1604 /* sign */ 1605 cc |= (b & 0xf) < 0xa ? 1 : 0; 1606 } else { 1607 /* digit */ 1608 cc |= (b & 0xf) > 0x9 ? 2 : 0; 1609 } 1610 } 1611 1612 return cc; 1613 } 1614 1615 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array, 1616 uint64_t trans, uintptr_t ra) 1617 { 1618 uint32_t i; 1619 1620 for (i = 0; i <= len; i++) { 1621 uint8_t byte = cpu_ldub_data_ra(env, array + i, ra); 1622 uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra); 1623 cpu_stb_data_ra(env, array + i, new_byte, ra); 1624 } 1625 1626 return env->cc_op; 1627 } 1628 1629 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array, 1630 uint64_t trans) 1631 { 1632 do_helper_tr(env, len, array, trans, GETPC()); 1633 } 1634 1635 Int128 HELPER(tre)(CPUS390XState *env, uint64_t array, 1636 uint64_t len, uint64_t trans) 1637 { 1638 uintptr_t ra = GETPC(); 1639 uint8_t end = env->regs[0] & 0xff; 1640 uint64_t l = len; 1641 uint64_t i; 1642 uint32_t cc = 0; 1643 1644 if (!(env->psw.mask & PSW_MASK_64)) { 1645 array &= 0x7fffffff; 1646 l = (uint32_t)l; 1647 } 1648 1649 /* Lest we fail to service interrupts in a timely manner, limit the 1650 amount of work we're willing to do. For now, let's cap at 8k. */ 1651 if (l > 0x2000) { 1652 l = 0x2000; 1653 cc = 3; 1654 } 1655 1656 for (i = 0; i < l; i++) { 1657 uint8_t byte, new_byte; 1658 1659 byte = cpu_ldub_data_ra(env, array + i, ra); 1660 1661 if (byte == end) { 1662 cc = 1; 1663 break; 1664 } 1665 1666 new_byte = cpu_ldub_data_ra(env, trans + byte, ra); 1667 cpu_stb_data_ra(env, array + i, new_byte, ra); 1668 } 1669 1670 env->cc_op = cc; 1671 return int128_make128(len - i, array + i); 1672 } 1673 1674 static inline uint32_t do_helper_trt(CPUS390XState *env, int len, 1675 uint64_t array, uint64_t trans, 1676 int inc, uintptr_t ra) 1677 { 1678 int i; 1679 1680 for (i = 0; i <= len; i++) { 1681 uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra); 1682 uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra); 1683 1684 if (sbyte != 0) { 1685 set_address(env, 1, array + i * inc); 1686 env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte); 1687 return (i == len) ? 2 : 1; 1688 } 1689 } 1690 1691 return 0; 1692 } 1693 1694 static uint32_t do_helper_trt_fwd(CPUS390XState *env, uint32_t len, 1695 uint64_t array, uint64_t trans, 1696 uintptr_t ra) 1697 { 1698 return do_helper_trt(env, len, array, trans, 1, ra); 1699 } 1700 1701 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array, 1702 uint64_t trans) 1703 { 1704 return do_helper_trt(env, len, array, trans, 1, GETPC()); 1705 } 1706 1707 static uint32_t do_helper_trt_bkwd(CPUS390XState *env, uint32_t len, 1708 uint64_t array, uint64_t trans, 1709 uintptr_t ra) 1710 { 1711 return do_helper_trt(env, len, array, trans, -1, ra); 1712 } 1713 1714 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array, 1715 uint64_t trans) 1716 { 1717 return do_helper_trt(env, len, array, trans, -1, GETPC()); 1718 } 1719 1720 /* Translate one/two to one/two */ 1721 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2, 1722 uint32_t tst, uint32_t sizes) 1723 { 1724 uintptr_t ra = GETPC(); 1725 int dsize = (sizes & 1) ? 1 : 2; 1726 int ssize = (sizes & 2) ? 1 : 2; 1727 uint64_t tbl = get_address(env, 1); 1728 uint64_t dst = get_address(env, r1); 1729 uint64_t len = get_length(env, r1 + 1); 1730 uint64_t src = get_address(env, r2); 1731 uint32_t cc = 3; 1732 int i; 1733 1734 /* The lower address bits of TBL are ignored. For TROO, TROT, it's 1735 the low 3 bits (double-word aligned). For TRTO, TRTT, it's either 1736 the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */ 1737 if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) { 1738 tbl &= -4096; 1739 } else { 1740 tbl &= -8; 1741 } 1742 1743 check_alignment(env, len, ssize, ra); 1744 1745 /* Lest we fail to service interrupts in a timely manner, */ 1746 /* limit the amount of work we're willing to do. */ 1747 for (i = 0; i < 0x2000; i++) { 1748 uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra); 1749 uint64_t tble = tbl + (sval * dsize); 1750 uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra); 1751 if (dval == tst) { 1752 cc = 1; 1753 break; 1754 } 1755 cpu_stsize_data_ra(env, dst, dval, dsize, ra); 1756 1757 len -= ssize; 1758 src += ssize; 1759 dst += dsize; 1760 1761 if (len == 0) { 1762 cc = 0; 1763 break; 1764 } 1765 } 1766 1767 set_address(env, r1, dst); 1768 set_length(env, r1 + 1, len); 1769 set_address(env, r2, src); 1770 1771 return cc; 1772 } 1773 1774 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, 1775 uint64_t a2, bool parallel) 1776 { 1777 uint32_t mem_idx = s390x_env_mmu_index(env, false); 1778 MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, mem_idx); 1779 MemOpIdx oi8 = make_memop_idx(MO_TE | MO_64, mem_idx); 1780 MemOpIdx oi4 = make_memop_idx(MO_TE | MO_32, mem_idx); 1781 MemOpIdx oi2 = make_memop_idx(MO_TE | MO_16, mem_idx); 1782 MemOpIdx oi1 = make_memop_idx(MO_8, mem_idx); 1783 uintptr_t ra = GETPC(); 1784 uint32_t fc = extract32(env->regs[0], 0, 8); 1785 uint32_t sc = extract32(env->regs[0], 8, 8); 1786 uint64_t pl = get_address(env, 1) & -16; 1787 uint64_t svh, svl; 1788 uint32_t cc; 1789 1790 /* Sanity check the function code and storage characteristic. */ 1791 if (fc > 1 || sc > 3) { 1792 if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) { 1793 goto spec_exception; 1794 } 1795 if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) { 1796 goto spec_exception; 1797 } 1798 } 1799 1800 /* Sanity check the alignments. */ 1801 if (extract32(a1, 0, fc + 2) || extract32(a2, 0, sc)) { 1802 goto spec_exception; 1803 } 1804 1805 /* Sanity check writability of the store address. */ 1806 probe_write(env, a2, 1 << sc, mem_idx, ra); 1807 1808 /* 1809 * Note that the compare-and-swap is atomic, and the store is atomic, 1810 * but the complete operation is not. Therefore we do not need to 1811 * assert serial context in order to implement this. That said, 1812 * restart early if we can't support either operation that is supposed 1813 * to be atomic. 1814 */ 1815 if (parallel) { 1816 uint32_t max = 2; 1817 #ifdef CONFIG_ATOMIC64 1818 max = 3; 1819 #endif 1820 if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) || 1821 (HAVE_ATOMIC128_RW ? 0 : sc > max)) { 1822 cpu_loop_exit_atomic(env_cpu(env), ra); 1823 } 1824 } 1825 1826 /* 1827 * All loads happen before all stores. For simplicity, load the entire 1828 * store value area from the parameter list. 1829 */ 1830 svh = cpu_ldq_mmu(env, pl + 16, oi8, ra); 1831 svl = cpu_ldq_mmu(env, pl + 24, oi8, ra); 1832 1833 switch (fc) { 1834 case 0: 1835 { 1836 uint32_t nv = cpu_ldl_mmu(env, pl, oi4, ra); 1837 uint32_t cv = env->regs[r3]; 1838 uint32_t ov; 1839 1840 if (parallel) { 1841 ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi4, ra); 1842 } else { 1843 ov = cpu_ldl_mmu(env, a1, oi4, ra); 1844 cpu_stl_mmu(env, a1, (ov == cv ? nv : ov), oi4, ra); 1845 } 1846 cc = (ov != cv); 1847 env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov); 1848 } 1849 break; 1850 1851 case 1: 1852 { 1853 uint64_t nv = cpu_ldq_mmu(env, pl, oi8, ra); 1854 uint64_t cv = env->regs[r3]; 1855 uint64_t ov; 1856 1857 if (parallel) { 1858 #ifdef CONFIG_ATOMIC64 1859 ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi8, ra); 1860 #else 1861 /* Note that we asserted !parallel above. */ 1862 g_assert_not_reached(); 1863 #endif 1864 } else { 1865 ov = cpu_ldq_mmu(env, a1, oi8, ra); 1866 cpu_stq_mmu(env, a1, (ov == cv ? nv : ov), oi8, ra); 1867 } 1868 cc = (ov != cv); 1869 env->regs[r3] = ov; 1870 } 1871 break; 1872 1873 case 2: 1874 { 1875 Int128 nv = cpu_ld16_mmu(env, pl, oi16, ra); 1876 Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]); 1877 Int128 ov; 1878 1879 if (!parallel) { 1880 ov = cpu_ld16_mmu(env, a1, oi16, ra); 1881 cc = !int128_eq(ov, cv); 1882 if (cc) { 1883 nv = ov; 1884 } 1885 cpu_st16_mmu(env, a1, nv, oi16, ra); 1886 } else if (HAVE_CMPXCHG128) { 1887 ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi16, ra); 1888 cc = !int128_eq(ov, cv); 1889 } else { 1890 /* Note that we asserted !parallel above. */ 1891 g_assert_not_reached(); 1892 } 1893 1894 env->regs[r3 + 0] = int128_gethi(ov); 1895 env->regs[r3 + 1] = int128_getlo(ov); 1896 } 1897 break; 1898 1899 default: 1900 g_assert_not_reached(); 1901 } 1902 1903 /* Store only if the comparison succeeded. Note that above we use a pair 1904 of 64-bit big-endian loads, so for sc < 3 we must extract the value 1905 from the most-significant bits of svh. */ 1906 if (cc == 0) { 1907 switch (sc) { 1908 case 0: 1909 cpu_stb_mmu(env, a2, svh >> 56, oi1, ra); 1910 break; 1911 case 1: 1912 cpu_stw_mmu(env, a2, svh >> 48, oi2, ra); 1913 break; 1914 case 2: 1915 cpu_stl_mmu(env, a2, svh >> 32, oi4, ra); 1916 break; 1917 case 3: 1918 cpu_stq_mmu(env, a2, svh, oi8, ra); 1919 break; 1920 case 4: 1921 cpu_st16_mmu(env, a2, int128_make128(svl, svh), oi16, ra); 1922 break; 1923 default: 1924 g_assert_not_reached(); 1925 } 1926 } 1927 1928 return cc; 1929 1930 spec_exception: 1931 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1932 } 1933 1934 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2) 1935 { 1936 return do_csst(env, r3, a1, a2, false); 1937 } 1938 1939 uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1, 1940 uint64_t a2) 1941 { 1942 return do_csst(env, r3, a1, a2, true); 1943 } 1944 1945 #if !defined(CONFIG_USER_ONLY) 1946 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1947 { 1948 uintptr_t ra = GETPC(); 1949 bool PERchanged = false; 1950 uint64_t src = a2; 1951 uint32_t i; 1952 1953 if (src & 0x7) { 1954 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1955 } 1956 1957 for (i = r1;; i = (i + 1) % 16) { 1958 uint64_t val = cpu_ldq_data_ra(env, src, ra); 1959 if (env->cregs[i] != val && i >= 9 && i <= 11) { 1960 PERchanged = true; 1961 } 1962 if (i == 0 && !(env->cregs[i] & CR0_CKC_SC) && (val & CR0_CKC_SC)) { 1963 BQL_LOCK_GUARD(); 1964 tcg_s390_tod_updated(env_cpu(env), RUN_ON_CPU_NULL); 1965 } 1966 env->cregs[i] = val; 1967 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n", 1968 i, src, val); 1969 src += sizeof(uint64_t); 1970 1971 if (i == r3) { 1972 break; 1973 } 1974 } 1975 1976 if (PERchanged && env->psw.mask & PSW_MASK_PER) { 1977 s390_cpu_recompute_watchpoints(env_cpu(env)); 1978 } 1979 1980 tlb_flush(env_cpu(env)); 1981 } 1982 1983 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1984 { 1985 uintptr_t ra = GETPC(); 1986 bool PERchanged = false; 1987 uint64_t src = a2; 1988 uint32_t i; 1989 1990 if (src & 0x3) { 1991 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1992 } 1993 1994 for (i = r1;; i = (i + 1) % 16) { 1995 uint32_t val = cpu_ldl_data_ra(env, src, ra); 1996 uint64_t val64 = deposit64(env->cregs[i], 0, 32, val); 1997 if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) { 1998 PERchanged = true; 1999 } 2000 if (i == 0 && !(env->cregs[i] & CR0_CKC_SC) && (val64 & CR0_CKC_SC)) { 2001 BQL_LOCK_GUARD(); 2002 tcg_s390_tod_updated(env_cpu(env), RUN_ON_CPU_NULL); 2003 } 2004 env->cregs[i] = val64; 2005 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val); 2006 src += sizeof(uint32_t); 2007 2008 if (i == r3) { 2009 break; 2010 } 2011 } 2012 2013 if (PERchanged && env->psw.mask & PSW_MASK_PER) { 2014 s390_cpu_recompute_watchpoints(env_cpu(env)); 2015 } 2016 2017 tlb_flush(env_cpu(env)); 2018 } 2019 2020 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 2021 { 2022 uintptr_t ra = GETPC(); 2023 uint64_t dest = a2; 2024 uint32_t i; 2025 2026 if (dest & 0x7) { 2027 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2028 } 2029 2030 for (i = r1;; i = (i + 1) % 16) { 2031 cpu_stq_data_ra(env, dest, env->cregs[i], ra); 2032 dest += sizeof(uint64_t); 2033 2034 if (i == r3) { 2035 break; 2036 } 2037 } 2038 } 2039 2040 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 2041 { 2042 uintptr_t ra = GETPC(); 2043 uint64_t dest = a2; 2044 uint32_t i; 2045 2046 if (dest & 0x3) { 2047 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2048 } 2049 2050 for (i = r1;; i = (i + 1) % 16) { 2051 cpu_stl_data_ra(env, dest, env->cregs[i], ra); 2052 dest += sizeof(uint32_t); 2053 2054 if (i == r3) { 2055 break; 2056 } 2057 } 2058 } 2059 2060 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr) 2061 { 2062 uintptr_t ra = GETPC(); 2063 int i; 2064 2065 real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK; 2066 2067 for (i = 0; i < TARGET_PAGE_SIZE; i += 8) { 2068 cpu_stq_mmuidx_ra(env, real_addr + i, 0, MMU_REAL_IDX, ra); 2069 } 2070 2071 return 0; 2072 } 2073 2074 uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2) 2075 { 2076 S390CPU *cpu = env_archcpu(env); 2077 CPUState *cs = env_cpu(env); 2078 2079 /* 2080 * TODO: we currently don't handle all access protection types 2081 * (including access-list and key-controlled) as well as AR mode. 2082 */ 2083 if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) { 2084 /* Fetching permitted; storing permitted */ 2085 return 0; 2086 } 2087 2088 if (env->int_pgm_code == PGM_PROTECTION) { 2089 /* retry if reading is possible */ 2090 cs->exception_index = -1; 2091 if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) { 2092 /* Fetching permitted; storing not permitted */ 2093 return 1; 2094 } 2095 } 2096 2097 switch (env->int_pgm_code) { 2098 case PGM_PROTECTION: 2099 /* Fetching not permitted; storing not permitted */ 2100 cs->exception_index = -1; 2101 return 2; 2102 case PGM_ADDRESSING: 2103 case PGM_TRANS_SPEC: 2104 /* exceptions forwarded to the guest */ 2105 s390_cpu_virt_mem_handle_exc(cpu, GETPC()); 2106 return 0; 2107 } 2108 2109 /* Translation not available */ 2110 cs->exception_index = -1; 2111 return 3; 2112 } 2113 2114 /* insert storage key extended */ 2115 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2) 2116 { 2117 static S390SKeysState *ss; 2118 static S390SKeysClass *skeyclass; 2119 uint64_t addr = wrap_address(env, r2); 2120 uint8_t key; 2121 int rc; 2122 2123 addr = mmu_real2abs(env, addr); 2124 if (!mmu_absolute_addr_valid(addr, false)) { 2125 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2126 } 2127 2128 if (unlikely(!ss)) { 2129 ss = s390_get_skeys_device(); 2130 skeyclass = S390_SKEYS_GET_CLASS(ss); 2131 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2132 tlb_flush_all_cpus_synced(env_cpu(env)); 2133 } 2134 } 2135 2136 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2137 if (rc) { 2138 return 0; 2139 } 2140 return key; 2141 } 2142 2143 /* set storage key extended */ 2144 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2) 2145 { 2146 static S390SKeysState *ss; 2147 static S390SKeysClass *skeyclass; 2148 uint64_t addr = wrap_address(env, r2); 2149 uint8_t key; 2150 2151 addr = mmu_real2abs(env, addr); 2152 if (!mmu_absolute_addr_valid(addr, false)) { 2153 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2154 } 2155 2156 if (unlikely(!ss)) { 2157 ss = s390_get_skeys_device(); 2158 skeyclass = S390_SKEYS_GET_CLASS(ss); 2159 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2160 tlb_flush_all_cpus_synced(env_cpu(env)); 2161 } 2162 } 2163 2164 key = r1 & 0xfe; 2165 s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2166 /* 2167 * As we can only flush by virtual address and not all the entries 2168 * that point to a physical address we have to flush the whole TLB. 2169 */ 2170 tlb_flush_all_cpus_synced(env_cpu(env)); 2171 } 2172 2173 /* reset reference bit extended */ 2174 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2) 2175 { 2176 uint64_t addr = wrap_address(env, r2); 2177 static S390SKeysState *ss; 2178 static S390SKeysClass *skeyclass; 2179 uint8_t re, key; 2180 int rc; 2181 2182 addr = mmu_real2abs(env, addr); 2183 if (!mmu_absolute_addr_valid(addr, false)) { 2184 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2185 } 2186 2187 if (unlikely(!ss)) { 2188 ss = s390_get_skeys_device(); 2189 skeyclass = S390_SKEYS_GET_CLASS(ss); 2190 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2191 tlb_flush_all_cpus_synced(env_cpu(env)); 2192 } 2193 } 2194 2195 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2196 if (rc) { 2197 return 0; 2198 } 2199 2200 re = key & (SK_R | SK_C); 2201 key &= ~SK_R; 2202 2203 rc = s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2204 if (rc) { 2205 return 0; 2206 } 2207 /* 2208 * As we can only flush by virtual address and not all the entries 2209 * that point to a physical address we have to flush the whole TLB. 2210 */ 2211 tlb_flush_all_cpus_synced(env_cpu(env)); 2212 2213 /* 2214 * cc 2215 * 2216 * 0 Reference bit zero; change bit zero 2217 * 1 Reference bit zero; change bit one 2218 * 2 Reference bit one; change bit zero 2219 * 3 Reference bit one; change bit one 2220 */ 2221 2222 return re >> 1; 2223 } 2224 2225 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2, 2226 uint64_t key) 2227 { 2228 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2229 S390Access srca, desta; 2230 uintptr_t ra = GETPC(); 2231 int cc = 0; 2232 2233 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n", 2234 __func__, l, a1, a2); 2235 2236 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) || 2237 psw_as == AS_HOME || psw_as == AS_ACCREG) { 2238 s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2239 } 2240 2241 if (!psw_key_valid(env, (key >> 4) & 0xf)) { 2242 s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2243 } 2244 2245 l = wrap_length32(env, l); 2246 if (l > 256) { 2247 /* max 256 */ 2248 l = 256; 2249 cc = 3; 2250 } else if (!l) { 2251 return cc; 2252 } 2253 2254 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_PRIMARY_IDX, ra); 2255 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_SECONDARY_IDX, ra); 2256 access_memmove(env, &desta, &srca, ra); 2257 return cc; 2258 } 2259 2260 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2, 2261 uint64_t key) 2262 { 2263 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2264 S390Access srca, desta; 2265 uintptr_t ra = GETPC(); 2266 int cc = 0; 2267 2268 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n", 2269 __func__, l, a1, a2); 2270 2271 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) || 2272 psw_as == AS_HOME || psw_as == AS_ACCREG) { 2273 s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2274 } 2275 2276 if (!psw_key_valid(env, (key >> 4) & 0xf)) { 2277 s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2278 } 2279 2280 l = wrap_length32(env, l); 2281 if (l > 256) { 2282 /* max 256 */ 2283 l = 256; 2284 cc = 3; 2285 } else if (!l) { 2286 return cc; 2287 } 2288 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_SECONDARY_IDX, ra); 2289 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_PRIMARY_IDX, ra); 2290 access_memmove(env, &desta, &srca, ra); 2291 return cc; 2292 } 2293 2294 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4) 2295 { 2296 CPUState *cs = env_cpu(env); 2297 const uintptr_t ra = GETPC(); 2298 uint64_t table, entry, raddr; 2299 uint16_t entries, i, index = 0; 2300 2301 if (r2 & 0xff000) { 2302 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2303 } 2304 2305 if (!(r2 & 0x800)) { 2306 /* invalidation-and-clearing operation */ 2307 table = r1 & ASCE_ORIGIN; 2308 entries = (r2 & 0x7ff) + 1; 2309 2310 switch (r1 & ASCE_TYPE_MASK) { 2311 case ASCE_TYPE_REGION1: 2312 index = (r2 >> 53) & 0x7ff; 2313 break; 2314 case ASCE_TYPE_REGION2: 2315 index = (r2 >> 42) & 0x7ff; 2316 break; 2317 case ASCE_TYPE_REGION3: 2318 index = (r2 >> 31) & 0x7ff; 2319 break; 2320 case ASCE_TYPE_SEGMENT: 2321 index = (r2 >> 20) & 0x7ff; 2322 break; 2323 } 2324 for (i = 0; i < entries; i++) { 2325 /* addresses are not wrapped in 24/31bit mode but table index is */ 2326 raddr = table + ((index + i) & 0x7ff) * sizeof(entry); 2327 entry = cpu_ldq_mmuidx_ra(env, raddr, MMU_REAL_IDX, ra); 2328 if (!(entry & REGION_ENTRY_I)) { 2329 /* we are allowed to not store if already invalid */ 2330 entry |= REGION_ENTRY_I; 2331 cpu_stq_mmuidx_ra(env, raddr, entry, MMU_REAL_IDX, ra); 2332 } 2333 } 2334 } 2335 2336 /* We simply flush the complete tlb, therefore we can ignore r3. */ 2337 if (m4 & 1) { 2338 tlb_flush(cs); 2339 } else { 2340 tlb_flush_all_cpus_synced(cs); 2341 } 2342 } 2343 2344 /* invalidate pte */ 2345 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr, 2346 uint32_t m4) 2347 { 2348 CPUState *cs = env_cpu(env); 2349 const uintptr_t ra = GETPC(); 2350 uint64_t page = vaddr & TARGET_PAGE_MASK; 2351 uint64_t pte_addr, pte; 2352 2353 /* Compute the page table entry address */ 2354 pte_addr = (pto & SEGMENT_ENTRY_ORIGIN); 2355 pte_addr += VADDR_PAGE_TX(vaddr) * 8; 2356 2357 /* Mark the page table entry as invalid */ 2358 pte = cpu_ldq_mmuidx_ra(env, pte_addr, MMU_REAL_IDX, ra); 2359 pte |= PAGE_ENTRY_I; 2360 cpu_stq_mmuidx_ra(env, pte_addr, pte, MMU_REAL_IDX, ra); 2361 2362 /* XXX we exploit the fact that Linux passes the exact virtual 2363 address here - it's not obliged to! */ 2364 if (m4 & 1) { 2365 if (vaddr & ~VADDR_PAGE_TX_MASK) { 2366 tlb_flush_page(cs, page); 2367 /* XXX 31-bit hack */ 2368 tlb_flush_page(cs, page ^ 0x80000000); 2369 } else { 2370 /* looks like we don't have a valid virtual address */ 2371 tlb_flush(cs); 2372 } 2373 } else { 2374 if (vaddr & ~VADDR_PAGE_TX_MASK) { 2375 tlb_flush_page_all_cpus_synced(cs, page); 2376 /* XXX 31-bit hack */ 2377 tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000); 2378 } else { 2379 /* looks like we don't have a valid virtual address */ 2380 tlb_flush_all_cpus_synced(cs); 2381 } 2382 } 2383 } 2384 2385 /* flush local tlb */ 2386 void HELPER(ptlb)(CPUS390XState *env) 2387 { 2388 tlb_flush(env_cpu(env)); 2389 } 2390 2391 /* flush global tlb */ 2392 void HELPER(purge)(CPUS390XState *env) 2393 { 2394 tlb_flush_all_cpus_synced(env_cpu(env)); 2395 } 2396 2397 /* load real address */ 2398 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t r1, uint64_t addr) 2399 { 2400 uint64_t asc = env->psw.mask & PSW_MASK_ASC; 2401 uint64_t ret, tec; 2402 int flags, exc, cc; 2403 2404 /* XXX incomplete - has more corner cases */ 2405 if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) { 2406 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, GETPC()); 2407 } 2408 2409 exc = mmu_translate(env, addr, MMU_S390_LRA, asc, &ret, &flags, &tec); 2410 if (exc) { 2411 cc = 3; 2412 ret = (r1 & 0xFFFFFFFF00000000ULL) | exc | 0x80000000; 2413 } else { 2414 cc = 0; 2415 ret |= addr & ~TARGET_PAGE_MASK; 2416 } 2417 2418 env->cc_op = cc; 2419 return ret; 2420 } 2421 #endif 2422 2423 /* Execute instruction. This instruction executes an insn modified with 2424 the contents of r1. It does not change the executed instruction in memory; 2425 it does not change the program counter. 2426 2427 Perform this by recording the modified instruction in env->ex_value. 2428 This will be noticed by cpu_get_tb_cpu_state and thus tb translation. 2429 */ 2430 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr) 2431 { 2432 uint64_t insn; 2433 uint8_t opc; 2434 2435 /* EXECUTE targets must be at even addresses. */ 2436 if (addr & 1) { 2437 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC()); 2438 } 2439 2440 insn = cpu_lduw_code(env, addr); 2441 opc = insn >> 8; 2442 2443 /* Or in the contents of R1[56:63]. */ 2444 insn |= r1 & 0xff; 2445 2446 /* Load the rest of the instruction. */ 2447 insn <<= 48; 2448 switch (get_ilen(opc)) { 2449 case 2: 2450 break; 2451 case 4: 2452 insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32; 2453 break; 2454 case 6: 2455 insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16; 2456 break; 2457 default: 2458 g_assert_not_reached(); 2459 } 2460 2461 /* The very most common cases can be sped up by avoiding a new TB. */ 2462 if ((opc & 0xf0) == 0xd0) { 2463 typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t, 2464 uint64_t, uintptr_t); 2465 static const dx_helper dx[16] = { 2466 [0x0] = do_helper_trt_bkwd, 2467 [0x2] = do_helper_mvc, 2468 [0x4] = do_helper_nc, 2469 [0x5] = do_helper_clc, 2470 [0x6] = do_helper_oc, 2471 [0x7] = do_helper_xc, 2472 [0xc] = do_helper_tr, 2473 [0xd] = do_helper_trt_fwd, 2474 }; 2475 dx_helper helper = dx[opc & 0xf]; 2476 2477 if (helper) { 2478 uint32_t l = extract64(insn, 48, 8); 2479 uint32_t b1 = extract64(insn, 44, 4); 2480 uint32_t d1 = extract64(insn, 32, 12); 2481 uint32_t b2 = extract64(insn, 28, 4); 2482 uint32_t d2 = extract64(insn, 16, 12); 2483 uint64_t a1 = wrap_address(env, (b1 ? env->regs[b1] : 0) + d1); 2484 uint64_t a2 = wrap_address(env, (b2 ? env->regs[b2] : 0) + d2); 2485 2486 env->cc_op = helper(env, l, a1, a2, 0); 2487 env->psw.addr += ilen; 2488 return; 2489 } 2490 } else if (opc == 0x0a) { 2491 env->int_svc_code = extract64(insn, 48, 8); 2492 env->int_svc_ilen = ilen; 2493 helper_exception(env, EXCP_SVC); 2494 g_assert_not_reached(); 2495 } 2496 2497 /* Record the insn we want to execute as well as the ilen to use 2498 during the execution of the target insn. This will also ensure 2499 that ex_value is non-zero, which flags that we are in a state 2500 that requires such execution. */ 2501 env->ex_value = insn | ilen; 2502 env->ex_target = addr; 2503 } 2504 2505 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src, 2506 uint64_t len) 2507 { 2508 const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY; 2509 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2510 const uint64_t r0 = env->regs[0]; 2511 const uintptr_t ra = GETPC(); 2512 uint8_t dest_key, dest_as, dest_k, dest_a; 2513 uint8_t src_key, src_as, src_k, src_a; 2514 uint64_t val; 2515 int cc = 0; 2516 2517 HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n", 2518 __func__, dest, src, len); 2519 2520 if (!(env->psw.mask & PSW_MASK_DAT)) { 2521 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2522 } 2523 2524 /* OAC (operand access control) for the first operand -> dest */ 2525 val = (r0 & 0xffff0000ULL) >> 16; 2526 dest_key = (val >> 12) & 0xf; 2527 dest_as = (val >> 6) & 0x3; 2528 dest_k = (val >> 1) & 0x1; 2529 dest_a = val & 0x1; 2530 2531 /* OAC (operand access control) for the second operand -> src */ 2532 val = (r0 & 0x0000ffffULL); 2533 src_key = (val >> 12) & 0xf; 2534 src_as = (val >> 6) & 0x3; 2535 src_k = (val >> 1) & 0x1; 2536 src_a = val & 0x1; 2537 2538 if (!dest_k) { 2539 dest_key = psw_key; 2540 } 2541 if (!src_k) { 2542 src_key = psw_key; 2543 } 2544 if (!dest_a) { 2545 dest_as = psw_as; 2546 } 2547 if (!src_a) { 2548 src_as = psw_as; 2549 } 2550 2551 if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) { 2552 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2553 } 2554 if (!(env->cregs[0] & CR0_SECONDARY) && 2555 (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) { 2556 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2557 } 2558 if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) { 2559 tcg_s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2560 } 2561 2562 len = wrap_length32(env, len); 2563 if (len > 4096) { 2564 cc = 3; 2565 len = 4096; 2566 } 2567 2568 /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */ 2569 if (src_as == AS_ACCREG || dest_as == AS_ACCREG || 2570 (env->psw.mask & PSW_MASK_PSTATE)) { 2571 qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n", 2572 __func__); 2573 tcg_s390_program_interrupt(env, PGM_ADDRESSING, ra); 2574 } 2575 2576 /* FIXME: Access using correct keys and AR-mode */ 2577 if (len) { 2578 S390Access srca, desta; 2579 2580 access_prepare(&srca, env, src, len, MMU_DATA_LOAD, 2581 mmu_idx_from_as(src_as), ra); 2582 access_prepare(&desta, env, dest, len, MMU_DATA_STORE, 2583 mmu_idx_from_as(dest_as), ra); 2584 2585 access_memmove(env, &desta, &srca, ra); 2586 } 2587 2588 return cc; 2589 } 2590 2591 /* Decode a Unicode character. A return value < 0 indicates success, storing 2592 the UTF-32 result into OCHAR and the input length into OLEN. A return 2593 value >= 0 indicates failure, and the CC value to be returned. */ 2594 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr, 2595 uint64_t ilen, bool enh_check, uintptr_t ra, 2596 uint32_t *ochar, uint32_t *olen); 2597 2598 /* Encode a Unicode character. A return value < 0 indicates success, storing 2599 the bytes into ADDR and the output length into OLEN. A return value >= 0 2600 indicates failure, and the CC value to be returned. */ 2601 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr, 2602 uint64_t ilen, uintptr_t ra, uint32_t c, 2603 uint32_t *olen); 2604 2605 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2606 bool enh_check, uintptr_t ra, 2607 uint32_t *ochar, uint32_t *olen) 2608 { 2609 uint8_t s0, s1, s2, s3; 2610 uint32_t c, l; 2611 2612 if (ilen < 1) { 2613 return 0; 2614 } 2615 s0 = cpu_ldub_data_ra(env, addr, ra); 2616 if (s0 <= 0x7f) { 2617 /* one byte character */ 2618 l = 1; 2619 c = s0; 2620 } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) { 2621 /* invalid character */ 2622 return 2; 2623 } else if (s0 <= 0xdf) { 2624 /* two byte character */ 2625 l = 2; 2626 if (ilen < 2) { 2627 return 0; 2628 } 2629 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2630 c = s0 & 0x1f; 2631 c = (c << 6) | (s1 & 0x3f); 2632 if (enh_check && (s1 & 0xc0) != 0x80) { 2633 return 2; 2634 } 2635 } else if (s0 <= 0xef) { 2636 /* three byte character */ 2637 l = 3; 2638 if (ilen < 3) { 2639 return 0; 2640 } 2641 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2642 s2 = cpu_ldub_data_ra(env, addr + 2, ra); 2643 c = s0 & 0x0f; 2644 c = (c << 6) | (s1 & 0x3f); 2645 c = (c << 6) | (s2 & 0x3f); 2646 /* Fold the byte-by-byte range descriptions in the PoO into 2647 tests against the complete value. It disallows encodings 2648 that could be smaller, and the UTF-16 surrogates. */ 2649 if (enh_check 2650 && ((s1 & 0xc0) != 0x80 2651 || (s2 & 0xc0) != 0x80 2652 || c < 0x1000 2653 || (c >= 0xd800 && c <= 0xdfff))) { 2654 return 2; 2655 } 2656 } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) { 2657 /* four byte character */ 2658 l = 4; 2659 if (ilen < 4) { 2660 return 0; 2661 } 2662 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2663 s2 = cpu_ldub_data_ra(env, addr + 2, ra); 2664 s3 = cpu_ldub_data_ra(env, addr + 3, ra); 2665 c = s0 & 0x07; 2666 c = (c << 6) | (s1 & 0x3f); 2667 c = (c << 6) | (s2 & 0x3f); 2668 c = (c << 6) | (s3 & 0x3f); 2669 /* See above. */ 2670 if (enh_check 2671 && ((s1 & 0xc0) != 0x80 2672 || (s2 & 0xc0) != 0x80 2673 || (s3 & 0xc0) != 0x80 2674 || c < 0x010000 2675 || c > 0x10ffff)) { 2676 return 2; 2677 } 2678 } else { 2679 /* invalid character */ 2680 return 2; 2681 } 2682 2683 *ochar = c; 2684 *olen = l; 2685 return -1; 2686 } 2687 2688 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2689 bool enh_check, uintptr_t ra, 2690 uint32_t *ochar, uint32_t *olen) 2691 { 2692 uint16_t s0, s1; 2693 uint32_t c, l; 2694 2695 if (ilen < 2) { 2696 return 0; 2697 } 2698 s0 = cpu_lduw_data_ra(env, addr, ra); 2699 if ((s0 & 0xfc00) != 0xd800) { 2700 /* one word character */ 2701 l = 2; 2702 c = s0; 2703 } else { 2704 /* two word character */ 2705 l = 4; 2706 if (ilen < 4) { 2707 return 0; 2708 } 2709 s1 = cpu_lduw_data_ra(env, addr + 2, ra); 2710 c = extract32(s0, 6, 4) + 1; 2711 c = (c << 6) | (s0 & 0x3f); 2712 c = (c << 10) | (s1 & 0x3ff); 2713 if (enh_check && (s1 & 0xfc00) != 0xdc00) { 2714 /* invalid surrogate character */ 2715 return 2; 2716 } 2717 } 2718 2719 *ochar = c; 2720 *olen = l; 2721 return -1; 2722 } 2723 2724 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2725 bool enh_check, uintptr_t ra, 2726 uint32_t *ochar, uint32_t *olen) 2727 { 2728 uint32_t c; 2729 2730 if (ilen < 4) { 2731 return 0; 2732 } 2733 c = cpu_ldl_data_ra(env, addr, ra); 2734 if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) { 2735 /* invalid unicode character */ 2736 return 2; 2737 } 2738 2739 *ochar = c; 2740 *olen = 4; 2741 return -1; 2742 } 2743 2744 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2745 uintptr_t ra, uint32_t c, uint32_t *olen) 2746 { 2747 uint8_t d[4]; 2748 uint32_t l, i; 2749 2750 if (c <= 0x7f) { 2751 /* one byte character */ 2752 l = 1; 2753 d[0] = c; 2754 } else if (c <= 0x7ff) { 2755 /* two byte character */ 2756 l = 2; 2757 d[1] = 0x80 | extract32(c, 0, 6); 2758 d[0] = 0xc0 | extract32(c, 6, 5); 2759 } else if (c <= 0xffff) { 2760 /* three byte character */ 2761 l = 3; 2762 d[2] = 0x80 | extract32(c, 0, 6); 2763 d[1] = 0x80 | extract32(c, 6, 6); 2764 d[0] = 0xe0 | extract32(c, 12, 4); 2765 } else { 2766 /* four byte character */ 2767 l = 4; 2768 d[3] = 0x80 | extract32(c, 0, 6); 2769 d[2] = 0x80 | extract32(c, 6, 6); 2770 d[1] = 0x80 | extract32(c, 12, 6); 2771 d[0] = 0xf0 | extract32(c, 18, 3); 2772 } 2773 2774 if (ilen < l) { 2775 return 1; 2776 } 2777 for (i = 0; i < l; ++i) { 2778 cpu_stb_data_ra(env, addr + i, d[i], ra); 2779 } 2780 2781 *olen = l; 2782 return -1; 2783 } 2784 2785 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2786 uintptr_t ra, uint32_t c, uint32_t *olen) 2787 { 2788 uint16_t d0, d1; 2789 2790 if (c <= 0xffff) { 2791 /* one word character */ 2792 if (ilen < 2) { 2793 return 1; 2794 } 2795 cpu_stw_data_ra(env, addr, c, ra); 2796 *olen = 2; 2797 } else { 2798 /* two word character */ 2799 if (ilen < 4) { 2800 return 1; 2801 } 2802 d1 = 0xdc00 | extract32(c, 0, 10); 2803 d0 = 0xd800 | extract32(c, 10, 6); 2804 d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1); 2805 cpu_stw_data_ra(env, addr + 0, d0, ra); 2806 cpu_stw_data_ra(env, addr + 2, d1, ra); 2807 *olen = 4; 2808 } 2809 2810 return -1; 2811 } 2812 2813 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2814 uintptr_t ra, uint32_t c, uint32_t *olen) 2815 { 2816 if (ilen < 4) { 2817 return 1; 2818 } 2819 cpu_stl_data_ra(env, addr, c, ra); 2820 *olen = 4; 2821 return -1; 2822 } 2823 2824 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1, 2825 uint32_t r2, uint32_t m3, uintptr_t ra, 2826 decode_unicode_fn decode, 2827 encode_unicode_fn encode) 2828 { 2829 uint64_t dst = get_address(env, r1); 2830 uint64_t dlen = get_length(env, r1 + 1); 2831 uint64_t src = get_address(env, r2); 2832 uint64_t slen = get_length(env, r2 + 1); 2833 bool enh_check = m3 & 1; 2834 int cc, i; 2835 2836 /* Lest we fail to service interrupts in a timely manner, limit the 2837 amount of work we're willing to do. For now, let's cap at 256. */ 2838 for (i = 0; i < 256; ++i) { 2839 uint32_t c, ilen, olen; 2840 2841 cc = decode(env, src, slen, enh_check, ra, &c, &ilen); 2842 if (unlikely(cc >= 0)) { 2843 break; 2844 } 2845 cc = encode(env, dst, dlen, ra, c, &olen); 2846 if (unlikely(cc >= 0)) { 2847 break; 2848 } 2849 2850 src += ilen; 2851 slen -= ilen; 2852 dst += olen; 2853 dlen -= olen; 2854 cc = 3; 2855 } 2856 2857 set_address(env, r1, dst); 2858 set_length(env, r1 + 1, dlen); 2859 set_address(env, r2, src); 2860 set_length(env, r2 + 1, slen); 2861 2862 return cc; 2863 } 2864 2865 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2866 { 2867 return convert_unicode(env, r1, r2, m3, GETPC(), 2868 decode_utf8, encode_utf16); 2869 } 2870 2871 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2872 { 2873 return convert_unicode(env, r1, r2, m3, GETPC(), 2874 decode_utf8, encode_utf32); 2875 } 2876 2877 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2878 { 2879 return convert_unicode(env, r1, r2, m3, GETPC(), 2880 decode_utf16, encode_utf8); 2881 } 2882 2883 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2884 { 2885 return convert_unicode(env, r1, r2, m3, GETPC(), 2886 decode_utf16, encode_utf32); 2887 } 2888 2889 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2890 { 2891 return convert_unicode(env, r1, r2, m3, GETPC(), 2892 decode_utf32, encode_utf8); 2893 } 2894 2895 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2896 { 2897 return convert_unicode(env, r1, r2, m3, GETPC(), 2898 decode_utf32, encode_utf16); 2899 } 2900 2901 void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len, 2902 uintptr_t ra) 2903 { 2904 const int mmu_idx = s390x_env_mmu_index(env, false); 2905 2906 /* test the actual access, not just any access to the page due to LAP */ 2907 while (len) { 2908 const uint64_t pagelen = -(addr | TARGET_PAGE_MASK); 2909 const uint64_t curlen = MIN(pagelen, len); 2910 2911 probe_write(env, addr, curlen, mmu_idx, ra); 2912 addr = wrap_address(env, addr + curlen); 2913 len -= curlen; 2914 } 2915 } 2916 2917 void HELPER(probe_write_access)(CPUS390XState *env, uint64_t addr, uint64_t len) 2918 { 2919 probe_write_access(env, addr, len, GETPC()); 2920 } 2921