1 /* 2 * S/390 memory access helper routines 3 * 4 * Copyright (c) 2009 Ulrich Hecht 5 * Copyright (c) 2009 Alexander Graf 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/osdep.h" 22 #include "qemu/log.h" 23 #include "cpu.h" 24 #include "s390x-internal.h" 25 #include "tcg_s390x.h" 26 #include "exec/helper-proto.h" 27 #include "exec/exec-all.h" 28 #include "exec/cpu_ldst.h" 29 #include "qemu/int128.h" 30 #include "qemu/atomic128.h" 31 #include "trace.h" 32 33 #if !defined(CONFIG_USER_ONLY) 34 #include "hw/s390x/storage-keys.h" 35 #include "hw/boards.h" 36 #endif 37 38 /*****************************************************************************/ 39 /* Softmmu support */ 40 41 /* #define DEBUG_HELPER */ 42 #ifdef DEBUG_HELPER 43 #define HELPER_LOG(x...) qemu_log(x) 44 #else 45 #define HELPER_LOG(x...) 46 #endif 47 48 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key) 49 { 50 uint16_t pkm = env->cregs[3] >> 16; 51 52 if (env->psw.mask & PSW_MASK_PSTATE) { 53 /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */ 54 return pkm & (0x8000 >> psw_key); 55 } 56 return true; 57 } 58 59 static bool is_destructive_overlap(CPUS390XState *env, uint64_t dest, 60 uint64_t src, uint32_t len) 61 { 62 if (!len || src == dest) { 63 return false; 64 } 65 /* Take care of wrapping at the end of address space. */ 66 if (unlikely(wrap_address(env, src + len - 1) < src)) { 67 return dest > src || dest <= wrap_address(env, src + len - 1); 68 } 69 return dest > src && dest <= src + len - 1; 70 } 71 72 /* Trigger a SPECIFICATION exception if an address or a length is not 73 naturally aligned. */ 74 static inline void check_alignment(CPUS390XState *env, uint64_t v, 75 int wordsize, uintptr_t ra) 76 { 77 if (v % wordsize) { 78 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 79 } 80 } 81 82 /* Load a value from memory according to its size. */ 83 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr, 84 int wordsize, uintptr_t ra) 85 { 86 switch (wordsize) { 87 case 1: 88 return cpu_ldub_data_ra(env, addr, ra); 89 case 2: 90 return cpu_lduw_data_ra(env, addr, ra); 91 default: 92 abort(); 93 } 94 } 95 96 /* Store a to memory according to its size. */ 97 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr, 98 uint64_t value, int wordsize, 99 uintptr_t ra) 100 { 101 switch (wordsize) { 102 case 1: 103 cpu_stb_data_ra(env, addr, value, ra); 104 break; 105 case 2: 106 cpu_stw_data_ra(env, addr, value, ra); 107 break; 108 default: 109 abort(); 110 } 111 } 112 113 /* An access covers at most 4096 bytes and therefore at most two pages. */ 114 typedef struct S390Access { 115 target_ulong vaddr1; 116 target_ulong vaddr2; 117 char *haddr1; 118 char *haddr2; 119 uint16_t size1; 120 uint16_t size2; 121 /* 122 * If we can't access the host page directly, we'll have to do I/O access 123 * via ld/st helpers. These are internal details, so we store the 124 * mmu idx to do the access here instead of passing it around in the 125 * helpers. Maybe, one day we can get rid of ld/st access - once we can 126 * handle TLB_NOTDIRTY differently. We don't expect these special accesses 127 * to trigger exceptions - only if we would have TLB_NOTDIRTY on LAP 128 * pages, we might trigger a new MMU translation - very unlikely that 129 * the mapping changes in between and we would trigger a fault. 130 */ 131 int mmu_idx; 132 } S390Access; 133 134 /* 135 * With nonfault=1, return the PGM_ exception that would have been injected 136 * into the guest; return 0 if no exception was detected. 137 * 138 * For !CONFIG_USER_ONLY, the TEC is stored stored to env->tlb_fill_tec. 139 * For CONFIG_USER_ONLY, the faulting address is stored to env->__excp_addr. 140 */ 141 static int s390_probe_access(CPUArchState *env, target_ulong addr, int size, 142 MMUAccessType access_type, int mmu_idx, 143 bool nonfault, void **phost, uintptr_t ra) 144 { 145 #if defined(CONFIG_USER_ONLY) 146 return probe_access_flags(env, addr, access_type, mmu_idx, 147 nonfault, phost, ra); 148 #else 149 int flags; 150 151 env->tlb_fill_exc = 0; 152 flags = probe_access_flags(env, addr, access_type, mmu_idx, nonfault, phost, 153 ra); 154 if (env->tlb_fill_exc) { 155 return env->tlb_fill_exc; 156 } 157 158 if (unlikely(flags & TLB_WATCHPOINT)) { 159 /* S390 does not presently use transaction attributes. */ 160 cpu_check_watchpoint(env_cpu(env), addr, size, 161 MEMTXATTRS_UNSPECIFIED, 162 (access_type == MMU_DATA_STORE 163 ? BP_MEM_WRITE : BP_MEM_READ), ra); 164 } 165 return 0; 166 #endif 167 } 168 169 static int access_prepare_nf(S390Access *access, CPUS390XState *env, 170 bool nonfault, vaddr vaddr1, int size, 171 MMUAccessType access_type, 172 int mmu_idx, uintptr_t ra) 173 { 174 void *haddr1, *haddr2 = NULL; 175 int size1, size2, exc; 176 vaddr vaddr2 = 0; 177 178 assert(size > 0 && size <= 4096); 179 180 size1 = MIN(size, -(vaddr1 | TARGET_PAGE_MASK)), 181 size2 = size - size1; 182 183 exc = s390_probe_access(env, vaddr1, size1, access_type, mmu_idx, nonfault, 184 &haddr1, ra); 185 if (exc) { 186 return exc; 187 } 188 if (unlikely(size2)) { 189 /* The access crosses page boundaries. */ 190 vaddr2 = wrap_address(env, vaddr1 + size1); 191 exc = s390_probe_access(env, vaddr2, size2, access_type, mmu_idx, 192 nonfault, &haddr2, ra); 193 if (exc) { 194 return exc; 195 } 196 } 197 198 *access = (S390Access) { 199 .vaddr1 = vaddr1, 200 .vaddr2 = vaddr2, 201 .haddr1 = haddr1, 202 .haddr2 = haddr2, 203 .size1 = size1, 204 .size2 = size2, 205 .mmu_idx = mmu_idx 206 }; 207 return 0; 208 } 209 210 static S390Access access_prepare(CPUS390XState *env, vaddr vaddr, int size, 211 MMUAccessType access_type, int mmu_idx, 212 uintptr_t ra) 213 { 214 S390Access ret; 215 int exc = access_prepare_nf(&ret, env, false, vaddr, size, 216 access_type, mmu_idx, ra); 217 assert(!exc); 218 return ret; 219 } 220 221 /* Helper to handle memset on a single page. */ 222 static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr, 223 uint8_t byte, uint16_t size, int mmu_idx, 224 uintptr_t ra) 225 { 226 #ifdef CONFIG_USER_ONLY 227 g_assert(haddr); 228 memset(haddr, byte, size); 229 #else 230 MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); 231 int i; 232 233 if (likely(haddr)) { 234 memset(haddr, byte, size); 235 } else { 236 /* 237 * Do a single access and test if we can then get access to the 238 * page. This is especially relevant to speed up TLB_NOTDIRTY. 239 */ 240 g_assert(size > 0); 241 cpu_stb_mmu(env, vaddr, byte, oi, ra); 242 haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx); 243 if (likely(haddr)) { 244 memset(haddr + 1, byte, size - 1); 245 } else { 246 for (i = 1; i < size; i++) { 247 cpu_stb_mmu(env, vaddr + i, byte, oi, ra); 248 } 249 } 250 } 251 #endif 252 } 253 254 static void access_memset(CPUS390XState *env, S390Access *desta, 255 uint8_t byte, uintptr_t ra) 256 { 257 258 do_access_memset(env, desta->vaddr1, desta->haddr1, byte, desta->size1, 259 desta->mmu_idx, ra); 260 if (likely(!desta->size2)) { 261 return; 262 } 263 do_access_memset(env, desta->vaddr2, desta->haddr2, byte, desta->size2, 264 desta->mmu_idx, ra); 265 } 266 267 static uint8_t do_access_get_byte(CPUS390XState *env, vaddr vaddr, char **haddr, 268 int offset, int mmu_idx, uintptr_t ra) 269 { 270 #ifdef CONFIG_USER_ONLY 271 return ldub_p(*haddr + offset); 272 #else 273 MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); 274 uint8_t byte; 275 276 if (likely(*haddr)) { 277 return ldub_p(*haddr + offset); 278 } 279 /* 280 * Do a single access and test if we can then get access to the 281 * page. This is especially relevant to speed up TLB_NOTDIRTY. 282 */ 283 byte = cpu_ldb_mmu(env, vaddr + offset, oi, ra); 284 *haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_LOAD, mmu_idx); 285 return byte; 286 #endif 287 } 288 289 static uint8_t access_get_byte(CPUS390XState *env, S390Access *access, 290 int offset, uintptr_t ra) 291 { 292 if (offset < access->size1) { 293 return do_access_get_byte(env, access->vaddr1, &access->haddr1, 294 offset, access->mmu_idx, ra); 295 } 296 return do_access_get_byte(env, access->vaddr2, &access->haddr2, 297 offset - access->size1, access->mmu_idx, ra); 298 } 299 300 static void do_access_set_byte(CPUS390XState *env, vaddr vaddr, char **haddr, 301 int offset, uint8_t byte, int mmu_idx, 302 uintptr_t ra) 303 { 304 #ifdef CONFIG_USER_ONLY 305 stb_p(*haddr + offset, byte); 306 #else 307 MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); 308 309 if (likely(*haddr)) { 310 stb_p(*haddr + offset, byte); 311 return; 312 } 313 /* 314 * Do a single access and test if we can then get access to the 315 * page. This is especially relevant to speed up TLB_NOTDIRTY. 316 */ 317 cpu_stb_mmu(env, vaddr + offset, byte, oi, ra); 318 *haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx); 319 #endif 320 } 321 322 static void access_set_byte(CPUS390XState *env, S390Access *access, 323 int offset, uint8_t byte, uintptr_t ra) 324 { 325 if (offset < access->size1) { 326 do_access_set_byte(env, access->vaddr1, &access->haddr1, offset, byte, 327 access->mmu_idx, ra); 328 } else { 329 do_access_set_byte(env, access->vaddr2, &access->haddr2, 330 offset - access->size1, byte, access->mmu_idx, ra); 331 } 332 } 333 334 /* 335 * Move data with the same semantics as memmove() in case ranges don't overlap 336 * or src > dest. Undefined behavior on destructive overlaps. 337 */ 338 static void access_memmove(CPUS390XState *env, S390Access *desta, 339 S390Access *srca, uintptr_t ra) 340 { 341 int diff; 342 343 g_assert(desta->size1 + desta->size2 == srca->size1 + srca->size2); 344 345 /* Fallback to slow access in case we don't have access to all host pages */ 346 if (unlikely(!desta->haddr1 || (desta->size2 && !desta->haddr2) || 347 !srca->haddr1 || (srca->size2 && !srca->haddr2))) { 348 int i; 349 350 for (i = 0; i < desta->size1 + desta->size2; i++) { 351 uint8_t byte = access_get_byte(env, srca, i, ra); 352 353 access_set_byte(env, desta, i, byte, ra); 354 } 355 return; 356 } 357 358 if (srca->size1 == desta->size1) { 359 memmove(desta->haddr1, srca->haddr1, srca->size1); 360 if (unlikely(srca->size2)) { 361 memmove(desta->haddr2, srca->haddr2, srca->size2); 362 } 363 } else if (srca->size1 < desta->size1) { 364 diff = desta->size1 - srca->size1; 365 memmove(desta->haddr1, srca->haddr1, srca->size1); 366 memmove(desta->haddr1 + srca->size1, srca->haddr2, diff); 367 if (likely(desta->size2)) { 368 memmove(desta->haddr2, srca->haddr2 + diff, desta->size2); 369 } 370 } else { 371 diff = srca->size1 - desta->size1; 372 memmove(desta->haddr1, srca->haddr1, desta->size1); 373 memmove(desta->haddr2, srca->haddr1 + desta->size1, diff); 374 if (likely(srca->size2)) { 375 memmove(desta->haddr2 + diff, srca->haddr2, srca->size2); 376 } 377 } 378 } 379 380 static int mmu_idx_from_as(uint8_t as) 381 { 382 switch (as) { 383 case AS_PRIMARY: 384 return MMU_PRIMARY_IDX; 385 case AS_SECONDARY: 386 return MMU_SECONDARY_IDX; 387 case AS_HOME: 388 return MMU_HOME_IDX; 389 default: 390 /* FIXME AS_ACCREG */ 391 g_assert_not_reached(); 392 } 393 } 394 395 /* and on array */ 396 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest, 397 uint64_t src, uintptr_t ra) 398 { 399 const int mmu_idx = cpu_mmu_index(env, false); 400 S390Access srca1, srca2, desta; 401 uint32_t i; 402 uint8_t c = 0; 403 404 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 405 __func__, l, dest, src); 406 407 /* NC always processes one more byte than specified - maximum is 256 */ 408 l++; 409 410 srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 411 srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 412 desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 413 for (i = 0; i < l; i++) { 414 const uint8_t x = access_get_byte(env, &srca1, i, ra) & 415 access_get_byte(env, &srca2, i, ra); 416 417 c |= x; 418 access_set_byte(env, &desta, i, x, ra); 419 } 420 return c != 0; 421 } 422 423 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest, 424 uint64_t src) 425 { 426 return do_helper_nc(env, l, dest, src, GETPC()); 427 } 428 429 /* xor on array */ 430 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest, 431 uint64_t src, uintptr_t ra) 432 { 433 const int mmu_idx = cpu_mmu_index(env, false); 434 S390Access srca1, srca2, desta; 435 uint32_t i; 436 uint8_t c = 0; 437 438 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 439 __func__, l, dest, src); 440 441 /* XC always processes one more byte than specified - maximum is 256 */ 442 l++; 443 444 srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 445 srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 446 desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 447 448 /* xor with itself is the same as memset(0) */ 449 if (src == dest) { 450 access_memset(env, &desta, 0, ra); 451 return 0; 452 } 453 454 for (i = 0; i < l; i++) { 455 const uint8_t x = access_get_byte(env, &srca1, i, ra) ^ 456 access_get_byte(env, &srca2, i, ra); 457 458 c |= x; 459 access_set_byte(env, &desta, i, x, ra); 460 } 461 return c != 0; 462 } 463 464 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest, 465 uint64_t src) 466 { 467 return do_helper_xc(env, l, dest, src, GETPC()); 468 } 469 470 /* or on array */ 471 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest, 472 uint64_t src, uintptr_t ra) 473 { 474 const int mmu_idx = cpu_mmu_index(env, false); 475 S390Access srca1, srca2, desta; 476 uint32_t i; 477 uint8_t c = 0; 478 479 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 480 __func__, l, dest, src); 481 482 /* OC always processes one more byte than specified - maximum is 256 */ 483 l++; 484 485 srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 486 srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 487 desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 488 for (i = 0; i < l; i++) { 489 const uint8_t x = access_get_byte(env, &srca1, i, ra) | 490 access_get_byte(env, &srca2, i, ra); 491 492 c |= x; 493 access_set_byte(env, &desta, i, x, ra); 494 } 495 return c != 0; 496 } 497 498 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest, 499 uint64_t src) 500 { 501 return do_helper_oc(env, l, dest, src, GETPC()); 502 } 503 504 /* memmove */ 505 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest, 506 uint64_t src, uintptr_t ra) 507 { 508 const int mmu_idx = cpu_mmu_index(env, false); 509 S390Access srca, desta; 510 uint32_t i; 511 512 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n", 513 __func__, l, dest, src); 514 515 /* MVC always copies one more byte than specified - maximum is 256 */ 516 l++; 517 518 srca = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 519 desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 520 521 /* 522 * "When the operands overlap, the result is obtained as if the operands 523 * were processed one byte at a time". Only non-destructive overlaps 524 * behave like memmove(). 525 */ 526 if (dest == src + 1) { 527 access_memset(env, &desta, access_get_byte(env, &srca, 0, ra), ra); 528 } else if (!is_destructive_overlap(env, dest, src, l)) { 529 access_memmove(env, &desta, &srca, ra); 530 } else { 531 for (i = 0; i < l; i++) { 532 uint8_t byte = access_get_byte(env, &srca, i, ra); 533 534 access_set_byte(env, &desta, i, byte, ra); 535 } 536 } 537 538 return env->cc_op; 539 } 540 541 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 542 { 543 do_helper_mvc(env, l, dest, src, GETPC()); 544 } 545 546 /* move right to left */ 547 void HELPER(mvcrl)(CPUS390XState *env, uint64_t l, uint64_t dest, uint64_t src) 548 { 549 const int mmu_idx = cpu_mmu_index(env, false); 550 const uint64_t ra = GETPC(); 551 S390Access srca, desta; 552 int32_t i; 553 554 /* MVCRL always copies one more byte than specified - maximum is 256 */ 555 l++; 556 557 srca = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 558 desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 559 560 for (i = l - 1; i >= 0; i--) { 561 uint8_t byte = access_get_byte(env, &srca, i, ra); 562 access_set_byte(env, &desta, i, byte, ra); 563 } 564 } 565 566 /* move inverse */ 567 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 568 { 569 const int mmu_idx = cpu_mmu_index(env, false); 570 S390Access srca, desta; 571 uintptr_t ra = GETPC(); 572 int i; 573 574 /* MVCIN always copies one more byte than specified - maximum is 256 */ 575 l++; 576 577 src = wrap_address(env, src - l + 1); 578 srca = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 579 desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 580 for (i = 0; i < l; i++) { 581 const uint8_t x = access_get_byte(env, &srca, l - i - 1, ra); 582 583 access_set_byte(env, &desta, i, x, ra); 584 } 585 } 586 587 /* move numerics */ 588 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 589 { 590 const int mmu_idx = cpu_mmu_index(env, false); 591 S390Access srca1, srca2, desta; 592 uintptr_t ra = GETPC(); 593 int i; 594 595 /* MVN always copies one more byte than specified - maximum is 256 */ 596 l++; 597 598 srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 599 srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 600 desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 601 for (i = 0; i < l; i++) { 602 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0x0f) | 603 (access_get_byte(env, &srca2, i, ra) & 0xf0); 604 605 access_set_byte(env, &desta, i, x, ra); 606 } 607 } 608 609 /* move with offset */ 610 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 611 { 612 const int mmu_idx = cpu_mmu_index(env, false); 613 /* MVO always processes one more byte than specified - maximum is 16 */ 614 const int len_dest = (l >> 4) + 1; 615 const int len_src = (l & 0xf) + 1; 616 uintptr_t ra = GETPC(); 617 uint8_t byte_dest, byte_src; 618 S390Access srca, desta; 619 int i, j; 620 621 srca = access_prepare(env, src, len_src, MMU_DATA_LOAD, mmu_idx, ra); 622 desta = access_prepare(env, dest, len_dest, MMU_DATA_STORE, mmu_idx, ra); 623 624 /* Handle rightmost byte */ 625 byte_dest = cpu_ldub_data_ra(env, dest + len_dest - 1, ra); 626 byte_src = access_get_byte(env, &srca, len_src - 1, ra); 627 byte_dest = (byte_dest & 0x0f) | (byte_src << 4); 628 access_set_byte(env, &desta, len_dest - 1, byte_dest, ra); 629 630 /* Process remaining bytes from right to left */ 631 for (i = len_dest - 2, j = len_src - 2; i >= 0; i--, j--) { 632 byte_dest = byte_src >> 4; 633 if (j >= 0) { 634 byte_src = access_get_byte(env, &srca, j, ra); 635 } else { 636 byte_src = 0; 637 } 638 byte_dest |= byte_src << 4; 639 access_set_byte(env, &desta, i, byte_dest, ra); 640 } 641 } 642 643 /* move zones */ 644 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src) 645 { 646 const int mmu_idx = cpu_mmu_index(env, false); 647 S390Access srca1, srca2, desta; 648 uintptr_t ra = GETPC(); 649 int i; 650 651 /* MVZ always copies one more byte than specified - maximum is 256 */ 652 l++; 653 654 srca1 = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra); 655 srca2 = access_prepare(env, dest, l, MMU_DATA_LOAD, mmu_idx, ra); 656 desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra); 657 for (i = 0; i < l; i++) { 658 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0xf0) | 659 (access_get_byte(env, &srca2, i, ra) & 0x0f); 660 661 access_set_byte(env, &desta, i, x, ra); 662 } 663 } 664 665 /* compare unsigned byte arrays */ 666 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1, 667 uint64_t s2, uintptr_t ra) 668 { 669 uint32_t i; 670 uint32_t cc = 0; 671 672 HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n", 673 __func__, l, s1, s2); 674 675 for (i = 0; i <= l; i++) { 676 uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra); 677 uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra); 678 HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y); 679 if (x < y) { 680 cc = 1; 681 break; 682 } else if (x > y) { 683 cc = 2; 684 break; 685 } 686 } 687 688 HELPER_LOG("\n"); 689 return cc; 690 } 691 692 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2) 693 { 694 return do_helper_clc(env, l, s1, s2, GETPC()); 695 } 696 697 /* compare logical under mask */ 698 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask, 699 uint64_t addr) 700 { 701 uintptr_t ra = GETPC(); 702 uint32_t cc = 0; 703 704 HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1, 705 mask, addr); 706 707 while (mask) { 708 if (mask & 8) { 709 uint8_t d = cpu_ldub_data_ra(env, addr, ra); 710 uint8_t r = extract32(r1, 24, 8); 711 HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d, 712 addr); 713 if (r < d) { 714 cc = 1; 715 break; 716 } else if (r > d) { 717 cc = 2; 718 break; 719 } 720 addr++; 721 } 722 mask = (mask << 1) & 0xf; 723 r1 <<= 8; 724 } 725 726 HELPER_LOG("\n"); 727 return cc; 728 } 729 730 static inline uint64_t get_address(CPUS390XState *env, int reg) 731 { 732 return wrap_address(env, env->regs[reg]); 733 } 734 735 /* 736 * Store the address to the given register, zeroing out unused leftmost 737 * bits in bit positions 32-63 (24-bit and 31-bit mode only). 738 */ 739 static inline void set_address_zero(CPUS390XState *env, int reg, 740 uint64_t address) 741 { 742 if (env->psw.mask & PSW_MASK_64) { 743 env->regs[reg] = address; 744 } else { 745 if (!(env->psw.mask & PSW_MASK_32)) { 746 address &= 0x00ffffff; 747 } else { 748 address &= 0x7fffffff; 749 } 750 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address); 751 } 752 } 753 754 static inline void set_address(CPUS390XState *env, int reg, uint64_t address) 755 { 756 if (env->psw.mask & PSW_MASK_64) { 757 /* 64-Bit mode */ 758 env->regs[reg] = address; 759 } else { 760 if (!(env->psw.mask & PSW_MASK_32)) { 761 /* 24-Bit mode. According to the PoO it is implementation 762 dependent if bits 32-39 remain unchanged or are set to 763 zeros. Choose the former so that the function can also be 764 used for TRT. */ 765 env->regs[reg] = deposit64(env->regs[reg], 0, 24, address); 766 } else { 767 /* 31-Bit mode. According to the PoO it is implementation 768 dependent if bit 32 remains unchanged or is set to zero. 769 Choose the latter so that the function can also be used for 770 TRT. */ 771 address &= 0x7fffffff; 772 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address); 773 } 774 } 775 } 776 777 static inline uint64_t wrap_length32(CPUS390XState *env, uint64_t length) 778 { 779 if (!(env->psw.mask & PSW_MASK_64)) { 780 return (uint32_t)length; 781 } 782 return length; 783 } 784 785 static inline uint64_t wrap_length31(CPUS390XState *env, uint64_t length) 786 { 787 if (!(env->psw.mask & PSW_MASK_64)) { 788 /* 24-Bit and 31-Bit mode */ 789 length &= 0x7fffffff; 790 } 791 return length; 792 } 793 794 static inline uint64_t get_length(CPUS390XState *env, int reg) 795 { 796 return wrap_length31(env, env->regs[reg]); 797 } 798 799 static inline void set_length(CPUS390XState *env, int reg, uint64_t length) 800 { 801 if (env->psw.mask & PSW_MASK_64) { 802 /* 64-Bit mode */ 803 env->regs[reg] = length; 804 } else { 805 /* 24-Bit and 31-Bit mode */ 806 env->regs[reg] = deposit64(env->regs[reg], 0, 32, length); 807 } 808 } 809 810 /* search string (c is byte to search, r2 is string, r1 end of string) */ 811 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2) 812 { 813 uintptr_t ra = GETPC(); 814 uint64_t end, str; 815 uint32_t len; 816 uint8_t v, c = env->regs[0]; 817 818 /* Bits 32-55 must contain all 0. */ 819 if (env->regs[0] & 0xffffff00u) { 820 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 821 } 822 823 str = get_address(env, r2); 824 end = get_address(env, r1); 825 826 /* Lest we fail to service interrupts in a timely manner, limit the 827 amount of work we're willing to do. For now, let's cap at 8k. */ 828 for (len = 0; len < 0x2000; ++len) { 829 if (str + len == end) { 830 /* Character not found. R1 & R2 are unmodified. */ 831 env->cc_op = 2; 832 return; 833 } 834 v = cpu_ldub_data_ra(env, str + len, ra); 835 if (v == c) { 836 /* Character found. Set R1 to the location; R2 is unmodified. */ 837 env->cc_op = 1; 838 set_address(env, r1, str + len); 839 return; 840 } 841 } 842 843 /* CPU-determined bytes processed. Advance R2 to next byte to process. */ 844 env->cc_op = 3; 845 set_address(env, r2, str + len); 846 } 847 848 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2) 849 { 850 uintptr_t ra = GETPC(); 851 uint32_t len; 852 uint16_t v, c = env->regs[0]; 853 uint64_t end, str, adj_end; 854 855 /* Bits 32-47 of R0 must be zero. */ 856 if (env->regs[0] & 0xffff0000u) { 857 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 858 } 859 860 str = get_address(env, r2); 861 end = get_address(env, r1); 862 863 /* If the LSB of the two addresses differ, use one extra byte. */ 864 adj_end = end + ((str ^ end) & 1); 865 866 /* Lest we fail to service interrupts in a timely manner, limit the 867 amount of work we're willing to do. For now, let's cap at 8k. */ 868 for (len = 0; len < 0x2000; len += 2) { 869 if (str + len == adj_end) { 870 /* End of input found. */ 871 env->cc_op = 2; 872 return; 873 } 874 v = cpu_lduw_data_ra(env, str + len, ra); 875 if (v == c) { 876 /* Character found. Set R1 to the location; R2 is unmodified. */ 877 env->cc_op = 1; 878 set_address(env, r1, str + len); 879 return; 880 } 881 } 882 883 /* CPU-determined bytes processed. Advance R2 to next byte to process. */ 884 env->cc_op = 3; 885 set_address(env, r2, str + len); 886 } 887 888 /* unsigned string compare (c is string terminator) */ 889 Int128 HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2) 890 { 891 uintptr_t ra = GETPC(); 892 uint32_t len; 893 894 c = c & 0xff; 895 s1 = wrap_address(env, s1); 896 s2 = wrap_address(env, s2); 897 898 /* Lest we fail to service interrupts in a timely manner, limit the 899 amount of work we're willing to do. For now, let's cap at 8k. */ 900 for (len = 0; len < 0x2000; ++len) { 901 uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra); 902 uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra); 903 if (v1 == v2) { 904 if (v1 == c) { 905 /* Equal. CC=0, and don't advance the registers. */ 906 env->cc_op = 0; 907 return int128_make128(s2, s1); 908 } 909 } else { 910 /* Unequal. CC={1,2}, and advance the registers. Note that 911 the terminator need not be zero, but the string that contains 912 the terminator is by definition "low". */ 913 env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2); 914 return int128_make128(s2 + len, s1 + len); 915 } 916 } 917 918 /* CPU-determined bytes equal; advance the registers. */ 919 env->cc_op = 3; 920 return int128_make128(s2 + len, s1 + len); 921 } 922 923 /* move page */ 924 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint32_t r1, uint32_t r2) 925 { 926 const uint64_t src = get_address(env, r2) & TARGET_PAGE_MASK; 927 const uint64_t dst = get_address(env, r1) & TARGET_PAGE_MASK; 928 const int mmu_idx = cpu_mmu_index(env, false); 929 const bool f = extract64(r0, 11, 1); 930 const bool s = extract64(r0, 10, 1); 931 const bool cco = extract64(r0, 8, 1); 932 uintptr_t ra = GETPC(); 933 S390Access srca, desta; 934 int exc; 935 936 if ((f && s) || extract64(r0, 12, 4)) { 937 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC()); 938 } 939 940 /* 941 * We always manually handle exceptions such that we can properly store 942 * r1/r2 to the lowcore on page-translation exceptions. 943 * 944 * TODO: Access key handling 945 */ 946 exc = access_prepare_nf(&srca, env, true, src, TARGET_PAGE_SIZE, 947 MMU_DATA_LOAD, mmu_idx, ra); 948 if (exc) { 949 if (cco) { 950 return 2; 951 } 952 goto inject_exc; 953 } 954 exc = access_prepare_nf(&desta, env, true, dst, TARGET_PAGE_SIZE, 955 MMU_DATA_STORE, mmu_idx, ra); 956 if (exc) { 957 if (cco && exc != PGM_PROTECTION) { 958 return 1; 959 } 960 goto inject_exc; 961 } 962 access_memmove(env, &desta, &srca, ra); 963 return 0; /* data moved */ 964 inject_exc: 965 #if !defined(CONFIG_USER_ONLY) 966 if (exc != PGM_ADDRESSING) { 967 stq_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, trans_exc_code), 968 env->tlb_fill_tec); 969 } 970 if (exc == PGM_PAGE_TRANS) { 971 stb_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, op_access_id), 972 r1 << 4 | r2); 973 } 974 #endif 975 tcg_s390_program_interrupt(env, exc, ra); 976 } 977 978 /* string copy */ 979 uint32_t HELPER(mvst)(CPUS390XState *env, uint32_t r1, uint32_t r2) 980 { 981 const int mmu_idx = cpu_mmu_index(env, false); 982 const uint64_t d = get_address(env, r1); 983 const uint64_t s = get_address(env, r2); 984 const uint8_t c = env->regs[0]; 985 const int len = MIN(-(d | TARGET_PAGE_MASK), -(s | TARGET_PAGE_MASK)); 986 S390Access srca, desta; 987 uintptr_t ra = GETPC(); 988 int i; 989 990 if (env->regs[0] & 0xffffff00ull) { 991 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 992 } 993 994 /* 995 * Our access should not exceed single pages, as we must not report access 996 * exceptions exceeding the actually copied range (which we don't know at 997 * this point). We might over-indicate watchpoints within the pages 998 * (if we ever care, we have to limit processing to a single byte). 999 */ 1000 srca = access_prepare(env, s, len, MMU_DATA_LOAD, mmu_idx, ra); 1001 desta = access_prepare(env, d, len, MMU_DATA_STORE, mmu_idx, ra); 1002 for (i = 0; i < len; i++) { 1003 const uint8_t v = access_get_byte(env, &srca, i, ra); 1004 1005 access_set_byte(env, &desta, i, v, ra); 1006 if (v == c) { 1007 set_address_zero(env, r1, d + i); 1008 return 1; 1009 } 1010 } 1011 set_address_zero(env, r1, d + len); 1012 set_address_zero(env, r2, s + len); 1013 return 3; 1014 } 1015 1016 /* load access registers r1 to r3 from memory at a2 */ 1017 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1018 { 1019 uintptr_t ra = GETPC(); 1020 int i; 1021 1022 if (a2 & 0x3) { 1023 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1024 } 1025 1026 for (i = r1;; i = (i + 1) % 16) { 1027 env->aregs[i] = cpu_ldl_data_ra(env, a2, ra); 1028 a2 += 4; 1029 1030 if (i == r3) { 1031 break; 1032 } 1033 } 1034 } 1035 1036 /* store access registers r1 to r3 in memory at a2 */ 1037 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 1038 { 1039 uintptr_t ra = GETPC(); 1040 int i; 1041 1042 if (a2 & 0x3) { 1043 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 1044 } 1045 1046 for (i = r1;; i = (i + 1) % 16) { 1047 cpu_stl_data_ra(env, a2, env->aregs[i], ra); 1048 a2 += 4; 1049 1050 if (i == r3) { 1051 break; 1052 } 1053 } 1054 } 1055 1056 /* move long helper */ 1057 static inline uint32_t do_mvcl(CPUS390XState *env, 1058 uint64_t *dest, uint64_t *destlen, 1059 uint64_t *src, uint64_t *srclen, 1060 uint16_t pad, int wordsize, uintptr_t ra) 1061 { 1062 const int mmu_idx = cpu_mmu_index(env, false); 1063 int len = MIN(*destlen, -(*dest | TARGET_PAGE_MASK)); 1064 S390Access srca, desta; 1065 int i, cc; 1066 1067 if (*destlen == *srclen) { 1068 cc = 0; 1069 } else if (*destlen < *srclen) { 1070 cc = 1; 1071 } else { 1072 cc = 2; 1073 } 1074 1075 if (!*destlen) { 1076 return cc; 1077 } 1078 1079 /* 1080 * Only perform one type of type of operation (move/pad) at a time. 1081 * Stay within single pages. 1082 */ 1083 if (*srclen) { 1084 /* Copy the src array */ 1085 len = MIN(MIN(*srclen, -(*src | TARGET_PAGE_MASK)), len); 1086 *destlen -= len; 1087 *srclen -= len; 1088 srca = access_prepare(env, *src, len, MMU_DATA_LOAD, mmu_idx, ra); 1089 desta = access_prepare(env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1090 access_memmove(env, &desta, &srca, ra); 1091 *src = wrap_address(env, *src + len); 1092 *dest = wrap_address(env, *dest + len); 1093 } else if (wordsize == 1) { 1094 /* Pad the remaining area */ 1095 *destlen -= len; 1096 desta = access_prepare(env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1097 access_memset(env, &desta, pad, ra); 1098 *dest = wrap_address(env, *dest + len); 1099 } else { 1100 desta = access_prepare(env, *dest, len, MMU_DATA_STORE, mmu_idx, ra); 1101 1102 /* The remaining length selects the padding byte. */ 1103 for (i = 0; i < len; (*destlen)--, i++) { 1104 if (*destlen & 1) { 1105 access_set_byte(env, &desta, i, pad, ra); 1106 } else { 1107 access_set_byte(env, &desta, i, pad >> 8, ra); 1108 } 1109 } 1110 *dest = wrap_address(env, *dest + len); 1111 } 1112 1113 return *destlen ? 3 : cc; 1114 } 1115 1116 /* move long */ 1117 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2) 1118 { 1119 const int mmu_idx = cpu_mmu_index(env, false); 1120 uintptr_t ra = GETPC(); 1121 uint64_t destlen = env->regs[r1 + 1] & 0xffffff; 1122 uint64_t dest = get_address(env, r1); 1123 uint64_t srclen = env->regs[r2 + 1] & 0xffffff; 1124 uint64_t src = get_address(env, r2); 1125 uint8_t pad = env->regs[r2 + 1] >> 24; 1126 CPUState *cs = env_cpu(env); 1127 S390Access srca, desta; 1128 uint32_t cc, cur_len; 1129 1130 if (is_destructive_overlap(env, dest, src, MIN(srclen, destlen))) { 1131 cc = 3; 1132 } else if (srclen == destlen) { 1133 cc = 0; 1134 } else if (destlen < srclen) { 1135 cc = 1; 1136 } else { 1137 cc = 2; 1138 } 1139 1140 /* We might have to zero-out some bits even if there was no action. */ 1141 if (unlikely(!destlen || cc == 3)) { 1142 set_address_zero(env, r2, src); 1143 set_address_zero(env, r1, dest); 1144 return cc; 1145 } else if (!srclen) { 1146 set_address_zero(env, r2, src); 1147 } 1148 1149 /* 1150 * Only perform one type of type of operation (move/pad) in one step. 1151 * Stay within single pages. 1152 */ 1153 while (destlen) { 1154 cur_len = MIN(destlen, -(dest | TARGET_PAGE_MASK)); 1155 if (!srclen) { 1156 desta = access_prepare(env, dest, cur_len, MMU_DATA_STORE, mmu_idx, 1157 ra); 1158 access_memset(env, &desta, pad, ra); 1159 } else { 1160 cur_len = MIN(MIN(srclen, -(src | TARGET_PAGE_MASK)), cur_len); 1161 1162 srca = access_prepare(env, src, cur_len, MMU_DATA_LOAD, mmu_idx, 1163 ra); 1164 desta = access_prepare(env, dest, cur_len, MMU_DATA_STORE, mmu_idx, 1165 ra); 1166 access_memmove(env, &desta, &srca, ra); 1167 src = wrap_address(env, src + cur_len); 1168 srclen -= cur_len; 1169 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen); 1170 set_address_zero(env, r2, src); 1171 } 1172 dest = wrap_address(env, dest + cur_len); 1173 destlen -= cur_len; 1174 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen); 1175 set_address_zero(env, r1, dest); 1176 1177 /* 1178 * MVCL is interruptible. Return to the main loop if requested after 1179 * writing back all state to registers. If no interrupt will get 1180 * injected, we'll end up back in this handler and continue processing 1181 * the remaining parts. 1182 */ 1183 if (destlen && unlikely(cpu_loop_exit_requested(cs))) { 1184 cpu_loop_exit_restore(cs, ra); 1185 } 1186 } 1187 return cc; 1188 } 1189 1190 /* move long extended */ 1191 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1192 uint32_t r3) 1193 { 1194 uintptr_t ra = GETPC(); 1195 uint64_t destlen = get_length(env, r1 + 1); 1196 uint64_t dest = get_address(env, r1); 1197 uint64_t srclen = get_length(env, r3 + 1); 1198 uint64_t src = get_address(env, r3); 1199 uint8_t pad = a2; 1200 uint32_t cc; 1201 1202 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra); 1203 1204 set_length(env, r1 + 1, destlen); 1205 set_length(env, r3 + 1, srclen); 1206 set_address(env, r1, dest); 1207 set_address(env, r3, src); 1208 1209 return cc; 1210 } 1211 1212 /* move long unicode */ 1213 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1214 uint32_t r3) 1215 { 1216 uintptr_t ra = GETPC(); 1217 uint64_t destlen = get_length(env, r1 + 1); 1218 uint64_t dest = get_address(env, r1); 1219 uint64_t srclen = get_length(env, r3 + 1); 1220 uint64_t src = get_address(env, r3); 1221 uint16_t pad = a2; 1222 uint32_t cc; 1223 1224 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra); 1225 1226 set_length(env, r1 + 1, destlen); 1227 set_length(env, r3 + 1, srclen); 1228 set_address(env, r1, dest); 1229 set_address(env, r3, src); 1230 1231 return cc; 1232 } 1233 1234 /* compare logical long helper */ 1235 static inline uint32_t do_clcl(CPUS390XState *env, 1236 uint64_t *src1, uint64_t *src1len, 1237 uint64_t *src3, uint64_t *src3len, 1238 uint16_t pad, uint64_t limit, 1239 int wordsize, uintptr_t ra) 1240 { 1241 uint64_t len = MAX(*src1len, *src3len); 1242 uint32_t cc = 0; 1243 1244 check_alignment(env, *src1len | *src3len, wordsize, ra); 1245 1246 if (!len) { 1247 return cc; 1248 } 1249 1250 /* Lest we fail to service interrupts in a timely manner, limit the 1251 amount of work we're willing to do. */ 1252 if (len > limit) { 1253 len = limit; 1254 cc = 3; 1255 } 1256 1257 for (; len; len -= wordsize) { 1258 uint16_t v1 = pad; 1259 uint16_t v3 = pad; 1260 1261 if (*src1len) { 1262 v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra); 1263 } 1264 if (*src3len) { 1265 v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra); 1266 } 1267 1268 if (v1 != v3) { 1269 cc = (v1 < v3) ? 1 : 2; 1270 break; 1271 } 1272 1273 if (*src1len) { 1274 *src1 += wordsize; 1275 *src1len -= wordsize; 1276 } 1277 if (*src3len) { 1278 *src3 += wordsize; 1279 *src3len -= wordsize; 1280 } 1281 } 1282 1283 return cc; 1284 } 1285 1286 1287 /* compare logical long */ 1288 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2) 1289 { 1290 uintptr_t ra = GETPC(); 1291 uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24); 1292 uint64_t src1 = get_address(env, r1); 1293 uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24); 1294 uint64_t src3 = get_address(env, r2); 1295 uint8_t pad = env->regs[r2 + 1] >> 24; 1296 uint32_t cc; 1297 1298 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra); 1299 1300 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len); 1301 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len); 1302 set_address(env, r1, src1); 1303 set_address(env, r2, src3); 1304 1305 return cc; 1306 } 1307 1308 /* compare logical long extended memcompare insn with padding */ 1309 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1310 uint32_t r3) 1311 { 1312 uintptr_t ra = GETPC(); 1313 uint64_t src1len = get_length(env, r1 + 1); 1314 uint64_t src1 = get_address(env, r1); 1315 uint64_t src3len = get_length(env, r3 + 1); 1316 uint64_t src3 = get_address(env, r3); 1317 uint8_t pad = a2; 1318 uint32_t cc; 1319 1320 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra); 1321 1322 set_length(env, r1 + 1, src1len); 1323 set_length(env, r3 + 1, src3len); 1324 set_address(env, r1, src1); 1325 set_address(env, r3, src3); 1326 1327 return cc; 1328 } 1329 1330 /* compare logical long unicode memcompare insn with padding */ 1331 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2, 1332 uint32_t r3) 1333 { 1334 uintptr_t ra = GETPC(); 1335 uint64_t src1len = get_length(env, r1 + 1); 1336 uint64_t src1 = get_address(env, r1); 1337 uint64_t src3len = get_length(env, r3 + 1); 1338 uint64_t src3 = get_address(env, r3); 1339 uint16_t pad = a2; 1340 uint32_t cc = 0; 1341 1342 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra); 1343 1344 set_length(env, r1 + 1, src1len); 1345 set_length(env, r3 + 1, src3len); 1346 set_address(env, r1, src1); 1347 set_address(env, r3, src3); 1348 1349 return cc; 1350 } 1351 1352 /* checksum */ 1353 Int128 HELPER(cksm)(CPUS390XState *env, uint64_t r1, 1354 uint64_t src, uint64_t src_len) 1355 { 1356 uintptr_t ra = GETPC(); 1357 uint64_t max_len, len; 1358 uint64_t cksm = (uint32_t)r1; 1359 1360 /* Lest we fail to service interrupts in a timely manner, limit the 1361 amount of work we're willing to do. For now, let's cap at 8k. */ 1362 max_len = (src_len > 0x2000 ? 0x2000 : src_len); 1363 1364 /* Process full words as available. */ 1365 for (len = 0; len + 4 <= max_len; len += 4, src += 4) { 1366 cksm += (uint32_t)cpu_ldl_data_ra(env, src, ra); 1367 } 1368 1369 switch (max_len - len) { 1370 case 1: 1371 cksm += cpu_ldub_data_ra(env, src, ra) << 24; 1372 len += 1; 1373 break; 1374 case 2: 1375 cksm += cpu_lduw_data_ra(env, src, ra) << 16; 1376 len += 2; 1377 break; 1378 case 3: 1379 cksm += cpu_lduw_data_ra(env, src, ra) << 16; 1380 cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8; 1381 len += 3; 1382 break; 1383 } 1384 1385 /* Fold the carry from the checksum. Note that we can see carry-out 1386 during folding more than once (but probably not more than twice). */ 1387 while (cksm > 0xffffffffull) { 1388 cksm = (uint32_t)cksm + (cksm >> 32); 1389 } 1390 1391 /* Indicate whether or not we've processed everything. */ 1392 env->cc_op = (len == src_len ? 0 : 3); 1393 1394 /* Return both cksm and processed length. */ 1395 return int128_make128(cksm, len); 1396 } 1397 1398 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src) 1399 { 1400 uintptr_t ra = GETPC(); 1401 int len_dest = len >> 4; 1402 int len_src = len & 0xf; 1403 uint8_t b; 1404 1405 dest += len_dest; 1406 src += len_src; 1407 1408 /* last byte is special, it only flips the nibbles */ 1409 b = cpu_ldub_data_ra(env, src, ra); 1410 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra); 1411 src--; 1412 len_src--; 1413 1414 /* now pack every value */ 1415 while (len_dest > 0) { 1416 b = 0; 1417 1418 if (len_src >= 0) { 1419 b = cpu_ldub_data_ra(env, src, ra) & 0x0f; 1420 src--; 1421 len_src--; 1422 } 1423 if (len_src >= 0) { 1424 b |= cpu_ldub_data_ra(env, src, ra) << 4; 1425 src--; 1426 len_src--; 1427 } 1428 1429 len_dest--; 1430 dest--; 1431 cpu_stb_data_ra(env, dest, b, ra); 1432 } 1433 } 1434 1435 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src, 1436 uint32_t srclen, int ssize, uintptr_t ra) 1437 { 1438 int i; 1439 /* The destination operand is always 16 bytes long. */ 1440 const int destlen = 16; 1441 1442 /* The operands are processed from right to left. */ 1443 src += srclen - 1; 1444 dest += destlen - 1; 1445 1446 for (i = 0; i < destlen; i++) { 1447 uint8_t b = 0; 1448 1449 /* Start with a positive sign */ 1450 if (i == 0) { 1451 b = 0xc; 1452 } else if (srclen > ssize) { 1453 b = cpu_ldub_data_ra(env, src, ra) & 0x0f; 1454 src -= ssize; 1455 srclen -= ssize; 1456 } 1457 1458 if (srclen > ssize) { 1459 b |= cpu_ldub_data_ra(env, src, ra) << 4; 1460 src -= ssize; 1461 srclen -= ssize; 1462 } 1463 1464 cpu_stb_data_ra(env, dest, b, ra); 1465 dest--; 1466 } 1467 } 1468 1469 1470 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src, 1471 uint32_t srclen) 1472 { 1473 do_pkau(env, dest, src, srclen, 1, GETPC()); 1474 } 1475 1476 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src, 1477 uint32_t srclen) 1478 { 1479 do_pkau(env, dest, src, srclen, 2, GETPC()); 1480 } 1481 1482 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest, 1483 uint64_t src) 1484 { 1485 uintptr_t ra = GETPC(); 1486 int len_dest = len >> 4; 1487 int len_src = len & 0xf; 1488 uint8_t b; 1489 int second_nibble = 0; 1490 1491 dest += len_dest; 1492 src += len_src; 1493 1494 /* last byte is special, it only flips the nibbles */ 1495 b = cpu_ldub_data_ra(env, src, ra); 1496 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra); 1497 src--; 1498 len_src--; 1499 1500 /* now pad every nibble with 0xf0 */ 1501 1502 while (len_dest > 0) { 1503 uint8_t cur_byte = 0; 1504 1505 if (len_src > 0) { 1506 cur_byte = cpu_ldub_data_ra(env, src, ra); 1507 } 1508 1509 len_dest--; 1510 dest--; 1511 1512 /* only advance one nibble at a time */ 1513 if (second_nibble) { 1514 cur_byte >>= 4; 1515 len_src--; 1516 src--; 1517 } 1518 second_nibble = !second_nibble; 1519 1520 /* digit */ 1521 cur_byte = (cur_byte & 0xf); 1522 /* zone bits */ 1523 cur_byte |= 0xf0; 1524 1525 cpu_stb_data_ra(env, dest, cur_byte, ra); 1526 } 1527 } 1528 1529 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest, 1530 uint32_t destlen, int dsize, uint64_t src, 1531 uintptr_t ra) 1532 { 1533 int i; 1534 uint32_t cc; 1535 uint8_t b; 1536 /* The source operand is always 16 bytes long. */ 1537 const int srclen = 16; 1538 1539 /* The operands are processed from right to left. */ 1540 src += srclen - 1; 1541 dest += destlen - dsize; 1542 1543 /* Check for the sign. */ 1544 b = cpu_ldub_data_ra(env, src, ra); 1545 src--; 1546 switch (b & 0xf) { 1547 case 0xa: 1548 case 0xc: 1549 case 0xe ... 0xf: 1550 cc = 0; /* plus */ 1551 break; 1552 case 0xb: 1553 case 0xd: 1554 cc = 1; /* minus */ 1555 break; 1556 default: 1557 case 0x0 ... 0x9: 1558 cc = 3; /* invalid */ 1559 break; 1560 } 1561 1562 /* Now pad every nibble with 0x30, advancing one nibble at a time. */ 1563 for (i = 0; i < destlen; i += dsize) { 1564 if (i == (31 * dsize)) { 1565 /* If length is 32/64 bytes, the leftmost byte is 0. */ 1566 b = 0; 1567 } else if (i % (2 * dsize)) { 1568 b = cpu_ldub_data_ra(env, src, ra); 1569 src--; 1570 } else { 1571 b >>= 4; 1572 } 1573 cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra); 1574 dest -= dsize; 1575 } 1576 1577 return cc; 1578 } 1579 1580 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen, 1581 uint64_t src) 1582 { 1583 return do_unpkau(env, dest, destlen, 1, src, GETPC()); 1584 } 1585 1586 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen, 1587 uint64_t src) 1588 { 1589 return do_unpkau(env, dest, destlen, 2, src, GETPC()); 1590 } 1591 1592 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen) 1593 { 1594 uintptr_t ra = GETPC(); 1595 uint32_t cc = 0; 1596 int i; 1597 1598 for (i = 0; i < destlen; i++) { 1599 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra); 1600 /* digit */ 1601 cc |= (b & 0xf0) > 0x90 ? 2 : 0; 1602 1603 if (i == (destlen - 1)) { 1604 /* sign */ 1605 cc |= (b & 0xf) < 0xa ? 1 : 0; 1606 } else { 1607 /* digit */ 1608 cc |= (b & 0xf) > 0x9 ? 2 : 0; 1609 } 1610 } 1611 1612 return cc; 1613 } 1614 1615 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array, 1616 uint64_t trans, uintptr_t ra) 1617 { 1618 uint32_t i; 1619 1620 for (i = 0; i <= len; i++) { 1621 uint8_t byte = cpu_ldub_data_ra(env, array + i, ra); 1622 uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra); 1623 cpu_stb_data_ra(env, array + i, new_byte, ra); 1624 } 1625 1626 return env->cc_op; 1627 } 1628 1629 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array, 1630 uint64_t trans) 1631 { 1632 do_helper_tr(env, len, array, trans, GETPC()); 1633 } 1634 1635 Int128 HELPER(tre)(CPUS390XState *env, uint64_t array, 1636 uint64_t len, uint64_t trans) 1637 { 1638 uintptr_t ra = GETPC(); 1639 uint8_t end = env->regs[0] & 0xff; 1640 uint64_t l = len; 1641 uint64_t i; 1642 uint32_t cc = 0; 1643 1644 if (!(env->psw.mask & PSW_MASK_64)) { 1645 array &= 0x7fffffff; 1646 l = (uint32_t)l; 1647 } 1648 1649 /* Lest we fail to service interrupts in a timely manner, limit the 1650 amount of work we're willing to do. For now, let's cap at 8k. */ 1651 if (l > 0x2000) { 1652 l = 0x2000; 1653 cc = 3; 1654 } 1655 1656 for (i = 0; i < l; i++) { 1657 uint8_t byte, new_byte; 1658 1659 byte = cpu_ldub_data_ra(env, array + i, ra); 1660 1661 if (byte == end) { 1662 cc = 1; 1663 break; 1664 } 1665 1666 new_byte = cpu_ldub_data_ra(env, trans + byte, ra); 1667 cpu_stb_data_ra(env, array + i, new_byte, ra); 1668 } 1669 1670 env->cc_op = cc; 1671 return int128_make128(len - i, array + i); 1672 } 1673 1674 static inline uint32_t do_helper_trt(CPUS390XState *env, int len, 1675 uint64_t array, uint64_t trans, 1676 int inc, uintptr_t ra) 1677 { 1678 int i; 1679 1680 for (i = 0; i <= len; i++) { 1681 uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra); 1682 uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra); 1683 1684 if (sbyte != 0) { 1685 set_address(env, 1, array + i * inc); 1686 env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte); 1687 return (i == len) ? 2 : 1; 1688 } 1689 } 1690 1691 return 0; 1692 } 1693 1694 static uint32_t do_helper_trt_fwd(CPUS390XState *env, uint32_t len, 1695 uint64_t array, uint64_t trans, 1696 uintptr_t ra) 1697 { 1698 return do_helper_trt(env, len, array, trans, 1, ra); 1699 } 1700 1701 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array, 1702 uint64_t trans) 1703 { 1704 return do_helper_trt(env, len, array, trans, 1, GETPC()); 1705 } 1706 1707 static uint32_t do_helper_trt_bkwd(CPUS390XState *env, uint32_t len, 1708 uint64_t array, uint64_t trans, 1709 uintptr_t ra) 1710 { 1711 return do_helper_trt(env, len, array, trans, -1, ra); 1712 } 1713 1714 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array, 1715 uint64_t trans) 1716 { 1717 return do_helper_trt(env, len, array, trans, -1, GETPC()); 1718 } 1719 1720 /* Translate one/two to one/two */ 1721 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2, 1722 uint32_t tst, uint32_t sizes) 1723 { 1724 uintptr_t ra = GETPC(); 1725 int dsize = (sizes & 1) ? 1 : 2; 1726 int ssize = (sizes & 2) ? 1 : 2; 1727 uint64_t tbl = get_address(env, 1); 1728 uint64_t dst = get_address(env, r1); 1729 uint64_t len = get_length(env, r1 + 1); 1730 uint64_t src = get_address(env, r2); 1731 uint32_t cc = 3; 1732 int i; 1733 1734 /* The lower address bits of TBL are ignored. For TROO, TROT, it's 1735 the low 3 bits (double-word aligned). For TRTO, TRTT, it's either 1736 the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */ 1737 if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) { 1738 tbl &= -4096; 1739 } else { 1740 tbl &= -8; 1741 } 1742 1743 check_alignment(env, len, ssize, ra); 1744 1745 /* Lest we fail to service interrupts in a timely manner, */ 1746 /* limit the amount of work we're willing to do. */ 1747 for (i = 0; i < 0x2000; i++) { 1748 uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra); 1749 uint64_t tble = tbl + (sval * dsize); 1750 uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra); 1751 if (dval == tst) { 1752 cc = 1; 1753 break; 1754 } 1755 cpu_stsize_data_ra(env, dst, dval, dsize, ra); 1756 1757 len -= ssize; 1758 src += ssize; 1759 dst += dsize; 1760 1761 if (len == 0) { 1762 cc = 0; 1763 break; 1764 } 1765 } 1766 1767 set_address(env, r1, dst); 1768 set_length(env, r1 + 1, len); 1769 set_address(env, r2, src); 1770 1771 return cc; 1772 } 1773 1774 void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, 1775 uint32_t r1, uint32_t r3) 1776 { 1777 uintptr_t ra = GETPC(); 1778 Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]); 1779 Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]); 1780 Int128 oldv; 1781 uint64_t oldh, oldl; 1782 bool fail; 1783 1784 check_alignment(env, addr, 16, ra); 1785 1786 oldh = cpu_ldq_data_ra(env, addr + 0, ra); 1787 oldl = cpu_ldq_data_ra(env, addr + 8, ra); 1788 1789 oldv = int128_make128(oldl, oldh); 1790 fail = !int128_eq(oldv, cmpv); 1791 if (fail) { 1792 newv = oldv; 1793 } 1794 1795 cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra); 1796 cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra); 1797 1798 env->cc_op = fail; 1799 env->regs[r1] = int128_gethi(oldv); 1800 env->regs[r1 + 1] = int128_getlo(oldv); 1801 } 1802 1803 void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, 1804 uint32_t r1, uint32_t r3) 1805 { 1806 uintptr_t ra = GETPC(); 1807 Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]); 1808 Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]); 1809 int mem_idx; 1810 MemOpIdx oi; 1811 Int128 oldv; 1812 bool fail; 1813 1814 assert(HAVE_CMPXCHG128); 1815 1816 mem_idx = cpu_mmu_index(env, false); 1817 oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx); 1818 oldv = cpu_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); 1819 fail = !int128_eq(oldv, cmpv); 1820 1821 env->cc_op = fail; 1822 env->regs[r1] = int128_gethi(oldv); 1823 env->regs[r1 + 1] = int128_getlo(oldv); 1824 } 1825 1826 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, 1827 uint64_t a2, bool parallel) 1828 { 1829 uint32_t mem_idx = cpu_mmu_index(env, false); 1830 uintptr_t ra = GETPC(); 1831 uint32_t fc = extract32(env->regs[0], 0, 8); 1832 uint32_t sc = extract32(env->regs[0], 8, 8); 1833 uint64_t pl = get_address(env, 1) & -16; 1834 uint64_t svh, svl; 1835 uint32_t cc; 1836 1837 /* Sanity check the function code and storage characteristic. */ 1838 if (fc > 1 || sc > 3) { 1839 if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) { 1840 goto spec_exception; 1841 } 1842 if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) { 1843 goto spec_exception; 1844 } 1845 } 1846 1847 /* Sanity check the alignments. */ 1848 if (extract32(a1, 0, fc + 2) || extract32(a2, 0, sc)) { 1849 goto spec_exception; 1850 } 1851 1852 /* Sanity check writability of the store address. */ 1853 probe_write(env, a2, 1 << sc, mem_idx, ra); 1854 1855 /* 1856 * Note that the compare-and-swap is atomic, and the store is atomic, 1857 * but the complete operation is not. Therefore we do not need to 1858 * assert serial context in order to implement this. That said, 1859 * restart early if we can't support either operation that is supposed 1860 * to be atomic. 1861 */ 1862 if (parallel) { 1863 uint32_t max = 2; 1864 #ifdef CONFIG_ATOMIC64 1865 max = 3; 1866 #endif 1867 if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) || 1868 (HAVE_ATOMIC128 ? 0 : sc > max)) { 1869 cpu_loop_exit_atomic(env_cpu(env), ra); 1870 } 1871 } 1872 1873 /* All loads happen before all stores. For simplicity, load the entire 1874 store value area from the parameter list. */ 1875 svh = cpu_ldq_data_ra(env, pl + 16, ra); 1876 svl = cpu_ldq_data_ra(env, pl + 24, ra); 1877 1878 switch (fc) { 1879 case 0: 1880 { 1881 uint32_t nv = cpu_ldl_data_ra(env, pl, ra); 1882 uint32_t cv = env->regs[r3]; 1883 uint32_t ov; 1884 1885 if (parallel) { 1886 #ifdef CONFIG_USER_ONLY 1887 uint32_t *haddr = g2h(env_cpu(env), a1); 1888 ov = qatomic_cmpxchg__nocheck(haddr, cv, nv); 1889 #else 1890 MemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx); 1891 ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra); 1892 #endif 1893 } else { 1894 ov = cpu_ldl_data_ra(env, a1, ra); 1895 cpu_stl_data_ra(env, a1, (ov == cv ? nv : ov), ra); 1896 } 1897 cc = (ov != cv); 1898 env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov); 1899 } 1900 break; 1901 1902 case 1: 1903 { 1904 uint64_t nv = cpu_ldq_data_ra(env, pl, ra); 1905 uint64_t cv = env->regs[r3]; 1906 uint64_t ov; 1907 1908 if (parallel) { 1909 #ifdef CONFIG_ATOMIC64 1910 MemOpIdx oi = make_memop_idx(MO_TEUQ | MO_ALIGN, mem_idx); 1911 ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra); 1912 #else 1913 /* Note that we asserted !parallel above. */ 1914 g_assert_not_reached(); 1915 #endif 1916 } else { 1917 ov = cpu_ldq_data_ra(env, a1, ra); 1918 cpu_stq_data_ra(env, a1, (ov == cv ? nv : ov), ra); 1919 } 1920 cc = (ov != cv); 1921 env->regs[r3] = ov; 1922 } 1923 break; 1924 1925 case 2: 1926 { 1927 uint64_t nvh = cpu_ldq_data_ra(env, pl, ra); 1928 uint64_t nvl = cpu_ldq_data_ra(env, pl + 8, ra); 1929 Int128 nv = int128_make128(nvl, nvh); 1930 Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]); 1931 Int128 ov; 1932 1933 if (!parallel) { 1934 uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra); 1935 uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra); 1936 1937 ov = int128_make128(ol, oh); 1938 cc = !int128_eq(ov, cv); 1939 if (cc) { 1940 nv = ov; 1941 } 1942 1943 cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra); 1944 cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra); 1945 } else if (HAVE_CMPXCHG128) { 1946 MemOpIdx oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx); 1947 ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra); 1948 cc = !int128_eq(ov, cv); 1949 } else { 1950 /* Note that we asserted !parallel above. */ 1951 g_assert_not_reached(); 1952 } 1953 1954 env->regs[r3 + 0] = int128_gethi(ov); 1955 env->regs[r3 + 1] = int128_getlo(ov); 1956 } 1957 break; 1958 1959 default: 1960 g_assert_not_reached(); 1961 } 1962 1963 /* Store only if the comparison succeeded. Note that above we use a pair 1964 of 64-bit big-endian loads, so for sc < 3 we must extract the value 1965 from the most-significant bits of svh. */ 1966 if (cc == 0) { 1967 switch (sc) { 1968 case 0: 1969 cpu_stb_data_ra(env, a2, svh >> 56, ra); 1970 break; 1971 case 1: 1972 cpu_stw_data_ra(env, a2, svh >> 48, ra); 1973 break; 1974 case 2: 1975 cpu_stl_data_ra(env, a2, svh >> 32, ra); 1976 break; 1977 case 3: 1978 cpu_stq_data_ra(env, a2, svh, ra); 1979 break; 1980 case 4: 1981 if (!parallel) { 1982 cpu_stq_data_ra(env, a2 + 0, svh, ra); 1983 cpu_stq_data_ra(env, a2 + 8, svl, ra); 1984 } else if (HAVE_ATOMIC128) { 1985 MemOpIdx oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx); 1986 Int128 sv = int128_make128(svl, svh); 1987 cpu_atomic_sto_be_mmu(env, a2, sv, oi, ra); 1988 } else { 1989 /* Note that we asserted !parallel above. */ 1990 g_assert_not_reached(); 1991 } 1992 break; 1993 default: 1994 g_assert_not_reached(); 1995 } 1996 } 1997 1998 return cc; 1999 2000 spec_exception: 2001 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2002 } 2003 2004 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2) 2005 { 2006 return do_csst(env, r3, a1, a2, false); 2007 } 2008 2009 uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1, 2010 uint64_t a2) 2011 { 2012 return do_csst(env, r3, a1, a2, true); 2013 } 2014 2015 #if !defined(CONFIG_USER_ONLY) 2016 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 2017 { 2018 uintptr_t ra = GETPC(); 2019 bool PERchanged = false; 2020 uint64_t src = a2; 2021 uint32_t i; 2022 2023 if (src & 0x7) { 2024 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2025 } 2026 2027 for (i = r1;; i = (i + 1) % 16) { 2028 uint64_t val = cpu_ldq_data_ra(env, src, ra); 2029 if (env->cregs[i] != val && i >= 9 && i <= 11) { 2030 PERchanged = true; 2031 } 2032 env->cregs[i] = val; 2033 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n", 2034 i, src, val); 2035 src += sizeof(uint64_t); 2036 2037 if (i == r3) { 2038 break; 2039 } 2040 } 2041 2042 if (PERchanged && env->psw.mask & PSW_MASK_PER) { 2043 s390_cpu_recompute_watchpoints(env_cpu(env)); 2044 } 2045 2046 tlb_flush(env_cpu(env)); 2047 } 2048 2049 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 2050 { 2051 uintptr_t ra = GETPC(); 2052 bool PERchanged = false; 2053 uint64_t src = a2; 2054 uint32_t i; 2055 2056 if (src & 0x3) { 2057 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2058 } 2059 2060 for (i = r1;; i = (i + 1) % 16) { 2061 uint32_t val = cpu_ldl_data_ra(env, src, ra); 2062 if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) { 2063 PERchanged = true; 2064 } 2065 env->cregs[i] = deposit64(env->cregs[i], 0, 32, val); 2066 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val); 2067 src += sizeof(uint32_t); 2068 2069 if (i == r3) { 2070 break; 2071 } 2072 } 2073 2074 if (PERchanged && env->psw.mask & PSW_MASK_PER) { 2075 s390_cpu_recompute_watchpoints(env_cpu(env)); 2076 } 2077 2078 tlb_flush(env_cpu(env)); 2079 } 2080 2081 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 2082 { 2083 uintptr_t ra = GETPC(); 2084 uint64_t dest = a2; 2085 uint32_t i; 2086 2087 if (dest & 0x7) { 2088 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2089 } 2090 2091 for (i = r1;; i = (i + 1) % 16) { 2092 cpu_stq_data_ra(env, dest, env->cregs[i], ra); 2093 dest += sizeof(uint64_t); 2094 2095 if (i == r3) { 2096 break; 2097 } 2098 } 2099 } 2100 2101 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) 2102 { 2103 uintptr_t ra = GETPC(); 2104 uint64_t dest = a2; 2105 uint32_t i; 2106 2107 if (dest & 0x3) { 2108 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2109 } 2110 2111 for (i = r1;; i = (i + 1) % 16) { 2112 cpu_stl_data_ra(env, dest, env->cregs[i], ra); 2113 dest += sizeof(uint32_t); 2114 2115 if (i == r3) { 2116 break; 2117 } 2118 } 2119 } 2120 2121 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr) 2122 { 2123 uintptr_t ra = GETPC(); 2124 int i; 2125 2126 real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK; 2127 2128 for (i = 0; i < TARGET_PAGE_SIZE; i += 8) { 2129 cpu_stq_mmuidx_ra(env, real_addr + i, 0, MMU_REAL_IDX, ra); 2130 } 2131 2132 return 0; 2133 } 2134 2135 uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2) 2136 { 2137 S390CPU *cpu = env_archcpu(env); 2138 CPUState *cs = env_cpu(env); 2139 2140 /* 2141 * TODO: we currently don't handle all access protection types 2142 * (including access-list and key-controlled) as well as AR mode. 2143 */ 2144 if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) { 2145 /* Fetching permitted; storing permitted */ 2146 return 0; 2147 } 2148 2149 if (env->int_pgm_code == PGM_PROTECTION) { 2150 /* retry if reading is possible */ 2151 cs->exception_index = -1; 2152 if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) { 2153 /* Fetching permitted; storing not permitted */ 2154 return 1; 2155 } 2156 } 2157 2158 switch (env->int_pgm_code) { 2159 case PGM_PROTECTION: 2160 /* Fetching not permitted; storing not permitted */ 2161 cs->exception_index = -1; 2162 return 2; 2163 case PGM_ADDRESSING: 2164 case PGM_TRANS_SPEC: 2165 /* exceptions forwarded to the guest */ 2166 s390_cpu_virt_mem_handle_exc(cpu, GETPC()); 2167 return 0; 2168 } 2169 2170 /* Translation not available */ 2171 cs->exception_index = -1; 2172 return 3; 2173 } 2174 2175 /* insert storage key extended */ 2176 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2) 2177 { 2178 static S390SKeysState *ss; 2179 static S390SKeysClass *skeyclass; 2180 uint64_t addr = wrap_address(env, r2); 2181 uint8_t key; 2182 int rc; 2183 2184 addr = mmu_real2abs(env, addr); 2185 if (!mmu_absolute_addr_valid(addr, false)) { 2186 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2187 } 2188 2189 if (unlikely(!ss)) { 2190 ss = s390_get_skeys_device(); 2191 skeyclass = S390_SKEYS_GET_CLASS(ss); 2192 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2193 tlb_flush_all_cpus_synced(env_cpu(env)); 2194 } 2195 } 2196 2197 rc = skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2198 if (rc) { 2199 trace_get_skeys_nonzero(rc); 2200 return 0; 2201 } 2202 return key; 2203 } 2204 2205 /* set storage key extended */ 2206 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2) 2207 { 2208 static S390SKeysState *ss; 2209 static S390SKeysClass *skeyclass; 2210 uint64_t addr = wrap_address(env, r2); 2211 uint8_t key; 2212 int rc; 2213 2214 addr = mmu_real2abs(env, addr); 2215 if (!mmu_absolute_addr_valid(addr, false)) { 2216 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2217 } 2218 2219 if (unlikely(!ss)) { 2220 ss = s390_get_skeys_device(); 2221 skeyclass = S390_SKEYS_GET_CLASS(ss); 2222 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2223 tlb_flush_all_cpus_synced(env_cpu(env)); 2224 } 2225 } 2226 2227 key = r1 & 0xfe; 2228 rc = skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2229 if (rc) { 2230 trace_set_skeys_nonzero(rc); 2231 } 2232 /* 2233 * As we can only flush by virtual address and not all the entries 2234 * that point to a physical address we have to flush the whole TLB. 2235 */ 2236 tlb_flush_all_cpus_synced(env_cpu(env)); 2237 } 2238 2239 /* reset reference bit extended */ 2240 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2) 2241 { 2242 uint64_t addr = wrap_address(env, r2); 2243 static S390SKeysState *ss; 2244 static S390SKeysClass *skeyclass; 2245 uint8_t re, key; 2246 int rc; 2247 2248 addr = mmu_real2abs(env, addr); 2249 if (!mmu_absolute_addr_valid(addr, false)) { 2250 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC()); 2251 } 2252 2253 if (unlikely(!ss)) { 2254 ss = s390_get_skeys_device(); 2255 skeyclass = S390_SKEYS_GET_CLASS(ss); 2256 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) { 2257 tlb_flush_all_cpus_synced(env_cpu(env)); 2258 } 2259 } 2260 2261 rc = skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2262 if (rc) { 2263 trace_get_skeys_nonzero(rc); 2264 return 0; 2265 } 2266 2267 re = key & (SK_R | SK_C); 2268 key &= ~SK_R; 2269 2270 rc = skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key); 2271 if (rc) { 2272 trace_set_skeys_nonzero(rc); 2273 return 0; 2274 } 2275 /* 2276 * As we can only flush by virtual address and not all the entries 2277 * that point to a physical address we have to flush the whole TLB. 2278 */ 2279 tlb_flush_all_cpus_synced(env_cpu(env)); 2280 2281 /* 2282 * cc 2283 * 2284 * 0 Reference bit zero; change bit zero 2285 * 1 Reference bit zero; change bit one 2286 * 2 Reference bit one; change bit zero 2287 * 3 Reference bit one; change bit one 2288 */ 2289 2290 return re >> 1; 2291 } 2292 2293 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2, 2294 uint64_t key) 2295 { 2296 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2297 S390Access srca, desta; 2298 uintptr_t ra = GETPC(); 2299 int cc = 0; 2300 2301 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n", 2302 __func__, l, a1, a2); 2303 2304 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) || 2305 psw_as == AS_HOME || psw_as == AS_ACCREG) { 2306 s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2307 } 2308 2309 if (!psw_key_valid(env, (key >> 4) & 0xf)) { 2310 s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2311 } 2312 2313 l = wrap_length32(env, l); 2314 if (l > 256) { 2315 /* max 256 */ 2316 l = 256; 2317 cc = 3; 2318 } else if (!l) { 2319 return cc; 2320 } 2321 2322 srca = access_prepare(env, a2, l, MMU_DATA_LOAD, MMU_PRIMARY_IDX, ra); 2323 desta = access_prepare(env, a1, l, MMU_DATA_STORE, MMU_SECONDARY_IDX, ra); 2324 access_memmove(env, &desta, &srca, ra); 2325 return cc; 2326 } 2327 2328 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2, 2329 uint64_t key) 2330 { 2331 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2332 S390Access srca, desta; 2333 uintptr_t ra = GETPC(); 2334 int cc = 0; 2335 2336 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n", 2337 __func__, l, a1, a2); 2338 2339 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) || 2340 psw_as == AS_HOME || psw_as == AS_ACCREG) { 2341 s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2342 } 2343 2344 if (!psw_key_valid(env, (key >> 4) & 0xf)) { 2345 s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2346 } 2347 2348 l = wrap_length32(env, l); 2349 if (l > 256) { 2350 /* max 256 */ 2351 l = 256; 2352 cc = 3; 2353 } else if (!l) { 2354 return cc; 2355 } 2356 2357 srca = access_prepare(env, a2, l, MMU_DATA_LOAD, MMU_SECONDARY_IDX, ra); 2358 desta = access_prepare(env, a1, l, MMU_DATA_STORE, MMU_PRIMARY_IDX, ra); 2359 access_memmove(env, &desta, &srca, ra); 2360 return cc; 2361 } 2362 2363 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4) 2364 { 2365 CPUState *cs = env_cpu(env); 2366 const uintptr_t ra = GETPC(); 2367 uint64_t table, entry, raddr; 2368 uint16_t entries, i, index = 0; 2369 2370 if (r2 & 0xff000) { 2371 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra); 2372 } 2373 2374 if (!(r2 & 0x800)) { 2375 /* invalidation-and-clearing operation */ 2376 table = r1 & ASCE_ORIGIN; 2377 entries = (r2 & 0x7ff) + 1; 2378 2379 switch (r1 & ASCE_TYPE_MASK) { 2380 case ASCE_TYPE_REGION1: 2381 index = (r2 >> 53) & 0x7ff; 2382 break; 2383 case ASCE_TYPE_REGION2: 2384 index = (r2 >> 42) & 0x7ff; 2385 break; 2386 case ASCE_TYPE_REGION3: 2387 index = (r2 >> 31) & 0x7ff; 2388 break; 2389 case ASCE_TYPE_SEGMENT: 2390 index = (r2 >> 20) & 0x7ff; 2391 break; 2392 } 2393 for (i = 0; i < entries; i++) { 2394 /* addresses are not wrapped in 24/31bit mode but table index is */ 2395 raddr = table + ((index + i) & 0x7ff) * sizeof(entry); 2396 entry = cpu_ldq_mmuidx_ra(env, raddr, MMU_REAL_IDX, ra); 2397 if (!(entry & REGION_ENTRY_I)) { 2398 /* we are allowed to not store if already invalid */ 2399 entry |= REGION_ENTRY_I; 2400 cpu_stq_mmuidx_ra(env, raddr, entry, MMU_REAL_IDX, ra); 2401 } 2402 } 2403 } 2404 2405 /* We simply flush the complete tlb, therefore we can ignore r3. */ 2406 if (m4 & 1) { 2407 tlb_flush(cs); 2408 } else { 2409 tlb_flush_all_cpus_synced(cs); 2410 } 2411 } 2412 2413 /* invalidate pte */ 2414 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr, 2415 uint32_t m4) 2416 { 2417 CPUState *cs = env_cpu(env); 2418 const uintptr_t ra = GETPC(); 2419 uint64_t page = vaddr & TARGET_PAGE_MASK; 2420 uint64_t pte_addr, pte; 2421 2422 /* Compute the page table entry address */ 2423 pte_addr = (pto & SEGMENT_ENTRY_ORIGIN); 2424 pte_addr += VADDR_PAGE_TX(vaddr) * 8; 2425 2426 /* Mark the page table entry as invalid */ 2427 pte = cpu_ldq_mmuidx_ra(env, pte_addr, MMU_REAL_IDX, ra); 2428 pte |= PAGE_ENTRY_I; 2429 cpu_stq_mmuidx_ra(env, pte_addr, pte, MMU_REAL_IDX, ra); 2430 2431 /* XXX we exploit the fact that Linux passes the exact virtual 2432 address here - it's not obliged to! */ 2433 if (m4 & 1) { 2434 if (vaddr & ~VADDR_PAGE_TX_MASK) { 2435 tlb_flush_page(cs, page); 2436 /* XXX 31-bit hack */ 2437 tlb_flush_page(cs, page ^ 0x80000000); 2438 } else { 2439 /* looks like we don't have a valid virtual address */ 2440 tlb_flush(cs); 2441 } 2442 } else { 2443 if (vaddr & ~VADDR_PAGE_TX_MASK) { 2444 tlb_flush_page_all_cpus_synced(cs, page); 2445 /* XXX 31-bit hack */ 2446 tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000); 2447 } else { 2448 /* looks like we don't have a valid virtual address */ 2449 tlb_flush_all_cpus_synced(cs); 2450 } 2451 } 2452 } 2453 2454 /* flush local tlb */ 2455 void HELPER(ptlb)(CPUS390XState *env) 2456 { 2457 tlb_flush(env_cpu(env)); 2458 } 2459 2460 /* flush global tlb */ 2461 void HELPER(purge)(CPUS390XState *env) 2462 { 2463 tlb_flush_all_cpus_synced(env_cpu(env)); 2464 } 2465 2466 /* load real address */ 2467 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr) 2468 { 2469 uint64_t asc = env->psw.mask & PSW_MASK_ASC; 2470 uint64_t ret, tec; 2471 int flags, exc, cc; 2472 2473 /* XXX incomplete - has more corner cases */ 2474 if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) { 2475 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, GETPC()); 2476 } 2477 2478 exc = mmu_translate(env, addr, MMU_S390_LRA, asc, &ret, &flags, &tec); 2479 if (exc) { 2480 cc = 3; 2481 ret = exc | 0x80000000; 2482 } else { 2483 cc = 0; 2484 ret |= addr & ~TARGET_PAGE_MASK; 2485 } 2486 2487 env->cc_op = cc; 2488 return ret; 2489 } 2490 #endif 2491 2492 /* load pair from quadword */ 2493 uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr) 2494 { 2495 uintptr_t ra = GETPC(); 2496 uint64_t hi, lo; 2497 2498 check_alignment(env, addr, 16, ra); 2499 hi = cpu_ldq_data_ra(env, addr + 0, ra); 2500 lo = cpu_ldq_data_ra(env, addr + 8, ra); 2501 2502 env->retxl = lo; 2503 return hi; 2504 } 2505 2506 uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) 2507 { 2508 uintptr_t ra = GETPC(); 2509 uint64_t hi, lo; 2510 int mem_idx; 2511 MemOpIdx oi; 2512 Int128 v; 2513 2514 assert(HAVE_ATOMIC128); 2515 2516 mem_idx = cpu_mmu_index(env, false); 2517 oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx); 2518 v = cpu_atomic_ldo_be_mmu(env, addr, oi, ra); 2519 hi = int128_gethi(v); 2520 lo = int128_getlo(v); 2521 2522 env->retxl = lo; 2523 return hi; 2524 } 2525 2526 /* store pair to quadword */ 2527 void HELPER(stpq)(CPUS390XState *env, uint64_t addr, 2528 uint64_t low, uint64_t high) 2529 { 2530 uintptr_t ra = GETPC(); 2531 2532 check_alignment(env, addr, 16, ra); 2533 cpu_stq_data_ra(env, addr + 0, high, ra); 2534 cpu_stq_data_ra(env, addr + 8, low, ra); 2535 } 2536 2537 void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, 2538 uint64_t low, uint64_t high) 2539 { 2540 uintptr_t ra = GETPC(); 2541 int mem_idx; 2542 MemOpIdx oi; 2543 Int128 v; 2544 2545 assert(HAVE_ATOMIC128); 2546 2547 mem_idx = cpu_mmu_index(env, false); 2548 oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx); 2549 v = int128_make128(low, high); 2550 cpu_atomic_sto_be_mmu(env, addr, v, oi, ra); 2551 } 2552 2553 /* Execute instruction. This instruction executes an insn modified with 2554 the contents of r1. It does not change the executed instruction in memory; 2555 it does not change the program counter. 2556 2557 Perform this by recording the modified instruction in env->ex_value. 2558 This will be noticed by cpu_get_tb_cpu_state and thus tb translation. 2559 */ 2560 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr) 2561 { 2562 uint64_t insn = cpu_lduw_code(env, addr); 2563 uint8_t opc = insn >> 8; 2564 2565 /* Or in the contents of R1[56:63]. */ 2566 insn |= r1 & 0xff; 2567 2568 /* Load the rest of the instruction. */ 2569 insn <<= 48; 2570 switch (get_ilen(opc)) { 2571 case 2: 2572 break; 2573 case 4: 2574 insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32; 2575 break; 2576 case 6: 2577 insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16; 2578 break; 2579 default: 2580 g_assert_not_reached(); 2581 } 2582 2583 /* The very most common cases can be sped up by avoiding a new TB. */ 2584 if ((opc & 0xf0) == 0xd0) { 2585 typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t, 2586 uint64_t, uintptr_t); 2587 static const dx_helper dx[16] = { 2588 [0x0] = do_helper_trt_bkwd, 2589 [0x2] = do_helper_mvc, 2590 [0x4] = do_helper_nc, 2591 [0x5] = do_helper_clc, 2592 [0x6] = do_helper_oc, 2593 [0x7] = do_helper_xc, 2594 [0xc] = do_helper_tr, 2595 [0xd] = do_helper_trt_fwd, 2596 }; 2597 dx_helper helper = dx[opc & 0xf]; 2598 2599 if (helper) { 2600 uint32_t l = extract64(insn, 48, 8); 2601 uint32_t b1 = extract64(insn, 44, 4); 2602 uint32_t d1 = extract64(insn, 32, 12); 2603 uint32_t b2 = extract64(insn, 28, 4); 2604 uint32_t d2 = extract64(insn, 16, 12); 2605 uint64_t a1 = wrap_address(env, (b1 ? env->regs[b1] : 0) + d1); 2606 uint64_t a2 = wrap_address(env, (b2 ? env->regs[b2] : 0) + d2); 2607 2608 env->cc_op = helper(env, l, a1, a2, 0); 2609 env->psw.addr += ilen; 2610 return; 2611 } 2612 } else if (opc == 0x0a) { 2613 env->int_svc_code = extract64(insn, 48, 8); 2614 env->int_svc_ilen = ilen; 2615 helper_exception(env, EXCP_SVC); 2616 g_assert_not_reached(); 2617 } 2618 2619 /* Record the insn we want to execute as well as the ilen to use 2620 during the execution of the target insn. This will also ensure 2621 that ex_value is non-zero, which flags that we are in a state 2622 that requires such execution. */ 2623 env->ex_value = insn | ilen; 2624 } 2625 2626 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src, 2627 uint64_t len) 2628 { 2629 const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY; 2630 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; 2631 const uint64_t r0 = env->regs[0]; 2632 const uintptr_t ra = GETPC(); 2633 uint8_t dest_key, dest_as, dest_k, dest_a; 2634 uint8_t src_key, src_as, src_k, src_a; 2635 uint64_t val; 2636 int cc = 0; 2637 2638 HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n", 2639 __func__, dest, src, len); 2640 2641 if (!(env->psw.mask & PSW_MASK_DAT)) { 2642 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2643 } 2644 2645 /* OAC (operand access control) for the first operand -> dest */ 2646 val = (r0 & 0xffff0000ULL) >> 16; 2647 dest_key = (val >> 12) & 0xf; 2648 dest_as = (val >> 6) & 0x3; 2649 dest_k = (val >> 1) & 0x1; 2650 dest_a = val & 0x1; 2651 2652 /* OAC (operand access control) for the second operand -> src */ 2653 val = (r0 & 0x0000ffffULL); 2654 src_key = (val >> 12) & 0xf; 2655 src_as = (val >> 6) & 0x3; 2656 src_k = (val >> 1) & 0x1; 2657 src_a = val & 0x1; 2658 2659 if (!dest_k) { 2660 dest_key = psw_key; 2661 } 2662 if (!src_k) { 2663 src_key = psw_key; 2664 } 2665 if (!dest_a) { 2666 dest_as = psw_as; 2667 } 2668 if (!src_a) { 2669 src_as = psw_as; 2670 } 2671 2672 if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) { 2673 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2674 } 2675 if (!(env->cregs[0] & CR0_SECONDARY) && 2676 (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) { 2677 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra); 2678 } 2679 if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) { 2680 tcg_s390_program_interrupt(env, PGM_PRIVILEGED, ra); 2681 } 2682 2683 len = wrap_length32(env, len); 2684 if (len > 4096) { 2685 cc = 3; 2686 len = 4096; 2687 } 2688 2689 /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */ 2690 if (src_as == AS_ACCREG || dest_as == AS_ACCREG || 2691 (env->psw.mask & PSW_MASK_PSTATE)) { 2692 qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n", 2693 __func__); 2694 tcg_s390_program_interrupt(env, PGM_ADDRESSING, ra); 2695 } 2696 2697 /* FIXME: Access using correct keys and AR-mode */ 2698 if (len) { 2699 S390Access srca = access_prepare(env, src, len, MMU_DATA_LOAD, 2700 mmu_idx_from_as(src_as), ra); 2701 S390Access desta = access_prepare(env, dest, len, MMU_DATA_STORE, 2702 mmu_idx_from_as(dest_as), ra); 2703 2704 access_memmove(env, &desta, &srca, ra); 2705 } 2706 2707 return cc; 2708 } 2709 2710 /* Decode a Unicode character. A return value < 0 indicates success, storing 2711 the UTF-32 result into OCHAR and the input length into OLEN. A return 2712 value >= 0 indicates failure, and the CC value to be returned. */ 2713 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr, 2714 uint64_t ilen, bool enh_check, uintptr_t ra, 2715 uint32_t *ochar, uint32_t *olen); 2716 2717 /* Encode a Unicode character. A return value < 0 indicates success, storing 2718 the bytes into ADDR and the output length into OLEN. A return value >= 0 2719 indicates failure, and the CC value to be returned. */ 2720 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr, 2721 uint64_t ilen, uintptr_t ra, uint32_t c, 2722 uint32_t *olen); 2723 2724 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2725 bool enh_check, uintptr_t ra, 2726 uint32_t *ochar, uint32_t *olen) 2727 { 2728 uint8_t s0, s1, s2, s3; 2729 uint32_t c, l; 2730 2731 if (ilen < 1) { 2732 return 0; 2733 } 2734 s0 = cpu_ldub_data_ra(env, addr, ra); 2735 if (s0 <= 0x7f) { 2736 /* one byte character */ 2737 l = 1; 2738 c = s0; 2739 } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) { 2740 /* invalid character */ 2741 return 2; 2742 } else if (s0 <= 0xdf) { 2743 /* two byte character */ 2744 l = 2; 2745 if (ilen < 2) { 2746 return 0; 2747 } 2748 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2749 c = s0 & 0x1f; 2750 c = (c << 6) | (s1 & 0x3f); 2751 if (enh_check && (s1 & 0xc0) != 0x80) { 2752 return 2; 2753 } 2754 } else if (s0 <= 0xef) { 2755 /* three byte character */ 2756 l = 3; 2757 if (ilen < 3) { 2758 return 0; 2759 } 2760 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2761 s2 = cpu_ldub_data_ra(env, addr + 2, ra); 2762 c = s0 & 0x0f; 2763 c = (c << 6) | (s1 & 0x3f); 2764 c = (c << 6) | (s2 & 0x3f); 2765 /* Fold the byte-by-byte range descriptions in the PoO into 2766 tests against the complete value. It disallows encodings 2767 that could be smaller, and the UTF-16 surrogates. */ 2768 if (enh_check 2769 && ((s1 & 0xc0) != 0x80 2770 || (s2 & 0xc0) != 0x80 2771 || c < 0x1000 2772 || (c >= 0xd800 && c <= 0xdfff))) { 2773 return 2; 2774 } 2775 } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) { 2776 /* four byte character */ 2777 l = 4; 2778 if (ilen < 4) { 2779 return 0; 2780 } 2781 s1 = cpu_ldub_data_ra(env, addr + 1, ra); 2782 s2 = cpu_ldub_data_ra(env, addr + 2, ra); 2783 s3 = cpu_ldub_data_ra(env, addr + 3, ra); 2784 c = s0 & 0x07; 2785 c = (c << 6) | (s1 & 0x3f); 2786 c = (c << 6) | (s2 & 0x3f); 2787 c = (c << 6) | (s3 & 0x3f); 2788 /* See above. */ 2789 if (enh_check 2790 && ((s1 & 0xc0) != 0x80 2791 || (s2 & 0xc0) != 0x80 2792 || (s3 & 0xc0) != 0x80 2793 || c < 0x010000 2794 || c > 0x10ffff)) { 2795 return 2; 2796 } 2797 } else { 2798 /* invalid character */ 2799 return 2; 2800 } 2801 2802 *ochar = c; 2803 *olen = l; 2804 return -1; 2805 } 2806 2807 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2808 bool enh_check, uintptr_t ra, 2809 uint32_t *ochar, uint32_t *olen) 2810 { 2811 uint16_t s0, s1; 2812 uint32_t c, l; 2813 2814 if (ilen < 2) { 2815 return 0; 2816 } 2817 s0 = cpu_lduw_data_ra(env, addr, ra); 2818 if ((s0 & 0xfc00) != 0xd800) { 2819 /* one word character */ 2820 l = 2; 2821 c = s0; 2822 } else { 2823 /* two word character */ 2824 l = 4; 2825 if (ilen < 4) { 2826 return 0; 2827 } 2828 s1 = cpu_lduw_data_ra(env, addr + 2, ra); 2829 c = extract32(s0, 6, 4) + 1; 2830 c = (c << 6) | (s0 & 0x3f); 2831 c = (c << 10) | (s1 & 0x3ff); 2832 if (enh_check && (s1 & 0xfc00) != 0xdc00) { 2833 /* invalid surrogate character */ 2834 return 2; 2835 } 2836 } 2837 2838 *ochar = c; 2839 *olen = l; 2840 return -1; 2841 } 2842 2843 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2844 bool enh_check, uintptr_t ra, 2845 uint32_t *ochar, uint32_t *olen) 2846 { 2847 uint32_t c; 2848 2849 if (ilen < 4) { 2850 return 0; 2851 } 2852 c = cpu_ldl_data_ra(env, addr, ra); 2853 if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) { 2854 /* invalid unicode character */ 2855 return 2; 2856 } 2857 2858 *ochar = c; 2859 *olen = 4; 2860 return -1; 2861 } 2862 2863 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2864 uintptr_t ra, uint32_t c, uint32_t *olen) 2865 { 2866 uint8_t d[4]; 2867 uint32_t l, i; 2868 2869 if (c <= 0x7f) { 2870 /* one byte character */ 2871 l = 1; 2872 d[0] = c; 2873 } else if (c <= 0x7ff) { 2874 /* two byte character */ 2875 l = 2; 2876 d[1] = 0x80 | extract32(c, 0, 6); 2877 d[0] = 0xc0 | extract32(c, 6, 5); 2878 } else if (c <= 0xffff) { 2879 /* three byte character */ 2880 l = 3; 2881 d[2] = 0x80 | extract32(c, 0, 6); 2882 d[1] = 0x80 | extract32(c, 6, 6); 2883 d[0] = 0xe0 | extract32(c, 12, 4); 2884 } else { 2885 /* four byte character */ 2886 l = 4; 2887 d[3] = 0x80 | extract32(c, 0, 6); 2888 d[2] = 0x80 | extract32(c, 6, 6); 2889 d[1] = 0x80 | extract32(c, 12, 6); 2890 d[0] = 0xf0 | extract32(c, 18, 3); 2891 } 2892 2893 if (ilen < l) { 2894 return 1; 2895 } 2896 for (i = 0; i < l; ++i) { 2897 cpu_stb_data_ra(env, addr + i, d[i], ra); 2898 } 2899 2900 *olen = l; 2901 return -1; 2902 } 2903 2904 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2905 uintptr_t ra, uint32_t c, uint32_t *olen) 2906 { 2907 uint16_t d0, d1; 2908 2909 if (c <= 0xffff) { 2910 /* one word character */ 2911 if (ilen < 2) { 2912 return 1; 2913 } 2914 cpu_stw_data_ra(env, addr, c, ra); 2915 *olen = 2; 2916 } else { 2917 /* two word character */ 2918 if (ilen < 4) { 2919 return 1; 2920 } 2921 d1 = 0xdc00 | extract32(c, 0, 10); 2922 d0 = 0xd800 | extract32(c, 10, 6); 2923 d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1); 2924 cpu_stw_data_ra(env, addr + 0, d0, ra); 2925 cpu_stw_data_ra(env, addr + 2, d1, ra); 2926 *olen = 4; 2927 } 2928 2929 return -1; 2930 } 2931 2932 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen, 2933 uintptr_t ra, uint32_t c, uint32_t *olen) 2934 { 2935 if (ilen < 4) { 2936 return 1; 2937 } 2938 cpu_stl_data_ra(env, addr, c, ra); 2939 *olen = 4; 2940 return -1; 2941 } 2942 2943 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1, 2944 uint32_t r2, uint32_t m3, uintptr_t ra, 2945 decode_unicode_fn decode, 2946 encode_unicode_fn encode) 2947 { 2948 uint64_t dst = get_address(env, r1); 2949 uint64_t dlen = get_length(env, r1 + 1); 2950 uint64_t src = get_address(env, r2); 2951 uint64_t slen = get_length(env, r2 + 1); 2952 bool enh_check = m3 & 1; 2953 int cc, i; 2954 2955 /* Lest we fail to service interrupts in a timely manner, limit the 2956 amount of work we're willing to do. For now, let's cap at 256. */ 2957 for (i = 0; i < 256; ++i) { 2958 uint32_t c, ilen, olen; 2959 2960 cc = decode(env, src, slen, enh_check, ra, &c, &ilen); 2961 if (unlikely(cc >= 0)) { 2962 break; 2963 } 2964 cc = encode(env, dst, dlen, ra, c, &olen); 2965 if (unlikely(cc >= 0)) { 2966 break; 2967 } 2968 2969 src += ilen; 2970 slen -= ilen; 2971 dst += olen; 2972 dlen -= olen; 2973 cc = 3; 2974 } 2975 2976 set_address(env, r1, dst); 2977 set_length(env, r1 + 1, dlen); 2978 set_address(env, r2, src); 2979 set_length(env, r2 + 1, slen); 2980 2981 return cc; 2982 } 2983 2984 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2985 { 2986 return convert_unicode(env, r1, r2, m3, GETPC(), 2987 decode_utf8, encode_utf16); 2988 } 2989 2990 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2991 { 2992 return convert_unicode(env, r1, r2, m3, GETPC(), 2993 decode_utf8, encode_utf32); 2994 } 2995 2996 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 2997 { 2998 return convert_unicode(env, r1, r2, m3, GETPC(), 2999 decode_utf16, encode_utf8); 3000 } 3001 3002 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 3003 { 3004 return convert_unicode(env, r1, r2, m3, GETPC(), 3005 decode_utf16, encode_utf32); 3006 } 3007 3008 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 3009 { 3010 return convert_unicode(env, r1, r2, m3, GETPC(), 3011 decode_utf32, encode_utf8); 3012 } 3013 3014 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) 3015 { 3016 return convert_unicode(env, r1, r2, m3, GETPC(), 3017 decode_utf32, encode_utf16); 3018 } 3019 3020 void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len, 3021 uintptr_t ra) 3022 { 3023 /* test the actual access, not just any access to the page due to LAP */ 3024 while (len) { 3025 const uint64_t pagelen = -(addr | TARGET_PAGE_MASK); 3026 const uint64_t curlen = MIN(pagelen, len); 3027 3028 probe_write(env, addr, curlen, cpu_mmu_index(env, false), ra); 3029 addr = wrap_address(env, addr + curlen); 3030 len -= curlen; 3031 } 3032 } 3033 3034 void HELPER(probe_write_access)(CPUS390XState *env, uint64_t addr, uint64_t len) 3035 { 3036 probe_write_access(env, addr, len, GETPC()); 3037 } 3038