xref: /openbmc/qemu/target/hexagon/op_helper.c (revision a6caeee8) 
1 /*
2  *  Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved.
3  *
4  *  This program is free software; you can redistribute it and/or modify
5  *  it under the terms of the GNU General Public License as published by
6  *  the Free Software Foundation; either version 2 of the License, or
7  *  (at your option) any later version.
8  *
9  *  This program is distributed in the hope that it will be useful,
10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  *  GNU General Public License for more details.
13  *
14  *  You should have received a copy of the GNU General Public License
15  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
16  */
17 
18 #include "qemu/osdep.h"
19 #include "qemu/log.h"
20 #include "exec/exec-all.h"
21 #include "exec/cpu_ldst.h"
22 #include "exec/helper-proto.h"
23 #include "fpu/softfloat.h"
24 #include "cpu.h"
25 #include "internal.h"
26 #include "macros.h"
27 #include "arch.h"
28 #include "hex_arch_types.h"
29 #include "fma_emu.h"
30 #include "mmvec/mmvec.h"
31 #include "mmvec/macros.h"
32 
33 #define SF_BIAS        127
34 #define SF_MANTBITS    23
35 
36 /* Exceptions processing helpers */
37 static G_NORETURN
38 void do_raise_exception_err(CPUHexagonState *env,
39                             uint32_t exception,
40                             uintptr_t pc)
41 {
42     CPUState *cs = env_cpu(env);
43     qemu_log_mask(CPU_LOG_INT, "%s: %d\n", __func__, exception);
44     cs->exception_index = exception;
45     cpu_loop_exit_restore(cs, pc);
46 }
47 
48 G_NORETURN void HELPER(raise_exception)(CPUHexagonState *env, uint32_t excp)
49 {
50     do_raise_exception_err(env, excp, 0);
51 }
52 
53 static void log_reg_write(CPUHexagonState *env, int rnum,
54                           target_ulong val, uint32_t slot)
55 {
56     HEX_DEBUG_LOG("log_reg_write[%d] = " TARGET_FMT_ld " (0x" TARGET_FMT_lx ")",
57                   rnum, val, val);
58     if (val == env->gpr[rnum]) {
59         HEX_DEBUG_LOG(" NO CHANGE");
60     }
61     HEX_DEBUG_LOG("\n");
62 
63     env->new_value[rnum] = val;
64     if (HEX_DEBUG) {
65         /* Do this so HELPER(debug_commit_end) will know */
66         env->reg_written[rnum] = 1;
67     }
68 }
69 
70 static void log_pred_write(CPUHexagonState *env, int pnum, target_ulong val)
71 {
72     HEX_DEBUG_LOG("log_pred_write[%d] = " TARGET_FMT_ld
73                   " (0x" TARGET_FMT_lx ")\n",
74                   pnum, val, val);
75 
76     /* Multiple writes to the same preg are and'ed together */
77     if (env->pred_written & (1 << pnum)) {
78         env->new_pred_value[pnum] &= val & 0xff;
79     } else {
80         env->new_pred_value[pnum] = val & 0xff;
81         env->pred_written |= 1 << pnum;
82     }
83 }
84 
85 static void log_store32(CPUHexagonState *env, target_ulong addr,
86                         target_ulong val, int width, int slot)
87 {
88     HEX_DEBUG_LOG("log_store%d(0x" TARGET_FMT_lx
89                   ", %" PRId32 " [0x08%" PRIx32 "])\n",
90                   width, addr, val, val);
91     env->mem_log_stores[slot].va = addr;
92     env->mem_log_stores[slot].width = width;
93     env->mem_log_stores[slot].data32 = val;
94 }
95 
96 static void log_store64(CPUHexagonState *env, target_ulong addr,
97                         int64_t val, int width, int slot)
98 {
99     HEX_DEBUG_LOG("log_store%d(0x" TARGET_FMT_lx
100                   ", %" PRId64 " [0x016%" PRIx64 "])\n",
101                    width, addr, val, val);
102     env->mem_log_stores[slot].va = addr;
103     env->mem_log_stores[slot].width = width;
104     env->mem_log_stores[slot].data64 = val;
105 }
106 
107 static void write_new_pc(CPUHexagonState *env, target_ulong addr)
108 {
109     HEX_DEBUG_LOG("write_new_pc(0x" TARGET_FMT_lx ")\n", addr);
110 
111     /*
112      * If more than one branch is taken in a packet, only the first one
113      * is actually done.
114      */
115     if (env->branch_taken) {
116         HEX_DEBUG_LOG("INFO: multiple branches taken in same packet, "
117                       "ignoring the second one\n");
118     } else {
119         fCHECK_PCALIGN(addr);
120         env->branch_taken = 1;
121         env->next_PC = addr;
122     }
123 }
124 
125 /* Handy place to set a breakpoint */
126 void HELPER(debug_start_packet)(CPUHexagonState *env)
127 {
128     HEX_DEBUG_LOG("Start packet: pc = 0x" TARGET_FMT_lx "\n",
129                   env->gpr[HEX_REG_PC]);
130 
131     for (int i = 0; i < TOTAL_PER_THREAD_REGS; i++) {
132         env->reg_written[i] = 0;
133     }
134 }
135 
136 /* Checks for bookkeeping errors between disassembly context and runtime */
137 void HELPER(debug_check_store_width)(CPUHexagonState *env, int slot, int check)
138 {
139     if (env->mem_log_stores[slot].width != check) {
140         HEX_DEBUG_LOG("ERROR: %d != %d\n",
141                       env->mem_log_stores[slot].width, check);
142         g_assert_not_reached();
143     }
144 }
145 
146 void HELPER(commit_store)(CPUHexagonState *env, int slot_num)
147 {
148     uintptr_t ra = GETPC();
149     uint8_t width = env->mem_log_stores[slot_num].width;
150     target_ulong va = env->mem_log_stores[slot_num].va;
151 
152     switch (width) {
153     case 1:
154         cpu_stb_data_ra(env, va, env->mem_log_stores[slot_num].data32, ra);
155         break;
156     case 2:
157         cpu_stw_data_ra(env, va, env->mem_log_stores[slot_num].data32, ra);
158         break;
159     case 4:
160         cpu_stl_data_ra(env, va, env->mem_log_stores[slot_num].data32, ra);
161         break;
162     case 8:
163         cpu_stq_data_ra(env, va, env->mem_log_stores[slot_num].data64, ra);
164         break;
165     default:
166         g_assert_not_reached();
167     }
168 }
169 
170 void HELPER(gather_store)(CPUHexagonState *env, uint32_t addr, int slot)
171 {
172     mem_gather_store(env, addr, slot);
173 }
174 
175 void HELPER(commit_hvx_stores)(CPUHexagonState *env)
176 {
177     uintptr_t ra = GETPC();
178     int i;
179 
180     /* Normal (possibly masked) vector store */
181     for (i = 0; i < VSTORES_MAX; i++) {
182         if (env->vstore_pending[i]) {
183             env->vstore_pending[i] = 0;
184             target_ulong va = env->vstore[i].va;
185             int size = env->vstore[i].size;
186             for (int j = 0; j < size; j++) {
187                 if (test_bit(j, env->vstore[i].mask)) {
188                     cpu_stb_data_ra(env, va + j, env->vstore[i].data.ub[j], ra);
189                 }
190             }
191         }
192     }
193 
194     /* Scatter store */
195     if (env->vtcm_pending) {
196         env->vtcm_pending = false;
197         if (env->vtcm_log.op) {
198             /* Need to perform the scatter read/modify/write at commit time */
199             if (env->vtcm_log.op_size == 2) {
200                 SCATTER_OP_WRITE_TO_MEM(uint16_t);
201             } else if (env->vtcm_log.op_size == 4) {
202                 /* Word Scatter += */
203                 SCATTER_OP_WRITE_TO_MEM(uint32_t);
204             } else {
205                 g_assert_not_reached();
206             }
207         } else {
208             for (i = 0; i < sizeof(MMVector); i++) {
209                 if (test_bit(i, env->vtcm_log.mask)) {
210                     cpu_stb_data_ra(env, env->vtcm_log.va[i],
211                                     env->vtcm_log.data.ub[i], ra);
212                     clear_bit(i, env->vtcm_log.mask);
213                     env->vtcm_log.data.ub[i] = 0;
214                 }
215 
216             }
217         }
218     }
219 }
220 
221 static void print_store(CPUHexagonState *env, int slot)
222 {
223     if (!(env->slot_cancelled & (1 << slot))) {
224         uint8_t width = env->mem_log_stores[slot].width;
225         if (width == 1) {
226             uint32_t data = env->mem_log_stores[slot].data32 & 0xff;
227             HEX_DEBUG_LOG("\tmemb[0x" TARGET_FMT_lx "] = %" PRId32
228                           " (0x%02" PRIx32 ")\n",
229                           env->mem_log_stores[slot].va, data, data);
230         } else if (width == 2) {
231             uint32_t data = env->mem_log_stores[slot].data32 & 0xffff;
232             HEX_DEBUG_LOG("\tmemh[0x" TARGET_FMT_lx "] = %" PRId32
233                           " (0x%04" PRIx32 ")\n",
234                           env->mem_log_stores[slot].va, data, data);
235         } else if (width == 4) {
236             uint32_t data = env->mem_log_stores[slot].data32;
237             HEX_DEBUG_LOG("\tmemw[0x" TARGET_FMT_lx "] = %" PRId32
238                           " (0x%08" PRIx32 ")\n",
239                           env->mem_log_stores[slot].va, data, data);
240         } else if (width == 8) {
241             HEX_DEBUG_LOG("\tmemd[0x" TARGET_FMT_lx "] = %" PRId64
242                           " (0x%016" PRIx64 ")\n",
243                           env->mem_log_stores[slot].va,
244                           env->mem_log_stores[slot].data64,
245                           env->mem_log_stores[slot].data64);
246         } else {
247             HEX_DEBUG_LOG("\tBad store width %d\n", width);
248             g_assert_not_reached();
249         }
250     }
251 }
252 
253 /* This function is a handy place to set a breakpoint */
254 void HELPER(debug_commit_end)(CPUHexagonState *env, int has_st0, int has_st1)
255 {
256     bool reg_printed = false;
257     bool pred_printed = false;
258     int i;
259 
260     HEX_DEBUG_LOG("Packet committed: pc = 0x" TARGET_FMT_lx "\n",
261                   env->this_PC);
262     HEX_DEBUG_LOG("slot_cancelled = %d\n", env->slot_cancelled);
263 
264     for (i = 0; i < TOTAL_PER_THREAD_REGS; i++) {
265         if (env->reg_written[i]) {
266             if (!reg_printed) {
267                 HEX_DEBUG_LOG("Regs written\n");
268                 reg_printed = true;
269             }
270             HEX_DEBUG_LOG("\tr%d = " TARGET_FMT_ld " (0x" TARGET_FMT_lx ")\n",
271                           i, env->new_value[i], env->new_value[i]);
272         }
273     }
274 
275     for (i = 0; i < NUM_PREGS; i++) {
276         if (env->pred_written & (1 << i)) {
277             if (!pred_printed) {
278                 HEX_DEBUG_LOG("Predicates written\n");
279                 pred_printed = true;
280             }
281             HEX_DEBUG_LOG("\tp%d = 0x" TARGET_FMT_lx "\n",
282                           i, env->new_pred_value[i]);
283         }
284     }
285 
286     if (has_st0 || has_st1) {
287         HEX_DEBUG_LOG("Stores\n");
288         if (has_st0) {
289             print_store(env, 0);
290         }
291         if (has_st1) {
292             print_store(env, 1);
293         }
294     }
295 
296     HEX_DEBUG_LOG("Next PC = " TARGET_FMT_lx "\n", env->next_PC);
297     HEX_DEBUG_LOG("Exec counters: pkt = " TARGET_FMT_lx
298                   ", insn = " TARGET_FMT_lx
299                   ", hvx = " TARGET_FMT_lx "\n",
300                   env->gpr[HEX_REG_QEMU_PKT_CNT],
301                   env->gpr[HEX_REG_QEMU_INSN_CNT],
302                   env->gpr[HEX_REG_QEMU_HVX_CNT]);
303 
304 }
305 
306 int32_t HELPER(fcircadd)(int32_t RxV, int32_t offset, int32_t M, int32_t CS)
307 {
308     uint32_t K_const = extract32(M, 24, 4);
309     uint32_t length = extract32(M, 0, 17);
310     uint32_t new_ptr = RxV + offset;
311     uint32_t start_addr;
312     uint32_t end_addr;
313 
314     if (K_const == 0 && length >= 4) {
315         start_addr = CS;
316         end_addr = start_addr + length;
317     } else {
318         /*
319          * Versions v3 and earlier used the K value to specify a power-of-2 size
320          * 2^(K+2) that is greater than the buffer length
321          */
322         int32_t mask = (1 << (K_const + 2)) - 1;
323         start_addr = RxV & (~mask);
324         end_addr = start_addr | length;
325     }
326 
327     if (new_ptr >= end_addr) {
328         new_ptr -= length;
329     } else if (new_ptr < start_addr) {
330         new_ptr += length;
331     }
332 
333     return new_ptr;
334 }
335 
336 uint32_t HELPER(fbrev)(uint32_t addr)
337 {
338     /*
339      *  Bit reverse the low 16 bits of the address
340      */
341     return deposit32(addr, 0, 16, revbit16(addr));
342 }
343 
344 static float32 build_float32(uint8_t sign, uint32_t exp, uint32_t mant)
345 {
346     return make_float32(
347         ((sign & 1) << 31) |
348         ((exp & 0xff) << SF_MANTBITS) |
349         (mant & ((1 << SF_MANTBITS) - 1)));
350 }
351 
352 /*
353  * sfrecipa, sfinvsqrta have two 32-bit results
354  *     r0,p0=sfrecipa(r1,r2)
355  *     r0,p0=sfinvsqrta(r1)
356  *
357  * Since helpers can only return a single value, we pack the two results
358  * into a 64-bit value.
359  */
360 uint64_t HELPER(sfrecipa)(CPUHexagonState *env, float32 RsV, float32 RtV)
361 {
362     int32_t PeV = 0;
363     float32 RdV;
364     int idx;
365     int adjust;
366     int mant;
367     int exp;
368 
369     arch_fpop_start(env);
370     if (arch_sf_recip_common(&RsV, &RtV, &RdV, &adjust, &env->fp_status)) {
371         PeV = adjust;
372         idx = (RtV >> 16) & 0x7f;
373         mant = (recip_lookup_table[idx] << 15) | 1;
374         exp = SF_BIAS - (float32_getexp(RtV) - SF_BIAS) - 1;
375         RdV = build_float32(extract32(RtV, 31, 1), exp, mant);
376     }
377     arch_fpop_end(env);
378     return ((uint64_t)RdV << 32) | PeV;
379 }
380 
381 uint64_t HELPER(sfinvsqrta)(CPUHexagonState *env, float32 RsV)
382 {
383     int PeV = 0;
384     float32 RdV;
385     int idx;
386     int adjust;
387     int mant;
388     int exp;
389 
390     arch_fpop_start(env);
391     if (arch_sf_invsqrt_common(&RsV, &RdV, &adjust, &env->fp_status)) {
392         PeV = adjust;
393         idx = (RsV >> 17) & 0x7f;
394         mant = (invsqrt_lookup_table[idx] << 15);
395         exp = SF_BIAS - ((float32_getexp(RsV) - SF_BIAS) >> 1) - 1;
396         RdV = build_float32(extract32(RsV, 31, 1), exp, mant);
397     }
398     arch_fpop_end(env);
399     return ((uint64_t)RdV << 32) | PeV;
400 }
401 
402 int64_t HELPER(vacsh_val)(CPUHexagonState *env,
403                            int64_t RxxV, int64_t RssV, int64_t RttV)
404 {
405     for (int i = 0; i < 4; i++) {
406         int xv = sextract64(RxxV, i * 16, 16);
407         int sv = sextract64(RssV, i * 16, 16);
408         int tv = sextract64(RttV, i * 16, 16);
409         int max;
410         xv = xv + tv;
411         sv = sv - tv;
412         max = xv > sv ? xv : sv;
413         /* Note that fSATH can set the OVF bit in usr */
414         RxxV = deposit64(RxxV, i * 16, 16, fSATH(max));
415     }
416     return RxxV;
417 }
418 
419 int32_t HELPER(vacsh_pred)(CPUHexagonState *env,
420                            int64_t RxxV, int64_t RssV, int64_t RttV)
421 {
422     int32_t PeV = 0;
423     for (int i = 0; i < 4; i++) {
424         int xv = sextract64(RxxV, i * 16, 16);
425         int sv = sextract64(RssV, i * 16, 16);
426         int tv = sextract64(RttV, i * 16, 16);
427         xv = xv + tv;
428         sv = sv - tv;
429         PeV = deposit32(PeV, i * 2, 1, (xv > sv));
430         PeV = deposit32(PeV, i * 2 + 1, 1, (xv > sv));
431     }
432     return PeV;
433 }
434 
435 static void probe_store(CPUHexagonState *env, int slot, int mmu_idx)
436 {
437     if (!(env->slot_cancelled & (1 << slot))) {
438         size1u_t width = env->mem_log_stores[slot].width;
439         target_ulong va = env->mem_log_stores[slot].va;
440         uintptr_t ra = GETPC();
441         probe_write(env, va, width, mmu_idx, ra);
442     }
443 }
444 
445 /* Called during packet commit when there are two scalar stores */
446 void HELPER(probe_pkt_scalar_store_s0)(CPUHexagonState *env, int mmu_idx)
447 {
448     probe_store(env, 0, mmu_idx);
449 }
450 
451 void HELPER(probe_hvx_stores)(CPUHexagonState *env, int mmu_idx)
452 {
453     uintptr_t retaddr = GETPC();
454     int i;
455 
456     /* Normal (possibly masked) vector store */
457     for (i = 0; i < VSTORES_MAX; i++) {
458         if (env->vstore_pending[i]) {
459             target_ulong va = env->vstore[i].va;
460             int size = env->vstore[i].size;
461             for (int j = 0; j < size; j++) {
462                 if (test_bit(j, env->vstore[i].mask)) {
463                     probe_write(env, va + j, 1, mmu_idx, retaddr);
464                 }
465             }
466         }
467     }
468 
469     /* Scatter store */
470     if (env->vtcm_pending) {
471         if (env->vtcm_log.op) {
472             /* Need to perform the scatter read/modify/write at commit time */
473             if (env->vtcm_log.op_size == 2) {
474                 SCATTER_OP_PROBE_MEM(size2u_t, mmu_idx, retaddr);
475             } else if (env->vtcm_log.op_size == 4) {
476                 /* Word Scatter += */
477                 SCATTER_OP_PROBE_MEM(size4u_t, mmu_idx, retaddr);
478             } else {
479                 g_assert_not_reached();
480             }
481         } else {
482             for (int i = 0; i < sizeof(MMVector); i++) {
483                 if (test_bit(i, env->vtcm_log.mask)) {
484                     probe_write(env, env->vtcm_log.va[i], 1, mmu_idx, retaddr);
485                 }
486 
487             }
488         }
489     }
490 }
491 
492 void HELPER(probe_pkt_scalar_hvx_stores)(CPUHexagonState *env, int mask,
493                                          int mmu_idx)
494 {
495     bool has_st0        = (mask >> 0) & 1;
496     bool has_st1        = (mask >> 1) & 1;
497     bool has_hvx_stores = (mask >> 2) & 1;
498 
499     if (has_st0) {
500         probe_store(env, 0, mmu_idx);
501     }
502     if (has_st1) {
503         probe_store(env, 1, mmu_idx);
504     }
505     if (has_hvx_stores) {
506         HELPER(probe_hvx_stores)(env, mmu_idx);
507     }
508 }
509 
510 /*
511  * mem_noshuf
512  * Section 5.5 of the Hexagon V67 Programmer's Reference Manual
513  *
514  * If the load is in slot 0 and there is a store in slot1 (that
515  * wasn't cancelled), we have to do the store first.
516  */
517 static void check_noshuf(CPUHexagonState *env, uint32_t slot)
518 {
519     if (slot == 0 && env->pkt_has_store_s1 &&
520         ((env->slot_cancelled & (1 << 1)) == 0)) {
521         HELPER(commit_store)(env, 1);
522     }
523 }
524 
525 static uint8_t mem_load1(CPUHexagonState *env, uint32_t slot,
526                          target_ulong vaddr)
527 {
528     uintptr_t ra = GETPC();
529     check_noshuf(env, slot);
530     return cpu_ldub_data_ra(env, vaddr, ra);
531 }
532 
533 static uint16_t mem_load2(CPUHexagonState *env, uint32_t slot,
534                           target_ulong vaddr)
535 {
536     uintptr_t ra = GETPC();
537     check_noshuf(env, slot);
538     return cpu_lduw_data_ra(env, vaddr, ra);
539 }
540 
541 static uint32_t mem_load4(CPUHexagonState *env, uint32_t slot,
542                           target_ulong vaddr)
543 {
544     uintptr_t ra = GETPC();
545     check_noshuf(env, slot);
546     return cpu_ldl_data_ra(env, vaddr, ra);
547 }
548 
549 static uint64_t mem_load8(CPUHexagonState *env, uint32_t slot,
550                           target_ulong vaddr)
551 {
552     uintptr_t ra = GETPC();
553     check_noshuf(env, slot);
554     return cpu_ldq_data_ra(env, vaddr, ra);
555 }
556 
557 /* Floating point */
558 float64 HELPER(conv_sf2df)(CPUHexagonState *env, float32 RsV)
559 {
560     float64 out_f64;
561     arch_fpop_start(env);
562     out_f64 = float32_to_float64(RsV, &env->fp_status);
563     arch_fpop_end(env);
564     return out_f64;
565 }
566 
567 float32 HELPER(conv_df2sf)(CPUHexagonState *env, float64 RssV)
568 {
569     float32 out_f32;
570     arch_fpop_start(env);
571     out_f32 = float64_to_float32(RssV, &env->fp_status);
572     arch_fpop_end(env);
573     return out_f32;
574 }
575 
576 float32 HELPER(conv_uw2sf)(CPUHexagonState *env, int32_t RsV)
577 {
578     float32 RdV;
579     arch_fpop_start(env);
580     RdV = uint32_to_float32(RsV, &env->fp_status);
581     arch_fpop_end(env);
582     return RdV;
583 }
584 
585 float64 HELPER(conv_uw2df)(CPUHexagonState *env, int32_t RsV)
586 {
587     float64 RddV;
588     arch_fpop_start(env);
589     RddV = uint32_to_float64(RsV, &env->fp_status);
590     arch_fpop_end(env);
591     return RddV;
592 }
593 
594 float32 HELPER(conv_w2sf)(CPUHexagonState *env, int32_t RsV)
595 {
596     float32 RdV;
597     arch_fpop_start(env);
598     RdV = int32_to_float32(RsV, &env->fp_status);
599     arch_fpop_end(env);
600     return RdV;
601 }
602 
603 float64 HELPER(conv_w2df)(CPUHexagonState *env, int32_t RsV)
604 {
605     float64 RddV;
606     arch_fpop_start(env);
607     RddV = int32_to_float64(RsV, &env->fp_status);
608     arch_fpop_end(env);
609     return RddV;
610 }
611 
612 float32 HELPER(conv_ud2sf)(CPUHexagonState *env, int64_t RssV)
613 {
614     float32 RdV;
615     arch_fpop_start(env);
616     RdV = uint64_to_float32(RssV, &env->fp_status);
617     arch_fpop_end(env);
618     return RdV;
619 }
620 
621 float64 HELPER(conv_ud2df)(CPUHexagonState *env, int64_t RssV)
622 {
623     float64 RddV;
624     arch_fpop_start(env);
625     RddV = uint64_to_float64(RssV, &env->fp_status);
626     arch_fpop_end(env);
627     return RddV;
628 }
629 
630 float32 HELPER(conv_d2sf)(CPUHexagonState *env, int64_t RssV)
631 {
632     float32 RdV;
633     arch_fpop_start(env);
634     RdV = int64_to_float32(RssV, &env->fp_status);
635     arch_fpop_end(env);
636     return RdV;
637 }
638 
639 float64 HELPER(conv_d2df)(CPUHexagonState *env, int64_t RssV)
640 {
641     float64 RddV;
642     arch_fpop_start(env);
643     RddV = int64_to_float64(RssV, &env->fp_status);
644     arch_fpop_end(env);
645     return RddV;
646 }
647 
648 uint32_t HELPER(conv_sf2uw)(CPUHexagonState *env, float32 RsV)
649 {
650     uint32_t RdV;
651     arch_fpop_start(env);
652     /* Hexagon checks the sign before rounding */
653     if (float32_is_neg(RsV) && !float32_is_any_nan(RsV)) {
654         float_raise(float_flag_invalid, &env->fp_status);
655         RdV = 0;
656     } else {
657         RdV = float32_to_uint32(RsV, &env->fp_status);
658     }
659     arch_fpop_end(env);
660     return RdV;
661 }
662 
663 int32_t HELPER(conv_sf2w)(CPUHexagonState *env, float32 RsV)
664 {
665     int32_t RdV;
666     arch_fpop_start(env);
667     /* Hexagon returns -1 for NaN */
668     if (float32_is_any_nan(RsV)) {
669         float_raise(float_flag_invalid, &env->fp_status);
670         RdV = -1;
671     } else {
672         RdV = float32_to_int32(RsV, &env->fp_status);
673     }
674     arch_fpop_end(env);
675     return RdV;
676 }
677 
678 uint64_t HELPER(conv_sf2ud)(CPUHexagonState *env, float32 RsV)
679 {
680     uint64_t RddV;
681     arch_fpop_start(env);
682     /* Hexagon checks the sign before rounding */
683     if (float32_is_neg(RsV) && !float32_is_any_nan(RsV)) {
684         float_raise(float_flag_invalid, &env->fp_status);
685         RddV = 0;
686     } else {
687         RddV = float32_to_uint64(RsV, &env->fp_status);
688     }
689     arch_fpop_end(env);
690     return RddV;
691 }
692 
693 int64_t HELPER(conv_sf2d)(CPUHexagonState *env, float32 RsV)
694 {
695     int64_t RddV;
696     arch_fpop_start(env);
697     /* Hexagon returns -1 for NaN */
698     if (float32_is_any_nan(RsV)) {
699         float_raise(float_flag_invalid, &env->fp_status);
700         RddV = -1;
701     } else {
702         RddV = float32_to_int64(RsV, &env->fp_status);
703     }
704     arch_fpop_end(env);
705     return RddV;
706 }
707 
708 uint32_t HELPER(conv_df2uw)(CPUHexagonState *env, float64 RssV)
709 {
710     uint32_t RdV;
711     arch_fpop_start(env);
712     /* Hexagon checks the sign before rounding */
713     if (float64_is_neg(RssV) && !float64_is_any_nan(RssV)) {
714         float_raise(float_flag_invalid, &env->fp_status);
715         RdV = 0;
716     } else {
717         RdV = float64_to_uint32(RssV, &env->fp_status);
718     }
719     arch_fpop_end(env);
720     return RdV;
721 }
722 
723 int32_t HELPER(conv_df2w)(CPUHexagonState *env, float64 RssV)
724 {
725     int32_t RdV;
726     arch_fpop_start(env);
727     /* Hexagon returns -1 for NaN */
728     if (float64_is_any_nan(RssV)) {
729         float_raise(float_flag_invalid, &env->fp_status);
730         RdV = -1;
731     } else {
732         RdV = float64_to_int32(RssV, &env->fp_status);
733     }
734     arch_fpop_end(env);
735     return RdV;
736 }
737 
738 uint64_t HELPER(conv_df2ud)(CPUHexagonState *env, float64 RssV)
739 {
740     uint64_t RddV;
741     arch_fpop_start(env);
742     /* Hexagon checks the sign before rounding */
743     if (float64_is_neg(RssV) && !float64_is_any_nan(RssV)) {
744         float_raise(float_flag_invalid, &env->fp_status);
745         RddV = 0;
746     } else {
747         RddV = float64_to_uint64(RssV, &env->fp_status);
748     }
749     arch_fpop_end(env);
750     return RddV;
751 }
752 
753 int64_t HELPER(conv_df2d)(CPUHexagonState *env, float64 RssV)
754 {
755     int64_t RddV;
756     arch_fpop_start(env);
757     /* Hexagon returns -1 for NaN */
758     if (float64_is_any_nan(RssV)) {
759         float_raise(float_flag_invalid, &env->fp_status);
760         RddV = -1;
761     } else {
762         RddV = float64_to_int64(RssV, &env->fp_status);
763     }
764     arch_fpop_end(env);
765     return RddV;
766 }
767 
768 uint32_t HELPER(conv_sf2uw_chop)(CPUHexagonState *env, float32 RsV)
769 {
770     uint32_t RdV;
771     arch_fpop_start(env);
772     /* Hexagon checks the sign before rounding */
773     if (float32_is_neg(RsV) && !float32_is_any_nan(RsV)) {
774         float_raise(float_flag_invalid, &env->fp_status);
775         RdV = 0;
776     } else {
777         RdV = float32_to_uint32_round_to_zero(RsV, &env->fp_status);
778     }
779     arch_fpop_end(env);
780     return RdV;
781 }
782 
783 int32_t HELPER(conv_sf2w_chop)(CPUHexagonState *env, float32 RsV)
784 {
785     int32_t RdV;
786     arch_fpop_start(env);
787     /* Hexagon returns -1 for NaN */
788     if (float32_is_any_nan(RsV)) {
789         float_raise(float_flag_invalid, &env->fp_status);
790         RdV = -1;
791     } else {
792         RdV = float32_to_int32_round_to_zero(RsV, &env->fp_status);
793     }
794     arch_fpop_end(env);
795     return RdV;
796 }
797 
798 uint64_t HELPER(conv_sf2ud_chop)(CPUHexagonState *env, float32 RsV)
799 {
800     uint64_t RddV;
801     arch_fpop_start(env);
802     /* Hexagon checks the sign before rounding */
803     if (float32_is_neg(RsV) && !float32_is_any_nan(RsV)) {
804         float_raise(float_flag_invalid, &env->fp_status);
805         RddV = 0;
806     } else {
807         RddV = float32_to_uint64_round_to_zero(RsV, &env->fp_status);
808     }
809     arch_fpop_end(env);
810     return RddV;
811 }
812 
813 int64_t HELPER(conv_sf2d_chop)(CPUHexagonState *env, float32 RsV)
814 {
815     int64_t RddV;
816     arch_fpop_start(env);
817     /* Hexagon returns -1 for NaN */
818     if (float32_is_any_nan(RsV)) {
819         float_raise(float_flag_invalid, &env->fp_status);
820         RddV = -1;
821     } else {
822         RddV = float32_to_int64_round_to_zero(RsV, &env->fp_status);
823     }
824     arch_fpop_end(env);
825     return RddV;
826 }
827 
828 uint32_t HELPER(conv_df2uw_chop)(CPUHexagonState *env, float64 RssV)
829 {
830     uint32_t RdV;
831     arch_fpop_start(env);
832     /* Hexagon checks the sign before rounding */
833     if (float64_is_neg(RssV) && !float64_is_any_nan(RssV)) {
834         float_raise(float_flag_invalid, &env->fp_status);
835         RdV = 0;
836     } else {
837         RdV = float64_to_uint32_round_to_zero(RssV, &env->fp_status);
838     }
839     arch_fpop_end(env);
840     return RdV;
841 }
842 
843 int32_t HELPER(conv_df2w_chop)(CPUHexagonState *env, float64 RssV)
844 {
845     int32_t RdV;
846     arch_fpop_start(env);
847     /* Hexagon returns -1 for NaN */
848     if (float64_is_any_nan(RssV)) {
849         float_raise(float_flag_invalid, &env->fp_status);
850         RdV = -1;
851     } else {
852         RdV = float64_to_int32_round_to_zero(RssV, &env->fp_status);
853     }
854     arch_fpop_end(env);
855     return RdV;
856 }
857 
858 uint64_t HELPER(conv_df2ud_chop)(CPUHexagonState *env, float64 RssV)
859 {
860     uint64_t RddV;
861     arch_fpop_start(env);
862     /* Hexagon checks the sign before rounding */
863     if (float64_is_neg(RssV) && !float64_is_any_nan(RssV)) {
864         float_raise(float_flag_invalid, &env->fp_status);
865         RddV = 0;
866     } else {
867         RddV = float64_to_uint64_round_to_zero(RssV, &env->fp_status);
868     }
869     arch_fpop_end(env);
870     return RddV;
871 }
872 
873 int64_t HELPER(conv_df2d_chop)(CPUHexagonState *env, float64 RssV)
874 {
875     int64_t RddV;
876     arch_fpop_start(env);
877     /* Hexagon returns -1 for NaN */
878     if (float64_is_any_nan(RssV)) {
879         float_raise(float_flag_invalid, &env->fp_status);
880         RddV = -1;
881     } else {
882         RddV = float64_to_int64_round_to_zero(RssV, &env->fp_status);
883     }
884     arch_fpop_end(env);
885     return RddV;
886 }
887 
888 float32 HELPER(sfadd)(CPUHexagonState *env, float32 RsV, float32 RtV)
889 {
890     float32 RdV;
891     arch_fpop_start(env);
892     RdV = float32_add(RsV, RtV, &env->fp_status);
893     arch_fpop_end(env);
894     return RdV;
895 }
896 
897 float32 HELPER(sfsub)(CPUHexagonState *env, float32 RsV, float32 RtV)
898 {
899     float32 RdV;
900     arch_fpop_start(env);
901     RdV = float32_sub(RsV, RtV, &env->fp_status);
902     arch_fpop_end(env);
903     return RdV;
904 }
905 
906 int32_t HELPER(sfcmpeq)(CPUHexagonState *env, float32 RsV, float32 RtV)
907 {
908     int32_t PdV;
909     arch_fpop_start(env);
910     PdV = f8BITSOF(float32_eq_quiet(RsV, RtV, &env->fp_status));
911     arch_fpop_end(env);
912     return PdV;
913 }
914 
915 int32_t HELPER(sfcmpgt)(CPUHexagonState *env, float32 RsV, float32 RtV)
916 {
917     int cmp;
918     int32_t PdV;
919     arch_fpop_start(env);
920     cmp = float32_compare_quiet(RsV, RtV, &env->fp_status);
921     PdV = f8BITSOF(cmp == float_relation_greater);
922     arch_fpop_end(env);
923     return PdV;
924 }
925 
926 int32_t HELPER(sfcmpge)(CPUHexagonState *env, float32 RsV, float32 RtV)
927 {
928     int cmp;
929     int32_t PdV;
930     arch_fpop_start(env);
931     cmp = float32_compare_quiet(RsV, RtV, &env->fp_status);
932     PdV = f8BITSOF(cmp == float_relation_greater ||
933                    cmp == float_relation_equal);
934     arch_fpop_end(env);
935     return PdV;
936 }
937 
938 int32_t HELPER(sfcmpuo)(CPUHexagonState *env, float32 RsV, float32 RtV)
939 {
940     int32_t PdV;
941     arch_fpop_start(env);
942     PdV = f8BITSOF(float32_unordered_quiet(RsV, RtV, &env->fp_status));
943     arch_fpop_end(env);
944     return PdV;
945 }
946 
947 float32 HELPER(sfmax)(CPUHexagonState *env, float32 RsV, float32 RtV)
948 {
949     float32 RdV;
950     arch_fpop_start(env);
951     RdV = float32_maximum_number(RsV, RtV, &env->fp_status);
952     arch_fpop_end(env);
953     return RdV;
954 }
955 
956 float32 HELPER(sfmin)(CPUHexagonState *env, float32 RsV, float32 RtV)
957 {
958     float32 RdV;
959     arch_fpop_start(env);
960     RdV = float32_minimum_number(RsV, RtV, &env->fp_status);
961     arch_fpop_end(env);
962     return RdV;
963 }
964 
965 int32_t HELPER(sfclass)(CPUHexagonState *env, float32 RsV, int32_t uiV)
966 {
967     int32_t PdV = 0;
968     arch_fpop_start(env);
969     if (fGETBIT(0, uiV) && float32_is_zero(RsV)) {
970         PdV = 0xff;
971     }
972     if (fGETBIT(1, uiV) && float32_is_normal(RsV)) {
973         PdV = 0xff;
974     }
975     if (fGETBIT(2, uiV) && float32_is_denormal(RsV)) {
976         PdV = 0xff;
977     }
978     if (fGETBIT(3, uiV) && float32_is_infinity(RsV)) {
979         PdV = 0xff;
980     }
981     if (fGETBIT(4, uiV) && float32_is_any_nan(RsV)) {
982         PdV = 0xff;
983     }
984     set_float_exception_flags(0, &env->fp_status);
985     arch_fpop_end(env);
986     return PdV;
987 }
988 
989 float32 HELPER(sffixupn)(CPUHexagonState *env, float32 RsV, float32 RtV)
990 {
991     float32 RdV = 0;
992     int adjust;
993     arch_fpop_start(env);
994     arch_sf_recip_common(&RsV, &RtV, &RdV, &adjust, &env->fp_status);
995     RdV = RsV;
996     arch_fpop_end(env);
997     return RdV;
998 }
999 
1000 float32 HELPER(sffixupd)(CPUHexagonState *env, float32 RsV, float32 RtV)
1001 {
1002     float32 RdV = 0;
1003     int adjust;
1004     arch_fpop_start(env);
1005     arch_sf_recip_common(&RsV, &RtV, &RdV, &adjust, &env->fp_status);
1006     RdV = RtV;
1007     arch_fpop_end(env);
1008     return RdV;
1009 }
1010 
1011 float32 HELPER(sffixupr)(CPUHexagonState *env, float32 RsV)
1012 {
1013     float32 RdV = 0;
1014     int adjust;
1015     arch_fpop_start(env);
1016     arch_sf_invsqrt_common(&RsV, &RdV, &adjust, &env->fp_status);
1017     RdV = RsV;
1018     arch_fpop_end(env);
1019     return RdV;
1020 }
1021 
1022 float64 HELPER(dfadd)(CPUHexagonState *env, float64 RssV, float64 RttV)
1023 {
1024     float64 RddV;
1025     arch_fpop_start(env);
1026     RddV = float64_add(RssV, RttV, &env->fp_status);
1027     arch_fpop_end(env);
1028     return RddV;
1029 }
1030 
1031 float64 HELPER(dfsub)(CPUHexagonState *env, float64 RssV, float64 RttV)
1032 {
1033     float64 RddV;
1034     arch_fpop_start(env);
1035     RddV = float64_sub(RssV, RttV, &env->fp_status);
1036     arch_fpop_end(env);
1037     return RddV;
1038 }
1039 
1040 float64 HELPER(dfmax)(CPUHexagonState *env, float64 RssV, float64 RttV)
1041 {
1042     float64 RddV;
1043     arch_fpop_start(env);
1044     RddV = float64_maximum_number(RssV, RttV, &env->fp_status);
1045     arch_fpop_end(env);
1046     return RddV;
1047 }
1048 
1049 float64 HELPER(dfmin)(CPUHexagonState *env, float64 RssV, float64 RttV)
1050 {
1051     float64 RddV;
1052     arch_fpop_start(env);
1053     RddV = float64_minimum_number(RssV, RttV, &env->fp_status);
1054     arch_fpop_end(env);
1055     return RddV;
1056 }
1057 
1058 int32_t HELPER(dfcmpeq)(CPUHexagonState *env, float64 RssV, float64 RttV)
1059 {
1060     int32_t PdV;
1061     arch_fpop_start(env);
1062     PdV = f8BITSOF(float64_eq_quiet(RssV, RttV, &env->fp_status));
1063     arch_fpop_end(env);
1064     return PdV;
1065 }
1066 
1067 int32_t HELPER(dfcmpgt)(CPUHexagonState *env, float64 RssV, float64 RttV)
1068 {
1069     int cmp;
1070     int32_t PdV;
1071     arch_fpop_start(env);
1072     cmp = float64_compare_quiet(RssV, RttV, &env->fp_status);
1073     PdV = f8BITSOF(cmp == float_relation_greater);
1074     arch_fpop_end(env);
1075     return PdV;
1076 }
1077 
1078 int32_t HELPER(dfcmpge)(CPUHexagonState *env, float64 RssV, float64 RttV)
1079 {
1080     int cmp;
1081     int32_t PdV;
1082     arch_fpop_start(env);
1083     cmp = float64_compare_quiet(RssV, RttV, &env->fp_status);
1084     PdV = f8BITSOF(cmp == float_relation_greater ||
1085                    cmp == float_relation_equal);
1086     arch_fpop_end(env);
1087     return PdV;
1088 }
1089 
1090 int32_t HELPER(dfcmpuo)(CPUHexagonState *env, float64 RssV, float64 RttV)
1091 {
1092     int32_t PdV;
1093     arch_fpop_start(env);
1094     PdV = f8BITSOF(float64_unordered_quiet(RssV, RttV, &env->fp_status));
1095     arch_fpop_end(env);
1096     return PdV;
1097 }
1098 
1099 int32_t HELPER(dfclass)(CPUHexagonState *env, float64 RssV, int32_t uiV)
1100 {
1101     int32_t PdV = 0;
1102     arch_fpop_start(env);
1103     if (fGETBIT(0, uiV) && float64_is_zero(RssV)) {
1104         PdV = 0xff;
1105     }
1106     if (fGETBIT(1, uiV) && float64_is_normal(RssV)) {
1107         PdV = 0xff;
1108     }
1109     if (fGETBIT(2, uiV) && float64_is_denormal(RssV)) {
1110         PdV = 0xff;
1111     }
1112     if (fGETBIT(3, uiV) && float64_is_infinity(RssV)) {
1113         PdV = 0xff;
1114     }
1115     if (fGETBIT(4, uiV) && float64_is_any_nan(RssV)) {
1116         PdV = 0xff;
1117     }
1118     set_float_exception_flags(0, &env->fp_status);
1119     arch_fpop_end(env);
1120     return PdV;
1121 }
1122 
1123 float32 HELPER(sfmpy)(CPUHexagonState *env, float32 RsV, float32 RtV)
1124 {
1125     float32 RdV;
1126     arch_fpop_start(env);
1127     RdV = internal_mpyf(RsV, RtV, &env->fp_status);
1128     arch_fpop_end(env);
1129     return RdV;
1130 }
1131 
1132 float32 HELPER(sffma)(CPUHexagonState *env, float32 RxV,
1133                       float32 RsV, float32 RtV)
1134 {
1135     arch_fpop_start(env);
1136     RxV = internal_fmafx(RsV, RtV, RxV, 0, &env->fp_status);
1137     arch_fpop_end(env);
1138     return RxV;
1139 }
1140 
1141 static bool is_zero_prod(float32 a, float32 b)
1142 {
1143     return ((float32_is_zero(a) && is_finite(b)) ||
1144             (float32_is_zero(b) && is_finite(a)));
1145 }
1146 
1147 static float32 check_nan(float32 dst, float32 x, float_status *fp_status)
1148 {
1149     float32 ret = dst;
1150     if (float32_is_any_nan(x)) {
1151         if (extract32(x, 22, 1) == 0) {
1152             float_raise(float_flag_invalid, fp_status);
1153         }
1154         ret = make_float32(0xffffffff);    /* nan */
1155     }
1156     return ret;
1157 }
1158 
1159 float32 HELPER(sffma_sc)(CPUHexagonState *env, float32 RxV,
1160                          float32 RsV, float32 RtV, float32 PuV)
1161 {
1162     size4s_t tmp;
1163     arch_fpop_start(env);
1164     RxV = check_nan(RxV, RxV, &env->fp_status);
1165     RxV = check_nan(RxV, RsV, &env->fp_status);
1166     RxV = check_nan(RxV, RtV, &env->fp_status);
1167     tmp = internal_fmafx(RsV, RtV, RxV, fSXTN(8, 64, PuV), &env->fp_status);
1168     if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) {
1169         RxV = tmp;
1170     }
1171     arch_fpop_end(env);
1172     return RxV;
1173 }
1174 
1175 float32 HELPER(sffms)(CPUHexagonState *env, float32 RxV,
1176                       float32 RsV, float32 RtV)
1177 {
1178     float32 neg_RsV;
1179     arch_fpop_start(env);
1180     neg_RsV = float32_sub(float32_zero, RsV, &env->fp_status);
1181     RxV = internal_fmafx(neg_RsV, RtV, RxV, 0, &env->fp_status);
1182     arch_fpop_end(env);
1183     return RxV;
1184 }
1185 
1186 static bool is_inf_prod(int32_t a, int32_t b)
1187 {
1188     return (float32_is_infinity(a) && float32_is_infinity(b)) ||
1189            (float32_is_infinity(a) && is_finite(b) && !float32_is_zero(b)) ||
1190            (float32_is_infinity(b) && is_finite(a) && !float32_is_zero(a));
1191 }
1192 
1193 float32 HELPER(sffma_lib)(CPUHexagonState *env, float32 RxV,
1194                           float32 RsV, float32 RtV)
1195 {
1196     bool infinp;
1197     bool infminusinf;
1198     float32 tmp;
1199 
1200     arch_fpop_start(env);
1201     set_float_rounding_mode(float_round_nearest_even, &env->fp_status);
1202     infminusinf = float32_is_infinity(RxV) &&
1203                   is_inf_prod(RsV, RtV) &&
1204                   (fGETBIT(31, RsV ^ RxV ^ RtV) != 0);
1205     infinp = float32_is_infinity(RxV) ||
1206              float32_is_infinity(RtV) ||
1207              float32_is_infinity(RsV);
1208     RxV = check_nan(RxV, RxV, &env->fp_status);
1209     RxV = check_nan(RxV, RsV, &env->fp_status);
1210     RxV = check_nan(RxV, RtV, &env->fp_status);
1211     tmp = internal_fmafx(RsV, RtV, RxV, 0, &env->fp_status);
1212     if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) {
1213         RxV = tmp;
1214     }
1215     set_float_exception_flags(0, &env->fp_status);
1216     if (float32_is_infinity(RxV) && !infinp) {
1217         RxV = RxV - 1;
1218     }
1219     if (infminusinf) {
1220         RxV = 0;
1221     }
1222     arch_fpop_end(env);
1223     return RxV;
1224 }
1225 
1226 float32 HELPER(sffms_lib)(CPUHexagonState *env, float32 RxV,
1227                           float32 RsV, float32 RtV)
1228 {
1229     bool infinp;
1230     bool infminusinf;
1231     float32 tmp;
1232 
1233     arch_fpop_start(env);
1234     set_float_rounding_mode(float_round_nearest_even, &env->fp_status);
1235     infminusinf = float32_is_infinity(RxV) &&
1236                   is_inf_prod(RsV, RtV) &&
1237                   (fGETBIT(31, RsV ^ RxV ^ RtV) == 0);
1238     infinp = float32_is_infinity(RxV) ||
1239              float32_is_infinity(RtV) ||
1240              float32_is_infinity(RsV);
1241     RxV = check_nan(RxV, RxV, &env->fp_status);
1242     RxV = check_nan(RxV, RsV, &env->fp_status);
1243     RxV = check_nan(RxV, RtV, &env->fp_status);
1244     float32 minus_RsV = float32_sub(float32_zero, RsV, &env->fp_status);
1245     tmp = internal_fmafx(minus_RsV, RtV, RxV, 0, &env->fp_status);
1246     if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) {
1247         RxV = tmp;
1248     }
1249     set_float_exception_flags(0, &env->fp_status);
1250     if (float32_is_infinity(RxV) && !infinp) {
1251         RxV = RxV - 1;
1252     }
1253     if (infminusinf) {
1254         RxV = 0;
1255     }
1256     arch_fpop_end(env);
1257     return RxV;
1258 }
1259 
1260 float64 HELPER(dfmpyfix)(CPUHexagonState *env, float64 RssV, float64 RttV)
1261 {
1262     int64_t RddV;
1263     arch_fpop_start(env);
1264     if (float64_is_denormal(RssV) &&
1265         (float64_getexp(RttV) >= 512) &&
1266         float64_is_normal(RttV)) {
1267         RddV = float64_mul(RssV, make_float64(0x4330000000000000),
1268                            &env->fp_status);
1269     } else if (float64_is_denormal(RttV) &&
1270                (float64_getexp(RssV) >= 512) &&
1271                float64_is_normal(RssV)) {
1272         RddV = float64_mul(RssV, make_float64(0x3cb0000000000000),
1273                            &env->fp_status);
1274     } else {
1275         RddV = RssV;
1276     }
1277     arch_fpop_end(env);
1278     return RddV;
1279 }
1280 
1281 float64 HELPER(dfmpyhh)(CPUHexagonState *env, float64 RxxV,
1282                         float64 RssV, float64 RttV)
1283 {
1284     arch_fpop_start(env);
1285     RxxV = internal_mpyhh(RssV, RttV, RxxV, &env->fp_status);
1286     arch_fpop_end(env);
1287     return RxxV;
1288 }
1289 
1290 /* Histogram instructions */
1291 
1292 void HELPER(vhist)(CPUHexagonState *env)
1293 {
1294     MMVector *input = &env->tmp_VRegs[0];
1295 
1296     for (int lane = 0; lane < 8; lane++) {
1297         for (int i = 0; i < sizeof(MMVector) / 8; ++i) {
1298             unsigned char value = input->ub[(sizeof(MMVector) / 8) * lane + i];
1299             unsigned char regno = value >> 3;
1300             unsigned char element = value & 7;
1301 
1302             env->VRegs[regno].uh[(sizeof(MMVector) / 16) * lane + element]++;
1303         }
1304     }
1305 }
1306 
1307 void HELPER(vhistq)(CPUHexagonState *env)
1308 {
1309     MMVector *input = &env->tmp_VRegs[0];
1310 
1311     for (int lane = 0; lane < 8; lane++) {
1312         for (int i = 0; i < sizeof(MMVector) / 8; ++i) {
1313             unsigned char value = input->ub[(sizeof(MMVector) / 8) * lane + i];
1314             unsigned char regno = value >> 3;
1315             unsigned char element = value & 7;
1316 
1317             if (fGETQBIT(env->qtmp, sizeof(MMVector) / 8 * lane + i)) {
1318                 env->VRegs[regno].uh[
1319                     (sizeof(MMVector) / 16) * lane + element]++;
1320             }
1321         }
1322     }
1323 }
1324 
1325 void HELPER(vwhist256)(CPUHexagonState *env)
1326 {
1327     MMVector *input = &env->tmp_VRegs[0];
1328 
1329     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1330         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1331         unsigned int weight = fGETUBYTE(1, input->h[i]);
1332         unsigned int vindex = (bucket >> 3) & 0x1F;
1333         unsigned int elindex = ((i >> 0) & (~7)) | ((bucket >> 0) & 7);
1334 
1335         env->VRegs[vindex].uh[elindex] =
1336             env->VRegs[vindex].uh[elindex] + weight;
1337     }
1338 }
1339 
1340 void HELPER(vwhist256q)(CPUHexagonState *env)
1341 {
1342     MMVector *input = &env->tmp_VRegs[0];
1343 
1344     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1345         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1346         unsigned int weight = fGETUBYTE(1, input->h[i]);
1347         unsigned int vindex = (bucket >> 3) & 0x1F;
1348         unsigned int elindex = ((i >> 0) & (~7)) | ((bucket >> 0) & 7);
1349 
1350         if (fGETQBIT(env->qtmp, 2 * i)) {
1351             env->VRegs[vindex].uh[elindex] =
1352                 env->VRegs[vindex].uh[elindex] + weight;
1353         }
1354     }
1355 }
1356 
1357 void HELPER(vwhist256_sat)(CPUHexagonState *env)
1358 {
1359     MMVector *input = &env->tmp_VRegs[0];
1360 
1361     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1362         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1363         unsigned int weight = fGETUBYTE(1, input->h[i]);
1364         unsigned int vindex = (bucket >> 3) & 0x1F;
1365         unsigned int elindex = ((i >> 0) & (~7)) | ((bucket >> 0) & 7);
1366 
1367         env->VRegs[vindex].uh[elindex] =
1368             fVSATUH(env->VRegs[vindex].uh[elindex] + weight);
1369     }
1370 }
1371 
1372 void HELPER(vwhist256q_sat)(CPUHexagonState *env)
1373 {
1374     MMVector *input = &env->tmp_VRegs[0];
1375 
1376     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1377         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1378         unsigned int weight = fGETUBYTE(1, input->h[i]);
1379         unsigned int vindex = (bucket >> 3) & 0x1F;
1380         unsigned int elindex = ((i >> 0) & (~7)) | ((bucket >> 0) & 7);
1381 
1382         if (fGETQBIT(env->qtmp, 2 * i)) {
1383             env->VRegs[vindex].uh[elindex] =
1384                 fVSATUH(env->VRegs[vindex].uh[elindex] + weight);
1385         }
1386     }
1387 }
1388 
1389 void HELPER(vwhist128)(CPUHexagonState *env)
1390 {
1391     MMVector *input = &env->tmp_VRegs[0];
1392 
1393     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1394         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1395         unsigned int weight = fGETUBYTE(1, input->h[i]);
1396         unsigned int vindex = (bucket >> 3) & 0x1F;
1397         unsigned int elindex = ((i >> 1) & (~3)) | ((bucket >> 1) & 3);
1398 
1399         env->VRegs[vindex].uw[elindex] =
1400             env->VRegs[vindex].uw[elindex] + weight;
1401     }
1402 }
1403 
1404 void HELPER(vwhist128q)(CPUHexagonState *env)
1405 {
1406     MMVector *input = &env->tmp_VRegs[0];
1407 
1408     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1409         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1410         unsigned int weight = fGETUBYTE(1, input->h[i]);
1411         unsigned int vindex = (bucket >> 3) & 0x1F;
1412         unsigned int elindex = ((i >> 1) & (~3)) | ((bucket >> 1) & 3);
1413 
1414         if (fGETQBIT(env->qtmp, 2 * i)) {
1415             env->VRegs[vindex].uw[elindex] =
1416                 env->VRegs[vindex].uw[elindex] + weight;
1417         }
1418     }
1419 }
1420 
1421 void HELPER(vwhist128m)(CPUHexagonState *env, int32_t uiV)
1422 {
1423     MMVector *input = &env->tmp_VRegs[0];
1424 
1425     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1426         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1427         unsigned int weight = fGETUBYTE(1, input->h[i]);
1428         unsigned int vindex = (bucket >> 3) & 0x1F;
1429         unsigned int elindex = ((i >> 1) & (~3)) | ((bucket >> 1) & 3);
1430 
1431         if ((bucket & 1) == uiV) {
1432             env->VRegs[vindex].uw[elindex] =
1433                 env->VRegs[vindex].uw[elindex] + weight;
1434         }
1435     }
1436 }
1437 
1438 void HELPER(vwhist128qm)(CPUHexagonState *env, int32_t uiV)
1439 {
1440     MMVector *input = &env->tmp_VRegs[0];
1441 
1442     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1443         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1444         unsigned int weight = fGETUBYTE(1, input->h[i]);
1445         unsigned int vindex = (bucket >> 3) & 0x1F;
1446         unsigned int elindex = ((i >> 1) & (~3)) | ((bucket >> 1) & 3);
1447 
1448         if (((bucket & 1) == uiV) && fGETQBIT(env->qtmp, 2 * i)) {
1449             env->VRegs[vindex].uw[elindex] =
1450                 env->VRegs[vindex].uw[elindex] + weight;
1451         }
1452     }
1453 }
1454 
1455 static void cancel_slot(CPUHexagonState *env, uint32_t slot)
1456 {
1457     HEX_DEBUG_LOG("Slot %d cancelled\n", slot);
1458     env->slot_cancelled |= (1 << slot);
1459 }
1460 
1461 /* These macros can be referenced in the generated helper functions */
1462 #define warn(...) /* Nothing */
1463 #define fatal(...) g_assert_not_reached();
1464 
1465 #define BOGUS_HELPER(tag) \
1466     printf("ERROR: bogus helper: " #tag "\n")
1467 
1468 #include "helper_funcs_generated.c.inc"
1469