Lines Matching full:pg
51 /* This is an iterative function, called for each Pd and Pg word
73 /* This is an iterative function, called for each Pd and Pg word
183 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \ in LOGICAL_PPPP()
185 if (pg & 1) { \ in LOGICAL_PPPP()
190 i += sizeof(TYPE), pg >>= sizeof(TYPE); \ in LOGICAL_PPPP()
201 uint8_t *pg = vg; \
203 if (pg[H1(i)] & 1) { \
668 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \ in DO_ZPZZ()
674 if (pg & 1) { \ in DO_ZPZZ()
677 i += sizeof(TYPE), pg >>= sizeof(TYPE); \ in DO_ZPZZ()
678 if (pg & 1) { \ in DO_ZPZZ()
681 i += sizeof(TYPE), pg >>= sizeof(TYPE); \ in DO_ZPZZ()
692 uint8_t *pg = vg; \
696 if (pg[H1(i)] & 1) { \
699 if (pg[H1(i + 1)] & 1) { \
739 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
745 if (pg & 1) { \
748 i += sizeof(TYPE), pg >>= sizeof(TYPE); \
749 if (pg & 1) { \
752 i += sizeof(TYPE), pg >>= sizeof(TYPE); \
788 uint8_t pg = *(uint8_t *)(vg + H1(i >> 3)); \
791 if (pg & 1) { \
795 i += sizeof(TYPE), pg >>= sizeof(TYPE); \
821 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
823 if (pg & 1) { \
827 i += sizeof(TYPE), pg >>= sizeof(TYPE); \
838 uint8_t *pg = vg; \
840 if (pg[H1(i)] & 1) { \
958 uint8_t *pg = vg; local
961 if (pg[H1(i)] & 1) {
1748 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
1750 if (pg & 1) { \
1754 i += sizeof(TYPEELT), pg >>= sizeof(TYPEELT); \
1765 uint8_t *pg = vg; \
1768 if (pg[H1(i)] & 1) { \
1831 uint16_t pg = g[H2(s)]; \
1832 for (uintptr_t e = 0; e < nsegelt; e++, pg >>= sizeof(TYPE)) { \
1833 if (pg & 1) { \
2055 uint8_t *pg = vg; in HELPER() local
2058 d[i] = n[i] & (expand_pred_b(pg[H1(i)]) ^ inv); in HELPER()
2067 uint8_t *pg = vg; in HELPER() local
2070 d[i] = n[i] & (expand_pred_h(pg[H1(i)]) ^ inv); in HELPER()
2079 uint8_t *pg = vg; in HELPER() local
2082 d[i] = n[i] & (expand_pred_s(pg[H1(i)]) ^ inv); in HELPER()
2090 uint8_t *pg = vg; in HELPER() local
2094 d[i] = n[i] & -(uint64_t)((pg[H1(i)] ^ inv) & 1); in HELPER()
2106 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
2108 if (pg & 1) { \
2112 i += sizeof(TYPE), pg >>= sizeof(TYPE); \
2124 uint8_t *pg = vg; \
2126 if (pg[H1(i)] & 1) { \
2387 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
2389 if (pg & 1) { \
2395 i += sizeof(TYPE), pg >>= sizeof(TYPE); \
2407 uint8_t *pg = vg; \
2409 if (pg[H1(i)] & 1) { \
2757 uint8_t *pg = vg; in HELPER() local
2762 uint64_t pp = expand_pred_b(pg[H1(i)]); in HELPER()
2772 uint8_t *pg = vg; in HELPER() local
2777 uint64_t pp = expand_pred_h(pg[H1(i)]); in HELPER()
2787 uint8_t *pg = vg; in HELPER() local
2792 uint64_t pp = expand_pred_s(pg[H1(i)]); in HELPER()
2802 uint8_t *pg = vg; in HELPER() local
2806 d[i] = (pg[H1(i)] & 1 ? mm : nn); in HELPER()
2814 uint8_t *pg = vg; in HELPER() local
2818 d[i] = val & expand_pred_b(pg[H1(i)]); in HELPER()
2826 uint8_t *pg = vg; in HELPER() local
2830 d[i] = val & expand_pred_h(pg[H1(i)]); in HELPER()
2838 uint8_t *pg = vg; in HELPER() local
2842 d[i] = val & expand_pred_s(pg[H1(i)]); in HELPER()
2850 uint8_t *pg = vg; in HELPER() local
2853 d[i] = (pg[H1(i)] & 1 ? val : 0); in HELPER()
3623 uint8_t *pg = vg; local
3626 if (pg[H1(i / 2)] & (i & 1 ? 0x10 : 0x01)) {
3640 uint8_t *pg = vg; in HELPER() local
3643 if (pg[H1(i)] & 1) { in HELPER()
3669 uint64_t pg, first_g, last_g, len, mask = pred_esz_masks[esz]; in HELPER() local
3678 pg = *(uint64_t *)(vg + i) & mask; in HELPER()
3679 if (pg) { in HELPER()
3681 last_g = pg; in HELPER()
3684 first_g = pg; in HELPER()
3707 uint8_t *pg = vg; in HELPER() local
3711 uint64_t pp = expand_pred_b(pg[H1(i)]); in HELPER()
3721 uint8_t *pg = vg; in HELPER() local
3725 uint64_t pp = expand_pred_h(pg[H1(i)]); in HELPER()
3735 uint8_t *pg = vg; in HELPER() local
3739 uint64_t pp = expand_pred_s(pg[H1(i)]); in HELPER()
3749 uint8_t *pg = vg; in HELPER() local
3753 d[i] = (pg[H1(i)] & 1 ? nn : mm); in HELPER()
3762 uint16_t *pg = vg; in HELPER() local
3765 d[i] = (pg[H2(i)] & 1 ? n : m)[i]; in HELPER()
3776 * and pg, %eax
3784 * and v0.8b, pg
3797 uint64_t out = 0, pg; \
3804 pg = *(uint64_t *)(vg + (i >> 3)) & MASK; \
3805 out &= pg; \
3807 flags = iter_predtest_bwd(out, pg, flags); \
3865 uint64_t out = 0, pg; \
3874 pg = *(uint64_t *)(vg + (i >> 3)) & MASK; \
3875 out &= pg; \
3877 flags = iter_predtest_bwd(out, pg, flags); \
3943 uint64_t out = 0, pg; \
3949 pg = *(uint64_t *)(vg + (i >> 3)) & MASK; \
3950 out &= pg; \
3952 flags = iter_predtest_bwd(out, pg, flags); \
4028 uint64_t pg = *(uint64_t *)(vg + i); in last_active_pred() local
4029 if (pg) { in last_active_pred()
4030 return (pow2floor(pg) & *(uint64_t *)(vd + i)) != 0; in last_active_pred()
4493 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
4496 *(TYPE *)((void *)data + i) = (pg & 1 ? nn : ident); \
4497 i += sizeof(TYPE), pg >>= sizeof(TYPE); \
4513 uint16_t pg = *(uint16_t *)(vg + H1_2(s * 2)); \
4515 data[s] = (pg >> e) & 1 ? nn : ident; \
4566 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); in DO_REDUCE() local
4568 if (pg & 1) { in DO_REDUCE()
4572 i += sizeof(float16), pg >>= sizeof(float16); in DO_REDUCE()
4586 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); in HELPER() local
4588 if (pg & 1) { in HELPER()
4592 i += sizeof(float32), pg >>= sizeof(float32); in HELPER()
4604 uint8_t *pg = vg; in HELPER() local
4607 if (pg[H1(i)] & 1) { in HELPER()
4625 uint64_t pg = g[(i - 1) >> 6]; \
4628 if (likely((pg >> (i & 63)) & 1)) { \
4754 uint64_t pg = g[(i - 1) >> 6]; \ in DO_ZPZZ_FP()
4757 if (likely((pg >> (i & 63)) & 1)) { \ in DO_ZPZZ_FP()
4830 uint64_t pg = g[(i - 1) >> 6]; \ in DO_ZPZS_FP()
4833 if (likely((pg >> (i & 63)) & 1)) { \ in DO_ZPZS_FP()
5115 uint64_t pg = g[(i - 1) >> 6]; in DO_ZPZ_FP() local
5118 if (likely((pg >> (i & 63)) & 1)) { in DO_ZPZ_FP()
5184 uint64_t pg = g[(i - 1) >> 6]; in do_fmla_zpzzz_h() local
5187 if (likely((pg >> (i & 63)) & 1)) { in do_fmla_zpzzz_h()
5253 uint64_t pg = g[(i - 1) >> 6]; in do_fmla_zpzzz_s() local
5256 if (likely((pg >> (i & 63)) & 1)) { in do_fmla_zpzzz_s()
5322 uint64_t pg = g[(i - 1) >> 6]; in do_fmla_zpzzz_d() local
5325 if (likely((pg >> (i & 63)) & 1)) { in do_fmla_zpzzz_d()
5395 uint64_t out = 0, pg = g[j]; \
5398 if (likely((pg >> (i & 63)) & 1)) { \
5457 uint64_t out = 0, pg = g[j]; \ in DO_FPCMP_PPZZ_ALL()
5460 if ((pg >> (i & 63)) & 1) { \ in DO_FPCMP_PPZZ_ALL()
5598 uint64_t pg = g[(i - 1) >> 6]; in HELPER() local
5617 if (likely((pg >> (i & 63)) & 1)) { in HELPER()
5620 if (likely((pg >> (j & 63)) & 1)) { in HELPER()
5636 uint64_t pg = g[(i - 1) >> 6]; in HELPER() local
5655 if (likely((pg >> (i & 63)) & 1)) { in HELPER()
5658 if (likely((pg >> (j & 63)) & 1)) { in HELPER()
5674 uint64_t pg = g[(i - 1) >> 6]; in HELPER() local
5693 if (likely((pg >> (i & 63)) & 1)) { in HELPER()
5696 if (likely((pg >> (j & 63)) & 1)) { in HELPER()
5725 uint64_t pg = g[(i - 1) >> 6]; in HELPER() local
5743 if (likely((pg >> (i & 63)) & 1)) { in HELPER()
5748 if (likely((pg >> (j & 63)) & 1)) { in HELPER()
5775 uint64_t pg = g[(i - 1) >> 6]; in HELPER() local
5793 if (likely((pg >> (i & 63)) & 1)) { in HELPER()
5798 if (likely((pg >> (j & 63)) & 1)) { in HELPER()
5825 uint64_t pg = g[(i - 1) >> 6]; in HELPER() local
5843 if (likely((pg >> (i & 63)) & 1)) { in HELPER()
5848 if (likely((pg >> (j & 63)) & 1)) { in HELPER()
5870 uint64_t pg = (vg[reg_off >> 6] & pg_mask) >> (reg_off & 63); in find_next_active() local
5873 if (likely(pg & 1)) { in find_next_active()
5877 if (pg == 0) { in find_next_active()
5885 pg = vg[reg_off >> 6] & pg_mask; in find_next_active()
5886 } while (pg == 0); in find_next_active()
5888 reg_off += ctz64(pg); in find_next_active()
5972 uint64_t pg = vg[i] & pg_mask; in sve_cont_ldst_elements() local
5973 if (pg) { in sve_cont_ldst_elements()
5974 reg_off_last = i * 64 + 63 - clz64(pg); in sve_cont_ldst_elements()
5976 reg_off_first = i * 64 + ctz64(pg); in sve_cont_ldst_elements()
6128 uint64_t pg = vg[reg_off >> 6]; in sve_cont_ldst_watchpoints() local
6130 if ((pg >> (reg_off & 63)) & 1) { in sve_cont_ldst_watchpoints()
6153 uint64_t pg = vg[reg_off >> 6]; in sve_cont_ldst_watchpoints() local
6155 if ((pg >> (reg_off & 63)) & 1) { in sve_cont_ldst_watchpoints()
6184 uint64_t pg = vg[reg_off >> 6]; in sve_cont_ldst_mte_check() local
6186 if ((pg >> (reg_off & 63)) & 1) { in sve_cont_ldst_mte_check()
6201 uint64_t pg = vg[reg_off >> 6]; in sve_cont_ldst_mte_check() local
6203 if ((pg >> (reg_off & 63)) & 1) { in sve_cont_ldst_mte_check()
6276 uint64_t pg = vg[reg_off >> 6]; in sve_ldN_r() local
6278 if ((pg >> (reg_off & 63)) & 1) { in sve_ldN_r()
6309 uint64_t pg = vg[reg_off >> 6]; in sve_ldN_r() local
6311 if ((pg >> (reg_off & 63)) & 1) { in sve_ldN_r()
6346 uint64_t pg = vg[reg_off >> 6]; in sve_ldN_r() local
6348 if ((pg >> (reg_off & 63)) & 1) { in sve_ldN_r()
6672 uint64_t pg = *(uint64_t *)(vg + (reg_off >> 3)); in sve_ldnfff1_r() local
6674 if ((pg >> (reg_off & 63)) & 1) { in sve_ldnfff1_r()
6903 uint64_t pg = vg[reg_off >> 6]; in DO_LDFF1_LDNF1_1() local
6905 if ((pg >> (reg_off & 63)) & 1) { in DO_LDFF1_LDNF1_1()
6926 uint64_t pg = vg[reg_off >> 6]; in DO_LDFF1_LDNF1_1() local
6928 if ((pg >> (reg_off & 63)) & 1) { in DO_LDFF1_LDNF1_1()
6963 uint64_t pg = vg[reg_off >> 6]; in DO_LDFF1_LDNF1_1() local
6965 if ((pg >> (reg_off & 63)) & 1) { in DO_LDFF1_LDNF1_1()
7127 uint64_t pg = vg[reg_off >> 6]; in sve_ld1_z() local
7129 if (likely(pg & 1)) { in sve_ld1_z()
7167 pg >>= esize; in sve_ld1_z()
7361 uint64_t pg = vg[reg_off >> 6]; in DO_LD1_ZPZ_S() local
7363 if (likely((pg >> (reg_off & 63)) & 1)) { in DO_LD1_ZPZ_S()
7537 uint64_t pg = vg[reg_off >> 6]; in DO_LDFF1_ZPZ_S() local
7543 if (likely((pg >> (reg_off & 63)) & 1)) { in DO_LDFF1_ZPZ_S()
8281 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)) & esz_mask; in do_match() local
8288 if (pg & (1 << (j + k))) { in do_match()
8295 flags = iter_predtest_fwd(out, pg, flags); in do_match()
8321 uint8_t *pg = vg; in DO_PPZZ_MATCH() local
8336 pred = pg[H1(i >> 3)] >> (i & 7); in DO_PPZZ_MATCH()
8341 pred = pg[H1(j >> 3)] >> (j & 7); in DO_PPZZ_MATCH()
8358 uint8_t *pg = vg; in HELPER() local
8371 if (pg[H1(i)] & 1) { in HELPER()
8374 if ((pg[H1(j)] & 1) && nn == m[j]) { in HELPER()
8565 uint64_t pg = g[(i - 1) >> 6]; \
8568 if (likely((pg >> (i & 63)) & 1)) { \
8587 uint64_t pg = g[(i - 1) >> 6]; \ in DO_FCVTNT()
8590 if (likely((pg >> (i & 63)) & 1)) { \ in DO_FCVTNT()