68     Double a = { .i = f64 };  in float64_getmant()  local
70         return a.mant | 1ULL << 52;  in float64_getmant()
76         return a.mant;  in float64_getmant()
83     Double a = { .i = f64 };  in float64_getexp()  local
85         return a.exp;  in float64_getexp()
88         return a.exp + 1;  in float64_getexp()
95     Float a = { .i = f32 };  in float32_getmant()  local
97         return a.mant | 1ULL << 23;  in float32_getmant()
103         return a.mant;  in float32_getmant()
110     Float a = { .i = f32 };  in float32_getexp()  local
112         return a.exp;  in float32_getexp()
115         return a.exp + 1;  in float32_getexp()
132     Int128 a, b;  in int128_mul_6464()  local
135     a = int128_make64(ai);  in int128_mul_6464()
137     pp0 = (uint64_t)int128_getw0(a) * (uint64_t)int128_getw0(b);  in int128_mul_6464()
138     pp1a = (uint64_t)int128_getw1(a) * (uint64_t)int128_getw0(b);  in int128_mul_6464()
139     pp1b = (uint64_t)int128_getw1(b) * (uint64_t)int128_getw0(a);  in int128_mul_6464()
140     pp2 = (uint64_t)int128_getw1(a) * (uint64_t)int128_getw1(b);  in int128_mul_6464()
154 static Int128 int128_sub_borrow(Int128 a, Int128 b, int borrow)  in int128_sub_borrow()  argument
156     Int128 ret = int128_sub(a, b);  in int128_sub_borrow()
182 static Accum accum_norm_left(Accum a)  in accum_norm_left()  argument
184     a.exp--;  in accum_norm_left()
185     a.mant = int128_lshift(a.mant, 1);  in accum_norm_left()
186     a.mant = int128_or(a.mant, int128_make64(a.guard));  in accum_norm_left()
187     a.guard = a.round;  in accum_norm_left()
188     a.round = a.sticky;  in accum_norm_left()
189     return a;  in accum_norm_left()
193 static inline Accum accum_norm_right(Accum a, int amt)  in accum_norm_right()  argument
196         a.sticky |=  in accum_norm_right()
197             a.round | a.guard | int128_nz(a.mant);  in accum_norm_right()
198         a.guard = a.round = 0;  in accum_norm_right()
199         a.mant = int128_zero();  in accum_norm_right()
200         a.exp += amt;  in accum_norm_right()
201         return a;  in accum_norm_right()
205         a.sticky |= a.round | a.guard | (int128_getlo(a.mant) != 0);  in accum_norm_right()
206         a.guard = (int128_getlo(a.mant) >> 63) & 1;  in accum_norm_right()
207         a.round = (int128_getlo(a.mant) >> 62) & 1;  in accum_norm_right()
208         a.mant = int128_make64(int128_gethi(a.mant));  in accum_norm_right()
209         a.exp += 64;  in accum_norm_right()
213         a.exp++;  in accum_norm_right()
214         a.sticky |= a.round;  in accum_norm_right()
215         a.round = a.guard;  in accum_norm_right()
216         a.guard = int128_getlo(a.mant) & 1;  in accum_norm_right()
217         a.mant = int128_rshift(a.mant, 1);  in accum_norm_right()
220     return a;  in accum_norm_right()
227 static Accum accum_add(Accum a, Accum b);
229 static Accum accum_sub(Accum a, Accum b, int negate)  in accum_sub()  argument
235     if (a.sign != b.sign) {  in accum_sub()
237         return accum_add(a, b);  in accum_sub()
239     if (b.exp > a.exp) {  in accum_sub()
241         return accum_sub(b, a, !negate);  in accum_sub()
243     if ((b.exp == a.exp) && (int128_gt(b.mant, a.mant))) {  in accum_sub()
245         return accum_sub(b, a, !negate);  in accum_sub()
248     while (a.exp > b.exp) {  in accum_sub()
250         if (int128_gethi(a.mant) & (1ULL << 62)) {  in accum_sub()
254             a = accum_norm_left(a);  in accum_sub()
258     while (a.exp > b.exp) {  in accum_sub()
261         b = accum_norm_right(b, a.exp - b.exp);  in accum_sub()
264     if ((int128_gt(b.mant, a.mant))) {  in accum_sub()
265         return accum_sub(b, a, !negate);  in accum_sub()
269     ret.sign = a.sign;  in accum_sub()
270     ret.exp = a.exp;  in accum_sub()
271     assert(!int128_gt(b.mant, a.mant));  in accum_sub()
273     ret.mant = int128_sub_borrow(a.mant, b.mant, (borrow != 0));  in accum_sub()
284 static Accum accum_add(Accum a, Accum b)  in accum_add()  argument
288     if (a.sign != b.sign) {  in accum_add()
290         return accum_sub(a, b, 0);  in accum_add()
292     if (b.exp > a.exp) {  in accum_add()
294         return accum_add(b, a);  in accum_add()
296     if ((b.exp == a.exp) && int128_gt(b.mant, a.mant)) {  in accum_add()
298         return accum_add(b, a);  in accum_add()
301     while (a.exp > b.exp) {  in accum_add()
303         if (int128_gethi(a.mant) & (1ULL << 62)) {  in accum_add()
307             a = accum_norm_left(a);  in accum_add()
311     while (a.exp > b.exp) {  in accum_add()
314         b = accum_norm_right(b, a.exp - b.exp);  in accum_add()
318     if (int128_gt(b.mant, a.mant)) {  in accum_add()
319         return accum_add(b, a);  in accum_add()
321     ret.sign = a.sign;  in accum_add()
322     ret.exp = a.exp;  in accum_add()
323     assert(!int128_gt(b.mant, a.mant));  in accum_add()
324     ret.mant = int128_add(a.mant, b.mant);  in accum_add()
392 static SUFFIX accum_round_##SUFFIX(Accum a, float_status * fp_status) \
394     if ((int128_gethi(a.mant) == 0) && (int128_getlo(a.mant) == 0) \
395         && ((a.guard | a.round | a.sticky) == 0)) { \
409     while ((int128_gethi(a.mant) != 0) || \
410            ((int128_getlo(a.mant) >> (MANTBITS + 1)) != 0)) { \
411         a = accum_norm_right(a, 1); \
420     while ((int128_getlo(a.mant) & (1ULL << MANTBITS)) == 0) { \
421         a = accum_norm_left(a); \
428     while (a.exp <= 0) { \
429         a = accum_norm_right(a, 1 - a.exp); \
435         if (a.guard || a.round || a.sticky) { \
440     if (a.guard || a.round || a.sticky) { \
447             if (a.sign == 0) { \
448                 a.mant = int128_add(a.mant, int128_one()); \
452             if (a.sign != 0) { \
453                 a.mant = int128_add(a.mant, int128_one()); \
457             if (a.round || a.sticky) { \
459                 a.mant = int128_add(a.mant, int128_make64(a.guard)); \
460             } else if (a.guard) { \
462                 a.mant = int128_add(a.mant, int128_and(a.mant, int128_one())); \
473     if ((int128_getlo(a.mant) >> (MANTBITS + 1)) != 0) { \
474         a = accum_norm_right(a, 1); \
477     if (a.exp >= INF_EXP) { \
483             return maxfinite_##SUFFIX(a.sign); \
485             if (a.sign == 0) { \
486                 return infinite_##SUFFIX(a.sign); \
488                 return maxfinite_##SUFFIX(a.sign); \
491             if (a.sign != 0) { \
492                 return infinite_##SUFFIX(a.sign); \
494                 return maxfinite_##SUFFIX(a.sign); \
497             return infinite_##SUFFIX(a.sign); \
501     if (int128_getlo(a.mant) & (1ULL << MANTBITS)) { \
505         ret.sign = a.sign; \
506         ret.exp = a.exp; \
507         ret.mant = int128_getlo(a.mant); \
510     assert(a.exp == 1); \
513     ret.sign = a.sign; \
515     ret.mant = int128_getlo(a.mant); \
522 static bool is_inf_prod(float64 a, float64 b)  in GEN_XF_ROUND()
524     return ((float64_is_infinity(a) && float64_is_infinity(b)) ||  in GEN_XF_ROUND()
525             (float64_is_infinity(a) && is_finite(b) && (!float64_is_zero(b))) ||  in GEN_XF_ROUND()
526             (float64_is_infinity(b) && is_finite(a) && (!float64_is_zero(a))));  in GEN_XF_ROUND()
529 static float64 special_fma(float64 a, float64 b, float64 c,  in special_fma()  argument
538     uint8_t a_sign = float64_is_neg(a);  in special_fma()
541     if (is_inf_prod(a, b) && float64_is_infinity(c)) {  in special_fma()
548     if ((float64_is_infinity(a) && float64_is_zero(b)) ||  in special_fma()
549         (float64_is_zero(a) && float64_is_infinity(b))) {  in special_fma()
559     if (float64_is_any_nan(a) ||  in special_fma()
562         if (float64_is_any_nan(a) && (fGETBIT(51, a) == 0)) {  in special_fma()
582     if (float64_is_infinity(a) || float64_is_infinity(b)) {  in special_fma()
589 static float32 special_fmaf(float32 a, float32 b, float32 c,  in special_fmaf()  argument
593     aa = float32_to_float64(a, fp_status);  in special_fmaf()
599 float32 internal_fmafx(float32 a, float32 b, float32 c, int scale,  in internal_fmafx()  argument
609     uint8_t a_sign = float32_is_neg(a);  in internal_fmafx()
612     if (float32_is_infinity(a) ||  in internal_fmafx()
615         return special_fmaf(a, b, c, fp_status);  in internal_fmafx()
617     if (float32_is_any_nan(a) ||  in internal_fmafx()
620         return special_fmaf(a, b, c, fp_status);  in internal_fmafx()
622     if ((scale == 0) && (float32_is_zero(a) || float32_is_zero(b))) {  in internal_fmafx()
623         float32 tmp = float32_mul(a, b, fp_status);  in internal_fmafx()
629     prod.mant = int128_mul_6464(float32_getmant(a), float32_getmant(b));  in internal_fmafx()
635     prod.exp = float32_getexp(a) + float32_getexp(b) - SF_BIAS - 23;  in internal_fmafx()
637     if (float32_is_zero(a) || float32_is_zero(b)) {  in internal_fmafx()
658 float32 internal_mpyf(float32 a, float32 b, float_status *fp_status)  in internal_mpyf()  argument
660     if (float32_is_zero(a) || float32_is_zero(b)) {  in internal_mpyf()
661         return float32_mul(a, b, fp_status);  in internal_mpyf()
663     return internal_fmafx(a, b, float32_zero, 0, fp_status);  in internal_mpyf()
666 float64 internal_mpyhh(float64 a, float64 b,  in internal_mpyhh()  argument
678     if (float64_is_zero(a) ||  in internal_mpyhh()
679         float64_is_any_nan(a) ||  in internal_mpyhh()
680         float64_is_infinity(a)) {  in internal_mpyhh()
681         return float64_mul(a, b, fp_status);  in internal_mpyhh()
686         return float64_mul(a, b, fp_status);  in internal_mpyhh()
690     prod = fGETUWORD(1, float64_getmant(a)) * fGETUWORD(1, float64_getmant(b));  in internal_mpyhh()
692     x.exp = float64_getexp(a) + float64_getexp(b) - DF_BIAS - 20;  in internal_mpyhh()
693     if (!float64_is_normal(a) || !float64_is_normal(b)) {  in internal_mpyhh()
698     a_sign = float64_is_neg(a);  in internal_mpyhh()