16253c22eSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-or-later */
21da177e4SLinus Torvalds /* multi_arith.h: multi-precision integer arithmetic functions, needed
31da177e4SLinus Torvalds to do extended-precision floating point.
41da177e4SLinus Torvalds
51da177e4SLinus Torvalds (c) 1998 David Huggins-Daines.
61da177e4SLinus Torvalds
71da177e4SLinus Torvalds Somewhat based on arch/alpha/math-emu/ieee-math.c, which is (c)
81da177e4SLinus Torvalds David Mosberger-Tang.
91da177e4SLinus Torvalds
106253c22eSThomas Gleixner */
111da177e4SLinus Torvalds
121da177e4SLinus Torvalds /* Note:
131da177e4SLinus Torvalds
141da177e4SLinus Torvalds These are not general multi-precision math routines. Rather, they
151da177e4SLinus Torvalds implement the subset of integer arithmetic that we need in order to
161da177e4SLinus Torvalds multiply, divide, and normalize 128-bit unsigned mantissae. */
171da177e4SLinus Torvalds
181da177e4SLinus Torvalds #ifndef MULTI_ARITH_H
191da177e4SLinus Torvalds #define MULTI_ARITH_H
201da177e4SLinus Torvalds
fp_denormalize(struct fp_ext * reg,unsigned int cnt)211da177e4SLinus Torvalds static inline void fp_denormalize(struct fp_ext *reg, unsigned int cnt)
221da177e4SLinus Torvalds {
231da177e4SLinus Torvalds reg->exp += cnt;
241da177e4SLinus Torvalds
251da177e4SLinus Torvalds switch (cnt) {
261da177e4SLinus Torvalds case 0 ... 8:
271da177e4SLinus Torvalds reg->lowmant = reg->mant.m32[1] << (8 - cnt);
281da177e4SLinus Torvalds reg->mant.m32[1] = (reg->mant.m32[1] >> cnt) |
291da177e4SLinus Torvalds (reg->mant.m32[0] << (32 - cnt));
301da177e4SLinus Torvalds reg->mant.m32[0] = reg->mant.m32[0] >> cnt;
311da177e4SLinus Torvalds break;
321da177e4SLinus Torvalds case 9 ... 32:
331da177e4SLinus Torvalds reg->lowmant = reg->mant.m32[1] >> (cnt - 8);
341da177e4SLinus Torvalds if (reg->mant.m32[1] << (40 - cnt))
351da177e4SLinus Torvalds reg->lowmant |= 1;
361da177e4SLinus Torvalds reg->mant.m32[1] = (reg->mant.m32[1] >> cnt) |
371da177e4SLinus Torvalds (reg->mant.m32[0] << (32 - cnt));
381da177e4SLinus Torvalds reg->mant.m32[0] = reg->mant.m32[0] >> cnt;
391da177e4SLinus Torvalds break;
401da177e4SLinus Torvalds case 33 ... 39:
411da177e4SLinus Torvalds asm volatile ("bfextu %1{%2,#8},%0" : "=d" (reg->lowmant)
421da177e4SLinus Torvalds : "m" (reg->mant.m32[0]), "d" (64 - cnt));
431da177e4SLinus Torvalds if (reg->mant.m32[1] << (40 - cnt))
441da177e4SLinus Torvalds reg->lowmant |= 1;
451da177e4SLinus Torvalds reg->mant.m32[1] = reg->mant.m32[0] >> (cnt - 32);
461da177e4SLinus Torvalds reg->mant.m32[0] = 0;
471da177e4SLinus Torvalds break;
481da177e4SLinus Torvalds case 40 ... 71:
491da177e4SLinus Torvalds reg->lowmant = reg->mant.m32[0] >> (cnt - 40);
501da177e4SLinus Torvalds if ((reg->mant.m32[0] << (72 - cnt)) || reg->mant.m32[1])
511da177e4SLinus Torvalds reg->lowmant |= 1;
521da177e4SLinus Torvalds reg->mant.m32[1] = reg->mant.m32[0] >> (cnt - 32);
531da177e4SLinus Torvalds reg->mant.m32[0] = 0;
541da177e4SLinus Torvalds break;
551da177e4SLinus Torvalds default:
561da177e4SLinus Torvalds reg->lowmant = reg->mant.m32[0] || reg->mant.m32[1];
571da177e4SLinus Torvalds reg->mant.m32[0] = 0;
581da177e4SLinus Torvalds reg->mant.m32[1] = 0;
591da177e4SLinus Torvalds break;
601da177e4SLinus Torvalds }
611da177e4SLinus Torvalds }
621da177e4SLinus Torvalds
fp_overnormalize(struct fp_ext * reg)631da177e4SLinus Torvalds static inline int fp_overnormalize(struct fp_ext *reg)
641da177e4SLinus Torvalds {
651da177e4SLinus Torvalds int shift;
661da177e4SLinus Torvalds
671da177e4SLinus Torvalds if (reg->mant.m32[0]) {
681da177e4SLinus Torvalds asm ("bfffo %1{#0,#32},%0" : "=d" (shift) : "dm" (reg->mant.m32[0]));
691da177e4SLinus Torvalds reg->mant.m32[0] = (reg->mant.m32[0] << shift) | (reg->mant.m32[1] >> (32 - shift));
701da177e4SLinus Torvalds reg->mant.m32[1] = (reg->mant.m32[1] << shift);
711da177e4SLinus Torvalds } else {
721da177e4SLinus Torvalds asm ("bfffo %1{#0,#32},%0" : "=d" (shift) : "dm" (reg->mant.m32[1]));
731da177e4SLinus Torvalds reg->mant.m32[0] = (reg->mant.m32[1] << shift);
741da177e4SLinus Torvalds reg->mant.m32[1] = 0;
751da177e4SLinus Torvalds shift += 32;
761da177e4SLinus Torvalds }
771da177e4SLinus Torvalds
781da177e4SLinus Torvalds return shift;
791da177e4SLinus Torvalds }
801da177e4SLinus Torvalds
fp_addmant(struct fp_ext * dest,struct fp_ext * src)811da177e4SLinus Torvalds static inline int fp_addmant(struct fp_ext *dest, struct fp_ext *src)
821da177e4SLinus Torvalds {
831da177e4SLinus Torvalds int carry;
841da177e4SLinus Torvalds
851da177e4SLinus Torvalds /* we assume here, gcc only insert move and a clr instr */
861da177e4SLinus Torvalds asm volatile ("add.b %1,%0" : "=d,g" (dest->lowmant)
871da177e4SLinus Torvalds : "g,d" (src->lowmant), "0,0" (dest->lowmant));
881da177e4SLinus Torvalds asm volatile ("addx.l %1,%0" : "=d" (dest->mant.m32[1])
891da177e4SLinus Torvalds : "d" (src->mant.m32[1]), "0" (dest->mant.m32[1]));
901da177e4SLinus Torvalds asm volatile ("addx.l %1,%0" : "=d" (dest->mant.m32[0])
911da177e4SLinus Torvalds : "d" (src->mant.m32[0]), "0" (dest->mant.m32[0]));
921da177e4SLinus Torvalds asm volatile ("addx.l %0,%0" : "=d" (carry) : "0" (0));
931da177e4SLinus Torvalds
941da177e4SLinus Torvalds return carry;
951da177e4SLinus Torvalds }
961da177e4SLinus Torvalds
fp_addcarry(struct fp_ext * reg)971da177e4SLinus Torvalds static inline int fp_addcarry(struct fp_ext *reg)
981da177e4SLinus Torvalds {
991da177e4SLinus Torvalds if (++reg->exp == 0x7fff) {
1001da177e4SLinus Torvalds if (reg->mant.m64)
1011da177e4SLinus Torvalds fp_set_sr(FPSR_EXC_INEX2);
1021da177e4SLinus Torvalds reg->mant.m64 = 0;
1031da177e4SLinus Torvalds fp_set_sr(FPSR_EXC_OVFL);
1041da177e4SLinus Torvalds return 0;
1051da177e4SLinus Torvalds }
1061da177e4SLinus Torvalds reg->lowmant = (reg->mant.m32[1] << 7) | (reg->lowmant ? 1 : 0);
1071da177e4SLinus Torvalds reg->mant.m32[1] = (reg->mant.m32[1] >> 1) |
1081da177e4SLinus Torvalds (reg->mant.m32[0] << 31);
1091da177e4SLinus Torvalds reg->mant.m32[0] = (reg->mant.m32[0] >> 1) | 0x80000000;
1101da177e4SLinus Torvalds
1111da177e4SLinus Torvalds return 1;
1121da177e4SLinus Torvalds }
1131da177e4SLinus Torvalds
fp_submant(struct fp_ext * dest,struct fp_ext * src1,struct fp_ext * src2)1141da177e4SLinus Torvalds static inline void fp_submant(struct fp_ext *dest, struct fp_ext *src1,
1151da177e4SLinus Torvalds struct fp_ext *src2)
1161da177e4SLinus Torvalds {
1171da177e4SLinus Torvalds /* we assume here, gcc only insert move and a clr instr */
1181da177e4SLinus Torvalds asm volatile ("sub.b %1,%0" : "=d,g" (dest->lowmant)
1191da177e4SLinus Torvalds : "g,d" (src2->lowmant), "0,0" (src1->lowmant));
1201da177e4SLinus Torvalds asm volatile ("subx.l %1,%0" : "=d" (dest->mant.m32[1])
1211da177e4SLinus Torvalds : "d" (src2->mant.m32[1]), "0" (src1->mant.m32[1]));
1221da177e4SLinus Torvalds asm volatile ("subx.l %1,%0" : "=d" (dest->mant.m32[0])
1231da177e4SLinus Torvalds : "d" (src2->mant.m32[0]), "0" (src1->mant.m32[0]));
1241da177e4SLinus Torvalds }
1251da177e4SLinus Torvalds
1261da177e4SLinus Torvalds #define fp_mul64(desth, destl, src1, src2) ({ \
1271da177e4SLinus Torvalds asm ("mulu.l %2,%1:%0" : "=d" (destl), "=d" (desth) \
1287f263338SAl Viro : "dm" (src1), "0" (src2)); \
1291da177e4SLinus Torvalds })
1301da177e4SLinus Torvalds #define fp_div64(quot, rem, srch, srcl, div) \
1311da177e4SLinus Torvalds asm ("divu.l %2,%1:%0" : "=d" (quot), "=d" (rem) \
1321da177e4SLinus Torvalds : "dm" (div), "1" (srch), "0" (srcl))
1331da177e4SLinus Torvalds #define fp_add64(dest1, dest2, src1, src2) ({ \
1341da177e4SLinus Torvalds asm ("add.l %1,%0" : "=d,dm" (dest2) \
1351da177e4SLinus Torvalds : "dm,d" (src2), "0,0" (dest2)); \
1361da177e4SLinus Torvalds asm ("addx.l %1,%0" : "=d" (dest1) \
1371da177e4SLinus Torvalds : "d" (src1), "0" (dest1)); \
1381da177e4SLinus Torvalds })
1391da177e4SLinus Torvalds #define fp_addx96(dest, src) ({ \
1401da177e4SLinus Torvalds /* we assume here, gcc only insert move and a clr instr */ \
1411da177e4SLinus Torvalds asm volatile ("add.l %1,%0" : "=d,g" (dest->m32[2]) \
1421da177e4SLinus Torvalds : "g,d" (temp.m32[1]), "0,0" (dest->m32[2])); \
1431da177e4SLinus Torvalds asm volatile ("addx.l %1,%0" : "=d" (dest->m32[1]) \
1441da177e4SLinus Torvalds : "d" (temp.m32[0]), "0" (dest->m32[1])); \
1451da177e4SLinus Torvalds asm volatile ("addx.l %1,%0" : "=d" (dest->m32[0]) \
1461da177e4SLinus Torvalds : "d" (0), "0" (dest->m32[0])); \
1471da177e4SLinus Torvalds })
1481da177e4SLinus Torvalds #define fp_sub64(dest, src) ({ \
1491da177e4SLinus Torvalds asm ("sub.l %1,%0" : "=d,dm" (dest.m32[1]) \
1501da177e4SLinus Torvalds : "dm,d" (src.m32[1]), "0,0" (dest.m32[1])); \
1511da177e4SLinus Torvalds asm ("subx.l %1,%0" : "=d" (dest.m32[0]) \
1521da177e4SLinus Torvalds : "d" (src.m32[0]), "0" (dest.m32[0])); \
1531da177e4SLinus Torvalds })
1541da177e4SLinus Torvalds #define fp_sub96c(dest, srch, srcm, srcl) ({ \
1551da177e4SLinus Torvalds char carry; \
1561da177e4SLinus Torvalds asm ("sub.l %1,%0" : "=d,dm" (dest.m32[2]) \
1571da177e4SLinus Torvalds : "dm,d" (srcl), "0,0" (dest.m32[2])); \
1581da177e4SLinus Torvalds asm ("subx.l %1,%0" : "=d" (dest.m32[1]) \
1591da177e4SLinus Torvalds : "d" (srcm), "0" (dest.m32[1])); \
1601da177e4SLinus Torvalds asm ("subx.l %2,%1; scs %0" : "=d" (carry), "=d" (dest.m32[0]) \
1611da177e4SLinus Torvalds : "d" (srch), "1" (dest.m32[0])); \
1621da177e4SLinus Torvalds carry; \
1631da177e4SLinus Torvalds })
1641da177e4SLinus Torvalds
fp_multiplymant(union fp_mant128 * dest,struct fp_ext * src1,struct fp_ext * src2)1651da177e4SLinus Torvalds static inline void fp_multiplymant(union fp_mant128 *dest, struct fp_ext *src1,
1661da177e4SLinus Torvalds struct fp_ext *src2)
1671da177e4SLinus Torvalds {
1681da177e4SLinus Torvalds union fp_mant64 temp;
1691da177e4SLinus Torvalds
1701da177e4SLinus Torvalds fp_mul64(dest->m32[0], dest->m32[1], src1->mant.m32[0], src2->mant.m32[0]);
1711da177e4SLinus Torvalds fp_mul64(dest->m32[2], dest->m32[3], src1->mant.m32[1], src2->mant.m32[1]);
1721da177e4SLinus Torvalds
1731da177e4SLinus Torvalds fp_mul64(temp.m32[0], temp.m32[1], src1->mant.m32[0], src2->mant.m32[1]);
1741da177e4SLinus Torvalds fp_addx96(dest, temp);
1751da177e4SLinus Torvalds
1761da177e4SLinus Torvalds fp_mul64(temp.m32[0], temp.m32[1], src1->mant.m32[1], src2->mant.m32[0]);
1771da177e4SLinus Torvalds fp_addx96(dest, temp);
1781da177e4SLinus Torvalds }
1791da177e4SLinus Torvalds
fp_dividemant(union fp_mant128 * dest,struct fp_ext * src,struct fp_ext * div)1801da177e4SLinus Torvalds static inline void fp_dividemant(union fp_mant128 *dest, struct fp_ext *src,
1811da177e4SLinus Torvalds struct fp_ext *div)
1821da177e4SLinus Torvalds {
1831da177e4SLinus Torvalds union fp_mant128 tmp;
1841da177e4SLinus Torvalds union fp_mant64 tmp64;
1851da177e4SLinus Torvalds unsigned long *mantp = dest->m32;
1861da177e4SLinus Torvalds unsigned long fix, rem, first, dummy;
1871da177e4SLinus Torvalds int i;
1881da177e4SLinus Torvalds
1891da177e4SLinus Torvalds /* the algorithm below requires dest to be smaller than div,
1901da177e4SLinus Torvalds but both have the high bit set */
1911da177e4SLinus Torvalds if (src->mant.m64 >= div->mant.m64) {
1921da177e4SLinus Torvalds fp_sub64(src->mant, div->mant);
1931da177e4SLinus Torvalds *mantp = 1;
1941da177e4SLinus Torvalds } else
1951da177e4SLinus Torvalds *mantp = 0;
1961da177e4SLinus Torvalds mantp++;
1971da177e4SLinus Torvalds
1981da177e4SLinus Torvalds /* basic idea behind this algorithm: we can't divide two 64bit numbers
1991da177e4SLinus Torvalds (AB/CD) directly, but we can calculate AB/C0, but this means this
2001da177e4SLinus Torvalds quotient is off by C0/CD, so we have to multiply the first result
2011da177e4SLinus Torvalds to fix the result, after that we have nearly the correct result
2021da177e4SLinus Torvalds and only a few corrections are needed. */
2031da177e4SLinus Torvalds
2041da177e4SLinus Torvalds /* C0/CD can be precalculated, but it's an 64bit division again, but
2051da177e4SLinus Torvalds we can make it a bit easier, by dividing first through C so we get
2061da177e4SLinus Torvalds 10/1D and now only a single shift and the value fits into 32bit. */
2071da177e4SLinus Torvalds fix = 0x80000000;
2081da177e4SLinus Torvalds dummy = div->mant.m32[1] / div->mant.m32[0] + 1;
2091da177e4SLinus Torvalds dummy = (dummy >> 1) | fix;
2101da177e4SLinus Torvalds fp_div64(fix, dummy, fix, 0, dummy);
2111da177e4SLinus Torvalds fix--;
2121da177e4SLinus Torvalds
2131da177e4SLinus Torvalds for (i = 0; i < 3; i++, mantp++) {
2141da177e4SLinus Torvalds if (src->mant.m32[0] == div->mant.m32[0]) {
2151da177e4SLinus Torvalds fp_div64(first, rem, 0, src->mant.m32[1], div->mant.m32[0]);
2161da177e4SLinus Torvalds
2171da177e4SLinus Torvalds fp_mul64(*mantp, dummy, first, fix);
2181da177e4SLinus Torvalds *mantp += fix;
2191da177e4SLinus Torvalds } else {
2201da177e4SLinus Torvalds fp_div64(first, rem, src->mant.m32[0], src->mant.m32[1], div->mant.m32[0]);
2211da177e4SLinus Torvalds
2221da177e4SLinus Torvalds fp_mul64(*mantp, dummy, first, fix);
2231da177e4SLinus Torvalds }
2241da177e4SLinus Torvalds
2251da177e4SLinus Torvalds fp_mul64(tmp.m32[0], tmp.m32[1], div->mant.m32[0], first - *mantp);
2261da177e4SLinus Torvalds fp_add64(tmp.m32[0], tmp.m32[1], 0, rem);
2271da177e4SLinus Torvalds tmp.m32[2] = 0;
2281da177e4SLinus Torvalds
2291da177e4SLinus Torvalds fp_mul64(tmp64.m32[0], tmp64.m32[1], *mantp, div->mant.m32[1]);
2301da177e4SLinus Torvalds fp_sub96c(tmp, 0, tmp64.m32[0], tmp64.m32[1]);
2311da177e4SLinus Torvalds
2321da177e4SLinus Torvalds src->mant.m32[0] = tmp.m32[1];
2331da177e4SLinus Torvalds src->mant.m32[1] = tmp.m32[2];
2341da177e4SLinus Torvalds
2351da177e4SLinus Torvalds while (!fp_sub96c(tmp, 0, div->mant.m32[0], div->mant.m32[1])) {
2361da177e4SLinus Torvalds src->mant.m32[0] = tmp.m32[1];
2371da177e4SLinus Torvalds src->mant.m32[1] = tmp.m32[2];
2381da177e4SLinus Torvalds *mantp += 1;
2391da177e4SLinus Torvalds }
2401da177e4SLinus Torvalds }
2411da177e4SLinus Torvalds }
2421da177e4SLinus Torvalds
fp_putmant128(struct fp_ext * dest,union fp_mant128 * src,int shift)2431da177e4SLinus Torvalds static inline void fp_putmant128(struct fp_ext *dest, union fp_mant128 *src,
2441da177e4SLinus Torvalds int shift)
2451da177e4SLinus Torvalds {
2461da177e4SLinus Torvalds unsigned long tmp;
2471da177e4SLinus Torvalds
2481da177e4SLinus Torvalds switch (shift) {
2491da177e4SLinus Torvalds case 0:
2501da177e4SLinus Torvalds dest->mant.m64 = src->m64[0];
2511da177e4SLinus Torvalds dest->lowmant = src->m32[2] >> 24;
2521da177e4SLinus Torvalds if (src->m32[3] || (src->m32[2] << 8))
2531da177e4SLinus Torvalds dest->lowmant |= 1;
2541da177e4SLinus Torvalds break;
2551da177e4SLinus Torvalds case 1:
2561da177e4SLinus Torvalds asm volatile ("lsl.l #1,%0"
2571da177e4SLinus Torvalds : "=d" (tmp) : "0" (src->m32[2]));
2581da177e4SLinus Torvalds asm volatile ("roxl.l #1,%0"
2591da177e4SLinus Torvalds : "=d" (dest->mant.m32[1]) : "0" (src->m32[1]));
2601da177e4SLinus Torvalds asm volatile ("roxl.l #1,%0"
2611da177e4SLinus Torvalds : "=d" (dest->mant.m32[0]) : "0" (src->m32[0]));
2621da177e4SLinus Torvalds dest->lowmant = tmp >> 24;
2631da177e4SLinus Torvalds if (src->m32[3] || (tmp << 8))
2641da177e4SLinus Torvalds dest->lowmant |= 1;
2651da177e4SLinus Torvalds break;
2661da177e4SLinus Torvalds case 31:
2671da177e4SLinus Torvalds asm volatile ("lsr.l #1,%1; roxr.l #1,%0"
2681da177e4SLinus Torvalds : "=d" (dest->mant.m32[0])
2691da177e4SLinus Torvalds : "d" (src->m32[0]), "0" (src->m32[1]));
2701da177e4SLinus Torvalds asm volatile ("roxr.l #1,%0"
2711da177e4SLinus Torvalds : "=d" (dest->mant.m32[1]) : "0" (src->m32[2]));
2721da177e4SLinus Torvalds asm volatile ("roxr.l #1,%0"
2731da177e4SLinus Torvalds : "=d" (tmp) : "0" (src->m32[3]));
2741da177e4SLinus Torvalds dest->lowmant = tmp >> 24;
2751da177e4SLinus Torvalds if (src->m32[3] << 7)
2761da177e4SLinus Torvalds dest->lowmant |= 1;
2771da177e4SLinus Torvalds break;
2781da177e4SLinus Torvalds case 32:
2791da177e4SLinus Torvalds dest->mant.m32[0] = src->m32[1];
2801da177e4SLinus Torvalds dest->mant.m32[1] = src->m32[2];
2811da177e4SLinus Torvalds dest->lowmant = src->m32[3] >> 24;
2821da177e4SLinus Torvalds if (src->m32[3] << 8)
2831da177e4SLinus Torvalds dest->lowmant |= 1;
2841da177e4SLinus Torvalds break;
2851da177e4SLinus Torvalds }
2861da177e4SLinus Torvalds }
2871da177e4SLinus Torvalds
2881da177e4SLinus Torvalds #endif /* MULTI_ARITH_H */
289