51 static __always_inline void fproduct_copy_from_wide_(u64 *output, u128 *input)  in fproduct_copy_from_wide_()  argument
55 		output[0] = ((u64)(xi));  in fproduct_copy_from_wide_()
59 		output[1] = ((u64)(xi));  in fproduct_copy_from_wide_()
63 		output[2] = ((u64)(xi));  in fproduct_copy_from_wide_()
67 		output[3] = ((u64)(xi));  in fproduct_copy_from_wide_()
71 		output[4] = ((u64)(xi));  in fproduct_copy_from_wide_()
76 fproduct_sum_scalar_multiplication_(u128 *output, u64 *input, u64 s)  in fproduct_sum_scalar_multiplication_()  argument
78 	output[0] += (u128)input[0] * s;  in fproduct_sum_scalar_multiplication_()
79 	output[1] += (u128)input[1] * s;  in fproduct_sum_scalar_multiplication_()
80 	output[2] += (u128)input[2] * s;  in fproduct_sum_scalar_multiplication_()
81 	output[3] += (u128)input[3] * s;  in fproduct_sum_scalar_multiplication_()
82 	output[4] += (u128)input[4] * s;  in fproduct_sum_scalar_multiplication_()
126 static __always_inline void fmul_shift_reduce(u64 *output)  in fmul_shift_reduce()  argument
128 	u64 tmp = output[4];  in fmul_shift_reduce()
132 		u64 z = output[ctr - 1];  in fmul_shift_reduce()
133 		output[ctr] = z;  in fmul_shift_reduce()
137 		u64 z = output[ctr - 1];  in fmul_shift_reduce()
138 		output[ctr] = z;  in fmul_shift_reduce()
142 		u64 z = output[ctr - 1];  in fmul_shift_reduce()
143 		output[ctr] = z;  in fmul_shift_reduce()
147 		u64 z = output[ctr - 1];  in fmul_shift_reduce()
148 		output[ctr] = z;  in fmul_shift_reduce()
150 	output[0] = tmp;  in fmul_shift_reduce()
151 	b0 = output[0];  in fmul_shift_reduce()
152 	output[0] = 19 * b0;  in fmul_shift_reduce()
155 static __always_inline void fmul_mul_shift_reduce_(u128 *output, u64 *input,  in fmul_mul_shift_reduce_()  argument
162 		fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
167 		fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
172 		fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
177 		fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
182 	fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
185 static __always_inline void fmul_fmul(u64 *output, u64 *input, u64 *input21)  in fmul_fmul()  argument
206 		fproduct_copy_from_wide_(output, t);  in fmul_fmul()
207 		i0 = output[0];  in fmul_fmul()
208 		i1 = output[1];  in fmul_fmul()
211 		output[0] = i0_;  in fmul_fmul()
212 		output[1] = i1_;  in fmul_fmul()
216 static __always_inline void fsquare_fsquare__(u128 *tmp, u64 *output)  in fsquare_fsquare__()  argument
218 	u64 r0 = output[0];  in fsquare_fsquare__()
219 	u64 r1 = output[1];  in fsquare_fsquare__()
220 	u64 r2 = output[2];  in fsquare_fsquare__()
221 	u64 r3 = output[3];  in fsquare_fsquare__()
222 	u64 r4 = output[4];  in fsquare_fsquare__()
245 static __always_inline void fsquare_fsquare_(u128 *tmp, u64 *output)  in fsquare_fsquare_()  argument
255 	fsquare_fsquare__(tmp, output);  in fsquare_fsquare_()
263 	fproduct_copy_from_wide_(output, tmp);  in fsquare_fsquare_()
264 	i0 = output[0];  in fsquare_fsquare_()
265 	i1 = output[1];  in fsquare_fsquare_()
268 	output[0] = i0_;  in fsquare_fsquare_()
269 	output[1] = i1_;  in fsquare_fsquare_()
272 static __always_inline void fsquare_fsquare_times_(u64 *output, u128 *tmp,  in fsquare_fsquare_times_()  argument
276 	fsquare_fsquare_(tmp, output);  in fsquare_fsquare_times_()
278 		fsquare_fsquare_(tmp, output);  in fsquare_fsquare_times_()
281 static __always_inline void fsquare_fsquare_times(u64 *output, u64 *input,  in fsquare_fsquare_times()  argument
285 	memcpy(output, input, 5 * sizeof(*input));  in fsquare_fsquare_times()
286 	fsquare_fsquare_times_(output, t, count1);  in fsquare_fsquare_times()
289 static __always_inline void fsquare_fsquare_times_inplace(u64 *output,  in fsquare_fsquare_times_inplace()  argument
293 	fsquare_fsquare_times_(output, t, count1);  in fsquare_fsquare_times_inplace()
395 static __always_inline void fscalar(u64 *output, u64 *b, u64 s)  in fscalar()  argument
429 	fproduct_copy_from_wide_(output, tmp);  in fscalar()
432 static __always_inline void fmul(u64 *output, u64 *a, u64 *b)  in fmul()  argument
434 	fmul_fmul(output, a, b);  in fmul()
437 static __always_inline void crecip(u64 *output, u64 *input)  in crecip()  argument
439 	crecip_crecip(output, input);  in crecip()
471 static __always_inline void point_copy(u64 *output, u64 *input)  in point_copy()  argument
473 	memcpy(output, input, 5 * sizeof(*input));  in point_copy()
474 	memcpy(output + 5, input + 5, 5 * sizeof(*input));  in point_copy()
601 static __always_inline void format_fexpand(u64 *output, const u8 *input)  in format_fexpand()  argument
618 	output[0] = output0;  in format_fexpand()
619 	output[1] = output1;  in format_fexpand()
620 	output[2] = output2;  in format_fexpand()
621 	output[3] = output3;  in format_fexpand()
622 	output[4] = output4;  in format_fexpand()
716 static __always_inline void format_fcontract_store(u8 *output, u64 *input)  in format_fcontract_store()  argument
727 	u8 *b0 = output;  in format_fcontract_store()
728 	u8 *b1 = output + 8;  in format_fcontract_store()
729 	u8 *b2 = output + 16;  in format_fcontract_store()
730 	u8 *b3 = output + 24;  in format_fcontract_store()
737 static __always_inline void format_fcontract(u8 *output, u64 *input)  in format_fcontract()  argument
742 	format_fcontract_store(output, input);  in format_fcontract()