1#!/usr/bin/env perl
2# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3#
4# Copyright (C) 2017-2018 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
5# Copyright (C) 2017-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
6# Copyright (C) 2006-2017 CRYPTOGAMS by <appro@openssl.org>. All Rights Reserved.
7#
8# This code is taken from the OpenSSL project but the author, Andy Polyakov,
9# has relicensed it under the licenses specified in the SPDX header above.
10# The original headers, including the original license headers, are
11# included below for completeness.
12#
13# ====================================================================
14# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
15# project. The module is, however, dual licensed under OpenSSL and
16# CRYPTOGAMS licenses depending on where you obtain it. For further
17# details see http://www.openssl.org/~appro/cryptogams/.
18# ====================================================================
19#
20# This module implements Poly1305 hash for x86_64.
21#
22# March 2015
23#
24# Initial release.
25#
26# December 2016
27#
28# Add AVX512F+VL+BW code path.
29#
30# November 2017
31#
32# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be
33# executed even on Knights Landing. Trigger for modification was
34# observation that AVX512 code paths can negatively affect overall
35# Skylake-X system performance. Since we are likely to suppress
36# AVX512F capability flag [at least on Skylake-X], conversion serves
37# as kind of "investment protection". Note that next *lake processor,
38# Cannonlake, has AVX512IFMA code path to execute...
39#
40# Numbers are cycles per processed byte with poly1305_blocks alone,
41# measured with rdtsc at fixed clock frequency.
42#
43#		IALU/gcc-4.8(*)	AVX(**)		AVX2	AVX-512
44# P4		4.46/+120%	-
45# Core 2	2.41/+90%	-
46# Westmere	1.88/+120%	-
47# Sandy Bridge	1.39/+140%	1.10
48# Haswell	1.14/+175%	1.11		0.65
49# Skylake[-X]	1.13/+120%	0.96		0.51	[0.35]
50# Silvermont	2.83/+95%	-
51# Knights L	3.60/?		1.65		1.10	0.41(***)
52# Goldmont	1.70/+180%	-
53# VIA Nano	1.82/+150%	-
54# Sledgehammer	1.38/+160%	-
55# Bulldozer	2.30/+130%	0.97
56# Ryzen		1.15/+200%	1.08		1.18
57#
58# (*)	improvement coefficients relative to clang are more modest and
59#	are ~50% on most processors, in both cases we are comparing to
60#	__int128 code;
61# (**)	SSE2 implementation was attempted, but among non-AVX processors
62#	it was faster than integer-only code only on older Intel P4 and
63#	Core processors, 50-30%, less newer processor is, but slower on
64#	contemporary ones, for example almost 2x slower on Atom, and as
65#	former are naturally disappearing, SSE2 is deemed unnecessary;
66# (***)	strangely enough performance seems to vary from core to core,
67#	listed result is best case;
68
69$flavour = shift;
70$output  = shift;
71if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
72
73$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
74$kernel=0; $kernel=1 if (!$flavour && !$output);
75
76if (!$kernel) {
77	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
78	( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
79	( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
80	die "can't locate x86_64-xlate.pl";
81
82	open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
83	*STDOUT=*OUT;
84
85	if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
86	    =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
87		$avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25);
88	}
89
90	if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
91	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
92		$avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12);
93		$avx += 1 if ($1==2.11 && $2>=8);
94	}
95
96	if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
97	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {
98		$avx = ($1>=10) + ($1>=11);
99	}
100
101	if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
102		$avx = ($2>=3.0) + ($2>3.0);
103	}
104} else {
105	$avx = 4; # The kernel uses ifdefs for this.
106}
107
108sub declare_function() {
109	my ($name, $align, $nargs) = @_;
110	if($kernel) {
111		$code .= ".align $align\n";
112		$code .= "SYM_FUNC_START($name)\n";
113		$code .= ".L$name:\n";
114	} else {
115		$code .= ".globl	$name\n";
116		$code .= ".type	$name,\@function,$nargs\n";
117		$code .= ".align	$align\n";
118		$code .= "$name:\n";
119	}
120}
121
122sub end_function() {
123	my ($name) = @_;
124	if($kernel) {
125		$code .= "SYM_FUNC_END($name)\n";
126	} else {
127		$code .= ".size   $name,.-$name\n";
128	}
129}
130
131$code.=<<___ if $kernel;
132#include <linux/linkage.h>
133___
134
135if ($avx) {
136$code.=<<___ if $kernel;
137.section .rodata
138___
139$code.=<<___;
140.align	64
141.Lconst:
142.Lmask24:
143.long	0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0
144.L129:
145.long	`1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0
146.Lmask26:
147.long	0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0
148.Lpermd_avx2:
149.long	2,2,2,3,2,0,2,1
150.Lpermd_avx512:
151.long	0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7
152
153.L2_44_inp_permd:
154.long	0,1,1,2,2,3,7,7
155.L2_44_inp_shift:
156.quad	0,12,24,64
157.L2_44_mask:
158.quad	0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff
159.L2_44_shift_rgt:
160.quad	44,44,42,64
161.L2_44_shift_lft:
162.quad	8,8,10,64
163
164.align	64
165.Lx_mask44:
166.quad	0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
167.quad	0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
168.Lx_mask42:
169.quad	0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
170.quad	0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
171___
172}
173$code.=<<___ if (!$kernel);
174.asciz	"Poly1305 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
175.align	16
176___
177
178my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx");
179my ($mac,$nonce)=($inp,$len);	# *_emit arguments
180my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13");
181my ($h0,$h1,$h2)=("%r14","%rbx","%r10");
182
183sub poly1305_iteration {
184# input:	copy of $r1 in %rax, $h0-$h2, $r0-$r1
185# output:	$h0-$h2 *= $r0-$r1
186$code.=<<___;
187	mulq	$h0			# h0*r1
188	mov	%rax,$d2
189	 mov	$r0,%rax
190	mov	%rdx,$d3
191
192	mulq	$h0			# h0*r0
193	mov	%rax,$h0		# future $h0
194	 mov	$r0,%rax
195	mov	%rdx,$d1
196
197	mulq	$h1			# h1*r0
198	add	%rax,$d2
199	 mov	$s1,%rax
200	adc	%rdx,$d3
201
202	mulq	$h1			# h1*s1
203	 mov	$h2,$h1			# borrow $h1
204	add	%rax,$h0
205	adc	%rdx,$d1
206
207	imulq	$s1,$h1			# h2*s1
208	add	$h1,$d2
209	 mov	$d1,$h1
210	adc	\$0,$d3
211
212	imulq	$r0,$h2			# h2*r0
213	add	$d2,$h1
214	mov	\$-4,%rax		# mask value
215	adc	$h2,$d3
216
217	and	$d3,%rax		# last reduction step
218	mov	$d3,$h2
219	shr	\$2,$d3
220	and	\$3,$h2
221	add	$d3,%rax
222	add	%rax,$h0
223	adc	\$0,$h1
224	adc	\$0,$h2
225___
226}
227
228########################################################################
229# Layout of opaque area is following.
230#
231#	unsigned __int64 h[3];		# current hash value base 2^64
232#	unsigned __int64 r[2];		# key value base 2^64
233
234$code.=<<___;
235.text
236___
237$code.=<<___ if (!$kernel);
238.extern	OPENSSL_ia32cap_P
239
240.globl	poly1305_init_x86_64
241.hidden	poly1305_init_x86_64
242.globl	poly1305_blocks_x86_64
243.hidden	poly1305_blocks_x86_64
244.globl	poly1305_emit_x86_64
245.hidden	poly1305_emit_x86_64
246___
247&declare_function("poly1305_init_x86_64", 32, 3);
248$code.=<<___;
249	xor	%eax,%eax
250	mov	%rax,0($ctx)		# initialize hash value
251	mov	%rax,8($ctx)
252	mov	%rax,16($ctx)
253
254	test	$inp,$inp
255	je	.Lno_key
256___
257$code.=<<___ if (!$kernel);
258	lea	poly1305_blocks_x86_64(%rip),%r10
259	lea	poly1305_emit_x86_64(%rip),%r11
260___
261$code.=<<___	if (!$kernel && $avx);
262	mov	OPENSSL_ia32cap_P+4(%rip),%r9
263	lea	poly1305_blocks_avx(%rip),%rax
264	lea	poly1305_emit_avx(%rip),%rcx
265	bt	\$`60-32`,%r9		# AVX?
266	cmovc	%rax,%r10
267	cmovc	%rcx,%r11
268___
269$code.=<<___	if (!$kernel && $avx>1);
270	lea	poly1305_blocks_avx2(%rip),%rax
271	bt	\$`5+32`,%r9		# AVX2?
272	cmovc	%rax,%r10
273___
274$code.=<<___	if (!$kernel && $avx>3);
275	mov	\$`(1<<31|1<<21|1<<16)`,%rax
276	shr	\$32,%r9
277	and	%rax,%r9
278	cmp	%rax,%r9
279	je	.Linit_base2_44
280___
281$code.=<<___;
282	mov	\$0x0ffffffc0fffffff,%rax
283	mov	\$0x0ffffffc0ffffffc,%rcx
284	and	0($inp),%rax
285	and	8($inp),%rcx
286	mov	%rax,24($ctx)
287	mov	%rcx,32($ctx)
288___
289$code.=<<___	if (!$kernel && $flavour !~ /elf32/);
290	mov	%r10,0(%rdx)
291	mov	%r11,8(%rdx)
292___
293$code.=<<___	if (!$kernel && $flavour =~ /elf32/);
294	mov	%r10d,0(%rdx)
295	mov	%r11d,4(%rdx)
296___
297$code.=<<___;
298	mov	\$1,%eax
299.Lno_key:
300	RET
301___
302&end_function("poly1305_init_x86_64");
303
304&declare_function("poly1305_blocks_x86_64", 32, 4);
305$code.=<<___;
306.cfi_startproc
307.Lblocks:
308	shr	\$4,$len
309	jz	.Lno_data		# too short
310
311	push	%rbx
312.cfi_push	%rbx
313	push	%r12
314.cfi_push	%r12
315	push	%r13
316.cfi_push	%r13
317	push	%r14
318.cfi_push	%r14
319	push	%r15
320.cfi_push	%r15
321	push	$ctx
322.cfi_push	$ctx
323.Lblocks_body:
324
325	mov	$len,%r15		# reassign $len
326
327	mov	24($ctx),$r0		# load r
328	mov	32($ctx),$s1
329
330	mov	0($ctx),$h0		# load hash value
331	mov	8($ctx),$h1
332	mov	16($ctx),$h2
333
334	mov	$s1,$r1
335	shr	\$2,$s1
336	mov	$r1,%rax
337	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
338	jmp	.Loop
339
340.align	32
341.Loop:
342	add	0($inp),$h0		# accumulate input
343	adc	8($inp),$h1
344	lea	16($inp),$inp
345	adc	$padbit,$h2
346___
347
348	&poly1305_iteration();
349
350$code.=<<___;
351	mov	$r1,%rax
352	dec	%r15			# len-=16
353	jnz	.Loop
354
355	mov	0(%rsp),$ctx
356.cfi_restore	$ctx
357
358	mov	$h0,0($ctx)		# store hash value
359	mov	$h1,8($ctx)
360	mov	$h2,16($ctx)
361
362	mov	8(%rsp),%r15
363.cfi_restore	%r15
364	mov	16(%rsp),%r14
365.cfi_restore	%r14
366	mov	24(%rsp),%r13
367.cfi_restore	%r13
368	mov	32(%rsp),%r12
369.cfi_restore	%r12
370	mov	40(%rsp),%rbx
371.cfi_restore	%rbx
372	lea	48(%rsp),%rsp
373.cfi_adjust_cfa_offset	-48
374.Lno_data:
375.Lblocks_epilogue:
376	RET
377.cfi_endproc
378___
379&end_function("poly1305_blocks_x86_64");
380
381&declare_function("poly1305_emit_x86_64", 32, 3);
382$code.=<<___;
383.Lemit:
384	mov	0($ctx),%r8	# load hash value
385	mov	8($ctx),%r9
386	mov	16($ctx),%r10
387
388	mov	%r8,%rax
389	add	\$5,%r8		# compare to modulus
390	mov	%r9,%rcx
391	adc	\$0,%r9
392	adc	\$0,%r10
393	shr	\$2,%r10	# did 130-bit value overflow?
394	cmovnz	%r8,%rax
395	cmovnz	%r9,%rcx
396
397	add	0($nonce),%rax	# accumulate nonce
398	adc	8($nonce),%rcx
399	mov	%rax,0($mac)	# write result
400	mov	%rcx,8($mac)
401
402	RET
403___
404&end_function("poly1305_emit_x86_64");
405if ($avx) {
406
407########################################################################
408# Layout of opaque area is following.
409#
410#	unsigned __int32 h[5];		# current hash value base 2^26
411#	unsigned __int32 is_base2_26;
412#	unsigned __int64 r[2];		# key value base 2^64
413#	unsigned __int64 pad;
414#	struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9];
415#
416# where r^n are base 2^26 digits of degrees of multiplier key. There are
417# 5 digits, but last four are interleaved with multiples of 5, totalling
418# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
419
420my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) =
421    map("%xmm$_",(0..15));
422
423$code.=<<___;
424.type	__poly1305_block,\@abi-omnipotent
425.align	32
426__poly1305_block:
427	push $ctx
428___
429	&poly1305_iteration();
430$code.=<<___;
431	pop $ctx
432	RET
433.size	__poly1305_block,.-__poly1305_block
434
435.type	__poly1305_init_avx,\@abi-omnipotent
436.align	32
437__poly1305_init_avx:
438	push %rbp
439	mov %rsp,%rbp
440	mov	$r0,$h0
441	mov	$r1,$h1
442	xor	$h2,$h2
443
444	lea	48+64($ctx),$ctx	# size optimization
445
446	mov	$r1,%rax
447	call	__poly1305_block	# r^2
448
449	mov	\$0x3ffffff,%eax	# save interleaved r^2 and r base 2^26
450	mov	\$0x3ffffff,%edx
451	mov	$h0,$d1
452	and	$h0#d,%eax
453	mov	$r0,$d2
454	and	$r0#d,%edx
455	mov	%eax,`16*0+0-64`($ctx)
456	shr	\$26,$d1
457	mov	%edx,`16*0+4-64`($ctx)
458	shr	\$26,$d2
459
460	mov	\$0x3ffffff,%eax
461	mov	\$0x3ffffff,%edx
462	and	$d1#d,%eax
463	and	$d2#d,%edx
464	mov	%eax,`16*1+0-64`($ctx)
465	lea	(%rax,%rax,4),%eax	# *5
466	mov	%edx,`16*1+4-64`($ctx)
467	lea	(%rdx,%rdx,4),%edx	# *5
468	mov	%eax,`16*2+0-64`($ctx)
469	shr	\$26,$d1
470	mov	%edx,`16*2+4-64`($ctx)
471	shr	\$26,$d2
472
473	mov	$h1,%rax
474	mov	$r1,%rdx
475	shl	\$12,%rax
476	shl	\$12,%rdx
477	or	$d1,%rax
478	or	$d2,%rdx
479	and	\$0x3ffffff,%eax
480	and	\$0x3ffffff,%edx
481	mov	%eax,`16*3+0-64`($ctx)
482	lea	(%rax,%rax,4),%eax	# *5
483	mov	%edx,`16*3+4-64`($ctx)
484	lea	(%rdx,%rdx,4),%edx	# *5
485	mov	%eax,`16*4+0-64`($ctx)
486	mov	$h1,$d1
487	mov	%edx,`16*4+4-64`($ctx)
488	mov	$r1,$d2
489
490	mov	\$0x3ffffff,%eax
491	mov	\$0x3ffffff,%edx
492	shr	\$14,$d1
493	shr	\$14,$d2
494	and	$d1#d,%eax
495	and	$d2#d,%edx
496	mov	%eax,`16*5+0-64`($ctx)
497	lea	(%rax,%rax,4),%eax	# *5
498	mov	%edx,`16*5+4-64`($ctx)
499	lea	(%rdx,%rdx,4),%edx	# *5
500	mov	%eax,`16*6+0-64`($ctx)
501	shr	\$26,$d1
502	mov	%edx,`16*6+4-64`($ctx)
503	shr	\$26,$d2
504
505	mov	$h2,%rax
506	shl	\$24,%rax
507	or	%rax,$d1
508	mov	$d1#d,`16*7+0-64`($ctx)
509	lea	($d1,$d1,4),$d1		# *5
510	mov	$d2#d,`16*7+4-64`($ctx)
511	lea	($d2,$d2,4),$d2		# *5
512	mov	$d1#d,`16*8+0-64`($ctx)
513	mov	$d2#d,`16*8+4-64`($ctx)
514
515	mov	$r1,%rax
516	call	__poly1305_block	# r^3
517
518	mov	\$0x3ffffff,%eax	# save r^3 base 2^26
519	mov	$h0,$d1
520	and	$h0#d,%eax
521	shr	\$26,$d1
522	mov	%eax,`16*0+12-64`($ctx)
523
524	mov	\$0x3ffffff,%edx
525	and	$d1#d,%edx
526	mov	%edx,`16*1+12-64`($ctx)
527	lea	(%rdx,%rdx,4),%edx	# *5
528	shr	\$26,$d1
529	mov	%edx,`16*2+12-64`($ctx)
530
531	mov	$h1,%rax
532	shl	\$12,%rax
533	or	$d1,%rax
534	and	\$0x3ffffff,%eax
535	mov	%eax,`16*3+12-64`($ctx)
536	lea	(%rax,%rax,4),%eax	# *5
537	mov	$h1,$d1
538	mov	%eax,`16*4+12-64`($ctx)
539
540	mov	\$0x3ffffff,%edx
541	shr	\$14,$d1
542	and	$d1#d,%edx
543	mov	%edx,`16*5+12-64`($ctx)
544	lea	(%rdx,%rdx,4),%edx	# *5
545	shr	\$26,$d1
546	mov	%edx,`16*6+12-64`($ctx)
547
548	mov	$h2,%rax
549	shl	\$24,%rax
550	or	%rax,$d1
551	mov	$d1#d,`16*7+12-64`($ctx)
552	lea	($d1,$d1,4),$d1		# *5
553	mov	$d1#d,`16*8+12-64`($ctx)
554
555	mov	$r1,%rax
556	call	__poly1305_block	# r^4
557
558	mov	\$0x3ffffff,%eax	# save r^4 base 2^26
559	mov	$h0,$d1
560	and	$h0#d,%eax
561	shr	\$26,$d1
562	mov	%eax,`16*0+8-64`($ctx)
563
564	mov	\$0x3ffffff,%edx
565	and	$d1#d,%edx
566	mov	%edx,`16*1+8-64`($ctx)
567	lea	(%rdx,%rdx,4),%edx	# *5
568	shr	\$26,$d1
569	mov	%edx,`16*2+8-64`($ctx)
570
571	mov	$h1,%rax
572	shl	\$12,%rax
573	or	$d1,%rax
574	and	\$0x3ffffff,%eax
575	mov	%eax,`16*3+8-64`($ctx)
576	lea	(%rax,%rax,4),%eax	# *5
577	mov	$h1,$d1
578	mov	%eax,`16*4+8-64`($ctx)
579
580	mov	\$0x3ffffff,%edx
581	shr	\$14,$d1
582	and	$d1#d,%edx
583	mov	%edx,`16*5+8-64`($ctx)
584	lea	(%rdx,%rdx,4),%edx	# *5
585	shr	\$26,$d1
586	mov	%edx,`16*6+8-64`($ctx)
587
588	mov	$h2,%rax
589	shl	\$24,%rax
590	or	%rax,$d1
591	mov	$d1#d,`16*7+8-64`($ctx)
592	lea	($d1,$d1,4),$d1		# *5
593	mov	$d1#d,`16*8+8-64`($ctx)
594
595	lea	-48-64($ctx),$ctx	# size [de-]optimization
596	pop %rbp
597	RET
598.size	__poly1305_init_avx,.-__poly1305_init_avx
599___
600
601&declare_function("poly1305_blocks_avx", 32, 4);
602$code.=<<___;
603.cfi_startproc
604	mov	20($ctx),%r8d		# is_base2_26
605	cmp	\$128,$len
606	jae	.Lblocks_avx
607	test	%r8d,%r8d
608	jz	.Lblocks
609
610.Lblocks_avx:
611	and	\$-16,$len
612	jz	.Lno_data_avx
613
614	vzeroupper
615
616	test	%r8d,%r8d
617	jz	.Lbase2_64_avx
618
619	test	\$31,$len
620	jz	.Leven_avx
621
622	push	%rbp
623.cfi_push	%rbp
624	mov 	%rsp,%rbp
625	push	%rbx
626.cfi_push	%rbx
627	push	%r12
628.cfi_push	%r12
629	push	%r13
630.cfi_push	%r13
631	push	%r14
632.cfi_push	%r14
633	push	%r15
634.cfi_push	%r15
635.Lblocks_avx_body:
636
637	mov	$len,%r15		# reassign $len
638
639	mov	0($ctx),$d1		# load hash value
640	mov	8($ctx),$d2
641	mov	16($ctx),$h2#d
642
643	mov	24($ctx),$r0		# load r
644	mov	32($ctx),$s1
645
646	################################# base 2^26 -> base 2^64
647	mov	$d1#d,$h0#d
648	and	\$`-1*(1<<31)`,$d1
649	mov	$d2,$r1			# borrow $r1
650	mov	$d2#d,$h1#d
651	and	\$`-1*(1<<31)`,$d2
652
653	shr	\$6,$d1
654	shl	\$52,$r1
655	add	$d1,$h0
656	shr	\$12,$h1
657	shr	\$18,$d2
658	add	$r1,$h0
659	adc	$d2,$h1
660
661	mov	$h2,$d1
662	shl	\$40,$d1
663	shr	\$24,$h2
664	add	$d1,$h1
665	adc	\$0,$h2			# can be partially reduced...
666
667	mov	\$-4,$d2		# ... so reduce
668	mov	$h2,$d1
669	and	$h2,$d2
670	shr	\$2,$d1
671	and	\$3,$h2
672	add	$d2,$d1			# =*5
673	add	$d1,$h0
674	adc	\$0,$h1
675	adc	\$0,$h2
676
677	mov	$s1,$r1
678	mov	$s1,%rax
679	shr	\$2,$s1
680	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
681
682	add	0($inp),$h0		# accumulate input
683	adc	8($inp),$h1
684	lea	16($inp),$inp
685	adc	$padbit,$h2
686
687	call	__poly1305_block
688
689	test	$padbit,$padbit		# if $padbit is zero,
690	jz	.Lstore_base2_64_avx	# store hash in base 2^64 format
691
692	################################# base 2^64 -> base 2^26
693	mov	$h0,%rax
694	mov	$h0,%rdx
695	shr	\$52,$h0
696	mov	$h1,$r0
697	mov	$h1,$r1
698	shr	\$26,%rdx
699	and	\$0x3ffffff,%rax	# h[0]
700	shl	\$12,$r0
701	and	\$0x3ffffff,%rdx	# h[1]
702	shr	\$14,$h1
703	or	$r0,$h0
704	shl	\$24,$h2
705	and	\$0x3ffffff,$h0		# h[2]
706	shr	\$40,$r1
707	and	\$0x3ffffff,$h1		# h[3]
708	or	$r1,$h2			# h[4]
709
710	sub	\$16,%r15
711	jz	.Lstore_base2_26_avx
712
713	vmovd	%rax#d,$H0
714	vmovd	%rdx#d,$H1
715	vmovd	$h0#d,$H2
716	vmovd	$h1#d,$H3
717	vmovd	$h2#d,$H4
718	jmp	.Lproceed_avx
719
720.align	32
721.Lstore_base2_64_avx:
722	mov	$h0,0($ctx)
723	mov	$h1,8($ctx)
724	mov	$h2,16($ctx)		# note that is_base2_26 is zeroed
725	jmp	.Ldone_avx
726
727.align	16
728.Lstore_base2_26_avx:
729	mov	%rax#d,0($ctx)		# store hash value base 2^26
730	mov	%rdx#d,4($ctx)
731	mov	$h0#d,8($ctx)
732	mov	$h1#d,12($ctx)
733	mov	$h2#d,16($ctx)
734.align	16
735.Ldone_avx:
736	pop 		%r15
737.cfi_restore	%r15
738	pop 		%r14
739.cfi_restore	%r14
740	pop 		%r13
741.cfi_restore	%r13
742	pop 		%r12
743.cfi_restore	%r12
744	pop 		%rbx
745.cfi_restore	%rbx
746	pop 		%rbp
747.cfi_restore	%rbp
748.Lno_data_avx:
749.Lblocks_avx_epilogue:
750	RET
751.cfi_endproc
752
753.align	32
754.Lbase2_64_avx:
755.cfi_startproc
756	push	%rbp
757.cfi_push	%rbp
758	mov 	%rsp,%rbp
759	push	%rbx
760.cfi_push	%rbx
761	push	%r12
762.cfi_push	%r12
763	push	%r13
764.cfi_push	%r13
765	push	%r14
766.cfi_push	%r14
767	push	%r15
768.cfi_push	%r15
769.Lbase2_64_avx_body:
770
771	mov	$len,%r15		# reassign $len
772
773	mov	24($ctx),$r0		# load r
774	mov	32($ctx),$s1
775
776	mov	0($ctx),$h0		# load hash value
777	mov	8($ctx),$h1
778	mov	16($ctx),$h2#d
779
780	mov	$s1,$r1
781	mov	$s1,%rax
782	shr	\$2,$s1
783	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
784
785	test	\$31,$len
786	jz	.Linit_avx
787
788	add	0($inp),$h0		# accumulate input
789	adc	8($inp),$h1
790	lea	16($inp),$inp
791	adc	$padbit,$h2
792	sub	\$16,%r15
793
794	call	__poly1305_block
795
796.Linit_avx:
797	################################# base 2^64 -> base 2^26
798	mov	$h0,%rax
799	mov	$h0,%rdx
800	shr	\$52,$h0
801	mov	$h1,$d1
802	mov	$h1,$d2
803	shr	\$26,%rdx
804	and	\$0x3ffffff,%rax	# h[0]
805	shl	\$12,$d1
806	and	\$0x3ffffff,%rdx	# h[1]
807	shr	\$14,$h1
808	or	$d1,$h0
809	shl	\$24,$h2
810	and	\$0x3ffffff,$h0		# h[2]
811	shr	\$40,$d2
812	and	\$0x3ffffff,$h1		# h[3]
813	or	$d2,$h2			# h[4]
814
815	vmovd	%rax#d,$H0
816	vmovd	%rdx#d,$H1
817	vmovd	$h0#d,$H2
818	vmovd	$h1#d,$H3
819	vmovd	$h2#d,$H4
820	movl	\$1,20($ctx)		# set is_base2_26
821
822	call	__poly1305_init_avx
823
824.Lproceed_avx:
825	mov	%r15,$len
826	pop 		%r15
827.cfi_restore	%r15
828	pop 		%r14
829.cfi_restore	%r14
830	pop 		%r13
831.cfi_restore	%r13
832	pop 		%r12
833.cfi_restore	%r12
834	pop 		%rbx
835.cfi_restore	%rbx
836	pop 		%rbp
837.cfi_restore	%rbp
838.Lbase2_64_avx_epilogue:
839	jmp	.Ldo_avx
840.cfi_endproc
841
842.align	32
843.Leven_avx:
844.cfi_startproc
845	vmovd		4*0($ctx),$H0		# load hash value
846	vmovd		4*1($ctx),$H1
847	vmovd		4*2($ctx),$H2
848	vmovd		4*3($ctx),$H3
849	vmovd		4*4($ctx),$H4
850
851.Ldo_avx:
852___
853$code.=<<___	if (!$win64);
854	lea		8(%rsp),%r10
855.cfi_def_cfa_register	%r10
856	and		\$-32,%rsp
857	sub		\$-8,%rsp
858	lea		-0x58(%rsp),%r11
859	sub		\$0x178,%rsp
860___
861$code.=<<___	if ($win64);
862	lea		-0xf8(%rsp),%r11
863	sub		\$0x218,%rsp
864	vmovdqa		%xmm6,0x50(%r11)
865	vmovdqa		%xmm7,0x60(%r11)
866	vmovdqa		%xmm8,0x70(%r11)
867	vmovdqa		%xmm9,0x80(%r11)
868	vmovdqa		%xmm10,0x90(%r11)
869	vmovdqa		%xmm11,0xa0(%r11)
870	vmovdqa		%xmm12,0xb0(%r11)
871	vmovdqa		%xmm13,0xc0(%r11)
872	vmovdqa		%xmm14,0xd0(%r11)
873	vmovdqa		%xmm15,0xe0(%r11)
874.Ldo_avx_body:
875___
876$code.=<<___;
877	sub		\$64,$len
878	lea		-32($inp),%rax
879	cmovc		%rax,$inp
880
881	vmovdqu		`16*3`($ctx),$D4	# preload r0^2
882	lea		`16*3+64`($ctx),$ctx	# size optimization
883	lea		.Lconst(%rip),%rcx
884
885	################################################################
886	# load input
887	vmovdqu		16*2($inp),$T0
888	vmovdqu		16*3($inp),$T1
889	vmovdqa		64(%rcx),$MASK		# .Lmask26
890
891	vpsrldq		\$6,$T0,$T2		# splat input
892	vpsrldq		\$6,$T1,$T3
893	vpunpckhqdq	$T1,$T0,$T4		# 4
894	vpunpcklqdq	$T1,$T0,$T0		# 0:1
895	vpunpcklqdq	$T3,$T2,$T3		# 2:3
896
897	vpsrlq		\$40,$T4,$T4		# 4
898	vpsrlq		\$26,$T0,$T1
899	vpand		$MASK,$T0,$T0		# 0
900	vpsrlq		\$4,$T3,$T2
901	vpand		$MASK,$T1,$T1		# 1
902	vpsrlq		\$30,$T3,$T3
903	vpand		$MASK,$T2,$T2		# 2
904	vpand		$MASK,$T3,$T3		# 3
905	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
906
907	jbe		.Lskip_loop_avx
908
909	# expand and copy pre-calculated table to stack
910	vmovdqu		`16*1-64`($ctx),$D1
911	vmovdqu		`16*2-64`($ctx),$D2
912	vpshufd		\$0xEE,$D4,$D3		# 34xx -> 3434
913	vpshufd		\$0x44,$D4,$D0		# xx12 -> 1212
914	vmovdqa		$D3,-0x90(%r11)
915	vmovdqa		$D0,0x00(%rsp)
916	vpshufd		\$0xEE,$D1,$D4
917	vmovdqu		`16*3-64`($ctx),$D0
918	vpshufd		\$0x44,$D1,$D1
919	vmovdqa		$D4,-0x80(%r11)
920	vmovdqa		$D1,0x10(%rsp)
921	vpshufd		\$0xEE,$D2,$D3
922	vmovdqu		`16*4-64`($ctx),$D1
923	vpshufd		\$0x44,$D2,$D2
924	vmovdqa		$D3,-0x70(%r11)
925	vmovdqa		$D2,0x20(%rsp)
926	vpshufd		\$0xEE,$D0,$D4
927	vmovdqu		`16*5-64`($ctx),$D2
928	vpshufd		\$0x44,$D0,$D0
929	vmovdqa		$D4,-0x60(%r11)
930	vmovdqa		$D0,0x30(%rsp)
931	vpshufd		\$0xEE,$D1,$D3
932	vmovdqu		`16*6-64`($ctx),$D0
933	vpshufd		\$0x44,$D1,$D1
934	vmovdqa		$D3,-0x50(%r11)
935	vmovdqa		$D1,0x40(%rsp)
936	vpshufd		\$0xEE,$D2,$D4
937	vmovdqu		`16*7-64`($ctx),$D1
938	vpshufd		\$0x44,$D2,$D2
939	vmovdqa		$D4,-0x40(%r11)
940	vmovdqa		$D2,0x50(%rsp)
941	vpshufd		\$0xEE,$D0,$D3
942	vmovdqu		`16*8-64`($ctx),$D2
943	vpshufd		\$0x44,$D0,$D0
944	vmovdqa		$D3,-0x30(%r11)
945	vmovdqa		$D0,0x60(%rsp)
946	vpshufd		\$0xEE,$D1,$D4
947	vpshufd		\$0x44,$D1,$D1
948	vmovdqa		$D4,-0x20(%r11)
949	vmovdqa		$D1,0x70(%rsp)
950	vpshufd		\$0xEE,$D2,$D3
951	 vmovdqa	0x00(%rsp),$D4		# preload r0^2
952	vpshufd		\$0x44,$D2,$D2
953	vmovdqa		$D3,-0x10(%r11)
954	vmovdqa		$D2,0x80(%rsp)
955
956	jmp		.Loop_avx
957
958.align	32
959.Loop_avx:
960	################################################################
961	# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
962	# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
963	#   \___________________/
964	# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
965	# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
966	#   \___________________/ \____________________/
967	#
968	# Note that we start with inp[2:3]*r^2. This is because it
969	# doesn't depend on reduction in previous iteration.
970	################################################################
971	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
972	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
973	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
974	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
975	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
976	#
977	# though note that $Tx and $Hx are "reversed" in this section,
978	# and $D4 is preloaded with r0^2...
979
980	vpmuludq	$T0,$D4,$D0		# d0 = h0*r0
981	vpmuludq	$T1,$D4,$D1		# d1 = h1*r0
982	  vmovdqa	$H2,0x20(%r11)				# offload hash
983	vpmuludq	$T2,$D4,$D2		# d3 = h2*r0
984	 vmovdqa	0x10(%rsp),$H2		# r1^2
985	vpmuludq	$T3,$D4,$D3		# d3 = h3*r0
986	vpmuludq	$T4,$D4,$D4		# d4 = h4*r0
987
988	  vmovdqa	$H0,0x00(%r11)				#
989	vpmuludq	0x20(%rsp),$T4,$H0	# h4*s1
990	  vmovdqa	$H1,0x10(%r11)				#
991	vpmuludq	$T3,$H2,$H1		# h3*r1
992	vpaddq		$H0,$D0,$D0		# d0 += h4*s1
993	vpaddq		$H1,$D4,$D4		# d4 += h3*r1
994	  vmovdqa	$H3,0x30(%r11)				#
995	vpmuludq	$T2,$H2,$H0		# h2*r1
996	vpmuludq	$T1,$H2,$H1		# h1*r1
997	vpaddq		$H0,$D3,$D3		# d3 += h2*r1
998	 vmovdqa	0x30(%rsp),$H3		# r2^2
999	vpaddq		$H1,$D2,$D2		# d2 += h1*r1
1000	  vmovdqa	$H4,0x40(%r11)				#
1001	vpmuludq	$T0,$H2,$H2		# h0*r1
1002	 vpmuludq	$T2,$H3,$H0		# h2*r2
1003	vpaddq		$H2,$D1,$D1		# d1 += h0*r1
1004
1005	 vmovdqa	0x40(%rsp),$H4		# s2^2
1006	vpaddq		$H0,$D4,$D4		# d4 += h2*r2
1007	vpmuludq	$T1,$H3,$H1		# h1*r2
1008	vpmuludq	$T0,$H3,$H3		# h0*r2
1009	vpaddq		$H1,$D3,$D3		# d3 += h1*r2
1010	 vmovdqa	0x50(%rsp),$H2		# r3^2
1011	vpaddq		$H3,$D2,$D2		# d2 += h0*r2
1012	vpmuludq	$T4,$H4,$H0		# h4*s2
1013	vpmuludq	$T3,$H4,$H4		# h3*s2
1014	vpaddq		$H0,$D1,$D1		# d1 += h4*s2
1015	 vmovdqa	0x60(%rsp),$H3		# s3^2
1016	vpaddq		$H4,$D0,$D0		# d0 += h3*s2
1017
1018	 vmovdqa	0x80(%rsp),$H4		# s4^2
1019	vpmuludq	$T1,$H2,$H1		# h1*r3
1020	vpmuludq	$T0,$H2,$H2		# h0*r3
1021	vpaddq		$H1,$D4,$D4		# d4 += h1*r3
1022	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
1023	vpmuludq	$T4,$H3,$H0		# h4*s3
1024	vpmuludq	$T3,$H3,$H1		# h3*s3
1025	vpaddq		$H0,$D2,$D2		# d2 += h4*s3
1026	 vmovdqu	16*0($inp),$H0				# load input
1027	vpaddq		$H1,$D1,$D1		# d1 += h3*s3
1028	vpmuludq	$T2,$H3,$H3		# h2*s3
1029	 vpmuludq	$T2,$H4,$T2		# h2*s4
1030	vpaddq		$H3,$D0,$D0		# d0 += h2*s3
1031
1032	 vmovdqu	16*1($inp),$H1				#
1033	vpaddq		$T2,$D1,$D1		# d1 += h2*s4
1034	vpmuludq	$T3,$H4,$T3		# h3*s4
1035	vpmuludq	$T4,$H4,$T4		# h4*s4
1036	 vpsrldq	\$6,$H0,$H2				# splat input
1037	vpaddq		$T3,$D2,$D2		# d2 += h3*s4
1038	vpaddq		$T4,$D3,$D3		# d3 += h4*s4
1039	 vpsrldq	\$6,$H1,$H3				#
1040	vpmuludq	0x70(%rsp),$T0,$T4	# h0*r4
1041	vpmuludq	$T1,$H4,$T0		# h1*s4
1042	 vpunpckhqdq	$H1,$H0,$H4		# 4
1043	vpaddq		$T4,$D4,$D4		# d4 += h0*r4
1044	 vmovdqa	-0x90(%r11),$T4		# r0^4
1045	vpaddq		$T0,$D0,$D0		# d0 += h1*s4
1046
1047	vpunpcklqdq	$H1,$H0,$H0		# 0:1
1048	vpunpcklqdq	$H3,$H2,$H3		# 2:3
1049
1050	#vpsrlq		\$40,$H4,$H4		# 4
1051	vpsrldq		\$`40/8`,$H4,$H4	# 4
1052	vpsrlq		\$26,$H0,$H1
1053	vpand		$MASK,$H0,$H0		# 0
1054	vpsrlq		\$4,$H3,$H2
1055	vpand		$MASK,$H1,$H1		# 1
1056	vpand		0(%rcx),$H4,$H4		# .Lmask24
1057	vpsrlq		\$30,$H3,$H3
1058	vpand		$MASK,$H2,$H2		# 2
1059	vpand		$MASK,$H3,$H3		# 3
1060	vpor		32(%rcx),$H4,$H4	# padbit, yes, always
1061
1062	vpaddq		0x00(%r11),$H0,$H0	# add hash value
1063	vpaddq		0x10(%r11),$H1,$H1
1064	vpaddq		0x20(%r11),$H2,$H2
1065	vpaddq		0x30(%r11),$H3,$H3
1066	vpaddq		0x40(%r11),$H4,$H4
1067
1068	lea		16*2($inp),%rax
1069	lea		16*4($inp),$inp
1070	sub		\$64,$len
1071	cmovc		%rax,$inp
1072
1073	################################################################
1074	# Now we accumulate (inp[0:1]+hash)*r^4
1075	################################################################
1076	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1077	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1078	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1079	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1080	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1081
1082	vpmuludq	$H0,$T4,$T0		# h0*r0
1083	vpmuludq	$H1,$T4,$T1		# h1*r0
1084	vpaddq		$T0,$D0,$D0
1085	vpaddq		$T1,$D1,$D1
1086	 vmovdqa	-0x80(%r11),$T2		# r1^4
1087	vpmuludq	$H2,$T4,$T0		# h2*r0
1088	vpmuludq	$H3,$T4,$T1		# h3*r0
1089	vpaddq		$T0,$D2,$D2
1090	vpaddq		$T1,$D3,$D3
1091	vpmuludq	$H4,$T4,$T4		# h4*r0
1092	 vpmuludq	-0x70(%r11),$H4,$T0	# h4*s1
1093	vpaddq		$T4,$D4,$D4
1094
1095	vpaddq		$T0,$D0,$D0		# d0 += h4*s1
1096	vpmuludq	$H2,$T2,$T1		# h2*r1
1097	vpmuludq	$H3,$T2,$T0		# h3*r1
1098	vpaddq		$T1,$D3,$D3		# d3 += h2*r1
1099	 vmovdqa	-0x60(%r11),$T3		# r2^4
1100	vpaddq		$T0,$D4,$D4		# d4 += h3*r1
1101	vpmuludq	$H1,$T2,$T1		# h1*r1
1102	vpmuludq	$H0,$T2,$T2		# h0*r1
1103	vpaddq		$T1,$D2,$D2		# d2 += h1*r1
1104	vpaddq		$T2,$D1,$D1		# d1 += h0*r1
1105
1106	 vmovdqa	-0x50(%r11),$T4		# s2^4
1107	vpmuludq	$H2,$T3,$T0		# h2*r2
1108	vpmuludq	$H1,$T3,$T1		# h1*r2
1109	vpaddq		$T0,$D4,$D4		# d4 += h2*r2
1110	vpaddq		$T1,$D3,$D3		# d3 += h1*r2
1111	 vmovdqa	-0x40(%r11),$T2		# r3^4
1112	vpmuludq	$H0,$T3,$T3		# h0*r2
1113	vpmuludq	$H4,$T4,$T0		# h4*s2
1114	vpaddq		$T3,$D2,$D2		# d2 += h0*r2
1115	vpaddq		$T0,$D1,$D1		# d1 += h4*s2
1116	 vmovdqa	-0x30(%r11),$T3		# s3^4
1117	vpmuludq	$H3,$T4,$T4		# h3*s2
1118	 vpmuludq	$H1,$T2,$T1		# h1*r3
1119	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
1120
1121	 vmovdqa	-0x10(%r11),$T4		# s4^4
1122	vpaddq		$T1,$D4,$D4		# d4 += h1*r3
1123	vpmuludq	$H0,$T2,$T2		# h0*r3
1124	vpmuludq	$H4,$T3,$T0		# h4*s3
1125	vpaddq		$T2,$D3,$D3		# d3 += h0*r3
1126	vpaddq		$T0,$D2,$D2		# d2 += h4*s3
1127	 vmovdqu	16*2($inp),$T0				# load input
1128	vpmuludq	$H3,$T3,$T2		# h3*s3
1129	vpmuludq	$H2,$T3,$T3		# h2*s3
1130	vpaddq		$T2,$D1,$D1		# d1 += h3*s3
1131	 vmovdqu	16*3($inp),$T1				#
1132	vpaddq		$T3,$D0,$D0		# d0 += h2*s3
1133
1134	vpmuludq	$H2,$T4,$H2		# h2*s4
1135	vpmuludq	$H3,$T4,$H3		# h3*s4
1136	 vpsrldq	\$6,$T0,$T2				# splat input
1137	vpaddq		$H2,$D1,$D1		# d1 += h2*s4
1138	vpmuludq	$H4,$T4,$H4		# h4*s4
1139	 vpsrldq	\$6,$T1,$T3				#
1140	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*s4
1141	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*s4
1142	vpmuludq	-0x20(%r11),$H0,$H4	# h0*r4
1143	vpmuludq	$H1,$T4,$H0
1144	 vpunpckhqdq	$T1,$T0,$T4		# 4
1145	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
1146	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
1147
1148	vpunpcklqdq	$T1,$T0,$T0		# 0:1
1149	vpunpcklqdq	$T3,$T2,$T3		# 2:3
1150
1151	#vpsrlq		\$40,$T4,$T4		# 4
1152	vpsrldq		\$`40/8`,$T4,$T4	# 4
1153	vpsrlq		\$26,$T0,$T1
1154	 vmovdqa	0x00(%rsp),$D4		# preload r0^2
1155	vpand		$MASK,$T0,$T0		# 0
1156	vpsrlq		\$4,$T3,$T2
1157	vpand		$MASK,$T1,$T1		# 1
1158	vpand		0(%rcx),$T4,$T4		# .Lmask24
1159	vpsrlq		\$30,$T3,$T3
1160	vpand		$MASK,$T2,$T2		# 2
1161	vpand		$MASK,$T3,$T3		# 3
1162	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
1163
1164	################################################################
1165	# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
1166	# and P. Schwabe
1167
1168	vpsrlq		\$26,$H3,$D3
1169	vpand		$MASK,$H3,$H3
1170	vpaddq		$D3,$H4,$H4		# h3 -> h4
1171
1172	vpsrlq		\$26,$H0,$D0
1173	vpand		$MASK,$H0,$H0
1174	vpaddq		$D0,$D1,$H1		# h0 -> h1
1175
1176	vpsrlq		\$26,$H4,$D0
1177	vpand		$MASK,$H4,$H4
1178
1179	vpsrlq		\$26,$H1,$D1
1180	vpand		$MASK,$H1,$H1
1181	vpaddq		$D1,$H2,$H2		# h1 -> h2
1182
1183	vpaddq		$D0,$H0,$H0
1184	vpsllq		\$2,$D0,$D0
1185	vpaddq		$D0,$H0,$H0		# h4 -> h0
1186
1187	vpsrlq		\$26,$H2,$D2
1188	vpand		$MASK,$H2,$H2
1189	vpaddq		$D2,$H3,$H3		# h2 -> h3
1190
1191	vpsrlq		\$26,$H0,$D0
1192	vpand		$MASK,$H0,$H0
1193	vpaddq		$D0,$H1,$H1		# h0 -> h1
1194
1195	vpsrlq		\$26,$H3,$D3
1196	vpand		$MASK,$H3,$H3
1197	vpaddq		$D3,$H4,$H4		# h3 -> h4
1198
1199	ja		.Loop_avx
1200
1201.Lskip_loop_avx:
1202	################################################################
1203	# multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1204
1205	vpshufd		\$0x10,$D4,$D4		# r0^n, xx12 -> x1x2
1206	add		\$32,$len
1207	jnz		.Long_tail_avx
1208
1209	vpaddq		$H2,$T2,$T2
1210	vpaddq		$H0,$T0,$T0
1211	vpaddq		$H1,$T1,$T1
1212	vpaddq		$H3,$T3,$T3
1213	vpaddq		$H4,$T4,$T4
1214
1215.Long_tail_avx:
1216	vmovdqa		$H2,0x20(%r11)
1217	vmovdqa		$H0,0x00(%r11)
1218	vmovdqa		$H1,0x10(%r11)
1219	vmovdqa		$H3,0x30(%r11)
1220	vmovdqa		$H4,0x40(%r11)
1221
1222	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1223	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1224	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1225	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1226	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1227
1228	vpmuludq	$T2,$D4,$D2		# d2 = h2*r0
1229	vpmuludq	$T0,$D4,$D0		# d0 = h0*r0
1230	 vpshufd	\$0x10,`16*1-64`($ctx),$H2		# r1^n
1231	vpmuludq	$T1,$D4,$D1		# d1 = h1*r0
1232	vpmuludq	$T3,$D4,$D3		# d3 = h3*r0
1233	vpmuludq	$T4,$D4,$D4		# d4 = h4*r0
1234
1235	vpmuludq	$T3,$H2,$H0		# h3*r1
1236	vpaddq		$H0,$D4,$D4		# d4 += h3*r1
1237	 vpshufd	\$0x10,`16*2-64`($ctx),$H3		# s1^n
1238	vpmuludq	$T2,$H2,$H1		# h2*r1
1239	vpaddq		$H1,$D3,$D3		# d3 += h2*r1
1240	 vpshufd	\$0x10,`16*3-64`($ctx),$H4		# r2^n
1241	vpmuludq	$T1,$H2,$H0		# h1*r1
1242	vpaddq		$H0,$D2,$D2		# d2 += h1*r1
1243	vpmuludq	$T0,$H2,$H2		# h0*r1
1244	vpaddq		$H2,$D1,$D1		# d1 += h0*r1
1245	vpmuludq	$T4,$H3,$H3		# h4*s1
1246	vpaddq		$H3,$D0,$D0		# d0 += h4*s1
1247
1248	 vpshufd	\$0x10,`16*4-64`($ctx),$H2		# s2^n
1249	vpmuludq	$T2,$H4,$H1		# h2*r2
1250	vpaddq		$H1,$D4,$D4		# d4 += h2*r2
1251	vpmuludq	$T1,$H4,$H0		# h1*r2
1252	vpaddq		$H0,$D3,$D3		# d3 += h1*r2
1253	 vpshufd	\$0x10,`16*5-64`($ctx),$H3		# r3^n
1254	vpmuludq	$T0,$H4,$H4		# h0*r2
1255	vpaddq		$H4,$D2,$D2		# d2 += h0*r2
1256	vpmuludq	$T4,$H2,$H1		# h4*s2
1257	vpaddq		$H1,$D1,$D1		# d1 += h4*s2
1258	 vpshufd	\$0x10,`16*6-64`($ctx),$H4		# s3^n
1259	vpmuludq	$T3,$H2,$H2		# h3*s2
1260	vpaddq		$H2,$D0,$D0		# d0 += h3*s2
1261
1262	vpmuludq	$T1,$H3,$H0		# h1*r3
1263	vpaddq		$H0,$D4,$D4		# d4 += h1*r3
1264	vpmuludq	$T0,$H3,$H3		# h0*r3
1265	vpaddq		$H3,$D3,$D3		# d3 += h0*r3
1266	 vpshufd	\$0x10,`16*7-64`($ctx),$H2		# r4^n
1267	vpmuludq	$T4,$H4,$H1		# h4*s3
1268	vpaddq		$H1,$D2,$D2		# d2 += h4*s3
1269	 vpshufd	\$0x10,`16*8-64`($ctx),$H3		# s4^n
1270	vpmuludq	$T3,$H4,$H0		# h3*s3
1271	vpaddq		$H0,$D1,$D1		# d1 += h3*s3
1272	vpmuludq	$T2,$H4,$H4		# h2*s3
1273	vpaddq		$H4,$D0,$D0		# d0 += h2*s3
1274
1275	vpmuludq	$T0,$H2,$H2		# h0*r4
1276	vpaddq		$H2,$D4,$D4		# h4 = d4 + h0*r4
1277	vpmuludq	$T4,$H3,$H1		# h4*s4
1278	vpaddq		$H1,$D3,$D3		# h3 = d3 + h4*s4
1279	vpmuludq	$T3,$H3,$H0		# h3*s4
1280	vpaddq		$H0,$D2,$D2		# h2 = d2 + h3*s4
1281	vpmuludq	$T2,$H3,$H1		# h2*s4
1282	vpaddq		$H1,$D1,$D1		# h1 = d1 + h2*s4
1283	vpmuludq	$T1,$H3,$H3		# h1*s4
1284	vpaddq		$H3,$D0,$D0		# h0 = d0 + h1*s4
1285
1286	jz		.Lshort_tail_avx
1287
1288	vmovdqu		16*0($inp),$H0		# load input
1289	vmovdqu		16*1($inp),$H1
1290
1291	vpsrldq		\$6,$H0,$H2		# splat input
1292	vpsrldq		\$6,$H1,$H3
1293	vpunpckhqdq	$H1,$H0,$H4		# 4
1294	vpunpcklqdq	$H1,$H0,$H0		# 0:1
1295	vpunpcklqdq	$H3,$H2,$H3		# 2:3
1296
1297	vpsrlq		\$40,$H4,$H4		# 4
1298	vpsrlq		\$26,$H0,$H1
1299	vpand		$MASK,$H0,$H0		# 0
1300	vpsrlq		\$4,$H3,$H2
1301	vpand		$MASK,$H1,$H1		# 1
1302	vpsrlq		\$30,$H3,$H3
1303	vpand		$MASK,$H2,$H2		# 2
1304	vpand		$MASK,$H3,$H3		# 3
1305	vpor		32(%rcx),$H4,$H4	# padbit, yes, always
1306
1307	vpshufd		\$0x32,`16*0-64`($ctx),$T4	# r0^n, 34xx -> x3x4
1308	vpaddq		0x00(%r11),$H0,$H0
1309	vpaddq		0x10(%r11),$H1,$H1
1310	vpaddq		0x20(%r11),$H2,$H2
1311	vpaddq		0x30(%r11),$H3,$H3
1312	vpaddq		0x40(%r11),$H4,$H4
1313
1314	################################################################
1315	# multiply (inp[0:1]+hash) by r^4:r^3 and accumulate
1316
1317	vpmuludq	$H0,$T4,$T0		# h0*r0
1318	vpaddq		$T0,$D0,$D0		# d0 += h0*r0
1319	vpmuludq	$H1,$T4,$T1		# h1*r0
1320	vpaddq		$T1,$D1,$D1		# d1 += h1*r0
1321	vpmuludq	$H2,$T4,$T0		# h2*r0
1322	vpaddq		$T0,$D2,$D2		# d2 += h2*r0
1323	 vpshufd	\$0x32,`16*1-64`($ctx),$T2		# r1^n
1324	vpmuludq	$H3,$T4,$T1		# h3*r0
1325	vpaddq		$T1,$D3,$D3		# d3 += h3*r0
1326	vpmuludq	$H4,$T4,$T4		# h4*r0
1327	vpaddq		$T4,$D4,$D4		# d4 += h4*r0
1328
1329	vpmuludq	$H3,$T2,$T0		# h3*r1
1330	vpaddq		$T0,$D4,$D4		# d4 += h3*r1
1331	 vpshufd	\$0x32,`16*2-64`($ctx),$T3		# s1
1332	vpmuludq	$H2,$T2,$T1		# h2*r1
1333	vpaddq		$T1,$D3,$D3		# d3 += h2*r1
1334	 vpshufd	\$0x32,`16*3-64`($ctx),$T4		# r2
1335	vpmuludq	$H1,$T2,$T0		# h1*r1
1336	vpaddq		$T0,$D2,$D2		# d2 += h1*r1
1337	vpmuludq	$H0,$T2,$T2		# h0*r1
1338	vpaddq		$T2,$D1,$D1		# d1 += h0*r1
1339	vpmuludq	$H4,$T3,$T3		# h4*s1
1340	vpaddq		$T3,$D0,$D0		# d0 += h4*s1
1341
1342	 vpshufd	\$0x32,`16*4-64`($ctx),$T2		# s2
1343	vpmuludq	$H2,$T4,$T1		# h2*r2
1344	vpaddq		$T1,$D4,$D4		# d4 += h2*r2
1345	vpmuludq	$H1,$T4,$T0		# h1*r2
1346	vpaddq		$T0,$D3,$D3		# d3 += h1*r2
1347	 vpshufd	\$0x32,`16*5-64`($ctx),$T3		# r3
1348	vpmuludq	$H0,$T4,$T4		# h0*r2
1349	vpaddq		$T4,$D2,$D2		# d2 += h0*r2
1350	vpmuludq	$H4,$T2,$T1		# h4*s2
1351	vpaddq		$T1,$D1,$D1		# d1 += h4*s2
1352	 vpshufd	\$0x32,`16*6-64`($ctx),$T4		# s3
1353	vpmuludq	$H3,$T2,$T2		# h3*s2
1354	vpaddq		$T2,$D0,$D0		# d0 += h3*s2
1355
1356	vpmuludq	$H1,$T3,$T0		# h1*r3
1357	vpaddq		$T0,$D4,$D4		# d4 += h1*r3
1358	vpmuludq	$H0,$T3,$T3		# h0*r3
1359	vpaddq		$T3,$D3,$D3		# d3 += h0*r3
1360	 vpshufd	\$0x32,`16*7-64`($ctx),$T2		# r4
1361	vpmuludq	$H4,$T4,$T1		# h4*s3
1362	vpaddq		$T1,$D2,$D2		# d2 += h4*s3
1363	 vpshufd	\$0x32,`16*8-64`($ctx),$T3		# s4
1364	vpmuludq	$H3,$T4,$T0		# h3*s3
1365	vpaddq		$T0,$D1,$D1		# d1 += h3*s3
1366	vpmuludq	$H2,$T4,$T4		# h2*s3
1367	vpaddq		$T4,$D0,$D0		# d0 += h2*s3
1368
1369	vpmuludq	$H0,$T2,$T2		# h0*r4
1370	vpaddq		$T2,$D4,$D4		# d4 += h0*r4
1371	vpmuludq	$H4,$T3,$T1		# h4*s4
1372	vpaddq		$T1,$D3,$D3		# d3 += h4*s4
1373	vpmuludq	$H3,$T3,$T0		# h3*s4
1374	vpaddq		$T0,$D2,$D2		# d2 += h3*s4
1375	vpmuludq	$H2,$T3,$T1		# h2*s4
1376	vpaddq		$T1,$D1,$D1		# d1 += h2*s4
1377	vpmuludq	$H1,$T3,$T3		# h1*s4
1378	vpaddq		$T3,$D0,$D0		# d0 += h1*s4
1379
1380.Lshort_tail_avx:
1381	################################################################
1382	# horizontal addition
1383
1384	vpsrldq		\$8,$D4,$T4
1385	vpsrldq		\$8,$D3,$T3
1386	vpsrldq		\$8,$D1,$T1
1387	vpsrldq		\$8,$D0,$T0
1388	vpsrldq		\$8,$D2,$T2
1389	vpaddq		$T3,$D3,$D3
1390	vpaddq		$T4,$D4,$D4
1391	vpaddq		$T0,$D0,$D0
1392	vpaddq		$T1,$D1,$D1
1393	vpaddq		$T2,$D2,$D2
1394
1395	################################################################
1396	# lazy reduction
1397
1398	vpsrlq		\$26,$D3,$H3
1399	vpand		$MASK,$D3,$D3
1400	vpaddq		$H3,$D4,$D4		# h3 -> h4
1401
1402	vpsrlq		\$26,$D0,$H0
1403	vpand		$MASK,$D0,$D0
1404	vpaddq		$H0,$D1,$D1		# h0 -> h1
1405
1406	vpsrlq		\$26,$D4,$H4
1407	vpand		$MASK,$D4,$D4
1408
1409	vpsrlq		\$26,$D1,$H1
1410	vpand		$MASK,$D1,$D1
1411	vpaddq		$H1,$D2,$D2		# h1 -> h2
1412
1413	vpaddq		$H4,$D0,$D0
1414	vpsllq		\$2,$H4,$H4
1415	vpaddq		$H4,$D0,$D0		# h4 -> h0
1416
1417	vpsrlq		\$26,$D2,$H2
1418	vpand		$MASK,$D2,$D2
1419	vpaddq		$H2,$D3,$D3		# h2 -> h3
1420
1421	vpsrlq		\$26,$D0,$H0
1422	vpand		$MASK,$D0,$D0
1423	vpaddq		$H0,$D1,$D1		# h0 -> h1
1424
1425	vpsrlq		\$26,$D3,$H3
1426	vpand		$MASK,$D3,$D3
1427	vpaddq		$H3,$D4,$D4		# h3 -> h4
1428
1429	vmovd		$D0,`4*0-48-64`($ctx)	# save partially reduced
1430	vmovd		$D1,`4*1-48-64`($ctx)
1431	vmovd		$D2,`4*2-48-64`($ctx)
1432	vmovd		$D3,`4*3-48-64`($ctx)
1433	vmovd		$D4,`4*4-48-64`($ctx)
1434___
1435$code.=<<___	if ($win64);
1436	vmovdqa		0x50(%r11),%xmm6
1437	vmovdqa		0x60(%r11),%xmm7
1438	vmovdqa		0x70(%r11),%xmm8
1439	vmovdqa		0x80(%r11),%xmm9
1440	vmovdqa		0x90(%r11),%xmm10
1441	vmovdqa		0xa0(%r11),%xmm11
1442	vmovdqa		0xb0(%r11),%xmm12
1443	vmovdqa		0xc0(%r11),%xmm13
1444	vmovdqa		0xd0(%r11),%xmm14
1445	vmovdqa		0xe0(%r11),%xmm15
1446	lea		0xf8(%r11),%rsp
1447.Ldo_avx_epilogue:
1448___
1449$code.=<<___	if (!$win64);
1450	lea		-8(%r10),%rsp
1451.cfi_def_cfa_register	%rsp
1452___
1453$code.=<<___;
1454	vzeroupper
1455	RET
1456.cfi_endproc
1457___
1458&end_function("poly1305_blocks_avx");
1459
1460&declare_function("poly1305_emit_avx", 32, 3);
1461$code.=<<___;
1462	cmpl	\$0,20($ctx)	# is_base2_26?
1463	je	.Lemit
1464
1465	mov	0($ctx),%eax	# load hash value base 2^26
1466	mov	4($ctx),%ecx
1467	mov	8($ctx),%r8d
1468	mov	12($ctx),%r11d
1469	mov	16($ctx),%r10d
1470
1471	shl	\$26,%rcx	# base 2^26 -> base 2^64
1472	mov	%r8,%r9
1473	shl	\$52,%r8
1474	add	%rcx,%rax
1475	shr	\$12,%r9
1476	add	%rax,%r8	# h0
1477	adc	\$0,%r9
1478
1479	shl	\$14,%r11
1480	mov	%r10,%rax
1481	shr	\$24,%r10
1482	add	%r11,%r9
1483	shl	\$40,%rax
1484	add	%rax,%r9	# h1
1485	adc	\$0,%r10	# h2
1486
1487	mov	%r10,%rax	# could be partially reduced, so reduce
1488	mov	%r10,%rcx
1489	and	\$3,%r10
1490	shr	\$2,%rax
1491	and	\$-4,%rcx
1492	add	%rcx,%rax
1493	add	%rax,%r8
1494	adc	\$0,%r9
1495	adc	\$0,%r10
1496
1497	mov	%r8,%rax
1498	add	\$5,%r8		# compare to modulus
1499	mov	%r9,%rcx
1500	adc	\$0,%r9
1501	adc	\$0,%r10
1502	shr	\$2,%r10	# did 130-bit value overflow?
1503	cmovnz	%r8,%rax
1504	cmovnz	%r9,%rcx
1505
1506	add	0($nonce),%rax	# accumulate nonce
1507	adc	8($nonce),%rcx
1508	mov	%rax,0($mac)	# write result
1509	mov	%rcx,8($mac)
1510
1511	RET
1512___
1513&end_function("poly1305_emit_avx");
1514
1515if ($avx>1) {
1516
1517my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) =
1518    map("%ymm$_",(0..15));
1519my $S4=$MASK;
1520
1521sub poly1305_blocks_avxN {
1522	my ($avx512) = @_;
1523	my $suffix = $avx512 ? "_avx512" : "";
1524$code.=<<___;
1525.cfi_startproc
1526	mov	20($ctx),%r8d		# is_base2_26
1527	cmp	\$128,$len
1528	jae	.Lblocks_avx2$suffix
1529	test	%r8d,%r8d
1530	jz	.Lblocks
1531
1532.Lblocks_avx2$suffix:
1533	and	\$-16,$len
1534	jz	.Lno_data_avx2$suffix
1535
1536	vzeroupper
1537
1538	test	%r8d,%r8d
1539	jz	.Lbase2_64_avx2$suffix
1540
1541	test	\$63,$len
1542	jz	.Leven_avx2$suffix
1543
1544	push	%rbp
1545.cfi_push	%rbp
1546	mov 	%rsp,%rbp
1547	push	%rbx
1548.cfi_push	%rbx
1549	push	%r12
1550.cfi_push	%r12
1551	push	%r13
1552.cfi_push	%r13
1553	push	%r14
1554.cfi_push	%r14
1555	push	%r15
1556.cfi_push	%r15
1557.Lblocks_avx2_body$suffix:
1558
1559	mov	$len,%r15		# reassign $len
1560
1561	mov	0($ctx),$d1		# load hash value
1562	mov	8($ctx),$d2
1563	mov	16($ctx),$h2#d
1564
1565	mov	24($ctx),$r0		# load r
1566	mov	32($ctx),$s1
1567
1568	################################# base 2^26 -> base 2^64
1569	mov	$d1#d,$h0#d
1570	and	\$`-1*(1<<31)`,$d1
1571	mov	$d2,$r1			# borrow $r1
1572	mov	$d2#d,$h1#d
1573	and	\$`-1*(1<<31)`,$d2
1574
1575	shr	\$6,$d1
1576	shl	\$52,$r1
1577	add	$d1,$h0
1578	shr	\$12,$h1
1579	shr	\$18,$d2
1580	add	$r1,$h0
1581	adc	$d2,$h1
1582
1583	mov	$h2,$d1
1584	shl	\$40,$d1
1585	shr	\$24,$h2
1586	add	$d1,$h1
1587	adc	\$0,$h2			# can be partially reduced...
1588
1589	mov	\$-4,$d2		# ... so reduce
1590	mov	$h2,$d1
1591	and	$h2,$d2
1592	shr	\$2,$d1
1593	and	\$3,$h2
1594	add	$d2,$d1			# =*5
1595	add	$d1,$h0
1596	adc	\$0,$h1
1597	adc	\$0,$h2
1598
1599	mov	$s1,$r1
1600	mov	$s1,%rax
1601	shr	\$2,$s1
1602	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
1603
1604.Lbase2_26_pre_avx2$suffix:
1605	add	0($inp),$h0		# accumulate input
1606	adc	8($inp),$h1
1607	lea	16($inp),$inp
1608	adc	$padbit,$h2
1609	sub	\$16,%r15
1610
1611	call	__poly1305_block
1612	mov	$r1,%rax
1613
1614	test	\$63,%r15
1615	jnz	.Lbase2_26_pre_avx2$suffix
1616
1617	test	$padbit,$padbit		# if $padbit is zero,
1618	jz	.Lstore_base2_64_avx2$suffix	# store hash in base 2^64 format
1619
1620	################################# base 2^64 -> base 2^26
1621	mov	$h0,%rax
1622	mov	$h0,%rdx
1623	shr	\$52,$h0
1624	mov	$h1,$r0
1625	mov	$h1,$r1
1626	shr	\$26,%rdx
1627	and	\$0x3ffffff,%rax	# h[0]
1628	shl	\$12,$r0
1629	and	\$0x3ffffff,%rdx	# h[1]
1630	shr	\$14,$h1
1631	or	$r0,$h0
1632	shl	\$24,$h2
1633	and	\$0x3ffffff,$h0		# h[2]
1634	shr	\$40,$r1
1635	and	\$0x3ffffff,$h1		# h[3]
1636	or	$r1,$h2			# h[4]
1637
1638	test	%r15,%r15
1639	jz	.Lstore_base2_26_avx2$suffix
1640
1641	vmovd	%rax#d,%x#$H0
1642	vmovd	%rdx#d,%x#$H1
1643	vmovd	$h0#d,%x#$H2
1644	vmovd	$h1#d,%x#$H3
1645	vmovd	$h2#d,%x#$H4
1646	jmp	.Lproceed_avx2$suffix
1647
1648.align	32
1649.Lstore_base2_64_avx2$suffix:
1650	mov	$h0,0($ctx)
1651	mov	$h1,8($ctx)
1652	mov	$h2,16($ctx)		# note that is_base2_26 is zeroed
1653	jmp	.Ldone_avx2$suffix
1654
1655.align	16
1656.Lstore_base2_26_avx2$suffix:
1657	mov	%rax#d,0($ctx)		# store hash value base 2^26
1658	mov	%rdx#d,4($ctx)
1659	mov	$h0#d,8($ctx)
1660	mov	$h1#d,12($ctx)
1661	mov	$h2#d,16($ctx)
1662.align	16
1663.Ldone_avx2$suffix:
1664	pop 		%r15
1665.cfi_restore	%r15
1666	pop 		%r14
1667.cfi_restore	%r14
1668	pop 		%r13
1669.cfi_restore	%r13
1670	pop 		%r12
1671.cfi_restore	%r12
1672	pop 		%rbx
1673.cfi_restore	%rbx
1674	pop 		%rbp
1675.cfi_restore 	%rbp
1676.Lno_data_avx2$suffix:
1677.Lblocks_avx2_epilogue$suffix:
1678	RET
1679.cfi_endproc
1680
1681.align	32
1682.Lbase2_64_avx2$suffix:
1683.cfi_startproc
1684	push	%rbp
1685.cfi_push	%rbp
1686	mov 	%rsp,%rbp
1687	push	%rbx
1688.cfi_push	%rbx
1689	push	%r12
1690.cfi_push	%r12
1691	push	%r13
1692.cfi_push	%r13
1693	push	%r14
1694.cfi_push	%r14
1695	push	%r15
1696.cfi_push	%r15
1697.Lbase2_64_avx2_body$suffix:
1698
1699	mov	$len,%r15		# reassign $len
1700
1701	mov	24($ctx),$r0		# load r
1702	mov	32($ctx),$s1
1703
1704	mov	0($ctx),$h0		# load hash value
1705	mov	8($ctx),$h1
1706	mov	16($ctx),$h2#d
1707
1708	mov	$s1,$r1
1709	mov	$s1,%rax
1710	shr	\$2,$s1
1711	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
1712
1713	test	\$63,$len
1714	jz	.Linit_avx2$suffix
1715
1716.Lbase2_64_pre_avx2$suffix:
1717	add	0($inp),$h0		# accumulate input
1718	adc	8($inp),$h1
1719	lea	16($inp),$inp
1720	adc	$padbit,$h2
1721	sub	\$16,%r15
1722
1723	call	__poly1305_block
1724	mov	$r1,%rax
1725
1726	test	\$63,%r15
1727	jnz	.Lbase2_64_pre_avx2$suffix
1728
1729.Linit_avx2$suffix:
1730	################################# base 2^64 -> base 2^26
1731	mov	$h0,%rax
1732	mov	$h0,%rdx
1733	shr	\$52,$h0
1734	mov	$h1,$d1
1735	mov	$h1,$d2
1736	shr	\$26,%rdx
1737	and	\$0x3ffffff,%rax	# h[0]
1738	shl	\$12,$d1
1739	and	\$0x3ffffff,%rdx	# h[1]
1740	shr	\$14,$h1
1741	or	$d1,$h0
1742	shl	\$24,$h2
1743	and	\$0x3ffffff,$h0		# h[2]
1744	shr	\$40,$d2
1745	and	\$0x3ffffff,$h1		# h[3]
1746	or	$d2,$h2			# h[4]
1747
1748	vmovd	%rax#d,%x#$H0
1749	vmovd	%rdx#d,%x#$H1
1750	vmovd	$h0#d,%x#$H2
1751	vmovd	$h1#d,%x#$H3
1752	vmovd	$h2#d,%x#$H4
1753	movl	\$1,20($ctx)		# set is_base2_26
1754
1755	call	__poly1305_init_avx
1756
1757.Lproceed_avx2$suffix:
1758	mov	%r15,$len			# restore $len
1759___
1760$code.=<<___ if (!$kernel);
1761	mov	OPENSSL_ia32cap_P+8(%rip),%r9d
1762	mov	\$`(1<<31|1<<30|1<<16)`,%r11d
1763___
1764$code.=<<___;
1765	pop 		%r15
1766.cfi_restore	%r15
1767	pop 		%r14
1768.cfi_restore	%r14
1769	pop 		%r13
1770.cfi_restore	%r13
1771	pop 		%r12
1772.cfi_restore	%r12
1773	pop 		%rbx
1774.cfi_restore	%rbx
1775	pop 		%rbp
1776.cfi_restore 	%rbp
1777.Lbase2_64_avx2_epilogue$suffix:
1778	jmp	.Ldo_avx2$suffix
1779.cfi_endproc
1780
1781.align	32
1782.Leven_avx2$suffix:
1783.cfi_startproc
1784___
1785$code.=<<___ if (!$kernel);
1786	mov		OPENSSL_ia32cap_P+8(%rip),%r9d
1787___
1788$code.=<<___;
1789	vmovd		4*0($ctx),%x#$H0	# load hash value base 2^26
1790	vmovd		4*1($ctx),%x#$H1
1791	vmovd		4*2($ctx),%x#$H2
1792	vmovd		4*3($ctx),%x#$H3
1793	vmovd		4*4($ctx),%x#$H4
1794
1795.Ldo_avx2$suffix:
1796___
1797$code.=<<___		if (!$kernel && $avx>2);
1798	cmp		\$512,$len
1799	jb		.Lskip_avx512
1800	and		%r11d,%r9d
1801	test		\$`1<<16`,%r9d		# check for AVX512F
1802	jnz		.Lblocks_avx512
1803.Lskip_avx512$suffix:
1804___
1805$code.=<<___ if ($avx > 2 && $avx512 && $kernel);
1806	cmp		\$512,$len
1807	jae		.Lblocks_avx512
1808___
1809$code.=<<___	if (!$win64);
1810	lea		8(%rsp),%r10
1811.cfi_def_cfa_register	%r10
1812	sub		\$0x128,%rsp
1813___
1814$code.=<<___	if ($win64);
1815	lea		8(%rsp),%r10
1816	sub		\$0x1c8,%rsp
1817	vmovdqa		%xmm6,-0xb0(%r10)
1818	vmovdqa		%xmm7,-0xa0(%r10)
1819	vmovdqa		%xmm8,-0x90(%r10)
1820	vmovdqa		%xmm9,-0x80(%r10)
1821	vmovdqa		%xmm10,-0x70(%r10)
1822	vmovdqa		%xmm11,-0x60(%r10)
1823	vmovdqa		%xmm12,-0x50(%r10)
1824	vmovdqa		%xmm13,-0x40(%r10)
1825	vmovdqa		%xmm14,-0x30(%r10)
1826	vmovdqa		%xmm15,-0x20(%r10)
1827.Ldo_avx2_body$suffix:
1828___
1829$code.=<<___;
1830	lea		.Lconst(%rip),%rcx
1831	lea		48+64($ctx),$ctx	# size optimization
1832	vmovdqa		96(%rcx),$T0		# .Lpermd_avx2
1833
1834	# expand and copy pre-calculated table to stack
1835	vmovdqu		`16*0-64`($ctx),%x#$T2
1836	and		\$-512,%rsp
1837	vmovdqu		`16*1-64`($ctx),%x#$T3
1838	vmovdqu		`16*2-64`($ctx),%x#$T4
1839	vmovdqu		`16*3-64`($ctx),%x#$D0
1840	vmovdqu		`16*4-64`($ctx),%x#$D1
1841	vmovdqu		`16*5-64`($ctx),%x#$D2
1842	lea		0x90(%rsp),%rax		# size optimization
1843	vmovdqu		`16*6-64`($ctx),%x#$D3
1844	vpermd		$T2,$T0,$T2		# 00003412 -> 14243444
1845	vmovdqu		`16*7-64`($ctx),%x#$D4
1846	vpermd		$T3,$T0,$T3
1847	vmovdqu		`16*8-64`($ctx),%x#$MASK
1848	vpermd		$T4,$T0,$T4
1849	vmovdqa		$T2,0x00(%rsp)
1850	vpermd		$D0,$T0,$D0
1851	vmovdqa		$T3,0x20-0x90(%rax)
1852	vpermd		$D1,$T0,$D1
1853	vmovdqa		$T4,0x40-0x90(%rax)
1854	vpermd		$D2,$T0,$D2
1855	vmovdqa		$D0,0x60-0x90(%rax)
1856	vpermd		$D3,$T0,$D3
1857	vmovdqa		$D1,0x80-0x90(%rax)
1858	vpermd		$D4,$T0,$D4
1859	vmovdqa		$D2,0xa0-0x90(%rax)
1860	vpermd		$MASK,$T0,$MASK
1861	vmovdqa		$D3,0xc0-0x90(%rax)
1862	vmovdqa		$D4,0xe0-0x90(%rax)
1863	vmovdqa		$MASK,0x100-0x90(%rax)
1864	vmovdqa		64(%rcx),$MASK		# .Lmask26
1865
1866	################################################################
1867	# load input
1868	vmovdqu		16*0($inp),%x#$T0
1869	vmovdqu		16*1($inp),%x#$T1
1870	vinserti128	\$1,16*2($inp),$T0,$T0
1871	vinserti128	\$1,16*3($inp),$T1,$T1
1872	lea		16*4($inp),$inp
1873
1874	vpsrldq		\$6,$T0,$T2		# splat input
1875	vpsrldq		\$6,$T1,$T3
1876	vpunpckhqdq	$T1,$T0,$T4		# 4
1877	vpunpcklqdq	$T3,$T2,$T2		# 2:3
1878	vpunpcklqdq	$T1,$T0,$T0		# 0:1
1879
1880	vpsrlq		\$30,$T2,$T3
1881	vpsrlq		\$4,$T2,$T2
1882	vpsrlq		\$26,$T0,$T1
1883	vpsrlq		\$40,$T4,$T4		# 4
1884	vpand		$MASK,$T2,$T2		# 2
1885	vpand		$MASK,$T0,$T0		# 0
1886	vpand		$MASK,$T1,$T1		# 1
1887	vpand		$MASK,$T3,$T3		# 3
1888	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
1889
1890	vpaddq		$H2,$T2,$H2		# accumulate input
1891	sub		\$64,$len
1892	jz		.Ltail_avx2$suffix
1893	jmp		.Loop_avx2$suffix
1894
1895.align	32
1896.Loop_avx2$suffix:
1897	################################################################
1898	# ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4
1899	# ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3
1900	# ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2
1901	# ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1
1902	#   \________/\__________/
1903	################################################################
1904	#vpaddq		$H2,$T2,$H2		# accumulate input
1905	vpaddq		$H0,$T0,$H0
1906	vmovdqa		`32*0`(%rsp),$T0	# r0^4
1907	vpaddq		$H1,$T1,$H1
1908	vmovdqa		`32*1`(%rsp),$T1	# r1^4
1909	vpaddq		$H3,$T3,$H3
1910	vmovdqa		`32*3`(%rsp),$T2	# r2^4
1911	vpaddq		$H4,$T4,$H4
1912	vmovdqa		`32*6-0x90`(%rax),$T3	# s3^4
1913	vmovdqa		`32*8-0x90`(%rax),$S4	# s4^4
1914
1915	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1916	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1917	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1918	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1919	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1920	#
1921	# however, as h2 is "chronologically" first one available pull
1922	# corresponding operations up, so it's
1923	#
1924	# d4 = h2*r2   + h4*r0 + h3*r1             + h1*r3   + h0*r4
1925	# d3 = h2*r1   + h3*r0           + h1*r2   + h0*r3   + h4*5*r4
1926	# d2 = h2*r0           + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1927	# d1 = h2*5*r4 + h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3
1928	# d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2           + h1*5*r4
1929
1930	vpmuludq	$H2,$T0,$D2		# d2 = h2*r0
1931	vpmuludq	$H2,$T1,$D3		# d3 = h2*r1
1932	vpmuludq	$H2,$T2,$D4		# d4 = h2*r2
1933	vpmuludq	$H2,$T3,$D0		# d0 = h2*s3
1934	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
1935
1936	vpmuludq	$H0,$T1,$T4		# h0*r1
1937	vpmuludq	$H1,$T1,$H2		# h1*r1, borrow $H2 as temp
1938	vpaddq		$T4,$D1,$D1		# d1 += h0*r1
1939	vpaddq		$H2,$D2,$D2		# d2 += h1*r1
1940	vpmuludq	$H3,$T1,$T4		# h3*r1
1941	vpmuludq	`32*2`(%rsp),$H4,$H2	# h4*s1
1942	vpaddq		$T4,$D4,$D4		# d4 += h3*r1
1943	vpaddq		$H2,$D0,$D0		# d0 += h4*s1
1944	 vmovdqa	`32*4-0x90`(%rax),$T1	# s2
1945
1946	vpmuludq	$H0,$T0,$T4		# h0*r0
1947	vpmuludq	$H1,$T0,$H2		# h1*r0
1948	vpaddq		$T4,$D0,$D0		# d0 += h0*r0
1949	vpaddq		$H2,$D1,$D1		# d1 += h1*r0
1950	vpmuludq	$H3,$T0,$T4		# h3*r0
1951	vpmuludq	$H4,$T0,$H2		# h4*r0
1952	 vmovdqu	16*0($inp),%x#$T0	# load input
1953	vpaddq		$T4,$D3,$D3		# d3 += h3*r0
1954	vpaddq		$H2,$D4,$D4		# d4 += h4*r0
1955	 vinserti128	\$1,16*2($inp),$T0,$T0
1956
1957	vpmuludq	$H3,$T1,$T4		# h3*s2
1958	vpmuludq	$H4,$T1,$H2		# h4*s2
1959	 vmovdqu	16*1($inp),%x#$T1
1960	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
1961	vpaddq		$H2,$D1,$D1		# d1 += h4*s2
1962	 vmovdqa	`32*5-0x90`(%rax),$H2	# r3
1963	vpmuludq	$H1,$T2,$T4		# h1*r2
1964	vpmuludq	$H0,$T2,$T2		# h0*r2
1965	vpaddq		$T4,$D3,$D3		# d3 += h1*r2
1966	vpaddq		$T2,$D2,$D2		# d2 += h0*r2
1967	 vinserti128	\$1,16*3($inp),$T1,$T1
1968	 lea		16*4($inp),$inp
1969
1970	vpmuludq	$H1,$H2,$T4		# h1*r3
1971	vpmuludq	$H0,$H2,$H2		# h0*r3
1972	 vpsrldq	\$6,$T0,$T2		# splat input
1973	vpaddq		$T4,$D4,$D4		# d4 += h1*r3
1974	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
1975	vpmuludq	$H3,$T3,$T4		# h3*s3
1976	vpmuludq	$H4,$T3,$H2		# h4*s3
1977	 vpsrldq	\$6,$T1,$T3
1978	vpaddq		$T4,$D1,$D1		# d1 += h3*s3
1979	vpaddq		$H2,$D2,$D2		# d2 += h4*s3
1980	 vpunpckhqdq	$T1,$T0,$T4		# 4
1981
1982	vpmuludq	$H3,$S4,$H3		# h3*s4
1983	vpmuludq	$H4,$S4,$H4		# h4*s4
1984	 vpunpcklqdq	$T1,$T0,$T0		# 0:1
1985	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*r4
1986	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*r4
1987	 vpunpcklqdq	$T3,$T2,$T3		# 2:3
1988	vpmuludq	`32*7-0x90`(%rax),$H0,$H4	# h0*r4
1989	vpmuludq	$H1,$S4,$H0		# h1*s4
1990	vmovdqa		64(%rcx),$MASK		# .Lmask26
1991	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
1992	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
1993
1994	################################################################
1995	# lazy reduction (interleaved with tail of input splat)
1996
1997	vpsrlq		\$26,$H3,$D3
1998	vpand		$MASK,$H3,$H3
1999	vpaddq		$D3,$H4,$H4		# h3 -> h4
2000
2001	vpsrlq		\$26,$H0,$D0
2002	vpand		$MASK,$H0,$H0
2003	vpaddq		$D0,$D1,$H1		# h0 -> h1
2004
2005	vpsrlq		\$26,$H4,$D4
2006	vpand		$MASK,$H4,$H4
2007
2008	 vpsrlq		\$4,$T3,$T2
2009
2010	vpsrlq		\$26,$H1,$D1
2011	vpand		$MASK,$H1,$H1
2012	vpaddq		$D1,$H2,$H2		# h1 -> h2
2013
2014	vpaddq		$D4,$H0,$H0
2015	vpsllq		\$2,$D4,$D4
2016	vpaddq		$D4,$H0,$H0		# h4 -> h0
2017
2018	 vpand		$MASK,$T2,$T2		# 2
2019	 vpsrlq		\$26,$T0,$T1
2020
2021	vpsrlq		\$26,$H2,$D2
2022	vpand		$MASK,$H2,$H2
2023	vpaddq		$D2,$H3,$H3		# h2 -> h3
2024
2025	 vpaddq		$T2,$H2,$H2		# modulo-scheduled
2026	 vpsrlq		\$30,$T3,$T3
2027
2028	vpsrlq		\$26,$H0,$D0
2029	vpand		$MASK,$H0,$H0
2030	vpaddq		$D0,$H1,$H1		# h0 -> h1
2031
2032	 vpsrlq		\$40,$T4,$T4		# 4
2033
2034	vpsrlq		\$26,$H3,$D3
2035	vpand		$MASK,$H3,$H3
2036	vpaddq		$D3,$H4,$H4		# h3 -> h4
2037
2038	 vpand		$MASK,$T0,$T0		# 0
2039	 vpand		$MASK,$T1,$T1		# 1
2040	 vpand		$MASK,$T3,$T3		# 3
2041	 vpor		32(%rcx),$T4,$T4	# padbit, yes, always
2042
2043	sub		\$64,$len
2044	jnz		.Loop_avx2$suffix
2045
2046	.byte		0x66,0x90
2047.Ltail_avx2$suffix:
2048	################################################################
2049	# while above multiplications were by r^4 in all lanes, in last
2050	# iteration we multiply least significant lane by r^4 and most
2051	# significant one by r, so copy of above except that references
2052	# to the precomputed table are displaced by 4...
2053
2054	#vpaddq		$H2,$T2,$H2		# accumulate input
2055	vpaddq		$H0,$T0,$H0
2056	vmovdqu		`32*0+4`(%rsp),$T0	# r0^4
2057	vpaddq		$H1,$T1,$H1
2058	vmovdqu		`32*1+4`(%rsp),$T1	# r1^4
2059	vpaddq		$H3,$T3,$H3
2060	vmovdqu		`32*3+4`(%rsp),$T2	# r2^4
2061	vpaddq		$H4,$T4,$H4
2062	vmovdqu		`32*6+4-0x90`(%rax),$T3	# s3^4
2063	vmovdqu		`32*8+4-0x90`(%rax),$S4	# s4^4
2064
2065	vpmuludq	$H2,$T0,$D2		# d2 = h2*r0
2066	vpmuludq	$H2,$T1,$D3		# d3 = h2*r1
2067	vpmuludq	$H2,$T2,$D4		# d4 = h2*r2
2068	vpmuludq	$H2,$T3,$D0		# d0 = h2*s3
2069	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
2070
2071	vpmuludq	$H0,$T1,$T4		# h0*r1
2072	vpmuludq	$H1,$T1,$H2		# h1*r1
2073	vpaddq		$T4,$D1,$D1		# d1 += h0*r1
2074	vpaddq		$H2,$D2,$D2		# d2 += h1*r1
2075	vpmuludq	$H3,$T1,$T4		# h3*r1
2076	vpmuludq	`32*2+4`(%rsp),$H4,$H2	# h4*s1
2077	vpaddq		$T4,$D4,$D4		# d4 += h3*r1
2078	vpaddq		$H2,$D0,$D0		# d0 += h4*s1
2079
2080	vpmuludq	$H0,$T0,$T4		# h0*r0
2081	vpmuludq	$H1,$T0,$H2		# h1*r0
2082	vpaddq		$T4,$D0,$D0		# d0 += h0*r0
2083	 vmovdqu	`32*4+4-0x90`(%rax),$T1	# s2
2084	vpaddq		$H2,$D1,$D1		# d1 += h1*r0
2085	vpmuludq	$H3,$T0,$T4		# h3*r0
2086	vpmuludq	$H4,$T0,$H2		# h4*r0
2087	vpaddq		$T4,$D3,$D3		# d3 += h3*r0
2088	vpaddq		$H2,$D4,$D4		# d4 += h4*r0
2089
2090	vpmuludq	$H3,$T1,$T4		# h3*s2
2091	vpmuludq	$H4,$T1,$H2		# h4*s2
2092	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
2093	vpaddq		$H2,$D1,$D1		# d1 += h4*s2
2094	 vmovdqu	`32*5+4-0x90`(%rax),$H2	# r3
2095	vpmuludq	$H1,$T2,$T4		# h1*r2
2096	vpmuludq	$H0,$T2,$T2		# h0*r2
2097	vpaddq		$T4,$D3,$D3		# d3 += h1*r2
2098	vpaddq		$T2,$D2,$D2		# d2 += h0*r2
2099
2100	vpmuludq	$H1,$H2,$T4		# h1*r3
2101	vpmuludq	$H0,$H2,$H2		# h0*r3
2102	vpaddq		$T4,$D4,$D4		# d4 += h1*r3
2103	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
2104	vpmuludq	$H3,$T3,$T4		# h3*s3
2105	vpmuludq	$H4,$T3,$H2		# h4*s3
2106	vpaddq		$T4,$D1,$D1		# d1 += h3*s3
2107	vpaddq		$H2,$D2,$D2		# d2 += h4*s3
2108
2109	vpmuludq	$H3,$S4,$H3		# h3*s4
2110	vpmuludq	$H4,$S4,$H4		# h4*s4
2111	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*r4
2112	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*r4
2113	vpmuludq	`32*7+4-0x90`(%rax),$H0,$H4		# h0*r4
2114	vpmuludq	$H1,$S4,$H0		# h1*s4
2115	vmovdqa		64(%rcx),$MASK		# .Lmask26
2116	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
2117	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
2118
2119	################################################################
2120	# horizontal addition
2121
2122	vpsrldq		\$8,$D1,$T1
2123	vpsrldq		\$8,$H2,$T2
2124	vpsrldq		\$8,$H3,$T3
2125	vpsrldq		\$8,$H4,$T4
2126	vpsrldq		\$8,$H0,$T0
2127	vpaddq		$T1,$D1,$D1
2128	vpaddq		$T2,$H2,$H2
2129	vpaddq		$T3,$H3,$H3
2130	vpaddq		$T4,$H4,$H4
2131	vpaddq		$T0,$H0,$H0
2132
2133	vpermq		\$0x2,$H3,$T3
2134	vpermq		\$0x2,$H4,$T4
2135	vpermq		\$0x2,$H0,$T0
2136	vpermq		\$0x2,$D1,$T1
2137	vpermq		\$0x2,$H2,$T2
2138	vpaddq		$T3,$H3,$H3
2139	vpaddq		$T4,$H4,$H4
2140	vpaddq		$T0,$H0,$H0
2141	vpaddq		$T1,$D1,$D1
2142	vpaddq		$T2,$H2,$H2
2143
2144	################################################################
2145	# lazy reduction
2146
2147	vpsrlq		\$26,$H3,$D3
2148	vpand		$MASK,$H3,$H3
2149	vpaddq		$D3,$H4,$H4		# h3 -> h4
2150
2151	vpsrlq		\$26,$H0,$D0
2152	vpand		$MASK,$H0,$H0
2153	vpaddq		$D0,$D1,$H1		# h0 -> h1
2154
2155	vpsrlq		\$26,$H4,$D4
2156	vpand		$MASK,$H4,$H4
2157
2158	vpsrlq		\$26,$H1,$D1
2159	vpand		$MASK,$H1,$H1
2160	vpaddq		$D1,$H2,$H2		# h1 -> h2
2161
2162	vpaddq		$D4,$H0,$H0
2163	vpsllq		\$2,$D4,$D4
2164	vpaddq		$D4,$H0,$H0		# h4 -> h0
2165
2166	vpsrlq		\$26,$H2,$D2
2167	vpand		$MASK,$H2,$H2
2168	vpaddq		$D2,$H3,$H3		# h2 -> h3
2169
2170	vpsrlq		\$26,$H0,$D0
2171	vpand		$MASK,$H0,$H0
2172	vpaddq		$D0,$H1,$H1		# h0 -> h1
2173
2174	vpsrlq		\$26,$H3,$D3
2175	vpand		$MASK,$H3,$H3
2176	vpaddq		$D3,$H4,$H4		# h3 -> h4
2177
2178	vmovd		%x#$H0,`4*0-48-64`($ctx)# save partially reduced
2179	vmovd		%x#$H1,`4*1-48-64`($ctx)
2180	vmovd		%x#$H2,`4*2-48-64`($ctx)
2181	vmovd		%x#$H3,`4*3-48-64`($ctx)
2182	vmovd		%x#$H4,`4*4-48-64`($ctx)
2183___
2184$code.=<<___	if ($win64);
2185	vmovdqa		-0xb0(%r10),%xmm6
2186	vmovdqa		-0xa0(%r10),%xmm7
2187	vmovdqa		-0x90(%r10),%xmm8
2188	vmovdqa		-0x80(%r10),%xmm9
2189	vmovdqa		-0x70(%r10),%xmm10
2190	vmovdqa		-0x60(%r10),%xmm11
2191	vmovdqa		-0x50(%r10),%xmm12
2192	vmovdqa		-0x40(%r10),%xmm13
2193	vmovdqa		-0x30(%r10),%xmm14
2194	vmovdqa		-0x20(%r10),%xmm15
2195	lea		-8(%r10),%rsp
2196.Ldo_avx2_epilogue$suffix:
2197___
2198$code.=<<___	if (!$win64);
2199	lea		-8(%r10),%rsp
2200.cfi_def_cfa_register	%rsp
2201___
2202$code.=<<___;
2203	vzeroupper
2204	RET
2205.cfi_endproc
2206___
2207if($avx > 2 && $avx512) {
2208my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24));
2209my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29));
2210my $PADBIT="%zmm30";
2211
2212map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3));		# switch to %zmm domain
2213map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4));
2214map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4));
2215map(s/%y/%z/,($MASK));
2216
2217$code.=<<___;
2218.cfi_startproc
2219.Lblocks_avx512:
2220	mov		\$15,%eax
2221	kmovw		%eax,%k2
2222___
2223$code.=<<___	if (!$win64);
2224	lea		8(%rsp),%r10
2225.cfi_def_cfa_register	%r10
2226	sub		\$0x128,%rsp
2227___
2228$code.=<<___	if ($win64);
2229	lea		8(%rsp),%r10
2230	sub		\$0x1c8,%rsp
2231	vmovdqa		%xmm6,-0xb0(%r10)
2232	vmovdqa		%xmm7,-0xa0(%r10)
2233	vmovdqa		%xmm8,-0x90(%r10)
2234	vmovdqa		%xmm9,-0x80(%r10)
2235	vmovdqa		%xmm10,-0x70(%r10)
2236	vmovdqa		%xmm11,-0x60(%r10)
2237	vmovdqa		%xmm12,-0x50(%r10)
2238	vmovdqa		%xmm13,-0x40(%r10)
2239	vmovdqa		%xmm14,-0x30(%r10)
2240	vmovdqa		%xmm15,-0x20(%r10)
2241.Ldo_avx512_body:
2242___
2243$code.=<<___;
2244	lea		.Lconst(%rip),%rcx
2245	lea		48+64($ctx),$ctx	# size optimization
2246	vmovdqa		96(%rcx),%y#$T2		# .Lpermd_avx2
2247
2248	# expand pre-calculated table
2249	vmovdqu		`16*0-64`($ctx),%x#$D0	# will become expanded ${R0}
2250	and		\$-512,%rsp
2251	vmovdqu		`16*1-64`($ctx),%x#$D1	# will become ... ${R1}
2252	mov		\$0x20,%rax
2253	vmovdqu		`16*2-64`($ctx),%x#$T0	# ... ${S1}
2254	vmovdqu		`16*3-64`($ctx),%x#$D2	# ... ${R2}
2255	vmovdqu		`16*4-64`($ctx),%x#$T1	# ... ${S2}
2256	vmovdqu		`16*5-64`($ctx),%x#$D3	# ... ${R3}
2257	vmovdqu		`16*6-64`($ctx),%x#$T3	# ... ${S3}
2258	vmovdqu		`16*7-64`($ctx),%x#$D4	# ... ${R4}
2259	vmovdqu		`16*8-64`($ctx),%x#$T4	# ... ${S4}
2260	vpermd		$D0,$T2,$R0		# 00003412 -> 14243444
2261	vpbroadcastq	64(%rcx),$MASK		# .Lmask26
2262	vpermd		$D1,$T2,$R1
2263	vpermd		$T0,$T2,$S1
2264	vpermd		$D2,$T2,$R2
2265	vmovdqa64	$R0,0x00(%rsp){%k2}	# save in case $len%128 != 0
2266	 vpsrlq		\$32,$R0,$T0		# 14243444 -> 01020304
2267	vpermd		$T1,$T2,$S2
2268	vmovdqu64	$R1,0x00(%rsp,%rax){%k2}
2269	 vpsrlq		\$32,$R1,$T1
2270	vpermd		$D3,$T2,$R3
2271	vmovdqa64	$S1,0x40(%rsp){%k2}
2272	vpermd		$T3,$T2,$S3
2273	vpermd		$D4,$T2,$R4
2274	vmovdqu64	$R2,0x40(%rsp,%rax){%k2}
2275	vpermd		$T4,$T2,$S4
2276	vmovdqa64	$S2,0x80(%rsp){%k2}
2277	vmovdqu64	$R3,0x80(%rsp,%rax){%k2}
2278	vmovdqa64	$S3,0xc0(%rsp){%k2}
2279	vmovdqu64	$R4,0xc0(%rsp,%rax){%k2}
2280	vmovdqa64	$S4,0x100(%rsp){%k2}
2281
2282	################################################################
2283	# calculate 5th through 8th powers of the key
2284	#
2285	# d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1
2286	# d1 = r0'*r1 + r1'*r0   + r2'*5*r4 + r3'*5*r3 + r4'*5*r2
2287	# d2 = r0'*r2 + r1'*r1   + r2'*r0   + r3'*5*r4 + r4'*5*r3
2288	# d3 = r0'*r3 + r1'*r2   + r2'*r1   + r3'*r0   + r4'*5*r4
2289	# d4 = r0'*r4 + r1'*r3   + r2'*r2   + r3'*r1   + r4'*r0
2290
2291	vpmuludq	$T0,$R0,$D0		# d0 = r0'*r0
2292	vpmuludq	$T0,$R1,$D1		# d1 = r0'*r1
2293	vpmuludq	$T0,$R2,$D2		# d2 = r0'*r2
2294	vpmuludq	$T0,$R3,$D3		# d3 = r0'*r3
2295	vpmuludq	$T0,$R4,$D4		# d4 = r0'*r4
2296	 vpsrlq		\$32,$R2,$T2
2297
2298	vpmuludq	$T1,$S4,$M0
2299	vpmuludq	$T1,$R0,$M1
2300	vpmuludq	$T1,$R1,$M2
2301	vpmuludq	$T1,$R2,$M3
2302	vpmuludq	$T1,$R3,$M4
2303	 vpsrlq		\$32,$R3,$T3
2304	vpaddq		$M0,$D0,$D0		# d0 += r1'*5*r4
2305	vpaddq		$M1,$D1,$D1		# d1 += r1'*r0
2306	vpaddq		$M2,$D2,$D2		# d2 += r1'*r1
2307	vpaddq		$M3,$D3,$D3		# d3 += r1'*r2
2308	vpaddq		$M4,$D4,$D4		# d4 += r1'*r3
2309
2310	vpmuludq	$T2,$S3,$M0
2311	vpmuludq	$T2,$S4,$M1
2312	vpmuludq	$T2,$R1,$M3
2313	vpmuludq	$T2,$R2,$M4
2314	vpmuludq	$T2,$R0,$M2
2315	 vpsrlq		\$32,$R4,$T4
2316	vpaddq		$M0,$D0,$D0		# d0 += r2'*5*r3
2317	vpaddq		$M1,$D1,$D1		# d1 += r2'*5*r4
2318	vpaddq		$M3,$D3,$D3		# d3 += r2'*r1
2319	vpaddq		$M4,$D4,$D4		# d4 += r2'*r2
2320	vpaddq		$M2,$D2,$D2		# d2 += r2'*r0
2321
2322	vpmuludq	$T3,$S2,$M0
2323	vpmuludq	$T3,$R0,$M3
2324	vpmuludq	$T3,$R1,$M4
2325	vpmuludq	$T3,$S3,$M1
2326	vpmuludq	$T3,$S4,$M2
2327	vpaddq		$M0,$D0,$D0		# d0 += r3'*5*r2
2328	vpaddq		$M3,$D3,$D3		# d3 += r3'*r0
2329	vpaddq		$M4,$D4,$D4		# d4 += r3'*r1
2330	vpaddq		$M1,$D1,$D1		# d1 += r3'*5*r3
2331	vpaddq		$M2,$D2,$D2		# d2 += r3'*5*r4
2332
2333	vpmuludq	$T4,$S4,$M3
2334	vpmuludq	$T4,$R0,$M4
2335	vpmuludq	$T4,$S1,$M0
2336	vpmuludq	$T4,$S2,$M1
2337	vpmuludq	$T4,$S3,$M2
2338	vpaddq		$M3,$D3,$D3		# d3 += r2'*5*r4
2339	vpaddq		$M4,$D4,$D4		# d4 += r2'*r0
2340	vpaddq		$M0,$D0,$D0		# d0 += r2'*5*r1
2341	vpaddq		$M1,$D1,$D1		# d1 += r2'*5*r2
2342	vpaddq		$M2,$D2,$D2		# d2 += r2'*5*r3
2343
2344	################################################################
2345	# load input
2346	vmovdqu64	16*0($inp),%z#$T3
2347	vmovdqu64	16*4($inp),%z#$T4
2348	lea		16*8($inp),$inp
2349
2350	################################################################
2351	# lazy reduction
2352
2353	vpsrlq		\$26,$D3,$M3
2354	vpandq		$MASK,$D3,$D3
2355	vpaddq		$M3,$D4,$D4		# d3 -> d4
2356
2357	vpsrlq		\$26,$D0,$M0
2358	vpandq		$MASK,$D0,$D0
2359	vpaddq		$M0,$D1,$D1		# d0 -> d1
2360
2361	vpsrlq		\$26,$D4,$M4
2362	vpandq		$MASK,$D4,$D4
2363
2364	vpsrlq		\$26,$D1,$M1
2365	vpandq		$MASK,$D1,$D1
2366	vpaddq		$M1,$D2,$D2		# d1 -> d2
2367
2368	vpaddq		$M4,$D0,$D0
2369	vpsllq		\$2,$M4,$M4
2370	vpaddq		$M4,$D0,$D0		# d4 -> d0
2371
2372	vpsrlq		\$26,$D2,$M2
2373	vpandq		$MASK,$D2,$D2
2374	vpaddq		$M2,$D3,$D3		# d2 -> d3
2375
2376	vpsrlq		\$26,$D0,$M0
2377	vpandq		$MASK,$D0,$D0
2378	vpaddq		$M0,$D1,$D1		# d0 -> d1
2379
2380	vpsrlq		\$26,$D3,$M3
2381	vpandq		$MASK,$D3,$D3
2382	vpaddq		$M3,$D4,$D4		# d3 -> d4
2383
2384	################################################################
2385	# at this point we have 14243444 in $R0-$S4 and 05060708 in
2386	# $D0-$D4, ...
2387
2388	vpunpcklqdq	$T4,$T3,$T0	# transpose input
2389	vpunpckhqdq	$T4,$T3,$T4
2390
2391	# ... since input 64-bit lanes are ordered as 73625140, we could
2392	# "vperm" it to 76543210 (here and in each loop iteration), *or*
2393	# we could just flow along, hence the goal for $R0-$S4 is
2394	# 1858286838784888 ...
2395
2396	vmovdqa32	128(%rcx),$M0		# .Lpermd_avx512:
2397	mov		\$0x7777,%eax
2398	kmovw		%eax,%k1
2399
2400	vpermd		$R0,$M0,$R0		# 14243444 -> 1---2---3---4---
2401	vpermd		$R1,$M0,$R1
2402	vpermd		$R2,$M0,$R2
2403	vpermd		$R3,$M0,$R3
2404	vpermd		$R4,$M0,$R4
2405
2406	vpermd		$D0,$M0,${R0}{%k1}	# 05060708 -> 1858286838784888
2407	vpermd		$D1,$M0,${R1}{%k1}
2408	vpermd		$D2,$M0,${R2}{%k1}
2409	vpermd		$D3,$M0,${R3}{%k1}
2410	vpermd		$D4,$M0,${R4}{%k1}
2411
2412	vpslld		\$2,$R1,$S1		# *5
2413	vpslld		\$2,$R2,$S2
2414	vpslld		\$2,$R3,$S3
2415	vpslld		\$2,$R4,$S4
2416	vpaddd		$R1,$S1,$S1
2417	vpaddd		$R2,$S2,$S2
2418	vpaddd		$R3,$S3,$S3
2419	vpaddd		$R4,$S4,$S4
2420
2421	vpbroadcastq	32(%rcx),$PADBIT	# .L129
2422
2423	vpsrlq		\$52,$T0,$T2		# splat input
2424	vpsllq		\$12,$T4,$T3
2425	vporq		$T3,$T2,$T2
2426	vpsrlq		\$26,$T0,$T1
2427	vpsrlq		\$14,$T4,$T3
2428	vpsrlq		\$40,$T4,$T4		# 4
2429	vpandq		$MASK,$T2,$T2		# 2
2430	vpandq		$MASK,$T0,$T0		# 0
2431	#vpandq		$MASK,$T1,$T1		# 1
2432	#vpandq		$MASK,$T3,$T3		# 3
2433	#vporq		$PADBIT,$T4,$T4		# padbit, yes, always
2434
2435	vpaddq		$H2,$T2,$H2		# accumulate input
2436	sub		\$192,$len
2437	jbe		.Ltail_avx512
2438	jmp		.Loop_avx512
2439
2440.align	32
2441.Loop_avx512:
2442	################################################################
2443	# ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8
2444	# ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7
2445	# ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6
2446	# ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5
2447	# ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4
2448	# ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3
2449	# ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2
2450	# ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1
2451	#   \________/\___________/
2452	################################################################
2453	#vpaddq		$H2,$T2,$H2		# accumulate input
2454
2455	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
2456	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
2457	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
2458	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
2459	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
2460	#
2461	# however, as h2 is "chronologically" first one available pull
2462	# corresponding operations up, so it's
2463	#
2464	# d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0 + h4*5*r4
2465	# d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1 + h4*r0
2466	# d0 = h2*5*r3 + h0*r0 + h1*5*r4         + h3*5*r2 + h4*5*r1
2467	# d1 = h2*5*r4 + h0*r1           + h1*r0 + h3*5*r3 + h4*5*r2
2468	# d2 = h2*r0           + h0*r2   + h1*r1 + h3*5*r4 + h4*5*r3
2469
2470	vpmuludq	$H2,$R1,$D3		# d3 = h2*r1
2471	 vpaddq		$H0,$T0,$H0
2472	vpmuludq	$H2,$R2,$D4		# d4 = h2*r2
2473	 vpandq		$MASK,$T1,$T1		# 1
2474	vpmuludq	$H2,$S3,$D0		# d0 = h2*s3
2475	 vpandq		$MASK,$T3,$T3		# 3
2476	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
2477	 vporq		$PADBIT,$T4,$T4		# padbit, yes, always
2478	vpmuludq	$H2,$R0,$D2		# d2 = h2*r0
2479	 vpaddq		$H1,$T1,$H1		# accumulate input
2480	 vpaddq		$H3,$T3,$H3
2481	 vpaddq		$H4,$T4,$H4
2482
2483	  vmovdqu64	16*0($inp),$T3		# load input
2484	  vmovdqu64	16*4($inp),$T4
2485	  lea		16*8($inp),$inp
2486	vpmuludq	$H0,$R3,$M3
2487	vpmuludq	$H0,$R4,$M4
2488	vpmuludq	$H0,$R0,$M0
2489	vpmuludq	$H0,$R1,$M1
2490	vpaddq		$M3,$D3,$D3		# d3 += h0*r3
2491	vpaddq		$M4,$D4,$D4		# d4 += h0*r4
2492	vpaddq		$M0,$D0,$D0		# d0 += h0*r0
2493	vpaddq		$M1,$D1,$D1		# d1 += h0*r1
2494
2495	vpmuludq	$H1,$R2,$M3
2496	vpmuludq	$H1,$R3,$M4
2497	vpmuludq	$H1,$S4,$M0
2498	vpmuludq	$H0,$R2,$M2
2499	vpaddq		$M3,$D3,$D3		# d3 += h1*r2
2500	vpaddq		$M4,$D4,$D4		# d4 += h1*r3
2501	vpaddq		$M0,$D0,$D0		# d0 += h1*s4
2502	vpaddq		$M2,$D2,$D2		# d2 += h0*r2
2503
2504	  vpunpcklqdq	$T4,$T3,$T0		# transpose input
2505	  vpunpckhqdq	$T4,$T3,$T4
2506
2507	vpmuludq	$H3,$R0,$M3
2508	vpmuludq	$H3,$R1,$M4
2509	vpmuludq	$H1,$R0,$M1
2510	vpmuludq	$H1,$R1,$M2
2511	vpaddq		$M3,$D3,$D3		# d3 += h3*r0
2512	vpaddq		$M4,$D4,$D4		# d4 += h3*r1
2513	vpaddq		$M1,$D1,$D1		# d1 += h1*r0
2514	vpaddq		$M2,$D2,$D2		# d2 += h1*r1
2515
2516	vpmuludq	$H4,$S4,$M3
2517	vpmuludq	$H4,$R0,$M4
2518	vpmuludq	$H3,$S2,$M0
2519	vpmuludq	$H3,$S3,$M1
2520	vpaddq		$M3,$D3,$D3		# d3 += h4*s4
2521	vpmuludq	$H3,$S4,$M2
2522	vpaddq		$M4,$D4,$D4		# d4 += h4*r0
2523	vpaddq		$M0,$D0,$D0		# d0 += h3*s2
2524	vpaddq		$M1,$D1,$D1		# d1 += h3*s3
2525	vpaddq		$M2,$D2,$D2		# d2 += h3*s4
2526
2527	vpmuludq	$H4,$S1,$M0
2528	vpmuludq	$H4,$S2,$M1
2529	vpmuludq	$H4,$S3,$M2
2530	vpaddq		$M0,$D0,$H0		# h0 = d0 + h4*s1
2531	vpaddq		$M1,$D1,$H1		# h1 = d2 + h4*s2
2532	vpaddq		$M2,$D2,$H2		# h2 = d3 + h4*s3
2533
2534	################################################################
2535	# lazy reduction (interleaved with input splat)
2536
2537	 vpsrlq		\$52,$T0,$T2		# splat input
2538	 vpsllq		\$12,$T4,$T3
2539
2540	vpsrlq		\$26,$D3,$H3
2541	vpandq		$MASK,$D3,$D3
2542	vpaddq		$H3,$D4,$H4		# h3 -> h4
2543
2544	 vporq		$T3,$T2,$T2
2545
2546	vpsrlq		\$26,$H0,$D0
2547	vpandq		$MASK,$H0,$H0
2548	vpaddq		$D0,$H1,$H1		# h0 -> h1
2549
2550	 vpandq		$MASK,$T2,$T2		# 2
2551
2552	vpsrlq		\$26,$H4,$D4
2553	vpandq		$MASK,$H4,$H4
2554
2555	vpsrlq		\$26,$H1,$D1
2556	vpandq		$MASK,$H1,$H1
2557	vpaddq		$D1,$H2,$H2		# h1 -> h2
2558
2559	vpaddq		$D4,$H0,$H0
2560	vpsllq		\$2,$D4,$D4
2561	vpaddq		$D4,$H0,$H0		# h4 -> h0
2562
2563	 vpaddq		$T2,$H2,$H2		# modulo-scheduled
2564	 vpsrlq		\$26,$T0,$T1
2565
2566	vpsrlq		\$26,$H2,$D2
2567	vpandq		$MASK,$H2,$H2
2568	vpaddq		$D2,$D3,$H3		# h2 -> h3
2569
2570	 vpsrlq		\$14,$T4,$T3
2571
2572	vpsrlq		\$26,$H0,$D0
2573	vpandq		$MASK,$H0,$H0
2574	vpaddq		$D0,$H1,$H1		# h0 -> h1
2575
2576	 vpsrlq		\$40,$T4,$T4		# 4
2577
2578	vpsrlq		\$26,$H3,$D3
2579	vpandq		$MASK,$H3,$H3
2580	vpaddq		$D3,$H4,$H4		# h3 -> h4
2581
2582	 vpandq		$MASK,$T0,$T0		# 0
2583	 #vpandq	$MASK,$T1,$T1		# 1
2584	 #vpandq	$MASK,$T3,$T3		# 3
2585	 #vporq		$PADBIT,$T4,$T4		# padbit, yes, always
2586
2587	sub		\$128,$len
2588	ja		.Loop_avx512
2589
2590.Ltail_avx512:
2591	################################################################
2592	# while above multiplications were by r^8 in all lanes, in last
2593	# iteration we multiply least significant lane by r^8 and most
2594	# significant one by r, that's why table gets shifted...
2595
2596	vpsrlq		\$32,$R0,$R0		# 0105020603070408
2597	vpsrlq		\$32,$R1,$R1
2598	vpsrlq		\$32,$R2,$R2
2599	vpsrlq		\$32,$S3,$S3
2600	vpsrlq		\$32,$S4,$S4
2601	vpsrlq		\$32,$R3,$R3
2602	vpsrlq		\$32,$R4,$R4
2603	vpsrlq		\$32,$S1,$S1
2604	vpsrlq		\$32,$S2,$S2
2605
2606	################################################################
2607	# load either next or last 64 byte of input
2608	lea		($inp,$len),$inp
2609
2610	#vpaddq		$H2,$T2,$H2		# accumulate input
2611	vpaddq		$H0,$T0,$H0
2612
2613	vpmuludq	$H2,$R1,$D3		# d3 = h2*r1
2614	vpmuludq	$H2,$R2,$D4		# d4 = h2*r2
2615	vpmuludq	$H2,$S3,$D0		# d0 = h2*s3
2616	 vpandq		$MASK,$T1,$T1		# 1
2617	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
2618	 vpandq		$MASK,$T3,$T3		# 3
2619	vpmuludq	$H2,$R0,$D2		# d2 = h2*r0
2620	 vporq		$PADBIT,$T4,$T4		# padbit, yes, always
2621	 vpaddq		$H1,$T1,$H1		# accumulate input
2622	 vpaddq		$H3,$T3,$H3
2623	 vpaddq		$H4,$T4,$H4
2624
2625	  vmovdqu	16*0($inp),%x#$T0
2626	vpmuludq	$H0,$R3,$M3
2627	vpmuludq	$H0,$R4,$M4
2628	vpmuludq	$H0,$R0,$M0
2629	vpmuludq	$H0,$R1,$M1
2630	vpaddq		$M3,$D3,$D3		# d3 += h0*r3
2631	vpaddq		$M4,$D4,$D4		# d4 += h0*r4
2632	vpaddq		$M0,$D0,$D0		# d0 += h0*r0
2633	vpaddq		$M1,$D1,$D1		# d1 += h0*r1
2634
2635	  vmovdqu	16*1($inp),%x#$T1
2636	vpmuludq	$H1,$R2,$M3
2637	vpmuludq	$H1,$R3,$M4
2638	vpmuludq	$H1,$S4,$M0
2639	vpmuludq	$H0,$R2,$M2
2640	vpaddq		$M3,$D3,$D3		# d3 += h1*r2
2641	vpaddq		$M4,$D4,$D4		# d4 += h1*r3
2642	vpaddq		$M0,$D0,$D0		# d0 += h1*s4
2643	vpaddq		$M2,$D2,$D2		# d2 += h0*r2
2644
2645	  vinserti128	\$1,16*2($inp),%y#$T0,%y#$T0
2646	vpmuludq	$H3,$R0,$M3
2647	vpmuludq	$H3,$R1,$M4
2648	vpmuludq	$H1,$R0,$M1
2649	vpmuludq	$H1,$R1,$M2
2650	vpaddq		$M3,$D3,$D3		# d3 += h3*r0
2651	vpaddq		$M4,$D4,$D4		# d4 += h3*r1
2652	vpaddq		$M1,$D1,$D1		# d1 += h1*r0
2653	vpaddq		$M2,$D2,$D2		# d2 += h1*r1
2654
2655	  vinserti128	\$1,16*3($inp),%y#$T1,%y#$T1
2656	vpmuludq	$H4,$S4,$M3
2657	vpmuludq	$H4,$R0,$M4
2658	vpmuludq	$H3,$S2,$M0
2659	vpmuludq	$H3,$S3,$M1
2660	vpmuludq	$H3,$S4,$M2
2661	vpaddq		$M3,$D3,$H3		# h3 = d3 + h4*s4
2662	vpaddq		$M4,$D4,$D4		# d4 += h4*r0
2663	vpaddq		$M0,$D0,$D0		# d0 += h3*s2
2664	vpaddq		$M1,$D1,$D1		# d1 += h3*s3
2665	vpaddq		$M2,$D2,$D2		# d2 += h3*s4
2666
2667	vpmuludq	$H4,$S1,$M0
2668	vpmuludq	$H4,$S2,$M1
2669	vpmuludq	$H4,$S3,$M2
2670	vpaddq		$M0,$D0,$H0		# h0 = d0 + h4*s1
2671	vpaddq		$M1,$D1,$H1		# h1 = d2 + h4*s2
2672	vpaddq		$M2,$D2,$H2		# h2 = d3 + h4*s3
2673
2674	################################################################
2675	# horizontal addition
2676
2677	mov		\$1,%eax
2678	vpermq		\$0xb1,$H3,$D3
2679	vpermq		\$0xb1,$D4,$H4
2680	vpermq		\$0xb1,$H0,$D0
2681	vpermq		\$0xb1,$H1,$D1
2682	vpermq		\$0xb1,$H2,$D2
2683	vpaddq		$D3,$H3,$H3
2684	vpaddq		$D4,$H4,$H4
2685	vpaddq		$D0,$H0,$H0
2686	vpaddq		$D1,$H1,$H1
2687	vpaddq		$D2,$H2,$H2
2688
2689	kmovw		%eax,%k3
2690	vpermq		\$0x2,$H3,$D3
2691	vpermq		\$0x2,$H4,$D4
2692	vpermq		\$0x2,$H0,$D0
2693	vpermq		\$0x2,$H1,$D1
2694	vpermq		\$0x2,$H2,$D2
2695	vpaddq		$D3,$H3,$H3
2696	vpaddq		$D4,$H4,$H4
2697	vpaddq		$D0,$H0,$H0
2698	vpaddq		$D1,$H1,$H1
2699	vpaddq		$D2,$H2,$H2
2700
2701	vextracti64x4	\$0x1,$H3,%y#$D3
2702	vextracti64x4	\$0x1,$H4,%y#$D4
2703	vextracti64x4	\$0x1,$H0,%y#$D0
2704	vextracti64x4	\$0x1,$H1,%y#$D1
2705	vextracti64x4	\$0x1,$H2,%y#$D2
2706	vpaddq		$D3,$H3,${H3}{%k3}{z}	# keep single qword in case
2707	vpaddq		$D4,$H4,${H4}{%k3}{z}	# it's passed to .Ltail_avx2
2708	vpaddq		$D0,$H0,${H0}{%k3}{z}
2709	vpaddq		$D1,$H1,${H1}{%k3}{z}
2710	vpaddq		$D2,$H2,${H2}{%k3}{z}
2711___
2712map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT));
2713map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK));
2714$code.=<<___;
2715	################################################################
2716	# lazy reduction (interleaved with input splat)
2717
2718	vpsrlq		\$26,$H3,$D3
2719	vpand		$MASK,$H3,$H3
2720	 vpsrldq	\$6,$T0,$T2		# splat input
2721	 vpsrldq	\$6,$T1,$T3
2722	 vpunpckhqdq	$T1,$T0,$T4		# 4
2723	vpaddq		$D3,$H4,$H4		# h3 -> h4
2724
2725	vpsrlq		\$26,$H0,$D0
2726	vpand		$MASK,$H0,$H0
2727	 vpunpcklqdq	$T3,$T2,$T2		# 2:3
2728	 vpunpcklqdq	$T1,$T0,$T0		# 0:1
2729	vpaddq		$D0,$H1,$H1		# h0 -> h1
2730
2731	vpsrlq		\$26,$H4,$D4
2732	vpand		$MASK,$H4,$H4
2733
2734	vpsrlq		\$26,$H1,$D1
2735	vpand		$MASK,$H1,$H1
2736	 vpsrlq		\$30,$T2,$T3
2737	 vpsrlq		\$4,$T2,$T2
2738	vpaddq		$D1,$H2,$H2		# h1 -> h2
2739
2740	vpaddq		$D4,$H0,$H0
2741	vpsllq		\$2,$D4,$D4
2742	 vpsrlq		\$26,$T0,$T1
2743	 vpsrlq		\$40,$T4,$T4		# 4
2744	vpaddq		$D4,$H0,$H0		# h4 -> h0
2745
2746	vpsrlq		\$26,$H2,$D2
2747	vpand		$MASK,$H2,$H2
2748	 vpand		$MASK,$T2,$T2		# 2
2749	 vpand		$MASK,$T0,$T0		# 0
2750	vpaddq		$D2,$H3,$H3		# h2 -> h3
2751
2752	vpsrlq		\$26,$H0,$D0
2753	vpand		$MASK,$H0,$H0
2754	 vpaddq		$H2,$T2,$H2		# accumulate input for .Ltail_avx2
2755	 vpand		$MASK,$T1,$T1		# 1
2756	vpaddq		$D0,$H1,$H1		# h0 -> h1
2757
2758	vpsrlq		\$26,$H3,$D3
2759	vpand		$MASK,$H3,$H3
2760	 vpand		$MASK,$T3,$T3		# 3
2761	 vpor		32(%rcx),$T4,$T4	# padbit, yes, always
2762	vpaddq		$D3,$H4,$H4		# h3 -> h4
2763
2764	lea		0x90(%rsp),%rax		# size optimization for .Ltail_avx2
2765	add		\$64,$len
2766	jnz		.Ltail_avx2$suffix
2767
2768	vpsubq		$T2,$H2,$H2		# undo input accumulation
2769	vmovd		%x#$H0,`4*0-48-64`($ctx)# save partially reduced
2770	vmovd		%x#$H1,`4*1-48-64`($ctx)
2771	vmovd		%x#$H2,`4*2-48-64`($ctx)
2772	vmovd		%x#$H3,`4*3-48-64`($ctx)
2773	vmovd		%x#$H4,`4*4-48-64`($ctx)
2774	vzeroall
2775___
2776$code.=<<___	if ($win64);
2777	movdqa		-0xb0(%r10),%xmm6
2778	movdqa		-0xa0(%r10),%xmm7
2779	movdqa		-0x90(%r10),%xmm8
2780	movdqa		-0x80(%r10),%xmm9
2781	movdqa		-0x70(%r10),%xmm10
2782	movdqa		-0x60(%r10),%xmm11
2783	movdqa		-0x50(%r10),%xmm12
2784	movdqa		-0x40(%r10),%xmm13
2785	movdqa		-0x30(%r10),%xmm14
2786	movdqa		-0x20(%r10),%xmm15
2787	lea		-8(%r10),%rsp
2788.Ldo_avx512_epilogue:
2789___
2790$code.=<<___	if (!$win64);
2791	lea		-8(%r10),%rsp
2792.cfi_def_cfa_register	%rsp
2793___
2794$code.=<<___;
2795	RET
2796.cfi_endproc
2797___
2798
2799}
2800
2801}
2802
2803&declare_function("poly1305_blocks_avx2", 32, 4);
2804poly1305_blocks_avxN(0);
2805&end_function("poly1305_blocks_avx2");
2806
2807#######################################################################
2808if ($avx>2) {
2809# On entry we have input length divisible by 64. But since inner loop
2810# processes 128 bytes per iteration, cases when length is not divisible
2811# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this
2812# reason stack layout is kept identical to poly1305_blocks_avx2. If not
2813# for this tail, we wouldn't have to even allocate stack frame...
2814
2815if($kernel) {
2816	$code .= "#ifdef CONFIG_AS_AVX512\n";
2817}
2818
2819&declare_function("poly1305_blocks_avx512", 32, 4);
2820poly1305_blocks_avxN(1);
2821&end_function("poly1305_blocks_avx512");
2822
2823if ($kernel) {
2824	$code .= "#endif\n";
2825}
2826
2827if (!$kernel && $avx>3) {
2828########################################################################
2829# VPMADD52 version using 2^44 radix.
2830#
2831# One can argue that base 2^52 would be more natural. Well, even though
2832# some operations would be more natural, one has to recognize couple of
2833# things. Base 2^52 doesn't provide advantage over base 2^44 if you look
2834# at amount of multiply-n-accumulate operations. Secondly, it makes it
2835# impossible to pre-compute multiples of 5 [referred to as s[]/sN in
2836# reference implementations], which means that more such operations
2837# would have to be performed in inner loop, which in turn makes critical
2838# path longer. In other words, even though base 2^44 reduction might
2839# look less elegant, overall critical path is actually shorter...
2840
2841########################################################################
2842# Layout of opaque area is following.
2843#
2844#	unsigned __int64 h[3];		# current hash value base 2^44
2845#	unsigned __int64 s[2];		# key value*20 base 2^44
2846#	unsigned __int64 r[3];		# key value base 2^44
2847#	struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4];
2848#					# r^n positions reflect
2849#					# placement in register, not
2850#					# memory, R[3] is R[1]*20
2851
2852$code.=<<___;
2853.type	poly1305_init_base2_44,\@function,3
2854.align	32
2855poly1305_init_base2_44:
2856	xor	%eax,%eax
2857	mov	%rax,0($ctx)		# initialize hash value
2858	mov	%rax,8($ctx)
2859	mov	%rax,16($ctx)
2860
2861.Linit_base2_44:
2862	lea	poly1305_blocks_vpmadd52(%rip),%r10
2863	lea	poly1305_emit_base2_44(%rip),%r11
2864
2865	mov	\$0x0ffffffc0fffffff,%rax
2866	mov	\$0x0ffffffc0ffffffc,%rcx
2867	and	0($inp),%rax
2868	mov	\$0x00000fffffffffff,%r8
2869	and	8($inp),%rcx
2870	mov	\$0x00000fffffffffff,%r9
2871	and	%rax,%r8
2872	shrd	\$44,%rcx,%rax
2873	mov	%r8,40($ctx)		# r0
2874	and	%r9,%rax
2875	shr	\$24,%rcx
2876	mov	%rax,48($ctx)		# r1
2877	lea	(%rax,%rax,4),%rax	# *5
2878	mov	%rcx,56($ctx)		# r2
2879	shl	\$2,%rax		# magic <<2
2880	lea	(%rcx,%rcx,4),%rcx	# *5
2881	shl	\$2,%rcx		# magic <<2
2882	mov	%rax,24($ctx)		# s1
2883	mov	%rcx,32($ctx)		# s2
2884	movq	\$-1,64($ctx)		# write impossible value
2885___
2886$code.=<<___	if ($flavour !~ /elf32/);
2887	mov	%r10,0(%rdx)
2888	mov	%r11,8(%rdx)
2889___
2890$code.=<<___	if ($flavour =~ /elf32/);
2891	mov	%r10d,0(%rdx)
2892	mov	%r11d,4(%rdx)
2893___
2894$code.=<<___;
2895	mov	\$1,%eax
2896	RET
2897.size	poly1305_init_base2_44,.-poly1305_init_base2_44
2898___
2899{
2900my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17));
2901my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21));
2902my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25));
2903
2904$code.=<<___;
2905.type	poly1305_blocks_vpmadd52,\@function,4
2906.align	32
2907poly1305_blocks_vpmadd52:
2908	shr	\$4,$len
2909	jz	.Lno_data_vpmadd52		# too short
2910
2911	shl	\$40,$padbit
2912	mov	64($ctx),%r8			# peek on power of the key
2913
2914	# if powers of the key are not calculated yet, process up to 3
2915	# blocks with this single-block subroutine, otherwise ensure that
2916	# length is divisible by 2 blocks and pass the rest down to next
2917	# subroutine...
2918
2919	mov	\$3,%rax
2920	mov	\$1,%r10
2921	cmp	\$4,$len			# is input long
2922	cmovae	%r10,%rax
2923	test	%r8,%r8				# is power value impossible?
2924	cmovns	%r10,%rax
2925
2926	and	$len,%rax			# is input of favourable length?
2927	jz	.Lblocks_vpmadd52_4x
2928
2929	sub		%rax,$len
2930	mov		\$7,%r10d
2931	mov		\$1,%r11d
2932	kmovw		%r10d,%k7
2933	lea		.L2_44_inp_permd(%rip),%r10
2934	kmovw		%r11d,%k1
2935
2936	vmovq		$padbit,%x#$PAD
2937	vmovdqa64	0(%r10),$inp_permd	# .L2_44_inp_permd
2938	vmovdqa64	32(%r10),$inp_shift	# .L2_44_inp_shift
2939	vpermq		\$0xcf,$PAD,$PAD
2940	vmovdqa64	64(%r10),$reduc_mask	# .L2_44_mask
2941
2942	vmovdqu64	0($ctx),${Dlo}{%k7}{z}		# load hash value
2943	vmovdqu64	40($ctx),${r2r1r0}{%k7}{z}	# load keys
2944	vmovdqu64	32($ctx),${r1r0s2}{%k7}{z}
2945	vmovdqu64	24($ctx),${r0s2s1}{%k7}{z}
2946
2947	vmovdqa64	96(%r10),$reduc_rght	# .L2_44_shift_rgt
2948	vmovdqa64	128(%r10),$reduc_left	# .L2_44_shift_lft
2949
2950	jmp		.Loop_vpmadd52
2951
2952.align	32
2953.Loop_vpmadd52:
2954	vmovdqu32	0($inp),%x#$T0		# load input as ----3210
2955	lea		16($inp),$inp
2956
2957	vpermd		$T0,$inp_permd,$T0	# ----3210 -> --322110
2958	vpsrlvq		$inp_shift,$T0,$T0
2959	vpandq		$reduc_mask,$T0,$T0
2960	vporq		$PAD,$T0,$T0
2961
2962	vpaddq		$T0,$Dlo,$Dlo		# accumulate input
2963
2964	vpermq		\$0,$Dlo,${H0}{%k7}{z}	# smash hash value
2965	vpermq		\$0b01010101,$Dlo,${H1}{%k7}{z}
2966	vpermq		\$0b10101010,$Dlo,${H2}{%k7}{z}
2967
2968	vpxord		$Dlo,$Dlo,$Dlo
2969	vpxord		$Dhi,$Dhi,$Dhi
2970
2971	vpmadd52luq	$r2r1r0,$H0,$Dlo
2972	vpmadd52huq	$r2r1r0,$H0,$Dhi
2973
2974	vpmadd52luq	$r1r0s2,$H1,$Dlo
2975	vpmadd52huq	$r1r0s2,$H1,$Dhi
2976
2977	vpmadd52luq	$r0s2s1,$H2,$Dlo
2978	vpmadd52huq	$r0s2s1,$H2,$Dhi
2979
2980	vpsrlvq		$reduc_rght,$Dlo,$T0	# 0 in topmost qword
2981	vpsllvq		$reduc_left,$Dhi,$Dhi	# 0 in topmost qword
2982	vpandq		$reduc_mask,$Dlo,$Dlo
2983
2984	vpaddq		$T0,$Dhi,$Dhi
2985
2986	vpermq		\$0b10010011,$Dhi,$Dhi	# 0 in lowest qword
2987
2988	vpaddq		$Dhi,$Dlo,$Dlo		# note topmost qword :-)
2989
2990	vpsrlvq		$reduc_rght,$Dlo,$T0	# 0 in topmost word
2991	vpandq		$reduc_mask,$Dlo,$Dlo
2992
2993	vpermq		\$0b10010011,$T0,$T0
2994
2995	vpaddq		$T0,$Dlo,$Dlo
2996
2997	vpermq		\$0b10010011,$Dlo,${T0}{%k1}{z}
2998
2999	vpaddq		$T0,$Dlo,$Dlo
3000	vpsllq		\$2,$T0,$T0
3001
3002	vpaddq		$T0,$Dlo,$Dlo
3003
3004	dec		%rax			# len-=16
3005	jnz		.Loop_vpmadd52
3006
3007	vmovdqu64	$Dlo,0($ctx){%k7}	# store hash value
3008
3009	test		$len,$len
3010	jnz		.Lblocks_vpmadd52_4x
3011
3012.Lno_data_vpmadd52:
3013	RET
3014.size	poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52
3015___
3016}
3017{
3018########################################################################
3019# As implied by its name 4x subroutine processes 4 blocks in parallel
3020# (but handles even 4*n+2 blocks lengths). It takes up to 4th key power
3021# and is handled in 256-bit %ymm registers.
3022
3023my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17));
3024my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23));
3025my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31));
3026
3027$code.=<<___;
3028.type	poly1305_blocks_vpmadd52_4x,\@function,4
3029.align	32
3030poly1305_blocks_vpmadd52_4x:
3031	shr	\$4,$len
3032	jz	.Lno_data_vpmadd52_4x		# too short
3033
3034	shl	\$40,$padbit
3035	mov	64($ctx),%r8			# peek on power of the key
3036
3037.Lblocks_vpmadd52_4x:
3038	vpbroadcastq	$padbit,$PAD
3039
3040	vmovdqa64	.Lx_mask44(%rip),$mask44
3041	mov		\$5,%eax
3042	vmovdqa64	.Lx_mask42(%rip),$mask42
3043	kmovw		%eax,%k1		# used in 2x path
3044
3045	test		%r8,%r8			# is power value impossible?
3046	js		.Linit_vpmadd52		# if it is, then init R[4]
3047
3048	vmovq		0($ctx),%x#$H0		# load current hash value
3049	vmovq		8($ctx),%x#$H1
3050	vmovq		16($ctx),%x#$H2
3051
3052	test		\$3,$len		# is length 4*n+2?
3053	jnz		.Lblocks_vpmadd52_2x_do
3054
3055.Lblocks_vpmadd52_4x_do:
3056	vpbroadcastq	64($ctx),$R0		# load 4th power of the key
3057	vpbroadcastq	96($ctx),$R1
3058	vpbroadcastq	128($ctx),$R2
3059	vpbroadcastq	160($ctx),$S1
3060
3061.Lblocks_vpmadd52_4x_key_loaded:
3062	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
3063	vpaddq		$R2,$S2,$S2
3064	vpsllq		\$2,$S2,$S2
3065
3066	test		\$7,$len		# is len 8*n?
3067	jz		.Lblocks_vpmadd52_8x
3068
3069	vmovdqu64	16*0($inp),$T2		# load data
3070	vmovdqu64	16*2($inp),$T3
3071	lea		16*4($inp),$inp
3072
3073	vpunpcklqdq	$T3,$T2,$T1		# transpose data
3074	vpunpckhqdq	$T3,$T2,$T3
3075
3076	# at this point 64-bit lanes are ordered as 3-1-2-0
3077
3078	vpsrlq		\$24,$T3,$T2		# splat the data
3079	vporq		$PAD,$T2,$T2
3080	 vpaddq		$T2,$H2,$H2		# accumulate input
3081	vpandq		$mask44,$T1,$T0
3082	vpsrlq		\$44,$T1,$T1
3083	vpsllq		\$20,$T3,$T3
3084	vporq		$T3,$T1,$T1
3085	vpandq		$mask44,$T1,$T1
3086
3087	sub		\$4,$len
3088	jz		.Ltail_vpmadd52_4x
3089	jmp		.Loop_vpmadd52_4x
3090	ud2
3091
3092.align	32
3093.Linit_vpmadd52:
3094	vmovq		24($ctx),%x#$S1		# load key
3095	vmovq		56($ctx),%x#$H2
3096	vmovq		32($ctx),%x#$S2
3097	vmovq		40($ctx),%x#$R0
3098	vmovq		48($ctx),%x#$R1
3099
3100	vmovdqa		$R0,$H0
3101	vmovdqa		$R1,$H1
3102	vmovdqa		$H2,$R2
3103
3104	mov		\$2,%eax
3105
3106.Lmul_init_vpmadd52:
3107	vpxorq		$D0lo,$D0lo,$D0lo
3108	vpmadd52luq	$H2,$S1,$D0lo
3109	vpxorq		$D0hi,$D0hi,$D0hi
3110	vpmadd52huq	$H2,$S1,$D0hi
3111	vpxorq		$D1lo,$D1lo,$D1lo
3112	vpmadd52luq	$H2,$S2,$D1lo
3113	vpxorq		$D1hi,$D1hi,$D1hi
3114	vpmadd52huq	$H2,$S2,$D1hi
3115	vpxorq		$D2lo,$D2lo,$D2lo
3116	vpmadd52luq	$H2,$R0,$D2lo
3117	vpxorq		$D2hi,$D2hi,$D2hi
3118	vpmadd52huq	$H2,$R0,$D2hi
3119
3120	vpmadd52luq	$H0,$R0,$D0lo
3121	vpmadd52huq	$H0,$R0,$D0hi
3122	vpmadd52luq	$H0,$R1,$D1lo
3123	vpmadd52huq	$H0,$R1,$D1hi
3124	vpmadd52luq	$H0,$R2,$D2lo
3125	vpmadd52huq	$H0,$R2,$D2hi
3126
3127	vpmadd52luq	$H1,$S2,$D0lo
3128	vpmadd52huq	$H1,$S2,$D0hi
3129	vpmadd52luq	$H1,$R0,$D1lo
3130	vpmadd52huq	$H1,$R0,$D1hi
3131	vpmadd52luq	$H1,$R1,$D2lo
3132	vpmadd52huq	$H1,$R1,$D2hi
3133
3134	################################################################
3135	# partial reduction
3136	vpsrlq		\$44,$D0lo,$tmp
3137	vpsllq		\$8,$D0hi,$D0hi
3138	vpandq		$mask44,$D0lo,$H0
3139	vpaddq		$tmp,$D0hi,$D0hi
3140
3141	vpaddq		$D0hi,$D1lo,$D1lo
3142
3143	vpsrlq		\$44,$D1lo,$tmp
3144	vpsllq		\$8,$D1hi,$D1hi
3145	vpandq		$mask44,$D1lo,$H1
3146	vpaddq		$tmp,$D1hi,$D1hi
3147
3148	vpaddq		$D1hi,$D2lo,$D2lo
3149
3150	vpsrlq		\$42,$D2lo,$tmp
3151	vpsllq		\$10,$D2hi,$D2hi
3152	vpandq		$mask42,$D2lo,$H2
3153	vpaddq		$tmp,$D2hi,$D2hi
3154
3155	vpaddq		$D2hi,$H0,$H0
3156	vpsllq		\$2,$D2hi,$D2hi
3157
3158	vpaddq		$D2hi,$H0,$H0
3159
3160	vpsrlq		\$44,$H0,$tmp		# additional step
3161	vpandq		$mask44,$H0,$H0
3162
3163	vpaddq		$tmp,$H1,$H1
3164
3165	dec		%eax
3166	jz		.Ldone_init_vpmadd52
3167
3168	vpunpcklqdq	$R1,$H1,$R1		# 1,2
3169	vpbroadcastq	%x#$H1,%x#$H1		# 2,2
3170	vpunpcklqdq	$R2,$H2,$R2
3171	vpbroadcastq	%x#$H2,%x#$H2
3172	vpunpcklqdq	$R0,$H0,$R0
3173	vpbroadcastq	%x#$H0,%x#$H0
3174
3175	vpsllq		\$2,$R1,$S1		# S1 = R1*5*4
3176	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
3177	vpaddq		$R1,$S1,$S1
3178	vpaddq		$R2,$S2,$S2
3179	vpsllq		\$2,$S1,$S1
3180	vpsllq		\$2,$S2,$S2
3181
3182	jmp		.Lmul_init_vpmadd52
3183	ud2
3184
3185.align	32
3186.Ldone_init_vpmadd52:
3187	vinserti128	\$1,%x#$R1,$H1,$R1	# 1,2,3,4
3188	vinserti128	\$1,%x#$R2,$H2,$R2
3189	vinserti128	\$1,%x#$R0,$H0,$R0
3190
3191	vpermq		\$0b11011000,$R1,$R1	# 1,3,2,4
3192	vpermq		\$0b11011000,$R2,$R2
3193	vpermq		\$0b11011000,$R0,$R0
3194
3195	vpsllq		\$2,$R1,$S1		# S1 = R1*5*4
3196	vpaddq		$R1,$S1,$S1
3197	vpsllq		\$2,$S1,$S1
3198
3199	vmovq		0($ctx),%x#$H0		# load current hash value
3200	vmovq		8($ctx),%x#$H1
3201	vmovq		16($ctx),%x#$H2
3202
3203	test		\$3,$len		# is length 4*n+2?
3204	jnz		.Ldone_init_vpmadd52_2x
3205
3206	vmovdqu64	$R0,64($ctx)		# save key powers
3207	vpbroadcastq	%x#$R0,$R0		# broadcast 4th power
3208	vmovdqu64	$R1,96($ctx)
3209	vpbroadcastq	%x#$R1,$R1
3210	vmovdqu64	$R2,128($ctx)
3211	vpbroadcastq	%x#$R2,$R2
3212	vmovdqu64	$S1,160($ctx)
3213	vpbroadcastq	%x#$S1,$S1
3214
3215	jmp		.Lblocks_vpmadd52_4x_key_loaded
3216	ud2
3217
3218.align	32
3219.Ldone_init_vpmadd52_2x:
3220	vmovdqu64	$R0,64($ctx)		# save key powers
3221	vpsrldq		\$8,$R0,$R0		# 0-1-0-2
3222	vmovdqu64	$R1,96($ctx)
3223	vpsrldq		\$8,$R1,$R1
3224	vmovdqu64	$R2,128($ctx)
3225	vpsrldq		\$8,$R2,$R2
3226	vmovdqu64	$S1,160($ctx)
3227	vpsrldq		\$8,$S1,$S1
3228	jmp		.Lblocks_vpmadd52_2x_key_loaded
3229	ud2
3230
3231.align	32
3232.Lblocks_vpmadd52_2x_do:
3233	vmovdqu64	128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers
3234	vmovdqu64	160+8($ctx),${S1}{%k1}{z}
3235	vmovdqu64	64+8($ctx),${R0}{%k1}{z}
3236	vmovdqu64	96+8($ctx),${R1}{%k1}{z}
3237
3238.Lblocks_vpmadd52_2x_key_loaded:
3239	vmovdqu64	16*0($inp),$T2		# load data
3240	vpxorq		$T3,$T3,$T3
3241	lea		16*2($inp),$inp
3242
3243	vpunpcklqdq	$T3,$T2,$T1		# transpose data
3244	vpunpckhqdq	$T3,$T2,$T3
3245
3246	# at this point 64-bit lanes are ordered as x-1-x-0
3247
3248	vpsrlq		\$24,$T3,$T2		# splat the data
3249	vporq		$PAD,$T2,$T2
3250	 vpaddq		$T2,$H2,$H2		# accumulate input
3251	vpandq		$mask44,$T1,$T0
3252	vpsrlq		\$44,$T1,$T1
3253	vpsllq		\$20,$T3,$T3
3254	vporq		$T3,$T1,$T1
3255	vpandq		$mask44,$T1,$T1
3256
3257	jmp		.Ltail_vpmadd52_2x
3258	ud2
3259
3260.align	32
3261.Loop_vpmadd52_4x:
3262	#vpaddq		$T2,$H2,$H2		# accumulate input
3263	vpaddq		$T0,$H0,$H0
3264	vpaddq		$T1,$H1,$H1
3265
3266	vpxorq		$D0lo,$D0lo,$D0lo
3267	vpmadd52luq	$H2,$S1,$D0lo
3268	vpxorq		$D0hi,$D0hi,$D0hi
3269	vpmadd52huq	$H2,$S1,$D0hi
3270	vpxorq		$D1lo,$D1lo,$D1lo
3271	vpmadd52luq	$H2,$S2,$D1lo
3272	vpxorq		$D1hi,$D1hi,$D1hi
3273	vpmadd52huq	$H2,$S2,$D1hi
3274	vpxorq		$D2lo,$D2lo,$D2lo
3275	vpmadd52luq	$H2,$R0,$D2lo
3276	vpxorq		$D2hi,$D2hi,$D2hi
3277	vpmadd52huq	$H2,$R0,$D2hi
3278
3279	 vmovdqu64	16*0($inp),$T2		# load data
3280	 vmovdqu64	16*2($inp),$T3
3281	 lea		16*4($inp),$inp
3282	vpmadd52luq	$H0,$R0,$D0lo
3283	vpmadd52huq	$H0,$R0,$D0hi
3284	vpmadd52luq	$H0,$R1,$D1lo
3285	vpmadd52huq	$H0,$R1,$D1hi
3286	vpmadd52luq	$H0,$R2,$D2lo
3287	vpmadd52huq	$H0,$R2,$D2hi
3288
3289	 vpunpcklqdq	$T3,$T2,$T1		# transpose data
3290	 vpunpckhqdq	$T3,$T2,$T3
3291	vpmadd52luq	$H1,$S2,$D0lo
3292	vpmadd52huq	$H1,$S2,$D0hi
3293	vpmadd52luq	$H1,$R0,$D1lo
3294	vpmadd52huq	$H1,$R0,$D1hi
3295	vpmadd52luq	$H1,$R1,$D2lo
3296	vpmadd52huq	$H1,$R1,$D2hi
3297
3298	################################################################
3299	# partial reduction (interleaved with data splat)
3300	vpsrlq		\$44,$D0lo,$tmp
3301	vpsllq		\$8,$D0hi,$D0hi
3302	vpandq		$mask44,$D0lo,$H0
3303	vpaddq		$tmp,$D0hi,$D0hi
3304
3305	 vpsrlq		\$24,$T3,$T2
3306	 vporq		$PAD,$T2,$T2
3307	vpaddq		$D0hi,$D1lo,$D1lo
3308
3309	vpsrlq		\$44,$D1lo,$tmp
3310	vpsllq		\$8,$D1hi,$D1hi
3311	vpandq		$mask44,$D1lo,$H1
3312	vpaddq		$tmp,$D1hi,$D1hi
3313
3314	 vpandq		$mask44,$T1,$T0
3315	 vpsrlq		\$44,$T1,$T1
3316	 vpsllq		\$20,$T3,$T3
3317	vpaddq		$D1hi,$D2lo,$D2lo
3318
3319	vpsrlq		\$42,$D2lo,$tmp
3320	vpsllq		\$10,$D2hi,$D2hi
3321	vpandq		$mask42,$D2lo,$H2
3322	vpaddq		$tmp,$D2hi,$D2hi
3323
3324	  vpaddq	$T2,$H2,$H2		# accumulate input
3325	vpaddq		$D2hi,$H0,$H0
3326	vpsllq		\$2,$D2hi,$D2hi
3327
3328	vpaddq		$D2hi,$H0,$H0
3329	 vporq		$T3,$T1,$T1
3330	 vpandq		$mask44,$T1,$T1
3331
3332	vpsrlq		\$44,$H0,$tmp		# additional step
3333	vpandq		$mask44,$H0,$H0
3334
3335	vpaddq		$tmp,$H1,$H1
3336
3337	sub		\$4,$len		# len-=64
3338	jnz		.Loop_vpmadd52_4x
3339
3340.Ltail_vpmadd52_4x:
3341	vmovdqu64	128($ctx),$R2		# load all key powers
3342	vmovdqu64	160($ctx),$S1
3343	vmovdqu64	64($ctx),$R0
3344	vmovdqu64	96($ctx),$R1
3345
3346.Ltail_vpmadd52_2x:
3347	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
3348	vpaddq		$R2,$S2,$S2
3349	vpsllq		\$2,$S2,$S2
3350
3351	#vpaddq		$T2,$H2,$H2		# accumulate input
3352	vpaddq		$T0,$H0,$H0
3353	vpaddq		$T1,$H1,$H1
3354
3355	vpxorq		$D0lo,$D0lo,$D0lo
3356	vpmadd52luq	$H2,$S1,$D0lo
3357	vpxorq		$D0hi,$D0hi,$D0hi
3358	vpmadd52huq	$H2,$S1,$D0hi
3359	vpxorq		$D1lo,$D1lo,$D1lo
3360	vpmadd52luq	$H2,$S2,$D1lo
3361	vpxorq		$D1hi,$D1hi,$D1hi
3362	vpmadd52huq	$H2,$S2,$D1hi
3363	vpxorq		$D2lo,$D2lo,$D2lo
3364	vpmadd52luq	$H2,$R0,$D2lo
3365	vpxorq		$D2hi,$D2hi,$D2hi
3366	vpmadd52huq	$H2,$R0,$D2hi
3367
3368	vpmadd52luq	$H0,$R0,$D0lo
3369	vpmadd52huq	$H0,$R0,$D0hi
3370	vpmadd52luq	$H0,$R1,$D1lo
3371	vpmadd52huq	$H0,$R1,$D1hi
3372	vpmadd52luq	$H0,$R2,$D2lo
3373	vpmadd52huq	$H0,$R2,$D2hi
3374
3375	vpmadd52luq	$H1,$S2,$D0lo
3376	vpmadd52huq	$H1,$S2,$D0hi
3377	vpmadd52luq	$H1,$R0,$D1lo
3378	vpmadd52huq	$H1,$R0,$D1hi
3379	vpmadd52luq	$H1,$R1,$D2lo
3380	vpmadd52huq	$H1,$R1,$D2hi
3381
3382	################################################################
3383	# horizontal addition
3384
3385	mov		\$1,%eax
3386	kmovw		%eax,%k1
3387	vpsrldq		\$8,$D0lo,$T0
3388	vpsrldq		\$8,$D0hi,$H0
3389	vpsrldq		\$8,$D1lo,$T1
3390	vpsrldq		\$8,$D1hi,$H1
3391	vpaddq		$T0,$D0lo,$D0lo
3392	vpaddq		$H0,$D0hi,$D0hi
3393	vpsrldq		\$8,$D2lo,$T2
3394	vpsrldq		\$8,$D2hi,$H2
3395	vpaddq		$T1,$D1lo,$D1lo
3396	vpaddq		$H1,$D1hi,$D1hi
3397	 vpermq		\$0x2,$D0lo,$T0
3398	 vpermq		\$0x2,$D0hi,$H0
3399	vpaddq		$T2,$D2lo,$D2lo
3400	vpaddq		$H2,$D2hi,$D2hi
3401
3402	vpermq		\$0x2,$D1lo,$T1
3403	vpermq		\$0x2,$D1hi,$H1
3404	vpaddq		$T0,$D0lo,${D0lo}{%k1}{z}
3405	vpaddq		$H0,$D0hi,${D0hi}{%k1}{z}
3406	vpermq		\$0x2,$D2lo,$T2
3407	vpermq		\$0x2,$D2hi,$H2
3408	vpaddq		$T1,$D1lo,${D1lo}{%k1}{z}
3409	vpaddq		$H1,$D1hi,${D1hi}{%k1}{z}
3410	vpaddq		$T2,$D2lo,${D2lo}{%k1}{z}
3411	vpaddq		$H2,$D2hi,${D2hi}{%k1}{z}
3412
3413	################################################################
3414	# partial reduction
3415	vpsrlq		\$44,$D0lo,$tmp
3416	vpsllq		\$8,$D0hi,$D0hi
3417	vpandq		$mask44,$D0lo,$H0
3418	vpaddq		$tmp,$D0hi,$D0hi
3419
3420	vpaddq		$D0hi,$D1lo,$D1lo
3421
3422	vpsrlq		\$44,$D1lo,$tmp
3423	vpsllq		\$8,$D1hi,$D1hi
3424	vpandq		$mask44,$D1lo,$H1
3425	vpaddq		$tmp,$D1hi,$D1hi
3426
3427	vpaddq		$D1hi,$D2lo,$D2lo
3428
3429	vpsrlq		\$42,$D2lo,$tmp
3430	vpsllq		\$10,$D2hi,$D2hi
3431	vpandq		$mask42,$D2lo,$H2
3432	vpaddq		$tmp,$D2hi,$D2hi
3433
3434	vpaddq		$D2hi,$H0,$H0
3435	vpsllq		\$2,$D2hi,$D2hi
3436
3437	vpaddq		$D2hi,$H0,$H0
3438
3439	vpsrlq		\$44,$H0,$tmp		# additional step
3440	vpandq		$mask44,$H0,$H0
3441
3442	vpaddq		$tmp,$H1,$H1
3443						# at this point $len is
3444						# either 4*n+2 or 0...
3445	sub		\$2,$len		# len-=32
3446	ja		.Lblocks_vpmadd52_4x_do
3447
3448	vmovq		%x#$H0,0($ctx)
3449	vmovq		%x#$H1,8($ctx)
3450	vmovq		%x#$H2,16($ctx)
3451	vzeroall
3452
3453.Lno_data_vpmadd52_4x:
3454	RET
3455.size	poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x
3456___
3457}
3458{
3459########################################################################
3460# As implied by its name 8x subroutine processes 8 blocks in parallel...
3461# This is intermediate version, as it's used only in cases when input
3462# length is either 8*n, 8*n+1 or 8*n+2...
3463
3464my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17));
3465my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23));
3466my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31));
3467my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10));
3468
3469$code.=<<___;
3470.type	poly1305_blocks_vpmadd52_8x,\@function,4
3471.align	32
3472poly1305_blocks_vpmadd52_8x:
3473	shr	\$4,$len
3474	jz	.Lno_data_vpmadd52_8x		# too short
3475
3476	shl	\$40,$padbit
3477	mov	64($ctx),%r8			# peek on power of the key
3478
3479	vmovdqa64	.Lx_mask44(%rip),$mask44
3480	vmovdqa64	.Lx_mask42(%rip),$mask42
3481
3482	test	%r8,%r8				# is power value impossible?
3483	js	.Linit_vpmadd52			# if it is, then init R[4]
3484
3485	vmovq	0($ctx),%x#$H0			# load current hash value
3486	vmovq	8($ctx),%x#$H1
3487	vmovq	16($ctx),%x#$H2
3488
3489.Lblocks_vpmadd52_8x:
3490	################################################################
3491	# fist we calculate more key powers
3492
3493	vmovdqu64	128($ctx),$R2		# load 1-3-2-4 powers
3494	vmovdqu64	160($ctx),$S1
3495	vmovdqu64	64($ctx),$R0
3496	vmovdqu64	96($ctx),$R1
3497
3498	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
3499	vpaddq		$R2,$S2,$S2
3500	vpsllq		\$2,$S2,$S2
3501
3502	vpbroadcastq	%x#$R2,$RR2		# broadcast 4th power
3503	vpbroadcastq	%x#$R0,$RR0
3504	vpbroadcastq	%x#$R1,$RR1
3505
3506	vpxorq		$D0lo,$D0lo,$D0lo
3507	vpmadd52luq	$RR2,$S1,$D0lo
3508	vpxorq		$D0hi,$D0hi,$D0hi
3509	vpmadd52huq	$RR2,$S1,$D0hi
3510	vpxorq		$D1lo,$D1lo,$D1lo
3511	vpmadd52luq	$RR2,$S2,$D1lo
3512	vpxorq		$D1hi,$D1hi,$D1hi
3513	vpmadd52huq	$RR2,$S2,$D1hi
3514	vpxorq		$D2lo,$D2lo,$D2lo
3515	vpmadd52luq	$RR2,$R0,$D2lo
3516	vpxorq		$D2hi,$D2hi,$D2hi
3517	vpmadd52huq	$RR2,$R0,$D2hi
3518
3519	vpmadd52luq	$RR0,$R0,$D0lo
3520	vpmadd52huq	$RR0,$R0,$D0hi
3521	vpmadd52luq	$RR0,$R1,$D1lo
3522	vpmadd52huq	$RR0,$R1,$D1hi
3523	vpmadd52luq	$RR0,$R2,$D2lo
3524	vpmadd52huq	$RR0,$R2,$D2hi
3525
3526	vpmadd52luq	$RR1,$S2,$D0lo
3527	vpmadd52huq	$RR1,$S2,$D0hi
3528	vpmadd52luq	$RR1,$R0,$D1lo
3529	vpmadd52huq	$RR1,$R0,$D1hi
3530	vpmadd52luq	$RR1,$R1,$D2lo
3531	vpmadd52huq	$RR1,$R1,$D2hi
3532
3533	################################################################
3534	# partial reduction
3535	vpsrlq		\$44,$D0lo,$tmp
3536	vpsllq		\$8,$D0hi,$D0hi
3537	vpandq		$mask44,$D0lo,$RR0
3538	vpaddq		$tmp,$D0hi,$D0hi
3539
3540	vpaddq		$D0hi,$D1lo,$D1lo
3541
3542	vpsrlq		\$44,$D1lo,$tmp
3543	vpsllq		\$8,$D1hi,$D1hi
3544	vpandq		$mask44,$D1lo,$RR1
3545	vpaddq		$tmp,$D1hi,$D1hi
3546
3547	vpaddq		$D1hi,$D2lo,$D2lo
3548
3549	vpsrlq		\$42,$D2lo,$tmp
3550	vpsllq		\$10,$D2hi,$D2hi
3551	vpandq		$mask42,$D2lo,$RR2
3552	vpaddq		$tmp,$D2hi,$D2hi
3553
3554	vpaddq		$D2hi,$RR0,$RR0
3555	vpsllq		\$2,$D2hi,$D2hi
3556
3557	vpaddq		$D2hi,$RR0,$RR0
3558
3559	vpsrlq		\$44,$RR0,$tmp		# additional step
3560	vpandq		$mask44,$RR0,$RR0
3561
3562	vpaddq		$tmp,$RR1,$RR1
3563
3564	################################################################
3565	# At this point Rx holds 1324 powers, RRx - 5768, and the goal
3566	# is 15263748, which reflects how data is loaded...
3567
3568	vpunpcklqdq	$R2,$RR2,$T2		# 3748
3569	vpunpckhqdq	$R2,$RR2,$R2		# 1526
3570	vpunpcklqdq	$R0,$RR0,$T0
3571	vpunpckhqdq	$R0,$RR0,$R0
3572	vpunpcklqdq	$R1,$RR1,$T1
3573	vpunpckhqdq	$R1,$RR1,$R1
3574___
3575######## switch to %zmm
3576map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2);
3577map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi);
3578map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD);
3579map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2);
3580
3581$code.=<<___;
3582	vshufi64x2	\$0x44,$R2,$T2,$RR2	# 15263748
3583	vshufi64x2	\$0x44,$R0,$T0,$RR0
3584	vshufi64x2	\$0x44,$R1,$T1,$RR1
3585
3586	vmovdqu64	16*0($inp),$T2		# load data
3587	vmovdqu64	16*4($inp),$T3
3588	lea		16*8($inp),$inp
3589
3590	vpsllq		\$2,$RR2,$SS2		# S2 = R2*5*4
3591	vpsllq		\$2,$RR1,$SS1		# S1 = R1*5*4
3592	vpaddq		$RR2,$SS2,$SS2
3593	vpaddq		$RR1,$SS1,$SS1
3594	vpsllq		\$2,$SS2,$SS2
3595	vpsllq		\$2,$SS1,$SS1
3596
3597	vpbroadcastq	$padbit,$PAD
3598	vpbroadcastq	%x#$mask44,$mask44
3599	vpbroadcastq	%x#$mask42,$mask42
3600
3601	vpbroadcastq	%x#$SS1,$S1		# broadcast 8th power
3602	vpbroadcastq	%x#$SS2,$S2
3603	vpbroadcastq	%x#$RR0,$R0
3604	vpbroadcastq	%x#$RR1,$R1
3605	vpbroadcastq	%x#$RR2,$R2
3606
3607	vpunpcklqdq	$T3,$T2,$T1		# transpose data
3608	vpunpckhqdq	$T3,$T2,$T3
3609
3610	# at this point 64-bit lanes are ordered as 73625140
3611
3612	vpsrlq		\$24,$T3,$T2		# splat the data
3613	vporq		$PAD,$T2,$T2
3614	 vpaddq		$T2,$H2,$H2		# accumulate input
3615	vpandq		$mask44,$T1,$T0
3616	vpsrlq		\$44,$T1,$T1
3617	vpsllq		\$20,$T3,$T3
3618	vporq		$T3,$T1,$T1
3619	vpandq		$mask44,$T1,$T1
3620
3621	sub		\$8,$len
3622	jz		.Ltail_vpmadd52_8x
3623	jmp		.Loop_vpmadd52_8x
3624
3625.align	32
3626.Loop_vpmadd52_8x:
3627	#vpaddq		$T2,$H2,$H2		# accumulate input
3628	vpaddq		$T0,$H0,$H0
3629	vpaddq		$T1,$H1,$H1
3630
3631	vpxorq		$D0lo,$D0lo,$D0lo
3632	vpmadd52luq	$H2,$S1,$D0lo
3633	vpxorq		$D0hi,$D0hi,$D0hi
3634	vpmadd52huq	$H2,$S1,$D0hi
3635	vpxorq		$D1lo,$D1lo,$D1lo
3636	vpmadd52luq	$H2,$S2,$D1lo
3637	vpxorq		$D1hi,$D1hi,$D1hi
3638	vpmadd52huq	$H2,$S2,$D1hi
3639	vpxorq		$D2lo,$D2lo,$D2lo
3640	vpmadd52luq	$H2,$R0,$D2lo
3641	vpxorq		$D2hi,$D2hi,$D2hi
3642	vpmadd52huq	$H2,$R0,$D2hi
3643
3644	 vmovdqu64	16*0($inp),$T2		# load data
3645	 vmovdqu64	16*4($inp),$T3
3646	 lea		16*8($inp),$inp
3647	vpmadd52luq	$H0,$R0,$D0lo
3648	vpmadd52huq	$H0,$R0,$D0hi
3649	vpmadd52luq	$H0,$R1,$D1lo
3650	vpmadd52huq	$H0,$R1,$D1hi
3651	vpmadd52luq	$H0,$R2,$D2lo
3652	vpmadd52huq	$H0,$R2,$D2hi
3653
3654	 vpunpcklqdq	$T3,$T2,$T1		# transpose data
3655	 vpunpckhqdq	$T3,$T2,$T3
3656	vpmadd52luq	$H1,$S2,$D0lo
3657	vpmadd52huq	$H1,$S2,$D0hi
3658	vpmadd52luq	$H1,$R0,$D1lo
3659	vpmadd52huq	$H1,$R0,$D1hi
3660	vpmadd52luq	$H1,$R1,$D2lo
3661	vpmadd52huq	$H1,$R1,$D2hi
3662
3663	################################################################
3664	# partial reduction (interleaved with data splat)
3665	vpsrlq		\$44,$D0lo,$tmp
3666	vpsllq		\$8,$D0hi,$D0hi
3667	vpandq		$mask44,$D0lo,$H0
3668	vpaddq		$tmp,$D0hi,$D0hi
3669
3670	 vpsrlq		\$24,$T3,$T2
3671	 vporq		$PAD,$T2,$T2
3672	vpaddq		$D0hi,$D1lo,$D1lo
3673
3674	vpsrlq		\$44,$D1lo,$tmp
3675	vpsllq		\$8,$D1hi,$D1hi
3676	vpandq		$mask44,$D1lo,$H1
3677	vpaddq		$tmp,$D1hi,$D1hi
3678
3679	 vpandq		$mask44,$T1,$T0
3680	 vpsrlq		\$44,$T1,$T1
3681	 vpsllq		\$20,$T3,$T3
3682	vpaddq		$D1hi,$D2lo,$D2lo
3683
3684	vpsrlq		\$42,$D2lo,$tmp
3685	vpsllq		\$10,$D2hi,$D2hi
3686	vpandq		$mask42,$D2lo,$H2
3687	vpaddq		$tmp,$D2hi,$D2hi
3688
3689	  vpaddq	$T2,$H2,$H2		# accumulate input
3690	vpaddq		$D2hi,$H0,$H0
3691	vpsllq		\$2,$D2hi,$D2hi
3692
3693	vpaddq		$D2hi,$H0,$H0
3694	 vporq		$T3,$T1,$T1
3695	 vpandq		$mask44,$T1,$T1
3696
3697	vpsrlq		\$44,$H0,$tmp		# additional step
3698	vpandq		$mask44,$H0,$H0
3699
3700	vpaddq		$tmp,$H1,$H1
3701
3702	sub		\$8,$len		# len-=128
3703	jnz		.Loop_vpmadd52_8x
3704
3705.Ltail_vpmadd52_8x:
3706	#vpaddq		$T2,$H2,$H2		# accumulate input
3707	vpaddq		$T0,$H0,$H0
3708	vpaddq		$T1,$H1,$H1
3709
3710	vpxorq		$D0lo,$D0lo,$D0lo
3711	vpmadd52luq	$H2,$SS1,$D0lo
3712	vpxorq		$D0hi,$D0hi,$D0hi
3713	vpmadd52huq	$H2,$SS1,$D0hi
3714	vpxorq		$D1lo,$D1lo,$D1lo
3715	vpmadd52luq	$H2,$SS2,$D1lo
3716	vpxorq		$D1hi,$D1hi,$D1hi
3717	vpmadd52huq	$H2,$SS2,$D1hi
3718	vpxorq		$D2lo,$D2lo,$D2lo
3719	vpmadd52luq	$H2,$RR0,$D2lo
3720	vpxorq		$D2hi,$D2hi,$D2hi
3721	vpmadd52huq	$H2,$RR0,$D2hi
3722
3723	vpmadd52luq	$H0,$RR0,$D0lo
3724	vpmadd52huq	$H0,$RR0,$D0hi
3725	vpmadd52luq	$H0,$RR1,$D1lo
3726	vpmadd52huq	$H0,$RR1,$D1hi
3727	vpmadd52luq	$H0,$RR2,$D2lo
3728	vpmadd52huq	$H0,$RR2,$D2hi
3729
3730	vpmadd52luq	$H1,$SS2,$D0lo
3731	vpmadd52huq	$H1,$SS2,$D0hi
3732	vpmadd52luq	$H1,$RR0,$D1lo
3733	vpmadd52huq	$H1,$RR0,$D1hi
3734	vpmadd52luq	$H1,$RR1,$D2lo
3735	vpmadd52huq	$H1,$RR1,$D2hi
3736
3737	################################################################
3738	# horizontal addition
3739
3740	mov		\$1,%eax
3741	kmovw		%eax,%k1
3742	vpsrldq		\$8,$D0lo,$T0
3743	vpsrldq		\$8,$D0hi,$H0
3744	vpsrldq		\$8,$D1lo,$T1
3745	vpsrldq		\$8,$D1hi,$H1
3746	vpaddq		$T0,$D0lo,$D0lo
3747	vpaddq		$H0,$D0hi,$D0hi
3748	vpsrldq		\$8,$D2lo,$T2
3749	vpsrldq		\$8,$D2hi,$H2
3750	vpaddq		$T1,$D1lo,$D1lo
3751	vpaddq		$H1,$D1hi,$D1hi
3752	 vpermq		\$0x2,$D0lo,$T0
3753	 vpermq		\$0x2,$D0hi,$H0
3754	vpaddq		$T2,$D2lo,$D2lo
3755	vpaddq		$H2,$D2hi,$D2hi
3756
3757	vpermq		\$0x2,$D1lo,$T1
3758	vpermq		\$0x2,$D1hi,$H1
3759	vpaddq		$T0,$D0lo,$D0lo
3760	vpaddq		$H0,$D0hi,$D0hi
3761	vpermq		\$0x2,$D2lo,$T2
3762	vpermq		\$0x2,$D2hi,$H2
3763	vpaddq		$T1,$D1lo,$D1lo
3764	vpaddq		$H1,$D1hi,$D1hi
3765	 vextracti64x4	\$1,$D0lo,%y#$T0
3766	 vextracti64x4	\$1,$D0hi,%y#$H0
3767	vpaddq		$T2,$D2lo,$D2lo
3768	vpaddq		$H2,$D2hi,$D2hi
3769
3770	vextracti64x4	\$1,$D1lo,%y#$T1
3771	vextracti64x4	\$1,$D1hi,%y#$H1
3772	vextracti64x4	\$1,$D2lo,%y#$T2
3773	vextracti64x4	\$1,$D2hi,%y#$H2
3774___
3775######## switch back to %ymm
3776map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2);
3777map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi);
3778map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD);
3779
3780$code.=<<___;
3781	vpaddq		$T0,$D0lo,${D0lo}{%k1}{z}
3782	vpaddq		$H0,$D0hi,${D0hi}{%k1}{z}
3783	vpaddq		$T1,$D1lo,${D1lo}{%k1}{z}
3784	vpaddq		$H1,$D1hi,${D1hi}{%k1}{z}
3785	vpaddq		$T2,$D2lo,${D2lo}{%k1}{z}
3786	vpaddq		$H2,$D2hi,${D2hi}{%k1}{z}
3787
3788	################################################################
3789	# partial reduction
3790	vpsrlq		\$44,$D0lo,$tmp
3791	vpsllq		\$8,$D0hi,$D0hi
3792	vpandq		$mask44,$D0lo,$H0
3793	vpaddq		$tmp,$D0hi,$D0hi
3794
3795	vpaddq		$D0hi,$D1lo,$D1lo
3796
3797	vpsrlq		\$44,$D1lo,$tmp
3798	vpsllq		\$8,$D1hi,$D1hi
3799	vpandq		$mask44,$D1lo,$H1
3800	vpaddq		$tmp,$D1hi,$D1hi
3801
3802	vpaddq		$D1hi,$D2lo,$D2lo
3803
3804	vpsrlq		\$42,$D2lo,$tmp
3805	vpsllq		\$10,$D2hi,$D2hi
3806	vpandq		$mask42,$D2lo,$H2
3807	vpaddq		$tmp,$D2hi,$D2hi
3808
3809	vpaddq		$D2hi,$H0,$H0
3810	vpsllq		\$2,$D2hi,$D2hi
3811
3812	vpaddq		$D2hi,$H0,$H0
3813
3814	vpsrlq		\$44,$H0,$tmp		# additional step
3815	vpandq		$mask44,$H0,$H0
3816
3817	vpaddq		$tmp,$H1,$H1
3818
3819	################################################################
3820
3821	vmovq		%x#$H0,0($ctx)
3822	vmovq		%x#$H1,8($ctx)
3823	vmovq		%x#$H2,16($ctx)
3824	vzeroall
3825
3826.Lno_data_vpmadd52_8x:
3827	RET
3828.size	poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x
3829___
3830}
3831$code.=<<___;
3832.type	poly1305_emit_base2_44,\@function,3
3833.align	32
3834poly1305_emit_base2_44:
3835	mov	0($ctx),%r8	# load hash value
3836	mov	8($ctx),%r9
3837	mov	16($ctx),%r10
3838
3839	mov	%r9,%rax
3840	shr	\$20,%r9
3841	shl	\$44,%rax
3842	mov	%r10,%rcx
3843	shr	\$40,%r10
3844	shl	\$24,%rcx
3845
3846	add	%rax,%r8
3847	adc	%rcx,%r9
3848	adc	\$0,%r10
3849
3850	mov	%r8,%rax
3851	add	\$5,%r8		# compare to modulus
3852	mov	%r9,%rcx
3853	adc	\$0,%r9
3854	adc	\$0,%r10
3855	shr	\$2,%r10	# did 130-bit value overflow?
3856	cmovnz	%r8,%rax
3857	cmovnz	%r9,%rcx
3858
3859	add	0($nonce),%rax	# accumulate nonce
3860	adc	8($nonce),%rcx
3861	mov	%rax,0($mac)	# write result
3862	mov	%rcx,8($mac)
3863
3864	RET
3865.size	poly1305_emit_base2_44,.-poly1305_emit_base2_44
3866___
3867}	}	}
3868}
3869
3870if (!$kernel)
3871{	# chacha20-poly1305 helpers
3872my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") :  # Win64 order
3873                                  ("%rdi","%rsi","%rdx","%rcx");  # Unix order
3874$code.=<<___;
3875.globl	xor128_encrypt_n_pad
3876.type	xor128_encrypt_n_pad,\@abi-omnipotent
3877.align	16
3878xor128_encrypt_n_pad:
3879	sub	$otp,$inp
3880	sub	$otp,$out
3881	mov	$len,%r10		# put len aside
3882	shr	\$4,$len		# len / 16
3883	jz	.Ltail_enc
3884	nop
3885.Loop_enc_xmm:
3886	movdqu	($inp,$otp),%xmm0
3887	pxor	($otp),%xmm0
3888	movdqu	%xmm0,($out,$otp)
3889	movdqa	%xmm0,($otp)
3890	lea	16($otp),$otp
3891	dec	$len
3892	jnz	.Loop_enc_xmm
3893
3894	and	\$15,%r10		# len % 16
3895	jz	.Ldone_enc
3896
3897.Ltail_enc:
3898	mov	\$16,$len
3899	sub	%r10,$len
3900	xor	%eax,%eax
3901.Loop_enc_byte:
3902	mov	($inp,$otp),%al
3903	xor	($otp),%al
3904	mov	%al,($out,$otp)
3905	mov	%al,($otp)
3906	lea	1($otp),$otp
3907	dec	%r10
3908	jnz	.Loop_enc_byte
3909
3910	xor	%eax,%eax
3911.Loop_enc_pad:
3912	mov	%al,($otp)
3913	lea	1($otp),$otp
3914	dec	$len
3915	jnz	.Loop_enc_pad
3916
3917.Ldone_enc:
3918	mov	$otp,%rax
3919	RET
3920.size	xor128_encrypt_n_pad,.-xor128_encrypt_n_pad
3921
3922.globl	xor128_decrypt_n_pad
3923.type	xor128_decrypt_n_pad,\@abi-omnipotent
3924.align	16
3925xor128_decrypt_n_pad:
3926	sub	$otp,$inp
3927	sub	$otp,$out
3928	mov	$len,%r10		# put len aside
3929	shr	\$4,$len		# len / 16
3930	jz	.Ltail_dec
3931	nop
3932.Loop_dec_xmm:
3933	movdqu	($inp,$otp),%xmm0
3934	movdqa	($otp),%xmm1
3935	pxor	%xmm0,%xmm1
3936	movdqu	%xmm1,($out,$otp)
3937	movdqa	%xmm0,($otp)
3938	lea	16($otp),$otp
3939	dec	$len
3940	jnz	.Loop_dec_xmm
3941
3942	pxor	%xmm1,%xmm1
3943	and	\$15,%r10		# len % 16
3944	jz	.Ldone_dec
3945
3946.Ltail_dec:
3947	mov	\$16,$len
3948	sub	%r10,$len
3949	xor	%eax,%eax
3950	xor	%r11d,%r11d
3951.Loop_dec_byte:
3952	mov	($inp,$otp),%r11b
3953	mov	($otp),%al
3954	xor	%r11b,%al
3955	mov	%al,($out,$otp)
3956	mov	%r11b,($otp)
3957	lea	1($otp),$otp
3958	dec	%r10
3959	jnz	.Loop_dec_byte
3960
3961	xor	%eax,%eax
3962.Loop_dec_pad:
3963	mov	%al,($otp)
3964	lea	1($otp),$otp
3965	dec	$len
3966	jnz	.Loop_dec_pad
3967
3968.Ldone_dec:
3969	mov	$otp,%rax
3970	RET
3971.size	xor128_decrypt_n_pad,.-xor128_decrypt_n_pad
3972___
3973}
3974
3975# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
3976#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
3977if ($win64) {
3978$rec="%rcx";
3979$frame="%rdx";
3980$context="%r8";
3981$disp="%r9";
3982
3983$code.=<<___;
3984.extern	__imp_RtlVirtualUnwind
3985.type	se_handler,\@abi-omnipotent
3986.align	16
3987se_handler:
3988	push	%rsi
3989	push	%rdi
3990	push	%rbx
3991	push	%rbp
3992	push	%r12
3993	push	%r13
3994	push	%r14
3995	push	%r15
3996	pushfq
3997	sub	\$64,%rsp
3998
3999	mov	120($context),%rax	# pull context->Rax
4000	mov	248($context),%rbx	# pull context->Rip
4001
4002	mov	8($disp),%rsi		# disp->ImageBase
4003	mov	56($disp),%r11		# disp->HandlerData
4004
4005	mov	0(%r11),%r10d		# HandlerData[0]
4006	lea	(%rsi,%r10),%r10	# prologue label
4007	cmp	%r10,%rbx		# context->Rip<.Lprologue
4008	jb	.Lcommon_seh_tail
4009
4010	mov	152($context),%rax	# pull context->Rsp
4011
4012	mov	4(%r11),%r10d		# HandlerData[1]
4013	lea	(%rsi,%r10),%r10	# epilogue label
4014	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
4015	jae	.Lcommon_seh_tail
4016
4017	lea	48(%rax),%rax
4018
4019	mov	-8(%rax),%rbx
4020	mov	-16(%rax),%rbp
4021	mov	-24(%rax),%r12
4022	mov	-32(%rax),%r13
4023	mov	-40(%rax),%r14
4024	mov	-48(%rax),%r15
4025	mov	%rbx,144($context)	# restore context->Rbx
4026	mov	%rbp,160($context)	# restore context->Rbp
4027	mov	%r12,216($context)	# restore context->R12
4028	mov	%r13,224($context)	# restore context->R13
4029	mov	%r14,232($context)	# restore context->R14
4030	mov	%r15,240($context)	# restore context->R14
4031
4032	jmp	.Lcommon_seh_tail
4033.size	se_handler,.-se_handler
4034
4035.type	avx_handler,\@abi-omnipotent
4036.align	16
4037avx_handler:
4038	push	%rsi
4039	push	%rdi
4040	push	%rbx
4041	push	%rbp
4042	push	%r12
4043	push	%r13
4044	push	%r14
4045	push	%r15
4046	pushfq
4047	sub	\$64,%rsp
4048
4049	mov	120($context),%rax	# pull context->Rax
4050	mov	248($context),%rbx	# pull context->Rip
4051
4052	mov	8($disp),%rsi		# disp->ImageBase
4053	mov	56($disp),%r11		# disp->HandlerData
4054
4055	mov	0(%r11),%r10d		# HandlerData[0]
4056	lea	(%rsi,%r10),%r10	# prologue label
4057	cmp	%r10,%rbx		# context->Rip<prologue label
4058	jb	.Lcommon_seh_tail
4059
4060	mov	152($context),%rax	# pull context->Rsp
4061
4062	mov	4(%r11),%r10d		# HandlerData[1]
4063	lea	(%rsi,%r10),%r10	# epilogue label
4064	cmp	%r10,%rbx		# context->Rip>=epilogue label
4065	jae	.Lcommon_seh_tail
4066
4067	mov	208($context),%rax	# pull context->R11
4068
4069	lea	0x50(%rax),%rsi
4070	lea	0xf8(%rax),%rax
4071	lea	512($context),%rdi	# &context.Xmm6
4072	mov	\$20,%ecx
4073	.long	0xa548f3fc		# cld; rep movsq
4074
4075.Lcommon_seh_tail:
4076	mov	8(%rax),%rdi
4077	mov	16(%rax),%rsi
4078	mov	%rax,152($context)	# restore context->Rsp
4079	mov	%rsi,168($context)	# restore context->Rsi
4080	mov	%rdi,176($context)	# restore context->Rdi
4081
4082	mov	40($disp),%rdi		# disp->ContextRecord
4083	mov	$context,%rsi		# context
4084	mov	\$154,%ecx		# sizeof(CONTEXT)
4085	.long	0xa548f3fc		# cld; rep movsq
4086
4087	mov	$disp,%rsi
4088	xor	%ecx,%ecx		# arg1, UNW_FLAG_NHANDLER
4089	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
4090	mov	0(%rsi),%r8		# arg3, disp->ControlPc
4091	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
4092	mov	40(%rsi),%r10		# disp->ContextRecord
4093	lea	56(%rsi),%r11		# &disp->HandlerData
4094	lea	24(%rsi),%r12		# &disp->EstablisherFrame
4095	mov	%r10,32(%rsp)		# arg5
4096	mov	%r11,40(%rsp)		# arg6
4097	mov	%r12,48(%rsp)		# arg7
4098	mov	%rcx,56(%rsp)		# arg8, (NULL)
4099	call	*__imp_RtlVirtualUnwind(%rip)
4100
4101	mov	\$1,%eax		# ExceptionContinueSearch
4102	add	\$64,%rsp
4103	popfq
4104	pop	%r15
4105	pop	%r14
4106	pop	%r13
4107	pop	%r12
4108	pop	%rbp
4109	pop	%rbx
4110	pop	%rdi
4111	pop	%rsi
4112	RET
4113.size	avx_handler,.-avx_handler
4114
4115.section	.pdata
4116.align	4
4117	.rva	.LSEH_begin_poly1305_init_x86_64
4118	.rva	.LSEH_end_poly1305_init_x86_64
4119	.rva	.LSEH_info_poly1305_init_x86_64
4120
4121	.rva	.LSEH_begin_poly1305_blocks_x86_64
4122	.rva	.LSEH_end_poly1305_blocks_x86_64
4123	.rva	.LSEH_info_poly1305_blocks_x86_64
4124
4125	.rva	.LSEH_begin_poly1305_emit_x86_64
4126	.rva	.LSEH_end_poly1305_emit_x86_64
4127	.rva	.LSEH_info_poly1305_emit_x86_64
4128___
4129$code.=<<___ if ($avx);
4130	.rva	.LSEH_begin_poly1305_blocks_avx
4131	.rva	.Lbase2_64_avx
4132	.rva	.LSEH_info_poly1305_blocks_avx_1
4133
4134	.rva	.Lbase2_64_avx
4135	.rva	.Leven_avx
4136	.rva	.LSEH_info_poly1305_blocks_avx_2
4137
4138	.rva	.Leven_avx
4139	.rva	.LSEH_end_poly1305_blocks_avx
4140	.rva	.LSEH_info_poly1305_blocks_avx_3
4141
4142	.rva	.LSEH_begin_poly1305_emit_avx
4143	.rva	.LSEH_end_poly1305_emit_avx
4144	.rva	.LSEH_info_poly1305_emit_avx
4145___
4146$code.=<<___ if ($avx>1);
4147	.rva	.LSEH_begin_poly1305_blocks_avx2
4148	.rva	.Lbase2_64_avx2
4149	.rva	.LSEH_info_poly1305_blocks_avx2_1
4150
4151	.rva	.Lbase2_64_avx2
4152	.rva	.Leven_avx2
4153	.rva	.LSEH_info_poly1305_blocks_avx2_2
4154
4155	.rva	.Leven_avx2
4156	.rva	.LSEH_end_poly1305_blocks_avx2
4157	.rva	.LSEH_info_poly1305_blocks_avx2_3
4158___
4159$code.=<<___ if ($avx>2);
4160	.rva	.LSEH_begin_poly1305_blocks_avx512
4161	.rva	.LSEH_end_poly1305_blocks_avx512
4162	.rva	.LSEH_info_poly1305_blocks_avx512
4163___
4164$code.=<<___;
4165.section	.xdata
4166.align	8
4167.LSEH_info_poly1305_init_x86_64:
4168	.byte	9,0,0,0
4169	.rva	se_handler
4170	.rva	.LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64
4171
4172.LSEH_info_poly1305_blocks_x86_64:
4173	.byte	9,0,0,0
4174	.rva	se_handler
4175	.rva	.Lblocks_body,.Lblocks_epilogue
4176
4177.LSEH_info_poly1305_emit_x86_64:
4178	.byte	9,0,0,0
4179	.rva	se_handler
4180	.rva	.LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64
4181___
4182$code.=<<___ if ($avx);
4183.LSEH_info_poly1305_blocks_avx_1:
4184	.byte	9,0,0,0
4185	.rva	se_handler
4186	.rva	.Lblocks_avx_body,.Lblocks_avx_epilogue		# HandlerData[]
4187
4188.LSEH_info_poly1305_blocks_avx_2:
4189	.byte	9,0,0,0
4190	.rva	se_handler
4191	.rva	.Lbase2_64_avx_body,.Lbase2_64_avx_epilogue	# HandlerData[]
4192
4193.LSEH_info_poly1305_blocks_avx_3:
4194	.byte	9,0,0,0
4195	.rva	avx_handler
4196	.rva	.Ldo_avx_body,.Ldo_avx_epilogue			# HandlerData[]
4197
4198.LSEH_info_poly1305_emit_avx:
4199	.byte	9,0,0,0
4200	.rva	se_handler
4201	.rva	.LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx
4202___
4203$code.=<<___ if ($avx>1);
4204.LSEH_info_poly1305_blocks_avx2_1:
4205	.byte	9,0,0,0
4206	.rva	se_handler
4207	.rva	.Lblocks_avx2_body,.Lblocks_avx2_epilogue	# HandlerData[]
4208
4209.LSEH_info_poly1305_blocks_avx2_2:
4210	.byte	9,0,0,0
4211	.rva	se_handler
4212	.rva	.Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue	# HandlerData[]
4213
4214.LSEH_info_poly1305_blocks_avx2_3:
4215	.byte	9,0,0,0
4216	.rva	avx_handler
4217	.rva	.Ldo_avx2_body,.Ldo_avx2_epilogue		# HandlerData[]
4218___
4219$code.=<<___ if ($avx>2);
4220.LSEH_info_poly1305_blocks_avx512:
4221	.byte	9,0,0,0
4222	.rva	avx_handler
4223	.rva	.Ldo_avx512_body,.Ldo_avx512_epilogue		# HandlerData[]
4224___
4225}
4226
4227open SELF,$0;
4228while(<SELF>) {
4229	next if (/^#!/);
4230	last if (!s/^#/\/\// and !/^$/);
4231	print;
4232}
4233close SELF;
4234
4235foreach (split('\n',$code)) {
4236	s/\`([^\`]*)\`/eval($1)/ge;
4237	s/%r([a-z]+)#d/%e$1/g;
4238	s/%r([0-9]+)#d/%r$1d/g;
4239	s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g;
4240
4241	if ($kernel) {
4242		s/(^\.type.*),[0-9]+$/\1/;
4243		s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/;
4244		next if /^\.cfi.*/;
4245	}
4246
4247	print $_,"\n";
4248}
4249close STDOUT;
4250