1#! /usr/bin/env perl
2# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
3#
4# Licensed under the OpenSSL license (the "License").  You may not use
5# this file except in compliance with the License.  You can obtain a copy
6# in the file LICENSE in the source distribution or at
7# https://www.openssl.org/source/license.html
8
9# ====================================================================
10# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
11# project. The module is, however, dual licensed under OpenSSL and
12# CRYPTOGAMS licenses depending on where you obtain it. For further
13# details see http://www.openssl.org/~appro/cryptogams/.
14#
15# Permission to use under GPLv2 terms is granted.
16# ====================================================================
17#
18# SHA256/512 for ARMv8.
19#
20# Performance in cycles per processed byte and improvement coefficient
21# over code generated with "default" compiler:
22#
23#		SHA256-hw	SHA256(*)	SHA512
24# Apple A7	1.97		10.5 (+33%)	6.73 (-1%(**))
25# Cortex-A53	2.38		15.5 (+115%)	10.0 (+150%(***))
26# Cortex-A57	2.31		11.6 (+86%)	7.51 (+260%(***))
27# Denver	2.01		10.5 (+26%)	6.70 (+8%)
28# X-Gene			20.0 (+100%)	12.8 (+300%(***))
29# Mongoose	2.36		13.0 (+50%)	8.36 (+33%)
30#
31# (*)	Software SHA256 results are of lesser relevance, presented
32#	mostly for informational purposes.
33# (**)	The result is a trade-off: it's possible to improve it by
34#	10% (or by 1 cycle per round), but at the cost of 20% loss
35#	on Cortex-A53 (or by 4 cycles per round).
36# (***)	Super-impressive coefficients over gcc-generated code are
37#	indication of some compiler "pathology", most notably code
38#	generated with -mgeneral-regs-only is significanty faster
39#	and the gap is only 40-90%.
40
41$output=pop;
42$flavour=pop;
43
44if ($flavour && $flavour ne "void") {
45    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
46    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
47    ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
48    die "can't locate arm-xlate.pl";
49
50    open OUT,"| \"$^X\" $xlate $flavour $output";
51    *STDOUT=*OUT;
52} else {
53    open STDOUT,">$output";
54}
55
56if ($output =~ /512/) {
57	$BITS=512;
58	$SZ=8;
59	@Sigma0=(28,34,39);
60	@Sigma1=(14,18,41);
61	@sigma0=(1,  8, 7);
62	@sigma1=(19,61, 6);
63	$rounds=80;
64	$reg_t="x";
65} else {
66	$BITS=256;
67	$SZ=4;
68	@Sigma0=( 2,13,22);
69	@Sigma1=( 6,11,25);
70	@sigma0=( 7,18, 3);
71	@sigma1=(17,19,10);
72	$rounds=64;
73	$reg_t="w";
74}
75
76$func="sha${BITS}_block_data_order";
77
78($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
79
80@X=map("$reg_t$_",(3..15,0..2));
81@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
82($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
83
84sub BODY_00_xx {
85my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
86my $j=($i+1)&15;
87my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
88   $T0=@X[$i+3] if ($i<11);
89
90$code.=<<___	if ($i<16);
91#ifndef	__ARMEB__
92	rev	@X[$i],@X[$i]			// $i
93#endif
94___
95$code.=<<___	if ($i<13 && ($i&1));
96	ldp	@X[$i+1],@X[$i+2],[$inp],#2*$SZ
97___
98$code.=<<___	if ($i==13);
99	ldp	@X[14],@X[15],[$inp]
100___
101$code.=<<___	if ($i>=14);
102	ldr	@X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
103___
104$code.=<<___	if ($i>0 && $i<16);
105	add	$a,$a,$t1			// h+=Sigma0(a)
106___
107$code.=<<___	if ($i>=11);
108	str	@X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
109___
110# While ARMv8 specifies merged rotate-n-logical operation such as
111# 'eor x,y,z,ror#n', it was found to negatively affect performance
112# on Apple A7. The reason seems to be that it requires even 'y' to
113# be available earlier. This means that such merged instruction is
114# not necessarily best choice on critical path... On the other hand
115# Cortex-A5x handles merged instructions much better than disjoint
116# rotate and logical... See (**) footnote above.
117$code.=<<___	if ($i<15);
118	ror	$t0,$e,#$Sigma1[0]
119	add	$h,$h,$t2			// h+=K[i]
120	eor	$T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
121	and	$t1,$f,$e
122	bic	$t2,$g,$e
123	add	$h,$h,@X[$i&15]			// h+=X[i]
124	orr	$t1,$t1,$t2			// Ch(e,f,g)
125	eor	$t2,$a,$b			// a^b, b^c in next round
126	eor	$t0,$t0,$T0,ror#$Sigma1[1]	// Sigma1(e)
127	ror	$T0,$a,#$Sigma0[0]
128	add	$h,$h,$t1			// h+=Ch(e,f,g)
129	eor	$t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
130	add	$h,$h,$t0			// h+=Sigma1(e)
131	and	$t3,$t3,$t2			// (b^c)&=(a^b)
132	add	$d,$d,$h			// d+=h
133	eor	$t3,$t3,$b			// Maj(a,b,c)
134	eor	$t1,$T0,$t1,ror#$Sigma0[1]	// Sigma0(a)
135	add	$h,$h,$t3			// h+=Maj(a,b,c)
136	ldr	$t3,[$Ktbl],#$SZ		// *K++, $t2 in next round
137	//add	$h,$h,$t1			// h+=Sigma0(a)
138___
139$code.=<<___	if ($i>=15);
140	ror	$t0,$e,#$Sigma1[0]
141	add	$h,$h,$t2			// h+=K[i]
142	ror	$T1,@X[($j+1)&15],#$sigma0[0]
143	and	$t1,$f,$e
144	ror	$T2,@X[($j+14)&15],#$sigma1[0]
145	bic	$t2,$g,$e
146	ror	$T0,$a,#$Sigma0[0]
147	add	$h,$h,@X[$i&15]			// h+=X[i]
148	eor	$t0,$t0,$e,ror#$Sigma1[1]
149	eor	$T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
150	orr	$t1,$t1,$t2			// Ch(e,f,g)
151	eor	$t2,$a,$b			// a^b, b^c in next round
152	eor	$t0,$t0,$e,ror#$Sigma1[2]	// Sigma1(e)
153	eor	$T0,$T0,$a,ror#$Sigma0[1]
154	add	$h,$h,$t1			// h+=Ch(e,f,g)
155	and	$t3,$t3,$t2			// (b^c)&=(a^b)
156	eor	$T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
157	eor	$T1,$T1,@X[($j+1)&15],lsr#$sigma0[2]	// sigma0(X[i+1])
158	add	$h,$h,$t0			// h+=Sigma1(e)
159	eor	$t3,$t3,$b			// Maj(a,b,c)
160	eor	$t1,$T0,$a,ror#$Sigma0[2]	// Sigma0(a)
161	eor	$T2,$T2,@X[($j+14)&15],lsr#$sigma1[2]	// sigma1(X[i+14])
162	add	@X[$j],@X[$j],@X[($j+9)&15]
163	add	$d,$d,$h			// d+=h
164	add	$h,$h,$t3			// h+=Maj(a,b,c)
165	ldr	$t3,[$Ktbl],#$SZ		// *K++, $t2 in next round
166	add	@X[$j],@X[$j],$T1
167	add	$h,$h,$t1			// h+=Sigma0(a)
168	add	@X[$j],@X[$j],$T2
169___
170	($t2,$t3)=($t3,$t2);
171}
172
173$code.=<<___;
174#ifndef	__KERNEL__
175# include <openssl/arm_arch.h>
176#endif
177
178.text
179
180.extern	OPENSSL_armcap_P
181.globl	$func
182.type	$func,%function
183.align	6
184$func:
185___
186$code.=<<___	if ($SZ==4);
187#ifndef	__KERNEL__
188	adrp	x16,:pg_hi21:OPENSSL_armcap_P
189	add	x16,x16,:lo12:OPENSSL_armcap_P
190	ldr	w16,[x16]
191	tst	w16,#ARMV8_SHA256
192	b.ne	.Lv8_entry
193#endif
194___
195$code.=<<___;
196	stp	x29,x30,[sp,#-128]!
197	add	x29,sp,#0
198
199	stp	x19,x20,[sp,#16]
200	stp	x21,x22,[sp,#32]
201	stp	x23,x24,[sp,#48]
202	stp	x25,x26,[sp,#64]
203	stp	x27,x28,[sp,#80]
204	sub	sp,sp,#4*$SZ
205
206	ldp	$A,$B,[$ctx]				// load context
207	ldp	$C,$D,[$ctx,#2*$SZ]
208	ldp	$E,$F,[$ctx,#4*$SZ]
209	add	$num,$inp,$num,lsl#`log(16*$SZ)/log(2)`	// end of input
210	ldp	$G,$H,[$ctx,#6*$SZ]
211	adrp	$Ktbl,:pg_hi21:.LK$BITS
212	add	$Ktbl,$Ktbl,:lo12:.LK$BITS
213	stp	$ctx,$num,[x29,#96]
214
215.Loop:
216	ldp	@X[0],@X[1],[$inp],#2*$SZ
217	ldr	$t2,[$Ktbl],#$SZ			// *K++
218	eor	$t3,$B,$C				// magic seed
219	str	$inp,[x29,#112]
220___
221for ($i=0;$i<16;$i++)	{ &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
222$code.=".Loop_16_xx:\n";
223for (;$i<32;$i++)	{ &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
224$code.=<<___;
225	cbnz	$t2,.Loop_16_xx
226
227	ldp	$ctx,$num,[x29,#96]
228	ldr	$inp,[x29,#112]
229	sub	$Ktbl,$Ktbl,#`$SZ*($rounds+1)`		// rewind
230
231	ldp	@X[0],@X[1],[$ctx]
232	ldp	@X[2],@X[3],[$ctx,#2*$SZ]
233	add	$inp,$inp,#14*$SZ			// advance input pointer
234	ldp	@X[4],@X[5],[$ctx,#4*$SZ]
235	add	$A,$A,@X[0]
236	ldp	@X[6],@X[7],[$ctx,#6*$SZ]
237	add	$B,$B,@X[1]
238	add	$C,$C,@X[2]
239	add	$D,$D,@X[3]
240	stp	$A,$B,[$ctx]
241	add	$E,$E,@X[4]
242	add	$F,$F,@X[5]
243	stp	$C,$D,[$ctx,#2*$SZ]
244	add	$G,$G,@X[6]
245	add	$H,$H,@X[7]
246	cmp	$inp,$num
247	stp	$E,$F,[$ctx,#4*$SZ]
248	stp	$G,$H,[$ctx,#6*$SZ]
249	b.ne	.Loop
250
251	ldp	x19,x20,[x29,#16]
252	add	sp,sp,#4*$SZ
253	ldp	x21,x22,[x29,#32]
254	ldp	x23,x24,[x29,#48]
255	ldp	x25,x26,[x29,#64]
256	ldp	x27,x28,[x29,#80]
257	ldp	x29,x30,[sp],#128
258	ret
259.size	$func,.-$func
260
261.section .rodata
262.align	6
263.type	.LK$BITS,%object
264.LK$BITS:
265___
266$code.=<<___ if ($SZ==8);
267	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
268	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
269	.quad	0x3956c25bf348b538,0x59f111f1b605d019
270	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
271	.quad	0xd807aa98a3030242,0x12835b0145706fbe
272	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
273	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
274	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
275	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
276	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
277	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
278	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
279	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
280	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
281	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
282	.quad	0x06ca6351e003826f,0x142929670a0e6e70
283	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
284	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
285	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
286	.quad	0x81c2c92e47edaee6,0x92722c851482353b
287	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
288	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
289	.quad	0xd192e819d6ef5218,0xd69906245565a910
290	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
291	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
292	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
293	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
294	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
295	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
296	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
297	.quad	0x90befffa23631e28,0xa4506cebde82bde9
298	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
299	.quad	0xca273eceea26619c,0xd186b8c721c0c207
300	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
301	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
302	.quad	0x113f9804bef90dae,0x1b710b35131c471b
303	.quad	0x28db77f523047d84,0x32caab7b40c72493
304	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
305	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
306	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
307	.quad	0	// terminator
308___
309$code.=<<___ if ($SZ==4);
310	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
311	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
312	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
313	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
314	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
315	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
316	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
317	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
318	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
319	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
320	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
321	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
322	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
323	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
324	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
325	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
326	.long	0	//terminator
327___
328$code.=<<___;
329.size	.LK$BITS,.-.LK$BITS
330.asciz	"SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
331.align	2
332___
333
334if ($SZ==4) {
335my $Ktbl="x3";
336
337my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
338my @MSG=map("v$_.16b",(4..7));
339my ($W0,$W1)=("v16.4s","v17.4s");
340my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
341
342$code.=<<___;
343.text
344#ifndef	__KERNEL__
345.type	sha256_block_armv8,%function
346.align	6
347sha256_block_armv8:
348.Lv8_entry:
349	stp		x29,x30,[sp,#-16]!
350	add		x29,sp,#0
351
352	ld1.32		{$ABCD,$EFGH},[$ctx]
353	adrp		$Ktbl,:pg_hi21:.LK256
354	add		$Ktbl,$Ktbl,:lo12:.LK256
355
356.Loop_hw:
357	ld1		{@MSG[0]-@MSG[3]},[$inp],#64
358	sub		$num,$num,#1
359	ld1.32		{$W0},[$Ktbl],#16
360	rev32		@MSG[0],@MSG[0]
361	rev32		@MSG[1],@MSG[1]
362	rev32		@MSG[2],@MSG[2]
363	rev32		@MSG[3],@MSG[3]
364	orr		$ABCD_SAVE,$ABCD,$ABCD		// offload
365	orr		$EFGH_SAVE,$EFGH,$EFGH
366___
367for($i=0;$i<12;$i++) {
368$code.=<<___;
369	ld1.32		{$W1},[$Ktbl],#16
370	add.i32		$W0,$W0,@MSG[0]
371	sha256su0	@MSG[0],@MSG[1]
372	orr		$abcd,$ABCD,$ABCD
373	sha256h		$ABCD,$EFGH,$W0
374	sha256h2	$EFGH,$abcd,$W0
375	sha256su1	@MSG[0],@MSG[2],@MSG[3]
376___
377	($W0,$W1)=($W1,$W0);	push(@MSG,shift(@MSG));
378}
379$code.=<<___;
380	ld1.32		{$W1},[$Ktbl],#16
381	add.i32		$W0,$W0,@MSG[0]
382	orr		$abcd,$ABCD,$ABCD
383	sha256h		$ABCD,$EFGH,$W0
384	sha256h2	$EFGH,$abcd,$W0
385
386	ld1.32		{$W0},[$Ktbl],#16
387	add.i32		$W1,$W1,@MSG[1]
388	orr		$abcd,$ABCD,$ABCD
389	sha256h		$ABCD,$EFGH,$W1
390	sha256h2	$EFGH,$abcd,$W1
391
392	ld1.32		{$W1},[$Ktbl]
393	add.i32		$W0,$W0,@MSG[2]
394	sub		$Ktbl,$Ktbl,#$rounds*$SZ-16	// rewind
395	orr		$abcd,$ABCD,$ABCD
396	sha256h		$ABCD,$EFGH,$W0
397	sha256h2	$EFGH,$abcd,$W0
398
399	add.i32		$W1,$W1,@MSG[3]
400	orr		$abcd,$ABCD,$ABCD
401	sha256h		$ABCD,$EFGH,$W1
402	sha256h2	$EFGH,$abcd,$W1
403
404	add.i32		$ABCD,$ABCD,$ABCD_SAVE
405	add.i32		$EFGH,$EFGH,$EFGH_SAVE
406
407	cbnz		$num,.Loop_hw
408
409	st1.32		{$ABCD,$EFGH},[$ctx]
410
411	ldr		x29,[sp],#16
412	ret
413.size	sha256_block_armv8,.-sha256_block_armv8
414#endif
415___
416}
417
418$code.=<<___;
419#ifndef	__KERNEL__
420.comm	OPENSSL_armcap_P,4,4
421.hidden	OPENSSL_armcap_P
422#endif
423___
424
425{   my  %opcode = (
426	"sha256h"	=> 0x5e004000,	"sha256h2"	=> 0x5e005000,
427	"sha256su0"	=> 0x5e282800,	"sha256su1"	=> 0x5e006000	);
428
429    sub unsha256 {
430	my ($mnemonic,$arg)=@_;
431
432	$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
433	&&
434	sprintf ".inst\t0x%08x\t//%s %s",
435			$opcode{$mnemonic}|$1|($2<<5)|($3<<16),
436			$mnemonic,$arg;
437    }
438}
439
440open SELF,$0;
441while(<SELF>) {
442        next if (/^#!/);
443        last if (!s/^#/\/\// and !/^$/);
444        print;
445}
446close SELF;
447
448foreach(split("\n",$code)) {
449
450	s/\`([^\`]*)\`/eval($1)/geo;
451
452	s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/geo;
453
454	s/\.\w?32\b//o		and s/\.16b/\.4s/go;
455	m/(ld|st)1[^\[]+\[0\]/o	and s/\.4s/\.s/go;
456
457	print $_,"\n";
458}
459
460close STDOUT;
461