Add AES-NI GCM stitch.
This commit is contained in:
Andy Polyakov
parent
b4a9d5bfe8
commit
4e049c5259
@ -128,7 +128,7 @@ my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o:des-586.o crypt5
|
||||
|
||||
my $x86_elf_asm="$x86_asm:elf";
|
||||
|
||||
my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o:e_padlock-x86_64.o";
|
||||
my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o aesni-gcm-x86_64.o:e_padlock-x86_64.o";
|
||||
my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o::void";
|
||||
my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o vis3-mont.o sparct4-mont.o sparcv9-gf2m.o:des_enc-sparc.o fcrypt_b.o:aes_core.o aes_cbc.o aes-sparcv9.o aest4-sparcv9.o::md5-sparcv9.o:sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o::::::camellia.o cmll_misc.o cmll_cbc.o cmllt4-sparcv9.o:ghash-sparcv9.o::void";
|
||||
my $sparcv8_asm=":sparcv8.o:des_enc-sparc.o fcrypt_b.o:::::::::::::void";
|
||||
|
||||
32
TABLE
32
TABLE
@ -318,7 +318,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -813,7 +813,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = auto
|
||||
$dso_scheme = win32
|
||||
@ -1506,7 +1506,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = macosx
|
||||
$dso_scheme = dlfcn
|
||||
@ -1671,7 +1671,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = auto
|
||||
$dso_scheme = win32
|
||||
@ -1770,7 +1770,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = macosx
|
||||
$dso_scheme = dlfcn
|
||||
@ -1836,7 +1836,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -2034,7 +2034,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -2562,7 +2562,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -2760,7 +2760,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -2826,7 +2826,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -4476,7 +4476,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -4509,7 +4509,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -4542,7 +4542,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -4740,7 +4740,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = mingw64
|
||||
$dso_scheme = win32
|
||||
@ -5730,7 +5730,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
@ -5763,7 +5763,7 @@ $rmd160_obj =
|
||||
$rc5_obj =
|
||||
$wp_obj = wp-x86_64.o
|
||||
$cmll_obj = cmll-x86_64.o cmll_misc.o
|
||||
$modes_obj = ghash-x86_64.o
|
||||
$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o
|
||||
$engines_obj = e_padlock-x86_64.o
|
||||
$perlasm_scheme = elf
|
||||
$dso_scheme = dlfcn
|
||||
|
||||
@ -227,6 +227,26 @@ void aesni_ccm64_decrypt_blocks (const unsigned char *in,
|
||||
const unsigned char ivec[16],
|
||||
unsigned char cmac[16]);
|
||||
|
||||
#if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
|
||||
size_t aesni_gcm_encrypt(const unsigned char *in,
|
||||
unsigned char *out,
|
||||
size_t len,
|
||||
const void *key,
|
||||
unsigned char ivec[16],
|
||||
u64 *Xi);
|
||||
#define AES_gcm_encrypt aesni_gcm_encrypt
|
||||
size_t aesni_gcm_decrypt(const unsigned char *in,
|
||||
unsigned char *out,
|
||||
size_t len,
|
||||
const void *key,
|
||||
unsigned char ivec[16],
|
||||
u64 *Xi);
|
||||
#define AES_gcm_decrypt aesni_gcm_decrypt
|
||||
void gcm_ghash_avx(u64 Xi[2],const u128 Htable[16],const u8 *in,size_t len);
|
||||
#define AES_GCM_ASM(gctx) (gctx->ctr==aesni_ctr32_encrypt_blocks && \
|
||||
gctx->gcm.ghash==gcm_ghash_avx)
|
||||
#endif
|
||||
|
||||
static int aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
|
||||
const unsigned char *iv, int enc)
|
||||
{
|
||||
@ -1297,8 +1317,24 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
|
||||
/* Encrypt payload */
|
||||
if (gctx->ctr)
|
||||
{
|
||||
size_t bulk=0;
|
||||
#if defined(AES_GCM_ASM)
|
||||
if (len>=32 && AES_GCM_ASM(gctx))
|
||||
{
|
||||
if (CRYPTO_gcm128_encrypt(&gctx->gcm,NULL,NULL,0))
|
||||
return -1;
|
||||
|
||||
bulk = AES_gcm_encrypt(in,out,len,
|
||||
gctx->gcm.key,
|
||||
gctx->gcm.Yi.c,
|
||||
gctx->gcm.Xi.u);
|
||||
gctx->gcm.len.u[1] += bulk;
|
||||
}
|
||||
#endif
|
||||
if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
|
||||
in, out, len,
|
||||
in +bulk,
|
||||
out+bulk,
|
||||
len-bulk,
|
||||
gctx->ctr))
|
||||
goto err;
|
||||
}
|
||||
@ -1316,8 +1352,24 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
|
||||
/* Decrypt */
|
||||
if (gctx->ctr)
|
||||
{
|
||||
size_t bulk=0;
|
||||
#if defined(AES_GCM_ASM)
|
||||
if (len>=16 && AES_GCM_ASM(gctx))
|
||||
{
|
||||
if (CRYPTO_gcm128_decrypt(&gctx->gcm,NULL,NULL,0))
|
||||
return -1;
|
||||
|
||||
bulk = AES_gcm_decrypt(in,out,len,
|
||||
gctx->gcm.key,
|
||||
gctx->gcm.Yi.c,
|
||||
gctx->gcm.Xi.u);
|
||||
gctx->gcm.len.u[1] += bulk;
|
||||
}
|
||||
#endif
|
||||
if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
|
||||
in, out, len,
|
||||
in +bulk,
|
||||
out+bulk,
|
||||
len-bulk,
|
||||
gctx->ctr))
|
||||
goto err;
|
||||
}
|
||||
@ -1367,8 +1419,28 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
|
||||
{
|
||||
if (gctx->ctr)
|
||||
{
|
||||
size_t bulk=0;
|
||||
#if defined(AES_GCM_ASM)
|
||||
if (len>=32 && AES_GCM_ASM(gctx))
|
||||
{
|
||||
size_t res = (16-gctx->gcm.mres)%16;
|
||||
|
||||
if (CRYPTO_gcm128_encrypt(&gctx->gcm,
|
||||
in,out,res))
|
||||
return -1;
|
||||
|
||||
bulk = AES_gcm_encrypt(in+res,
|
||||
out+res,len-res, gctx->gcm.key,
|
||||
gctx->gcm.Yi.c,
|
||||
gctx->gcm.Xi.u);
|
||||
gctx->gcm.len.u[1] += bulk;
|
||||
bulk += res;
|
||||
}
|
||||
#endif
|
||||
if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
|
||||
in, out, len,
|
||||
in +bulk,
|
||||
out+bulk,
|
||||
len-bulk,
|
||||
gctx->ctr))
|
||||
return -1;
|
||||
}
|
||||
@ -1381,8 +1453,29 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
|
||||
{
|
||||
if (gctx->ctr)
|
||||
{
|
||||
size_t bulk=0;
|
||||
#if defined(AES_GCM_ASM)
|
||||
if (len>=16 && AES_GCM_ASM(gctx))
|
||||
{
|
||||
size_t res = (16-gctx->gcm.mres)%16;
|
||||
|
||||
if (CRYPTO_gcm128_decrypt(&gctx->gcm,
|
||||
in,out,res))
|
||||
return -1;
|
||||
|
||||
bulk = AES_gcm_decrypt(in+res,
|
||||
out+res,len-res,
|
||||
gctx->gcm.key,
|
||||
gctx->gcm.Yi.c,
|
||||
gctx->gcm.Xi.u);
|
||||
gctx->gcm.len.u[1] += bulk;
|
||||
bulk += res;
|
||||
}
|
||||
#endif
|
||||
if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
|
||||
in, out, len,
|
||||
in +bulk,
|
||||
out+bulk,
|
||||
len-bulk,
|
||||
gctx->ctr))
|
||||
return -1;
|
||||
}
|
||||
|
||||
@ -367,5 +367,7 @@ aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:0000000000
|
||||
aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:0388dace60b6a392f328c2b971b2fe78f795aaab494b5923f7fd89ff948bc1e0200211214e7394da2089b6acd093abe0c94da219118e297d7b7ebcbcc9c388f28ade7d85a8ee35616f7124a9d527029195b84d1b96c690ff2f2de30bf2ec89e00253786e126504f0dab90c48a30321de3345e6b0461e7c9e6c6b7afedde83f40::cac45f60e31efd3b5a43b98a22ce1aa1
|
||||
# 192 bytes plaintext, iv is chosen so that initial counter LSB is 0xFF
|
||||
aes-128-gcm:00000000000000000000000000000000:ffffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:56b3373ca9ef6e4a2b64fe1e9a17b61425f10d47a75a5fce13efc6bc784af24f4141bdd48cf7c770887afd573cca5418a9aeffcd7c5ceddfc6a78397b9a85b499da558257267caab2ad0b23ca476a53cb17fb41c4b8b475cb4f3f7165094c229c9e8c4dc0a2a5ff1903e501511221376a1cdb8364c5061a20cae74bc4acd76ceb0abc9fd3217ef9f8c90be402ddf6d8697f4f880dff15bfb7a6b28241ec8fe183c2d59e3f9dfff653c7126f0acb9e64211f42bae12af462b1070bef1ab5e3606::566f8ef683078bfdeeffa869d751a017
|
||||
# 288 bytes plaintext, iv is chosen so that initial counter LSB is 0xFF
|
||||
aes-128-gcm:00000000000000000000000000000000:ffffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:56b3373ca9ef6e4a2b64fe1e9a17b61425f10d47a75a5fce13efc6bc784af24f4141bdd48cf7c770887afd573cca5418a9aeffcd7c5ceddfc6a78397b9a85b499da558257267caab2ad0b23ca476a53cb17fb41c4b8b475cb4f3f7165094c229c9e8c4dc0a2a5ff1903e501511221376a1cdb8364c5061a20cae74bc4acd76ceb0abc9fd3217ef9f8c90be402ddf6d8697f4f880dff15bfb7a6b28241ec8fe183c2d59e3f9dfff653c7126f0acb9e64211f42bae12af462b1070bef1ab5e3606872ca10dee15b3249b1a1b958f23134c4bccb7d03200bce420a2f8eb66dcf3644d1423c1b5699003c13ecef4bf38a3b60eedc34033bac1902783dc6d89e2e774188a439c7ebcc0672dbda4ddcfb2794613b0be41315ef778708a70ee7d75165c::8b307f6b33286d0ab026a9ed3fe1e85f
|
||||
# 80 bytes plaintext, submitted by Intel
|
||||
aes-128-gcm:843ffcf5d2b72694d19ed01d01249412:dbcca32ebf9b804617c3aa9e:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f:6268c6fa2a80b2d137467f092f657ac04d89be2beaa623d61b5a868c8f03ff95d3dcee23ad2f1ab3a6c80eaf4b140eb05de3457f0fbc111a6b43d0763aa422a3013cf1dc37fe417d1fbfc449b75d4cc5:00000000000000000000000000000000101112131415161718191a1b1c1d1e1f:3b629ccfbc1119b7319e1dce2cd6fd6d
|
||||
|
||||
@ -50,6 +50,8 @@ ghash-x86.s: asm/ghash-x86.pl
|
||||
$(PERL) asm/ghash-x86.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@
|
||||
ghash-x86_64.s: asm/ghash-x86_64.pl
|
||||
$(PERL) asm/ghash-x86_64.pl $(PERLASM_SCHEME) > $@
|
||||
aesni-gcm-x86_64.s: asm/aesni-gcm-x86_64.pl
|
||||
$(PERL) asm/aesni-gcm-x86_64.pl $(PERLASM_SCHEME) > $@
|
||||
ghash-sparcv9.s: asm/ghash-sparcv9.pl
|
||||
$(PERL) asm/ghash-sparcv9.pl $@ $(CFLAGS)
|
||||
ghash-alpha.s: asm/ghash-alpha.pl
|
||||
|
||||
1028
crypto/modes/asm/aesni-gcm-x86_64.pl
Normal file
1028
crypto/modes/asm/aesni-gcm-x86_64.pl
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user