a10260ca5f
wolfssl/wolfcrypt/aes.h: add key_C_fallback[] to struct Aes, and remove comment that "AESNI needs key first, rounds 2nd, not sure why yet" now that AES_128_Key_Expansion_AESNI no longer writes rounds after the expanded key.
wolfcrypt/src/aes.c:
* add _AESNI or _aesni suffixes/infixes to AESNI implementations that were missing them: AES_CBC_encrypt(), AES_CBC_decrypt_by*(), AES_ECB_encrypt(), AES_*_Key_Expansion(), AES_set_encrypt_key(), AES_set_decrypt_key(), AES_GCM_encrypt(), AES_GCM_decrypt(), AES_XTS_encrypt(), and AES_XTS_decrypt().
* move key size check from to start of wc_AesSetKeyLocal().
* refactor pure-C AES setkey and cipher implementations to use aes->key_C_fallback when defined(WC_AES_C_DYNAMIC_FALLBACK).
* refactor wc_AesSetKeyLocal() to set up both AESNI and pure-C expanded keys when defined(WC_AES_C_DYNAMIC_FALLBACK).
* refactor all (haveAESNI && aes->use_aesni) conditions to just (aes->use_aesni).
* add macros VECTOR_REGISTERS_PUSH and VECTOR_REGISTERS_POP, which do nothing but push a brace level when !defined(WC_AES_C_DYNAMIC_FALLBACK), but when defined(WC_AES_C_DYNAMIC_FALLBACK), they call SAVE_VECTOR_REGISTERS2() and on failure, temporarily clear aes->use_aesni and restore at _POP().
* refactor all invocations of SAVE_VECTOR_REGISTERS() and RESTORE_VECTOR_REGISTERS() to VECTOR_REGISTERS_PUSH and VECTOR_REGISTERS_POP, except in wc_AesSetKeyLocal(), wc_AesXtsEncrypt(), and wc_AesXtsDecrypt(), which are refactored to use SAVE_VECTOR_REGISTERS2(), with graceful failure concealment if defined(WC_AES_C_DYNAMIC_FALLBACK).
* orthogonalize cleanup code in wc_AesCbcEncrypt(), wc_AesCcmEncrypt() and wc_AesCcmDecrypt().
* streamline fallthrough software definitions of wc_AesEncryptDirect() and wc_AesDecryptDirect(), and remove special-casing for defined(WOLFSSL_LINUXKM)&&defined(WOLFSSL_AESNI).
wolfcrypt/src/aes_asm.{S,asm}:
* remove errant "movl $10, 240(%rsi)" from AES_128_Key_Expansion_AESNI.
* add _AESNI suffixes/infixes to implementations that needed them.
wolfcrypt/src/{aes_gcm_asm.{S,asm},aes_xts_asm.S}: regenerate from revisions in scripts#357 -- adds _aesni suffixes to implementations that were missing them.
wolfssl/wolfcrypt/types.h: remove DEBUG_VECTOR_REGISTER_ACCESS macros, and add dummy fallthrough definitions for SAVE_VECTOR_REGISTERS2 and WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL.
wolfssl/wolfcrypt/memory.h: adopt DEBUG_VECTOR_REGISTER_ACCESS code from types.h, and add definitions for WC_DEBUG_VECTOR_REGISTERS_RETVAL_INITVAL and WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL.
linuxkm/linuxkm_wc_port.h: add arch-specific macro definitions for SAVE_VECTOR_REGISTERS2().
wolfcrypt/benchmark/benchmark.c: add missing gates around calls to RESTORE_VECTOR_REGISTERS().
configure.ac:
* cover various interdependencies in enable-all/enable-all-crypto, for better behavior in combination with --disable-aesgcm, --disable-ecc, --disable-ocsp, --disable-hmac, --disable-chacha, --disable-ed25519, and --disable-ed448.
* inhibit aesgcm_stream in enable-all/enable-all-crypto when ENABLED_LINUXKM_DEFAULTS, because it is currently incompatible with WC_AES_C_DYNAMIC_FALLBACK.
* add -DWC_AES_C_DYNAMIC_FALLBACK when ENABLED_LINUXKM_DEFAULTS.
* add 3 new interdependency checks: "ECCSI requires ECC.", "SAKKE requires ECC.", "WOLFSSH requires HMAC."
wolfcrypt/src/asn.c: tweak gating to accommodate defined(NO_RSA) && !defined(HAVE_ECC).
wolfcrypt/src/evp.c: tweak gating to accommodate defined(NO_HMAC).
wolfcrypt/src/logging.c: remove DEBUG_VECTOR_REGISTER_ACCESS code (moved to memory.c).
wolfcrypt/src/memory.c: change #include of settings.h to types.h; adopt DEBUG_VECTOR_REGISTER_ACCESS code from logging.c; add implementation of SAVE_VECTOR_REGISTERS2_fuzzer().
wolfcrypt/src/pwdbased.c: add explanatory #error scrypt requires HMAC.
wolfcrypt/test/test.c:
* add DEBUG_VECTOR_REGISTER_ACCESS clauses to aes_xts_128_test(), aesecb_test(), aesctr_test(), aes_test() CBC section, aes256_test() CBC section, and aesgcm_default_test_helper()
* remove duplicate wc_AesEcbDecrypt() in aesecb_test().
* add gating for pbkdf2_test().
* fix cleanup code in dsa_test().
* fix gating in pkcs7authenveloped_run_vectors() to accommodate !defined(HAVE_AESGCM).
* fix gating in cryptocb_test() to accommodate defined(NO_HMAC).
wolfssl/wolfcrypt/cryptocb.h: remove gates around "pk" sub-struct of struct wc_CryptoInfo -- wc_CryptoInfo.pk.type (an int) is used unconditionally when --enable-debug, and is used with DH.
wolfssl/wolfcrypt/error-crypt.h: fix whitespace.
1450 lines
45 KiB
ArmAsm
1450 lines
45 KiB
ArmAsm
/* aes_xts_asm.S */
|
|
/*
|
|
* Copyright (C) 2006-2023 wolfSSL Inc.
|
|
*
|
|
* This file is part of wolfSSL.
|
|
*
|
|
* wolfSSL is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* wolfSSL is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
|
|
*/
|
|
|
|
#ifdef WOLFSSL_USER_SETTINGS
|
|
#ifdef WOLFSSL_USER_SETTINGS_ASM
|
|
/*
|
|
* user_settings_asm.h is a file generated by the script user_settings_asm.sh.
|
|
* The script takes in a user_settings.h and produces user_settings_asm.h, which
|
|
* is a stripped down version of user_settings.h containing only preprocessor
|
|
* directives. This makes the header safe to include in assembly (.S) files.
|
|
*/
|
|
#include "user_settings_asm.h"
|
|
#else
|
|
/*
|
|
* Note: if user_settings.h contains any C code (e.g. a typedef or function
|
|
* prototype), including it here in an assembly (.S) file will cause an
|
|
* assembler failure. See user_settings_asm.h above.
|
|
*/
|
|
#include "user_settings.h"
|
|
#endif /* WOLFSSL_USER_SETTINGS_ASM */
|
|
#endif /* WOLFSSL_USER_SETTINGS */
|
|
|
|
#ifndef HAVE_INTEL_AVX1
|
|
#define HAVE_INTEL_AVX1
|
|
#endif /* HAVE_INTEL_AVX1 */
|
|
#ifndef NO_AVX2_SUPPORT
|
|
#define HAVE_INTEL_AVX2
|
|
#endif /* NO_AVX2_SUPPORT */
|
|
|
|
#ifdef WOLFSSL_AES_XTS
|
|
#ifdef WOLFSSL_X86_64_BUILD
|
|
#ifndef __APPLE__
|
|
.data
|
|
#else
|
|
.section __DATA,__data
|
|
#endif /* __APPLE__ */
|
|
L_aes_xts_gc_xts:
|
|
.long 0x87,0x1,0x1,0x1
|
|
#ifndef __APPLE__
|
|
.text
|
|
.globl AES_XTS_encrypt_aesni
|
|
.type AES_XTS_encrypt_aesni,@function
|
|
.align 16
|
|
AES_XTS_encrypt_aesni:
|
|
#else
|
|
.section __TEXT,__text
|
|
.globl _AES_XTS_encrypt_aesni
|
|
.p2align 4
|
|
_AES_XTS_encrypt_aesni:
|
|
#endif /* __APPLE__ */
|
|
pushq %r12
|
|
pushq %r13
|
|
movq %rdx, %rax
|
|
movq %rcx, %r12
|
|
movl 24(%rsp), %r10d
|
|
subq $0x40, %rsp
|
|
movdqu L_aes_xts_gc_xts(%rip), %xmm12
|
|
movdqu (%r12), %xmm0
|
|
# aes_enc_block
|
|
pxor (%r9), %xmm0
|
|
movdqu 16(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 32(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 48(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 64(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 80(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 96(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 112(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 128(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 144(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r9), %xmm5
|
|
jl L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 176(%r9), %xmm6
|
|
aesenc %xmm6, %xmm0
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r9), %xmm5
|
|
jl L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 208(%r9), %xmm6
|
|
aesenc %xmm6, %xmm0
|
|
movdqu 224(%r9), %xmm5
|
|
L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last:
|
|
aesenclast %xmm5, %xmm0
|
|
xorl %r13d, %r13d
|
|
cmpl $0x40, %eax
|
|
movl %eax, %r11d
|
|
jl L_AES_XTS_encrypt_aesni_done_64
|
|
andl $0xffffffc0, %r11d
|
|
L_AES_XTS_encrypt_aesni_enc_64:
|
|
# 64 bytes of input
|
|
# aes_enc_64
|
|
leaq (%rdi,%r13,1), %rcx
|
|
leaq (%rsi,%r13,1), %rdx
|
|
movdqu (%rcx), %xmm8
|
|
movdqu 16(%rcx), %xmm9
|
|
movdqu 32(%rcx), %xmm10
|
|
movdqu 48(%rcx), %xmm11
|
|
movdqa %xmm0, %xmm4
|
|
movdqa %xmm0, %xmm1
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm1
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm1
|
|
movdqa %xmm1, %xmm4
|
|
movdqa %xmm1, %xmm2
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm2
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm2
|
|
movdqa %xmm2, %xmm4
|
|
movdqa %xmm2, %xmm3
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm3
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm3
|
|
pxor %xmm0, %xmm8
|
|
pxor %xmm1, %xmm9
|
|
pxor %xmm2, %xmm10
|
|
pxor %xmm3, %xmm11
|
|
# aes_enc_block
|
|
movdqu (%r8), %xmm4
|
|
pxor %xmm4, %xmm8
|
|
pxor %xmm4, %xmm9
|
|
pxor %xmm4, %xmm10
|
|
pxor %xmm4, %xmm11
|
|
movdqu 16(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 32(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 48(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 64(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 80(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 96(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 112(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 128(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 144(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r8), %xmm4
|
|
jl L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 176(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r8), %xmm4
|
|
jl L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 208(%r8), %xmm4
|
|
aesenc %xmm4, %xmm8
|
|
aesenc %xmm4, %xmm9
|
|
aesenc %xmm4, %xmm10
|
|
aesenc %xmm4, %xmm11
|
|
movdqu 224(%r8), %xmm4
|
|
L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last:
|
|
aesenclast %xmm4, %xmm8
|
|
aesenclast %xmm4, %xmm9
|
|
aesenclast %xmm4, %xmm10
|
|
aesenclast %xmm4, %xmm11
|
|
pxor %xmm0, %xmm8
|
|
pxor %xmm1, %xmm9
|
|
pxor %xmm2, %xmm10
|
|
pxor %xmm3, %xmm11
|
|
movdqu %xmm8, (%rdx)
|
|
movdqu %xmm9, 16(%rdx)
|
|
movdqu %xmm10, 32(%rdx)
|
|
movdqu %xmm11, 48(%rdx)
|
|
movdqa %xmm3, %xmm4
|
|
movdqa %xmm3, %xmm0
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm0
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm0
|
|
addl $0x40, %r13d
|
|
cmpl %r11d, %r13d
|
|
jl L_AES_XTS_encrypt_aesni_enc_64
|
|
L_AES_XTS_encrypt_aesni_done_64:
|
|
cmpl %eax, %r13d
|
|
movl %eax, %r11d
|
|
je L_AES_XTS_encrypt_aesni_done_enc
|
|
subl %r13d, %r11d
|
|
cmpl $16, %r11d
|
|
movl %eax, %r11d
|
|
jl L_AES_XTS_encrypt_aesni_last_15
|
|
andl $0xfffffff0, %r11d
|
|
# 16 bytes of input
|
|
L_AES_XTS_encrypt_aesni_enc_16:
|
|
leaq (%rdi,%r13,1), %rcx
|
|
movdqu (%rcx), %xmm8
|
|
pxor %xmm0, %xmm8
|
|
# aes_enc_block
|
|
pxor (%r8), %xmm8
|
|
movdqu 16(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 32(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 48(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 64(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 80(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 96(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 112(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 128(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 144(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_encrypt_aesni_aes_enc_block_last
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 176(%r8), %xmm6
|
|
aesenc %xmm6, %xmm8
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_encrypt_aesni_aes_enc_block_last
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 208(%r8), %xmm6
|
|
aesenc %xmm6, %xmm8
|
|
movdqu 224(%r8), %xmm5
|
|
L_AES_XTS_encrypt_aesni_aes_enc_block_last:
|
|
aesenclast %xmm5, %xmm8
|
|
pxor %xmm0, %xmm8
|
|
leaq (%rsi,%r13,1), %rcx
|
|
movdqu %xmm8, (%rcx)
|
|
movdqa %xmm0, %xmm4
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm0
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm0
|
|
addl $16, %r13d
|
|
cmpl %r11d, %r13d
|
|
jl L_AES_XTS_encrypt_aesni_enc_16
|
|
cmpl %eax, %r13d
|
|
je L_AES_XTS_encrypt_aesni_done_enc
|
|
L_AES_XTS_encrypt_aesni_last_15:
|
|
subq $16, %r13
|
|
leaq (%rsi,%r13,1), %rcx
|
|
movdqu (%rcx), %xmm8
|
|
addq $16, %r13
|
|
movdqu %xmm8, (%rsp)
|
|
xorq %rdx, %rdx
|
|
L_AES_XTS_encrypt_aesni_last_15_byte_loop:
|
|
movb (%rsp,%rdx,1), %r11b
|
|
movb (%rdi,%r13,1), %cl
|
|
movb %r11b, (%rsi,%r13,1)
|
|
movb %cl, (%rsp,%rdx,1)
|
|
incl %r13d
|
|
incl %edx
|
|
cmpl %eax, %r13d
|
|
jl L_AES_XTS_encrypt_aesni_last_15_byte_loop
|
|
subq %rdx, %r13
|
|
movdqu (%rsp), %xmm8
|
|
subq $16, %r13
|
|
pxor %xmm0, %xmm8
|
|
# aes_enc_block
|
|
pxor (%r8), %xmm8
|
|
movdqu 16(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 32(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 48(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 64(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 80(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 96(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 112(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 128(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 144(%r8), %xmm5
|
|
aesenc %xmm5, %xmm8
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 176(%r8), %xmm6
|
|
aesenc %xmm6, %xmm8
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last
|
|
aesenc %xmm5, %xmm8
|
|
movdqu 208(%r8), %xmm6
|
|
aesenc %xmm6, %xmm8
|
|
movdqu 224(%r8), %xmm5
|
|
L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last:
|
|
aesenclast %xmm5, %xmm8
|
|
pxor %xmm0, %xmm8
|
|
leaq (%rsi,%r13,1), %rcx
|
|
movdqu %xmm8, (%rcx)
|
|
L_AES_XTS_encrypt_aesni_done_enc:
|
|
addq $0x40, %rsp
|
|
popq %r13
|
|
popq %r12
|
|
repz retq
|
|
#ifndef __APPLE__
|
|
.size AES_XTS_encrypt_aesni,.-AES_XTS_encrypt_aesni
|
|
#endif /* __APPLE__ */
|
|
#ifndef __APPLE__
|
|
.text
|
|
.globl AES_XTS_decrypt_aesni
|
|
.type AES_XTS_decrypt_aesni,@function
|
|
.align 16
|
|
AES_XTS_decrypt_aesni:
|
|
#else
|
|
.section __TEXT,__text
|
|
.globl _AES_XTS_decrypt_aesni
|
|
.p2align 4
|
|
_AES_XTS_decrypt_aesni:
|
|
#endif /* __APPLE__ */
|
|
pushq %r12
|
|
pushq %r13
|
|
movq %rdx, %rax
|
|
movq %rcx, %r12
|
|
movl 24(%rsp), %r10d
|
|
subq $16, %rsp
|
|
movdqu L_aes_xts_gc_xts(%rip), %xmm12
|
|
movdqu (%r12), %xmm0
|
|
# aes_enc_block
|
|
pxor (%r9), %xmm0
|
|
movdqu 16(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 32(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 48(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 64(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 80(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 96(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 112(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 128(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 144(%r9), %xmm5
|
|
aesenc %xmm5, %xmm0
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r9), %xmm5
|
|
jl L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 176(%r9), %xmm6
|
|
aesenc %xmm6, %xmm0
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r9), %xmm5
|
|
jl L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last
|
|
aesenc %xmm5, %xmm0
|
|
movdqu 208(%r9), %xmm6
|
|
aesenc %xmm6, %xmm0
|
|
movdqu 224(%r9), %xmm5
|
|
L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last:
|
|
aesenclast %xmm5, %xmm0
|
|
xorl %r13d, %r13d
|
|
movl %eax, %r11d
|
|
andl $0xfffffff0, %r11d
|
|
cmpl %eax, %r11d
|
|
je L_AES_XTS_decrypt_aesni_mul16_64
|
|
subl $16, %r11d
|
|
cmpl $16, %r11d
|
|
jl L_AES_XTS_decrypt_aesni_last_31_start
|
|
L_AES_XTS_decrypt_aesni_mul16_64:
|
|
cmpl $0x40, %r11d
|
|
jl L_AES_XTS_decrypt_aesni_done_64
|
|
andl $0xffffffc0, %r11d
|
|
L_AES_XTS_decrypt_aesni_dec_64:
|
|
# 64 bytes of input
|
|
# aes_dec_64
|
|
leaq (%rdi,%r13,1), %rcx
|
|
leaq (%rsi,%r13,1), %rdx
|
|
movdqu (%rcx), %xmm8
|
|
movdqu 16(%rcx), %xmm9
|
|
movdqu 32(%rcx), %xmm10
|
|
movdqu 48(%rcx), %xmm11
|
|
movdqa %xmm0, %xmm4
|
|
movdqa %xmm0, %xmm1
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm1
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm1
|
|
movdqa %xmm1, %xmm4
|
|
movdqa %xmm1, %xmm2
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm2
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm2
|
|
movdqa %xmm2, %xmm4
|
|
movdqa %xmm2, %xmm3
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm3
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm3
|
|
pxor %xmm0, %xmm8
|
|
pxor %xmm1, %xmm9
|
|
pxor %xmm2, %xmm10
|
|
pxor %xmm3, %xmm11
|
|
# aes_dec_block
|
|
movdqu (%r8), %xmm4
|
|
pxor %xmm4, %xmm8
|
|
pxor %xmm4, %xmm9
|
|
pxor %xmm4, %xmm10
|
|
pxor %xmm4, %xmm11
|
|
movdqu 16(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 32(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 48(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 64(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 80(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 96(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 112(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 128(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 144(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r8), %xmm4
|
|
jl L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 176(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r8), %xmm4
|
|
jl L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 208(%r8), %xmm4
|
|
aesdec %xmm4, %xmm8
|
|
aesdec %xmm4, %xmm9
|
|
aesdec %xmm4, %xmm10
|
|
aesdec %xmm4, %xmm11
|
|
movdqu 224(%r8), %xmm4
|
|
L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last:
|
|
aesdeclast %xmm4, %xmm8
|
|
aesdeclast %xmm4, %xmm9
|
|
aesdeclast %xmm4, %xmm10
|
|
aesdeclast %xmm4, %xmm11
|
|
pxor %xmm0, %xmm8
|
|
pxor %xmm1, %xmm9
|
|
pxor %xmm2, %xmm10
|
|
pxor %xmm3, %xmm11
|
|
movdqu %xmm8, (%rdx)
|
|
movdqu %xmm9, 16(%rdx)
|
|
movdqu %xmm10, 32(%rdx)
|
|
movdqu %xmm11, 48(%rdx)
|
|
movdqa %xmm3, %xmm4
|
|
movdqa %xmm3, %xmm0
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm0
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm0
|
|
addl $0x40, %r13d
|
|
cmpl %r11d, %r13d
|
|
jl L_AES_XTS_decrypt_aesni_dec_64
|
|
L_AES_XTS_decrypt_aesni_done_64:
|
|
cmpl %eax, %r13d
|
|
movl %eax, %r11d
|
|
je L_AES_XTS_decrypt_aesni_done_dec
|
|
andl $0xfffffff0, %r11d
|
|
cmpl %eax, %r11d
|
|
je L_AES_XTS_decrypt_aesni_mul16
|
|
subl $16, %r11d
|
|
subl %r13d, %r11d
|
|
cmpl $16, %r11d
|
|
jl L_AES_XTS_decrypt_aesni_last_31_start
|
|
addl %r13d, %r11d
|
|
L_AES_XTS_decrypt_aesni_mul16:
|
|
L_AES_XTS_decrypt_aesni_dec_16:
|
|
# 16 bytes of input
|
|
leaq (%rdi,%r13,1), %rcx
|
|
movdqu (%rcx), %xmm8
|
|
pxor %xmm0, %xmm8
|
|
# aes_dec_block
|
|
pxor (%r8), %xmm8
|
|
movdqu 16(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 32(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 48(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 64(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 80(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 96(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 112(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 128(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 144(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_aesni_aes_dec_block_last
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 176(%r8), %xmm6
|
|
aesdec %xmm6, %xmm8
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_aesni_aes_dec_block_last
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 208(%r8), %xmm6
|
|
aesdec %xmm6, %xmm8
|
|
movdqu 224(%r8), %xmm5
|
|
L_AES_XTS_decrypt_aesni_aes_dec_block_last:
|
|
aesdeclast %xmm5, %xmm8
|
|
pxor %xmm0, %xmm8
|
|
leaq (%rsi,%r13,1), %rcx
|
|
movdqu %xmm8, (%rcx)
|
|
movdqa %xmm0, %xmm4
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm0
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm0
|
|
addl $16, %r13d
|
|
cmpl %r11d, %r13d
|
|
jl L_AES_XTS_decrypt_aesni_dec_16
|
|
cmpl %eax, %r13d
|
|
je L_AES_XTS_decrypt_aesni_done_dec
|
|
L_AES_XTS_decrypt_aesni_last_31_start:
|
|
movdqa %xmm0, %xmm4
|
|
movdqa %xmm0, %xmm7
|
|
psrad $31, %xmm4
|
|
pslld $0x01, %xmm7
|
|
pshufd $0x93, %xmm4, %xmm4
|
|
pand %xmm12, %xmm4
|
|
pxor %xmm4, %xmm7
|
|
leaq (%rdi,%r13,1), %rcx
|
|
movdqu (%rcx), %xmm8
|
|
pxor %xmm7, %xmm8
|
|
# aes_dec_block
|
|
pxor (%r8), %xmm8
|
|
movdqu 16(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 32(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 48(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 64(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 80(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 96(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 112(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 128(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 144(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 176(%r8), %xmm6
|
|
aesdec %xmm6, %xmm8
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 208(%r8), %xmm6
|
|
aesdec %xmm6, %xmm8
|
|
movdqu 224(%r8), %xmm5
|
|
L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last:
|
|
aesdeclast %xmm5, %xmm8
|
|
pxor %xmm7, %xmm8
|
|
movdqu %xmm8, (%rsp)
|
|
addq $16, %r13
|
|
xorq %rdx, %rdx
|
|
L_AES_XTS_decrypt_aesni_last_31_byte_loop:
|
|
movb (%rsp,%rdx,1), %r11b
|
|
movb (%rdi,%r13,1), %cl
|
|
movb %r11b, (%rsi,%r13,1)
|
|
movb %cl, (%rsp,%rdx,1)
|
|
incl %r13d
|
|
incl %edx
|
|
cmpl %eax, %r13d
|
|
jl L_AES_XTS_decrypt_aesni_last_31_byte_loop
|
|
subq %rdx, %r13
|
|
movdqu (%rsp), %xmm8
|
|
pxor %xmm0, %xmm8
|
|
# aes_dec_block
|
|
pxor (%r8), %xmm8
|
|
movdqu 16(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 32(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 48(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 64(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 80(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 96(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 112(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 128(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 144(%r8), %xmm5
|
|
aesdec %xmm5, %xmm8
|
|
cmpl $11, %r10d
|
|
movdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 176(%r8), %xmm6
|
|
aesdec %xmm6, %xmm8
|
|
cmpl $13, %r10d
|
|
movdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last
|
|
aesdec %xmm5, %xmm8
|
|
movdqu 208(%r8), %xmm6
|
|
aesdec %xmm6, %xmm8
|
|
movdqu 224(%r8), %xmm5
|
|
L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last:
|
|
aesdeclast %xmm5, %xmm8
|
|
pxor %xmm0, %xmm8
|
|
subq $16, %r13
|
|
leaq (%rsi,%r13,1), %rcx
|
|
movdqu %xmm8, (%rcx)
|
|
L_AES_XTS_decrypt_aesni_done_dec:
|
|
addq $16, %rsp
|
|
popq %r13
|
|
popq %r12
|
|
repz retq
|
|
#ifndef __APPLE__
|
|
.size AES_XTS_decrypt_aesni,.-AES_XTS_decrypt_aesni
|
|
#endif /* __APPLE__ */
|
|
#ifdef HAVE_INTEL_AVX1
|
|
#ifndef __APPLE__
|
|
.data
|
|
#else
|
|
.section __DATA,__data
|
|
#endif /* __APPLE__ */
|
|
L_avx1_aes_xts_gc_xts:
|
|
.long 0x87,0x1,0x1,0x1
|
|
#ifndef __APPLE__
|
|
.text
|
|
.globl AES_XTS_encrypt_avx1
|
|
.type AES_XTS_encrypt_avx1,@function
|
|
.align 16
|
|
AES_XTS_encrypt_avx1:
|
|
#else
|
|
.section __TEXT,__text
|
|
.globl _AES_XTS_encrypt_avx1
|
|
.p2align 4
|
|
_AES_XTS_encrypt_avx1:
|
|
#endif /* __APPLE__ */
|
|
pushq %r12
|
|
pushq %r13
|
|
movq %rdx, %rax
|
|
movq %rcx, %r12
|
|
movl 24(%rsp), %r10d
|
|
subq $0x40, %rsp
|
|
vmovdqu L_avx1_aes_xts_gc_xts(%rip), %xmm12
|
|
vmovdqu (%r12), %xmm0
|
|
# aes_enc_block
|
|
vpxor (%r9), %xmm0, %xmm0
|
|
vmovdqu 16(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 32(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 48(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 64(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 80(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 96(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 112(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 128(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 144(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r9), %xmm5
|
|
jl L_AES_XTS_encrypt_avx1_tweak_aes_enc_block_last
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 176(%r9), %xmm6
|
|
vaesenc %xmm6, %xmm0, %xmm0
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r9), %xmm5
|
|
jl L_AES_XTS_encrypt_avx1_tweak_aes_enc_block_last
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 208(%r9), %xmm6
|
|
vaesenc %xmm6, %xmm0, %xmm0
|
|
vmovdqu 224(%r9), %xmm5
|
|
L_AES_XTS_encrypt_avx1_tweak_aes_enc_block_last:
|
|
vaesenclast %xmm5, %xmm0, %xmm0
|
|
xorl %r13d, %r13d
|
|
cmpl $0x40, %eax
|
|
movl %eax, %r11d
|
|
jl L_AES_XTS_encrypt_avx1_done_64
|
|
andl $0xffffffc0, %r11d
|
|
L_AES_XTS_encrypt_avx1_enc_64:
|
|
# 64 bytes of input
|
|
# aes_enc_64
|
|
leaq (%rdi,%r13,1), %rcx
|
|
leaq (%rsi,%r13,1), %rdx
|
|
vmovdqu (%rcx), %xmm8
|
|
vmovdqu 16(%rcx), %xmm9
|
|
vmovdqu 32(%rcx), %xmm10
|
|
vmovdqu 48(%rcx), %xmm11
|
|
vpsrad $31, %xmm0, %xmm4
|
|
vpslld $0x01, %xmm0, %xmm1
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm1, %xmm1
|
|
vpsrad $31, %xmm1, %xmm4
|
|
vpslld $0x01, %xmm1, %xmm2
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm2, %xmm2
|
|
vpsrad $31, %xmm2, %xmm4
|
|
vpslld $0x01, %xmm2, %xmm3
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm3, %xmm3
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
vpxor %xmm1, %xmm9, %xmm9
|
|
vpxor %xmm2, %xmm10, %xmm10
|
|
vpxor %xmm3, %xmm11, %xmm11
|
|
# aes_enc_block
|
|
vmovdqu (%r8), %xmm4
|
|
vpxor %xmm4, %xmm8, %xmm8
|
|
vpxor %xmm4, %xmm9, %xmm9
|
|
vpxor %xmm4, %xmm10, %xmm10
|
|
vpxor %xmm4, %xmm11, %xmm11
|
|
vmovdqu 16(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 32(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 48(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 64(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 80(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 96(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 112(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 128(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 144(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r8), %xmm4
|
|
jl L_AES_XTS_encrypt_avx1_aes_enc_64_aes_enc_block_last
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 176(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r8), %xmm4
|
|
jl L_AES_XTS_encrypt_avx1_aes_enc_64_aes_enc_block_last
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 208(%r8), %xmm4
|
|
vaesenc %xmm4, %xmm8, %xmm8
|
|
vaesenc %xmm4, %xmm9, %xmm9
|
|
vaesenc %xmm4, %xmm10, %xmm10
|
|
vaesenc %xmm4, %xmm11, %xmm11
|
|
vmovdqu 224(%r8), %xmm4
|
|
L_AES_XTS_encrypt_avx1_aes_enc_64_aes_enc_block_last:
|
|
vaesenclast %xmm4, %xmm8, %xmm8
|
|
vaesenclast %xmm4, %xmm9, %xmm9
|
|
vaesenclast %xmm4, %xmm10, %xmm10
|
|
vaesenclast %xmm4, %xmm11, %xmm11
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
vpxor %xmm1, %xmm9, %xmm9
|
|
vpxor %xmm2, %xmm10, %xmm10
|
|
vpxor %xmm3, %xmm11, %xmm11
|
|
vmovdqu %xmm8, (%rdx)
|
|
vmovdqu %xmm9, 16(%rdx)
|
|
vmovdqu %xmm10, 32(%rdx)
|
|
vmovdqu %xmm11, 48(%rdx)
|
|
vpsrad $31, %xmm3, %xmm4
|
|
vpslld $0x01, %xmm3, %xmm0
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm0, %xmm0
|
|
addl $0x40, %r13d
|
|
cmpl %r11d, %r13d
|
|
jl L_AES_XTS_encrypt_avx1_enc_64
|
|
L_AES_XTS_encrypt_avx1_done_64:
|
|
cmpl %eax, %r13d
|
|
movl %eax, %r11d
|
|
je L_AES_XTS_encrypt_avx1_done_enc
|
|
subl %r13d, %r11d
|
|
cmpl $16, %r11d
|
|
movl %eax, %r11d
|
|
jl L_AES_XTS_encrypt_avx1_last_15
|
|
andl $0xfffffff0, %r11d
|
|
# 16 bytes of input
|
|
L_AES_XTS_encrypt_avx1_enc_16:
|
|
leaq (%rdi,%r13,1), %rcx
|
|
vmovdqu (%rcx), %xmm8
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
# aes_enc_block
|
|
vpxor (%r8), %xmm8, %xmm8
|
|
vmovdqu 16(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 32(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 48(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 64(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 80(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 96(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 112(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 128(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 144(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_encrypt_avx1_aes_enc_block_last
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 176(%r8), %xmm6
|
|
vaesenc %xmm6, %xmm8, %xmm8
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_encrypt_avx1_aes_enc_block_last
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 208(%r8), %xmm6
|
|
vaesenc %xmm6, %xmm8, %xmm8
|
|
vmovdqu 224(%r8), %xmm5
|
|
L_AES_XTS_encrypt_avx1_aes_enc_block_last:
|
|
vaesenclast %xmm5, %xmm8, %xmm8
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
leaq (%rsi,%r13,1), %rcx
|
|
vmovdqu %xmm8, (%rcx)
|
|
vpsrad $31, %xmm0, %xmm4
|
|
vpslld $0x01, %xmm0, %xmm0
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm0, %xmm0
|
|
addl $16, %r13d
|
|
cmpl %r11d, %r13d
|
|
jl L_AES_XTS_encrypt_avx1_enc_16
|
|
cmpl %eax, %r13d
|
|
je L_AES_XTS_encrypt_avx1_done_enc
|
|
L_AES_XTS_encrypt_avx1_last_15:
|
|
subq $16, %r13
|
|
leaq (%rsi,%r13,1), %rcx
|
|
vmovdqu (%rcx), %xmm8
|
|
addq $16, %r13
|
|
vmovdqu %xmm8, (%rsp)
|
|
xorq %rdx, %rdx
|
|
L_AES_XTS_encrypt_avx1_last_15_byte_loop:
|
|
movb (%rsp,%rdx,1), %r11b
|
|
movb (%rdi,%r13,1), %cl
|
|
movb %r11b, (%rsi,%r13,1)
|
|
movb %cl, (%rsp,%rdx,1)
|
|
incl %r13d
|
|
incl %edx
|
|
cmpl %eax, %r13d
|
|
jl L_AES_XTS_encrypt_avx1_last_15_byte_loop
|
|
subq %rdx, %r13
|
|
vmovdqu (%rsp), %xmm8
|
|
subq $16, %r13
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
# aes_enc_block
|
|
vpxor (%r8), %xmm8, %xmm8
|
|
vmovdqu 16(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 32(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 48(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 64(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 80(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 96(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 112(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 128(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 144(%r8), %xmm5
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_encrypt_avx1_last_15_aes_enc_block_last
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 176(%r8), %xmm6
|
|
vaesenc %xmm6, %xmm8, %xmm8
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_encrypt_avx1_last_15_aes_enc_block_last
|
|
vaesenc %xmm5, %xmm8, %xmm8
|
|
vmovdqu 208(%r8), %xmm6
|
|
vaesenc %xmm6, %xmm8, %xmm8
|
|
vmovdqu 224(%r8), %xmm5
|
|
L_AES_XTS_encrypt_avx1_last_15_aes_enc_block_last:
|
|
vaesenclast %xmm5, %xmm8, %xmm8
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
leaq (%rsi,%r13,1), %rcx
|
|
vmovdqu %xmm8, (%rcx)
|
|
L_AES_XTS_encrypt_avx1_done_enc:
|
|
vzeroupper
|
|
addq $0x40, %rsp
|
|
popq %r13
|
|
popq %r12
|
|
repz retq
|
|
#ifndef __APPLE__
|
|
.size AES_XTS_encrypt_avx1,.-AES_XTS_encrypt_avx1
|
|
#endif /* __APPLE__ */
|
|
#ifndef __APPLE__
|
|
.text
|
|
.globl AES_XTS_decrypt_avx1
|
|
.type AES_XTS_decrypt_avx1,@function
|
|
.align 16
|
|
AES_XTS_decrypt_avx1:
|
|
#else
|
|
.section __TEXT,__text
|
|
.globl _AES_XTS_decrypt_avx1
|
|
.p2align 4
|
|
_AES_XTS_decrypt_avx1:
|
|
#endif /* __APPLE__ */
|
|
pushq %r12
|
|
pushq %r13
|
|
movq %rdx, %rax
|
|
movq %rcx, %r12
|
|
movl 24(%rsp), %r10d
|
|
subq $16, %rsp
|
|
vmovdqu L_avx1_aes_xts_gc_xts(%rip), %xmm12
|
|
vmovdqu (%r12), %xmm0
|
|
# aes_enc_block
|
|
vpxor (%r9), %xmm0, %xmm0
|
|
vmovdqu 16(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 32(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 48(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 64(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 80(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 96(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 112(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 128(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 144(%r9), %xmm5
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r9), %xmm5
|
|
jl L_AES_XTS_decrypt_avx1_tweak_aes_enc_block_last
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 176(%r9), %xmm6
|
|
vaesenc %xmm6, %xmm0, %xmm0
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r9), %xmm5
|
|
jl L_AES_XTS_decrypt_avx1_tweak_aes_enc_block_last
|
|
vaesenc %xmm5, %xmm0, %xmm0
|
|
vmovdqu 208(%r9), %xmm6
|
|
vaesenc %xmm6, %xmm0, %xmm0
|
|
vmovdqu 224(%r9), %xmm5
|
|
L_AES_XTS_decrypt_avx1_tweak_aes_enc_block_last:
|
|
vaesenclast %xmm5, %xmm0, %xmm0
|
|
xorl %r13d, %r13d
|
|
movl %eax, %r11d
|
|
andl $0xfffffff0, %r11d
|
|
cmpl %eax, %r11d
|
|
je L_AES_XTS_decrypt_avx1_mul16_64
|
|
subl $16, %r11d
|
|
cmpl $16, %r11d
|
|
jl L_AES_XTS_decrypt_avx1_last_31_start
|
|
L_AES_XTS_decrypt_avx1_mul16_64:
|
|
cmpl $0x40, %r11d
|
|
jl L_AES_XTS_decrypt_avx1_done_64
|
|
andl $0xffffffc0, %r11d
|
|
L_AES_XTS_decrypt_avx1_dec_64:
|
|
# 64 bytes of input
|
|
# aes_dec_64
|
|
leaq (%rdi,%r13,1), %rcx
|
|
leaq (%rsi,%r13,1), %rdx
|
|
vmovdqu (%rcx), %xmm8
|
|
vmovdqu 16(%rcx), %xmm9
|
|
vmovdqu 32(%rcx), %xmm10
|
|
vmovdqu 48(%rcx), %xmm11
|
|
vpsrad $31, %xmm0, %xmm4
|
|
vpslld $0x01, %xmm0, %xmm1
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm1, %xmm1
|
|
vpsrad $31, %xmm1, %xmm4
|
|
vpslld $0x01, %xmm1, %xmm2
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm2, %xmm2
|
|
vpsrad $31, %xmm2, %xmm4
|
|
vpslld $0x01, %xmm2, %xmm3
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm3, %xmm3
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
vpxor %xmm1, %xmm9, %xmm9
|
|
vpxor %xmm2, %xmm10, %xmm10
|
|
vpxor %xmm3, %xmm11, %xmm11
|
|
# aes_dec_block
|
|
vmovdqu (%r8), %xmm4
|
|
vpxor %xmm4, %xmm8, %xmm8
|
|
vpxor %xmm4, %xmm9, %xmm9
|
|
vpxor %xmm4, %xmm10, %xmm10
|
|
vpxor %xmm4, %xmm11, %xmm11
|
|
vmovdqu 16(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 32(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 48(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 64(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 80(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 96(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 112(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 128(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 144(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r8), %xmm4
|
|
jl L_AES_XTS_decrypt_avx1_aes_dec_64_aes_dec_block_last
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 176(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r8), %xmm4
|
|
jl L_AES_XTS_decrypt_avx1_aes_dec_64_aes_dec_block_last
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 208(%r8), %xmm4
|
|
vaesdec %xmm4, %xmm8, %xmm8
|
|
vaesdec %xmm4, %xmm9, %xmm9
|
|
vaesdec %xmm4, %xmm10, %xmm10
|
|
vaesdec %xmm4, %xmm11, %xmm11
|
|
vmovdqu 224(%r8), %xmm4
|
|
L_AES_XTS_decrypt_avx1_aes_dec_64_aes_dec_block_last:
|
|
vaesdeclast %xmm4, %xmm8, %xmm8
|
|
vaesdeclast %xmm4, %xmm9, %xmm9
|
|
vaesdeclast %xmm4, %xmm10, %xmm10
|
|
vaesdeclast %xmm4, %xmm11, %xmm11
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
vpxor %xmm1, %xmm9, %xmm9
|
|
vpxor %xmm2, %xmm10, %xmm10
|
|
vpxor %xmm3, %xmm11, %xmm11
|
|
vmovdqu %xmm8, (%rdx)
|
|
vmovdqu %xmm9, 16(%rdx)
|
|
vmovdqu %xmm10, 32(%rdx)
|
|
vmovdqu %xmm11, 48(%rdx)
|
|
vpsrad $31, %xmm3, %xmm4
|
|
vpslld $0x01, %xmm3, %xmm0
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm0, %xmm0
|
|
addl $0x40, %r13d
|
|
cmpl %r11d, %r13d
|
|
jl L_AES_XTS_decrypt_avx1_dec_64
|
|
L_AES_XTS_decrypt_avx1_done_64:
|
|
cmpl %eax, %r13d
|
|
movl %eax, %r11d
|
|
je L_AES_XTS_decrypt_avx1_done_dec
|
|
andl $0xfffffff0, %r11d
|
|
cmpl %eax, %r11d
|
|
je L_AES_XTS_decrypt_avx1_mul16
|
|
subl $16, %r11d
|
|
subl %r13d, %r11d
|
|
cmpl $16, %r11d
|
|
jl L_AES_XTS_decrypt_avx1_last_31_start
|
|
addl %r13d, %r11d
|
|
L_AES_XTS_decrypt_avx1_mul16:
|
|
L_AES_XTS_decrypt_avx1_dec_16:
|
|
# 16 bytes of input
|
|
leaq (%rdi,%r13,1), %rcx
|
|
vmovdqu (%rcx), %xmm8
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
# aes_dec_block
|
|
vpxor (%r8), %xmm8, %xmm8
|
|
vmovdqu 16(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 32(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 48(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 64(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 80(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 96(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 112(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 128(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 144(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_avx1_aes_dec_block_last
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 176(%r8), %xmm6
|
|
vaesdec %xmm6, %xmm8, %xmm8
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_avx1_aes_dec_block_last
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 208(%r8), %xmm6
|
|
vaesdec %xmm6, %xmm8, %xmm8
|
|
vmovdqu 224(%r8), %xmm5
|
|
L_AES_XTS_decrypt_avx1_aes_dec_block_last:
|
|
vaesdeclast %xmm5, %xmm8, %xmm8
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
leaq (%rsi,%r13,1), %rcx
|
|
vmovdqu %xmm8, (%rcx)
|
|
vpsrad $31, %xmm0, %xmm4
|
|
vpslld $0x01, %xmm0, %xmm0
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm0, %xmm0
|
|
addl $16, %r13d
|
|
cmpl %r11d, %r13d
|
|
jl L_AES_XTS_decrypt_avx1_dec_16
|
|
cmpl %eax, %r13d
|
|
je L_AES_XTS_decrypt_avx1_done_dec
|
|
L_AES_XTS_decrypt_avx1_last_31_start:
|
|
vpsrad $31, %xmm0, %xmm4
|
|
vpslld $0x01, %xmm0, %xmm7
|
|
vpshufd $0x93, %xmm4, %xmm4
|
|
vpand %xmm12, %xmm4, %xmm4
|
|
vpxor %xmm4, %xmm7, %xmm7
|
|
leaq (%rdi,%r13,1), %rcx
|
|
vmovdqu (%rcx), %xmm8
|
|
vpxor %xmm7, %xmm8, %xmm8
|
|
# aes_dec_block
|
|
vpxor (%r8), %xmm8, %xmm8
|
|
vmovdqu 16(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 32(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 48(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 64(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 80(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 96(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 112(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 128(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 144(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_avx1_last_31_aes_dec_block_last
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 176(%r8), %xmm6
|
|
vaesdec %xmm6, %xmm8, %xmm8
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_avx1_last_31_aes_dec_block_last
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 208(%r8), %xmm6
|
|
vaesdec %xmm6, %xmm8, %xmm8
|
|
vmovdqu 224(%r8), %xmm5
|
|
L_AES_XTS_decrypt_avx1_last_31_aes_dec_block_last:
|
|
vaesdeclast %xmm5, %xmm8, %xmm8
|
|
vpxor %xmm7, %xmm8, %xmm8
|
|
vmovdqu %xmm8, (%rsp)
|
|
addq $16, %r13
|
|
xorq %rdx, %rdx
|
|
L_AES_XTS_decrypt_avx1_last_31_byte_loop:
|
|
movb (%rsp,%rdx,1), %r11b
|
|
movb (%rdi,%r13,1), %cl
|
|
movb %r11b, (%rsi,%r13,1)
|
|
movb %cl, (%rsp,%rdx,1)
|
|
incl %r13d
|
|
incl %edx
|
|
cmpl %eax, %r13d
|
|
jl L_AES_XTS_decrypt_avx1_last_31_byte_loop
|
|
subq %rdx, %r13
|
|
vmovdqu (%rsp), %xmm8
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
# aes_dec_block
|
|
vpxor (%r8), %xmm8, %xmm8
|
|
vmovdqu 16(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 32(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 48(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 64(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 80(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 96(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 112(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 128(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 144(%r8), %xmm5
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
cmpl $11, %r10d
|
|
vmovdqu 160(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_avx1_last_31_2_aes_dec_block_last
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 176(%r8), %xmm6
|
|
vaesdec %xmm6, %xmm8, %xmm8
|
|
cmpl $13, %r10d
|
|
vmovdqu 192(%r8), %xmm5
|
|
jl L_AES_XTS_decrypt_avx1_last_31_2_aes_dec_block_last
|
|
vaesdec %xmm5, %xmm8, %xmm8
|
|
vmovdqu 208(%r8), %xmm6
|
|
vaesdec %xmm6, %xmm8, %xmm8
|
|
vmovdqu 224(%r8), %xmm5
|
|
L_AES_XTS_decrypt_avx1_last_31_2_aes_dec_block_last:
|
|
vaesdeclast %xmm5, %xmm8, %xmm8
|
|
vpxor %xmm0, %xmm8, %xmm8
|
|
subq $16, %r13
|
|
leaq (%rsi,%r13,1), %rcx
|
|
vmovdqu %xmm8, (%rcx)
|
|
L_AES_XTS_decrypt_avx1_done_dec:
|
|
vzeroupper
|
|
addq $16, %rsp
|
|
popq %r13
|
|
popq %r12
|
|
repz retq
|
|
#ifndef __APPLE__
|
|
.size AES_XTS_decrypt_avx1,.-AES_XTS_decrypt_avx1
|
|
#endif /* __APPLE__ */
|
|
#endif /* HAVE_INTEL_AVX1 */
|
|
#endif /* WOLFSSL_X86_64_BUILD */
|
|
#endif /* WOLFSSL_AES_XTS */
|
|
|
|
#if defined(__linux__) && defined(__ELF__)
|
|
.section .note.GNU-stack,"",%progbits
|
|
#endif
|