0b47811c46
Changes made to test.h to allow interop of PSK with OpenSSL. Changes to allow server to pre-generate key share and perform other operations at later time. Fix ChaCha20 code header to have bigger state to support assembly code for AVX1. Fix Curve25519 code to use define instead. Change Curve25519 to memset all object data on init. Change Poly1305 to put both sizes into one buffer to avoid a second call to wc_Poly1305Update(). Added WOLFSSL_START and WOLFSSL_END API and calls to show time of protocol message function enter and leave to analyse performance differences. Moved Curve25519 code in KeyShare extension out of general ECC code.
1516 lines
68 KiB
C
1516 lines
68 KiB
C
/* chacha.c
|
|
*
|
|
* Copyright (C) 2006-2017 wolfSSL Inc.
|
|
*
|
|
* This file is part of wolfSSL.
|
|
*
|
|
* wolfSSL is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* wolfSSL is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
|
|
*
|
|
* based from
|
|
* chacha-ref.c version 20080118
|
|
* D. J. Bernstein
|
|
* Public domain.
|
|
*/
|
|
|
|
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
#include <config.h>
|
|
#endif
|
|
|
|
#include <wolfssl/wolfcrypt/settings.h>
|
|
|
|
#ifdef HAVE_CHACHA
|
|
|
|
#include <wolfssl/wolfcrypt/chacha.h>
|
|
#include <wolfssl/wolfcrypt/error-crypt.h>
|
|
#include <wolfssl/wolfcrypt/logging.h>
|
|
#include <wolfssl/wolfcrypt/cpuid.h>
|
|
#ifdef NO_INLINE
|
|
#include <wolfssl/wolfcrypt/misc.h>
|
|
#else
|
|
#define WOLFSSL_MISC_INCLUDED
|
|
#include <wolfcrypt/src/misc.c>
|
|
#endif
|
|
|
|
#ifdef CHACHA_AEAD_TEST
|
|
#include <stdio.h>
|
|
#endif
|
|
|
|
#ifdef USE_INTEL_CHACHA_SPEEDUP
|
|
#include <emmintrin.h>
|
|
#include <immintrin.h>
|
|
|
|
#if defined(__GNUC__) && ((__GNUC__ < 4) || \
|
|
(__GNUC__ == 4 && __GNUC_MINOR__ <= 8))
|
|
#define NO_AVX2_SUPPORT
|
|
#endif
|
|
#if defined(__clang__) && ((__clang_major__ < 3) || \
|
|
(__clang_major__ == 3 && __clang_minor__ <= 5))
|
|
#define NO_AVX2_SUPPORT
|
|
#elif defined(__clang__) && defined(NO_AVX2_SUPPORT)
|
|
#undef NO_AVX2_SUPPORT
|
|
#endif
|
|
|
|
#ifndef NO_AVX2_SUPPORT
|
|
#define HAVE_INTEL_AVX2
|
|
#endif
|
|
|
|
#if defined(_MSC_VER)
|
|
#define CHACHA20_NOINLINE __declspec(noinline)
|
|
#elif defined(__GNUC__)
|
|
#define CHACHA20_NOINLINE __attribute__((noinline))
|
|
#else
|
|
#define CHACHA20_NOINLINE
|
|
#endif
|
|
|
|
static int cpuidFlagsSet = 0;
|
|
static int cpuidFlags = 0;
|
|
#endif
|
|
|
|
#ifdef BIG_ENDIAN_ORDER
|
|
#define LITTLE32(x) ByteReverseWord32(x)
|
|
#else
|
|
#define LITTLE32(x) (x)
|
|
#endif
|
|
|
|
/* Number of rounds */
|
|
#define ROUNDS 20
|
|
|
|
#define U32C(v) (v##U)
|
|
#define U32V(v) ((word32)(v) & U32C(0xFFFFFFFF))
|
|
#define U8TO32_LITTLE(p) LITTLE32(((word32*)(p))[0])
|
|
|
|
#define ROTATE(v,c) rotlFixed(v, c)
|
|
#define XOR(v,w) ((v) ^ (w))
|
|
#define PLUS(v,w) (U32V((v) + (w)))
|
|
#define PLUSONE(v) (PLUS((v),1))
|
|
|
|
#define QUARTERROUND(a,b,c,d) \
|
|
x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]),16); \
|
|
x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]),12); \
|
|
x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]), 8); \
|
|
x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]), 7);
|
|
|
|
|
|
|
|
#define QUARTERROUND_INTEL_ASM(a0,b0,c0,d0, \
|
|
a1,b1,c1,d1, \
|
|
a2,b2,c2,d2, \
|
|
a3,b3,c3,d3, \
|
|
t1,o1) \
|
|
"vpaddd "#b0", "#a0", "#a0"\n\t" \
|
|
"vpxor "#a0", "#d0", "#d0"\n\t" \
|
|
"vmovdqa "#o1"(%[x]), "#c3"\n\t" \
|
|
"vpshufb %[rotl16], "#d0", "#d0"\n\t" \
|
|
"vpaddd "#d0", "#c0", "#c0"\n\t" \
|
|
"vpxor "#c0", "#b0", "#b0"\n\t" \
|
|
"vpaddd "#b1", "#a1", "#a1"\n\t" \
|
|
"vpxor "#a1", "#d1", "#d1"\n\t" \
|
|
"vpshufb %[rotl16], "#d1", "#d1"\n\t" \
|
|
"vpaddd "#d1", "#c1", "#c1"\n\t" \
|
|
"vpxor "#c1", "#b1", "#b1"\n\t" \
|
|
"vpaddd "#b2", "#a2", "#a2"\n\t" \
|
|
"vpxor "#a2", "#d2", "#d2"\n\t" \
|
|
"vpshufb %[rotl16], "#d2", "#d2"\n\t" \
|
|
"vpaddd "#d2", "#c2", "#c2"\n\t" \
|
|
"vpxor "#c2", "#b2", "#b2"\n\t" \
|
|
"vpaddd "#b3", "#a3", "#a3"\n\t" \
|
|
"vpxor "#a3", "#d3", "#d3"\n\t" \
|
|
"vpshufb %[rotl16], "#d3", "#d3"\n\t" \
|
|
"vpaddd "#d3", "#c3", "#c3"\n\t" \
|
|
"vpxor "#c3", "#b3", "#b3"\n\t" \
|
|
"vmovdqa "#c3", "#o1"(%[x])\n\t" \
|
|
"vpsrld $20, "#b0", "#t1"\n\t" \
|
|
"vpslld $12, "#b0", "#b0"\n\t" \
|
|
"vpxor "#t1", "#b0", "#b0"\n\t" \
|
|
"vpsrld $20, "#b1", "#t1"\n\t" \
|
|
"vpslld $12, "#b1", "#b1"\n\t" \
|
|
"vpxor "#t1", "#b1", "#b1"\n\t" \
|
|
"vpsrld $20, "#b2", "#t1"\n\t" \
|
|
"vpslld $12, "#b2", "#b2"\n\t" \
|
|
"vpxor "#t1", "#b2", "#b2"\n\t" \
|
|
"vpsrld $20, "#b3", "#t1"\n\t" \
|
|
"vpslld $12, "#b3", "#b3"\n\t" \
|
|
"vpxor "#t1", "#b3", "#b3"\n\t" \
|
|
"vpaddd "#b0", "#a0", "#a0"\n\t" \
|
|
"vpxor "#a0", "#d0", "#d0"\n\t" \
|
|
"vmovdqa "#o1"(%[x]), "#c3"\n\t" \
|
|
"vpshufb %[rotl8], "#d0", "#d0"\n\t" \
|
|
"vpaddd "#d0", "#c0", "#c0"\n\t" \
|
|
"vpxor "#c0", "#b0", "#b0"\n\t" \
|
|
"vpaddd "#b1", "#a1", "#a1"\n\t" \
|
|
"vpxor "#a1", "#d1", "#d1"\n\t" \
|
|
"vpshufb %[rotl8], "#d1", "#d1"\n\t" \
|
|
"vpaddd "#d1", "#c1", "#c1"\n\t" \
|
|
"vpxor "#c1", "#b1", "#b1"\n\t" \
|
|
"vpaddd "#b2", "#a2", "#a2"\n\t" \
|
|
"vpxor "#a2", "#d2", "#d2"\n\t" \
|
|
"vpshufb %[rotl8], "#d2", "#d2"\n\t" \
|
|
"vpaddd "#d2", "#c2", "#c2"\n\t" \
|
|
"vpxor "#c2", "#b2", "#b2"\n\t" \
|
|
"vpaddd "#b3", "#a3", "#a3"\n\t" \
|
|
"vpxor "#a3", "#d3", "#d3"\n\t" \
|
|
"vpshufb %[rotl8], "#d3", "#d3"\n\t" \
|
|
"vpaddd "#d3", "#c3", "#c3"\n\t" \
|
|
"vpxor "#c3", "#b3", "#b3"\n\t" \
|
|
"vmovdqa "#c3", "#o1"(%[x])\n\t" \
|
|
"vpsrld $25, "#b0", "#t1"\n\t" \
|
|
"vpslld $7, "#b0", "#b0"\n\t" \
|
|
"vpxor "#t1", "#b0", "#b0"\n\t" \
|
|
"vpsrld $25, "#b1", "#t1"\n\t" \
|
|
"vpslld $7, "#b1", "#b1"\n\t" \
|
|
"vpxor "#t1", "#b1", "#b1"\n\t" \
|
|
"vpsrld $25, "#b2", "#t1"\n\t" \
|
|
"vpslld $7, "#b2", "#b2"\n\t" \
|
|
"vpxor "#t1", "#b2", "#b2"\n\t" \
|
|
"vpsrld $25, "#b3", "#t1"\n\t" \
|
|
"vpslld $7, "#b3", "#b3"\n\t" \
|
|
"vpxor "#t1", "#b3", "#b3"\n\t"
|
|
|
|
#define QUARTERROUND_INTEL_ASM_2(a0,b0,c0,d0, \
|
|
a1,b1,c1,d1, \
|
|
a2,b2,c2,d2, \
|
|
a3,b3,c3,d3, \
|
|
t1,o1) \
|
|
"vpaddd "#b0", "#a0", "#a0"\n\t" \
|
|
"vpxor "#a0", "#d0", "#d0"\n\t" \
|
|
"vmovdqa "#o1"(%[x]), "#c1"\n\t" \
|
|
"vpshufb %[rotl16], "#d0", "#d0"\n\t" \
|
|
"vpaddd "#d0", "#c0", "#c0"\n\t" \
|
|
"vpxor "#c0", "#b0", "#b0"\n\t" \
|
|
"vpaddd "#b1", "#a1", "#a1"\n\t" \
|
|
"vpxor "#a1", "#d1", "#d1"\n\t" \
|
|
"vpshufb %[rotl16], "#d1", "#d1"\n\t" \
|
|
"vpaddd "#d1", "#c1", "#c1"\n\t" \
|
|
"vpxor "#c1", "#b1", "#b1"\n\t" \
|
|
"vpaddd "#b2", "#a2", "#a2"\n\t" \
|
|
"vpxor "#a2", "#d2", "#d2"\n\t" \
|
|
"vpshufb %[rotl16], "#d2", "#d2"\n\t" \
|
|
"vpaddd "#d2", "#c2", "#c2"\n\t" \
|
|
"vpxor "#c2", "#b2", "#b2"\n\t" \
|
|
"vpaddd "#b3", "#a3", "#a3"\n\t" \
|
|
"vpxor "#a3", "#d3", "#d3"\n\t" \
|
|
"vpshufb %[rotl16], "#d3", "#d3"\n\t" \
|
|
"vpaddd "#d3", "#c3", "#c3"\n\t" \
|
|
"vpxor "#c3", "#b3", "#b3"\n\t" \
|
|
"vmovdqa "#c1", "#o1"(%[x])\n\t" \
|
|
"vpsrld $20, "#b0", "#t1"\n\t" \
|
|
"vpslld $12, "#b0", "#b0"\n\t" \
|
|
"vpxor "#t1", "#b0", "#b0"\n\t" \
|
|
"vpsrld $20, "#b1", "#t1"\n\t" \
|
|
"vpslld $12, "#b1", "#b1"\n\t" \
|
|
"vpxor "#t1", "#b1", "#b1"\n\t" \
|
|
"vpsrld $20, "#b2", "#t1"\n\t" \
|
|
"vpslld $12, "#b2", "#b2"\n\t" \
|
|
"vpxor "#t1", "#b2", "#b2"\n\t" \
|
|
"vpsrld $20, "#b3", "#t1"\n\t" \
|
|
"vpslld $12, "#b3", "#b3"\n\t" \
|
|
"vpxor "#t1", "#b3", "#b3"\n\t" \
|
|
"vpaddd "#b0", "#a0", "#a0"\n\t" \
|
|
"vpxor "#a0", "#d0", "#d0"\n\t" \
|
|
"vmovdqa "#o1"(%[x]), "#c1"\n\t" \
|
|
"vpshufb %[rotl8], "#d0", "#d0"\n\t" \
|
|
"vpaddd "#d0", "#c0", "#c0"\n\t" \
|
|
"vpxor "#c0", "#b0", "#b0"\n\t" \
|
|
"vpaddd "#b1", "#a1", "#a1"\n\t" \
|
|
"vpxor "#a1", "#d1", "#d1"\n\t" \
|
|
"vpshufb %[rotl8], "#d1", "#d1"\n\t" \
|
|
"vpaddd "#d1", "#c1", "#c1"\n\t" \
|
|
"vpxor "#c1", "#b1", "#b1"\n\t" \
|
|
"vpaddd "#b2", "#a2", "#a2"\n\t" \
|
|
"vpxor "#a2", "#d2", "#d2"\n\t" \
|
|
"vpshufb %[rotl8], "#d2", "#d2"\n\t" \
|
|
"vpaddd "#d2", "#c2", "#c2"\n\t" \
|
|
"vpxor "#c2", "#b2", "#b2"\n\t" \
|
|
"vpaddd "#b3", "#a3", "#a3"\n\t" \
|
|
"vpxor "#a3", "#d3", "#d3"\n\t" \
|
|
"vpshufb %[rotl8], "#d3", "#d3"\n\t" \
|
|
"vpaddd "#d3", "#c3", "#c3"\n\t" \
|
|
"vpxor "#c3", "#b3", "#b3"\n\t" \
|
|
"vmovdqa "#c1", "#o1"(%[x])\n\t" \
|
|
"vpsrld $25, "#b0", "#t1"\n\t" \
|
|
"vpslld $7, "#b0", "#b0"\n\t" \
|
|
"vpxor "#t1", "#b0", "#b0"\n\t" \
|
|
"vpsrld $25, "#b1", "#t1"\n\t" \
|
|
"vpslld $7, "#b1", "#b1"\n\t" \
|
|
"vpxor "#t1", "#b1", "#b1"\n\t" \
|
|
"vpsrld $25, "#b2", "#t1"\n\t" \
|
|
"vpslld $7, "#b2", "#b2"\n\t" \
|
|
"vpxor "#t1", "#b2", "#b2"\n\t" \
|
|
"vpsrld $25, "#b3", "#t1"\n\t" \
|
|
"vpslld $7, "#b3", "#b3"\n\t" \
|
|
"vpxor "#t1", "#b3", "#b3"\n\t"
|
|
|
|
|
|
#define QUARTERROUND_XMM() \
|
|
QUARTERROUND_INTEL_ASM(%%xmm0,%%xmm4,%%xmm8,%%xmm12, \
|
|
%%xmm1,%%xmm5,%%xmm9,%%xmm13, \
|
|
%%xmm2,%%xmm6,%%xmm10,%%xmm14, \
|
|
%%xmm3,%%xmm7,%%xmm11,%%xmm15, \
|
|
%%xmm11,48)
|
|
#define QUARTERROUND_XMM_2() \
|
|
QUARTERROUND_INTEL_ASM_2(%%xmm0,%%xmm5,%%xmm10,%%xmm15, \
|
|
%%xmm1,%%xmm6,%%xmm11,%%xmm12, \
|
|
%%xmm2,%%xmm7,%%xmm8,%%xmm13, \
|
|
%%xmm3,%%xmm4,%%xmm9,%%xmm14, \
|
|
%%xmm11,48)
|
|
|
|
#define QUARTERROUND_YMM() \
|
|
QUARTERROUND_INTEL_ASM(%%ymm0,%%ymm4,%%ymm8,%%ymm12, \
|
|
%%ymm1,%%ymm5,%%ymm9,%%ymm13, \
|
|
%%ymm2,%%ymm6,%%ymm10,%%ymm14, \
|
|
%%ymm3,%%ymm7,%%ymm11,%%ymm15, \
|
|
%%ymm11,96)
|
|
#define QUARTERROUND_YMM_2() \
|
|
QUARTERROUND_INTEL_ASM_2(%%ymm0,%%ymm5,%%ymm10,%%ymm15, \
|
|
%%ymm1,%%ymm6,%%ymm11,%%ymm12, \
|
|
%%ymm2,%%ymm7,%%ymm8,%%ymm13, \
|
|
%%ymm3,%%ymm4,%%ymm9,%%ymm14, \
|
|
%%ymm11,96)
|
|
|
|
/**
|
|
* Set up iv(nonce). Earlier versions used 64 bits instead of 96, this version
|
|
* uses the typical AEAD 96 bit nonce and can do record sizes of 256 GB.
|
|
*/
|
|
int wc_Chacha_SetIV(ChaCha* ctx, const byte* inIv, word32 counter)
|
|
{
|
|
word32 temp[CHACHA_IV_WORDS];/* used for alignment of memory */
|
|
|
|
#ifdef CHACHA_AEAD_TEST
|
|
word32 i;
|
|
printf("NONCE : ");
|
|
for (i = 0; i < CHACHA_IV_BYTES; i++) {
|
|
printf("%02x", inIv[i]);
|
|
}
|
|
printf("\n\n");
|
|
#endif
|
|
|
|
if (ctx == NULL)
|
|
return BAD_FUNC_ARG;
|
|
|
|
XMEMCPY(temp, inIv, CHACHA_IV_BYTES);
|
|
|
|
ctx->X[CHACHA_IV_BYTES+0] = counter; /* block counter */
|
|
ctx->X[CHACHA_IV_BYTES+1] = LITTLE32(temp[0]); /* fixed variable from nonce */
|
|
ctx->X[CHACHA_IV_BYTES+2] = LITTLE32(temp[1]); /* counter from nonce */
|
|
ctx->X[CHACHA_IV_BYTES+3] = LITTLE32(temp[2]); /* counter from nonce */
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* "expand 32-byte k" as unsigned 32 byte */
|
|
static const word32 sigma[4] = {0x61707865, 0x3320646e, 0x79622d32, 0x6b206574};
|
|
/* "expand 16-byte k" as unsigned 16 byte */
|
|
static const word32 tau[4] = {0x61707865, 0x3120646e, 0x79622d36, 0x6b206574};
|
|
|
|
/**
|
|
* Key setup. 8 word iv (nonce)
|
|
*/
|
|
int wc_Chacha_SetKey(ChaCha* ctx, const byte* key, word32 keySz)
|
|
{
|
|
const word32* constants;
|
|
const byte* k;
|
|
|
|
#ifdef XSTREAM_ALIGN
|
|
word32 alignKey[8];
|
|
#endif
|
|
|
|
if (ctx == NULL)
|
|
return BAD_FUNC_ARG;
|
|
|
|
if (keySz != (CHACHA_MAX_KEY_SZ/2) && keySz != CHACHA_MAX_KEY_SZ)
|
|
return BAD_FUNC_ARG;
|
|
|
|
#ifdef XSTREAM_ALIGN
|
|
if ((wolfssl_word)key % 4) {
|
|
WOLFSSL_MSG("wc_ChachaSetKey unaligned key");
|
|
XMEMCPY(alignKey, key, keySz);
|
|
k = (byte*)alignKey;
|
|
}
|
|
else {
|
|
k = key;
|
|
}
|
|
#else
|
|
k = key;
|
|
#endif /* XSTREAM_ALIGN */
|
|
|
|
#ifdef CHACHA_AEAD_TEST
|
|
word32 i;
|
|
printf("ChaCha key used :\n");
|
|
for (i = 0; i < keySz; i++) {
|
|
printf("%02x", key[i]);
|
|
if ((i + 1) % 8 == 0)
|
|
printf("\n");
|
|
}
|
|
printf("\n\n");
|
|
#endif
|
|
|
|
ctx->X[4] = U8TO32_LITTLE(k + 0);
|
|
ctx->X[5] = U8TO32_LITTLE(k + 4);
|
|
ctx->X[6] = U8TO32_LITTLE(k + 8);
|
|
ctx->X[7] = U8TO32_LITTLE(k + 12);
|
|
if (keySz == CHACHA_MAX_KEY_SZ) {
|
|
k += 16;
|
|
constants = sigma;
|
|
}
|
|
else {
|
|
constants = tau;
|
|
}
|
|
ctx->X[ 8] = U8TO32_LITTLE(k + 0);
|
|
ctx->X[ 9] = U8TO32_LITTLE(k + 4);
|
|
ctx->X[10] = U8TO32_LITTLE(k + 8);
|
|
ctx->X[11] = U8TO32_LITTLE(k + 12);
|
|
ctx->X[ 0] = constants[0];
|
|
ctx->X[ 1] = constants[1];
|
|
ctx->X[ 2] = constants[2];
|
|
ctx->X[ 3] = constants[3];
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Converts word into bytes with rotations having been done.
|
|
*/
|
|
static INLINE void wc_Chacha_wordtobyte(word32 output[CHACHA_CHUNK_WORDS],
|
|
const word32 input[CHACHA_CHUNK_WORDS])
|
|
{
|
|
word32 x[CHACHA_CHUNK_WORDS];
|
|
word32 i;
|
|
|
|
for (i = 0; i < CHACHA_CHUNK_WORDS; i++) {
|
|
x[i] = input[i];
|
|
}
|
|
|
|
for (i = (ROUNDS); i > 0; i -= 2) {
|
|
QUARTERROUND(0, 4, 8, 12)
|
|
QUARTERROUND(1, 5, 9, 13)
|
|
QUARTERROUND(2, 6, 10, 14)
|
|
QUARTERROUND(3, 7, 11, 15)
|
|
QUARTERROUND(0, 5, 10, 15)
|
|
QUARTERROUND(1, 6, 11, 12)
|
|
QUARTERROUND(2, 7, 8, 13)
|
|
QUARTERROUND(3, 4, 9, 14)
|
|
}
|
|
|
|
for (i = 0; i < CHACHA_CHUNK_WORDS; i++) {
|
|
x[i] = PLUS(x[i], input[i]);
|
|
}
|
|
|
|
for (i = 0; i < CHACHA_CHUNK_WORDS; i++) {
|
|
output[i] = LITTLE32(x[i]);
|
|
}
|
|
}
|
|
|
|
|
|
#ifdef USE_INTEL_CHACHA_SPEEDUP
|
|
|
|
#define QUARTERROUND_2_X64(r11, r12, r13, r14, r21, r22, r23, r24) \
|
|
"addl "#r12", "#r11"\n\t" \
|
|
"addl "#r22", "#r21"\n\t" \
|
|
"xorl "#r11", "#r14"\n\t" \
|
|
"xorl "#r21", "#r24"\n\t" \
|
|
"roll $16, "#r14"\n\t" \
|
|
"roll $16, "#r24"\n\t" \
|
|
"addl "#r14", "#r13"\n\t" \
|
|
"addl "#r24", "#r23"\n\t" \
|
|
"xorl "#r13", "#r12"\n\t" \
|
|
"xorl "#r23", "#r22"\n\t" \
|
|
"roll $12, "#r12"\n\t" \
|
|
"roll $12, "#r22"\n\t" \
|
|
"addl "#r12", "#r11"\n\t" \
|
|
"addl "#r22", "#r21"\n\t" \
|
|
"xorl "#r11", "#r14"\n\t" \
|
|
"xorl "#r21", "#r24"\n\t" \
|
|
"roll $8, "#r14"\n\t" \
|
|
"roll $8, "#r24"\n\t" \
|
|
"addl "#r14", "#r13"\n\t" \
|
|
"addl "#r24", "#r23"\n\t" \
|
|
"xorl "#r13", "#r12"\n\t" \
|
|
"xorl "#r23", "#r22"\n\t" \
|
|
"roll $7, "#r12"\n\t" \
|
|
"roll $7, "#r22"\n\t" \
|
|
|
|
#define CHACHA_CRYPT_X64() \
|
|
"subq $40, %%rsp\n\t" \
|
|
"movq 32(%[input]), %%rax\n\t" \
|
|
"movq 40(%[input]), %%rdx\n\t" \
|
|
"movq %%rax, 8(%%rsp)\n\t" \
|
|
"movq %%rdx, 16(%%rsp)\n\t" \
|
|
"movl 0(%[input]), %%eax\n\t" \
|
|
"movl 4(%[input]), %%ebx\n\t" \
|
|
"movl 8(%[input]), %%ecx\n\t" \
|
|
"movl 12(%[input]), %%edx\n\t" \
|
|
"movl 16(%[input]), %%r8d\n\t" \
|
|
"movl 20(%[input]), %%r9d\n\t" \
|
|
"movl 24(%[input]), %%r10d\n\t" \
|
|
"movl 28(%[input]), %%r11d\n\t" \
|
|
"movl 48(%[input]), %%r12d\n\t" \
|
|
"movl 52(%[input]), %%r13d\n\t" \
|
|
"movl 56(%[input]), %%r14d\n\t" \
|
|
"movl 60(%[input]), %%r15d\n\t" \
|
|
"movb $10, (%%rsp)\n\t" \
|
|
"movq %%rsi, 32(%%rsp)\n\t" \
|
|
"movq %%rdi, 24(%%rsp)\n\t" \
|
|
"movl 8(%%rsp), %%esi\n\t" \
|
|
"movl 12(%%rsp), %%edi\n\t" \
|
|
"\n" \
|
|
"1:\n\t" \
|
|
QUARTERROUND_2_X64(%%eax, %%r8d, %%esi, %%r12d, \
|
|
%%ebx, %%r9d, %%edi, %%r13d) \
|
|
"movl %%esi, 8(%%rsp)\n\t" \
|
|
"movl %%edi, 12(%%rsp)\n\t" \
|
|
"movl 16(%%rsp), %%esi\n\t" \
|
|
"movl 20(%%rsp), %%edi\n\t" \
|
|
QUARTERROUND_2_X64(%%ecx, %%r10d, %%esi, %%r14d, \
|
|
%%edx, %%r11d, %%edi, %%r15d) \
|
|
QUARTERROUND_2_X64(%%eax, %%r9d, %%esi, %%r15d, \
|
|
%%ebx, %%r10d, %%edi, %%r12d) \
|
|
"movl %%esi, 16(%%rsp)\n\t" \
|
|
"movl %%edi, 20(%%rsp)\n\t" \
|
|
"movl 8(%%rsp), %%esi\n\t" \
|
|
"movl 12(%%rsp), %%edi\n\t" \
|
|
QUARTERROUND_2_X64(%%ecx, %%r11d, %%esi, %%r13d, \
|
|
%%edx, %%r8d, %%edi, %%r14d) \
|
|
"decb (%%rsp)\n\t" \
|
|
"jnz 1b\n\t" \
|
|
"movl %%esi, 8(%%rsp)\n\t" \
|
|
"movl %%edi, 12(%%rsp)\n\t" \
|
|
"movq 32(%%rsp), %%rsi\n\t" \
|
|
"movq 24(%%rsp), %%rdi\n\t" \
|
|
"addl 0(%[input]), %%eax\n\t" \
|
|
"addl 4(%[input]), %%ebx\n\t" \
|
|
"addl 8(%[input]), %%ecx\n\t" \
|
|
"addl 12(%[input]), %%edx\n\t" \
|
|
"addl 16(%[input]), %%r8d\n\t" \
|
|
"addl 20(%[input]), %%r9d\n\t" \
|
|
"addl 24(%[input]), %%r10d\n\t" \
|
|
"addl 28(%[input]), %%r11d\n\t" \
|
|
"addl 48(%[input]), %%r12d\n\t" \
|
|
"addl 52(%[input]), %%r13d\n\t" \
|
|
"addl 56(%[input]), %%r14d\n\t" \
|
|
"addl 60(%[input]), %%r15d\n\t" \
|
|
|
|
#define CHACHA_PARTIAL_CHUNK_X64() \
|
|
__asm__ __volatile__ ( \
|
|
CHACHA_CRYPT_X64() \
|
|
"movl %%eax , 0(%[c])\n\t" \
|
|
"movl %%ebx , 4(%[c])\n\t" \
|
|
"movl %%ecx , 8(%[c])\n\t" \
|
|
"movl %%edx , 12(%[c])\n\t" \
|
|
"movl %%r8d , 16(%[c])\n\t" \
|
|
"movl %%r9d , 20(%[c])\n\t" \
|
|
"movl %%r10d, 24(%[c])\n\t" \
|
|
"movl %%r11d, 28(%[c])\n\t" \
|
|
"movl %%r12d, 48(%[c])\n\t" \
|
|
"movl %%r13d, 52(%[c])\n\t" \
|
|
"movl %%r14d, 56(%[c])\n\t" \
|
|
"movl %%r15d, 60(%[c])\n\t" \
|
|
"movl 8(%%rsp), %%eax\n\t" \
|
|
"movl 12(%%rsp), %%ebx\n\t" \
|
|
"movl 16(%%rsp), %%ecx\n\t" \
|
|
"movl 20(%%rsp), %%edx\n\t" \
|
|
"addl 32(%[input]), %%eax\n\t" \
|
|
"addl 36(%[input]), %%ebx\n\t" \
|
|
"addl 40(%[input]), %%ecx\n\t" \
|
|
"addl 44(%[input]), %%edx\n\t" \
|
|
"movl %%eax , 32(%[c])\n\t" \
|
|
"movl %%ebx , 36(%[c])\n\t" \
|
|
"movl %%ecx , 40(%[c])\n\t" \
|
|
"movl %%edx , 44(%[c])\n\t" \
|
|
"addl $1, 48(%[input])\n\t" \
|
|
"addq $40, %%rsp\n\t" \
|
|
"movq %[output], %%rax\n\t" \
|
|
"movq %[m], %%rbx\n\t" \
|
|
"movl %[bytes], %%r8d\n\t" \
|
|
"xorq %%rdx, %%rdx\n\t" \
|
|
"movl %%r8d, %%r9d\n\t" \
|
|
"andl $7, %%r9d\n\t" \
|
|
"jz 4f\n\t" \
|
|
"\n" \
|
|
"2:\n\t" \
|
|
"movzbl (%[c],%%rdx,1), %%ecx\n\t" \
|
|
"xorb (%%rbx,%%rdx,1), %%cl\n\t" \
|
|
"movb %%cl, (%%rax,%%rdx,1)\n\t" \
|
|
"incl %%edx\n\t" \
|
|
"cmpl %%r9d, %%edx\n\t" \
|
|
"jne 2b\n\t" \
|
|
"je 3f\n\t" \
|
|
"\n" \
|
|
"4:\n\t" \
|
|
"movq (%[c],%%rdx,1), %%rcx\n\t" \
|
|
"xorq (%%rbx,%%rdx,1), %%rcx\n\t" \
|
|
"movq %%rcx, (%%rax,%%rdx,1)\n\t" \
|
|
"addl $8, %%edx\n\t" \
|
|
"\n" \
|
|
"3:\n\t" \
|
|
"cmpl %%r8d, %%edx\n\t" \
|
|
"jne 4b\n\t" \
|
|
: \
|
|
: [input] "r" (ctx->X), [c] "r" (x), \
|
|
[output] "m" (c), [bytes] "m" (bytes), [m] "m" (m) \
|
|
: "eax", "ebx", "ecx", "edx", "r8", "r9", "r10", "r11", "r12", "r13", \
|
|
"r14", "r15", "memory" \
|
|
)
|
|
|
|
|
|
#define CHACHA_CHUNK_X64() \
|
|
__asm__ __volatile__ ( \
|
|
CHACHA_CRYPT_X64() \
|
|
"movq %%rsi, 32(%%rsp)\n\t" \
|
|
"addq $40, %%rsp\n\t" \
|
|
"movq %[m], %%rsi\n\t" \
|
|
"subq $40, %%rsp\n\t" \
|
|
"xorl 0(%%rsi), %%eax\n\t" \
|
|
"xorl 4(%%rsi), %%ebx\n\t" \
|
|
"xorl 8(%%rsi), %%ecx\n\t" \
|
|
"xorl 12(%%rsi), %%edx\n\t" \
|
|
"xorl 16(%%rsi), %%r8d\n\t" \
|
|
"xorl 20(%%rsi), %%r9d\n\t" \
|
|
"xorl 24(%%rsi), %%r10d\n\t" \
|
|
"xorl 28(%%rsi), %%r11d\n\t" \
|
|
"xorl 48(%%rsi), %%r12d\n\t" \
|
|
"xorl 52(%%rsi), %%r13d\n\t" \
|
|
"xorl 56(%%rsi), %%r14d\n\t" \
|
|
"xorl 60(%%rsi), %%r15d\n\t" \
|
|
"movq 32(%%rsp), %%rsi\n\t" \
|
|
"movl %%eax , 0(%[c])\n\t" \
|
|
"movl %%ebx , 4(%[c])\n\t" \
|
|
"movl %%ecx , 8(%[c])\n\t" \
|
|
"movl %%edx , 12(%[c])\n\t" \
|
|
"movl %%r8d , 16(%[c])\n\t" \
|
|
"movl %%r9d , 20(%[c])\n\t" \
|
|
"movl %%r10d, 24(%[c])\n\t" \
|
|
"movl %%r11d, 28(%[c])\n\t" \
|
|
"movl %%r12d, 48(%[c])\n\t" \
|
|
"movl %%r13d, 52(%[c])\n\t" \
|
|
"movl %%r14d, 56(%[c])\n\t" \
|
|
"movl %%r15d, 60(%[c])\n\t" \
|
|
"addq $40, %%rsp\n\t" \
|
|
"movq %[m], %%r8\n\t" \
|
|
"subq $40, %%rsp\n\t" \
|
|
"movl 8(%%rsp), %%eax\n\t" \
|
|
"movl 12(%%rsp), %%ebx\n\t" \
|
|
"movl 16(%%rsp), %%ecx\n\t" \
|
|
"movl 20(%%rsp), %%edx\n\t" \
|
|
"addl 32(%[input]), %%eax\n\t" \
|
|
"addl 36(%[input]), %%ebx\n\t" \
|
|
"addl 40(%[input]), %%ecx\n\t" \
|
|
"addl 44(%[input]), %%edx\n\t" \
|
|
"xorl 32(%%r8), %%eax\n\t" \
|
|
"xorl 36(%%r8), %%ebx\n\t" \
|
|
"xorl 40(%%r8), %%ecx\n\t" \
|
|
"xorl 44(%%r8), %%edx\n\t" \
|
|
"movl %%eax , 32(%[c])\n\t" \
|
|
"movl %%ebx , 36(%[c])\n\t" \
|
|
"movl %%ecx , 40(%[c])\n\t" \
|
|
"movl %%edx , 44(%[c])\n\t" \
|
|
"addl $1, 48(%[input])\n\t" \
|
|
"addq $40, %%rsp\n\t" \
|
|
: \
|
|
: [input] "r" (ctx->X), [c] "r" (c), [m] "m" (m) \
|
|
: "eax", "ebx", "ecx", "edx", "r8", "r9", "r10", "r11", "r12", "r13", \
|
|
"r14", "r15", "memory" \
|
|
)
|
|
|
|
|
|
static void chacha_encrypt_x64(ChaCha* ctx, const byte* m, byte* c,
|
|
word32 bytes)
|
|
{
|
|
word32 x[CHACHA_CHUNK_WORDS];
|
|
|
|
if (bytes == 0)
|
|
return;
|
|
|
|
for (; bytes >= CHACHA_CHUNK_BYTES;) {
|
|
CHACHA_CHUNK_X64();
|
|
bytes -= CHACHA_CHUNK_BYTES;
|
|
c += CHACHA_CHUNK_BYTES;
|
|
m += CHACHA_CHUNK_BYTES;
|
|
}
|
|
if (bytes > 0) {
|
|
CHACHA_PARTIAL_CHUNK_X64();
|
|
}
|
|
}
|
|
|
|
#if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2)
|
|
static const __m128i rotl8 = { 0x0605040702010003UL,0x0e0d0c0f0a09080bUL };
|
|
static const __m128i rotl16 = { 0x0504070601000302UL,0x0d0c0f0e09080b0aUL };
|
|
#endif /* HAVE_INTEL_AVX1 || HAVE_INTEL_AVX2 */
|
|
|
|
#ifdef HAVE_INTEL_AVX1
|
|
#define QUARTERROUND_2_AVX() \
|
|
"paddd %%xmm1, %%xmm0\n\t" \
|
|
"pxor %%xmm0, %%xmm3\n\t" \
|
|
"pshufb %[rotl16], %%xmm3\n\t" \
|
|
"paddd %%xmm3, %%xmm2\n\t" \
|
|
"pxor %%xmm2, %%xmm1\n\t" \
|
|
"movdqa %%xmm1, %%xmm4\n\t" \
|
|
"pslld $12, %%xmm1\n\t" \
|
|
"psrld $20, %%xmm4\n\t" \
|
|
"pxor %%xmm4, %%xmm1\n\t" \
|
|
"paddd %%xmm1, %%xmm0\n\t" \
|
|
"pxor %%xmm0, %%xmm3\n\t" \
|
|
"pshufb %[rotl8], %%xmm3\n\t" \
|
|
"paddd %%xmm3, %%xmm2\n\t" \
|
|
"pxor %%xmm2, %%xmm1\n\t" \
|
|
"movdqa %%xmm1, %%xmm4\n\t" \
|
|
"pslld $7, %%xmm1\n\t" \
|
|
"psrld $25, %%xmm4\n\t" \
|
|
"pxor %%xmm4, %%xmm1\n\t" \
|
|
"# Swap words for next round\n\t" \
|
|
"pshufd $0x39, %%xmm1, %%xmm1\n\t" \
|
|
"pshufd $0x4e, %%xmm2, %%xmm2\n\t" \
|
|
"pshufd $0x93, %%xmm3, %%xmm3\n\t" \
|
|
"paddd %%xmm1, %%xmm0\n\t" \
|
|
"pxor %%xmm0, %%xmm3\n\t" \
|
|
"pshufb %[rotl16], %%xmm3\n\t" \
|
|
"paddd %%xmm3, %%xmm2\n\t" \
|
|
"pxor %%xmm2, %%xmm1\n\t" \
|
|
"movdqa %%xmm1, %%xmm4\n\t" \
|
|
"pslld $12, %%xmm1\n\t" \
|
|
"psrld $20, %%xmm4\n\t" \
|
|
"pxor %%xmm4, %%xmm1\n\t" \
|
|
"paddd %%xmm1, %%xmm0\n\t" \
|
|
"pxor %%xmm0, %%xmm3\n\t" \
|
|
"pshufb %[rotl8], %%xmm3\n\t" \
|
|
"paddd %%xmm3, %%xmm2\n\t" \
|
|
"pxor %%xmm2, %%xmm1\n\t" \
|
|
"movdqa %%xmm1, %%xmm4\n\t" \
|
|
"pslld $7, %%xmm1\n\t" \
|
|
"psrld $25, %%xmm4\n\t" \
|
|
"pxor %%xmm4, %%xmm1\n\t" \
|
|
"# Swap words back\n\t" \
|
|
"pshufd $0x93, %%xmm1, %%xmm1\n\t" \
|
|
"pshufd $0x4e, %%xmm2, %%xmm2\n\t" \
|
|
"pshufd $0x39, %%xmm3, %%xmm3\n\t" \
|
|
|
|
#define CHACHA_CRYPT_AVX() \
|
|
"movdqu 0(%[input]), %%xmm0\n\t" \
|
|
"movdqu 16(%[input]), %%xmm1\n\t" \
|
|
"movdqu 32(%[input]), %%xmm2\n\t" \
|
|
"movdqu 48(%[input]), %%xmm3\n\t" \
|
|
"movb $10, %%al\n\t" \
|
|
"\n" \
|
|
"1:\n\t" \
|
|
QUARTERROUND_2_AVX() \
|
|
"decb %%al\n\t" \
|
|
"jnz 1b\n\t" \
|
|
"movdqu 0(%[input]), %%xmm4\n\t" \
|
|
"movdqu 16(%[input]), %%xmm5\n\t" \
|
|
"movdqu 32(%[input]), %%xmm6\n\t" \
|
|
"movdqu 48(%[input]), %%xmm7\n\t" \
|
|
"paddd %%xmm4, %%xmm0\n\t" \
|
|
"paddd %%xmm5, %%xmm1\n\t" \
|
|
"paddd %%xmm6, %%xmm2\n\t" \
|
|
"paddd %%xmm7, %%xmm3\n\t" \
|
|
|
|
#define CHACHA_PARTIAL_CHUNK_AVX() \
|
|
__asm__ __volatile__ ( \
|
|
CHACHA_CRYPT_AVX() \
|
|
"movdqu %%xmm0, 0(%[c])\n\t" \
|
|
"movdqu %%xmm1, 16(%[c])\n\t" \
|
|
"movdqu %%xmm2, 32(%[c])\n\t" \
|
|
"movdqu %%xmm3, 48(%[c])\n\t" \
|
|
"addl $1, 48(%[input])\n\t" \
|
|
"movl %[bytes], %%r8d\n\t" \
|
|
"xorq %%rdx, %%rdx\n\t" \
|
|
"movl %%r8d, %%r9d\n\t" \
|
|
"andl $7, %%r9d\n\t" \
|
|
"jz 4f\n\t" \
|
|
"\n" \
|
|
"2:\n\t" \
|
|
"movzbl (%[c],%%rdx,1), %%ecx\n\t" \
|
|
"xorb (%[m],%%rdx,1), %%cl\n\t" \
|
|
"movb %%cl, (%[output],%%rdx,1)\n\t" \
|
|
"incl %%edx\n\t" \
|
|
"cmpl %%r9d, %%edx\n\t" \
|
|
"jne 2b\n\t" \
|
|
"je 3f\n\t" \
|
|
"\n" \
|
|
"4:\n\t" \
|
|
"movq (%[c],%%rdx,1), %%rcx\n\t" \
|
|
"xorq (%[m],%%rdx,1), %%rcx\n\t" \
|
|
"movq %%rcx, (%[output],%%rdx,1)\n\t" \
|
|
"addl $8, %%edx\n\t" \
|
|
"\n" \
|
|
"3:\n\t" \
|
|
"cmpl %%r8d, %%edx\n\t" \
|
|
"jne 4b\n\t" \
|
|
: \
|
|
: [input] "r" (ctx->X), [c] "r" (x), \
|
|
[output] "r" (c), [bytes] "r" (bytes), [m] "r" (m), \
|
|
[rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) \
|
|
: "eax", "ecx", "edx", "r8", "r9", "memory", \
|
|
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" \
|
|
)
|
|
|
|
|
|
#define CHACHA_CHUNK_AVX() \
|
|
__asm__ __volatile__ ( \
|
|
CHACHA_CRYPT_AVX() \
|
|
"movdqu 0(%[m]), %%xmm4\n\t" \
|
|
"movdqu 16(%[m]), %%xmm5\n\t" \
|
|
"movdqu 32(%[m]), %%xmm6\n\t" \
|
|
"movdqu 48(%[m]), %%xmm7\n\t" \
|
|
"pxor %%xmm4, %%xmm0\n\t" \
|
|
"pxor %%xmm5, %%xmm1\n\t" \
|
|
"pxor %%xmm6, %%xmm2\n\t" \
|
|
"pxor %%xmm7, %%xmm3\n\t" \
|
|
"movdqu %%xmm0, 0(%[c])\n\t" \
|
|
"movdqu %%xmm1, 16(%[c])\n\t" \
|
|
"movdqu %%xmm2, 32(%[c])\n\t" \
|
|
"movdqu %%xmm3, 48(%[c])\n\t" \
|
|
"addl $1, 48(%[input])\n\t" \
|
|
: \
|
|
: [input] "r" (ctx->X), [c] "r" (c), [m] "r" (m), \
|
|
[rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) \
|
|
: "rax", "memory", \
|
|
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" \
|
|
)
|
|
|
|
CHACHA20_NOINLINE static void chacha_encrypt_avx(ChaCha* ctx, const byte* m,
|
|
byte* c, word32 bytes)
|
|
{
|
|
ALIGN128 word32 X[4*CHACHA_CHUNK_WORDS]; /* used to make sure aligned */
|
|
ALIGN128 word32 x[2*CHACHA_CHUNK_WORDS]; /* used to make sure aligned */
|
|
word32 cnt = 0;
|
|
static const __m128i add = { 0x0000000100000000UL,0x0000000300000002UL };
|
|
static const __m128i four = { 0x0000000400000004UL,0x0000000400000004UL };
|
|
|
|
if (bytes == 0)
|
|
return;
|
|
|
|
__asm__ __volatile__ (
|
|
"movl %[bytes], %[cnt]\n\t"
|
|
"shrl $8, %[cnt]\n\t"
|
|
"jz L_end128\n\t"
|
|
|
|
"vpshufd $0, (%[key]), %%xmm0\n\t"
|
|
"vpshufd $0, 4(%[key]), %%xmm1\n\t"
|
|
"vpshufd $0, 8(%[key]), %%xmm2\n\t"
|
|
"vpshufd $0, 12(%[key]), %%xmm3\n\t"
|
|
"vpshufd $0, 16(%[key]), %%xmm4\n\t"
|
|
"vpshufd $0, 20(%[key]), %%xmm5\n\t"
|
|
"vpshufd $0, 24(%[key]), %%xmm6\n\t"
|
|
"vpshufd $0, 28(%[key]), %%xmm7\n\t"
|
|
"vpshufd $0, 32(%[key]), %%xmm8\n\t"
|
|
"vpshufd $0, 36(%[key]), %%xmm9\n\t"
|
|
"vpshufd $0, 40(%[key]), %%xmm10\n\t"
|
|
"vpshufd $0, 44(%[key]), %%xmm11\n\t"
|
|
"vpshufd $0, 48(%[key]), %%xmm12\n\t"
|
|
"vpshufd $0, 52(%[key]), %%xmm13\n\t"
|
|
"vpshufd $0, 56(%[key]), %%xmm14\n\t"
|
|
"vpshufd $0, 60(%[key]), %%xmm15\n\t"
|
|
|
|
"vpaddd %[add], %%xmm12, %%xmm12\n\t"
|
|
|
|
"vmovdqa %%xmm0, (%[X])\n\t"
|
|
"vmovdqa %%xmm1, 16(%[X])\n\t"
|
|
"vmovdqa %%xmm2, 32(%[X])\n\t"
|
|
"vmovdqa %%xmm3, 48(%[X])\n\t"
|
|
"vmovdqa %%xmm4, 64(%[X])\n\t"
|
|
"vmovdqa %%xmm5, 80(%[X])\n\t"
|
|
"vmovdqa %%xmm6, 96(%[X])\n\t"
|
|
"vmovdqa %%xmm7, 112(%[X])\n\t"
|
|
"vmovdqa %%xmm8, 128(%[X])\n\t"
|
|
"vmovdqa %%xmm9, 144(%[X])\n\t"
|
|
"vmovdqa %%xmm10, 160(%[X])\n\t"
|
|
"vmovdqa %%xmm11, 176(%[X])\n\t"
|
|
"vmovdqa %%xmm12, 192(%[X])\n\t"
|
|
"vmovdqa %%xmm13, 208(%[X])\n\t"
|
|
"vmovdqa %%xmm14, 224(%[X])\n\t"
|
|
"vmovdqa %%xmm15, 240(%[X])\n\t"
|
|
"\n"
|
|
"L_enc128_loop:\n\t"
|
|
"vmovdqa %%xmm11, 48(%[x])\n\t"
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
QUARTERROUND_XMM()
|
|
QUARTERROUND_XMM_2()
|
|
"vmovdqa 48(%[x]), %%xmm11\n\t"
|
|
|
|
"vpaddd (%[X]), %%xmm0, %%xmm0\n\t"
|
|
"vpaddd 16(%[X]), %%xmm1, %%xmm1\n\t"
|
|
"vpaddd 32(%[X]), %%xmm2, %%xmm2\n\t"
|
|
"vpaddd 48(%[X]), %%xmm3, %%xmm3\n\t"
|
|
"vpaddd 64(%[X]), %%xmm4, %%xmm4\n\t"
|
|
"vpaddd 80(%[X]), %%xmm5, %%xmm5\n\t"
|
|
"vpaddd 96(%[X]), %%xmm6, %%xmm6\n\t"
|
|
"vpaddd 112(%[X]), %%xmm7, %%xmm7\n\t"
|
|
"vpaddd 128(%[X]), %%xmm8, %%xmm8\n\t"
|
|
"vpaddd 144(%[X]), %%xmm9, %%xmm9\n\t"
|
|
"vpaddd 160(%[X]), %%xmm10, %%xmm10\n\t"
|
|
"vpaddd 176(%[X]), %%xmm11, %%xmm11\n\t"
|
|
"vpaddd 192(%[X]), %%xmm12, %%xmm12\n\t"
|
|
"vpaddd 208(%[X]), %%xmm13, %%xmm13\n\t"
|
|
"vpaddd 224(%[X]), %%xmm14, %%xmm14\n\t"
|
|
"vpaddd 240(%[X]), %%xmm15, %%xmm15\n\t"
|
|
|
|
"vmovdqa %%xmm8, (%[x])\n\t"
|
|
"vmovdqa %%xmm9, 16(%[x])\n\t"
|
|
"vmovdqa %%xmm10, 32(%[x])\n\t"
|
|
"vmovdqa %%xmm11, 48(%[x])\n\t"
|
|
"vmovdqa %%xmm12, 64(%[x])\n\t"
|
|
"vmovdqa %%xmm13, 80(%[x])\n\t"
|
|
"vmovdqa %%xmm14, 96(%[x])\n\t"
|
|
"vmovdqa %%xmm15, 112(%[x])\n\t"
|
|
|
|
"vpunpckldq %%xmm1, %%xmm0, %%xmm8\n\t"
|
|
"vpunpckldq %%xmm3, %%xmm2, %%xmm9\n\t"
|
|
"vpunpckhdq %%xmm1, %%xmm0, %%xmm12\n\t"
|
|
"vpunpckhdq %%xmm3, %%xmm2, %%xmm13\n\t"
|
|
"vpunpckldq %%xmm5, %%xmm4, %%xmm10\n\t"
|
|
"vpunpckldq %%xmm7, %%xmm6, %%xmm11\n\t"
|
|
"vpunpckhdq %%xmm5, %%xmm4, %%xmm14\n\t"
|
|
"vpunpckhdq %%xmm7, %%xmm6, %%xmm15\n\t"
|
|
"vpunpcklqdq %%xmm9, %%xmm8, %%xmm0\n\t"
|
|
"vpunpcklqdq %%xmm11, %%xmm10, %%xmm1\n\t"
|
|
"vpunpckhqdq %%xmm9, %%xmm8, %%xmm2\n\t"
|
|
"vpunpckhqdq %%xmm11, %%xmm10, %%xmm3\n\t"
|
|
"vpunpcklqdq %%xmm13, %%xmm12, %%xmm4\n\t"
|
|
"vpunpcklqdq %%xmm15, %%xmm14, %%xmm5\n\t"
|
|
"vpunpckhqdq %%xmm13, %%xmm12, %%xmm6\n\t"
|
|
"vpunpckhqdq %%xmm15, %%xmm14, %%xmm7\n\t"
|
|
"vmovdqu (%[in]), %%xmm8\n\t"
|
|
"vmovdqu 16(%[in]), %%xmm9\n\t"
|
|
"vmovdqu 64(%[in]), %%xmm10\n\t"
|
|
"vmovdqu 80(%[in]), %%xmm11\n\t"
|
|
"vmovdqu 128(%[in]), %%xmm12\n\t"
|
|
"vmovdqu 144(%[in]), %%xmm13\n\t"
|
|
"vmovdqu 192(%[in]), %%xmm14\n\t"
|
|
"vmovdqu 208(%[in]), %%xmm15\n\t"
|
|
"vpxor %%xmm8, %%xmm0, %%xmm0\n\t"
|
|
"vpxor %%xmm9, %%xmm1, %%xmm1\n\t"
|
|
"vpxor %%xmm10, %%xmm2, %%xmm2\n\t"
|
|
"vpxor %%xmm11, %%xmm3, %%xmm3\n\t"
|
|
"vpxor %%xmm12, %%xmm4, %%xmm4\n\t"
|
|
"vpxor %%xmm13, %%xmm5, %%xmm5\n\t"
|
|
"vpxor %%xmm14, %%xmm6, %%xmm6\n\t"
|
|
"vpxor %%xmm15, %%xmm7, %%xmm7\n\t"
|
|
"vmovdqu %%xmm0, (%[out])\n\t"
|
|
"vmovdqu %%xmm1, 16(%[out])\n\t"
|
|
"vmovdqu %%xmm2, 64(%[out])\n\t"
|
|
"vmovdqu %%xmm3, 80(%[out])\n\t"
|
|
"vmovdqu %%xmm4, 128(%[out])\n\t"
|
|
"vmovdqu %%xmm5, 144(%[out])\n\t"
|
|
"vmovdqu %%xmm6, 192(%[out])\n\t"
|
|
"vmovdqu %%xmm7, 208(%[out])\n\t"
|
|
|
|
"vmovdqa (%[x]), %%xmm0\n\t"
|
|
"vmovdqa 16(%[x]), %%xmm1\n\t"
|
|
"vmovdqa 32(%[x]), %%xmm2\n\t"
|
|
"vmovdqa 48(%[x]), %%xmm3\n\t"
|
|
"vmovdqa 64(%[x]), %%xmm4\n\t"
|
|
"vmovdqa 80(%[x]), %%xmm5\n\t"
|
|
"vmovdqa 96(%[x]), %%xmm6\n\t"
|
|
"vmovdqa 112(%[x]), %%xmm7\n\t"
|
|
|
|
"vpunpckldq %%xmm1, %%xmm0, %%xmm8\n\t"
|
|
"vpunpckldq %%xmm3, %%xmm2, %%xmm9\n\t"
|
|
"vpunpckhdq %%xmm1, %%xmm0, %%xmm12\n\t"
|
|
"vpunpckhdq %%xmm3, %%xmm2, %%xmm13\n\t"
|
|
"vpunpckldq %%xmm5, %%xmm4, %%xmm10\n\t"
|
|
"vpunpckldq %%xmm7, %%xmm6, %%xmm11\n\t"
|
|
"vpunpckhdq %%xmm5, %%xmm4, %%xmm14\n\t"
|
|
"vpunpckhdq %%xmm7, %%xmm6, %%xmm15\n\t"
|
|
"vpunpcklqdq %%xmm9, %%xmm8, %%xmm0\n\t"
|
|
"vpunpcklqdq %%xmm11, %%xmm10, %%xmm1\n\t"
|
|
"vpunpckhqdq %%xmm9, %%xmm8, %%xmm2\n\t"
|
|
"vpunpckhqdq %%xmm11, %%xmm10, %%xmm3\n\t"
|
|
"vpunpcklqdq %%xmm13, %%xmm12, %%xmm4\n\t"
|
|
"vpunpcklqdq %%xmm15, %%xmm14, %%xmm5\n\t"
|
|
"vpunpckhqdq %%xmm13, %%xmm12, %%xmm6\n\t"
|
|
"vpunpckhqdq %%xmm15, %%xmm14, %%xmm7\n\t"
|
|
"vmovdqu 32(%[in]), %%xmm8\n\t"
|
|
"vmovdqu 48(%[in]), %%xmm9\n\t"
|
|
"vmovdqu 96(%[in]), %%xmm10\n\t"
|
|
"vmovdqu 112(%[in]), %%xmm11\n\t"
|
|
"vmovdqu 160(%[in]), %%xmm12\n\t"
|
|
"vmovdqu 176(%[in]), %%xmm13\n\t"
|
|
"vmovdqu 224(%[in]), %%xmm14\n\t"
|
|
"vmovdqu 240(%[in]), %%xmm15\n\t"
|
|
"vpxor %%xmm8, %%xmm0, %%xmm0\n\t"
|
|
"vpxor %%xmm9, %%xmm1, %%xmm1\n\t"
|
|
"vpxor %%xmm10, %%xmm2, %%xmm2\n\t"
|
|
"vpxor %%xmm11, %%xmm3, %%xmm3\n\t"
|
|
"vpxor %%xmm12, %%xmm4, %%xmm4\n\t"
|
|
"vpxor %%xmm13, %%xmm5, %%xmm5\n\t"
|
|
"vpxor %%xmm14, %%xmm6, %%xmm6\n\t"
|
|
"vpxor %%xmm15, %%xmm7, %%xmm7\n\t"
|
|
"vmovdqu %%xmm0, 32(%[out])\n\t"
|
|
"vmovdqu %%xmm1, 48(%[out])\n\t"
|
|
"vmovdqu %%xmm2, 96(%[out])\n\t"
|
|
"vmovdqu %%xmm3, 112(%[out])\n\t"
|
|
"vmovdqu %%xmm4, 160(%[out])\n\t"
|
|
"vmovdqu %%xmm5, 176(%[out])\n\t"
|
|
"vmovdqu %%xmm6, 224(%[out])\n\t"
|
|
"vmovdqu %%xmm7, 240(%[out])\n\t"
|
|
|
|
"vmovdqa 192(%[X]), %%xmm12\n\t"
|
|
"add $256, %[in]\n\t"
|
|
"add $256, %[out]\n\t"
|
|
"vpaddd %[four], %%xmm12, %%xmm12\n\t"
|
|
"sub $256, %[bytes]\n\t"
|
|
"vmovdqa %%xmm12, 192(%[X])\n\t"
|
|
"cmp $256, %[bytes]\n\t"
|
|
"jl L_done\n\t"
|
|
|
|
"vmovdqa (%[X]), %%xmm0\n\t"
|
|
"vmovdqa 16(%[X]), %%xmm1\n\t"
|
|
"vmovdqa 32(%[X]), %%xmm2\n\t"
|
|
"vmovdqa 48(%[X]), %%xmm3\n\t"
|
|
"vmovdqa 64(%[X]), %%xmm4\n\t"
|
|
"vmovdqa 80(%[X]), %%xmm5\n\t"
|
|
"vmovdqa 96(%[X]), %%xmm6\n\t"
|
|
"vmovdqa 112(%[X]), %%xmm7\n\t"
|
|
"vmovdqa 128(%[X]), %%xmm8\n\t"
|
|
"vmovdqa 144(%[X]), %%xmm9\n\t"
|
|
"vmovdqa 160(%[X]), %%xmm10\n\t"
|
|
"vmovdqa 176(%[X]), %%xmm11\n\t"
|
|
"vmovdqa 192(%[X]), %%xmm12\n\t"
|
|
"vmovdqa 208(%[X]), %%xmm13\n\t"
|
|
"vmovdqa 224(%[X]), %%xmm14\n\t"
|
|
"vmovdqa 240(%[X]), %%xmm15\n\t"
|
|
"jmp L_enc128_loop\n\t"
|
|
|
|
"\n"
|
|
"L_done:\n\t"
|
|
|
|
"shl $2, %[cnt]\n\t"
|
|
"add 48(%[key]), %[cnt]\n\t"
|
|
"movl %[cnt], 48(%[key])\n\t"
|
|
"\n"
|
|
"L_end128:\n\t"
|
|
: [bytes] "+r" (bytes), [cnt] "+r" (cnt),
|
|
[in] "+r" (m), [out] "+r" (c)
|
|
: [X] "r" (X), [x] "r" (x), [key] "r" (ctx->X),
|
|
[add] "xrm" (add), [four] "xrm" (four),
|
|
[rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16)
|
|
: "xmm0", "xmm1", "xmm2", "xmm3",
|
|
"xmm4", "xmm5", "xmm6", "xmm7",
|
|
"xmm8", "xmm9", "xmm10", "xmm11",
|
|
"xmm12", "xmm13", "xmm14", "xmm15", "memory"
|
|
);
|
|
|
|
for (; bytes >= CHACHA_CHUNK_BYTES;) {
|
|
CHACHA_CHUNK_AVX();
|
|
bytes -= CHACHA_CHUNK_BYTES;
|
|
c += CHACHA_CHUNK_BYTES;
|
|
m += CHACHA_CHUNK_BYTES;
|
|
}
|
|
if (bytes > 0) {
|
|
CHACHA_PARTIAL_CHUNK_AVX();
|
|
}
|
|
}
|
|
#endif /* HAVE_INTEL_AVX1 */
|
|
|
|
#ifdef HAVE_INTEL_AVX2
|
|
#define QUARTERROUND_2_AVX2() \
|
|
"vpaddd %%xmm1, %%xmm0, %%xmm0\n\t" \
|
|
"vpxor %%xmm0, %%xmm3, %%xmm3\n\t" \
|
|
"vpshufb %[rotl16], %%xmm3, %%xmm3\n\t" \
|
|
"vpaddd %%xmm3, %%xmm2, %%xmm2\n\t" \
|
|
"vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \
|
|
"vpsrld $20, %%xmm1, %%xmm4\n\t" \
|
|
"vpslld $12, %%xmm1, %%xmm1\n\t" \
|
|
"vpxor %%xmm4, %%xmm1, %%xmm1\n\t" \
|
|
"vpaddd %%xmm1, %%xmm0, %%xmm0\n\t" \
|
|
"vpxor %%xmm0, %%xmm3, %%xmm3\n\t" \
|
|
"vpshufb %[rotl8], %%xmm3, %%xmm3\n\t" \
|
|
"vpaddd %%xmm3, %%xmm2, %%xmm2\n\t" \
|
|
"vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \
|
|
"vpsrld $25, %%xmm1, %%xmm4\n\t" \
|
|
"vpslld $7, %%xmm1, %%xmm1\n\t" \
|
|
"vpxor %%xmm4, %%xmm1, %%xmm1\n\t" \
|
|
"# Swap words for next round\n\t" \
|
|
"vpshufd $0x39, %%xmm1, %%xmm1\n\t" \
|
|
"vpshufd $0x4e, %%xmm2, %%xmm2\n\t" \
|
|
"vpshufd $0x93, %%xmm3, %%xmm3\n\t" \
|
|
"vpaddd %%xmm1, %%xmm0, %%xmm0\n\t" \
|
|
"vpxor %%xmm0, %%xmm3, %%xmm3\n\t" \
|
|
"vpshufb %[rotl16], %%xmm3, %%xmm3\n\t" \
|
|
"vpaddd %%xmm3, %%xmm2, %%xmm2\n\t" \
|
|
"vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \
|
|
"vpsrld $20, %%xmm1, %%xmm4\n\t" \
|
|
"vpslld $12, %%xmm1, %%xmm1\n\t" \
|
|
"vpxor %%xmm4, %%xmm1, %%xmm1\n\t" \
|
|
"vpaddd %%xmm1, %%xmm0, %%xmm0\n\t" \
|
|
"vpxor %%xmm0, %%xmm3, %%xmm3\n\t" \
|
|
"vpshufb %[rotl8], %%xmm3, %%xmm3\n\t" \
|
|
"vpaddd %%xmm3, %%xmm2, %%xmm2\n\t" \
|
|
"vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \
|
|
"vpsrld $25, %%Xmm1, %%xmm4\n\t" \
|
|
"vpslld $7, %%xmm1, %%xmm1\n\t" \
|
|
"vpxor %%xmm4, %%xmm1, %%xmm1\n\t" \
|
|
"# Swap words back\n\t" \
|
|
"vpshufd $0x93, %%xmm1, %%xmm1\n\t" \
|
|
"vpshufd $0x4e, %%xmm2, %%xmm2\n\t" \
|
|
"vpshufd $0x39, %%xmm3, %%xmm3\n\t" \
|
|
|
|
#define CHACHA_CRYPT_AVX2() \
|
|
"vmovdqu 0(%[input]), %%xmm8\n\t" \
|
|
"vmovdqu 16(%[input]), %%xmm9\n\t" \
|
|
"vmovdqu 32(%[input]), %%xmm10\n\t" \
|
|
"vmovdqu 48(%[input]), %%xmm11\n\t" \
|
|
"vmovdqu %%xmm8, %%xmm0\n\t" \
|
|
"vmovdqu %%xmm9, %%xmm1\n\t" \
|
|
"vmovdqu %%xmm10, %%xmm2\n\t" \
|
|
"vmovdqu %%xmm11, %%xmm3\n\t" \
|
|
"movb $10, %%al\n\t" \
|
|
"\n" \
|
|
"1:\n\t" \
|
|
QUARTERROUND_2_AVX2() \
|
|
"decb %%al\n\t" \
|
|
"jnz 1b\n\t" \
|
|
"vpaddd %%xmm8, %%xmm0, %%xmm0\n\t" \
|
|
"vpaddd %%xmm9, %%xmm1, %%xmm1\n\t" \
|
|
"vpaddd %%xmm10, %%xmm2, %%xmm2\n\t" \
|
|
"vpaddd %%xmm11, %%xmm3, %%xmm3\n\t" \
|
|
|
|
#define CHACHA_PARTIAL_CHUNK_AVX2() \
|
|
__asm__ __volatile__ ( \
|
|
CHACHA_CRYPT_AVX2() \
|
|
"vmovdqu %%xmm0, 0(%[c])\n\t" \
|
|
"vmovdqu %%xmm1, 16(%[c])\n\t" \
|
|
"vmovdqu %%xmm2, 32(%[c])\n\t" \
|
|
"vmovdqu %%xmm3, 48(%[c])\n\t" \
|
|
"addl $1, 48(%[input])\n\t" \
|
|
"movl %[bytes], %%r8d\n\t" \
|
|
"xorq %%rdx, %%rdx\n\t" \
|
|
"movl %%r8d, %%r9d\n\t" \
|
|
"andl $7, %%r9d\n\t" \
|
|
"jz 4f\n\t" \
|
|
"\n" \
|
|
"2:\n\t" \
|
|
"movzbl (%[c],%%rdx,1), %%ecx\n\t" \
|
|
"xorb (%[m],%%rdx,1), %%cl\n\t" \
|
|
"movb %%cl, (%[output],%%rdx,1)\n\t" \
|
|
"incl %%edx\n\t" \
|
|
"cmpl %%r9d, %%edx\n\t" \
|
|
"jne 2b\n\t" \
|
|
"je 3f\n\t" \
|
|
"\n" \
|
|
"4:\n\t" \
|
|
"movq (%[c],%%rdx,1), %%rcx\n\t" \
|
|
"xorq (%[m],%%rdx,1), %%rcx\n\t" \
|
|
"movq %%rcx, (%[output],%%rdx,1)\n\t" \
|
|
"addl $8, %%edx\n\t" \
|
|
"\n" \
|
|
"3:\n\t" \
|
|
"cmpl %%r8d, %%edx\n\t" \
|
|
"jne 4b\n\t" \
|
|
: \
|
|
: [input] "r" (ctx->X), [c] "r" (x), \
|
|
[output] "r" (c), [bytes] "r" (bytes), [m] "r" (m), \
|
|
[rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) \
|
|
: "eax", "ecx", "edx", "r8", "r9", "memory", \
|
|
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", \
|
|
"xmm8", "xmm9", "xmm10", "xmm11" \
|
|
)
|
|
|
|
|
|
#define CHACHA_CHUNK_AVX2() \
|
|
__asm__ __volatile__ ( \
|
|
CHACHA_CRYPT_AVX2() \
|
|
"vmovdqu 0(%[m]), %%xmm4\n\t" \
|
|
"vmovdqu 16(%[m]), %%xmm5\n\t" \
|
|
"vmovdqu 32(%[m]), %%xmm6\n\t" \
|
|
"vmovdqu 48(%[m]), %%xmm7\n\t" \
|
|
"vpxor %%xmm4, %%xmm0, %%xmm0\n\t" \
|
|
"vpxor %%xmm5, %%xmm1, %%xmm1\n\t" \
|
|
"vpxor %%xmm6, %%xmm2, %%xmm2\n\t" \
|
|
"vpxor %%xmm7, %%xmm3, %%xmm3\n\t" \
|
|
"vmovdqu %%xmm0, 0(%[c])\n\t" \
|
|
"vmovdqu %%xmm1, 16(%[c])\n\t" \
|
|
"vmovdqu %%xmm2, 32(%[c])\n\t" \
|
|
"vmovdqu %%xmm3, 48(%[c])\n\t" \
|
|
"addl $1, 48(%[input])\n\t" \
|
|
: \
|
|
: [input] "r" (ctx->X), [c] "r" (c), [m] "r" (m), \
|
|
[rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) \
|
|
: "rax", "memory", \
|
|
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", \
|
|
"xmm8", "xmm9", "xmm10", "xmm11" \
|
|
)
|
|
|
|
|
|
static void chacha_encrypt_avx2(ChaCha* ctx, const byte* m, byte* c,
|
|
word32 bytes)
|
|
{
|
|
ALIGN256 word32 X[8*CHACHA_CHUNK_WORDS]; /* used to make sure aligned */
|
|
ALIGN256 word32 x[4*CHACHA_CHUNK_WORDS]; /* used to make sure aligned */
|
|
word32 cnt = 0;
|
|
static const __m256i add = { 0x0000000100000000UL,0x0000000300000002UL,
|
|
0x0000000500000004UL,0x0000000700000006UL };
|
|
static const __m256i eight = { 0x0000000800000008UL,0x0000000800000008UL,
|
|
0x0000000800000008UL,0x0000000800000008UL };
|
|
static const __m256i rotl8_256 =
|
|
{ 0x0605040702010003UL,0x0e0d0c0f0a09080bUL,
|
|
0x0605040702010003UL,0x0e0d0c0f0a09080bUL };
|
|
static const __m256i rotl16_256 =
|
|
{ 0x0504070601000302UL,0x0d0c0f0e09080b0aUL,
|
|
0x0504070601000302UL,0x0d0c0f0e09080b0aUL };
|
|
|
|
if (bytes == 0)
|
|
return;
|
|
|
|
__asm__ __volatile__ (
|
|
"movl %[bytes], %[cnt]\n\t"
|
|
"shrl $9, %[cnt]\n\t"
|
|
"jz L_end256\n\t"
|
|
|
|
"vpbroadcastd (%[key]), %%ymm0\n\t"
|
|
"vpbroadcastd 4(%[key]), %%ymm1\n\t"
|
|
"vpbroadcastd 8(%[key]), %%ymm2\n\t"
|
|
"vpbroadcastd 12(%[key]), %%ymm3\n\t"
|
|
"vpbroadcastd 16(%[key]), %%ymm4\n\t"
|
|
"vpbroadcastd 20(%[key]), %%ymm5\n\t"
|
|
"vpbroadcastd 24(%[key]), %%ymm6\n\t"
|
|
"vpbroadcastd 28(%[key]), %%ymm7\n\t"
|
|
"vpbroadcastd 32(%[key]), %%ymm8\n\t"
|
|
"vpbroadcastd 36(%[key]), %%ymm9\n\t"
|
|
"vpbroadcastd 40(%[key]), %%ymm10\n\t"
|
|
"vpbroadcastd 44(%[key]), %%ymm11\n\t"
|
|
"vpbroadcastd 48(%[key]), %%ymm12\n\t"
|
|
"vpbroadcastd 52(%[key]), %%ymm13\n\t"
|
|
"vpbroadcastd 56(%[key]), %%ymm14\n\t"
|
|
"vpbroadcastd 60(%[key]), %%ymm15\n\t"
|
|
|
|
"vpaddd %[add], %%ymm12, %%ymm12\n\t"
|
|
|
|
"vmovdqa %%ymm0, (%[X])\n\t"
|
|
"vmovdqa %%ymm1, 32(%[X])\n\t"
|
|
"vmovdqa %%ymm2, 64(%[X])\n\t"
|
|
"vmovdqa %%ymm3, 96(%[X])\n\t"
|
|
"vmovdqa %%ymm4, 128(%[X])\n\t"
|
|
"vmovdqa %%ymm5, 160(%[X])\n\t"
|
|
"vmovdqa %%ymm6, 192(%[X])\n\t"
|
|
"vmovdqa %%ymm7, 224(%[X])\n\t"
|
|
"vmovdqa %%ymm8, 256(%[X])\n\t"
|
|
"vmovdqa %%ymm9, 288(%[X])\n\t"
|
|
"vmovdqa %%ymm10, 320(%[X])\n\t"
|
|
"vmovdqa %%ymm11, 352(%[X])\n\t"
|
|
"vmovdqa %%ymm12, 384(%[X])\n\t"
|
|
"vmovdqa %%ymm13, 416(%[X])\n\t"
|
|
"vmovdqa %%ymm14, 448(%[X])\n\t"
|
|
"vmovdqa %%ymm15, 480(%[X])\n\t"
|
|
"\n"
|
|
"L_enc256_loop:\n\t"
|
|
"vmovdqa %%ymm11, 96(%[x])\n\t"
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
QUARTERROUND_YMM()
|
|
QUARTERROUND_YMM_2()
|
|
"vmovdqa 96(%[x]), %%ymm11\n\t"
|
|
|
|
"vpaddd (%[X]), %%ymm0, %%ymm0\n\t"
|
|
"vpaddd 32(%[X]), %%ymm1, %%ymm1\n\t"
|
|
"vpaddd 64(%[X]), %%ymm2, %%ymm2\n\t"
|
|
"vpaddd 96(%[X]), %%ymm3, %%ymm3\n\t"
|
|
"vpaddd 128(%[X]), %%ymm4, %%ymm4\n\t"
|
|
"vpaddd 160(%[X]), %%ymm5, %%ymm5\n\t"
|
|
"vpaddd 192(%[X]), %%ymm6, %%ymm6\n\t"
|
|
"vpaddd 224(%[X]), %%ymm7, %%ymm7\n\t"
|
|
"vpaddd 256(%[X]), %%ymm8, %%ymm8\n\t"
|
|
"vpaddd 288(%[X]), %%ymm9, %%ymm9\n\t"
|
|
"vpaddd 320(%[X]), %%ymm10, %%ymm10\n\t"
|
|
"vpaddd 352(%[X]), %%ymm11, %%ymm11\n\t"
|
|
"vpaddd 384(%[X]), %%ymm12, %%ymm12\n\t"
|
|
"vpaddd 416(%[X]), %%ymm13, %%ymm13\n\t"
|
|
"vpaddd 448(%[X]), %%ymm14, %%ymm14\n\t"
|
|
"vpaddd 480(%[X]), %%ymm15, %%ymm15\n\t"
|
|
|
|
"vmovdqa %%ymm8, (%[x])\n\t"
|
|
"vmovdqa %%ymm9, 32(%[x])\n\t"
|
|
"vmovdqa %%ymm10, 64(%[x])\n\t"
|
|
"vmovdqa %%ymm11, 96(%[x])\n\t"
|
|
"vmovdqa %%ymm12, 128(%[x])\n\t"
|
|
"vmovdqa %%ymm13, 160(%[x])\n\t"
|
|
"vmovdqa %%ymm14, 192(%[x])\n\t"
|
|
"vmovdqa %%ymm15, 224(%[x])\n\t"
|
|
|
|
"vpunpckldq %%ymm1, %%ymm0, %%ymm8\n\t"
|
|
"vpunpckldq %%ymm3, %%ymm2, %%ymm9\n\t"
|
|
"vpunpckhdq %%ymm1, %%ymm0, %%ymm12\n\t"
|
|
"vpunpckhdq %%ymm3, %%ymm2, %%ymm13\n\t"
|
|
"vpunpckldq %%ymm5, %%ymm4, %%ymm10\n\t"
|
|
"vpunpckldq %%ymm7, %%ymm6, %%ymm11\n\t"
|
|
"vpunpckhdq %%ymm5, %%ymm4, %%ymm14\n\t"
|
|
"vpunpckhdq %%ymm7, %%ymm6, %%ymm15\n\t"
|
|
"vpunpcklqdq %%ymm9, %%ymm8, %%ymm0\n\t"
|
|
"vpunpcklqdq %%ymm11, %%ymm10, %%ymm1\n\t"
|
|
"vpunpckhqdq %%ymm9, %%ymm8, %%ymm2\n\t"
|
|
"vpunpckhqdq %%ymm11, %%ymm10, %%ymm3\n\t"
|
|
"vpunpcklqdq %%ymm13, %%ymm12, %%ymm4\n\t"
|
|
"vpunpcklqdq %%ymm15, %%ymm14, %%ymm5\n\t"
|
|
"vpunpckhqdq %%ymm13, %%ymm12, %%ymm6\n\t"
|
|
"vpunpckhqdq %%ymm15, %%ymm14, %%ymm7\n\t"
|
|
"vperm2i128 $0x20, %%ymm1, %%ymm0, %%ymm8\n\t"
|
|
"vperm2i128 $0x20, %%ymm3, %%ymm2, %%ymm9\n\t"
|
|
"vperm2i128 $0x31, %%ymm1, %%ymm0, %%ymm12\n\t"
|
|
"vperm2i128 $0x31, %%ymm3, %%ymm2, %%ymm13\n\t"
|
|
"vperm2i128 $0x20, %%ymm5, %%ymm4, %%ymm10\n\t"
|
|
"vperm2i128 $0x20, %%ymm7, %%ymm6, %%ymm11\n\t"
|
|
"vperm2i128 $0x31, %%ymm5, %%ymm4, %%ymm14\n\t"
|
|
"vperm2i128 $0x31, %%ymm7, %%ymm6, %%ymm15\n\t"
|
|
|
|
"vmovdqu (%[in]), %%ymm0\n\t"
|
|
"vmovdqu 64(%[in]), %%ymm1\n\t"
|
|
"vmovdqu 128(%[in]), %%ymm2\n\t"
|
|
"vmovdqu 192(%[in]), %%ymm3\n\t"
|
|
"vmovdqu 256(%[in]), %%ymm4\n\t"
|
|
"vmovdqu 320(%[in]), %%ymm5\n\t"
|
|
"vmovdqu 384(%[in]), %%ymm6\n\t"
|
|
"vmovdqu 448(%[in]), %%ymm7\n\t"
|
|
"vpxor %%ymm0, %%ymm8, %%ymm8\n\t"
|
|
"vpxor %%ymm1, %%ymm9, %%ymm9\n\t"
|
|
"vpxor %%ymm2, %%ymm10, %%ymm10\n\t"
|
|
"vpxor %%ymm3, %%ymm11, %%ymm11\n\t"
|
|
"vpxor %%ymm4, %%ymm12, %%ymm12\n\t"
|
|
"vpxor %%ymm5, %%ymm13, %%ymm13\n\t"
|
|
"vpxor %%ymm6, %%ymm14, %%ymm14\n\t"
|
|
"vpxor %%ymm7, %%ymm15, %%ymm15\n\t"
|
|
"vmovdqu %%ymm8, (%[out])\n\t"
|
|
"vmovdqu %%ymm9, 64(%[out])\n\t"
|
|
"vmovdqu %%ymm10, 128(%[out])\n\t"
|
|
"vmovdqu %%ymm11, 192(%[out])\n\t"
|
|
"vmovdqu %%ymm12, 256(%[out])\n\t"
|
|
"vmovdqu %%ymm13, 320(%[out])\n\t"
|
|
"vmovdqu %%ymm14, 384(%[out])\n\t"
|
|
"vmovdqu %%ymm15, 448(%[out])\n\t"
|
|
|
|
"vmovdqa (%[x]), %%ymm0\n\t"
|
|
"vmovdqa 32(%[x]), %%ymm1\n\t"
|
|
"vmovdqa 64(%[x]), %%ymm2\n\t"
|
|
"vmovdqa 96(%[x]), %%ymm3\n\t"
|
|
"vmovdqa 128(%[x]), %%ymm4\n\t"
|
|
"vmovdqa 160(%[x]), %%ymm5\n\t"
|
|
"vmovdqa 192(%[x]), %%ymm6\n\t"
|
|
"vmovdqa 224(%[x]), %%ymm7\n\t"
|
|
|
|
"vpunpckldq %%ymm1, %%ymm0, %%ymm8\n\t"
|
|
"vpunpckldq %%ymm3, %%ymm2, %%ymm9\n\t"
|
|
"vpunpckhdq %%ymm1, %%ymm0, %%ymm12\n\t"
|
|
"vpunpckhdq %%ymm3, %%ymm2, %%ymm13\n\t"
|
|
"vpunpckldq %%ymm5, %%ymm4, %%ymm10\n\t"
|
|
"vpunpckldq %%ymm7, %%ymm6, %%ymm11\n\t"
|
|
"vpunpckhdq %%ymm5, %%ymm4, %%ymm14\n\t"
|
|
"vpunpckhdq %%ymm7, %%ymm6, %%ymm15\n\t"
|
|
"vpunpcklqdq %%ymm9, %%ymm8, %%ymm0\n\t"
|
|
"vpunpcklqdq %%ymm11, %%ymm10, %%ymm1\n\t"
|
|
"vpunpckhqdq %%ymm9 , %%ymm8, %%ymm2\n\t"
|
|
"vpunpckhqdq %%ymm11, %%ymm10, %%ymm3\n\t"
|
|
"vpunpcklqdq %%ymm13, %%ymm12, %%ymm4\n\t"
|
|
"vpunpcklqdq %%ymm15, %%ymm14, %%ymm5\n\t"
|
|
"vpunpckhqdq %%ymm13, %%ymm12, %%ymm6\n\t"
|
|
"vpunpckhqdq %%ymm15, %%ymm14, %%ymm7\n\t"
|
|
"vperm2i128 $0x20, %%ymm1, %%ymm0, %%ymm8\n\t"
|
|
"vperm2i128 $0x20, %%ymm3, %%ymm2, %%ymm9\n\t"
|
|
"vperm2i128 $0x31, %%ymm1, %%ymm0, %%ymm12\n\t"
|
|
"vperm2i128 $0x31, %%ymm3, %%ymm2, %%ymm13\n\t"
|
|
"vperm2i128 $0x20, %%ymm5, %%ymm4, %%ymm10\n\t"
|
|
"vperm2i128 $0x20, %%ymm7, %%ymm6, %%ymm11\n\t"
|
|
"vperm2i128 $0x31, %%ymm5, %%ymm4, %%ymm14\n\t"
|
|
"vperm2i128 $0x31, %%ymm7, %%ymm6, %%ymm15\n\t"
|
|
|
|
"vmovdqu 32(%[in]), %%ymm0\n\t"
|
|
"vmovdqu 96(%[in]), %%ymm1\n\t"
|
|
"vmovdqu 160(%[in]), %%ymm2\n\t"
|
|
"vmovdqu 224(%[in]), %%ymm3\n\t"
|
|
"vmovdqu 288(%[in]), %%ymm4\n\t"
|
|
"vmovdqu 352(%[in]), %%ymm5\n\t"
|
|
"vmovdqu 416(%[in]), %%ymm6\n\t"
|
|
"vmovdqu 480(%[in]), %%ymm7\n\t"
|
|
"vpxor %%ymm0, %%ymm8, %%ymm8\n\t"
|
|
"vpxor %%ymm1, %%ymm9, %%ymm9\n\t"
|
|
"vpxor %%ymm2, %%ymm10, %%ymm10\n\t"
|
|
"vpxor %%ymm3, %%ymm11, %%ymm11\n\t"
|
|
"vpxor %%ymm4, %%ymm12, %%ymm12\n\t"
|
|
"vpxor %%ymm5, %%ymm13, %%ymm13\n\t"
|
|
"vpxor %%ymm6, %%ymm14, %%ymm14\n\t"
|
|
"vpxor %%ymm7, %%ymm15, %%ymm15\n\t"
|
|
"vmovdqu %%ymm8, 32(%[out])\n\t"
|
|
"vmovdqu %%ymm9, 96(%[out])\n\t"
|
|
"vmovdqu %%ymm10, 160(%[out])\n\t"
|
|
"vmovdqu %%ymm11, 224(%[out])\n\t"
|
|
"vmovdqu %%ymm12, 288(%[out])\n\t"
|
|
"vmovdqu %%ymm13, 352(%[out])\n\t"
|
|
"vmovdqu %%ymm14, 416(%[out])\n\t"
|
|
"vmovdqu %%ymm15, 480(%[out])\n\t"
|
|
|
|
"vmovdqa 384(%[X]), %%ymm12\n\t"
|
|
"add $512, %[in]\n\t"
|
|
"add $512, %[out]\n\t"
|
|
"vpaddd %[eight], %%ymm12, %%ymm12\n\t"
|
|
"sub $512, %[bytes]\n\t"
|
|
"vmovdqa %%ymm12, 384(%[X])\n\t"
|
|
"cmp $512, %[bytes]\n\t"
|
|
"jl L_done256\n\t"
|
|
|
|
"vmovdqa (%[X]), %%ymm0\n\t"
|
|
"vmovdqa 32(%[X]), %%ymm1\n\t"
|
|
"vmovdqa 64(%[X]), %%ymm2\n\t"
|
|
"vmovdqa 96(%[X]), %%ymm3\n\t"
|
|
"vmovdqa 128(%[X]), %%ymm4\n\t"
|
|
"vmovdqa 160(%[X]), %%ymm5\n\t"
|
|
"vmovdqa 192(%[X]), %%ymm6\n\t"
|
|
"vmovdqa 224(%[X]), %%ymm7\n\t"
|
|
"vmovdqa 256(%[X]), %%ymm8\n\t"
|
|
"vmovdqa 288(%[X]), %%ymm9\n\t"
|
|
"vmovdqa 320(%[X]), %%ymm10\n\t"
|
|
"vmovdqa 352(%[X]), %%ymm11\n\t"
|
|
"vmovdqa 384(%[X]), %%ymm12\n\t"
|
|
"vmovdqa 416(%[X]), %%ymm13\n\t"
|
|
"vmovdqa 448(%[X]), %%ymm14\n\t"
|
|
"vmovdqa 480(%[X]), %%ymm15\n\t"
|
|
"jmp L_enc256_loop\n\t"
|
|
"\n"
|
|
"L_done256:\n\t"
|
|
"shl $3, %[cnt]\n\t"
|
|
"add 48(%[key]), %[cnt]\n\t"
|
|
"movl %[cnt], 48(%[key])\n\t"
|
|
"\n"
|
|
"L_end256:\n\t"
|
|
: [bytes] "+r" (bytes), [cnt] "+r" (cnt),
|
|
[in] "+r" (m), [out] "+r" (c)
|
|
: [X] "r" (X), [x] "r" (x), [key] "r" (ctx->X),
|
|
[add] "m" (add), [eight] "m" (eight),
|
|
[rotl8] "m" (rotl8_256), [rotl16] "m" (rotl16_256)
|
|
: "ymm0", "ymm1", "ymm2", "ymm3",
|
|
"ymm4", "ymm5", "ymm6", "ymm7",
|
|
"ymm8", "ymm9", "ymm10", "ymm11",
|
|
"ymm12", "ymm13", "ymm14", "ymm15", "memory"
|
|
);
|
|
|
|
/* AVX code optimised for multiples of 256 bytes. */
|
|
if (bytes == 256) {
|
|
chacha_encrypt_avx(ctx, m, c, bytes);
|
|
bytes -= 256;
|
|
}
|
|
|
|
for (; bytes >= CHACHA_CHUNK_BYTES;) {
|
|
CHACHA_CHUNK_AVX2();
|
|
bytes -= CHACHA_CHUNK_BYTES;
|
|
c += CHACHA_CHUNK_BYTES;
|
|
m += CHACHA_CHUNK_BYTES;
|
|
}
|
|
if (bytes > 0) {
|
|
CHACHA_PARTIAL_CHUNK_AVX2();
|
|
}
|
|
}
|
|
#endif /* HAVE_INTEL_AVX2 */
|
|
#endif /* USE_INTEL_CHACHA_SPEEDUP */
|
|
|
|
/**
|
|
* Encrypt a stream of bytes
|
|
*/
|
|
static void wc_Chacha_encrypt_bytes(ChaCha* ctx, const byte* m, byte* c,
|
|
word32 bytes)
|
|
{
|
|
byte* output;
|
|
word32 temp[CHACHA_CHUNK_WORDS]; /* used to make sure aligned */
|
|
word32 i;
|
|
|
|
output = (byte*)temp;
|
|
|
|
for (; bytes > 0;) {
|
|
wc_Chacha_wordtobyte(temp, ctx->X);
|
|
ctx->X[CHACHA_IV_BYTES] = PLUSONE(ctx->X[CHACHA_IV_BYTES]);
|
|
if (bytes <= CHACHA_CHUNK_BYTES) {
|
|
for (i = 0; i < bytes; ++i) {
|
|
c[i] = m[i] ^ output[i];
|
|
}
|
|
return;
|
|
}
|
|
for (i = 0; i < CHACHA_CHUNK_BYTES; ++i) {
|
|
c[i] = m[i] ^ output[i];
|
|
}
|
|
bytes -= CHACHA_CHUNK_BYTES;
|
|
c += CHACHA_CHUNK_BYTES;
|
|
m += CHACHA_CHUNK_BYTES;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* API to encrypt/decrypt a message of any size.
|
|
*/
|
|
int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input,
|
|
word32 msglen)
|
|
{
|
|
if (ctx == NULL)
|
|
return BAD_FUNC_ARG;
|
|
|
|
#ifdef USE_INTEL_CHACHA_SPEEDUP
|
|
if (!cpuidFlagsSet) {
|
|
cpuidFlags = cpuid_get_flags();
|
|
cpuidFlagsSet = 1;
|
|
}
|
|
|
|
#ifdef HAVE_INTEL_AVX2
|
|
if (IS_INTEL_AVX2(cpuidFlags)) {
|
|
chacha_encrypt_avx2(ctx, input, output, msglen);
|
|
return 0;
|
|
}
|
|
#endif
|
|
if (IS_INTEL_AVX1(cpuidFlags)) {
|
|
chacha_encrypt_avx(ctx, input, output, msglen);
|
|
return 0;
|
|
}
|
|
else {
|
|
chacha_encrypt_x64(ctx, input, output, msglen);
|
|
return 0;
|
|
}
|
|
#endif
|
|
wc_Chacha_encrypt_bytes(ctx, input, output, msglen);
|
|
|
|
return 0;
|
|
}
|
|
|
|
#endif /* HAVE_CHACHA*/
|
|
|