Cyclic0007/wolfssl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
2021bcb188f7d1a47381e8c7e452befdc93faa23
wolfssl/wolfcrypt/src/poly1305.c
Sean Parkinson 0b47811c46 Changes for interop and performance 
Changes made to test.h to allow interop of PSK with OpenSSL.
Changes to allow server to pre-generate key share and perform other
operations at later time.
Fix ChaCha20 code header to have bigger state to support assembly code
for AVX1.
Fix Curve25519 code to use define instead.
Change Curve25519 to memset all object data on init.
Change Poly1305 to put both sizes into one buffer to avoid a second call
to wc_Poly1305Update().
Added WOLFSSL_START and WOLFSSL_END API and calls to show time of
protocol message function enter and leave to analyse performance
differences.
Moved Curve25519 code in KeyShare extension out of general ECC code.
2018-04-13 12:01:20 +10:00

1645 lines
53 KiB
C
Raw Blame History

/* poly1305.c
*
* Copyright (C) 2006-2017 wolfSSL Inc.
*
* This file is part of wolfSSL.
*
* wolfSSL is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* wolfSSL is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
*/
/*
* Based off the public domain implementations by Andrew Moon
* and Daniel J. Bernstein
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <wolfssl/wolfcrypt/settings.h>
#ifdef HAVE_POLY1305
#include <wolfssl/wolfcrypt/poly1305.h>
#include <wolfssl/wolfcrypt/error-crypt.h>
#include <wolfssl/wolfcrypt/logging.h>
#include <wolfssl/wolfcrypt/cpuid.h>
#ifdef NO_INLINE
#include <wolfssl/wolfcrypt/misc.h>
#else
#define WOLFSSL_MISC_INCLUDED
#include <wolfcrypt/src/misc.c>
#endif
#ifdef CHACHA_AEAD_TEST
#include <stdio.h>
#endif
#ifdef _MSC_VER
/* 4127 warning constant while(1) */
#pragma warning(disable: 4127)
#endif
#ifdef USE_INTEL_SPEEDUP
#include <emmintrin.h>
#include <immintrin.h>
#if defined(__GNUC__) && ((__GNUC__ < 4) || \
(__GNUC__ == 4 && __GNUC_MINOR__ <= 8))
#define NO_AVX2_SUPPORT
#endif
#if defined(__clang__) && ((__clang_major__ < 3) || \
(__clang_major__ == 3 && __clang_minor__ <= 5))
#define NO_AVX2_SUPPORT
#elif defined(__clang__) && defined(NO_AVX2_SUPPORT)
#undef NO_AVX2_SUPPORT
#endif
#define HAVE_INTEL_AVX1
#ifndef NO_AVX2_SUPPORT
#define HAVE_INTEL_AVX2
#endif
#endif
#ifdef USE_INTEL_SPEEDUP
static word32 intel_flags = 0;
static word32 cpu_flags_set = 0;
#endif
#if defined(USE_INTEL_SPEEDUP) || defined(POLY130564)
#if defined(_MSC_VER)
#define POLY1305_NOINLINE __declspec(noinline)
#elif defined(__GNUC__)
#define POLY1305_NOINLINE __attribute__((noinline))
#else
#define POLY1305_NOINLINE
#endif
#if defined(_MSC_VER)
#include <intrin.h>
typedef struct word128 {
word64 lo;
word64 hi;
} word128;
#define MUL(out, x, y) out.lo = _umul128((x), (y), &out.hi)
#define ADD(out, in) { word64 t = out.lo; out.lo += in.lo; \
out.hi += (out.lo < t) + in.hi; }
#define ADDLO(out, in) { word64 t = out.lo; out.lo += in; \
out.hi += (out.lo < t); }
#define SHR(in, shift) (__shiftright128(in.lo, in.hi, (shift)))
#define LO(in) (in.lo)
#elif defined(__GNUC__)
#if defined(__SIZEOF_INT128__)
typedef unsigned __int128 word128;
#else
typedef unsigned word128 __attribute__((mode(TI)));
#endif
#define MUL(out, x, y) out = ((word128)x * y)
#define ADD(out, in) out += in
#define ADDLO(out, in) out += in
#define SHR(in, shift) (word64)(in >> (shift))
#define LO(in) (word64)(in)
#endif
#endif
#ifdef USE_INTEL_SPEEDUP
#ifdef HAVE_INTEL_AVX1
/* Process one block (16 bytes) of data.
*
* ctx Poly1305 context.
* m One block of message data.
*/
static void poly1305_block_avx(Poly1305* ctx, const unsigned char *m)
{
__asm__ __volatile__ (
"movq (%[ctx]), %%r15\n\t"
"movq 24(%[ctx]), %%r8\n\t"
"movq 32(%[ctx]), %%r9\n\t"
"movq 40(%[ctx]), %%r10\n\t"
"xorq %%rbx, %%rbx\n\t"
"movb %[nfin], %%bl\n\t"
"# h += m\n\t"
"movq (%[m]), %%r11\n\t"
"movq 8(%[m]), %%r12\n\t"
"addq %%r11, %%r8\n\t"
"adcq %%r12, %%r9\n\t"
"movq 8(%[ctx]), %%rax\n\t"
"adcq %%rbx, %%r10\n\t"
"# r[1] * h[0] => rdx, rax ==> t2, t1\n\t"
"mulq %%r8\n\t"
"movq %%rax, %%r12\n\t"
"movq %%rdx, %%r13\n\t"
"# r[0] * h[1] => rdx, rax ++> t2, t1\n\t"
"movq %%r15, %%rax\n\t"
"mulq %%r9\n\t"
"addq %%rax, %%r12\n\t"
"movq %%r15, %%rax\n\t"
"adcq %%rdx, %%r13\n\t"
"# r[0] * h[0] => rdx, rax ==> t4, t0\n\t"
"mulq %%r8\n\t"
"movq %%rax, %%r11\n\t"
"movq %%rdx, %%r8\n\t"
"# r[1] * h[1] => rdx, rax =+> t3, t2\n\t"
"movq 8(%[ctx]), %%rax\n\t"
"mulq %%r9\n\t"
"# r[0] * h[2] +> t2\n\t"
"addq 352(%[ctx],%%r10,8), %%r13\n\t"
"movq %%rdx, %%r14\n\t"
"addq %%r8, %%r12\n\t"
"adcq %%rax, %%r13\n\t"
"# r[1] * h[2] +> t3\n\t"
"adcq 408(%[ctx],%%r10,8), %%r14\n\t"
"# r * h in r14, r13, r12, r11 \n\t"
"# h = (r * h) mod 2^130 - 5\n\t"
"movq %%r13, %%r10\n\t"
"andq $-4, %%r13\n\t"
"andq $3, %%r10\n\t"
"addq %%r13, %%r11\n\t"
"movq %%r13, %%r8\n\t"
"adcq %%r14, %%r12\n\t"
"adcq $0, %%r10\n\t"
"shrdq $2, %%r14, %%r8\n\t"
"shrq $2, %%r14\n\t"
"addq %%r11, %%r8\n\t"
"adcq %%r14, %%r12\n\t"
"movq %%r12, %%r9\n\t"
"adcq $0, %%r10\n\t"
"# h in r10, r9, r8 \n\t"
"# Store h to ctx\n\t"
"movq %%r8, 24(%[ctx])\n\t"
"movq %%r9, 32(%[ctx])\n\t"
"movq %%r10, 40(%[ctx])\n\t"
:
: [m] "r" (m), [ctx] "r" (ctx), [nfin] "m" (ctx->finished)
: "rax", "rdx", "r11", "r12", "r13", "r14", "r15", "rbx",
"r8", "r9", "r10", "memory"
);
}
/* Process multiple blocks (n * 16 bytes) of data.
*
* ctx Poly1305 context.
* m Blocks of message data.
* bytes The number of bytes to process.
*/
POLY1305_NOINLINE static void poly1305_blocks_avx(Poly1305* ctx,
const unsigned char* m, size_t bytes)
{
__asm__ __volatile__ (
"movq (%[ctx]), %%r15\n\t"
"movq 24(%[ctx]), %%r8\n\t"
"movq 32(%[ctx]), %%r9\n\t"
"movq 40(%[ctx]), %%r10\n"
"L_avx_start:\n\t"
"# h += m\n\t"
"movq (%[m]), %%r11\n\t"
"movq 8(%[m]), %%r12\n\t"
"addq %%r11, %%r8\n\t"
"adcq %%r12, %%r9\n\t"
"movq 8(%[ctx]), %%rax\n\t"
"adcq $0, %%r10\n\t"
"# r[1] * h[0] => rdx, rax ==> t2, t1\n\t"
"mulq %%r8\n\t"
"movq %%rax, %%r12\n\t"
"movq %%rdx, %%r13\n\t"
"# r[0] * h[1] => rdx, rax ++> t2, t1\n\t"
"movq %%r15, %%rax\n\t"
"mulq %%r9\n\t"
"addq %%rax, %%r12\n\t"
"movq %%r15, %%rax\n\t"
"adcq %%rdx, %%r13\n\t"
"# r[0] * h[0] => rdx, rax ==> t4, t0\n\t"
"mulq %%r8\n\t"
"movq %%rax, %%r11\n\t"
"movq %%rdx, %%r8\n\t"
"# r[1] * h[1] => rdx, rax =+> t3, t2\n\t"
"movq 8(%[ctx]), %%rax\n\t"
"mulq %%r9\n\t"
"# r[0] * h[2] +> t2\n\t"
"addq 360(%[ctx],%%r10,8), %%r13\n\t"
"movq %%rdx, %%r14\n\t"
"addq %%r8, %%r12\n\t"
"adcq %%rax, %%r13\n\t"
"# r[1] * h[2] +> t3\n\t"
"adcq 416(%[ctx],%%r10,8), %%r14\n\t"
"# r * h in r14, r13, r12, r11 \n\t"
"# h = (r * h) mod 2^130 - 5\n\t"
"movq %%r13, %%r10\n\t"
"andq $-4, %%r13\n\t"
"andq $3, %%r10\n\t"
"addq %%r13, %%r11\n\t"
"movq %%r13, %%r8\n\t"
"adcq %%r14, %%r12\n\t"
"adcq $0, %%r10\n\t"
"shrdq $2, %%r14, %%r8\n\t"
"shrq $2, %%r14\n\t"
"addq %%r11, %%r8\n\t"
"adcq %%r14, %%r12\n\t"
"movq %%r12, %%r9\n\t"
"adcq $0, %%r10\n\t"
"# h in r10, r9, r8 \n\t"
"# Next block from message\n\t"
"addq $16, %[m]\n\t"
"subq $16, %[bytes]\n\t"
"cmp $16, %[bytes]\n\t"
"jge L_avx_start\n\t"
"# Store h to ctx\n\t"
"movq %%r8, 24(%[ctx])\n\t"
"movq %%r9, 32(%[ctx])\n\t"
"movq %%r10, 40(%[ctx])\n\t"
: [m] "+r" (m), [bytes] "+r" (bytes)
: [ctx] "r" (ctx)
: "rax", "rdx", "r11", "r12", "r13", "r14", "r15",
"r8", "r9", "r10", "memory"
);
}
/* Set the key to use when processing data.
* Initialize the context.
*
* ctx Poly1305 context.
* key The key data (16 bytes).
*/
static void poly1305_setkey_avx(Poly1305* ctx, const byte* key)
{
int i;
ctx->r[0] = *(word64*)(key + 0) & 0x0ffffffc0fffffffL;
ctx->r[1] = *(word64*)(key + 8) & 0x0ffffffc0ffffffcL;
for (i=0; i<7; i++) {
ctx->hm[i + 0] = ctx->r[0] * i;
ctx->hm[i + 7] = ctx->r[1] * i;
}
/* h (accumulator) = 0 */
ctx->h[0] = 0;
ctx->h[1] = 0;
ctx->h[2] = 0;
/* save pad for later */
ctx->pad[0] = *(word64*)(key + 16);
ctx->pad[1] = *(word64*)(key + 24);
ctx->leftover = 0;
ctx->finished = 1;
}
/* Calculate the final result - authentication data.
* Zeros out the private data in the context.
*
* ctx Poly1305 context.
* mac Buffer to hold 16 bytes.
*/
static void poly1305_final_avx(Poly1305* ctx, byte* mac)
{
word64 h0, h1, h2;
/* process the remaining block */
if (ctx->leftover) {
size_t i = ctx->leftover;
ctx->buffer[i] = 1;
for (i = i + 1; i < POLY1305_BLOCK_SIZE; i++)
ctx->buffer[i] = 0;
ctx->finished = 0;
poly1305_block_avx(ctx, ctx->buffer);
}
h0 = ctx->h[0];
h1 = ctx->h[1];
h2 = ctx->h[2];
/* h %= p */
/* h = (h + pad) */
__asm__ __volatile__ (
"# mod 2^130 - 5\n\t"
"movq %[h2], %%r13\n\t"
"andq $0x3, %[h2]\n\t"
"shrq $0x2, %%r13\n\t"
"leaq (%%r13, %%r13, 4), %%r13\n\t"
"add %%r13, %[h0]\n\t"
"adc $0, %[h1]\n\t"
"adc $0, %[h2]\n\t"
"# Fixup when between (1 << 130) - 1 and (1 << 130) - 5\n\t"
"movq %[h0], %%r13\n\t"
"movq %[h1], %%r14\n\t"
"movq %[h2], %%r15\n\t"
"addq $5, %%r13\n\t"
"adcq $0, %%r14\n\t"
"adcq $0, %%r15\n\t"
"movq %%r15, %%r12\n\t"
"andq $3, %%r15\n\t"
"cmpq $4, %%r12\n\t"
"cmove %%r13, %[h0]\n\t"
"cmove %%r14, %[h1]\n\t"
"cmove %%r15, %[h2]\n\t"
"# h += pad\n\t"
"add %[p0], %[h0]\n\t"
"adc %[p1], %[h1]\n\t"
"movq %[h0], (%[m])\n\t"
"movq %[h1], 8(%[m])\n\t"
: [h0] "+r" (h0), [h1] "+r" (h1), [h2] "+r" (h2),
[p0] "+r" (ctx->pad[0]), [p1] "+r" (ctx->pad[1])
: [m] "r" (mac)
: "memory", "r15", "r14", "r13", "r12"
);
/* zero out the state */
ctx->h[0] = 0;
ctx->h[1] = 0;
ctx->h[2] = 0;
ctx->r[0] = 0;
ctx->r[1] = 0;
ctx->pad[0] = 0;
ctx->pad[1] = 0;
}
#endif
#ifdef HAVE_INTEL_AVX2
#if defined(_MSC_VER)
#define POLY1305_NOINLINE __declspec(noinline)
#elif defined(__GNUC__)
#define POLY1305_NOINLINE __attribute__((noinline))
#else
#define POLY1305_NOINLINE
#endif
/* Load H into five 256-bit registers.
*
* h is the memory location of the data - 26 of 32 bits.
* h0-h4 the 4 H values with 26 bits stored in 64 for multiply.
*/
#define LOAD_H(h, h0, h1, h2, h3, h4) \
"vmovdqu ("#h"), "#h0"\n\t" \
"vmovdqu 32("#h"), "#h1"\n\t" \
"vmovdqu 64("#h"), "#h2"\n\t" \
"vmovdqu 96("#h"), "#h3"\n\t" \
"vmovdqu 128("#h"), "#h4"\n\t"
/* Store H, five 256-bit registers, packed.
*
* h is the memory location of the data - 26 bits in 32.
* h0-h4 the 4 H values with 26 bits stored in 64.
* x4 is the xmm register of h4.
*/
#define STORE_H(h, h0, h1, h2, h3, h4, x4) \
"vmovdqu "#h0", ("#h")\n\t" \
"vmovdqu "#h1", 32("#h")\n\t" \
"vmovdqu "#h2", 64("#h")\n\t" \
"vmovdqu "#h3", 96("#h")\n\t" \
"vmovdqu "#h4", 128("#h")\n\t"
/* Load four powers of r into position to be multiplied by the 4 H values.
*
* r0-r4 holds the loaded values with 26 bits stored in 64 for multiply.
* t0-t3 are temporary registers.
*/
#define LOAD_Rx4(r0, r1, r2, r3, r4, \
t0, t1, t2, t3) \
"vmovdqu 224(%[ctx]), "#r3"\n\t" \
"vmovdqu 256(%[ctx]), "#r2"\n\t" \
"vmovdqu 288(%[ctx]), "#r1"\n\t" \
"vmovdqu 320(%[ctx]), "#r0"\n\t" \
"vpermq $0xd8, "#r0", "#r0"\n\t" \
"vpermq $0xd8, "#r1", "#r1"\n\t" \
"vpermq $0xd8, "#r2", "#r2"\n\t" \
"vpermq $0xd8, "#r3", "#r3"\n\t" \
"vpunpcklqdq "#r1", "#r0", "#t0"\n\t" \
"vpunpckhqdq "#r1", "#r0", "#t1"\n\t" \
"vpunpcklqdq "#r3", "#r2", "#t2"\n\t" \
"vpunpckhqdq "#r3", "#r2", "#t3"\n\t" \
"vperm2i128 $0x20, "#t2", "#t0", "#r0"\n\t" \
"vperm2i128 $0x31, "#t2", "#t0", "#r2"\n\t" \
"vperm2i128 $0x20, "#t3", "#t1", "#r4"\n\t" \
"vpsrlq $32, "#r0", "#r1"\n\t" \
"vpsrlq $32, "#r2", "#r3"\n\t"
/* Load the r^4 value into position to be multiplied by all 4 H values.
*
* r4 holds r^4 as five 26 bits each in 32.
* r0-r4 holds the loaded values with 26 bits stored in 64 for multiply.
* t0-t1 are temporary registers.
*/
#define LOAD_R4(r4, r40, r41, r42, r43, r44, \
t0, t1) \
"vmovdqu "#r4", "#t0"\n\t" \
"vpermq $0x0, "#t0", "#r40"\n\t" \
"vpsrlq $32, "#t0", "#t1"\n\t" \
"vpermq $0x55, "#t0", "#r42"\n\t" \
"vpermq $0xaa, "#t0", "#r44"\n\t" \
"vpermq $0x0, "#t1", "#r41"\n\t" \
"vpermq $0x55, "#t1", "#r43"\n\t"
/* Multiply the top 4 26-bit values in 64 bits of each H by 5 for reduction in
* multiply.
*
* s1-s4 are each 64 bit value in r1-r4 multiplied by 5.
* r1-r4 are the top 4
*/
#define MUL5(s1, s2, s3, s4, r1, r2, r3, r4) \
"vpslld $2, "#r1", "#s1"\n\t" \
"vpslld $2, "#r2", "#s2"\n\t" \
"vpslld $2, "#r3", "#s3"\n\t" \
"vpslld $2, "#r4", "#s4"\n\t" \
"vpaddq "#s1", "#r1", "#s1"\n\t" \
"vpaddq "#s2", "#r2", "#s2"\n\t" \
"vpaddq "#s3", "#r3", "#s3"\n\t" \
"vpaddq "#s4", "#r4", "#s4"\n\t"
/* Add the 4 H values together.
* Each 64 bits in a register is 26 bits of one of the H values.
*
* h0-h4 contains the 4 H values.
* t1-t4 are temporary registers.
*/
#define FINALIZE_H(h0, h1, h2, h3, h4, \
t0, t1, t2, t3, t4) \
"vpsrldq $8, "#h0", "#t0"\n\t" \
"vpsrldq $8, "#h1", "#t1"\n\t" \
"vpsrldq $8, "#h2", "#t2"\n\t" \
"vpsrldq $8, "#h3", "#t3"\n\t" \
"vpsrldq $8, "#h4", "#t4"\n\t" \
"vpaddq "#h0", "#t0", "#h0"\n\t" \
"vpaddq "#h1", "#t1", "#h1"\n\t" \
"vpaddq "#h2", "#t2", "#h2"\n\t" \
"vpaddq "#h3", "#t3", "#h3"\n\t" \
"vpaddq "#h4", "#t4", "#h4"\n\t" \
"vpermq $0x02, "#h0", "#t0"\n\t" \
"vpermq $0x02, "#h1", "#t1"\n\t" \
"vpermq $0x02, "#h2", "#t2"\n\t" \
"vpermq $0x02, "#h3", "#t3"\n\t" \
"vpermq $0x02, "#h4", "#t4"\n\t" \
"vpaddq "#h0", "#t0", "#h0"\n\t" \
"vpaddq "#h1", "#t1", "#h1"\n\t" \
"vpaddq "#h2", "#t2", "#h2"\n\t" \
"vpaddq "#h3", "#t3", "#h3"\n\t" \
"vpaddq "#h4", "#t4", "#h4"\n\t"
/* Move 32 bits from each xmm register to a 32 bit register.
*
* x0-x4 are the xmm version of the ymm registers used.
* t0-t4 are the 32-bit registers to store data in.
*/
#define MOVE_TO_32(x0, x1, x2, x3, x4, \
t0, t1, t2, t3, t4) \
"vmovd "#x0", "#t0"\n\t" \
"vmovd "#x1", "#t1"\n\t" \
"vmovd "#x2", "#t2"\n\t" \
"vmovd "#x3", "#t3"\n\t" \
"vmovd "#x4", "#t4"\n\t"
/* Multiply using AVX2 instructions.
* Each register contains up to 32 bits of data in 64 bits.
* This is a 4 way parallel multiply.
*
* h0-h4 contain 4 H values with the 32 bits of each per register.
* r0-r4 contain the 4 powers of r.
* s1-s4 contain r1-r4 times 5.
* t0-t4 and v0-v3 are temporary registers.
*/
#define MUL_AVX2(h0, h1, h2, h3, h4, \
r0, r1, r2, r3, r4, \
s1, s2, s3, s4, \
t0, t1, t2, t3, t4, \
v0, v1, v2, v3) \
"vpmuludq "#s1", "#h4", "#t0"\n\t" \
"vpmuludq "#s2", "#h3", "#v0"\n\t" \
"vpmuludq "#s2", "#h4", "#t1"\n\t" \
"vpmuludq "#s3", "#h3", "#v1"\n\t" \
"vpmuludq "#s3", "#h4", "#t2"\n\t" \
"vpaddq "#t0", "#v0", "#t0"\n\t" \
"vpmuludq "#s3", "#h2", "#v2"\n\t" \
"vpmuludq "#s4", "#h4", "#t3"\n\t" \
"vpaddq "#t1", "#v1", "#t1"\n\t" \
"vpmuludq "#s4", "#h1", "#v3"\n\t" \
"vpmuludq "#s4", "#h2", "#v0"\n\t" \
"vpaddq "#t0", "#v2", "#t0"\n\t" \
"vpmuludq "#s4", "#h3", "#v1"\n\t" \
"vpmuludq "#r0", "#h3", "#v2"\n\t" \
"vpaddq "#t0", "#v3", "#t0"\n\t" \
"vpmuludq "#r0", "#h4", "#t4"\n\t" \
"vpaddq "#t1", "#v0", "#t1"\n\t" \
"vpmuludq "#r0", "#h0", "#v3"\n\t" \
"vpaddq "#t2", "#v1", "#t2"\n\t" \
"vpmuludq "#r0", "#h1", "#v0"\n\t" \
"vpaddq "#t3", "#v2", "#t3"\n\t" \
"vpmuludq "#r0", "#h2", "#v1"\n\t" \
"vpmuludq "#r1", "#h2", "#v2"\n\t" \
"vpaddq "#t0", "#v3", "#t0"\n\t" \
"vpmuludq "#r1", "#h3", "#v3"\n\t" \
"vpaddq "#t1", "#v0", "#t1"\n\t" \
"vpmuludq "#r1", "#h0", "#v0"\n\t" \
"vpaddq "#t2", "#v1", "#t2"\n\t" \
"vpmuludq "#r1", "#h1", "#v1"\n\t" \
"vpaddq "#t3", "#v2", "#t3"\n\t" \
"vpmuludq "#r2", "#h1", "#v2"\n\t" \
"vpaddq "#t4", "#v3", "#t4"\n\t" \
"vpmuludq "#r2", "#h2", "#v3"\n\t" \
"vpaddq "#t1", "#v0", "#t1"\n\t" \
"vpmuludq "#r2", "#h0", "#v0"\n\t" \
"vpaddq "#t2", "#v1", "#t2"\n\t" \
"vpmuludq "#r3", "#h0", "#v1"\n\t" \
"vpaddq "#t3", "#v2", "#t3"\n\t" \
"vpmuludq "#r3", "#h1", "#v2"\n\t" \
"vpaddq "#t4", "#v3", "#t4"\n\t" \
"vpmuludq "#r4", "#h0", "#v3"\n\t" \
"vpaddq "#t2", "#v0", "#t2"\n\t" \
"vpaddq "#t3", "#v1", "#t3"\n\t" \
"vpaddq "#t4", "#v2", "#t4"\n\t" \
"vpaddq "#t4", "#v3", "#t4"\n\t"
/* Load the 4 blocks of the message.
*
* m the address of the message to load.
* m0-m4 is the loaded message with 32 bits in 64. Loaded so data is parallel.
* hi is the high bits of the 4 m (1 << 128 as not final block).
* z is zero.
*/
#define LOAD_M(m, m0, m1, m2, m3, m4, hi, z) \
"vmovdqu (%[m]), "#m0"\n\t" \
"vmovdqu 32(%[m]), "#m1"\n\t" \
"vperm2i128 $0x20, "#m1", "#m0", "#m2"\n\t" \
"vperm2i128 $0x31, "#m1", "#m0", "#m0"\n\t" \
"vpunpckldq "#m0", "#m2", "#m1"\n\t" \
"vpunpckhdq "#m0", "#m2", "#m3"\n\t" \
"vpunpckldq "#z", "#m1", "#m0"\n\t" \
"vpunpckhdq "#z", "#m1", "#m1"\n\t" \
"vpunpckldq "#z", "#m3", "#m2"\n\t" \
"vpunpckhdq "#z", "#m3", "#m3"\n\t" \
"vmovdqu "#hi", "#m4"\n\t" \
"vpsllq $6, "#m1", "#m1"\n\t" \
"vpsllq $12, "#m2", "#m2"\n\t" \
"vpsllq $18, "#m3", "#m3"\n\t"
/* Multiply using AVX2 instructions - adding with message.
* Each register contains up to 32 bits of data in 64 bits.
* This is a 4 way parallel multiply.
* The message data is loaded first and the multiplication adds into it.
*
* h0-h4 contain 4 H values with the 32 bits of each per register.
* r0-r4 contain the 4 powers of r.
* s1-s4 contain r1-r4 times 5.
* t0-t4 and v0-v3 are temporary registers.
* hi is the high bits of the 4 m (1 << 128 as not final block).
* z is zero.
*/
#define MUL_ADD_AVX2(h0, h1, h2, h3, h4, \
r0, r1, r2, r3, r4, \
s1, s2, s3, s4, \
t0, t1, t2, t3, t4, \
v0, v1, v2, v3, \
hi, z) \
"vmovdqu (%[m]), "#t0"\n\t" \
"vmovdqu 32(%[m]), "#t1"\n\t" \
"vperm2i128 $0x20, "#t1", "#t0", "#t2"\n\t" \
"vperm2i128 $0x31, "#t1", "#t0", "#t0"\n\t" \
"vpunpckldq "#t0", "#t2", "#t1"\n\t" \
"vpunpckhdq "#t0", "#t2", "#t3"\n\t" \
"vpunpckldq "#z", "#t1", "#t0"\n\t" \
"vpunpckhdq "#z", "#t1", "#t1"\n\t" \
"vpunpckldq "#z", "#t3", "#t2"\n\t" \
"vpunpckhdq "#z", "#t3", "#t3"\n\t" \
"vmovdqu "#hi", "#t4"\n\t" \
"vpsllq $6, "#t1", "#t1"\n\t" \
"vpsllq $12, "#t2", "#t2"\n\t" \
"vpsllq $18, "#t3", "#t3"\n\t" \
"vpmuludq "#s1", "#h4", "#v0"\n\t" \
"vpaddq "#t0", "#v0", "#t0"\n\t" \
"vpmuludq "#s2", "#h3", "#v0"\n\t" \
"vpmuludq "#s2", "#h4", "#v1"\n\t" \
"vpaddq "#t1", "#v1", "#t1"\n\t" \
"vpmuludq "#s3", "#h3", "#v1"\n\t" \
"vpmuludq "#s3", "#h4", "#v2"\n\t" \
"vpaddq "#t2", "#v2", "#t2"\n\t" \
"vpaddq "#t0", "#v0", "#t0"\n\t" \
"vpmuludq "#s3", "#h2", "#v2"\n\t" \
"vpmuludq "#s4", "#h4", "#v3"\n\t" \
"vpaddq "#t3", "#v3", "#t3"\n\t" \
"vpaddq "#t1", "#v1", "#t1"\n\t" \
"vpmuludq "#s4", "#h1", "#v3"\n\t" \
"vpmuludq "#s4", "#h2", "#v0"\n\t" \
"vpaddq "#t0", "#v2", "#t0"\n\t" \
"vpmuludq "#s4", "#h3", "#v1"\n\t" \
"vpmuludq "#r0", "#h3", "#v2"\n\t" \
"vpaddq "#t0", "#v3", "#t0"\n\t" \
"vpmuludq "#r0", "#h4", "#v3"\n\t" \
"vpaddq "#t4", "#v3", "#t4"\n\t" \
"vpaddq "#t1", "#v0", "#t1"\n\t" \
"vpmuludq "#r0", "#h0", "#v3"\n\t" \
"vpaddq "#t2", "#v1", "#t2"\n\t" \
"vpmuludq "#r0", "#h1", "#v0"\n\t" \
"vpaddq "#t3", "#v2", "#t3"\n\t" \
"vpmuludq "#r0", "#h2", "#v1"\n\t" \
"vpmuludq "#r1", "#h2", "#v2"\n\t" \
"vpaddq "#t0", "#v3", "#t0"\n\t" \
"vpmuludq "#r1", "#h3", "#v3"\n\t" \
"vpaddq "#t1", "#v0", "#t1"\n\t" \
"vpmuludq "#r1", "#h0", "#v0"\n\t" \
"vpaddq "#t2", "#v1", "#t2"\n\t" \
"vpmuludq "#r1", "#h1", "#v1"\n\t" \
"vpaddq "#t3", "#v2", "#t3"\n\t" \
"vpmuludq "#r2", "#h1", "#v2"\n\t" \
"vpaddq "#t4", "#v3", "#t4"\n\t" \
"vpmuludq "#r2", "#h2", "#v3"\n\t" \
"vpaddq "#t1", "#v0", "#t1"\n\t" \
"vpmuludq "#r2", "#h0", "#v0"\n\t" \
"vpaddq "#t2", "#v1", "#t2"\n\t" \
"vpmuludq "#r3", "#h0", "#v1"\n\t" \
"vpaddq "#t3", "#v2", "#t3"\n\t" \
"vpmuludq "#r3", "#h1", "#v2"\n\t" \
"vpaddq "#t4", "#v3", "#t4"\n\t" \
"vpmuludq "#r4", "#h0", "#v3"\n\t" \
"vpaddq "#t2", "#v0", "#t2"\n\t" \
"vpaddq "#t3", "#v1", "#t3"\n\t" \
"vpaddq "#t4", "#v2", "#t4"\n\t" \
"vpaddq "#t4", "#v3", "#t4"\n\t"
/* Reduce the 64 bits of data to 26 bits.
*
* h0-h4 contain the reduced H values.
* m0-m4 contain the 4 H values to reduce.
* t0-t2 are temporaries.
* mask contains the 26-bit mask for each 64 bit value in the 256 bit register.
*/
#define REDUCE(h0, h1, h2, h3, h4, \
m0, m1, m2, m3, m4, \
t0, t1, t2, mask) \
"vpsrlq $26, "#m0", "#t0"\n\t" \
"vpsrlq $26, "#m3", "#t1"\n\t" \
"vpand "#mask", "#m0", "#m0"\n\t" \
"vpand "#mask", "#m3", "#m3"\n\t" \
"vpaddq "#m1", "#t0", "#m1"\n\t" \
"vpaddq "#m4", "#t1", "#m4"\n\t" \
\
"vpsrlq $26, "#m1", "#t0"\n\t" \
"vpsrlq $26, "#m4", "#t1"\n\t" \
"vpand "#mask", "#m1", "#h1"\n\t" \
"vpand "#mask", "#m4", "#h4"\n\t" \
"vpaddq "#m2", "#t0", "#m2"\n\t" \
"vpslld $2, "#t1", "#t2"\n\t" \
"vpaddd "#t2", "#t1", "#t2"\n\t" \
\
"vpsrlq $26, "#m2", "#t0"\n\t" \
"vpaddq "#m0", "#t2", "#m0"\n\t" \
"vpsrlq $26, "#m0", "#t1"\n\t" \
"vpand "#mask", "#m2", "#h2"\n\t" \
"vpand "#mask", "#m0", "#h0"\n\t" \
"vpaddq "#m3", "#t0", "#m3"\n\t" \
"vpaddq "#h1", "#t1", "#h1"\n\t" \
\
"vpsrlq $26, "#m3", "#t0"\n\t" \
"vpand "#mask", "#m3", "#h3"\n\t" \
"vpaddq "#h4", "#t0", "#h4"\n\t" \
/* Process multiple blocks (n * 16 bytes) of data.
*
* ctx Poly1305 context.
* m Blocks of message data.
* bytes The number of bytes to process.
*/
POLY1305_NOINLINE static void poly1305_blocks_avx2(Poly1305* ctx,
const unsigned char* m, size_t bytes)
{
ALIGN256 word64 r4[5][4];
ALIGN256 word64 s[4][4];
register word32 t0 asm("r8") = 0;
register word32 t1 asm("r9") = 0;
register word32 t2 asm("r10") = 0;
register word32 t3 asm("r11") = 0;
register word32 t4 asm("r12") = 0;
static const word64 mask[4] = { 0x0000000003ffffff, 0x0000000003ffffff,
0x0000000003ffffff, 0x0000000003ffffff };
static const word64 hibit[4] = { 0x1000000, 0x1000000,
0x1000000, 0x1000000 };
__asm__ __volatile__ (
"vpxor %%ymm15, %%ymm15, %%ymm15\n\t"
"cmpb $1, %[started]\n\t"
"je L_begin\n\t"
"cmpb $1, %[fin]\n\t"
"je L_begin\n\t"
"# Load the message data\n\t"
LOAD_M(m, %%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, %[hibit], %%ymm15)
"vmovdqu %[mask], %%ymm14\n\t"
"# Reduce, in place, the message data\n\t"
REDUCE(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4,
%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4,
%%ymm10, %%ymm11, %%ymm12, %%ymm14)
"addq $64, %[m]\n\t"
"subq $64, %[bytes]\n\t"
"jz L_store\n\t"
"jmp L_load_r4\n\t"
"\n"
"L_begin:\n\t"
"# Load the H values.\n\t"
LOAD_H(%[h], %%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4)
"# Check if there is a power of r to load - otherwise use r^4.\n\t"
"cmpb $0, %[fin]\n\t"
"je L_load_r4\n\t"
"\n\t"
"# Load the 4 powers of r - r^4, r^3, r^2, r^1.\n\t"
LOAD_Rx4(%%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9,
%%ymm10, %%ymm11, %%ymm12, %%ymm13)
"jmp L_mul_5\n\t"
"\n"
"L_load_r4:\n\t"
"# Load r^4 into all four positions.\n\t"
LOAD_R4(320(%[ctx]), %%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9,
%%ymm13, %%ymm14)
"\n"
"L_mul_5:\n\t"
"# Multiply top 4 26-bit values of all four H by 5\n\t"
MUL5(%%ymm10, %%ymm11, %%ymm12, %%ymm13, %%ymm6, %%ymm7, %%ymm8, %%ymm9)
"# Store powers of r and multiple of 5 for use in multiply.\n\t"
"vmovdqa %%ymm10, (%[s])\n\t"
"vmovdqa %%ymm11, 32(%[s])\n\t"
"vmovdqa %%ymm12, 64(%[s])\n\t"
"vmovdqa %%ymm13, 96(%[s])\n\t"
"vmovdqa %%ymm5 , (%[r4])\n\t"
"vmovdqa %%ymm6 , 32(%[r4])\n\t"
"vmovdqa %%ymm7 , 64(%[r4])\n\t"
"vmovdqa %%ymm8 , 96(%[r4])\n\t"
"vmovdqa %%ymm9 , 128(%[r4])\n\t"
"vmovdqu %[mask], %%ymm14\n\t"
"\n"
"# If not finished then loop over data\n\t"
"cmpb $0x1, %[fin]\n\t"
"jne L_start\n\t"
"# Do last multiply, reduce, add the four H together and move to\n\t"
"# 32-bit registers\n\t"
MUL_AVX2(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4,
(%[r4]), 32(%[r4]), 64(%[r4]), 96(%[r4]), 128(%[r4]),
(%[s]), 32(%[s]), 64(%[s]), 96(%[s]),
%%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9,
%%ymm10, %%ymm11, %%ymm12, %%ymm13)
REDUCE(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4,
%%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9,
%%ymm10, %%ymm11, %%ymm12, %%ymm14)
FINALIZE_H(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4,
%%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9)
MOVE_TO_32(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4,
%[t0], %[t1], %[t2], %[t3], %[t4])
"jmp L_end\n\t"
"\n"
"L_start:\n\t"
MUL_ADD_AVX2(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4,
(%[r4]), 32(%[r4]), 64(%[r4]), 96(%[r4]), 128(%[r4]),
(%[s]), 32(%[s]), 64(%[s]), 96(%[s]),
%%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9,
%%ymm10, %%ymm11, %%ymm12, %%ymm13,
%[hibit], %%ymm15)
REDUCE(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4,
%%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9,
%%ymm10, %%ymm11, %%ymm12, %%ymm14)
"addq $64, %[m]\n\t"
"subq $64, %[bytes]\n\t"
"jnz L_start\n\t"
"\n"
"L_store:\n\t"
"# Store four H values - state\n\t"
STORE_H(%[h], %%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, %%xmm4)
"\n"
"L_end:\n\t"
: [m] "+r" (m), [bytes] "+r" (bytes),
[t0] "+r" (t0), [t1] "+r" (t1), [t2] "+r" (t2),
[t3] "+r" (t3), [t4] "+r" (t4)
: [ctx] "r" (ctx), [h] "r" (ctx->hh),
[r4] "r" (r4), [s] "r" (s),
[fin] "m" (ctx->finished), [started] "m" (ctx->started),
[mask] "m" (mask), [hibit] "m" (hibit)
: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7",
"ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15",
"memory"
);
if (ctx->finished)
{
word64 h0, h1, h2, c;
/* Convert to 64-bit form. */
h0 = (((word64)(t1 & 0x3FFFF)) << 26) + t0;
h1 = (((word64)(t3 & 0x3FF)) << 34) +
(((word64) t2 ) << 8) + (t1 >> 18);
h2 = (((word64) t4 ) << 16) + (t3 >> 10);
/* Perform modulur reduction. */
c = (h1 >> 44); h1 &= 0xfffffffffff;
h2 += c; c = (h2 >> 42); h2 &= 0x3ffffffffff;
h0 += c * 5; c = (h0 >> 44); h0 &= 0xfffffffffff;
h1 += c; c = (h1 >> 44); h1 &= 0xfffffffffff;
h2 += c; c = (h2 >> 42); h2 &= 0x3ffffffffff;
h0 += c * 5; c = (h0 >> 44); h0 &= 0xfffffffffff;
h1 += c;
/* Convert from 42/44/44 to 2/64/64 bits used and store result. */
ctx->h[0] = h0 | (h1 << 44);
ctx->h[1] = (h1 >> 20) | (h2 << 24);
ctx->h[2] = h2 >> 40;
}
ctx->started = 1;
}
/* Multiply two 130-bit numbers in 64-bit registers and reduce.
* 44 + 44 + 42 = 130 bits
*
* r0-r2 are the first operand and the result.
* a0-a2 are the second operand.
*/
#define MUL_64(r0, r1, r2, a0, a1, a2) \
s1 = a1 * (5 << 2); \
s2 = a2 * (5 << 2); \
MUL(d0, r0, a0); MUL(d, r1, s2); ADD(d0, d); MUL(d, r2, s1); ADD(d0, d); \
MUL(d1, r0, a1); MUL(d, r1, a0); ADD(d1, d); MUL(d, r2, s2); ADD(d1, d); \
MUL(d2, r0, a2); MUL(d, r1, a1); ADD(d2, d); MUL(d, r2, a0); ADD(d2, d); \
\
c = SHR(d0, 44); r0 = LO(d0) & 0xfffffffffff; \
ADDLO(d1, c); c = SHR(d1, 44); r1 = LO(d1) & 0xfffffffffff; \
ADDLO(d2, c); c = SHR(d2, 42); r2 = LO(d2) & 0x3ffffffffff; \
r0 += c * 5; c = (r0 >> 44); r0 = r0 & 0xfffffffffff; \
r1 += c
#define SQR_64(r0, r1, r2) \
s2 = r2 * (5 << 2); \
MUL(d0, r1, s2); ADD(d0, d0); MUL(d, r0, r0); ADD(d0, d); \
MUL(d1, r0, r1); ADD(d1, d1); MUL(d, r2, s2); ADD(d1, d); \
MUL(d2, r0, r2); ADD(d2, d2); MUL(d, r1, r1); ADD(d2, d); \
\
c = SHR(d0, 44); r0 = LO(d0) & 0xfffffffffff; \
ADDLO(d1, c); c = SHR(d1, 44); r1 = LO(d1) & 0xfffffffffff; \
ADDLO(d2, c); c = SHR(d2, 42); r2 = LO(d2) & 0x3ffffffffff; \
r0 += c * 5; c = (r0 >> 44); r0 = r0 & 0xfffffffffff; \
r1 += c
/* Store the 130-bit number in 64-bit registers as 26-bit values in 32 bits.
*
* r0-r2 contains the 130-bit number in 64-bit registers.
* r is the address of where to store the 26 of 32 bits result.
*/
#define CONV_64_TO_32(r0, r1, r2, r) \
r[0] = (word32)( r0 ) & 0x3ffffff; \
r[1] = (word32)((r0 >> 26) | (r1 << 18)) & 0x3ffffff; \
r[2] = (word32)( r1 >> 8 ) & 0x3ffffff; \
r[3] = (word32)((r1 >> 34) | (r2 << 10)) & 0x3ffffff; \
r[4] = (word32)( r2 >> 16 )
/* Calculate R^1, R^2, R^3 and R^4 and store them in the context.
*
* ctx Poly1305 context.
*/
static void poly1305_calc_powers(Poly1305* ctx)
{
word64 r0, r1, r2, t0, t1, c;
word64 r20, r21, r22;
word64 r30, r31, r32;
word64 r40, r41, r42;
word64 s1, s2;
word128 d0, d1, d2, d;
t0 = ctx->r[0];
t1 = ctx->r[1];
r0 = ( t0 ) & 0xfffffffffff;
r1 = ((t0 >> 44) | (t1 << 20)) & 0xfffffffffff;
r2 = ((t1 >> 24) ) & 0x00fffffffff;
/* Store r^1 */
CONV_64_TO_32(r0, r1, r2, ctx->r1);
/* Calc and store r^2 */
r20 = r0; r21 = r1; r22 = r2;
SQR_64(r20, r21, r22);
CONV_64_TO_32(r20, r21, r22, ctx->r2);
/* Calc and store r^3 */
r30 = r20; r31 = r21; r32 = r22;
MUL_64(r30, r31, r32, r0, r1, r2);
CONV_64_TO_32(r30, r31, r32, ctx->r3);
/* Calc and store r^4 */
r40 = r20; r41 = r21; r42 = r22;
SQR_64(r40, r41, r42);
CONV_64_TO_32(r40, r41, r42, ctx->r4);
}
/* Set the key to use when processing data.
* Initialize the context.
* Calls AVX set key function as final function calls AVX code.
*
* ctx Poly1305 context.
* key The key data (16 bytes).
*/
static void poly1305_setkey_avx2(Poly1305* ctx, const byte* key)
{
poly1305_setkey_avx(ctx, key);
__asm__ __volatile__ (
"vpxor %%ymm0, %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, (%[hh])\n\t"
"vmovdqu %%ymm0, 32(%[hh])\n\t"
"vmovdqu %%ymm0, 64(%[hh])\n\t"
"vmovdqu %%ymm0, 96(%[hh])\n\t"
"vmovdqu %%ymm0, 128(%[hh])\n\t"
:
: [hh] "r" (ctx->hh)
: "memory", "ymm0"
);
ctx->leftover = 0;
ctx->finished = 0;
ctx->started = 0;
}
/* Calculate the final result - authentication data.
* Zeros out the private data in the context.
* Calls AVX final function to quickly process last blocks.
*
* ctx Poly1305 context.
* mac Buffer to hold 16 bytes - authentication data.
*/
static void poly1305_final_avx2(Poly1305* ctx, byte* mac)
{
int i, j;
int l = (int)ctx->leftover;
ctx->finished = 1;
if (ctx->started)
poly1305_blocks_avx2(ctx, ctx->buffer, POLY1305_BLOCK_SIZE * 4);
i = l & ~(POLY1305_BLOCK_SIZE - 1);
if (i > 0)
poly1305_blocks_avx(ctx, ctx->buffer, i);
ctx->leftover -= i;
for (j = 0; i < l; i++, j++)
ctx->buffer[j] = ctx->buffer[i];
poly1305_final_avx(ctx, mac);
/* zero out the state */
__asm__ __volatile__ (
"vpxor %%ymm0, %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, (%[hh])\n\t"
"vmovdqu %%ymm0, 32(%[hh])\n\t"
"vmovdqu %%ymm0, 64(%[hh])\n\t"
"vmovdqu %%ymm0, 96(%[hh])\n\t"
"vmovdqu %%ymm0, 128(%[hh])\n\t"
"vmovdqu %%ymm0, (%[r1])\n\t"
"vmovdqu %%ymm0, (%[r2])\n\t"
"vmovdqu %%ymm0, (%[r3])\n\t"
"vmovdqu %%ymm0, (%[r4])\n\t"
:
: [hh] "r" (ctx->hh), [r1] "r" (ctx->r1), [r2] "r" (ctx->r2),
[r3] "r" (ctx->r3), [r4] "r" (ctx->r4)
: "memory", "ymm0"
);
ctx->leftover = 0;
ctx->finished = 0;
ctx->started = 0;
}
#endif
#elif defined(POLY130564)
static word64 U8TO64(const byte* p)
{
return
(((word64)(p[0] & 0xff) ) |
((word64)(p[1] & 0xff) << 8) |
((word64)(p[2] & 0xff) << 16) |
((word64)(p[3] & 0xff) << 24) |
((word64)(p[4] & 0xff) << 32) |
((word64)(p[5] & 0xff) << 40) |
((word64)(p[6] & 0xff) << 48) |
((word64)(p[7] & 0xff) << 56));
}
static void U64TO8(byte* p, word64 v) {
p[0] = (v ) & 0xff;
p[1] = (v >> 8) & 0xff;
p[2] = (v >> 16) & 0xff;
p[3] = (v >> 24) & 0xff;
p[4] = (v >> 32) & 0xff;
p[5] = (v >> 40) & 0xff;
p[6] = (v >> 48) & 0xff;
p[7] = (v >> 56) & 0xff;
}
#else /* if not 64 bit then use 32 bit */
static word32 U8TO32(const byte *p)
{
return
(((word32)(p[0] & 0xff) ) |
((word32)(p[1] & 0xff) << 8) |
((word32)(p[2] & 0xff) << 16) |
((word32)(p[3] & 0xff) << 24));
}
static void U32TO8(byte *p, word32 v) {
p[0] = (v ) & 0xff;
p[1] = (v >> 8) & 0xff;
p[2] = (v >> 16) & 0xff;
p[3] = (v >> 24) & 0xff;
}
#endif
static void U32TO64(word32 v, byte* p)
{
XMEMSET(p, 0, 8);
p[0] = (v & 0xFF);
p[1] = (v >> 8) & 0xFF;
p[2] = (v >> 16) & 0xFF;
p[3] = (v >> 24) & 0xFF;
}
static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
size_t bytes)
{
#ifdef USE_INTEL_SPEEDUP
/* AVX2 is handled in wc_Poly1305Update. */
poly1305_blocks_avx(ctx, m, bytes);
#elif defined(POLY130564)
const word64 hibit = (ctx->finished) ? 0 : ((word64)1 << 40); /* 1 << 128 */
word64 r0,r1,r2;
word64 s1,s2;
word64 h0,h1,h2;
word64 c;
word128 d0,d1,d2,d;
r0 = ctx->r[0];
r1 = ctx->r[1];
r2 = ctx->r[2];
h0 = ctx->h[0];
h1 = ctx->h[1];
h2 = ctx->h[2];
s1 = r1 * (5 << 2);
s2 = r2 * (5 << 2);
while (bytes >= POLY1305_BLOCK_SIZE) {
word64 t0,t1;
/* h += m[i] */
t0 = U8TO64(&m[0]);
t1 = U8TO64(&m[8]);
h0 += (( t0 ) & 0xfffffffffff);
h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffff);
h2 += (((t1 >> 24) ) & 0x3ffffffffff) | hibit;
/* h *= r */
MUL(d0, h0, r0); MUL(d, h1, s2); ADD(d0, d); MUL(d, h2, s1); ADD(d0, d);
MUL(d1, h0, r1); MUL(d, h1, r0); ADD(d1, d); MUL(d, h2, s2); ADD(d1, d);
MUL(d2, h0, r2); MUL(d, h1, r1); ADD(d2, d); MUL(d, h2, r0); ADD(d2, d);
/* (partial) h %= p */
c = SHR(d0, 44); h0 = LO(d0) & 0xfffffffffff;
ADDLO(d1, c); c = SHR(d1, 44); h1 = LO(d1) & 0xfffffffffff;
ADDLO(d2, c); c = SHR(d2, 42); h2 = LO(d2) & 0x3ffffffffff;
h0 += c * 5; c = (h0 >> 44); h0 = h0 & 0xfffffffffff;
h1 += c;
m += POLY1305_BLOCK_SIZE;
bytes -= POLY1305_BLOCK_SIZE;
}
ctx->h[0] = h0;
ctx->h[1] = h1;
ctx->h[2] = h2;
#else /* if not 64 bit then use 32 bit */
const word32 hibit = (ctx->finished) ? 0 : (1 << 24); /* 1 << 128 */
word32 r0,r1,r2,r3,r4;
word32 s1,s2,s3,s4;
word32 h0,h1,h2,h3,h4;
word64 d0,d1,d2,d3,d4;
word32 c;
r0 = ctx->r[0];
r1 = ctx->r[1];
r2 = ctx->r[2];
r3 = ctx->r[3];
r4 = ctx->r[4];
s1 = r1 * 5;
s2 = r2 * 5;
s3 = r3 * 5;
s4 = r4 * 5;
h0 = ctx->h[0];
h1 = ctx->h[1];
h2 = ctx->h[2];
h3 = ctx->h[3];
h4 = ctx->h[4];
while (bytes >= POLY1305_BLOCK_SIZE) {
/* h += m[i] */
h0 += (U8TO32(m+ 0) ) & 0x3ffffff;
h1 += (U8TO32(m+ 3) >> 2) & 0x3ffffff;
h2 += (U8TO32(m+ 6) >> 4) & 0x3ffffff;
h3 += (U8TO32(m+ 9) >> 6) & 0x3ffffff;
h4 += (U8TO32(m+12) >> 8) | hibit;
/* h *= r */
d0 = ((word64)h0 * r0) + ((word64)h1 * s4) + ((word64)h2 * s3) +
((word64)h3 * s2) + ((word64)h4 * s1);
d1 = ((word64)h0 * r1) + ((word64)h1 * r0) + ((word64)h2 * s4) +
((word64)h3 * s3) + ((word64)h4 * s2);
d2 = ((word64)h0 * r2) + ((word64)h1 * r1) + ((word64)h2 * r0) +
((word64)h3 * s4) + ((word64)h4 * s3);
d3 = ((word64)h0 * r3) + ((word64)h1 * r2) + ((word64)h2 * r1) +
((word64)h3 * r0) + ((word64)h4 * s4);
d4 = ((word64)h0 * r4) + ((word64)h1 * r3) + ((word64)h2 * r2) +
((word64)h3 * r1) + ((word64)h4 * r0);
/* (partial) h %= p */
c = (word32)(d0 >> 26); h0 = (word32)d0 & 0x3ffffff;
d1 += c; c = (word32)(d1 >> 26); h1 = (word32)d1 & 0x3ffffff;
d2 += c; c = (word32)(d2 >> 26); h2 = (word32)d2 & 0x3ffffff;
d3 += c; c = (word32)(d3 >> 26); h3 = (word32)d3 & 0x3ffffff;
d4 += c; c = (word32)(d4 >> 26); h4 = (word32)d4 & 0x3ffffff;
h0 += c * 5; c = (h0 >> 26); h0 = h0 & 0x3ffffff;
h1 += c;
m += POLY1305_BLOCK_SIZE;
bytes -= POLY1305_BLOCK_SIZE;
}
ctx->h[0] = h0;
ctx->h[1] = h1;
ctx->h[2] = h2;
ctx->h[3] = h3;
ctx->h[4] = h4;
#endif /* end of 64 bit cpu blocks or 32 bit cpu */
}
static void poly1305_block(Poly1305* ctx, const unsigned char *m)
{
#ifdef USE_INTEL_SPEEDUP
/* No call to poly1305_block when AVX2, AVX2 does 4 blocks at a time. */
poly1305_block_avx(ctx, m);
#else
poly1305_blocks(ctx, m, POLY1305_BLOCK_SIZE);
#endif
}
int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz)
{
#if defined(POLY130564)
word64 t0,t1;
#endif
#ifdef CHACHA_AEAD_TEST
word32 k;
printf("Poly key used:\n");
for (k = 0; k < keySz; k++) {
printf("%02x", key[k]);
if ((k+1) % 8 == 0)
printf("\n");
}
printf("\n");
#endif
if (keySz != 32 || ctx == NULL)
return BAD_FUNC_ARG;
#ifdef USE_INTEL_SPEEDUP
if (!cpu_flags_set) {
intel_flags = cpuid_get_flags();
cpu_flags_set = 1;
}
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags))
poly1305_setkey_avx2(ctx, key);
else
#endif
poly1305_setkey_avx(ctx, key);
#elif defined(POLY130564)
/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
t0 = U8TO64(key + 0);
t1 = U8TO64(key + 8);
ctx->r[0] = ( t0 ) & 0xffc0fffffff;
ctx->r[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffff;
ctx->r[2] = ((t1 >> 24) ) & 0x00ffffffc0f;
/* h (accumulator) = 0 */
ctx->h[0] = 0;
ctx->h[1] = 0;
ctx->h[2] = 0;
/* save pad for later */
ctx->pad[0] = U8TO64(key + 16);
ctx->pad[1] = U8TO64(key + 24);
ctx->leftover = 0;
ctx->finished = 0;
#else /* if not 64 bit then use 32 bit */
/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
ctx->r[0] = (U8TO32(key + 0) ) & 0x3ffffff;
ctx->r[1] = (U8TO32(key + 3) >> 2) & 0x3ffff03;
ctx->r[2] = (U8TO32(key + 6) >> 4) & 0x3ffc0ff;
ctx->r[3] = (U8TO32(key + 9) >> 6) & 0x3f03fff;
ctx->r[4] = (U8TO32(key + 12) >> 8) & 0x00fffff;
/* h = 0 */
ctx->h[0] = 0;
ctx->h[1] = 0;
ctx->h[2] = 0;
ctx->h[3] = 0;
ctx->h[4] = 0;
/* save pad for later */
ctx->pad[0] = U8TO32(key + 16);
ctx->pad[1] = U8TO32(key + 20);
ctx->pad[2] = U8TO32(key + 24);
ctx->pad[3] = U8TO32(key + 28);
ctx->leftover = 0;
ctx->finished = 0;
#endif
return 0;
}
int wc_Poly1305Final(Poly1305* ctx, byte* mac)
{
#ifdef USE_INTEL_SPEEDUP
#elif defined(POLY130564)
word64 h0,h1,h2,c;
word64 g0,g1,g2;
word64 t0,t1;
#else
word32 h0,h1,h2,h3,h4,c;
word32 g0,g1,g2,g3,g4;
word64 f;
word32 mask;
#endif
if (ctx == NULL)
return BAD_FUNC_ARG;
#ifdef USE_INTEL_SPEEDUP
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags))
poly1305_final_avx2(ctx, mac);
else
#endif
poly1305_final_avx(ctx, mac);
#elif defined(POLY130564)
/* process the remaining block */
if (ctx->leftover) {
size_t i = ctx->leftover;
ctx->buffer[i] = 1;
for (i = i + 1; i < POLY1305_BLOCK_SIZE; i++)
ctx->buffer[i] = 0;
ctx->finished = 1;
poly1305_block(ctx, ctx->buffer);
}
/* fully carry h */
h0 = ctx->h[0];
h1 = ctx->h[1];
h2 = ctx->h[2];
c = (h1 >> 44); h1 &= 0xfffffffffff;
h2 += c; c = (h2 >> 42); h2 &= 0x3ffffffffff;
h0 += c * 5; c = (h0 >> 44); h0 &= 0xfffffffffff;
h1 += c; c = (h1 >> 44); h1 &= 0xfffffffffff;
h2 += c; c = (h2 >> 42); h2 &= 0x3ffffffffff;
h0 += c * 5; c = (h0 >> 44); h0 &= 0xfffffffffff;
h1 += c;
/* compute h + -p */
g0 = h0 + 5; c = (g0 >> 44); g0 &= 0xfffffffffff;
g1 = h1 + c; c = (g1 >> 44); g1 &= 0xfffffffffff;
g2 = h2 + c - ((word64)1 << 42);
/* select h if h < p, or h + -p if h >= p */
c = (g2 >> ((sizeof(word64) * 8) - 1)) - 1;
g0 &= c;
g1 &= c;
g2 &= c;
c = ~c;
h0 = (h0 & c) | g0;
h1 = (h1 & c) | g1;
h2 = (h2 & c) | g2;
/* h = (h + pad) */
t0 = ctx->pad[0];
t1 = ctx->pad[1];
h0 += (( t0 ) & 0xfffffffffff) ;
c = (h0 >> 44); h0 &= 0xfffffffffff;
h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffff) + c;
c = (h1 >> 44); h1 &= 0xfffffffffff;
h2 += (((t1 >> 24) ) & 0x3ffffffffff) + c;
h2 &= 0x3ffffffffff;
/* mac = h % (2^128) */
h0 = ((h0 ) | (h1 << 44));
h1 = ((h1 >> 20) | (h2 << 24));
U64TO8(mac + 0, h0);
U64TO8(mac + 8, h1);
/* zero out the state */
ctx->h[0] = 0;
ctx->h[1] = 0;
ctx->h[2] = 0;
ctx->r[0] = 0;
ctx->r[1] = 0;
ctx->r[2] = 0;
ctx->pad[0] = 0;
ctx->pad[1] = 0;
#else /* if not 64 bit then use 32 bit */
/* process the remaining block */
if (ctx->leftover) {
size_t i = ctx->leftover;
ctx->buffer[i++] = 1;
for (; i < POLY1305_BLOCK_SIZE; i++)
ctx->buffer[i] = 0;
ctx->finished = 1;
poly1305_block(ctx, ctx->buffer);
}
/* fully carry h */
h0 = ctx->h[0];
h1 = ctx->h[1];
h2 = ctx->h[2];
h3 = ctx->h[3];
h4 = ctx->h[4];
c = h1 >> 26; h1 = h1 & 0x3ffffff;
h2 += c; c = h2 >> 26; h2 = h2 & 0x3ffffff;
h3 += c; c = h3 >> 26; h3 = h3 & 0x3ffffff;
h4 += c; c = h4 >> 26; h4 = h4 & 0x3ffffff;
h0 += c * 5; c = h0 >> 26; h0 = h0 & 0x3ffffff;
h1 += c;
/* compute h + -p */
g0 = h0 + 5; c = g0 >> 26; g0 &= 0x3ffffff;
g1 = h1 + c; c = g1 >> 26; g1 &= 0x3ffffff;
g2 = h2 + c; c = g2 >> 26; g2 &= 0x3ffffff;
g3 = h3 + c; c = g3 >> 26; g3 &= 0x3ffffff;
g4 = h4 + c - (1 << 26);
/* select h if h < p, or h + -p if h >= p */
mask = (g4 >> ((sizeof(word32) * 8) - 1)) - 1;
g0 &= mask;
g1 &= mask;
g2 &= mask;
g3 &= mask;
g4 &= mask;
mask = ~mask;
h0 = (h0 & mask) | g0;
h1 = (h1 & mask) | g1;
h2 = (h2 & mask) | g2;
h3 = (h3 & mask) | g3;
h4 = (h4 & mask) | g4;
/* h = h % (2^128) */
h0 = ((h0 ) | (h1 << 26)) & 0xffffffff;
h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff;
h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff;
h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff;
/* mac = (h + pad) % (2^128) */
f = (word64)h0 + ctx->pad[0] ; h0 = (word32)f;
f = (word64)h1 + ctx->pad[1] + (f >> 32); h1 = (word32)f;
f = (word64)h2 + ctx->pad[2] + (f >> 32); h2 = (word32)f;
f = (word64)h3 + ctx->pad[3] + (f >> 32); h3 = (word32)f;
U32TO8(mac + 0, h0);
U32TO8(mac + 4, h1);
U32TO8(mac + 8, h2);
U32TO8(mac + 12, h3);
/* zero out the state */
ctx->h[0] = 0;
ctx->h[1] = 0;
ctx->h[2] = 0;
ctx->h[3] = 0;
ctx->h[4] = 0;
ctx->r[0] = 0;
ctx->r[1] = 0;
ctx->r[2] = 0;
ctx->r[3] = 0;
ctx->r[4] = 0;
ctx->pad[0] = 0;
ctx->pad[1] = 0;
ctx->pad[2] = 0;
ctx->pad[3] = 0;
#endif
return 0;
}
int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes)
{
size_t i;
#ifdef CHACHA_AEAD_TEST
word32 k;
printf("Raw input to poly:\n");
for (k = 0; k < bytes; k++) {
printf("%02x", m[k]);
if ((k+1) % 16 == 0)
printf("\n");
}
printf("\n");
#endif
if (ctx == NULL)
return BAD_FUNC_ARG;
#ifdef USE_INTEL_SPEEDUP
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags)) {
/* handle leftover */
if (ctx->leftover) {
size_t want = sizeof(ctx->buffer) - ctx->leftover;
if (want > bytes)
want = bytes;
for (i = 0; i < want; i++)
ctx->buffer[ctx->leftover + i] = m[i];
bytes -= (word32)want;
m += want;
ctx->leftover += want;
if (ctx->leftover < sizeof(ctx->buffer))
return 0;
if (!ctx->started)
poly1305_calc_powers(ctx);
poly1305_blocks_avx2(ctx, ctx->buffer, sizeof(ctx->buffer));
ctx->leftover = 0;
}
/* process full blocks */
if (bytes >= sizeof(ctx->buffer)) {
size_t want = bytes & ~(sizeof(ctx->buffer) - 1);
if (!ctx->started)
poly1305_calc_powers(ctx);
poly1305_blocks_avx2(ctx, m, want);
m += want;
bytes -= (word32)want;
}
/* store leftover */
if (bytes) {
for (i = 0; i < bytes; i++)
ctx->buffer[ctx->leftover + i] = m[i];
ctx->leftover += bytes;
}
}
else
#endif
#endif
{
/* handle leftover */
if (ctx->leftover) {
size_t want = (POLY1305_BLOCK_SIZE - ctx->leftover);
if (want > bytes)
want = bytes;
for (i = 0; i < want; i++)
ctx->buffer[ctx->leftover + i] = m[i];
bytes -= (word32)want;
m += want;
ctx->leftover += want;
if (ctx->leftover < POLY1305_BLOCK_SIZE)
return 0;
poly1305_block(ctx, ctx->buffer);
ctx->leftover = 0;
}
/* process full blocks */
if (bytes >= POLY1305_BLOCK_SIZE) {
size_t want = (bytes & ~(POLY1305_BLOCK_SIZE - 1));
poly1305_blocks(ctx, m, want);
m += want;
bytes -= (word32)want;
}
/* store leftover */
if (bytes) {
for (i = 0; i < bytes; i++)
ctx->buffer[ctx->leftover + i] = m[i];
ctx->leftover += bytes;
}
}
return 0;
}
/* Takes in an initialized Poly1305 struct that has a key loaded and creates
a MAC (tag) using recent TLS AEAD padding scheme.
ctx : Initialized Poly1305 struct to use
additional : Additional data to use
addSz : Size of additional buffer
input : Input buffer to create tag from
sz : Size of input buffer
tag : Buffer to hold created tag
tagSz : Size of input tag buffer (must be at least
WC_POLY1305_MAC_SZ(16))
*/
int wc_Poly1305_MAC(Poly1305* ctx, byte* additional, word32 addSz,
byte* input, word32 sz, byte* tag, word32 tagSz)
{
int ret;
byte padding[WC_POLY1305_PAD_SZ - 1];
word32 paddingLen;
byte little64[16];
XMEMSET(padding, 0, sizeof(padding));
/* sanity check on arguments */
if (ctx == NULL || input == NULL || tag == NULL ||
tagSz < WC_POLY1305_MAC_SZ) {
return BAD_FUNC_ARG;
}
/* additional allowed to be 0 */
if (addSz > 0) {
if (additional == NULL)
return BAD_FUNC_ARG;
/* additional data plus padding */
if ((ret = wc_Poly1305Update(ctx, additional, addSz)) != 0) {
return ret;
}
paddingLen = -((int)addSz) & (WC_POLY1305_PAD_SZ - 1);
if (paddingLen) {
if ((ret = wc_Poly1305Update(ctx, padding, paddingLen)) != 0) {
return ret;
}
}
}
/* input plus padding */
if ((ret = wc_Poly1305Update(ctx, input, sz)) != 0) {
return ret;
}
paddingLen = -((int)sz) & (WC_POLY1305_PAD_SZ - 1);
if (paddingLen) {
if ((ret = wc_Poly1305Update(ctx, padding, paddingLen)) != 0) {
return ret;
}
}
/* size of additional data and input as little endian 64 bit types */
U32TO64(addSz, little64);
U32TO64(sz, little64 + 8);
ret = wc_Poly1305Update(ctx, little64, sizeof(little64));
if (ret)
{
return ret;
}
/* Finalize the auth tag */
ret = wc_Poly1305Final(ctx, tag);
return ret;
}
#endif /* HAVE_POLY1305 */
Reference in New Issue View Git Blame Copy Permalink