Added ChaCha20 implementation with SSE2 opcodes.
This commit is contained in:
Thomas Pornin
parent
5414fd525e
commit
24c6f09bf8
9 changed files with 429 additions and 17 deletions
10
src/config.h
10
src/config.h
|
|
@ -162,6 +162,16 @@
|
|||
#define BR_AES_X86NI 1
|
||||
*/
|
||||
|
||||
/*
|
||||
* When BR_SSE2 is enabled, SSE2 intrinsics will be used for some
|
||||
* algorithm implementations that use them (e.g. chacha20_sse2). If this
|
||||
* is not enabled explicitly, then support for SSE2 intrinsics will be
|
||||
* automatically detected. If set explicitly to 0, then SSE2 code will
|
||||
* not be compiled at all.
|
||||
*
|
||||
#define BR_SSE2 1
|
||||
*/
|
||||
|
||||
/*
|
||||
* When BR_POWER8 is enabled, the AES implementation using the POWER ISA
|
||||
* 2.07 opcodes (available on POWER8 processors and later) is compiled.
|
||||
|
|
|
|||
42
src/inner.h
42
src/inner.h
|
|
@ -144,10 +144,46 @@
|
|||
#endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Determine whether SSE2 intrinsics are understood by the compiler.
|
||||
* Right now, we restrict ourselves to compiler versions where things
|
||||
* are documented to work well:
|
||||
* -- GCC 4.4+ and Clang 3.7+ understand the function attribute "target"
|
||||
* -- MS Visual Studio 2005 documents the existence of <emmintrin.h>
|
||||
* SSE2-powered code _might_ work with older versions, but there is no
|
||||
* pressing need to do so right now.
|
||||
*/
|
||||
#ifndef BR_SSE2
|
||||
#if (__i386__ || __x86_64__) \
|
||||
&& ((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) \
|
||||
|| (__clang_major__ > 3 \
|
||||
|| (__clang_major__ == 3 && __clang_minor__ >= 7)))
|
||||
#define BR_SSE2 1
|
||||
#elif (_M_IX86 || _M_X64) && (_MSC_VER >= 1400)
|
||||
#define BR_SSE2 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* If we use SSE2 intrinsics, determine the compiler brand.
|
||||
*/
|
||||
#if BR_SSE2
|
||||
#ifndef BR_SSE2_GCC
|
||||
#if __GNUC__
|
||||
#define BR_SSE2_GCC 1
|
||||
#endif
|
||||
#endif
|
||||
#ifndef BR_SSE2_MSC
|
||||
#if _MSC_VER >= 1400
|
||||
#define BR_SSE2_MSC 1
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* A macro to tag a function with a "target" attribute (for GCC and Clang).
|
||||
*/
|
||||
#if BR_AES_X86NI_GCC
|
||||
#if BR_AES_X86NI_GCC || BR_SSE2_GCC
|
||||
#define BR_TARGET(x) __attribute__((target(x)))
|
||||
#else
|
||||
#define BR_TARGET(x)
|
||||
|
|
@ -156,8 +192,8 @@
|
|||
/*
|
||||
* GCC versions from 4.4 to 4.8 (inclusive) must use a special #pragma
|
||||
* to activate extra opcodes before including the relevant intrinsic
|
||||
* headers. But these don't work with Clang (which does not need them
|
||||
* either). We also need that #pragma for GCC 4.9 in order to work
|
||||
* AES-NI headers. But these don't work with Clang (which does not need
|
||||
* them either). We also need that #pragma for GCC 4.9 in order to work
|
||||
* around a compiler bug (it tends to blow up on ghash_pclmul code
|
||||
* otherwise).
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -31,21 +31,35 @@ br_ssl_engine_set_default_chapol(br_ssl_engine_context *cc)
|
|||
#if BR_INT128 || BR_UMUL128
|
||||
br_poly1305_run bp;
|
||||
#endif
|
||||
#if BR_SSE2
|
||||
br_chacha20_run bc;
|
||||
#endif
|
||||
|
||||
br_ssl_engine_set_chapol(cc,
|
||||
&br_sslrec_in_chapol_vtable,
|
||||
&br_sslrec_out_chapol_vtable);
|
||||
br_ssl_engine_set_chacha20(cc, &br_chacha20_ct_run);
|
||||
#if BR_SSE2
|
||||
bc = br_chacha20_sse2_get();
|
||||
if (bc) {
|
||||
br_ssl_engine_set_chacha20(cc, bc);
|
||||
} else {
|
||||
#endif
|
||||
br_ssl_engine_set_chacha20(cc, &br_chacha20_ct_run);
|
||||
#if BR_SSE2
|
||||
}
|
||||
#endif
|
||||
#if BR_INT128 || BR_UMUL128
|
||||
bp = br_poly1305_ctmulq_get();
|
||||
if (bp) {
|
||||
br_ssl_engine_set_poly1305(cc, bp);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
#endif
|
||||
#if BR_LOMUL
|
||||
br_ssl_engine_set_poly1305(cc, &br_poly1305_ctmul32_run);
|
||||
br_ssl_engine_set_poly1305(cc, &br_poly1305_ctmul32_run);
|
||||
#else
|
||||
br_ssl_engine_set_poly1305(cc, &br_poly1305_ctmul_run);
|
||||
br_ssl_engine_set_poly1305(cc, &br_poly1305_ctmul_run);
|
||||
#endif
|
||||
#if BR_INT128 || BR_UMUL128
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
|
|
|||
257
src/symcipher/chacha20_sse2.c
Normal file
257
src/symcipher/chacha20_sse2.c
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
/*
|
||||
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
||||
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
||||
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "inner.h"
|
||||
|
||||
/*
|
||||
* This file contains a ChaCha20 implementation that leverages SSE2
|
||||
* opcodes for better performance.
|
||||
*/
|
||||
|
||||
#if BR_SSE2
|
||||
|
||||
#if BR_SSE2_GCC
|
||||
#include <emmintrin.h>
|
||||
#include <cpuid.h>
|
||||
#endif
|
||||
#if BR_SSE2_MSC
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
|
||||
/* see bearssl_block.h */
|
||||
BR_TARGET("sse2")
|
||||
uint32_t
|
||||
br_chacha20_sse2_run(const void *key,
|
||||
const void *iv, uint32_t cc, void *data, size_t len)
|
||||
{
|
||||
unsigned char *buf;
|
||||
uint32_t ivtmp[4];
|
||||
__m128i kw0, kw1;
|
||||
__m128i iw, cw;
|
||||
__m128i one;
|
||||
|
||||
static const uint32_t CW[] = {
|
||||
0x61707865, 0x3320646e, 0x79622d32, 0x6b206574
|
||||
};
|
||||
|
||||
buf = data;
|
||||
kw0 = _mm_loadu_si128(key);
|
||||
kw1 = _mm_loadu_si128((const void *)((const unsigned char *)key + 16));
|
||||
ivtmp[0] = cc;
|
||||
memcpy(ivtmp + 1, iv, 12);
|
||||
iw = _mm_loadu_si128((const void *)ivtmp);
|
||||
cw = _mm_loadu_si128((const void *)CW);
|
||||
one = _mm_set_epi32(0, 0, 0, 1);
|
||||
|
||||
while (len > 0) {
|
||||
/*
|
||||
* sj contains state words 4*j to 4*j+3.
|
||||
*/
|
||||
__m128i s0, s1, s2, s3;
|
||||
int i;
|
||||
|
||||
s0 = cw;
|
||||
s1 = kw0;
|
||||
s2 = kw1;
|
||||
s3 = iw;
|
||||
for (i = 0; i < 10; i ++) {
|
||||
/*
|
||||
* Even round is straightforward application on
|
||||
* the state words.
|
||||
*/
|
||||
s0 = _mm_add_epi32(s0, s1);
|
||||
s3 = _mm_xor_si128(s3, s0);
|
||||
s3 = _mm_or_si128(
|
||||
_mm_slli_epi32(s3, 16),
|
||||
_mm_srli_epi32(s3, 16));
|
||||
|
||||
s2 = _mm_add_epi32(s2, s3);
|
||||
s1 = _mm_xor_si128(s1, s2);
|
||||
s1 = _mm_or_si128(
|
||||
_mm_slli_epi32(s1, 12),
|
||||
_mm_srli_epi32(s1, 20));
|
||||
|
||||
s0 = _mm_add_epi32(s0, s1);
|
||||
s3 = _mm_xor_si128(s3, s0);
|
||||
s3 = _mm_or_si128(
|
||||
_mm_slli_epi32(s3, 8),
|
||||
_mm_srli_epi32(s3, 24));
|
||||
|
||||
s2 = _mm_add_epi32(s2, s3);
|
||||
s1 = _mm_xor_si128(s1, s2);
|
||||
s1 = _mm_or_si128(
|
||||
_mm_slli_epi32(s1, 7),
|
||||
_mm_srli_epi32(s1, 25));
|
||||
|
||||
/*
|
||||
* For the odd round, we must rotate some state
|
||||
* words so that the computations apply on the
|
||||
* right combinations of words.
|
||||
*/
|
||||
s1 = _mm_shuffle_epi32(s1, 0x39);
|
||||
s2 = _mm_shuffle_epi32(s2, 0x4E);
|
||||
s3 = _mm_shuffle_epi32(s3, 0x93);
|
||||
|
||||
s0 = _mm_add_epi32(s0, s1);
|
||||
s3 = _mm_xor_si128(s3, s0);
|
||||
s3 = _mm_or_si128(
|
||||
_mm_slli_epi32(s3, 16),
|
||||
_mm_srli_epi32(s3, 16));
|
||||
|
||||
s2 = _mm_add_epi32(s2, s3);
|
||||
s1 = _mm_xor_si128(s1, s2);
|
||||
s1 = _mm_or_si128(
|
||||
_mm_slli_epi32(s1, 12),
|
||||
_mm_srli_epi32(s1, 20));
|
||||
|
||||
s0 = _mm_add_epi32(s0, s1);
|
||||
s3 = _mm_xor_si128(s3, s0);
|
||||
s3 = _mm_or_si128(
|
||||
_mm_slli_epi32(s3, 8),
|
||||
_mm_srli_epi32(s3, 24));
|
||||
|
||||
s2 = _mm_add_epi32(s2, s3);
|
||||
s1 = _mm_xor_si128(s1, s2);
|
||||
s1 = _mm_or_si128(
|
||||
_mm_slli_epi32(s1, 7),
|
||||
_mm_srli_epi32(s1, 25));
|
||||
|
||||
/*
|
||||
* After the odd round, we rotate back the values
|
||||
* to undo the rotate at the start of the odd round.
|
||||
*/
|
||||
s1 = _mm_shuffle_epi32(s1, 0x93);
|
||||
s2 = _mm_shuffle_epi32(s2, 0x4E);
|
||||
s3 = _mm_shuffle_epi32(s3, 0x39);
|
||||
}
|
||||
|
||||
/*
|
||||
* Addition with the initial state.
|
||||
*/
|
||||
s0 = _mm_add_epi32(s0, cw);
|
||||
s1 = _mm_add_epi32(s1, kw0);
|
||||
s2 = _mm_add_epi32(s2, kw1);
|
||||
s3 = _mm_add_epi32(s3, iw);
|
||||
|
||||
/*
|
||||
* Increment block counter.
|
||||
*/
|
||||
iw = _mm_add_epi32(iw, one);
|
||||
|
||||
/*
|
||||
* XOR final state with the data.
|
||||
*/
|
||||
if (len < 64) {
|
||||
unsigned char tmp[64];
|
||||
size_t u;
|
||||
|
||||
_mm_storeu_si128((void *)(tmp + 0), s0);
|
||||
_mm_storeu_si128((void *)(tmp + 16), s1);
|
||||
_mm_storeu_si128((void *)(tmp + 32), s2);
|
||||
_mm_storeu_si128((void *)(tmp + 48), s3);
|
||||
for (u = 0; u < len; u ++) {
|
||||
buf[u] ^= tmp[u];
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
__m128i b0, b1, b2, b3;
|
||||
|
||||
b0 = _mm_loadu_si128((const void *)(buf + 0));
|
||||
b1 = _mm_loadu_si128((const void *)(buf + 16));
|
||||
b2 = _mm_loadu_si128((const void *)(buf + 32));
|
||||
b3 = _mm_loadu_si128((const void *)(buf + 48));
|
||||
b0 = _mm_xor_si128(b0, s0);
|
||||
b1 = _mm_xor_si128(b1, s1);
|
||||
b2 = _mm_xor_si128(b2, s2);
|
||||
b3 = _mm_xor_si128(b3, s3);
|
||||
_mm_storeu_si128((void *)(buf + 0), b0);
|
||||
_mm_storeu_si128((void *)(buf + 16), b1);
|
||||
_mm_storeu_si128((void *)(buf + 32), b2);
|
||||
_mm_storeu_si128((void *)(buf + 48), b3);
|
||||
buf += 64;
|
||||
len -= 64;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* _mm_extract_epi32() requires SSE4.1. We prefer to stick to
|
||||
* raw SSE2, thus we use _mm_extract_epi16().
|
||||
*/
|
||||
return (uint32_t)_mm_extract_epi16(iw, 0)
|
||||
| ((uint32_t)_mm_extract_epi16(iw, 1) << 16);
|
||||
}
|
||||
|
||||
/* see bearssl_block.h */
|
||||
br_chacha20_run
|
||||
br_chacha20_sse2_get(void)
|
||||
{
|
||||
/*
|
||||
* If using 64-bit mode, then SSE2 opcodes should be automatically
|
||||
* available, since they are part of the ABI.
|
||||
*
|
||||
* In 32-bit mode, we use CPUID to detect the SSE2 feature.
|
||||
*/
|
||||
|
||||
#if __x86_64__ || _M_X64
|
||||
|
||||
return &br_chacha20_sse2_run;
|
||||
|
||||
#else
|
||||
|
||||
/*
|
||||
* SSE2 support is indicated by bit 26 in EDX.
|
||||
*/
|
||||
#define MASK 0x04000000
|
||||
|
||||
#if BR_SSE2_GCC
|
||||
unsigned eax, ebx, ecx, edx;
|
||||
|
||||
if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
|
||||
if ((edx & MASK) == MASK) {
|
||||
return &br_chacha20_sse2_run;
|
||||
}
|
||||
}
|
||||
#elif BR_SSE2_MSC
|
||||
int info[4];
|
||||
|
||||
__cpuid(info, 1);
|
||||
if (((uint32_t)info[3] & MASK) == MASK) {
|
||||
return &br_chacha20_sse2_run;
|
||||
}
|
||||
#endif
|
||||
return 0;
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
/* see bearssl_block.h */
|
||||
br_chacha20_run
|
||||
br_chacha20_sse2_get(void)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
Loading…
Add table
Add a link
Reference in a new issue