j3d1/bearssl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added ChaCha20 implementation with SSE2 opcodes.

Browse source
This commit is contained in:
Thomas Pornin 2017-07-26 15:58:01 +02:00
parent 5414fd525e
commit 24c6f09bf8
9 changed files with 429 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

50
inc/bearssl_block.h
View file

@ -259,9 +259,26 @@ extern "C" {
* `chacha20_ct` is a straightforward implementation of ChaCha20 in
* plain C; it is constant-time, small, and reasonably fast.
*
* `chacha20_sse2` leverages SSE2 opcodes (on x86 architectures that
* support these opcodes). It is faster than `chacha20_ct`.
*
* `poly1305_ctmul` is an implementation of the ChaCha20+Poly1305 AEAD
* construction, where the Poly1305 part is performed with mixed 32-bit
* multiplications (operands are 32-bit, result is 64-bit).
*
* `poly1305_ctmul32` implements ChaCha20+Poly1305 using pure 32-bit
* multiplications (32-bit operands, 32-bit result). It is slower than
* `poly1305_ctmul`, except on some specific architectures such as
* the ARM Cortex M0+.
*
* `poly1305_ctmulq` implements ChaCha20+Poly1305 with mixed 64-bit
* multiplications (operands are 64-bit, result is 128-bit) on 64-bit
* platforms that support such operations.
*
* `poly1305_i15` implements ChaCha20+Poly1305 with the generic "i15"
* big integer implementation. It is meant mostly for testing purposes,
* although it can help with saving a few hundred bytes of code footprint
* on systems where code size is scarce.
*/
/**
@ -1684,6 +1701,39 @@ typedef uint32_t (*br_chacha20_run)(const void *key,
uint32_t br_chacha20_ct_run(const void *key,
const void *iv, uint32_t cc, void *data, size_t len);
/**
* \brief ChaCha20 implementation (SSE2 code, constant-time).
*
* This implementation is available only on x86 platforms, depending on
* compiler support. Moreover, in 32-bit mode, it might not actually run,
* if the underlying hardware does not implement the SSE2 opcode (in
* 64-bit mode, SSE2 is part of the ABI, so if the code could be compiled
* at all, then it can run). Use `br_chacha20_sse2_get()` to safely obtain
* a pointer to that function.
*
* \see br_chacha20_run
*
* \param key secret key (32 bytes).
* \param iv IV (12 bytes).
* \param cc initial counter value.
* \param data data to encrypt or decrypt.
* \param len data length (in bytes).
*/
uint32_t br_chacha20_sse2_run(const void *key,
const void *iv, uint32_t cc, void *data, size_t len);
/**
* \brief Obtain the `sse2` ChaCha20 implementation, if available.
*
* This function returns a pointer to `br_chacha20_sse2_run`, if
* that implementation was compiled in the library _and_ the SSE2
* opcodes are available on the currently running CPU. If either of
* these conditions is not met, then this function returns `0`.
*
* \return the `sse2` ChaCha20 implementation, or `0`.
*/
br_chacha20_run br_chacha20_sse2_get(void);
/**
* \brief Type for a ChaCha20+Poly1305 AEAD implementation.
*

5
mk/Rules.mk
View file

File diff suppressed because one or more lines are too long

1
mk/mkrules.sh
View file

@ -258,6 +258,7 @@ coresrc=" \
src/symcipher/aes_x86ni_cbcenc.c \
src/symcipher/aes_x86ni_ctr.c \
src/symcipher/chacha20_ct.c \
src/symcipher/chacha20_sse2.c \
src/symcipher/des_ct.c \
src/symcipher/des_ct_cbcdec.c \
src/symcipher/des_ct_cbcenc.c \

10
src/config.h
View file

@ -162,6 +162,16 @@
#define BR_AES_X86NI 1
*/
/*
* When BR_SSE2 is enabled, SSE2 intrinsics will be used for some
* algorithm implementations that use them (e.g. chacha20_sse2). If this
* is not enabled explicitly, then support for SSE2 intrinsics will be
* automatically detected. If set explicitly to 0, then SSE2 code will
* not be compiled at all.
*
#define BR_SSE2 1
*/
/*
* When BR_POWER8 is enabled, the AES implementation using the POWER ISA
* 2.07 opcodes (available on POWER8 processors and later) is compiled.

42
src/inner.h
View file

@ -144,10 +144,46 @@
#endif
#endif
/*
* Determine whether SSE2 intrinsics are understood by the compiler.
* Right now, we restrict ourselves to compiler versions where things
* are documented to work well:
* -- GCC 4.4+ and Clang 3.7+ understand the function attribute "target"
* -- MS Visual Studio 2005 documents the existence of <emmintrin.h>
* SSE2-powered code _might_ work with older versions, but there is no
* pressing need to do so right now.
*/
#ifndef BR_SSE2
#if (__i386__ || __x86_64__) \
&& ((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) \
|| (__clang_major__ > 3 \
|| (__clang_major__ == 3 && __clang_minor__ >= 7)))
#define BR_SSE2 1
#elif (_M_IX86 || _M_X64) && (_MSC_VER >= 1400)
#define BR_SSE2 1
#endif
#endif
/*
* If we use SSE2 intrinsics, determine the compiler brand.
*/
#if BR_SSE2
#ifndef BR_SSE2_GCC
#if __GNUC__
#define BR_SSE2_GCC 1
#endif
#endif
#ifndef BR_SSE2_MSC
#if _MSC_VER >= 1400
#define BR_SSE2_MSC 1
#endif
#endif
#endif
/*
* A macro to tag a function with a "target" attribute (for GCC and Clang).
*/
#if BR_AES_X86NI_GCC
#if BR_AES_X86NI_GCC || BR_SSE2_GCC
#define BR_TARGET(x) __attribute__((target(x)))
#else
#define BR_TARGET(x)
@ -156,8 +192,8 @@
/*
* GCC versions from 4.4 to 4.8 (inclusive) must use a special #pragma
* to activate extra opcodes before including the relevant intrinsic
* headers. But these don't work with Clang (which does not need them
* either). We also need that #pragma for GCC 4.9 in order to work
* AES-NI headers. But these don't work with Clang (which does not need
* them either). We also need that #pragma for GCC 4.9 in order to work
* around a compiler bug (it tends to blow up on ghash_pclmul code
* otherwise).
*/

24
src/ssl/ssl_engine_default_chapol.c
View file

@ -31,21 +31,35 @@ br_ssl_engine_set_default_chapol(br_ssl_engine_context *cc)
#if BR_INT128 || BR_UMUL128
br_poly1305_run bp;
#endif
#if BR_SSE2
br_chacha20_run bc;
#endif
br_ssl_engine_set_chapol(cc,
&br_sslrec_in_chapol_vtable,
&br_sslrec_out_chapol_vtable);
br_ssl_engine_set_chacha20(cc, &br_chacha20_ct_run);
#if BR_SSE2
bc = br_chacha20_sse2_get();
if (bc) {
br_ssl_engine_set_chacha20(cc, bc);
} else {
#endif
br_ssl_engine_set_chacha20(cc, &br_chacha20_ct_run);
#if BR_SSE2
}
#endif
#if BR_INT128 || BR_UMUL128
bp = br_poly1305_ctmulq_get();
if (bp) {
br_ssl_engine_set_poly1305(cc, bp);
return;
}
} else {
#endif
#if BR_LOMUL
br_ssl_engine_set_poly1305(cc, &br_poly1305_ctmul32_run);
br_ssl_engine_set_poly1305(cc, &br_poly1305_ctmul32_run);
#else
br_ssl_engine_set_poly1305(cc, &br_poly1305_ctmul_run);
br_ssl_engine_set_poly1305(cc, &br_poly1305_ctmul_run);
#endif
#if BR_INT128 || BR_UMUL128
}
#endif
}

257
src/symcipher/chacha20_sse2.c Normal file
View file

@ -0,0 +1,257 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "inner.h"
/*
* This file contains a ChaCha20 implementation that leverages SSE2
* opcodes for better performance.
*/
#if BR_SSE2
#if BR_SSE2_GCC
#include <emmintrin.h>
#include <cpuid.h>
#endif
#if BR_SSE2_MSC
#include <intrin.h>
#endif
/* see bearssl_block.h */
BR_TARGET("sse2")
uint32_t
br_chacha20_sse2_run(const void *key,
const void *iv, uint32_t cc, void *data, size_t len)
{
unsigned char *buf;
uint32_t ivtmp[4];
__m128i kw0, kw1;
__m128i iw, cw;
__m128i one;
static const uint32_t CW[] = {
0x61707865, 0x3320646e, 0x79622d32, 0x6b206574
};
buf = data;
kw0 = _mm_loadu_si128(key);
kw1 = _mm_loadu_si128((const void *)((const unsigned char *)key + 16));
ivtmp[0] = cc;
memcpy(ivtmp + 1, iv, 12);
iw = _mm_loadu_si128((const void *)ivtmp);
cw = _mm_loadu_si128((const void *)CW);
one = _mm_set_epi32(0, 0, 0, 1);
while (len > 0) {
/*
* sj contains state words 4*j to 4*j+3.
*/
__m128i s0, s1, s2, s3;
int i;
s0 = cw;
s1 = kw0;
s2 = kw1;
s3 = iw;
for (i = 0; i < 10; i ++) {
/*
* Even round is straightforward application on
* the state words.
*/
s0 = _mm_add_epi32(s0, s1);
s3 = _mm_xor_si128(s3, s0);
s3 = _mm_or_si128(
_mm_slli_epi32(s3, 16),
_mm_srli_epi32(s3, 16));
s2 = _mm_add_epi32(s2, s3);
s1 = _mm_xor_si128(s1, s2);
s1 = _mm_or_si128(
_mm_slli_epi32(s1, 12),
_mm_srli_epi32(s1, 20));
s0 = _mm_add_epi32(s0, s1);
s3 = _mm_xor_si128(s3, s0);
s3 = _mm_or_si128(
_mm_slli_epi32(s3, 8),
_mm_srli_epi32(s3, 24));
s2 = _mm_add_epi32(s2, s3);
s1 = _mm_xor_si128(s1, s2);
s1 = _mm_or_si128(
_mm_slli_epi32(s1, 7),
_mm_srli_epi32(s1, 25));
/*
* For the odd round, we must rotate some state
* words so that the computations apply on the
* right combinations of words.
*/
s1 = _mm_shuffle_epi32(s1, 0x39);
s2 = _mm_shuffle_epi32(s2, 0x4E);
s3 = _mm_shuffle_epi32(s3, 0x93);
s0 = _mm_add_epi32(s0, s1);
s3 = _mm_xor_si128(s3, s0);
s3 = _mm_or_si128(
_mm_slli_epi32(s3, 16),
_mm_srli_epi32(s3, 16));
s2 = _mm_add_epi32(s2, s3);
s1 = _mm_xor_si128(s1, s2);
s1 = _mm_or_si128(
_mm_slli_epi32(s1, 12),
_mm_srli_epi32(s1, 20));
s0 = _mm_add_epi32(s0, s1);
s3 = _mm_xor_si128(s3, s0);
s3 = _mm_or_si128(
_mm_slli_epi32(s3, 8),
_mm_srli_epi32(s3, 24));
s2 = _mm_add_epi32(s2, s3);
s1 = _mm_xor_si128(s1, s2);
s1 = _mm_or_si128(
_mm_slli_epi32(s1, 7),
_mm_srli_epi32(s1, 25));
/*
* After the odd round, we rotate back the values
* to undo the rotate at the start of the odd round.
*/
s1 = _mm_shuffle_epi32(s1, 0x93);
s2 = _mm_shuffle_epi32(s2, 0x4E);
s3 = _mm_shuffle_epi32(s3, 0x39);
}
/*
* Addition with the initial state.
*/
s0 = _mm_add_epi32(s0, cw);
s1 = _mm_add_epi32(s1, kw0);
s2 = _mm_add_epi32(s2, kw1);
s3 = _mm_add_epi32(s3, iw);
/*
* Increment block counter.
*/
iw = _mm_add_epi32(iw, one);
/*
* XOR final state with the data.
*/
if (len < 64) {
unsigned char tmp[64];
size_t u;
_mm_storeu_si128((void *)(tmp + 0), s0);
_mm_storeu_si128((void *)(tmp + 16), s1);
_mm_storeu_si128((void *)(tmp + 32), s2);
_mm_storeu_si128((void *)(tmp + 48), s3);
for (u = 0; u < len; u ++) {
buf[u] ^= tmp[u];
}
break;
} else {
__m128i b0, b1, b2, b3;
b0 = _mm_loadu_si128((const void *)(buf + 0));
b1 = _mm_loadu_si128((const void *)(buf + 16));
b2 = _mm_loadu_si128((const void *)(buf + 32));
b3 = _mm_loadu_si128((const void *)(buf + 48));
b0 = _mm_xor_si128(b0, s0);
b1 = _mm_xor_si128(b1, s1);
b2 = _mm_xor_si128(b2, s2);
b3 = _mm_xor_si128(b3, s3);
_mm_storeu_si128((void *)(buf + 0), b0);
_mm_storeu_si128((void *)(buf + 16), b1);
_mm_storeu_si128((void *)(buf + 32), b2);
_mm_storeu_si128((void *)(buf + 48), b3);
buf += 64;
len -= 64;
}
}
/*
* _mm_extract_epi32() requires SSE4.1. We prefer to stick to
* raw SSE2, thus we use _mm_extract_epi16().
*/
return (uint32_t)_mm_extract_epi16(iw, 0)
| ((uint32_t)_mm_extract_epi16(iw, 1) << 16);
}
/* see bearssl_block.h */
br_chacha20_run
br_chacha20_sse2_get(void)
{
/*
* If using 64-bit mode, then SSE2 opcodes should be automatically
* available, since they are part of the ABI.
*
* In 32-bit mode, we use CPUID to detect the SSE2 feature.
*/
#if __x86_64__ || _M_X64
return &br_chacha20_sse2_run;
#else
/*
* SSE2 support is indicated by bit 26 in EDX.
*/
#define MASK 0x04000000
#if BR_SSE2_GCC
unsigned eax, ebx, ecx, edx;
if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
if ((edx & MASK) == MASK) {
return &br_chacha20_sse2_run;
}
}
#elif BR_SSE2_MSC
int info[4];
__cpuid(info, 1);
if (((uint32_t)info[3] & MASK) == MASK) {
return &br_chacha20_sse2_run;
}
#endif
return 0;
#endif
}
#else
/* see bearssl_block.h */
br_chacha20_run
br_chacha20_sse2_get(void)
{
return 0;
}
#endif

40
test/test_crypto.c
View file

@ -4178,12 +4178,16 @@ static const struct {
};
static void
test_ChaCha20_ct(void)
test_ChaCha20_generic(const char *name, br_chacha20_run cr)
{
size_t u;
printf("Test ChaCha20_ct: ");
printf("Test %s: ", name);
fflush(stdout);
if (cr == 0) {
printf("UNAVAILABLE\n");
return;
}
for (u = 0; KAT_CHACHA20[u].skey; u ++) {
unsigned char key[32], nonce[12], plain[400], cipher[400];
@ -4199,10 +4203,11 @@ test_ChaCha20_ct(void)
for (v = 0; v < len; v ++) {
unsigned char tmp[400];
size_t w;
uint32_t cc2;
memset(tmp, 0, sizeof tmp);
memcpy(tmp, plain, v);
if (br_chacha20_ct_run(key, nonce, cc, tmp, v)
if (cr(key, nonce, cc, tmp, v)
!= cc + (uint32_t)((v + 63) >> 6))
{
fprintf(stderr, "ChaCha20: wrong counter\n");
@ -4218,7 +4223,21 @@ test_ChaCha20_ct(void)
exit(EXIT_FAILURE);
}
}
br_chacha20_ct_run(key, nonce, cc, tmp, v);
for (w = 0, cc2 = cc; w < v; w += 64, cc2 ++) {
size_t x;
x = v - w;
if (x > 64) {
x = 64;
}
if (cr(key, nonce, cc2, tmp + w, x)
!= (cc2 + 1))
{
fprintf(stderr, "ChaCha20:"
" wrong counter (2)\n");
exit(EXIT_FAILURE);
}
}
if (memcmp(tmp, plain, v) != 0) {
fprintf(stderr, "ChaCha20 KAT fail (2)\n");
exit(EXIT_FAILURE);
@ -4233,6 +4252,18 @@ test_ChaCha20_ct(void)
fflush(stdout);
}
static void
test_ChaCha20_ct(void)
{
test_ChaCha20_generic("ChaCha20_ct", &br_chacha20_ct_run);
}
static void
test_ChaCha20_sse2(void)
{
test_ChaCha20_generic("ChaCha20_sse2", br_chacha20_sse2_get());
}
static const struct {
const char *splain;
const char *saad;
@ -6173,6 +6204,7 @@ static const struct {
STU(DES_tab),
STU(DES_ct),
STU(ChaCha20_ct),
STU(ChaCha20_sse2),
STU(Poly1305_ctmul),
STU(Poly1305_ctmul32),
STU(Poly1305_ctmulq),

17
test/test_speed.c
View file

@ -175,17 +175,24 @@ test_speed_ ## fname(void) \
static void \
test_speed_ ## fname(void) \
{ \
br_chacha20_run bc; \
unsigned char key[32]; \
unsigned char buf[8192]; \
unsigned char iv[12]; \
int i; \
long num; \
\
bc = br_ ## fname ## _get(); \
if (bc == 0) { \
printf("%-30s UNAVAILABLE\n", #Name); \
fflush(stdout); \
return; \
} \
memset(key, 'T', sizeof key); \
memset(buf, 'P', sizeof buf); \
memset(iv, 'X', sizeof iv); \
for (i = 0; i < 10; i ++) { \
br_ ## fname ## _run(key, iv, i, buf, sizeof buf); \
bc(key, iv, i, buf, sizeof buf); \
} \
num = 10; \
for (;;) { \
@ -195,8 +202,7 @@ test_speed_ ## fname(void) \
\
begin = clock(); \
for (k = num; k > 0; k --) { \
br_ ## fname ## _run(key, iv, \
(uint32_t)k, buf, sizeof buf); \
bc(key, iv, (uint32_t)k, buf, sizeof buf); \
} \
end = clock(); \
tt = (double)(end - begin) / CLOCKS_PER_SEC; \
@ -232,6 +238,7 @@ SPEED_HASH(SHA-512, sha512)
#define br_aes_ct64_cbcenc_get_vtable() (&br_aes_ct64_cbcenc_vtable)
#define br_aes_ct64_cbcdec_get_vtable() (&br_aes_ct64_cbcdec_vtable)
#define br_aes_ct64_ctr_get_vtable() (&br_aes_ct64_ctr_vtable)
#define br_chacha20_ct_get() (&br_chacha20_ct_run)
#define SPEED_AES(iname) \
SPEED_BLOCKCIPHER_CBC(AES-128 CBC encrypt (iname), aes128_ ## iname ## _cbcenc, aes_ ## iname, 16, enc) \
@ -265,7 +272,8 @@ SPEED_BLOCKCIPHER_CBC(3DES CBC decrypt (iname), 3des_ ## iname ## _cbcdec, des_
SPEED_DES(tab)
SPEED_DES(ct)
SPEED_CHACHA20(ChaCha20, chacha20_ct)
SPEED_CHACHA20(ChaCha20 (ct), chacha20_ct)
SPEED_CHACHA20(ChaCha20 (sse2), chacha20_sse2)
static void
test_speed_ghash_inner(char *name, br_ghash gh)
@ -1279,6 +1287,7 @@ static const struct {
STU(3des_ct_cbcdec),
STU(chacha20_ct),
STU(chacha20_sse2),
STU(ghash_ctmul),
STU(ghash_ctmul32),