j3d1/bearssl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added AES+GHASH implementation using AES-NI opcodes; also ARM-Thumb assembly for faster Montgomery multiplication on Cortex-M0+. Added selection functions for "default" implementations.

Browse source
This commit is contained in:
Thomas Pornin 2017-01-29 21:46:33 +01:00
parent 556e525d62
commit 5f045c7599
58 changed files with 3012 additions and 395 deletions
Download patch file Download diff file Expand all files Collapse all files

263
src/symcipher/aes_x86ni.c Normal file
View file

@ -0,0 +1,263 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "inner.h"
/*
* This code contains the AES key schedule implementation using the
* AES-NI opcodes.
*/
#if BR_AES_X86NI
#if BR_AES_X86NI_GCC
#include <wmmintrin.h>
#include <cpuid.h>
#endif
#if BR_AES_X86NI_MSC
#include <intrin.h>
#endif
/* see inner.h */
int
br_aes_x86ni_supported(void)
{
/*
* Bit mask for features in ECX:
* 19 SSE4.1 (used for _mm_insert_epi32(), for AES-CTR)
* 25 AES-NI
*/
#define MASK 0x02080000
#if BR_AES_X86NI_GCC
unsigned eax, ebx, ecx, edx;
if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
return (ecx & MASK) == MASK;
} else {
return 0;
}
#elif BR_AES_X86NI_MSC
int info[4];
__cpuid(info, 1);
return ((uint32_t)info[2] & MASK) == MASK;
#else
return 0;
#endif
#undef MASK
}
BR_TARGET("sse2,aes")
static inline __m128i
expand_step128(__m128i k, __m128i k2)
{
k = _mm_xor_si128(k, _mm_slli_si128(k, 4));
k = _mm_xor_si128(k, _mm_slli_si128(k, 4));
k = _mm_xor_si128(k, _mm_slli_si128(k, 4));
k2 = _mm_shuffle_epi32(k2, 0xFF);
return _mm_xor_si128(k, k2);
}
BR_TARGET("sse2,aes")
static inline void
expand_step192(__m128i *t1, __m128i *t2, __m128i *t3)
{
__m128i t4;
*t2 = _mm_shuffle_epi32(*t2, 0x55);
t4 = _mm_slli_si128(*t1, 0x4);
*t1 = _mm_xor_si128(*t1, t4);
t4 = _mm_slli_si128(t4, 0x4);
*t1 = _mm_xor_si128(*t1, t4);
t4 = _mm_slli_si128(t4, 0x4);
*t1 = _mm_xor_si128(*t1, t4);
*t1 = _mm_xor_si128(*t1, *t2);
*t2 = _mm_shuffle_epi32(*t1, 0xFF);
t4 = _mm_slli_si128(*t3, 0x4);
*t3 = _mm_xor_si128(*t3, t4);
*t3 = _mm_xor_si128(*t3, *t2);
}
BR_TARGET("sse2,aes")
static inline void
expand_step256_1(__m128i *t1, __m128i *t2)
{
__m128i t4;
*t2 = _mm_shuffle_epi32(*t2, 0xFF);
t4 = _mm_slli_si128(*t1, 0x4);
*t1 = _mm_xor_si128(*t1, t4);
t4 = _mm_slli_si128(t4, 0x4);
*t1 = _mm_xor_si128(*t1, t4);
t4 = _mm_slli_si128(t4, 0x4);
*t1 = _mm_xor_si128(*t1, t4);
*t1 = _mm_xor_si128(*t1, *t2);
}
BR_TARGET("sse2,aes")
static inline void
expand_step256_2(__m128i *t1, __m128i *t3)
{
__m128i t2, t4;
t4 = _mm_aeskeygenassist_si128(*t1, 0x0);
t2 = _mm_shuffle_epi32(t4, 0xAA);
t4 = _mm_slli_si128(*t3, 0x4);
*t3 = _mm_xor_si128(*t3, t4);
t4 = _mm_slli_si128(t4, 0x4);
*t3 = _mm_xor_si128(*t3, t4);
t4 = _mm_slli_si128(t4, 0x4);
*t3 = _mm_xor_si128(*t3, t4);
*t3 = _mm_xor_si128(*t3, t2);
}
/*
* Perform key schedule for AES, encryption direction. Subkeys are written
* in sk[], and the number of rounds is returned. Key length MUST be 16,
* 24 or 32 bytes.
*/
BR_TARGET("sse2,aes")
static unsigned
x86ni_keysched(__m128i *sk, const void *key, size_t len)
{
const unsigned char *kb;
#define KEXP128(k, i, rcon) do { \
k = expand_step128(k, _mm_aeskeygenassist_si128(k, rcon)); \
sk[i] = k; \
} while (0)
#define KEXP192(i, rcon1, rcon2) do { \
sk[(i) + 0] = t1; \
sk[(i) + 1] = t3; \
t2 = _mm_aeskeygenassist_si128(t3, rcon1); \
expand_step192(&t1, &t2, &t3); \
sk[(i) + 1] = _mm_castpd_si128(_mm_shuffle_pd( \
_mm_castsi128_pd(sk[(i) + 1]), \
_mm_castsi128_pd(t1), 0)); \
sk[(i) + 2] = _mm_castpd_si128(_mm_shuffle_pd( \
_mm_castsi128_pd(t1), \
_mm_castsi128_pd(t3), 1)); \
t2 = _mm_aeskeygenassist_si128(t3, rcon2); \
expand_step192(&t1, &t2, &t3); \
} while (0)
#define KEXP256(i, rcon) do { \
sk[(i) + 0] = t3; \
t2 = _mm_aeskeygenassist_si128(t3, rcon); \
expand_step256_1(&t1, &t2); \
sk[(i) + 1] = t1; \
expand_step256_2(&t1, &t3); \
} while (0)
kb = key;
switch (len) {
__m128i t1, t2, t3;
case 16:
t1 = _mm_loadu_si128((const void *)kb);
sk[0] = t1;
KEXP128(t1, 1, 0x01);
KEXP128(t1, 2, 0x02);
KEXP128(t1, 3, 0x04);
KEXP128(t1, 4, 0x08);
KEXP128(t1, 5, 0x10);
KEXP128(t1, 6, 0x20);
KEXP128(t1, 7, 0x40);
KEXP128(t1, 8, 0x80);
KEXP128(t1, 9, 0x1B);
KEXP128(t1, 10, 0x36);
return 10;
case 24:
t1 = _mm_loadu_si128((const void *)kb);
t3 = _mm_loadu_si128((const void *)(kb + 8));
t3 = _mm_shuffle_epi32(t3, 0x4E);
KEXP192(0, 0x01, 0x02);
KEXP192(3, 0x04, 0x08);
KEXP192(6, 0x10, 0x20);
KEXP192(9, 0x40, 0x80);
sk[12] = t1;
return 12;
case 32:
t1 = _mm_loadu_si128((const void *)kb);
t3 = _mm_loadu_si128((const void *)(kb + 16));
sk[0] = t1;
KEXP256( 1, 0x01);
KEXP256( 3, 0x02);
KEXP256( 5, 0x04);
KEXP256( 7, 0x08);
KEXP256( 9, 0x10);
KEXP256(11, 0x20);
sk[13] = t3;
t2 = _mm_aeskeygenassist_si128(t3, 0x40);
expand_step256_1(&t1, &t2);
sk[14] = t1;
return 14;
default:
return 0;
}
#undef KEXP128
#undef KEXP192
#undef KEXP256
}
/* see inner.h */
BR_TARGET("sse2,aes")
unsigned
br_aes_x86ni_keysched_enc(unsigned char *skni, const void *key, size_t len)
{
__m128i sk[15];
unsigned num_rounds;
num_rounds = x86ni_keysched(sk, key, len);
memcpy(skni, sk, (num_rounds + 1) << 4);
return num_rounds;
}
/* see inner.h */
BR_TARGET("sse2,aes")
unsigned
br_aes_x86ni_keysched_dec(unsigned char *skni, const void *key, size_t len)
{
__m128i sk[15];
unsigned u, num_rounds;
num_rounds = x86ni_keysched(sk, key, len);
_mm_storeu_si128((void *)skni, sk[num_rounds]);
for (u = 1; u < num_rounds; u ++) {
_mm_storeu_si128((void *)(skni + (u << 4)),
_mm_aesimc_si128(sk[num_rounds - u]));
}
_mm_storeu_si128((void *)(skni + (num_rounds << 4)), sk[0]);
return num_rounds;
}
#endif

224
src/symcipher/aes_x86ni_cbcdec.c Normal file
View file

@ -0,0 +1,224 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "inner.h"
#if BR_AES_X86NI
#if BR_AES_X86NI_GCC
#include <wmmintrin.h>
#endif
#if BR_AES_X86NI_MSC
#include <intrin.h>
#endif
/* see bearssl_block.h */
void
br_aes_x86ni_cbcdec_init(br_aes_x86ni_cbcdec_keys *ctx,
const void *key, size_t len)
{
ctx->vtable = &br_aes_x86ni_cbcdec_vtable;
ctx->num_rounds = br_aes_x86ni_keysched_dec(ctx->skey.skni, key, len);
}
/* see bearssl_block.h */
BR_TARGET("sse2,aes")
void
br_aes_x86ni_cbcdec_run(const br_aes_x86ni_cbcdec_keys *ctx,
void *iv, void *data, size_t len)
{
unsigned char *buf;
unsigned num_rounds;
__m128i sk[15], ivx;
unsigned u;
buf = data;
ivx = _mm_loadu_si128(iv);
num_rounds = ctx->num_rounds;
for (u = 0; u <= num_rounds; u ++) {
sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
}
while (len > 0) {
__m128i x0, x1, x2, x3, e0, e1, e2, e3;
x0 = _mm_loadu_si128((void *)(buf + 0));
if (len >= 64) {
x1 = _mm_loadu_si128((void *)(buf + 16));
x2 = _mm_loadu_si128((void *)(buf + 32));
x3 = _mm_loadu_si128((void *)(buf + 48));
} else {
x0 = _mm_loadu_si128((void *)(buf + 0));
if (len >= 32) {
x1 = _mm_loadu_si128((void *)(buf + 16));
if (len >= 48) {
x2 = _mm_loadu_si128(
(void *)(buf + 32));
} else {
x2 = x0;
}
} else {
x1 = x0;
x2 = x0;
}
x3 = x0;
}
e0 = x0;
e1 = x1;
e2 = x2;
e3 = x3;
x0 = _mm_xor_si128(x0, sk[0]);
x1 = _mm_xor_si128(x1, sk[0]);
x2 = _mm_xor_si128(x2, sk[0]);
x3 = _mm_xor_si128(x3, sk[0]);
x0 = _mm_aesdec_si128(x0, sk[1]);
x1 = _mm_aesdec_si128(x1, sk[1]);
x2 = _mm_aesdec_si128(x2, sk[1]);
x3 = _mm_aesdec_si128(x3, sk[1]);
x0 = _mm_aesdec_si128(x0, sk[2]);
x1 = _mm_aesdec_si128(x1, sk[2]);
x2 = _mm_aesdec_si128(x2, sk[2]);
x3 = _mm_aesdec_si128(x3, sk[2]);
x0 = _mm_aesdec_si128(x0, sk[3]);
x1 = _mm_aesdec_si128(x1, sk[3]);
x2 = _mm_aesdec_si128(x2, sk[3]);
x3 = _mm_aesdec_si128(x3, sk[3]);
x0 = _mm_aesdec_si128(x0, sk[4]);
x1 = _mm_aesdec_si128(x1, sk[4]);
x2 = _mm_aesdec_si128(x2, sk[4]);
x3 = _mm_aesdec_si128(x3, sk[4]);
x0 = _mm_aesdec_si128(x0, sk[5]);
x1 = _mm_aesdec_si128(x1, sk[5]);
x2 = _mm_aesdec_si128(x2, sk[5]);
x3 = _mm_aesdec_si128(x3, sk[5]);
x0 = _mm_aesdec_si128(x0, sk[6]);
x1 = _mm_aesdec_si128(x1, sk[6]);
x2 = _mm_aesdec_si128(x2, sk[6]);
x3 = _mm_aesdec_si128(x3, sk[6]);
x0 = _mm_aesdec_si128(x0, sk[7]);
x1 = _mm_aesdec_si128(x1, sk[7]);
x2 = _mm_aesdec_si128(x2, sk[7]);
x3 = _mm_aesdec_si128(x3, sk[7]);
x0 = _mm_aesdec_si128(x0, sk[8]);
x1 = _mm_aesdec_si128(x1, sk[8]);
x2 = _mm_aesdec_si128(x2, sk[8]);
x3 = _mm_aesdec_si128(x3, sk[8]);
x0 = _mm_aesdec_si128(x0, sk[9]);
x1 = _mm_aesdec_si128(x1, sk[9]);
x2 = _mm_aesdec_si128(x2, sk[9]);
x3 = _mm_aesdec_si128(x3, sk[9]);
if (num_rounds == 10) {
x0 = _mm_aesdeclast_si128(x0, sk[10]);
x1 = _mm_aesdeclast_si128(x1, sk[10]);
x2 = _mm_aesdeclast_si128(x2, sk[10]);
x3 = _mm_aesdeclast_si128(x3, sk[10]);
} else if (num_rounds == 12) {
x0 = _mm_aesdec_si128(x0, sk[10]);
x1 = _mm_aesdec_si128(x1, sk[10]);
x2 = _mm_aesdec_si128(x2, sk[10]);
x3 = _mm_aesdec_si128(x3, sk[10]);
x0 = _mm_aesdec_si128(x0, sk[11]);
x1 = _mm_aesdec_si128(x1, sk[11]);
x2 = _mm_aesdec_si128(x2, sk[11]);
x3 = _mm_aesdec_si128(x3, sk[11]);
x0 = _mm_aesdeclast_si128(x0, sk[12]);
x1 = _mm_aesdeclast_si128(x1, sk[12]);
x2 = _mm_aesdeclast_si128(x2, sk[12]);
x3 = _mm_aesdeclast_si128(x3, sk[12]);
} else {
x0 = _mm_aesdec_si128(x0, sk[10]);
x1 = _mm_aesdec_si128(x1, sk[10]);
x2 = _mm_aesdec_si128(x2, sk[10]);
x3 = _mm_aesdec_si128(x3, sk[10]);
x0 = _mm_aesdec_si128(x0, sk[11]);
x1 = _mm_aesdec_si128(x1, sk[11]);
x2 = _mm_aesdec_si128(x2, sk[11]);
x3 = _mm_aesdec_si128(x3, sk[11]);
x0 = _mm_aesdec_si128(x0, sk[12]);
x1 = _mm_aesdec_si128(x1, sk[12]);
x2 = _mm_aesdec_si128(x2, sk[12]);
x3 = _mm_aesdec_si128(x3, sk[12]);
x0 = _mm_aesdec_si128(x0, sk[13]);
x1 = _mm_aesdec_si128(x1, sk[13]);
x2 = _mm_aesdec_si128(x2, sk[13]);
x3 = _mm_aesdec_si128(x3, sk[13]);
x0 = _mm_aesdeclast_si128(x0, sk[14]);
x1 = _mm_aesdeclast_si128(x1, sk[14]);
x2 = _mm_aesdeclast_si128(x2, sk[14]);
x3 = _mm_aesdeclast_si128(x3, sk[14]);
}
x0 = _mm_xor_si128(x0, ivx);
x1 = _mm_xor_si128(x1, e0);
x2 = _mm_xor_si128(x2, e1);
x3 = _mm_xor_si128(x3, e2);
ivx = e3;
_mm_storeu_si128((void *)(buf + 0), x0);
if (len >= 64) {
_mm_storeu_si128((void *)(buf + 16), x1);
_mm_storeu_si128((void *)(buf + 32), x2);
_mm_storeu_si128((void *)(buf + 48), x3);
buf += 64;
len -= 64;
} else {
if (len >= 32) {
_mm_storeu_si128((void *)(buf + 16), x1);
if (len >= 48) {
_mm_storeu_si128(
(void *)(buf + 32), x2);
}
}
break;
}
}
_mm_storeu_si128(iv, ivx);
}
/* see bearssl_block.h */
const br_block_cbcdec_class br_aes_x86ni_cbcdec_vtable = {
sizeof(br_aes_x86ni_cbcdec_keys),
16,
4,
(void (*)(const br_block_cbcdec_class **, const void *, size_t))
&br_aes_x86ni_cbcdec_init,
(void (*)(const br_block_cbcdec_class *const *, void *, void *, size_t))
&br_aes_x86ni_cbcdec_run
};
/* see bearssl_block.h */
const br_block_cbcdec_class *
br_aes_x86ni_cbcdec_get_vtable(void)
{
return br_aes_x86ni_supported() ? &br_aes_x86ni_cbcdec_vtable : NULL;
}
#else
/* see bearssl_block.h */
const br_block_cbcdec_class *
br_aes_x86ni_cbcdec_get_vtable(void)
{
return NULL;
}
#endif

125
src/symcipher/aes_x86ni_cbcenc.c Normal file
View file

@ -0,0 +1,125 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "inner.h"
#if BR_AES_X86NI
#if BR_AES_X86NI_GCC
#include <wmmintrin.h>
#endif
#if BR_AES_X86NI_MSC
#include <intrin.h>
#endif
/* see bearssl_block.h */
void
br_aes_x86ni_cbcenc_init(br_aes_x86ni_cbcenc_keys *ctx,
const void *key, size_t len)
{
ctx->vtable = &br_aes_x86ni_cbcenc_vtable;
ctx->num_rounds = br_aes_x86ni_keysched_enc(ctx->skey.skni, key, len);
}
/* see bearssl_block.h */
BR_TARGET("sse2,aes")
void
br_aes_x86ni_cbcenc_run(const br_aes_x86ni_cbcenc_keys *ctx,
void *iv, void *data, size_t len)
{
unsigned char *buf;
unsigned num_rounds;
__m128i sk[15], ivx;
unsigned u;
buf = data;
ivx = _mm_loadu_si128(iv);
num_rounds = ctx->num_rounds;
for (u = 0; u <= num_rounds; u ++) {
sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
}
while (len > 0) {
__m128i x;
x = _mm_xor_si128(_mm_loadu_si128((void *)buf), ivx);
x = _mm_xor_si128(x, sk[0]);
x = _mm_aesenc_si128(x, sk[1]);
x = _mm_aesenc_si128(x, sk[2]);
x = _mm_aesenc_si128(x, sk[3]);
x = _mm_aesenc_si128(x, sk[4]);
x = _mm_aesenc_si128(x, sk[5]);
x = _mm_aesenc_si128(x, sk[6]);
x = _mm_aesenc_si128(x, sk[7]);
x = _mm_aesenc_si128(x, sk[8]);
x = _mm_aesenc_si128(x, sk[9]);
if (num_rounds == 10) {
x = _mm_aesenclast_si128(x, sk[10]);
} else if (num_rounds == 12) {
x = _mm_aesenc_si128(x, sk[10]);
x = _mm_aesenc_si128(x, sk[11]);
x = _mm_aesenclast_si128(x, sk[12]);
} else {
x = _mm_aesenc_si128(x, sk[10]);
x = _mm_aesenc_si128(x, sk[11]);
x = _mm_aesenc_si128(x, sk[12]);
x = _mm_aesenc_si128(x, sk[13]);
x = _mm_aesenclast_si128(x, sk[14]);
}
ivx = x;
_mm_storeu_si128((void *)buf, x);
buf += 16;
len -= 16;
}
_mm_storeu_si128(iv, ivx);
}
/* see bearssl_block.h */
const br_block_cbcenc_class br_aes_x86ni_cbcenc_vtable = {
sizeof(br_aes_x86ni_cbcenc_keys),
16,
4,
(void (*)(const br_block_cbcenc_class **, const void *, size_t))
&br_aes_x86ni_cbcenc_init,
(void (*)(const br_block_cbcenc_class *const *, void *, void *, size_t))
&br_aes_x86ni_cbcenc_run
};
/* see bearssl_block.h */
const br_block_cbcenc_class *
br_aes_x86ni_cbcenc_get_vtable(void)
{
return br_aes_x86ni_supported() ? &br_aes_x86ni_cbcenc_vtable : NULL;
}
#else
/* see bearssl_block.h */
const br_block_cbcenc_class *
br_aes_x86ni_cbcenc_get_vtable(void)
{
return NULL;
}
#endif

218
src/symcipher/aes_x86ni_ctr.c Normal file
View file

@ -0,0 +1,218 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "inner.h"
#if BR_AES_X86NI
#if BR_AES_X86NI_GCC
#include <smmintrin.h>
#include <wmmintrin.h>
#define bswap32 __builtin_bswap32
#endif
#if BR_AES_X86NI_MSC
#include <stdlib.h>
#include <intrin.h>
#define bswap32 _byteswap_ulong
#endif
/* see bearssl_block.h */
void
br_aes_x86ni_ctr_init(br_aes_x86ni_ctr_keys *ctx,
const void *key, size_t len)
{
ctx->vtable = &br_aes_x86ni_ctr_vtable;
ctx->num_rounds = br_aes_x86ni_keysched_enc(ctx->skey.skni, key, len);
}
/* see bearssl_block.h */
BR_TARGET("sse2,sse4.1,aes")
uint32_t
br_aes_x86ni_ctr_run(const br_aes_x86ni_ctr_keys *ctx,
const void *iv, uint32_t cc, void *data, size_t len)
{
unsigned char *buf;
unsigned char ivbuf[16];
unsigned num_rounds;
__m128i sk[15];
__m128i ivx;
unsigned u;
buf = data;
memcpy(ivbuf, iv, 12);
num_rounds = ctx->num_rounds;
for (u = 0; u <= num_rounds; u ++) {
sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
}
ivx = _mm_loadu_si128((void *)ivbuf);
while (len > 0) {
__m128i x0, x1, x2, x3;
x0 = _mm_insert_epi32(ivx, bswap32(cc + 0), 3);
x1 = _mm_insert_epi32(ivx, bswap32(cc + 1), 3);
x2 = _mm_insert_epi32(ivx, bswap32(cc + 2), 3);
x3 = _mm_insert_epi32(ivx, bswap32(cc + 3), 3);
x0 = _mm_xor_si128(x0, sk[0]);
x1 = _mm_xor_si128(x1, sk[0]);
x2 = _mm_xor_si128(x2, sk[0]);
x3 = _mm_xor_si128(x3, sk[0]);
x0 = _mm_aesenc_si128(x0, sk[1]);
x1 = _mm_aesenc_si128(x1, sk[1]);
x2 = _mm_aesenc_si128(x2, sk[1]);
x3 = _mm_aesenc_si128(x3, sk[1]);
x0 = _mm_aesenc_si128(x0, sk[2]);
x1 = _mm_aesenc_si128(x1, sk[2]);
x2 = _mm_aesenc_si128(x2, sk[2]);
x3 = _mm_aesenc_si128(x3, sk[2]);
x0 = _mm_aesenc_si128(x0, sk[3]);
x1 = _mm_aesenc_si128(x1, sk[3]);
x2 = _mm_aesenc_si128(x2, sk[3]);
x3 = _mm_aesenc_si128(x3, sk[3]);
x0 = _mm_aesenc_si128(x0, sk[4]);
x1 = _mm_aesenc_si128(x1, sk[4]);
x2 = _mm_aesenc_si128(x2, sk[4]);
x3 = _mm_aesenc_si128(x3, sk[4]);
x0 = _mm_aesenc_si128(x0, sk[5]);
x1 = _mm_aesenc_si128(x1, sk[5]);
x2 = _mm_aesenc_si128(x2, sk[5]);
x3 = _mm_aesenc_si128(x3, sk[5]);
x0 = _mm_aesenc_si128(x0, sk[6]);
x1 = _mm_aesenc_si128(x1, sk[6]);
x2 = _mm_aesenc_si128(x2, sk[6]);
x3 = _mm_aesenc_si128(x3, sk[6]);
x0 = _mm_aesenc_si128(x0, sk[7]);
x1 = _mm_aesenc_si128(x1, sk[7]);
x2 = _mm_aesenc_si128(x2, sk[7]);
x3 = _mm_aesenc_si128(x3, sk[7]);
x0 = _mm_aesenc_si128(x0, sk[8]);
x1 = _mm_aesenc_si128(x1, sk[8]);
x2 = _mm_aesenc_si128(x2, sk[8]);
x3 = _mm_aesenc_si128(x3, sk[8]);
x0 = _mm_aesenc_si128(x0, sk[9]);
x1 = _mm_aesenc_si128(x1, sk[9]);
x2 = _mm_aesenc_si128(x2, sk[9]);
x3 = _mm_aesenc_si128(x3, sk[9]);
if (num_rounds == 10) {
x0 = _mm_aesenclast_si128(x0, sk[10]);
x1 = _mm_aesenclast_si128(x1, sk[10]);
x2 = _mm_aesenclast_si128(x2, sk[10]);
x3 = _mm_aesenclast_si128(x3, sk[10]);
} else if (num_rounds == 12) {
x0 = _mm_aesenc_si128(x0, sk[10]);
x1 = _mm_aesenc_si128(x1, sk[10]);
x2 = _mm_aesenc_si128(x2, sk[10]);
x3 = _mm_aesenc_si128(x3, sk[10]);
x0 = _mm_aesenc_si128(x0, sk[11]);
x1 = _mm_aesenc_si128(x1, sk[11]);
x2 = _mm_aesenc_si128(x2, sk[11]);
x3 = _mm_aesenc_si128(x3, sk[11]);
x0 = _mm_aesenclast_si128(x0, sk[12]);
x1 = _mm_aesenclast_si128(x1, sk[12]);
x2 = _mm_aesenclast_si128(x2, sk[12]);
x3 = _mm_aesenclast_si128(x3, sk[12]);
} else {
x0 = _mm_aesenc_si128(x0, sk[10]);
x1 = _mm_aesenc_si128(x1, sk[10]);
x2 = _mm_aesenc_si128(x2, sk[10]);
x3 = _mm_aesenc_si128(x3, sk[10]);
x0 = _mm_aesenc_si128(x0, sk[11]);
x1 = _mm_aesenc_si128(x1, sk[11]);
x2 = _mm_aesenc_si128(x2, sk[11]);
x3 = _mm_aesenc_si128(x3, sk[11]);
x0 = _mm_aesenc_si128(x0, sk[12]);
x1 = _mm_aesenc_si128(x1, sk[12]);
x2 = _mm_aesenc_si128(x2, sk[12]);
x3 = _mm_aesenc_si128(x3, sk[12]);
x0 = _mm_aesenc_si128(x0, sk[13]);
x1 = _mm_aesenc_si128(x1, sk[13]);
x2 = _mm_aesenc_si128(x2, sk[13]);
x3 = _mm_aesenc_si128(x3, sk[13]);
x0 = _mm_aesenclast_si128(x0, sk[14]);
x1 = _mm_aesenclast_si128(x1, sk[14]);
x2 = _mm_aesenclast_si128(x2, sk[14]);
x3 = _mm_aesenclast_si128(x3, sk[14]);
}
if (len >= 64) {
x0 = _mm_xor_si128(x0,
_mm_loadu_si128((void *)(buf + 0)));
x1 = _mm_xor_si128(x1,
_mm_loadu_si128((void *)(buf + 16)));
x2 = _mm_xor_si128(x2,
_mm_loadu_si128((void *)(buf + 32)));
x3 = _mm_xor_si128(x3,
_mm_loadu_si128((void *)(buf + 48)));
_mm_storeu_si128((void *)(buf + 0), x0);
_mm_storeu_si128((void *)(buf + 16), x1);
_mm_storeu_si128((void *)(buf + 32), x2);
_mm_storeu_si128((void *)(buf + 48), x3);
buf += 64;
len -= 64;
cc += 4;
} else {
unsigned char tmp[64];
_mm_storeu_si128((void *)(tmp + 0), x0);
_mm_storeu_si128((void *)(tmp + 16), x1);
_mm_storeu_si128((void *)(tmp + 32), x2);
_mm_storeu_si128((void *)(tmp + 48), x3);
for (u = 0; u < len; u ++) {
buf[u] ^= tmp[u];
}
cc += (uint32_t)len >> 4;
break;
}
}
return cc;
}
/* see bearssl_block.h */
const br_block_ctr_class br_aes_x86ni_ctr_vtable = {
sizeof(br_aes_x86ni_ctr_keys),
16,
4,
(void (*)(const br_block_ctr_class **, const void *, size_t))
&br_aes_x86ni_ctr_init,
(uint32_t (*)(const br_block_ctr_class *const *,
const void *, uint32_t, void *, size_t))
&br_aes_x86ni_ctr_run
};
/* see bearssl_block.h */
const br_block_ctr_class *
br_aes_x86ni_ctr_get_vtable(void)
{
return br_aes_x86ni_supported() ? &br_aes_x86ni_ctr_vtable : NULL;
}
#else
/* see bearssl_block.h */
const br_block_ctr_class *
br_aes_x86ni_ctr_get_vtable(void)
{
return NULL;
}
#endif