New AES and GHASH implementations using POWER8 crypto opcodes.

This commit is contained in:
Thomas Pornin 2017-02-15 14:08:37 +00:00
parent f0c0046601
commit db8f1b6645
16 changed files with 3186 additions and 27 deletions

445
src/symcipher/aes_pwr8.c Normal file
View file

@ -0,0 +1,445 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#define BR_POWER_ASM_MACROS 1
#include "inner.h"
/*
* This code contains the AES key schedule implementation using the
* POWER8 opcodes.
*/
#if BR_POWER8
static void
key_schedule_128(unsigned char *sk, const unsigned char *key)
{
long cc;
static const uint32_t fmod[] = { 0x11B, 0x11B, 0x11B, 0x11B };
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc = 0;
/*
* We use the VSX instructions for loading and storing the
* key/subkeys, since they support unaligned accesses. The rest
* of the computation is VMX only. VMX register 0 is VSX
* register 32.
*/
asm volatile (
/*
* v0 = all-zero word
* v1 = constant -8 / +8, copied into four words
* v2 = current subkey
* v3 = Rcon (x4 words)
* v6 = constant 8, copied into four words
* v7 = constant 0x11B, copied into four words
* v8 = constant for byteswapping words
*/
vspltisw(0, 0)
#if BR_POWER8_LE
vspltisw(1, -8)
#else
vspltisw(1, 8)
#endif
lxvw4x(34, 0, %[key])
vspltisw(3, 1)
vspltisw(6, 8)
lxvw4x(39, 0, %[fmod])
#if BR_POWER8_LE
lxvw4x(40, 0, %[idx2be])
#endif
/*
* First subkey is a copy of the key itself.
*/
#if BR_POWER8_LE
vperm(4, 2, 2, 8)
stxvw4x(36, 0, %[sk])
#else
stxvw4x(34, 0, %[sk])
#endif
/*
* Loop must run 10 times.
*/
li(%[cc], 10)
mtctr(%[cc])
label(loop)
/* Increment subkey address */
addi(%[sk], %[sk], 16)
/* Compute SubWord(RotWord(temp)) xor Rcon (into v4, splat) */
vrlw(4, 2, 1)
vsbox(4, 4)
#if BR_POWER8_LE
vxor(4, 4, 3)
#else
vsldoi(5, 3, 0, 3)
vxor(4, 4, 5)
#endif
vspltw(4, 4, 3)
/* XOR words for next subkey */
vsldoi(5, 0, 2, 12)
vxor(2, 2, 5)
vsldoi(5, 0, 2, 12)
vxor(2, 2, 5)
vsldoi(5, 0, 2, 12)
vxor(2, 2, 5)
vxor(2, 2, 4)
/* Store next subkey */
#if BR_POWER8_LE
vperm(4, 2, 2, 8)
stxvw4x(36, 0, %[sk])
#else
stxvw4x(34, 0, %[sk])
#endif
/* Update Rcon */
vadduwm(3, 3, 3)
vsrw(4, 3, 6)
vsubuwm(4, 0, 4)
vand(4, 4, 7)
vxor(3, 3, 4)
bdnz(loop)
: [sk] "+b" (sk), [cc] "+b" (cc)
: [key] "b" (key), [fmod] "b" (fmod)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "ctr", "memory"
);
}
static void
key_schedule_192(unsigned char *sk, const unsigned char *key)
{
long cc;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc = 0;
/*
* We use the VSX instructions for loading and storing the
* key/subkeys, since they support unaligned accesses. The rest
* of the computation is VMX only. VMX register 0 is VSX
* register 32.
*/
asm volatile (
/*
* v0 = all-zero word
* v1 = constant -8 / +8, copied into four words
* v2, v3 = current subkey
* v5 = Rcon (x4 words) (already shifted on big-endian)
* v6 = constant 8, copied into four words
* v8 = constant for byteswapping words
*
* The left two words of v3 are ignored.
*/
vspltisw(0, 0)
#if BR_POWER8_LE
vspltisw(1, -8)
#else
vspltisw(1, 8)
#endif
li(%[cc], 8)
lxvw4x(34, 0, %[key])
lxvw4x(35, %[cc], %[key])
vsldoi(3, 3, 0, 8)
vspltisw(5, 1)
#if !BR_POWER8_LE
vsldoi(5, 5, 0, 3)
#endif
vspltisw(6, 8)
#if BR_POWER8_LE
lxvw4x(40, 0, %[idx2be])
#endif
/*
* Loop must run 8 times. Each iteration produces 256
* bits of subkeys, with a 64-bit overlap.
*/
li(%[cc], 8)
mtctr(%[cc])
li(%[cc], 16)
label(loop)
/*
* Last 6 words in v2:v3l. Compute next 6 words into
* v3r:v4.
*/
vrlw(10, 3, 1)
vsbox(10, 10)
vxor(10, 10, 5)
vspltw(10, 10, 1)
vsldoi(11, 0, 10, 8)
vsldoi(12, 0, 2, 12)
vxor(12, 2, 12)
vsldoi(13, 0, 12, 12)
vxor(12, 12, 13)
vsldoi(13, 0, 12, 12)
vxor(12, 12, 13)
vspltw(13, 12, 3)
vxor(13, 13, 3)
vsldoi(14, 0, 3, 12)
vxor(13, 13, 14)
vsldoi(4, 12, 13, 8)
vsldoi(14, 0, 3, 8)
vsldoi(3, 14, 12, 8)
vxor(3, 3, 11)
vxor(4, 4, 10)
/*
* Update Rcon. Since for a 192-bit key, we use only 8
* such constants, we will not hit the field modulus,
* so a simple shift (addition) works well.
*/
vadduwm(5, 5, 5)
/*
* Write out the two left 128-bit words
*/
#if BR_POWER8_LE
vperm(10, 2, 2, 8)
vperm(11, 3, 3, 8)
stxvw4x(42, 0, %[sk])
stxvw4x(43, %[cc], %[sk])
#else
stxvw4x(34, 0, %[sk])
stxvw4x(35, %[cc], %[sk])
#endif
addi(%[sk], %[sk], 24)
/*
* Shift words for next iteration.
*/
vsldoi(2, 3, 4, 8)
vsldoi(3, 4, 0, 8)
bdnz(loop)
/*
* The loop wrote the first 50 subkey words, but we need
* to produce 52, so we must do one last write.
*/
#if BR_POWER8_LE
vperm(10, 2, 2, 8)
stxvw4x(42, 0, %[sk])
#else
stxvw4x(34, 0, %[sk])
#endif
: [sk] "+b" (sk), [cc] "+b" (cc)
: [key] "b" (key)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v8", "v9", "v10", "v11", "v12", "v13", "v14", "ctr", "memory"
);
}
static void
key_schedule_256(unsigned char *sk, const unsigned char *key)
{
long cc;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc = 0;
/*
* We use the VSX instructions for loading and storing the
* key/subkeys, since they support unaligned accesses. The rest
* of the computation is VMX only. VMX register 0 is VSX
* register 32.
*/
asm volatile (
/*
* v0 = all-zero word
* v1 = constant -8 / +8, copied into four words
* v2, v3 = current subkey
* v6 = Rcon (x4 words) (already shifted on big-endian)
* v7 = constant 8, copied into four words
* v8 = constant for byteswapping words
*
* The left two words of v3 are ignored.
*/
vspltisw(0, 0)
#if BR_POWER8_LE
vspltisw(1, -8)
#else
vspltisw(1, 8)
#endif
li(%[cc], 16)
lxvw4x(34, 0, %[key])
lxvw4x(35, %[cc], %[key])
vspltisw(6, 1)
#if !BR_POWER8_LE
vsldoi(6, 6, 0, 3)
#endif
vspltisw(7, 8)
#if BR_POWER8_LE
lxvw4x(40, 0, %[idx2be])
#endif
/*
* Loop must run 7 times. Each iteration produces two
* subkeys.
*/
li(%[cc], 7)
mtctr(%[cc])
li(%[cc], 16)
label(loop)
/*
* Current words are in v2:v3. Compute next word in v4.
*/
vrlw(10, 3, 1)
vsbox(10, 10)
vxor(10, 10, 6)
vspltw(10, 10, 3)
vsldoi(4, 0, 2, 12)
vxor(4, 2, 4)
vsldoi(5, 0, 4, 12)
vxor(4, 4, 5)
vsldoi(5, 0, 4, 12)
vxor(4, 4, 5)
vxor(4, 4, 10)
/*
* Then other word in v5.
*/
vsbox(10, 4)
vspltw(10, 10, 3)
vsldoi(5, 0, 3, 12)
vxor(5, 3, 5)
vsldoi(11, 0, 5, 12)
vxor(5, 5, 11)
vsldoi(11, 0, 5, 12)
vxor(5, 5, 11)
vxor(5, 5, 10)
/*
* Update Rcon. Since for a 256-bit key, we use only 7
* such constants, we will not hit the field modulus,
* so a simple shift (addition) works well.
*/
vadduwm(6, 6, 6)
/*
* Write out the two left 128-bit words
*/
#if BR_POWER8_LE
vperm(10, 2, 2, 8)
vperm(11, 3, 3, 8)
stxvw4x(42, 0, %[sk])
stxvw4x(43, %[cc], %[sk])
#else
stxvw4x(34, 0, %[sk])
stxvw4x(35, %[cc], %[sk])
#endif
addi(%[sk], %[sk], 32)
/*
* Replace v2:v3 with v4:v5.
*/
vxor(2, 0, 4)
vxor(3, 0, 5)
bdnz(loop)
/*
* The loop wrote the first 14 subkeys, but we need 15,
* so we must do an extra write.
*/
#if BR_POWER8_LE
vperm(10, 2, 2, 8)
stxvw4x(42, 0, %[sk])
#else
stxvw4x(34, 0, %[sk])
#endif
: [sk] "+b" (sk), [cc] "+b" (cc)
: [key] "b" (key)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v8", "v9", "v10", "v11", "v12", "v13", "v14", "ctr", "memory"
);
}
/* see inner.h */
int
br_aes_pwr8_supported(void)
{
return 1;
}
/* see inner.h */
unsigned
br_aes_pwr8_keysched(unsigned char *sk, const void *key, size_t len)
{
switch (len) {
case 16:
key_schedule_128(sk, key);
return 10;
case 24:
key_schedule_192(sk, key);
return 12;
default:
key_schedule_256(sk, key);
return 14;
}
}
#endif

View file

@ -0,0 +1,670 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#define BR_POWER_ASM_MACROS 1
#include "inner.h"
#if BR_POWER8
/* see bearssl_block.h */
void
br_aes_pwr8_cbcdec_init(br_aes_pwr8_cbcdec_keys *ctx,
const void *key, size_t len)
{
ctx->vtable = &br_aes_pwr8_cbcdec_vtable;
ctx->num_rounds = br_aes_pwr8_keysched(ctx->skey.skni, key, len);
}
static void
cbcdec_128(const unsigned char *sk,
const unsigned char *iv, unsigned char *buf, size_t num_blocks)
{
long cc0, cc1, cc2, cc3;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc0 = 0;
cc1 = 16;
cc2 = 32;
cc3 = 48;
asm volatile (
/*
* Load subkeys into v0..v10
*/
lxvw4x(32, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(33, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(34, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(35, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(36, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(37, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(38, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(39, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(40, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(41, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(42, %[cc0], %[sk])
li(%[cc0], 0)
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* Load IV into v24.
*/
lxvw4x(56, 0, %[iv])
#if BR_POWER8_LE
vperm(24, 24, 24, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Load next ciphertext words in v16..v19. Also save them
* in v20..v23.
*/
lxvw4x(48, %[cc0], %[buf])
lxvw4x(49, %[cc1], %[buf])
lxvw4x(50, %[cc2], %[buf])
lxvw4x(51, %[cc3], %[buf])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
vand(20, 16, 16)
vand(21, 17, 17)
vand(22, 18, 18)
vand(23, 19, 19)
/*
* Decrypt the blocks.
*/
vxor(16, 16, 10)
vxor(17, 17, 10)
vxor(18, 18, 10)
vxor(19, 19, 10)
vncipher(16, 16, 9)
vncipher(17, 17, 9)
vncipher(18, 18, 9)
vncipher(19, 19, 9)
vncipher(16, 16, 8)
vncipher(17, 17, 8)
vncipher(18, 18, 8)
vncipher(19, 19, 8)
vncipher(16, 16, 7)
vncipher(17, 17, 7)
vncipher(18, 18, 7)
vncipher(19, 19, 7)
vncipher(16, 16, 6)
vncipher(17, 17, 6)
vncipher(18, 18, 6)
vncipher(19, 19, 6)
vncipher(16, 16, 5)
vncipher(17, 17, 5)
vncipher(18, 18, 5)
vncipher(19, 19, 5)
vncipher(16, 16, 4)
vncipher(17, 17, 4)
vncipher(18, 18, 4)
vncipher(19, 19, 4)
vncipher(16, 16, 3)
vncipher(17, 17, 3)
vncipher(18, 18, 3)
vncipher(19, 19, 3)
vncipher(16, 16, 2)
vncipher(17, 17, 2)
vncipher(18, 18, 2)
vncipher(19, 19, 2)
vncipher(16, 16, 1)
vncipher(17, 17, 1)
vncipher(18, 18, 1)
vncipher(19, 19, 1)
vncipherlast(16, 16, 0)
vncipherlast(17, 17, 0)
vncipherlast(18, 18, 0)
vncipherlast(19, 19, 0)
/*
* XOR decrypted blocks with IV / previous block.
*/
vxor(16, 16, 24)
vxor(17, 17, 20)
vxor(18, 18, 21)
vxor(19, 19, 22)
/*
* Store back result (with byteswap)
*/
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
stxvw4x(48, %[cc0], %[buf])
stxvw4x(49, %[cc1], %[buf])
stxvw4x(50, %[cc2], %[buf])
stxvw4x(51, %[cc3], %[buf])
/*
* Fourth encrypted block is IV for next run.
*/
vand(24, 23, 23)
addi(%[buf], %[buf], 64)
bdnz(loop)
: [cc0] "+b" (cc0), [cc1] "+b" (cc1), [cc2] "+b" (cc2), [cc3] "+b" (cc3),
[buf] "+b" (buf)
: [sk] "b" (sk), [iv] "b" (iv), [num_blocks] "b" (num_blocks >> 2)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
"ctr", "memory"
);
}
static void
cbcdec_192(const unsigned char *sk,
const unsigned char *iv, unsigned char *buf, size_t num_blocks)
{
long cc0, cc1, cc2, cc3;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc0 = 0;
cc1 = 16;
cc2 = 32;
cc3 = 48;
asm volatile (
/*
* Load subkeys into v0..v12
*/
lxvw4x(32, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(33, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(34, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(35, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(36, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(37, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(38, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(39, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(40, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(41, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(42, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(43, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(44, %[cc0], %[sk])
li(%[cc0], 0)
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* Load IV into v24.
*/
lxvw4x(56, 0, %[iv])
#if BR_POWER8_LE
vperm(24, 24, 24, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Load next ciphertext words in v16..v19. Also save them
* in v20..v23.
*/
lxvw4x(48, %[cc0], %[buf])
lxvw4x(49, %[cc1], %[buf])
lxvw4x(50, %[cc2], %[buf])
lxvw4x(51, %[cc3], %[buf])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
vand(20, 16, 16)
vand(21, 17, 17)
vand(22, 18, 18)
vand(23, 19, 19)
/*
* Decrypt the blocks.
*/
vxor(16, 16, 12)
vxor(17, 17, 12)
vxor(18, 18, 12)
vxor(19, 19, 12)
vncipher(16, 16, 11)
vncipher(17, 17, 11)
vncipher(18, 18, 11)
vncipher(19, 19, 11)
vncipher(16, 16, 10)
vncipher(17, 17, 10)
vncipher(18, 18, 10)
vncipher(19, 19, 10)
vncipher(16, 16, 9)
vncipher(17, 17, 9)
vncipher(18, 18, 9)
vncipher(19, 19, 9)
vncipher(16, 16, 8)
vncipher(17, 17, 8)
vncipher(18, 18, 8)
vncipher(19, 19, 8)
vncipher(16, 16, 7)
vncipher(17, 17, 7)
vncipher(18, 18, 7)
vncipher(19, 19, 7)
vncipher(16, 16, 6)
vncipher(17, 17, 6)
vncipher(18, 18, 6)
vncipher(19, 19, 6)
vncipher(16, 16, 5)
vncipher(17, 17, 5)
vncipher(18, 18, 5)
vncipher(19, 19, 5)
vncipher(16, 16, 4)
vncipher(17, 17, 4)
vncipher(18, 18, 4)
vncipher(19, 19, 4)
vncipher(16, 16, 3)
vncipher(17, 17, 3)
vncipher(18, 18, 3)
vncipher(19, 19, 3)
vncipher(16, 16, 2)
vncipher(17, 17, 2)
vncipher(18, 18, 2)
vncipher(19, 19, 2)
vncipher(16, 16, 1)
vncipher(17, 17, 1)
vncipher(18, 18, 1)
vncipher(19, 19, 1)
vncipherlast(16, 16, 0)
vncipherlast(17, 17, 0)
vncipherlast(18, 18, 0)
vncipherlast(19, 19, 0)
/*
* XOR decrypted blocks with IV / previous block.
*/
vxor(16, 16, 24)
vxor(17, 17, 20)
vxor(18, 18, 21)
vxor(19, 19, 22)
/*
* Store back result (with byteswap)
*/
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
stxvw4x(48, %[cc0], %[buf])
stxvw4x(49, %[cc1], %[buf])
stxvw4x(50, %[cc2], %[buf])
stxvw4x(51, %[cc3], %[buf])
/*
* Fourth encrypted block is IV for next run.
*/
vand(24, 23, 23)
addi(%[buf], %[buf], 64)
bdnz(loop)
: [cc0] "+b" (cc0), [cc1] "+b" (cc1), [cc2] "+b" (cc2), [cc3] "+b" (cc3),
[buf] "+b" (buf)
: [sk] "b" (sk), [iv] "b" (iv), [num_blocks] "b" (num_blocks >> 2)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
"ctr", "memory"
);
}
static void
cbcdec_256(const unsigned char *sk,
const unsigned char *iv, unsigned char *buf, size_t num_blocks)
{
long cc0, cc1, cc2, cc3;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc0 = 0;
cc1 = 16;
cc2 = 32;
cc3 = 48;
asm volatile (
/*
* Load subkeys into v0..v14
*/
lxvw4x(32, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(33, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(34, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(35, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(36, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(37, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(38, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(39, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(40, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(41, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(42, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(43, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(44, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(45, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(46, %[cc0], %[sk])
li(%[cc0], 0)
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* Load IV into v24.
*/
lxvw4x(56, 0, %[iv])
#if BR_POWER8_LE
vperm(24, 24, 24, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Load next ciphertext words in v16..v19. Also save them
* in v20..v23.
*/
lxvw4x(48, %[cc0], %[buf])
lxvw4x(49, %[cc1], %[buf])
lxvw4x(50, %[cc2], %[buf])
lxvw4x(51, %[cc3], %[buf])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
vand(20, 16, 16)
vand(21, 17, 17)
vand(22, 18, 18)
vand(23, 19, 19)
/*
* Decrypt the blocks.
*/
vxor(16, 16, 14)
vxor(17, 17, 14)
vxor(18, 18, 14)
vxor(19, 19, 14)
vncipher(16, 16, 13)
vncipher(17, 17, 13)
vncipher(18, 18, 13)
vncipher(19, 19, 13)
vncipher(16, 16, 12)
vncipher(17, 17, 12)
vncipher(18, 18, 12)
vncipher(19, 19, 12)
vncipher(16, 16, 11)
vncipher(17, 17, 11)
vncipher(18, 18, 11)
vncipher(19, 19, 11)
vncipher(16, 16, 10)
vncipher(17, 17, 10)
vncipher(18, 18, 10)
vncipher(19, 19, 10)
vncipher(16, 16, 9)
vncipher(17, 17, 9)
vncipher(18, 18, 9)
vncipher(19, 19, 9)
vncipher(16, 16, 8)
vncipher(17, 17, 8)
vncipher(18, 18, 8)
vncipher(19, 19, 8)
vncipher(16, 16, 7)
vncipher(17, 17, 7)
vncipher(18, 18, 7)
vncipher(19, 19, 7)
vncipher(16, 16, 6)
vncipher(17, 17, 6)
vncipher(18, 18, 6)
vncipher(19, 19, 6)
vncipher(16, 16, 5)
vncipher(17, 17, 5)
vncipher(18, 18, 5)
vncipher(19, 19, 5)
vncipher(16, 16, 4)
vncipher(17, 17, 4)
vncipher(18, 18, 4)
vncipher(19, 19, 4)
vncipher(16, 16, 3)
vncipher(17, 17, 3)
vncipher(18, 18, 3)
vncipher(19, 19, 3)
vncipher(16, 16, 2)
vncipher(17, 17, 2)
vncipher(18, 18, 2)
vncipher(19, 19, 2)
vncipher(16, 16, 1)
vncipher(17, 17, 1)
vncipher(18, 18, 1)
vncipher(19, 19, 1)
vncipherlast(16, 16, 0)
vncipherlast(17, 17, 0)
vncipherlast(18, 18, 0)
vncipherlast(19, 19, 0)
/*
* XOR decrypted blocks with IV / previous block.
*/
vxor(16, 16, 24)
vxor(17, 17, 20)
vxor(18, 18, 21)
vxor(19, 19, 22)
/*
* Store back result (with byteswap)
*/
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
stxvw4x(48, %[cc0], %[buf])
stxvw4x(49, %[cc1], %[buf])
stxvw4x(50, %[cc2], %[buf])
stxvw4x(51, %[cc3], %[buf])
/*
* Fourth encrypted block is IV for next run.
*/
vand(24, 23, 23)
addi(%[buf], %[buf], 64)
bdnz(loop)
: [cc0] "+b" (cc0), [cc1] "+b" (cc1), [cc2] "+b" (cc2), [cc3] "+b" (cc3),
[buf] "+b" (buf)
: [sk] "b" (sk), [iv] "b" (iv), [num_blocks] "b" (num_blocks >> 2)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
"ctr", "memory"
);
}
/* see bearssl_block.h */
void
br_aes_pwr8_cbcdec_run(const br_aes_pwr8_cbcdec_keys *ctx,
void *iv, void *data, size_t len)
{
unsigned char nextiv[16];
unsigned char *buf;
if (len == 0) {
return;
}
buf = data;
memcpy(nextiv, buf + len - 16, 16);
if (len >= 64) {
size_t num_blocks;
unsigned char tmp[16];
num_blocks = (len >> 4) & ~(size_t)3;
memcpy(tmp, buf + (num_blocks << 4) - 16, 16);
switch (ctx->num_rounds) {
case 10:
cbcdec_128(ctx->skey.skni, iv, buf, num_blocks);
break;
case 12:
cbcdec_192(ctx->skey.skni, iv, buf, num_blocks);
break;
default:
cbcdec_256(ctx->skey.skni, iv, buf, num_blocks);
break;
}
buf += num_blocks << 4;
len &= 63;
memcpy(iv, tmp, 16);
}
if (len > 0) {
unsigned char tmp[64];
memcpy(tmp, buf, len);
memset(tmp + len, 0, (sizeof tmp) - len);
switch (ctx->num_rounds) {
case 10:
cbcdec_128(ctx->skey.skni, iv, tmp, 4);
break;
case 12:
cbcdec_192(ctx->skey.skni, iv, tmp, 4);
break;
default:
cbcdec_256(ctx->skey.skni, iv, tmp, 4);
break;
}
memcpy(buf, tmp, len);
}
memcpy(iv, nextiv, 16);
}
/* see bearssl_block.h */
const br_block_cbcdec_class br_aes_pwr8_cbcdec_vtable = {
sizeof(br_aes_pwr8_cbcdec_keys),
16,
4,
(void (*)(const br_block_cbcdec_class **, const void *, size_t))
&br_aes_pwr8_cbcdec_init,
(void (*)(const br_block_cbcdec_class *const *, void *, void *, size_t))
&br_aes_pwr8_cbcdec_run
};
/* see bearssl_block.h */
const br_block_cbcdec_class *
br_aes_pwr8_cbcdec_get_vtable(void)
{
return br_aes_pwr8_supported() ? &br_aes_pwr8_cbcdec_vtable : NULL;
}
#else
/* see bearssl_block.h */
const br_block_cbcdec_class *
br_aes_pwr8_cbcdec_get_vtable(void)
{
return NULL;
}
#endif

View file

@ -0,0 +1,417 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#define BR_POWER_ASM_MACROS 1
#include "inner.h"
#if BR_POWER8
/* see bearssl_block.h */
void
br_aes_pwr8_cbcenc_init(br_aes_pwr8_cbcenc_keys *ctx,
const void *key, size_t len)
{
ctx->vtable = &br_aes_pwr8_cbcenc_vtable;
ctx->num_rounds = br_aes_pwr8_keysched(ctx->skey.skni, key, len);
}
static void
cbcenc_128(const unsigned char *sk,
const unsigned char *iv, unsigned char *buf, size_t len)
{
long cc;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc = 0;
asm volatile (
/*
* Load subkeys into v0..v10
*/
lxvw4x(32, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(33, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(34, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(35, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(36, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(37, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(38, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(39, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(40, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(41, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(42, %[cc], %[sk])
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* Load IV into v16.
*/
lxvw4x(48, 0, %[iv])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Load next plaintext word and XOR with current IV.
*/
lxvw4x(49, 0, %[buf])
#if BR_POWER8_LE
vperm(17, 17, 17, 15)
#endif
vxor(16, 16, 17)
/*
* Encrypt the block.
*/
vxor(16, 16, 0)
vcipher(16, 16, 1)
vcipher(16, 16, 2)
vcipher(16, 16, 3)
vcipher(16, 16, 4)
vcipher(16, 16, 5)
vcipher(16, 16, 6)
vcipher(16, 16, 7)
vcipher(16, 16, 8)
vcipher(16, 16, 9)
vcipherlast(16, 16, 10)
/*
* Store back result (with byteswap)
*/
#if BR_POWER8_LE
vperm(17, 16, 16, 15)
stxvw4x(49, 0, %[buf])
#else
stxvw4x(48, 0, %[buf])
#endif
addi(%[buf], %[buf], 16)
bdnz(loop)
: [cc] "+b" (cc), [buf] "+b" (buf)
: [sk] "b" (sk), [iv] "b" (iv), [num_blocks] "b" (len >> 4)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"ctr", "memory"
);
}
static void
cbcenc_192(const unsigned char *sk,
const unsigned char *iv, unsigned char *buf, size_t len)
{
long cc;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc = 0;
asm volatile (
/*
* Load subkeys into v0..v12
*/
lxvw4x(32, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(33, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(34, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(35, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(36, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(37, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(38, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(39, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(40, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(41, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(42, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(43, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(44, %[cc], %[sk])
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* Load IV into v16.
*/
lxvw4x(48, 0, %[iv])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Load next plaintext word and XOR with current IV.
*/
lxvw4x(49, 0, %[buf])
#if BR_POWER8_LE
vperm(17, 17, 17, 15)
#endif
vxor(16, 16, 17)
/*
* Encrypt the block.
*/
vxor(16, 16, 0)
vcipher(16, 16, 1)
vcipher(16, 16, 2)
vcipher(16, 16, 3)
vcipher(16, 16, 4)
vcipher(16, 16, 5)
vcipher(16, 16, 6)
vcipher(16, 16, 7)
vcipher(16, 16, 8)
vcipher(16, 16, 9)
vcipher(16, 16, 10)
vcipher(16, 16, 11)
vcipherlast(16, 16, 12)
/*
* Store back result (with byteswap)
*/
#if BR_POWER8_LE
vperm(17, 16, 16, 15)
stxvw4x(49, 0, %[buf])
#else
stxvw4x(48, 0, %[buf])
#endif
addi(%[buf], %[buf], 16)
bdnz(loop)
: [cc] "+b" (cc), [buf] "+b" (buf)
: [sk] "b" (sk), [iv] "b" (iv), [num_blocks] "b" (len >> 4)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"ctr", "memory"
);
}
static void
cbcenc_256(const unsigned char *sk,
const unsigned char *iv, unsigned char *buf, size_t len)
{
long cc;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
cc = 0;
asm volatile (
/*
* Load subkeys into v0..v14
*/
lxvw4x(32, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(33, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(34, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(35, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(36, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(37, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(38, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(39, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(40, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(41, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(42, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(43, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(44, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(45, %[cc], %[sk])
addi(%[cc], %[cc], 16)
lxvw4x(46, %[cc], %[sk])
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* Load IV into v16.
*/
lxvw4x(48, 0, %[iv])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Load next plaintext word and XOR with current IV.
*/
lxvw4x(49, 0, %[buf])
#if BR_POWER8_LE
vperm(17, 17, 17, 15)
#endif
vxor(16, 16, 17)
/*
* Encrypt the block.
*/
vxor(16, 16, 0)
vcipher(16, 16, 1)
vcipher(16, 16, 2)
vcipher(16, 16, 3)
vcipher(16, 16, 4)
vcipher(16, 16, 5)
vcipher(16, 16, 6)
vcipher(16, 16, 7)
vcipher(16, 16, 8)
vcipher(16, 16, 9)
vcipher(16, 16, 10)
vcipher(16, 16, 11)
vcipher(16, 16, 12)
vcipher(16, 16, 13)
vcipherlast(16, 16, 14)
/*
* Store back result (with byteswap)
*/
#if BR_POWER8_LE
vperm(17, 16, 16, 15)
stxvw4x(49, 0, %[buf])
#else
stxvw4x(48, 0, %[buf])
#endif
addi(%[buf], %[buf], 16)
bdnz(loop)
: [cc] "+b" (cc), [buf] "+b" (buf)
: [sk] "b" (sk), [iv] "b" (iv), [num_blocks] "b" (len >> 4)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"ctr", "memory"
);
}
/* see bearssl_block.h */
void
br_aes_pwr8_cbcenc_run(const br_aes_pwr8_cbcenc_keys *ctx,
void *iv, void *data, size_t len)
{
if (len > 0) {
switch (ctx->num_rounds) {
case 10:
cbcenc_128(ctx->skey.skni, iv, data, len);
break;
case 12:
cbcenc_192(ctx->skey.skni, iv, data, len);
break;
default:
cbcenc_256(ctx->skey.skni, iv, data, len);
break;
}
memcpy(iv, (unsigned char *)data + (len - 16), 16);
}
}
/* see bearssl_block.h */
const br_block_cbcenc_class br_aes_pwr8_cbcenc_vtable = {
sizeof(br_aes_pwr8_cbcenc_keys),
16,
4,
(void (*)(const br_block_cbcenc_class **, const void *, size_t))
&br_aes_pwr8_cbcenc_init,
(void (*)(const br_block_cbcenc_class *const *, void *, void *, size_t))
&br_aes_pwr8_cbcenc_run
};
/* see bearssl_block.h */
const br_block_cbcenc_class *
br_aes_pwr8_cbcenc_get_vtable(void)
{
return br_aes_pwr8_supported() ? &br_aes_pwr8_cbcenc_vtable : NULL;
}
#else
/* see bearssl_block.h */
const br_block_cbcenc_class *
br_aes_pwr8_cbcenc_get_vtable(void)
{
return NULL;
}
#endif

View file

@ -0,0 +1,717 @@
/*
* Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#define BR_POWER_ASM_MACROS 1
#include "inner.h"
#if BR_POWER8
/* see bearssl_block.h */
void
br_aes_pwr8_ctr_init(br_aes_pwr8_ctr_keys *ctx,
const void *key, size_t len)
{
ctx->vtable = &br_aes_pwr8_ctr_vtable;
ctx->num_rounds = br_aes_pwr8_keysched(ctx->skey.skni, key, len);
}
static void
ctr_128(const unsigned char *sk, const unsigned char *ivbuf,
unsigned char *buf, size_t num_blocks)
{
long cc0, cc1, cc2, cc3;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
static const uint32_t ctrinc[] = {
0, 0, 0, 4
};
cc0 = 0;
cc1 = 16;
cc2 = 32;
cc3 = 48;
asm volatile (
/*
* Load subkeys into v0..v10
*/
lxvw4x(32, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(33, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(34, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(35, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(36, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(37, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(38, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(39, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(40, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(41, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(42, %[cc0], %[sk])
li(%[cc0], 0)
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* v28 = increment for IV counter.
*/
lxvw4x(60, 0, %[ctrinc])
/*
* Load IV into v16..v19
*/
lxvw4x(48, %[cc0], %[ivbuf])
lxvw4x(49, %[cc1], %[ivbuf])
lxvw4x(50, %[cc2], %[ivbuf])
lxvw4x(51, %[cc3], %[ivbuf])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Compute next IV into v24..v27
*/
vadduwm(24, 16, 28)
vadduwm(25, 17, 28)
vadduwm(26, 18, 28)
vadduwm(27, 19, 28)
/*
* Load next data blocks. We do this early on but we
* won't need them until IV encryption is done.
*/
lxvw4x(52, %[cc0], %[buf])
lxvw4x(53, %[cc1], %[buf])
lxvw4x(54, %[cc2], %[buf])
lxvw4x(55, %[cc3], %[buf])
/*
* Encrypt the current IV.
*/
vxor(16, 16, 0)
vxor(17, 17, 0)
vxor(18, 18, 0)
vxor(19, 19, 0)
vcipher(16, 16, 1)
vcipher(17, 17, 1)
vcipher(18, 18, 1)
vcipher(19, 19, 1)
vcipher(16, 16, 2)
vcipher(17, 17, 2)
vcipher(18, 18, 2)
vcipher(19, 19, 2)
vcipher(16, 16, 3)
vcipher(17, 17, 3)
vcipher(18, 18, 3)
vcipher(19, 19, 3)
vcipher(16, 16, 4)
vcipher(17, 17, 4)
vcipher(18, 18, 4)
vcipher(19, 19, 4)
vcipher(16, 16, 5)
vcipher(17, 17, 5)
vcipher(18, 18, 5)
vcipher(19, 19, 5)
vcipher(16, 16, 6)
vcipher(17, 17, 6)
vcipher(18, 18, 6)
vcipher(19, 19, 6)
vcipher(16, 16, 7)
vcipher(17, 17, 7)
vcipher(18, 18, 7)
vcipher(19, 19, 7)
vcipher(16, 16, 8)
vcipher(17, 17, 8)
vcipher(18, 18, 8)
vcipher(19, 19, 8)
vcipher(16, 16, 9)
vcipher(17, 17, 9)
vcipher(18, 18, 9)
vcipher(19, 19, 9)
vcipherlast(16, 16, 10)
vcipherlast(17, 17, 10)
vcipherlast(18, 18, 10)
vcipherlast(19, 19, 10)
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
/*
* Load next plaintext word and XOR with encrypted IV.
*/
vxor(16, 20, 16)
vxor(17, 21, 17)
vxor(18, 22, 18)
vxor(19, 23, 19)
stxvw4x(48, %[cc0], %[buf])
stxvw4x(49, %[cc1], %[buf])
stxvw4x(50, %[cc2], %[buf])
stxvw4x(51, %[cc3], %[buf])
addi(%[buf], %[buf], 64)
/*
* Update IV.
*/
vand(16, 24, 24)
vand(17, 25, 25)
vand(18, 26, 26)
vand(19, 27, 27)
bdnz(loop)
: [cc0] "+b" (cc0), [cc1] "+b" (cc1), [cc2] "+b" (cc2), [cc3] "+b" (cc3),
[buf] "+b" (buf)
: [sk] "b" (sk), [ivbuf] "b" (ivbuf), [num_blocks] "b" (num_blocks >> 2),
[ctrinc] "b" (ctrinc)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
"ctr", "memory"
);
}
static void
ctr_192(const unsigned char *sk, const unsigned char *ivbuf,
unsigned char *buf, size_t num_blocks)
{
long cc0, cc1, cc2, cc3;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
static const uint32_t ctrinc[] = {
0, 0, 0, 4
};
cc0 = 0;
cc1 = 16;
cc2 = 32;
cc3 = 48;
asm volatile (
/*
* Load subkeys into v0..v12
*/
lxvw4x(32, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(33, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(34, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(35, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(36, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(37, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(38, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(39, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(40, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(41, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(42, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(43, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(44, %[cc0], %[sk])
li(%[cc0], 0)
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* v28 = increment for IV counter.
*/
lxvw4x(60, 0, %[ctrinc])
/*
* Load IV into v16..v19
*/
lxvw4x(48, %[cc0], %[ivbuf])
lxvw4x(49, %[cc1], %[ivbuf])
lxvw4x(50, %[cc2], %[ivbuf])
lxvw4x(51, %[cc3], %[ivbuf])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Compute next IV into v24..v27
*/
vadduwm(24, 16, 28)
vadduwm(25, 17, 28)
vadduwm(26, 18, 28)
vadduwm(27, 19, 28)
/*
* Load next data blocks. We do this early on but we
* won't need them until IV encryption is done.
*/
lxvw4x(52, %[cc0], %[buf])
lxvw4x(53, %[cc1], %[buf])
lxvw4x(54, %[cc2], %[buf])
lxvw4x(55, %[cc3], %[buf])
/*
* Encrypt the current IV.
*/
vxor(16, 16, 0)
vxor(17, 17, 0)
vxor(18, 18, 0)
vxor(19, 19, 0)
vcipher(16, 16, 1)
vcipher(17, 17, 1)
vcipher(18, 18, 1)
vcipher(19, 19, 1)
vcipher(16, 16, 2)
vcipher(17, 17, 2)
vcipher(18, 18, 2)
vcipher(19, 19, 2)
vcipher(16, 16, 3)
vcipher(17, 17, 3)
vcipher(18, 18, 3)
vcipher(19, 19, 3)
vcipher(16, 16, 4)
vcipher(17, 17, 4)
vcipher(18, 18, 4)
vcipher(19, 19, 4)
vcipher(16, 16, 5)
vcipher(17, 17, 5)
vcipher(18, 18, 5)
vcipher(19, 19, 5)
vcipher(16, 16, 6)
vcipher(17, 17, 6)
vcipher(18, 18, 6)
vcipher(19, 19, 6)
vcipher(16, 16, 7)
vcipher(17, 17, 7)
vcipher(18, 18, 7)
vcipher(19, 19, 7)
vcipher(16, 16, 8)
vcipher(17, 17, 8)
vcipher(18, 18, 8)
vcipher(19, 19, 8)
vcipher(16, 16, 9)
vcipher(17, 17, 9)
vcipher(18, 18, 9)
vcipher(19, 19, 9)
vcipher(16, 16, 10)
vcipher(17, 17, 10)
vcipher(18, 18, 10)
vcipher(19, 19, 10)
vcipher(16, 16, 11)
vcipher(17, 17, 11)
vcipher(18, 18, 11)
vcipher(19, 19, 11)
vcipherlast(16, 16, 12)
vcipherlast(17, 17, 12)
vcipherlast(18, 18, 12)
vcipherlast(19, 19, 12)
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
/*
* Load next plaintext word and XOR with encrypted IV.
*/
vxor(16, 20, 16)
vxor(17, 21, 17)
vxor(18, 22, 18)
vxor(19, 23, 19)
stxvw4x(48, %[cc0], %[buf])
stxvw4x(49, %[cc1], %[buf])
stxvw4x(50, %[cc2], %[buf])
stxvw4x(51, %[cc3], %[buf])
addi(%[buf], %[buf], 64)
/*
* Update IV.
*/
vand(16, 24, 24)
vand(17, 25, 25)
vand(18, 26, 26)
vand(19, 27, 27)
bdnz(loop)
: [cc0] "+b" (cc0), [cc1] "+b" (cc1), [cc2] "+b" (cc2), [cc3] "+b" (cc3),
[buf] "+b" (buf)
: [sk] "b" (sk), [ivbuf] "b" (ivbuf), [num_blocks] "b" (num_blocks >> 2),
[ctrinc] "b" (ctrinc)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
"ctr", "memory"
);
}
static void
ctr_256(const unsigned char *sk, const unsigned char *ivbuf,
unsigned char *buf, size_t num_blocks)
{
long cc0, cc1, cc2, cc3;
#if BR_POWER8_LE
static const uint32_t idx2be[] = {
0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
};
#endif
static const uint32_t ctrinc[] = {
0, 0, 0, 4
};
cc0 = 0;
cc1 = 16;
cc2 = 32;
cc3 = 48;
asm volatile (
/*
* Load subkeys into v0..v14
*/
lxvw4x(32, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(33, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(34, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(35, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(36, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(37, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(38, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(39, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(40, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(41, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(42, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(43, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(44, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(45, %[cc0], %[sk])
addi(%[cc0], %[cc0], 16)
lxvw4x(46, %[cc0], %[sk])
li(%[cc0], 0)
#if BR_POWER8_LE
/*
* v15 = constant for byteswapping words
*/
lxvw4x(47, 0, %[idx2be])
#endif
/*
* v28 = increment for IV counter.
*/
lxvw4x(60, 0, %[ctrinc])
/*
* Load IV into v16..v19
*/
lxvw4x(48, %[cc0], %[ivbuf])
lxvw4x(49, %[cc1], %[ivbuf])
lxvw4x(50, %[cc2], %[ivbuf])
lxvw4x(51, %[cc3], %[ivbuf])
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
mtctr(%[num_blocks])
label(loop)
/*
* Compute next IV into v24..v27
*/
vadduwm(24, 16, 28)
vadduwm(25, 17, 28)
vadduwm(26, 18, 28)
vadduwm(27, 19, 28)
/*
* Load next data blocks. We do this early on but we
* won't need them until IV encryption is done.
*/
lxvw4x(52, %[cc0], %[buf])
lxvw4x(53, %[cc1], %[buf])
lxvw4x(54, %[cc2], %[buf])
lxvw4x(55, %[cc3], %[buf])
/*
* Encrypt the current IV.
*/
vxor(16, 16, 0)
vxor(17, 17, 0)
vxor(18, 18, 0)
vxor(19, 19, 0)
vcipher(16, 16, 1)
vcipher(17, 17, 1)
vcipher(18, 18, 1)
vcipher(19, 19, 1)
vcipher(16, 16, 2)
vcipher(17, 17, 2)
vcipher(18, 18, 2)
vcipher(19, 19, 2)
vcipher(16, 16, 3)
vcipher(17, 17, 3)
vcipher(18, 18, 3)
vcipher(19, 19, 3)
vcipher(16, 16, 4)
vcipher(17, 17, 4)
vcipher(18, 18, 4)
vcipher(19, 19, 4)
vcipher(16, 16, 5)
vcipher(17, 17, 5)
vcipher(18, 18, 5)
vcipher(19, 19, 5)
vcipher(16, 16, 6)
vcipher(17, 17, 6)
vcipher(18, 18, 6)
vcipher(19, 19, 6)
vcipher(16, 16, 7)
vcipher(17, 17, 7)
vcipher(18, 18, 7)
vcipher(19, 19, 7)
vcipher(16, 16, 8)
vcipher(17, 17, 8)
vcipher(18, 18, 8)
vcipher(19, 19, 8)
vcipher(16, 16, 9)
vcipher(17, 17, 9)
vcipher(18, 18, 9)
vcipher(19, 19, 9)
vcipher(16, 16, 10)
vcipher(17, 17, 10)
vcipher(18, 18, 10)
vcipher(19, 19, 10)
vcipher(16, 16, 11)
vcipher(17, 17, 11)
vcipher(18, 18, 11)
vcipher(19, 19, 11)
vcipher(16, 16, 12)
vcipher(17, 17, 12)
vcipher(18, 18, 12)
vcipher(19, 19, 12)
vcipher(16, 16, 13)
vcipher(17, 17, 13)
vcipher(18, 18, 13)
vcipher(19, 19, 13)
vcipherlast(16, 16, 14)
vcipherlast(17, 17, 14)
vcipherlast(18, 18, 14)
vcipherlast(19, 19, 14)
#if BR_POWER8_LE
vperm(16, 16, 16, 15)
vperm(17, 17, 17, 15)
vperm(18, 18, 18, 15)
vperm(19, 19, 19, 15)
#endif
/*
* Load next plaintext word and XOR with encrypted IV.
*/
vxor(16, 20, 16)
vxor(17, 21, 17)
vxor(18, 22, 18)
vxor(19, 23, 19)
stxvw4x(48, %[cc0], %[buf])
stxvw4x(49, %[cc1], %[buf])
stxvw4x(50, %[cc2], %[buf])
stxvw4x(51, %[cc3], %[buf])
addi(%[buf], %[buf], 64)
/*
* Update IV.
*/
vand(16, 24, 24)
vand(17, 25, 25)
vand(18, 26, 26)
vand(19, 27, 27)
bdnz(loop)
: [cc0] "+b" (cc0), [cc1] "+b" (cc1), [cc2] "+b" (cc2), [cc3] "+b" (cc3),
[buf] "+b" (buf)
: [sk] "b" (sk), [ivbuf] "b" (ivbuf), [num_blocks] "b" (num_blocks >> 2),
[ctrinc] "b" (ctrinc)
#if BR_POWER8_LE
, [idx2be] "b" (idx2be)
#endif
: "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
"v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
"v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
"ctr", "memory"
);
}
/* see bearssl_block.h */
uint32_t
br_aes_pwr8_ctr_run(const br_aes_pwr8_ctr_keys *ctx,
const void *iv, uint32_t cc, void *data, size_t len)
{
unsigned char *buf;
unsigned char ivbuf[64];
buf = data;
memcpy(ivbuf + 0, iv, 12);
memcpy(ivbuf + 16, iv, 12);
memcpy(ivbuf + 32, iv, 12);
memcpy(ivbuf + 48, iv, 12);
if (len >= 64) {
br_enc32be(ivbuf + 12, cc + 0);
br_enc32be(ivbuf + 28, cc + 1);
br_enc32be(ivbuf + 44, cc + 2);
br_enc32be(ivbuf + 60, cc + 3);
switch (ctx->num_rounds) {
case 10:
ctr_128(ctx->skey.skni, ivbuf, buf,
(len >> 4) & ~(size_t)3);
break;
case 12:
ctr_192(ctx->skey.skni, ivbuf, buf,
(len >> 4) & ~(size_t)3);
break;
default:
ctr_256(ctx->skey.skni, ivbuf, buf,
(len >> 4) & ~(size_t)3);
break;
}
cc += (len >> 4) & ~(size_t)3;
buf += len & ~(size_t)63;
len &= 63;
}
if (len > 0) {
unsigned char tmp[64];
memcpy(tmp, buf, len);
memset(tmp + len, 0, (sizeof tmp) - len);
br_enc32be(ivbuf + 12, cc + 0);
br_enc32be(ivbuf + 28, cc + 1);
br_enc32be(ivbuf + 44, cc + 2);
br_enc32be(ivbuf + 60, cc + 3);
switch (ctx->num_rounds) {
case 10:
ctr_128(ctx->skey.skni, ivbuf, tmp, 4);
break;
case 12:
ctr_192(ctx->skey.skni, ivbuf, tmp, 4);
break;
default:
ctr_256(ctx->skey.skni, ivbuf, tmp, 4);
break;
}
memcpy(buf, tmp, len);
cc += (len + 15) >> 4;
}
return cc;
}
/* see bearssl_block.h */
const br_block_ctr_class br_aes_pwr8_ctr_vtable = {
sizeof(br_aes_pwr8_ctr_keys),
16,
4,
(void (*)(const br_block_ctr_class **, const void *, size_t))
&br_aes_pwr8_ctr_init,
(uint32_t (*)(const br_block_ctr_class *const *,
const void *, uint32_t, void *, size_t))
&br_aes_pwr8_ctr_run
};
/* see bearssl_block.h */
const br_block_ctr_class *
br_aes_pwr8_ctr_get_vtable(void)
{
return br_aes_pwr8_supported() ? &br_aes_pwr8_ctr_vtable : NULL;
}
#else
/* see bearssl_block.h */
const br_block_ctr_class *
br_aes_pwr8_ctr_get_vtable(void)
{
return NULL;
}
#endif