forked from Minki/linux
Revert "crypto: aegis128 - add support for SIMD acceleration"
This reverts commitecc8bc81f2("crypto: aegis128 - provide a SIMD implementation based on NEON intrinsics") and commit7cdc0ddbf7("crypto: aegis128 - add support for SIMD acceleration"). They cause compile errors on platforms other than ARM because the mechanism to selectively compile the SIMD code is broken. Repoted-by: Heiko Carstens <heiko.carstens@de.ibm.com> Reported-by: Stephen Rothwell <sfr@canb.auug.org.au> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Herbert Xu
parent
82cb548568
commit
c9f1fd4f2f
@ -306,11 +306,6 @@ config CRYPTO_AEGIS128
|
||||
help
|
||||
Support for the AEGIS-128 dedicated AEAD algorithm.
|
||||
|
||||
config CRYPTO_AEGIS128_SIMD
|
||||
bool "Support SIMD acceleration for AEGIS-128"
|
||||
depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON)
|
||||
default y
|
||||
|
||||
config CRYPTO_AEGIS128_AESNI_SSE2
|
||||
tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)"
|
||||
depends on X86 && 64BIT
|
||||
|
||||
@ -90,18 +90,6 @@ obj-$(CONFIG_CRYPTO_GCM) += gcm.o
|
||||
obj-$(CONFIG_CRYPTO_CCM) += ccm.o
|
||||
obj-$(CONFIG_CRYPTO_CHACHA20POLY1305) += chacha20poly1305.o
|
||||
obj-$(CONFIG_CRYPTO_AEGIS128) += aegis128.o
|
||||
aegis128-y := aegis128-core.o
|
||||
|
||||
ifeq ($(ARCH),arm)
|
||||
CFLAGS_aegis128-neon-inner.o += -ffreestanding -march=armv7-a -mfloat-abi=softfp -mfpu=crypto-neon-fp-armv8
|
||||
aegis128-$(CONFIG_CRYPTO_AEGIS128_SIMD) += aegis128-neon.o aegis128-neon-inner.o
|
||||
endif
|
||||
ifeq ($(ARCH),arm64)
|
||||
CFLAGS_aegis128-neon-inner.o += -ffreestanding -mcpu=generic+crypto
|
||||
CFLAGS_REMOVE_aegis128-neon-inner.o += -mgeneral-regs-only
|
||||
aegis128-$(CONFIG_CRYPTO_AEGIS128_SIMD) += aegis128-neon.o aegis128-neon-inner.o
|
||||
endif
|
||||
|
||||
obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
|
||||
obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
|
||||
obj-$(CONFIG_CRYPTO_DES) += des_generic.o
|
||||
|
||||
@ -1,149 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
|
||||
*/
|
||||
|
||||
#ifdef CONFIG_ARM64
|
||||
#include <asm/neon-intrinsics.h>
|
||||
|
||||
#define AES_ROUND "aese %0.16b, %1.16b \n\t aesmc %0.16b, %0.16b"
|
||||
#else
|
||||
#include <arm_neon.h>
|
||||
|
||||
#define AES_ROUND "aese.8 %q0, %q1 \n\t aesmc.8 %q0, %q0"
|
||||
#endif
|
||||
|
||||
#define AEGIS_BLOCK_SIZE 16
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
void *memcpy(void *dest, const void *src, size_t n);
|
||||
void *memset(void *s, int c, size_t n);
|
||||
|
||||
struct aegis128_state {
|
||||
uint8x16_t v[5];
|
||||
};
|
||||
|
||||
static struct aegis128_state aegis128_load_state_neon(const void *state)
|
||||
{
|
||||
return (struct aegis128_state){ {
|
||||
vld1q_u8(state),
|
||||
vld1q_u8(state + 16),
|
||||
vld1q_u8(state + 32),
|
||||
vld1q_u8(state + 48),
|
||||
vld1q_u8(state + 64)
|
||||
} };
|
||||
}
|
||||
|
||||
static void aegis128_save_state_neon(struct aegis128_state st, void *state)
|
||||
{
|
||||
vst1q_u8(state, st.v[0]);
|
||||
vst1q_u8(state + 16, st.v[1]);
|
||||
vst1q_u8(state + 32, st.v[2]);
|
||||
vst1q_u8(state + 48, st.v[3]);
|
||||
vst1q_u8(state + 64, st.v[4]);
|
||||
}
|
||||
|
||||
static uint8x16_t aegis_aes_round(uint8x16_t w)
|
||||
{
|
||||
uint8x16_t z = {};
|
||||
|
||||
/*
|
||||
* We use inline asm here instead of the vaeseq_u8/vaesmcq_u8 intrinsics
|
||||
* to force the compiler to issue the aese/aesmc instructions in pairs.
|
||||
* This is much faster on many cores, where the instruction pair can
|
||||
* execute in a single cycle.
|
||||
*/
|
||||
asm(AES_ROUND : "+w"(w) : "w"(z));
|
||||
return w;
|
||||
}
|
||||
|
||||
static struct aegis128_state aegis128_update_neon(struct aegis128_state st,
|
||||
uint8x16_t m)
|
||||
{
|
||||
uint8x16_t t;
|
||||
|
||||
t = aegis_aes_round(st.v[3]);
|
||||
st.v[3] ^= aegis_aes_round(st.v[2]);
|
||||
st.v[2] ^= aegis_aes_round(st.v[1]);
|
||||
st.v[1] ^= aegis_aes_round(st.v[0]);
|
||||
st.v[0] ^= aegis_aes_round(st.v[4]) ^ m;
|
||||
st.v[4] ^= t;
|
||||
|
||||
return st;
|
||||
}
|
||||
|
||||
void crypto_aegis128_update_neon(void *state, const void *msg)
|
||||
{
|
||||
struct aegis128_state st = aegis128_load_state_neon(state);
|
||||
|
||||
st = aegis128_update_neon(st, vld1q_u8(msg));
|
||||
|
||||
aegis128_save_state_neon(st, state);
|
||||
}
|
||||
|
||||
void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src,
|
||||
unsigned int size)
|
||||
{
|
||||
struct aegis128_state st = aegis128_load_state_neon(state);
|
||||
uint8x16_t tmp;
|
||||
|
||||
while (size >= AEGIS_BLOCK_SIZE) {
|
||||
uint8x16_t s = vld1q_u8(src);
|
||||
|
||||
tmp = s ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4];
|
||||
st = aegis128_update_neon(st, s);
|
||||
vst1q_u8(dst, tmp);
|
||||
|
||||
size -= AEGIS_BLOCK_SIZE;
|
||||
src += AEGIS_BLOCK_SIZE;
|
||||
dst += AEGIS_BLOCK_SIZE;
|
||||
}
|
||||
|
||||
if (size > 0) {
|
||||
uint8_t buf[AEGIS_BLOCK_SIZE] = {};
|
||||
uint8x16_t msg;
|
||||
|
||||
memcpy(buf, src, size);
|
||||
msg = vld1q_u8(buf);
|
||||
tmp = msg ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4];
|
||||
st = aegis128_update_neon(st, msg);
|
||||
vst1q_u8(buf, tmp);
|
||||
memcpy(dst, buf, size);
|
||||
}
|
||||
|
||||
aegis128_save_state_neon(st, state);
|
||||
}
|
||||
|
||||
void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src,
|
||||
unsigned int size)
|
||||
{
|
||||
struct aegis128_state st = aegis128_load_state_neon(state);
|
||||
uint8x16_t tmp;
|
||||
|
||||
while (size >= AEGIS_BLOCK_SIZE) {
|
||||
tmp = vld1q_u8(src) ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4];
|
||||
st = aegis128_update_neon(st, tmp);
|
||||
vst1q_u8(dst, tmp);
|
||||
|
||||
size -= AEGIS_BLOCK_SIZE;
|
||||
src += AEGIS_BLOCK_SIZE;
|
||||
dst += AEGIS_BLOCK_SIZE;
|
||||
}
|
||||
|
||||
if (size > 0) {
|
||||
uint8_t buf[AEGIS_BLOCK_SIZE] = {};
|
||||
uint8x16_t msg;
|
||||
|
||||
memcpy(buf, src, size);
|
||||
msg = vld1q_u8(buf) ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4];
|
||||
vst1q_u8(buf, msg);
|
||||
memcpy(dst, buf, size);
|
||||
|
||||
memset(buf + size, 0, AEGIS_BLOCK_SIZE - size);
|
||||
msg = vld1q_u8(buf);
|
||||
st = aegis128_update_neon(st, msg);
|
||||
}
|
||||
|
||||
aegis128_save_state_neon(st, state);
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* Copyright (C) 2019 Linaro Ltd <ard.biesheuvel@linaro.org>
|
||||
*/
|
||||
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/neon.h>
|
||||
|
||||
#include "aegis.h"
|
||||
|
||||
void crypto_aegis128_update_neon(void *state, const void *msg);
|
||||
void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src,
|
||||
unsigned int size);
|
||||
void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src,
|
||||
unsigned int size);
|
||||
|
||||
bool crypto_aegis128_have_simd(void)
|
||||
{
|
||||
return cpu_have_feature(cpu_feature(AES));
|
||||
}
|
||||
|
||||
void crypto_aegis128_update_simd(union aegis_block *state, const void *msg)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
crypto_aegis128_update_neon(state, msg);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
void crypto_aegis128_encrypt_chunk_simd(union aegis_block *state, u8 *dst,
|
||||
const u8 *src, unsigned int size)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
crypto_aegis128_encrypt_chunk_neon(state, dst, src, size);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
void crypto_aegis128_decrypt_chunk_simd(union aegis_block *state, u8 *dst,
|
||||
const u8 *src, unsigned int size)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
crypto_aegis128_decrypt_chunk_neon(state, dst, src, size);
|
||||
kernel_neon_end();
|
||||
}
|
||||
@ -8,7 +8,6 @@
|
||||
|
||||
#include <crypto/algapi.h>
|
||||
#include <crypto/internal/aead.h>
|
||||
#include <crypto/internal/simd.h>
|
||||
#include <crypto/internal/skcipher.h>
|
||||
#include <crypto/scatterwalk.h>
|
||||
#include <linux/err.h>
|
||||
@ -16,7 +15,6 @@
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <asm/simd.h>
|
||||
|
||||
#include "aegis.h"
|
||||
|
||||
@ -42,15 +40,6 @@ struct aegis128_ops {
|
||||
const u8 *src, unsigned int size);
|
||||
};
|
||||
|
||||
static bool have_simd;
|
||||
|
||||
bool crypto_aegis128_have_simd(void);
|
||||
void crypto_aegis128_update_simd(struct aegis_state *state, const void *msg);
|
||||
void crypto_aegis128_encrypt_chunk_simd(struct aegis_state *state, u8 *dst,
|
||||
const u8 *src, unsigned int size);
|
||||
void crypto_aegis128_decrypt_chunk_simd(struct aegis_state *state, u8 *dst,
|
||||
const u8 *src, unsigned int size);
|
||||
|
||||
static void crypto_aegis128_update(struct aegis_state *state)
|
||||
{
|
||||
union aegis_block tmp;
|
||||
@ -66,22 +55,12 @@ static void crypto_aegis128_update(struct aegis_state *state)
|
||||
static void crypto_aegis128_update_a(struct aegis_state *state,
|
||||
const union aegis_block *msg)
|
||||
{
|
||||
if (have_simd && crypto_simd_usable()) {
|
||||
crypto_aegis128_update_simd(state, msg);
|
||||
return;
|
||||
}
|
||||
|
||||
crypto_aegis128_update(state);
|
||||
crypto_aegis_block_xor(&state->blocks[0], msg);
|
||||
}
|
||||
|
||||
static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg)
|
||||
{
|
||||
if (have_simd && crypto_simd_usable()) {
|
||||
crypto_aegis128_update_simd(state, msg);
|
||||
return;
|
||||
}
|
||||
|
||||
crypto_aegis128_update(state);
|
||||
crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE);
|
||||
}
|
||||
@ -386,7 +365,7 @@ static void crypto_aegis128_crypt(struct aead_request *req,
|
||||
|
||||
static int crypto_aegis128_encrypt(struct aead_request *req)
|
||||
{
|
||||
const struct aegis128_ops *ops = &(struct aegis128_ops){
|
||||
static const struct aegis128_ops ops = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_encrypt,
|
||||
.crypt_chunk = crypto_aegis128_encrypt_chunk,
|
||||
};
|
||||
@ -396,12 +375,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req)
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen;
|
||||
|
||||
if (have_simd && crypto_simd_usable())
|
||||
ops = &(struct aegis128_ops){
|
||||
.skcipher_walk_init = skcipher_walk_aead_encrypt,
|
||||
.crypt_chunk = crypto_aegis128_encrypt_chunk_simd };
|
||||
|
||||
crypto_aegis128_crypt(req, &tag, cryptlen, ops);
|
||||
crypto_aegis128_crypt(req, &tag, cryptlen, &ops);
|
||||
|
||||
scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
|
||||
authsize, 1);
|
||||
@ -410,7 +384,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req)
|
||||
|
||||
static int crypto_aegis128_decrypt(struct aead_request *req)
|
||||
{
|
||||
const struct aegis128_ops *ops = &(struct aegis128_ops){
|
||||
static const struct aegis128_ops ops = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_decrypt,
|
||||
.crypt_chunk = crypto_aegis128_decrypt_chunk,
|
||||
};
|
||||
@ -424,12 +398,7 @@ static int crypto_aegis128_decrypt(struct aead_request *req)
|
||||
scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
|
||||
authsize, 0);
|
||||
|
||||
if (have_simd && crypto_simd_usable())
|
||||
ops = &(struct aegis128_ops){
|
||||
.skcipher_walk_init = skcipher_walk_aead_decrypt,
|
||||
.crypt_chunk = crypto_aegis128_decrypt_chunk_simd };
|
||||
|
||||
crypto_aegis128_crypt(req, &tag, cryptlen, ops);
|
||||
crypto_aegis128_crypt(req, &tag, cryptlen, &ops);
|
||||
|
||||
return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
|
||||
}
|
||||
@ -460,9 +429,6 @@ static struct aead_alg crypto_aegis128_alg = {
|
||||
|
||||
static int __init crypto_aegis128_module_init(void)
|
||||
{
|
||||
if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD))
|
||||
have_simd = crypto_aegis128_have_simd();
|
||||
|
||||
return crypto_register_aead(&crypto_aegis128_alg);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user