code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * aes-ce-glue.c - wrapper code for ARMv8 AES
 *
 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h>
#include <asm/unaligned.h>
#include <crypto/aes.h>
#include <crypto/ctr.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/cpufeature.h>
#include <linux/module.h>
#include <crypto/xts.h>

MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");

/* defined in aes-ce-core.S */
asmlinkage u32 ce_aes_sub(u32 input);
asmlinkage void ce_aes_invert(void *dst, void *src);

asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
				   int rounds, int blocks);
asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
				   int rounds, int blocks);

asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
				   int rounds, int blocks, u8 iv[]);
asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
				   int rounds, int blocks, u8 iv[]);
asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
				   int rounds, int bytes, u8 const iv[]);
asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
				   int rounds, int bytes, u8 const iv[]);

asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
				   int rounds, int blocks, u8 ctr[]);

asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
				   int rounds, int bytes, u8 iv[],
				   u32 const rk2[], int first);
asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
				   int rounds, int bytes, u8 iv[],
				   u32 const rk2[], int first);

struct aes_block {
	u8 b[AES_BLOCK_SIZE];
};

static int num_rounds(struct crypto_aes_ctx *ctx)
{
	/*
	 * # of rounds specified by AES:
	 * 128 bit key		10 rounds
	 * 192 bit key		12 rounds
	 * 256 bit key		14 rounds
	 * => n byte key	=> 6 + (n/4) rounds
	 */
	return 6 + ctx->key_length / 4;
}

static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
			    unsigned int key_len)
{
	/*
	 * The AES key schedule round constants
	 */
	static u8 const rcon[] = {
		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
	};

	u32 kwords = key_len / sizeof(u32);
	struct aes_block *key_enc, *key_dec;
	int i, j;

	if (key_len != AES_KEYSIZE_128 &&
	    key_len != AES_KEYSIZE_192 &&
	    key_len != AES_KEYSIZE_256)
		return -EINVAL;

	ctx->key_length = key_len;
	for (i = 0; i < kwords; i++)
		ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));

	kernel_neon_begin();
	for (i = 0; i < sizeof(rcon); i++) {
		u32 *rki = ctx->key_enc + (i * kwords);
		u32 *rko = rki + kwords;

		rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8);
		rko[0] = rko[0] ^ rki[0] ^ rcon[i];
		rko[1] = rko[0] ^ rki[1];
		rko[2] = rko[1] ^ rki[2];
		rko[3] = rko[2] ^ rki[3];

		if (key_len == AES_KEYSIZE_192) {
			if (i >= 7)
				break;
			rko[4] = rko[3] ^ rki[4];
			rko[5] = rko[4] ^ rki[5];
		} else if (key_len == AES_KEYSIZE_256) {
			if (i >= 6)
				break;
			rko[4] = ce_aes_sub(rko[3]) ^ rki[4];
			rko[5] = rko[4] ^ rki[5];
			rko[6] = rko[5] ^ rki[6];
			rko[7] = rko[6] ^ rki[7];
		}
	}

	/*
	 * Generate the decryption keys for the Equivalent Inverse Cipher.
	 * This involves reversing the order of the round keys, and applying
	 * the Inverse Mix Columns transformation on all but the first and
	 * the last one.
	 */
	key_enc = (struct aes_block *)ctx->key_enc;
	key_dec = (struct aes_block *)ctx->key_dec;
	j = num_rounds(ctx);

	key_dec[0] = key_enc[j];
	for (i = 1, j--; j > 0; i++, j--)
		ce_aes_invert(key_dec + i, key_enc + j);
	key_dec[i] = key_enc[0];

	kernel_neon_end();
	return 0;
}

static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
			 unsigned int key_len)
{
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);

	return ce_aes_expandkey(ctx, in_key, key_len);
}

struct crypto_aes_xts_ctx {
	struct crypto_aes_ctx key1;
	struct crypto_aes_ctx __aligned(8) key2;
};

static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
		       unsigned int key_len)
{
	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
	int ret;

	ret = xts_verify_key(tfm, in_key, key_len);
	if (ret)
		return ret;

	ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2);
	if (!ret)
		ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2],
				       key_len / 2);
	return ret;
}

static int ecb_encrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	unsigned int blocks;
	int err;

	err = skcipher_walk_virt(&walk, req, false);

	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
		kernel_neon_begin();
		ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
				   ctx->key_enc, num_rounds(ctx), blocks);
		kernel_neon_end();
		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
	}
	return err;
}

static int ecb_decrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	unsigned int blocks;
	int err;

	err = skcipher_walk_virt(&walk, req, false);

	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
		kernel_neon_begin();
		ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
				   ctx->key_dec, num_rounds(ctx), blocks);
		kernel_neon_end();
		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
	}
	return err;
}

static int cbc_encrypt_walk(struct skcipher_request *req,
			    struct skcipher_walk *walk)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	unsigned int blocks;
	int err = 0;

	while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
		kernel_neon_begin();
		ce_aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
				   ctx->key_enc, num_rounds(ctx), blocks,
				   walk->iv);
		kernel_neon_end();
		err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
	}
	return err;
}

static int cbc_encrypt(struct skcipher_request *req)
{
	struct skcipher_walk walk;
	int err;

	err = skcipher_walk_virt(&walk, req, false);
	if (err)
		return err;
	return cbc_encrypt_walk(req, &walk);
}

static int cbc_decrypt_walk(struct skcipher_request *req,
			    struct skcipher_walk *walk)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	unsigned int blocks;
	int err = 0;

	while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
		kernel_neon_begin();
		ce_aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
				   ctx->key_dec, num_rounds(ctx), blocks,
				   walk->iv);
		kernel_neon_end();
		err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
	}
	return err;
}

static int cbc_decrypt(struct skcipher_request *req)
{
	struct skcipher_walk walk;
	int err;

	err = skcipher_walk_virt(&walk, req, false);
	if (err)
		return err;
	return cbc_decrypt_walk(req, &walk);
}

static int cts_cbc_encrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
	struct scatterlist *src = req->src, *dst = req->dst;
	struct scatterlist sg_src[2], sg_dst[2];
	struct skcipher_request subreq;
	struct skcipher_walk walk;
	int err;

	skcipher_request_set_tfm(&subreq, tfm);
	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
				      NULL, NULL);

	if (req->cryptlen <= AES_BLOCK_SIZE) {
		if (req->cryptlen < AES_BLOCK_SIZE)
			return -EINVAL;
		cbc_blocks = 1;
	}

	if (cbc_blocks > 0) {
		skcipher_request_set_crypt(&subreq, req->src, req->dst,
					   cbc_blocks * AES_BLOCK_SIZE,
					   req->iv);

		err = skcipher_walk_virt(&walk, &subreq, false) ?:
		      cbc_encrypt_walk(&subreq, &walk);
		if (err)
			return err;

		if (req->cryptlen == AES_BLOCK_SIZE)
			return 0;

		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
		if (req->dst != req->src)
			dst = scatterwalk_ffwd(sg_dst, req->dst,
					       subreq.cryptlen);
	}

	/* handle ciphertext stealing */
	skcipher_request_set_crypt(&subreq, src, dst,
				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
				   req->iv);

	err = skcipher_walk_virt(&walk, &subreq, false);
	if (err)
		return err;

	kernel_neon_begin();
	ce_aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
			       ctx->key_enc, num_rounds(ctx), walk.nbytes,
			       walk.iv);
	kernel_neon_end();

	return skcipher_walk_done(&walk, 0);
}

static int cts_cbc_decrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
	struct scatterlist *src = req->src, *dst = req->dst;
	struct scatterlist sg_src[2], sg_dst[2];
	struct skcipher_request subreq;
	struct skcipher_walk walk;
	int err;

	skcipher_request_set_tfm(&subreq, tfm);
	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
				      NULL, NULL);

	if (req->cryptlen <= AES_BLOCK_SIZE) {
		if (req->cryptlen < AES_BLOCK_SIZE)
			return -EINVAL;
		cbc_blocks = 1;
	}

	if (cbc_blocks > 0) {
		skcipher_request_set_crypt(&subreq, req->src, req->dst,
					   cbc_blocks * AES_BLOCK_SIZE,
					   req->iv);

		err = skcipher_walk_virt(&walk, &subreq, false) ?:
		      cbc_decrypt_walk(&subreq, &walk);
		if (err)
			return err;

		if (req->cryptlen == AES_BLOCK_SIZE)
			return 0;

		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
		if (req->dst != req->src)
			dst = scatterwalk_ffwd(sg_dst, req->dst,
					       subreq.cryptlen);
	}

	/* handle ciphertext stealing */
	skcipher_request_set_crypt(&subreq, src, dst,
				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
				   req->iv);

	err = skcipher_walk_virt(&walk, &subreq, false);
	if (err)
		return err;

	kernel_neon_begin();
	ce_aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
			       ctx->key_dec, num_rounds(ctx), walk.nbytes,
			       walk.iv);
	kernel_neon_end();

	return skcipher_walk_done(&walk, 0);
}

static int ctr_encrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	int err, blocks;

	err = skcipher_walk_virt(&walk, req, false);

	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
		kernel_neon_begin();
		ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
				   ctx->key_enc, num_rounds(ctx), blocks,
				   walk.iv);
		kernel_neon_end();
		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
	}
	if (walk.nbytes) {
		u8 __aligned(8) tail[AES_BLOCK_SIZE];
		unsigned int nbytes = walk.nbytes;
		u8 *tdst = walk.dst.virt.addr;
		u8 *tsrc = walk.src.virt.addr;

		/*
		 * Tell aes_ctr_encrypt() to process a tail block.
		 */
		blocks = -1;

		kernel_neon_begin();
		ce_aes_ctr_encrypt(tail, NULL, ctx->key_enc, num_rounds(ctx),
				   blocks, walk.iv);
		kernel_neon_end();
		crypto_xor_cpy(tdst, tsrc, tail, nbytes);
		err = skcipher_walk_done(&walk, 0);
	}
	return err;
}

static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
{
	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	unsigned long flags;

	/*
	 * Temporarily disable interrupts to avoid races where
	 * cachelines are evicted when the CPU is interrupted
	 * to do something else.
	 */
	local_irq_save(flags);
	aes_encrypt(ctx, dst, src);
	local_irq_restore(flags);
}

static int ctr_encrypt_sync(struct skcipher_request *req)
{
	if (!crypto_simd_usable())
		return crypto_ctr_encrypt_walk(req, ctr_encrypt_one);

	return ctr_encrypt(req);
}

static int xts_encrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
	int err, first, rounds = num_rounds(&ctx->key1);
	int tail = req->cryptlen % AES_BLOCK_SIZE;
	struct scatterlist sg_src[2], sg_dst[2];
	struct skcipher_request subreq;
	struct scatterlist *src, *dst;
	struct skcipher_walk walk;

	if (req->cryptlen < AES_BLOCK_SIZE)
		return -EINVAL;

	err = skcipher_walk_virt(&walk, req, false);

	if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
		int xts_blocks = DIV_ROUND_UP(req->cryptlen,
					      AES_BLOCK_SIZE) - 2;

		skcipher_walk_abort(&walk);

		skcipher_request_set_tfm(&subreq, tfm);
		skcipher_request_set_callback(&subreq,
					      skcipher_request_flags(req),
					      NULL, NULL);
		skcipher_request_set_crypt(&subreq, req->src, req->dst,
					   xts_blocks * AES_BLOCK_SIZE,
					   req->iv);
		req = &subreq;
		err = skcipher_walk_virt(&walk, req, false);
	} else {
		tail = 0;
	}

	for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
		int nbytes = walk.nbytes;

		if (walk.nbytes < walk.total)
			nbytes &= ~(AES_BLOCK_SIZE - 1);

		kernel_neon_begin();
		ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
				   ctx->key1.key_enc, rounds, nbytes, walk.iv,
				   ctx->key2.key_enc, first);
		kernel_neon_end();
		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
	}

	if (err || likely(!tail))
		return err;

	dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
	if (req->dst != req->src)
		dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);

	skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
				   req->iv);

	err = skcipher_walk_virt(&walk, req, false);
	if (err)
		return err;

	kernel_neon_begin();
	ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
			   ctx->key1.key_enc, rounds, walk.nbytes, walk.iv,
			   ctx->key2.key_enc, first);
	kernel_neon_end();

	return skcipher_walk_done(&walk, 0);
}

static int xts_decrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
	int err, first, rounds = num_rounds(&ctx->key1);
	int tail = req->cryptlen % AES_BLOCK_SIZE;
	struct scatterlist sg_src[2], sg_dst[2];
	struct skcipher_request subreq;
	struct scatterlist *src, *dst;
	struct skcipher_walk walk;

	if (req->cryptlen < AES_BLOCK_SIZE)
		return -EINVAL;

	err = skcipher_walk_virt(&walk, req, false);

	if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
		int xts_blocks = DIV_ROUND_UP(req->cryptlen,
					      AES_BLOCK_SIZE) - 2;

		skcipher_walk_abort(&walk);

		skcipher_request_set_tfm(&subreq, tfm);
		skcipher_request_set_callback(&subreq,
					      skcipher_request_flags(req),
					      NULL, NULL);
		skcipher_request_set_crypt(&subreq, req->src, req->dst,
					   xts_blocks * AES_BLOCK_SIZE,
					   req->iv);
		req = &subreq;
		err = skcipher_walk_virt(&walk, req, false);
	} else {
		tail = 0;
	}

	for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
		int nbytes = walk.nbytes;

		if (walk.nbytes < walk.total)
			nbytes &= ~(AES_BLOCK_SIZE - 1);

		kernel_neon_begin();
		ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
				   ctx->key1.key_dec, rounds, nbytes, walk.iv,
				   ctx->key2.key_enc, first);
		kernel_neon_end();
		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
	}

	if (err || likely(!tail))
		return err;

	dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
	if (req->dst != req->src)
		dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);

	skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
				   req->iv);

	err = skcipher_walk_virt(&walk, req, false);
	if (err)
		return err;

	kernel_neon_begin();
	ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
			   ctx->key1.key_dec, rounds, walk.nbytes, walk.iv,
			   ctx->key2.key_enc, first);
	kernel_neon_end();

	return skcipher_walk_done(&walk, 0);
}

static struct skcipher_alg aes_algs[] = { {
	.base.cra_name		= "__ecb(aes)",
	.base.cra_driver_name	= "__ecb-aes-ce",
	.base.cra_priority	= 300,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
	.base.cra_blocksize	= AES_BLOCK_SIZE,
	.base.cra_ctxsize	= sizeof(struct crypto_aes_ctx),
	.base.cra_module	= THIS_MODULE,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.setkey			= ce_aes_setkey,
	.encrypt		= ecb_encrypt,
	.decrypt		= ecb_decrypt,
}, {
	.base.cra_name		= "__cbc(aes)",
	.base.cra_driver_name	= "__cbc-aes-ce",
	.base.cra_priority	= 300,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
	.base.cra_blocksize	= AES_BLOCK_SIZE,
	.base.cra_ctxsize	= sizeof(struct crypto_aes_ctx),
	.base.cra_module	= THIS_MODULE,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.setkey			= ce_aes_setkey,
	.encrypt		= cbc_encrypt,
	.decrypt		= cbc_decrypt,
}, {
	.base.cra_name		= "__cts(cbc(aes))",
	.base.cra_driver_name	= "__cts-cbc-aes-ce",
	.base.cra_priority	= 300,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
	.base.cra_blocksize	= AES_BLOCK_SIZE,
	.base.cra_ctxsize	= sizeof(struct crypto_aes_ctx),
	.base.cra_module	= THIS_MODULE,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.walksize		= 2 * AES_BLOCK_SIZE,
	.setkey			= ce_aes_setkey,
	.encrypt		= cts_cbc_encrypt,
	.decrypt		= cts_cbc_decrypt,
}, {
	.base.cra_name		= "__ctr(aes)",
	.base.cra_driver_name	= "__ctr-aes-ce",
	.base.cra_priority	= 300,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
	.base.cra_blocksize	= 1,
	.base.cra_ctxsize	= sizeof(struct crypto_aes_ctx),
	.base.cra_module	= THIS_MODULE,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.chunksize		= AES_BLOCK_SIZE,
	.setkey			= ce_aes_setkey,
	.encrypt		= ctr_encrypt,
	.decrypt		= ctr_encrypt,
}, {
	.base.cra_name		= "ctr(aes)",
	.base.cra_driver_name	= "ctr-aes-ce-sync",
	.base.cra_priority	= 300 - 1,
	.base.cra_blocksize	= 1,
	.base.cra_ctxsize	= sizeof(struct crypto_aes_ctx),
	.base.cra_module	= THIS_MODULE,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.chunksize		= AES_BLOCK_SIZE,
	.setkey			= ce_aes_setkey,
	.encrypt		= ctr_encrypt_sync,
	.decrypt		= ctr_encrypt_sync,
}, {
	.base.cra_name		= "__xts(aes)",
	.base.cra_driver_name	= "__xts-aes-ce",
	.base.cra_priority	= 300,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
	.base.cra_blocksize	= AES_BLOCK_SIZE,
	.base.cra_ctxsize	= sizeof(struct crypto_aes_xts_ctx),
	.base.cra_module	= THIS_MODULE,

	.min_keysize		= 2 * AES_MIN_KEY_SIZE,
	.max_keysize		= 2 * AES_MAX_KEY_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.walksize		= 2 * AES_BLOCK_SIZE,
	.setkey			= xts_set_key,
	.encrypt		= xts_encrypt,
	.decrypt		= xts_decrypt,
} };

static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];

static void aes_exit(void)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
		simd_skcipher_free(aes_simd_algs[i]);

	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}

static int __init aes_init(void)
{
	struct simd_skcipher_alg *simd;
	const char *basename;
	const char *algname;
	const char *drvname;
	int err;
	int i;

	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
	if (err)
		return err;

	for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
		if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
			continue;

		algname = aes_algs[i].base.cra_name + 2;
		drvname = aes_algs[i].base.cra_driver_name + 2;
		basename = aes_algs[i].base.cra_driver_name;
		simd = simd_skcipher_create_compat(algname, drvname, basename);
		err = PTR_ERR(simd);
		if (IS_ERR(simd))
			goto unregister_simds;

		aes_simd_algs[i] = simd;
	}

	return 0;

unregister_simds:
	aes_exit();
	return err;
}

module_cpu_feature_match(AES, aes_init);
module_exit(aes_exit);


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Scalar AES core transform
 *
 * Copyright (C) 2017 Linaro Ltd.
 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/cache.h>

	.text
	.align		5

	rk		.req	r0
	rounds		.req	r1
	in		.req	r2
	out		.req	r3
	ttab		.req	ip

	t0		.req	lr
	t1		.req	r2
	t2		.req	r3

	.macro		__select, out, in, idx
	.if		__LINUX_ARM_ARCH__ < 7
	and		\out, \in, #0xff << (8 * \idx)
	.else
	ubfx		\out, \in, #(8 * \idx), #8
	.endif
	.endm

	.macro		__load, out, in, idx, sz, op
	.if		__LINUX_ARM_ARCH__ < 7 && \idx > 0
	ldr\op		\out, [ttab, \in, lsr #(8 * \idx) - \sz]
	.else
	ldr\op		\out, [ttab, \in, lsl #\sz]
	.endif
	.endm

	.macro		__hround, out0, out1, in0, in1, in2, in3, t3, t4, enc, sz, op, oldcpsr
	__select	\out0, \in0, 0
	__select	t0, \in1, 1
	__load		\out0, \out0, 0, \sz, \op
	__load		t0, t0, 1, \sz, \op

	.if		\enc
	__select	\out1, \in1, 0
	__select	t1, \in2, 1
	.else
	__select	\out1, \in3, 0
	__select	t1, \in0, 1
	.endif
	__load		\out1, \out1, 0, \sz, \op
	__select	t2, \in2, 2
	__load		t1, t1, 1, \sz, \op
	__load		t2, t2, 2, \sz, \op

	eor		\out0, \out0, t0, ror #24

	__select	t0, \in3, 3
	.if		\enc
	__select	\t3, \in3, 2
	__select	\t4, \in0, 3
	.else
	__select	\t3, \in1, 2
	__select	\t4, \in2, 3
	.endif
	__load		\t3, \t3, 2, \sz, \op
	__load		t0, t0, 3, \sz, \op
	__load		\t4, \t4, 3, \sz, \op

	.ifnb		\oldcpsr
	/*
	 * This is the final round and we're done with all data-dependent table
	 * lookups, so we can safely re-enable interrupts.
	 */
	restore_irqs	\oldcpsr
	.endif

	eor		\out1, \out1, t1, ror #24
	eor		\out0, \out0, t2, ror #16
	ldm		rk!, {t1, t2}
	eor		\out1, \out1, \t3, ror #16
	eor		\out0, \out0, t0, ror #8
	eor		\out1, \out1, \t4, ror #8
	eor		\out0, \out0, t1
	eor		\out1, \out1, t2
	.endm

	.macro		fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op, oldcpsr
	__hround	\out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
	__hround	\out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op, \oldcpsr
	.endm

	.macro		iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op, oldcpsr
	__hround	\out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
	__hround	\out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op, \oldcpsr
	.endm

	.macro		do_crypt, round, ttab, ltab, bsz
	push		{r3-r11, lr}

	// Load keys first, to reduce latency in case they're not cached yet.
	ldm		rk!, {r8-r11}

	ldr		r4, [in]
	ldr		r5, [in, #4]
	ldr		r6, [in, #8]
	ldr		r7, [in, #12]

#ifdef CONFIG_CPU_BIG_ENDIAN
	rev_l		r4, t0
	rev_l		r5, t0
	rev_l		r6, t0
	rev_l		r7, t0
#endif

	eor		r4, r4, r8
	eor		r5, r5, r9
	eor		r6, r6, r10
	eor		r7, r7, r11

	mov_l		ttab, \ttab
	/*
	 * Disable interrupts and prefetch the 1024-byte 'ft' or 'it' table into
	 * L1 cache, assuming cacheline size >= 32.  This is a hardening measure
	 * intended to make cache-timing attacks more difficult.  They may not
	 * be fully prevented, however; see the paper
	 * https://cr.yp.to/antiforgery/cachetiming-20050414.pdf
	 * ("Cache-timing attacks on AES") for a discussion of the many
	 * difficulties involved in writing truly constant-time AES software.
	 */
	 save_and_disable_irqs	t0
	.set		i, 0
	.rept		1024 / 128
	ldr		r8, [ttab, #i + 0]
	ldr		r9, [ttab, #i + 32]
	ldr		r10, [ttab, #i + 64]
	ldr		r11, [ttab, #i + 96]
	.set		i, i + 128
	.endr
	push		{t0}		// oldcpsr

	tst		rounds, #2
	bne		1f

0:	\round		r8, r9, r10, r11, r4, r5, r6, r7
	\round		r4, r5, r6, r7, r8, r9, r10, r11

1:	subs		rounds, rounds, #4
	\round		r8, r9, r10, r11, r4, r5, r6, r7
	bls		2f
	\round		r4, r5, r6, r7, r8, r9, r10, r11
	b		0b

2:	.ifb		\ltab
	add		ttab, ttab, #1
	.else
	mov_l		ttab, \ltab
	// Prefetch inverse S-box for final round; see explanation above
	.set		i, 0
	.rept		256 / 64
	ldr		t0, [ttab, #i + 0]
	ldr		t1, [ttab, #i + 32]
	.set		i, i + 64
	.endr
	.endif

	pop		{rounds}	// oldcpsr
	\round		r4, r5, r6, r7, r8, r9, r10, r11, \bsz, b, rounds

#ifdef CONFIG_CPU_BIG_ENDIAN
	rev_l		r4, t0
	rev_l		r5, t0
	rev_l		r6, t0
	rev_l		r7, t0
#endif

	ldr		out, [sp]

	str		r4, [out]
	str		r5, [out, #4]
	str		r6, [out, #8]
	str		r7, [out, #12]

	pop		{r3-r11, pc}

	.align		3
	.ltorg
	.endm

ENTRY(__aes_arm_encrypt)
	do_crypt	fround, crypto_ft_tab,, 2
ENDPROC(__aes_arm_encrypt)

	.align		5
ENTRY(__aes_arm_decrypt)
	do_crypt	iround, crypto_it_tab, crypto_aes_inv_sbox, 0
ENDPROC(__aes_arm_decrypt)

repo_name: angery/linux


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Scalar AES core transform
 *
 * Copyright (C) 2017 Linaro Ltd.
 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
 */

#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <linux/module.h>

asmlinkage void __aes_arm_encrypt(u32 *rk, int rounds, const u8 *in, u8 *out);
asmlinkage void __aes_arm_decrypt(u32 *rk, int rounds, const u8 *in, u8 *out);

static void aes_arm_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
	int rounds = 6 + ctx->key_length / 4;

	__aes_arm_encrypt(ctx->key_enc, rounds, in, out);
}

static void aes_arm_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
	int rounds = 6 + ctx->key_length / 4;

	__aes_arm_decrypt(ctx->key_dec, rounds, in, out);
}

static struct crypto_alg aes_alg = {
	.cra_name			= "aes",
	.cra_driver_name		= "aes-arm",
	.cra_priority			= 200,
	.cra_flags			= CRYPTO_ALG_TYPE_CIPHER,
	.cra_blocksize			= AES_BLOCK_SIZE,
	.cra_ctxsize			= sizeof(struct crypto_aes_ctx),
	.cra_module			= THIS_MODULE,

	.cra_cipher.cia_min_keysize	= AES_MIN_KEY_SIZE,
	.cra_cipher.cia_max_keysize	= AES_MAX_KEY_SIZE,
	.cra_cipher.cia_setkey		= crypto_aes_set_key,
	.cra_cipher.cia_encrypt		= aes_arm_encrypt,
	.cra_cipher.cia_decrypt		= aes_arm_decrypt,

#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
	.cra_alignmask			= 3,
#endif
};

static int __init aes_init(void)
{
	return crypto_register_alg(&aes_alg);
}

static void __exit aes_fini(void)
{
	crypto_unregister_alg(&aes_alg);
}

module_init(aes_init);
module_exit(aes_fini);

MODULE_DESCRIPTION("Scalar AES cipher for ARM");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_CRYPTO("aes");


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Bit sliced AES using NEON instructions
 *
 * Copyright (C) 2017 Linaro Ltd.
 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
 */

/*
 * The algorithm implemented here is described in detail by the paper
 * 'Faster and Timing-Attack Resistant AES-GCM' by Emilia Kaesper and
 * Peter Schwabe (https://eprint.iacr.org/2009/129.pdf)
 *
 * This implementation is based primarily on the OpenSSL implementation
 * for 32-bit ARM written by Andy Polyakov <appro@openssl.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

	.text
	.fpu		neon

	rounds		.req	ip
	bskey		.req	r4

	q0l		.req	d0
	q0h		.req	d1
	q1l		.req	d2
	q1h		.req	d3
	q2l		.req	d4
	q2h		.req	d5
	q3l		.req	d6
	q3h		.req	d7
	q4l		.req	d8
	q4h		.req	d9
	q5l		.req	d10
	q5h		.req	d11
	q6l		.req	d12
	q6h		.req	d13
	q7l		.req	d14
	q7h		.req	d15
	q8l		.req	d16
	q8h		.req	d17
	q9l		.req	d18
	q9h		.req	d19
	q10l		.req	d20
	q10h		.req	d21
	q11l		.req	d22
	q11h		.req	d23
	q12l		.req	d24
	q12h		.req	d25
	q13l		.req	d26
	q13h		.req	d27
	q14l		.req	d28
	q14h		.req	d29
	q15l		.req	d30
	q15h		.req	d31

	.macro		__tbl, out, tbl, in, tmp
	.ifc		\out, \tbl
	.ifb		\tmp
	.error		__tbl needs temp register if out == tbl
	.endif
	vmov		\tmp, \out
	.endif
	vtbl.8		\out\()l, {\tbl}, \in\()l
	.ifc		\out, \tbl
	vtbl.8		\out\()h, {\tmp}, \in\()h
	.else
	vtbl.8		\out\()h, {\tbl}, \in\()h
	.endif
	.endm

	.macro		__ldr, out, sym
	vldr		\out\()l, \sym
	vldr		\out\()h, \sym + 8
	.endm

	.macro		in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
	veor		\b2, \b2, \b1
	veor		\b5, \b5, \b6
	veor		\b3, \b3, \b0
	veor		\b6, \b6, \b2
	veor		\b5, \b5, \b0
	veor		\b6, \b6, \b3
	veor		\b3, \b3, \b7
	veor		\b7, \b7, \b5
	veor		\b3, \b3, \b4
	veor		\b4, \b4, \b5
	veor		\b2, \b2, \b7
	veor		\b3, \b3, \b1
	veor		\b1, \b1, \b5
	.endm

	.macro		out_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
	veor		\b0, \b0, \b6
	veor		\b1, \b1, \b4
	veor		\b4, \b4, \b6
	veor		\b2, \b2, \b0
	veor		\b6, \b6, \b1
	veor		\b1, \b1, \b5
	veor		\b5, \b5, \b3
	veor		\b3, \b3, \b7
	veor		\b7, \b7, \b5
	veor		\b2, \b2, \b5
	veor		\b4, \b4, \b7
	.endm

	.macro		inv_in_bs_ch, b6, b1, b2, b4, b7, b0, b3, b5
	veor		\b1, \b1, \b7
	veor		\b4, \b4, \b7
	veor		\b7, \b7, \b5
	veor		\b1, \b1, \b3
	veor		\b2, \b2, \b5
	veor		\b3, \b3, \b7
	veor		\b6, \b6, \b1
	veor		\b2, \b2, \b0
	veor		\b5, \b5, \b3
	veor		\b4, \b4, \b6
	veor		\b0, \b0, \b6
	veor		\b1, \b1, \b4
	.endm

	.macro		inv_out_bs_ch, b6, b5, b0, b3, b7, b1, b4, b2
	veor		\b1, \b1, \b5
	veor		\b2, \b2, \b7
	veor		\b3, \b3, \b1
	veor		\b4, \b4, \b5
	veor		\b7, \b7, \b5
	veor		\b3, \b3, \b4
	veor 		\b5, \b5, \b0
	veor		\b3, \b3, \b7
	veor		\b6, \b6, \b2
	veor		\b2, \b2, \b1
	veor		\b6, \b6, \b3
	veor		\b3, \b3, \b0
	veor		\b5, \b5, \b6
	.endm

	.macro		mul_gf4, x0, x1, y0, y1, t0, t1
	veor 		\t0, \y0, \y1
	vand		\t0, \t0, \x0
	veor		\x0, \x0, \x1
	vand		\t1, \x1, \y0
	vand		\x0, \x0, \y1
	veor		\x1, \t1, \t0
	veor		\x0, \x0, \t1
	.endm

	.macro		mul_gf4_n_gf4, x0, x1, y0, y1, t0, x2, x3, y2, y3, t1
	veor		\t0, \y0, \y1
	veor 		\t1, \y2, \y3
	vand		\t0, \t0, \x0
	vand		\t1, \t1, \x2
	veor		\x0, \x0, \x1
	veor		\x2, \x2, \x3
	vand		\x1, \x1, \y0
	vand		\x3, \x3, \y2
	vand		\x0, \x0, \y1
	vand		\x2, \x2, \y3
	veor		\x1, \x1, \x0
	veor		\x2, \x2, \x3
	veor		\x0, \x0, \t0
	veor		\x3, \x3, \t1
	.endm

	.macro		mul_gf16_2, x0, x1, x2, x3, x4, x5, x6, x7, \
				    y0, y1, y2, y3, t0, t1, t2, t3
	veor		\t0, \x0, \x2
	veor		\t1, \x1, \x3
	mul_gf4  	\x0, \x1, \y0, \y1, \t2, \t3
	veor		\y0, \y0, \y2
	veor		\y1, \y1, \y3
	mul_gf4_n_gf4	\t0, \t1, \y0, \y1, \t3, \x2, \x3, \y2, \y3, \t2
	veor		\x0, \x0, \t0
	veor		\x2, \x2, \t0
	veor		\x1, \x1, \t1
	veor		\x3, \x3, \t1
	veor		\t0, \x4, \x6
	veor		\t1, \x5, \x7
	mul_gf4_n_gf4	\t0, \t1, \y0, \y1, \t3, \x6, \x7, \y2, \y3, \t2
	veor		\y0, \y0, \y2
	veor		\y1, \y1, \y3
	mul_gf4  	\x4, \x5, \y0, \y1, \t2, \t3
	veor		\x4, \x4, \t0
	veor		\x6, \x6, \t0
	veor		\x5, \x5, \t1
	veor		\x7, \x7, \t1
	.endm

	.macro		inv_gf256, x0, x1, x2, x3, x4, x5, x6, x7, \
				   t0, t1, t2, t3, s0, s1, s2, s3
	veor		\t3, \x4, \x6
	veor		\t0, \x5, \x7
	veor		\t1, \x1, \x3
	veor		\s1, \x7, \x6
	veor		\s0, \x0, \x2
	veor		\s3, \t3, \t0
	vorr		\t2, \t0, \t1
	vand		\s2, \t3, \s0
	vorr		\t3, \t3, \s0
	veor		\s0, \s0, \t1
	vand		\t0, \t0, \t1
	veor		\t1, \x3, \x2
	vand		\s3, \s3, \s0
	vand		\s1, \s1, \t1
	veor		\t1, \x4, \x5
	veor		\s0, \x1, \x0
	veor		\t3, \t3, \s1
	veor		\t2, \t2, \s1
	vand		\s1, \t1, \s0
	vorr		\t1, \t1, \s0
	veor		\t3, \t3, \s3
	veor		\t0, \t0, \s1
	veor		\t2, \t2, \s2
	veor		\t1, \t1, \s3
	veor		\t0, \t0, \s2
	vand		\s0, \x7, \x3
	veor		\t1, \t1, \s2
	vand		\s1, \x6, \x2
	vand		\s2, \x5, \x1
	vorr		\s3, \x4, \x0
	veor		\t3, \t3, \s0
	veor		\t1, \t1, \s2
	veor		\s0, \t0, \s3
	veor		\t2, \t2, \s1
	vand		\s2, \t3, \t1
	veor		\s1, \t2, \s2
	veor		\s3, \s0, \s2
	vbsl		\s1, \t1, \s0
	vmvn		\t0, \s0
	vbsl		\s0, \s1, \s3
	vbsl		\t0, \s1, \s3
	vbsl		\s3, \t3, \t2
	veor		\t3, \t3, \t2
	vand		\s2, \s0, \s3
	veor		\t1, \t1, \t0
	veor		\s2, \s2, \t3
	mul_gf16_2	\x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
			\s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
	.endm

	.macro		sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
			      t0, t1, t2, t3, s0, s1, s2, s3
	in_bs_ch	\b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
	inv_gf256	\b6, \b5, \b0, \b3, \b7, \b1, \b4, \b2, \
			\t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
	out_bs_ch	\b7, \b1, \b4, \b2, \b6, \b5, \b0, \b3
	.endm

	.macro		inv_sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
				  t0, t1, t2, t3, s0, s1, s2, s3
	inv_in_bs_ch	\b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
	inv_gf256	\b5, \b1, \b2, \b6, \b3, \b7, \b0, \b4, \
			\t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
	inv_out_bs_ch	\b3, \b7, \b0, \b4, \b5, \b1, \b2, \b6
	.endm

	.macro		shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, \
				    t0, t1, t2, t3, mask
	vld1.8		{\t0-\t1}, [bskey, :256]!
	veor		\t0, \t0, \x0
	vld1.8		{\t2-\t3}, [bskey, :256]!
	veor		\t1, \t1, \x1
	__tbl		\x0, \t0, \mask
	veor		\t2, \t2, \x2
	__tbl		\x1, \t1, \mask
	vld1.8		{\t0-\t1}, [bskey, :256]!
	veor		\t3, \t3, \x3
	__tbl		\x2, \t2, \mask
	__tbl		\x3, \t3, \mask
	vld1.8		{\t2-\t3}, [bskey, :256]!
	veor		\t0, \t0, \x4
	veor		\t1, \t1, \x5
	__tbl		\x4, \t0, \mask
	veor		\t2, \t2, \x6
	__tbl		\x5, \t1, \mask
	veor		\t3, \t3, \x7
	__tbl		\x6, \t2, \mask
	__tbl		\x7, \t3, \mask
	.endm

	.macro		inv_shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, \
					t0, t1, t2, t3, mask
	__tbl		\x0, \x0, \mask, \t0
	__tbl		\x1, \x1, \mask, \t1
	__tbl		\x2, \x2, \mask, \t2
	__tbl		\x3, \x3, \mask, \t3
	__tbl		\x4, \x4, \mask, \t0
	__tbl		\x5, \x5, \mask, \t1
	__tbl		\x6, \x6, \mask, \t2
	__tbl		\x7, \x7, \mask, \t3
	.endm

	.macro		mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
				  t0, t1, t2, t3, t4, t5, t6, t7, inv
	vext.8		\t0, \x0, \x0, #12
	vext.8		\t1, \x1, \x1, #12
	veor		\x0, \x0, \t0
	vext.8		\t2, \x2, \x2, #12
	veor		\x1, \x1, \t1
	vext.8		\t3, \x3, \x3, #12
	veor		\x2, \x2, \t2
	vext.8		\t4, \x4, \x4, #12
	veor		\x3, \x3, \t3
	vext.8		\t5, \x5, \x5, #12
	veor		\x4, \x4, \t4
	vext.8		\t6, \x6, \x6, #12
	veor		\x5, \x5, \t5
	vext.8		\t7, \x7, \x7, #12
	veor		\x6, \x6, \t6
	veor		\t1, \t1, \x0
	veor.8		\x7, \x7, \t7
	vext.8		\x0, \x0, \x0, #8
	veor		\t2, \t2, \x1
	veor		\t0, \t0, \x7
	veor		\t1, \t1, \x7
	vext.8		\x1, \x1, \x1, #8
	veor		\t5, \t5, \x4
	veor		\x0, \x0, \t0
	veor		\t6, \t6, \x5
	veor		\x1, \x1, \t1
	vext.8		\t0, \x4, \x4, #8
	veor		\t4, \t4, \x3
	vext.8		\t1, \x5, \x5, #8
	veor		\t7, \t7, \x6
	vext.8		\x4, \x3, \x3, #8
	veor		\t3, \t3, \x2
	vext.8		\x5, \x7, \x7, #8
	veor		\t4, \t4, \x7
	vext.8		\x3, \x6, \x6, #8
	veor		\t3, \t3, \x7
	vext.8		\x6, \x2, \x2, #8
	veor		\x7, \t1, \t5
	.ifb		\inv
	veor		\x2, \t0, \t4
	veor		\x4, \x4, \t3
	veor		\x5, \x5, \t7
	veor		\x3, \x3, \t6
	veor		\x6, \x6, \t2
	.else
	veor		\t3, \t3, \x4
	veor		\x5, \x5, \t7
	veor		\x2, \x3, \t6
	veor		\x3, \t0, \t4
	veor		\x4, \x6, \t2
	vmov		\x6, \t3
	.endif
	.endm

	.macro		inv_mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
				      t0, t1, t2, t3, t4, t5, t6, t7
	vld1.8		{\t0-\t1}, [bskey, :256]!
	veor		\x0, \x0, \t0
	vld1.8		{\t2-\t3}, [bskey, :256]!
	veor		\x1, \x1, \t1
	vld1.8		{\t4-\t5}, [bskey, :256]!
	veor		\x2, \x2, \t2
	vld1.8		{\t6-\t7}, [bskey, :256]
	sub		bskey, bskey, #224
	veor		\x3, \x3, \t3
	veor		\x4, \x4, \t4
	veor		\x5, \x5, \t5
	veor		\x6, \x6, \t6
	veor		\x7, \x7, \t7
	vext.8		\t0, \x0, \x0, #8
	vext.8		\t6, \x6, \x6, #8
	vext.8		\t7, \x7, \x7, #8
	veor		\t0, \t0, \x0
	vext.8		\t1, \x1, \x1, #8
	veor		\t6, \t6, \x6
	vext.8		\t2, \x2, \x2, #8
	veor		\t7, \t7, \x7
	vext.8		\t3, \x3, \x3, #8
	veor		\t1, \t1, \x1
	vext.8		\t4, \x4, \x4, #8
	veor		\t2, \t2, \x2
	vext.8		\t5, \x5, \x5, #8
	veor		\t3, \t3, \x3
	veor		\t4, \t4, \x4
	veor		\t5, \t5, \x5
	veor		\x0, \x0, \t6
	veor		\x1, \x1, \t6
	veor		\x2, \x2, \t0
	veor		\x4, \x4, \t2
	veor		\x3, \x3, \t1
	veor		\x1, \x1, \t7
	veor		\x2, \x2, \t7
	veor		\x4, \x4, \t6
	veor		\x5, \x5, \t3
	veor		\x3, \x3, \t6
	veor		\x6, \x6, \t4
	veor		\x4, \x4, \t7
	veor		\x5, \x5, \t7
	veor		\x7, \x7, \t5
	mix_cols	\x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
			\t0, \t1, \t2, \t3, \t4, \t5, \t6, \t7, 1
	.endm

	.macro		swapmove_2x, a0, b0, a1, b1, n, mask, t0, t1
	vshr.u64	\t0, \b0, #\n
	vshr.u64	\t1, \b1, #\n
	veor		\t0, \t0, \a0
	veor		\t1, \t1, \a1
	vand		\t0, \t0, \mask
	vand		\t1, \t1, \mask
	veor		\a0, \a0, \t0
	vshl.s64	\t0, \t0, #\n
	veor		\a1, \a1, \t1
	vshl.s64	\t1, \t1, #\n
	veor		\b0, \b0, \t0
	veor		\b1, \b1, \t1
	.endm

	.macro		bitslice, x7, x6, x5, x4, x3, x2, x1, x0, t0, t1, t2, t3
	vmov.i8		\t0, #0x55
	vmov.i8		\t1, #0x33
	swapmove_2x	\x0, \x1, \x2, \x3, 1, \t0, \t2, \t3
	swapmove_2x	\x4, \x5, \x6, \x7, 1, \t0, \t2, \t3
	vmov.i8		\t0, #0x0f
	swapmove_2x	\x0, \x2, \x1, \x3, 2, \t1, \t2, \t3
	swapmove_2x	\x4, \x6, \x5, \x7, 2, \t1, \t2, \t3
	swapmove_2x	\x0, \x4, \x1, \x5, 4, \t0, \t2, \t3
	swapmove_2x	\x2, \x6, \x3, \x7, 4, \t0, \t2, \t3
	.endm

	.align		4
M0:	.quad		0x02060a0e03070b0f, 0x0004080c0105090d

	/*
	 * void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
	 */
ENTRY(aesbs_convert_key)
	vld1.32		{q7}, [r1]!		// load round 0 key
	vld1.32		{q15}, [r1]!		// load round 1 key

	vmov.i8		q8,  #0x01		// bit masks
	vmov.i8		q9,  #0x02
	vmov.i8		q10, #0x04
	vmov.i8		q11, #0x08
	vmov.i8		q12, #0x10
	vmov.i8		q13, #0x20
	__ldr		q14, M0

	sub		r2, r2, #1
	vst1.8		{q7}, [r0, :128]!	// save round 0 key

.Lkey_loop:
	__tbl		q7, q15, q14
	vmov.i8		q6, #0x40
	vmov.i8		q15, #0x80

	vtst.8		q0, q7, q8
	vtst.8		q1, q7, q9
	vtst.8		q2, q7, q10
	vtst.8		q3, q7, q11
	vtst.8		q4, q7, q12
	vtst.8		q5, q7, q13
	vtst.8		q6, q7, q6
	vtst.8		q7, q7, q15
	vld1.32		{q15}, [r1]!		// load next round key
	vmvn		q0, q0
	vmvn		q1, q1
	vmvn		q5, q5
	vmvn		q6, q6

	subs		r2, r2, #1
	vst1.8		{q0-q1}, [r0, :256]!
	vst1.8		{q2-q3}, [r0, :256]!
	vst1.8		{q4-q5}, [r0, :256]!
	vst1.8		{q6-q7}, [r0, :256]!
	bne		.Lkey_loop

	vmov.i8		q7, #0x63		// compose .L63
	veor		q15, q15, q7
	vst1.8		{q15}, [r0, :128]
	bx		lr
ENDPROC(aesbs_convert_key)

	.align		4
M0SR:	.quad		0x0a0e02060f03070b, 0x0004080c05090d01

aesbs_encrypt8:
	vld1.8		{q9}, [bskey, :128]!	// round 0 key
	__ldr		q8, M0SR

	veor		q10, q0, q9		// xor with round0 key
	veor		q11, q1, q9
	__tbl		q0, q10, q8
	veor		q12, q2, q9
	__tbl		q1, q11, q8
	veor		q13, q3, q9
	__tbl		q2, q12, q8
	veor		q14, q4, q9
	__tbl		q3, q13, q8
	veor		q15, q5, q9
	__tbl		q4, q14, q8
	veor		q10, q6, q9
	__tbl		q5, q15, q8
	veor		q11, q7, q9
	__tbl		q6, q10, q8
	__tbl		q7, q11, q8

	bitslice	q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11

	sub		rounds, rounds, #1
	b		.Lenc_sbox

	.align		5
SR:	.quad		0x0504070600030201, 0x0f0e0d0c0a09080b
SRM0:	.quad		0x0304090e00050a0f, 0x01060b0c0207080d

.Lenc_last:
	__ldr		q12, SRM0
.Lenc_loop:
	shift_rows	q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12
.Lenc_sbox:
	sbox		q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, \
								q13, q14, q15
	subs		rounds, rounds, #1
	bcc		.Lenc_done

	mix_cols	q0, q1, q4, q6, q3, q7, q2, q5, q8, q9, q10, q11, q12, \
								q13, q14, q15

	beq		.Lenc_last
	__ldr		q12, SR
	b		.Lenc_loop

.Lenc_done:
	vld1.8		{q12}, [bskey, :128]	// last round key

	bitslice	q0, q1, q4, q6, q3, q7, q2, q5, q8, q9, q10, q11

	veor		q0, q0, q12
	veor		q1, q1, q12
	veor		q4, q4, q12
	veor		q6, q6, q12
	veor		q3, q3, q12
	veor		q7, q7, q12
	veor		q2, q2, q12
	veor		q5, q5, q12
	bx		lr
ENDPROC(aesbs_encrypt8)

	.align		4
M0ISR:	.quad		0x0a0e0206070b0f03, 0x0004080c0d010509

aesbs_decrypt8:
	add		bskey, bskey, rounds, lsl #7
	sub		bskey, bskey, #112
	vld1.8		{q9}, [bskey, :128]	// round 0 key
	sub		bskey, bskey, #128
	__ldr		q8, M0ISR

	veor		q10, q0, q9		// xor with round0 key
	veor		q11, q1, q9
	__tbl		q0, q10, q8
	veor		q12, q2, q9
	__tbl		q1, q11, q8
	veor		q13, q3, q9
	__tbl		q2, q12, q8
	veor		q14, q4, q9
	__tbl		q3, q13, q8
	veor		q15, q5, q9
	__tbl		q4, q14, q8
	veor		q10, q6, q9
	__tbl		q5, q15, q8
	veor		q11, q7, q9
	__tbl		q6, q10, q8
	__tbl		q7, q11, q8

	bitslice	q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11

	sub		rounds, rounds, #1
	b		.Ldec_sbox

	.align		5
ISR:	.quad		0x0504070602010003, 0x0f0e0d0c080b0a09
ISRM0:	.quad		0x01040b0e0205080f, 0x0306090c00070a0d

.Ldec_last:
	__ldr		q12, ISRM0
.Ldec_loop:
	inv_shift_rows	q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12
.Ldec_sbox:
	inv_sbox	q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, \
								q13, q14, q15
	subs		rounds, rounds, #1
	bcc		.Ldec_done

	inv_mix_cols	q0, q1, q6, q4, q2, q7, q3, q5, q8, q9, q10, q11, q12, \
								q13, q14, q15

	beq		.Ldec_last
	__ldr		q12, ISR
	b		.Ldec_loop

.Ldec_done:
	add		bskey, bskey, #112
	vld1.8		{q12}, [bskey, :128]	// last round key

	bitslice	q0, q1, q6, q4, q2, q7, q3, q5, q8, q9, q10, q11

	veor		q0, q0, q12
	veor		q1, q1, q12
	veor		q6, q6, q12
	veor		q4, q4, q12
	veor		q2, q2, q12
	veor		q7, q7, q12
	veor		q3, q3, q12
	veor		q5, q5, q12
	bx		lr
ENDPROC(aesbs_decrypt8)

	/*
	 * aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		     int blocks)
	 * aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		     int blocks)
	 */
	.macro		__ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
	push		{r4-r6, lr}
	ldr		r5, [sp, #16]		// number of blocks

99:	adr		ip, 0f
	and		lr, r5, #7
	cmp		r5, #8
	sub		ip, ip, lr, lsl #2
	movlt		pc, ip			// computed goto if blocks < 8

	vld1.8		{q0}, [r1]!
	vld1.8		{q1}, [r1]!
	vld1.8		{q2}, [r1]!
	vld1.8		{q3}, [r1]!
	vld1.8		{q4}, [r1]!
	vld1.8		{q5}, [r1]!
	vld1.8		{q6}, [r1]!
	vld1.8		{q7}, [r1]!

0:	mov		bskey, r2
	mov		rounds, r3
	bl		\do8

	adr		ip, 1f
	and		lr, r5, #7
	cmp		r5, #8
	sub		ip, ip, lr, lsl #2
	movlt		pc, ip			// computed goto if blocks < 8

	vst1.8		{\o0}, [r0]!
	vst1.8		{\o1}, [r0]!
	vst1.8		{\o2}, [r0]!
	vst1.8		{\o3}, [r0]!
	vst1.8		{\o4}, [r0]!
	vst1.8		{\o5}, [r0]!
	vst1.8		{\o6}, [r0]!
	vst1.8		{\o7}, [r0]!

1:	subs		r5, r5, #8
	bgt		99b

	pop		{r4-r6, pc}
	.endm

	.align		4
ENTRY(aesbs_ecb_encrypt)
	__ecb_crypt	aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5
ENDPROC(aesbs_ecb_encrypt)

	.align		4
ENTRY(aesbs_ecb_decrypt)
	__ecb_crypt	aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5
ENDPROC(aesbs_ecb_decrypt)

	/*
	 * aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
	 *		     int rounds, int blocks, u8 iv[])
	 */
	.align		4
ENTRY(aesbs_cbc_decrypt)
	mov		ip, sp
	push		{r4-r6, lr}
	ldm		ip, {r5-r6}		// load args 4-5

99:	adr		ip, 0f
	and		lr, r5, #7
	cmp		r5, #8
	sub		ip, ip, lr, lsl #2
	mov		lr, r1
	movlt		pc, ip			// computed goto if blocks < 8

	vld1.8		{q0}, [lr]!
	vld1.8		{q1}, [lr]!
	vld1.8		{q2}, [lr]!
	vld1.8		{q3}, [lr]!
	vld1.8		{q4}, [lr]!
	vld1.8		{q5}, [lr]!
	vld1.8		{q6}, [lr]!
	vld1.8		{q7}, [lr]

0:	mov		bskey, r2
	mov		rounds, r3
	bl		aesbs_decrypt8

	vld1.8		{q8}, [r6]
	vmov		q9, q8
	vmov		q10, q8
	vmov		q11, q8
	vmov		q12, q8
	vmov		q13, q8
	vmov		q14, q8
	vmov		q15, q8

	adr		ip, 1f
	and		lr, r5, #7
	cmp		r5, #8
	sub		ip, ip, lr, lsl #2
	movlt		pc, ip			// computed goto if blocks < 8

	vld1.8		{q9}, [r1]!
	vld1.8		{q10}, [r1]!
	vld1.8		{q11}, [r1]!
	vld1.8		{q12}, [r1]!
	vld1.8		{q13}, [r1]!
	vld1.8		{q14}, [r1]!
	vld1.8		{q15}, [r1]!
	W(nop)

1:	adr		ip, 2f
	sub		ip, ip, lr, lsl #3
	movlt		pc, ip			// computed goto if blocks < 8

	veor		q0, q0, q8
	vst1.8		{q0}, [r0]!
	veor		q1, q1, q9
	vst1.8		{q1}, [r0]!
	veor		q6, q6, q10
	vst1.8		{q6}, [r0]!
	veor		q4, q4, q11
	vst1.8		{q4}, [r0]!
	veor		q2, q2, q12
	vst1.8		{q2}, [r0]!
	veor		q7, q7, q13
	vst1.8		{q7}, [r0]!
	veor		q3, q3, q14
	vst1.8		{q3}, [r0]!
	veor		q5, q5, q15
	vld1.8		{q8}, [r1]!		// load next round's iv
2:	vst1.8		{q5}, [r0]!

	subs		r5, r5, #8
	vst1.8		{q8}, [r6]		// store next round's iv
	bgt		99b

	pop		{r4-r6, pc}
ENDPROC(aesbs_cbc_decrypt)

	.macro		next_ctr, q
	vmov		\q\()h, r9, r10
	adds		r10, r10, #1
	adcs		r9, r9, #0
	vmov		\q\()l, r7, r8
	adcs		r8, r8, #0
	adc		r7, r7, #0
	vrev32.8	\q, \q
	.endm

	/*
	 * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
	 *		     int rounds, int bytes, u8 ctr[])
	 */
ENTRY(aesbs_ctr_encrypt)
	mov		ip, sp
	push		{r4-r10, lr}

	ldm		ip, {r5, r6}		// load args 4-5
	vld1.8		{q0}, [r6]		// load counter
	vrev32.8	q1, q0
	vmov		r9, r10, d3
	vmov		r7, r8, d2

	adds		r10, r10, #1
	adcs		r9, r9, #0
	adcs		r8, r8, #0
	adc		r7, r7, #0

99:	vmov		q1, q0
	sub		lr, r5, #1
	vmov		q2, q0
	adr		ip, 0f
	vmov		q3, q0
	and		lr, lr, #112
	vmov		q4, q0
	cmp		r5, #112
	vmov		q5, q0
	sub		ip, ip, lr, lsl #1
	vmov		q6, q0
	add		ip, ip, lr, lsr #2
	vmov		q7, q0
	movle		pc, ip			// computed goto if bytes < 112

	next_ctr	q1
	next_ctr	q2
	next_ctr	q3
	next_ctr	q4
	next_ctr	q5
	next_ctr	q6
	next_ctr	q7

0:	mov		bskey, r2
	mov		rounds, r3
	bl		aesbs_encrypt8

	adr		ip, 1f
	sub		lr, r5, #1
	cmp		r5, #128
	bic		lr, lr, #15
	ands		r4, r5, #15		// preserves C flag
	teqcs		r5, r5			// set Z flag if not last iteration
	sub		ip, ip, lr, lsr #2
	rsb		r4, r4, #16
	movcc		pc, ip			// computed goto if bytes < 128

	vld1.8		{q8}, [r1]!
	vld1.8		{q9}, [r1]!
	vld1.8		{q10}, [r1]!
	vld1.8		{q11}, [r1]!
	vld1.8		{q12}, [r1]!
	vld1.8		{q13}, [r1]!
	vld1.8		{q14}, [r1]!
1:	subne		r1, r1, r4
	vld1.8		{q15}, [r1]!

	add		ip, ip, #2f - 1b

	veor		q0, q0, q8
	veor		q1, q1, q9
	veor		q4, q4, q10
	veor		q6, q6, q11
	veor		q3, q3, q12
	veor		q7, q7, q13
	veor		q2, q2, q14
	bne		3f
	veor		q5, q5, q15

	movcc		pc, ip			// computed goto if bytes < 128

	vst1.8		{q0}, [r0]!
	vst1.8		{q1}, [r0]!
	vst1.8		{q4}, [r0]!
	vst1.8		{q6}, [r0]!
	vst1.8		{q3}, [r0]!
	vst1.8		{q7}, [r0]!
	vst1.8		{q2}, [r0]!
2:	subne		r0, r0, r4
	vst1.8		{q5}, [r0]!

	next_ctr	q0

	subs		r5, r5, #128
	bgt		99b

	vst1.8		{q0}, [r6]
	pop		{r4-r10, pc}

3:	adr		lr, .Lpermute_table + 16
	cmp		r5, #16			// Z flag remains cleared
	sub		lr, lr, r4
	vld1.8		{q8-q9}, [lr]
	vtbl.8		d16, {q5}, d16
	vtbl.8		d17, {q5}, d17
	veor		q5, q8, q15
	bcc		4f			// have to reload prev if R5 < 16
	vtbx.8		d10, {q2}, d18
	vtbx.8		d11, {q2}, d19
	mov		pc, ip			// branch back to VST sequence

4:	sub		r0, r0, r4
	vshr.s8		q9, q9, #7		// create mask for VBIF
	vld1.8		{q8}, [r0]		// reload
	vbif		q5, q8, q9
	vst1.8		{q5}, [r0]
	pop		{r4-r10, pc}
ENDPROC(aesbs_ctr_encrypt)

	.align		6
.Lpermute_table:
	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
	.byte		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
	.byte		0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff

	.macro		next_tweak, out, in, const, tmp
	vshr.s64	\tmp, \in, #63
	vand		\tmp, \tmp, \const
	vadd.u64	\out, \in, \in
	vext.8		\tmp, \tmp, \tmp, #8
	veor		\out, \out, \tmp
	.endm

	/*
	 * aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		     int blocks, u8 iv[], int reorder_last_tweak)
	 * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		     int blocks, u8 iv[], int reorder_last_tweak)
	 */
	.align		6
__xts_prepare8:
	vld1.8		{q14}, [r7]		// load iv
	vmov.i32	d30, #0x87		// compose tweak mask vector
	vmovl.u32	q15, d30
	vshr.u64	d30, d31, #7
	vmov		q12, q14

	adr		ip, 0f
	and		r4, r6, #7
	cmp		r6, #8
	sub		ip, ip, r4, lsl #5
	mov		r4, sp
	movlt		pc, ip			// computed goto if blocks < 8

	vld1.8		{q0}, [r1]!
	next_tweak	q12, q14, q15, q13
	veor		q0, q0, q14
	vst1.8		{q14}, [r4, :128]!

	vld1.8		{q1}, [r1]!
	next_tweak	q14, q12, q15, q13
	veor		q1, q1, q12
	vst1.8		{q12}, [r4, :128]!

	vld1.8		{q2}, [r1]!
	next_tweak	q12, q14, q15, q13
	veor		q2, q2, q14
	vst1.8		{q14}, [r4, :128]!

	vld1.8		{q3}, [r1]!
	next_tweak	q14, q12, q15, q13
	veor		q3, q3, q12
	vst1.8		{q12}, [r4, :128]!

	vld1.8		{q4}, [r1]!
	next_tweak	q12, q14, q15, q13
	veor		q4, q4, q14
	vst1.8		{q14}, [r4, :128]!

	vld1.8		{q5}, [r1]!
	next_tweak	q14, q12, q15, q13
	veor		q5, q5, q12
	vst1.8		{q12}, [r4, :128]!

	vld1.8		{q6}, [r1]!
	next_tweak	q12, q14, q15, q13
	veor		q6, q6, q14
	vst1.8		{q14}, [r4, :128]!

	vld1.8		{q7}, [r1]!
	next_tweak	q14, q12, q15, q13
THUMB(	itt		le		)
	W(cmple)	r8, #0
	ble		1f
0:	veor		q7, q7, q12
	vst1.8		{q12}, [r4, :128]

	vst1.8		{q14}, [r7]		// store next iv
	bx		lr

1:	vswp		q12, q14
	b		0b
ENDPROC(__xts_prepare8)

	.macro		__xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
	push		{r4-r8, lr}
	mov		r5, sp			// preserve sp
	ldrd		r6, r7, [sp, #24]	// get blocks and iv args
	rsb		r8, ip, #1
	sub		ip, sp, #128		// make room for 8x tweak
	bic		ip, ip, #0xf		// align sp to 16 bytes
	mov		sp, ip

99:	bl		__xts_prepare8

	mov		bskey, r2
	mov		rounds, r3
	bl		\do8

	adr		ip, 0f
	and		lr, r6, #7
	cmp		r6, #8
	sub		ip, ip, lr, lsl #2
	mov		r4, sp
	movlt		pc, ip			// computed goto if blocks < 8

	vld1.8		{q8}, [r4, :128]!
	vld1.8		{q9}, [r4, :128]!
	vld1.8		{q10}, [r4, :128]!
	vld1.8		{q11}, [r4, :128]!
	vld1.8		{q12}, [r4, :128]!
	vld1.8		{q13}, [r4, :128]!
	vld1.8		{q14}, [r4, :128]!
	vld1.8		{q15}, [r4, :128]

0:	adr		ip, 1f
	sub		ip, ip, lr, lsl #3
	movlt		pc, ip			// computed goto if blocks < 8

	veor		\o0, \o0, q8
	vst1.8		{\o0}, [r0]!
	veor		\o1, \o1, q9
	vst1.8		{\o1}, [r0]!
	veor		\o2, \o2, q10
	vst1.8		{\o2}, [r0]!
	veor		\o3, \o3, q11
	vst1.8		{\o3}, [r0]!
	veor		\o4, \o4, q12
	vst1.8		{\o4}, [r0]!
	veor		\o5, \o5, q13
	vst1.8		{\o5}, [r0]!
	veor		\o6, \o6, q14
	vst1.8		{\o6}, [r0]!
	veor		\o7, \o7, q15
	vst1.8		{\o7}, [r0]!

1:	subs		r6, r6, #8
	bgt		99b

	mov		sp, r5
	pop		{r4-r8, pc}
	.endm

ENTRY(aesbs_xts_encrypt)
	mov		ip, #0			// never reorder final tweak
	__xts_crypt	aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5
ENDPROC(aesbs_xts_encrypt)

ENTRY(aesbs_xts_decrypt)
	ldr		ip, [sp, #8]		// reorder final tweak?
	__xts_crypt	aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5
ENDPROC(aesbs_xts_decrypt)


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Bit sliced AES using NEON instructions
 *
 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/aes.h>
#include <crypto/ctr.h>
#include <crypto/internal/cipher.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <crypto/xts.h>
#include <linux/module.h>

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");

MODULE_ALIAS_CRYPTO("ecb(aes)");
MODULE_ALIAS_CRYPTO("cbc(aes)-all");
MODULE_ALIAS_CRYPTO("ctr(aes)");
MODULE_ALIAS_CRYPTO("xts(aes)");

MODULE_IMPORT_NS(CRYPTO_INTERNAL);

asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);

asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
				  int rounds, int blocks);
asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
				  int rounds, int blocks);

asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
				  int rounds, int blocks, u8 iv[]);

asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
				  int rounds, int blocks, u8 ctr[]);

asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
				  int rounds, int blocks, u8 iv[], int);
asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
				  int rounds, int blocks, u8 iv[], int);

struct aesbs_ctx {
	int	rounds;
	u8	rk[13 * (8 * AES_BLOCK_SIZE) + 32] __aligned(AES_BLOCK_SIZE);
};

struct aesbs_cbc_ctx {
	struct aesbs_ctx	key;
	struct crypto_skcipher	*enc_tfm;
};

struct aesbs_xts_ctx {
	struct aesbs_ctx	key;
	struct crypto_cipher	*cts_tfm;
	struct crypto_cipher	*tweak_tfm;
};

struct aesbs_ctr_ctx {
	struct aesbs_ctx	key;		/* must be first member */
	struct crypto_aes_ctx	fallback;
};

static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
			unsigned int key_len)
{
	struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct crypto_aes_ctx rk;
	int err;

	err = aes_expandkey(&rk, in_key, key_len);
	if (err)
		return err;

	ctx->rounds = 6 + key_len / 4;

	kernel_neon_begin();
	aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds);
	kernel_neon_end();

	return 0;
}

static int __ecb_crypt(struct skcipher_request *req,
		       void (*fn)(u8 out[], u8 const in[], u8 const rk[],
				  int rounds, int blocks))
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	int err;

	err = skcipher_walk_virt(&walk, req, false);

	while (walk.nbytes >= AES_BLOCK_SIZE) {
		unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;

		if (walk.nbytes < walk.total)
			blocks = round_down(blocks,
					    walk.stride / AES_BLOCK_SIZE);

		kernel_neon_begin();
		fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk,
		   ctx->rounds, blocks);
		kernel_neon_end();
		err = skcipher_walk_done(&walk,
					 walk.nbytes - blocks * AES_BLOCK_SIZE);
	}

	return err;
}

static int ecb_encrypt(struct skcipher_request *req)
{
	return __ecb_crypt(req, aesbs_ecb_encrypt);
}

static int ecb_decrypt(struct skcipher_request *req)
{
	return __ecb_crypt(req, aesbs_ecb_decrypt);
}

static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
			    unsigned int key_len)
{
	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct crypto_aes_ctx rk;
	int err;

	err = aes_expandkey(&rk, in_key, key_len);
	if (err)
		return err;

	ctx->key.rounds = 6 + key_len / 4;

	kernel_neon_begin();
	aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds);
	kernel_neon_end();
	memzero_explicit(&rk, sizeof(rk));

	return crypto_skcipher_setkey(ctx->enc_tfm, in_key, key_len);
}

static int cbc_encrypt(struct skcipher_request *req)
{
	struct skcipher_request *subreq = skcipher_request_ctx(req);
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);

	skcipher_request_set_tfm(subreq, ctx->enc_tfm);
	skcipher_request_set_callback(subreq,
				      skcipher_request_flags(req),
				      NULL, NULL);
	skcipher_request_set_crypt(subreq, req->src, req->dst,
				   req->cryptlen, req->iv);

	return crypto_skcipher_encrypt(subreq);
}

static int cbc_decrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	int err;

	err = skcipher_walk_virt(&walk, req, false);

	while (walk.nbytes >= AES_BLOCK_SIZE) {
		unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;

		if (walk.nbytes < walk.total)
			blocks = round_down(blocks,
					    walk.stride / AES_BLOCK_SIZE);

		kernel_neon_begin();
		aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
				  ctx->key.rk, ctx->key.rounds, blocks,
				  walk.iv);
		kernel_neon_end();
		err = skcipher_walk_done(&walk,
					 walk.nbytes - blocks * AES_BLOCK_SIZE);
	}

	return err;
}

static int cbc_init(struct crypto_skcipher *tfm)
{
	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
	unsigned int reqsize;

	ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC |
					     CRYPTO_ALG_NEED_FALLBACK);
	if (IS_ERR(ctx->enc_tfm))
		return PTR_ERR(ctx->enc_tfm);

	reqsize = sizeof(struct skcipher_request);
	reqsize += crypto_skcipher_reqsize(ctx->enc_tfm);
	crypto_skcipher_set_reqsize(tfm, reqsize);

	return 0;
}

static void cbc_exit(struct crypto_skcipher *tfm)
{
	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);

	crypto_free_skcipher(ctx->enc_tfm);
}

static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key,
				 unsigned int key_len)
{
	struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
	int err;

	err = aes_expandkey(&ctx->fallback, in_key, key_len);
	if (err)
		return err;

	ctx->key.rounds = 6 + key_len / 4;

	kernel_neon_begin();
	aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds);
	kernel_neon_end();

	return 0;
}

static int ctr_encrypt(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	u8 buf[AES_BLOCK_SIZE];
	int err;

	err = skcipher_walk_virt(&walk, req, false);

	while (walk.nbytes > 0) {
		const u8 *src = walk.src.virt.addr;
		u8 *dst = walk.dst.virt.addr;
		int bytes = walk.nbytes;

		if (unlikely(bytes < AES_BLOCK_SIZE))
			src = dst = memcpy(buf + sizeof(buf) - bytes,
					   src, bytes);
		else if (walk.nbytes < walk.total)
			bytes &= ~(8 * AES_BLOCK_SIZE - 1);

		kernel_neon_begin();
		aesbs_ctr_encrypt(dst, src, ctx->rk, ctx->rounds, bytes, walk.iv);
		kernel_neon_end();

		if (unlikely(bytes < AES_BLOCK_SIZE))
			memcpy(walk.dst.virt.addr,
			       buf + sizeof(buf) - bytes, bytes);

		err = skcipher_walk_done(&walk, walk.nbytes - bytes);
	}

	return err;
}

static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
{
	struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
	unsigned long flags;

	/*
	 * Temporarily disable interrupts to avoid races where
	 * cachelines are evicted when the CPU is interrupted
	 * to do something else.
	 */
	local_irq_save(flags);
	aes_encrypt(&ctx->fallback, dst, src);
	local_irq_restore(flags);
}

static int ctr_encrypt_sync(struct skcipher_request *req)
{
	if (!crypto_simd_usable())
		return crypto_ctr_encrypt_walk(req, ctr_encrypt_one);

	return ctr_encrypt(req);
}

static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
			    unsigned int key_len)
{
	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
	int err;

	err = xts_verify_key(tfm, in_key, key_len);
	if (err)
		return err;

	key_len /= 2;
	err = crypto_cipher_setkey(ctx->cts_tfm, in_key, key_len);
	if (err)
		return err;
	err = crypto_cipher_setkey(ctx->tweak_tfm, in_key + key_len, key_len);
	if (err)
		return err;

	return aesbs_setkey(tfm, in_key, key_len);
}

static int xts_init(struct crypto_skcipher *tfm)
{
	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

	ctx->cts_tfm = crypto_alloc_cipher("aes", 0, 0);
	if (IS_ERR(ctx->cts_tfm))
		return PTR_ERR(ctx->cts_tfm);

	ctx->tweak_tfm = crypto_alloc_cipher("aes", 0, 0);
	if (IS_ERR(ctx->tweak_tfm))
		crypto_free_cipher(ctx->cts_tfm);

	return PTR_ERR_OR_ZERO(ctx->tweak_tfm);
}

static void xts_exit(struct crypto_skcipher *tfm)
{
	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

	crypto_free_cipher(ctx->tweak_tfm);
	crypto_free_cipher(ctx->cts_tfm);
}

static int __xts_crypt(struct skcipher_request *req, bool encrypt,
		       void (*fn)(u8 out[], u8 const in[], u8 const rk[],
				  int rounds, int blocks, u8 iv[], int))
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
	int tail = req->cryptlen % AES_BLOCK_SIZE;
	struct skcipher_request subreq;
	u8 buf[2 * AES_BLOCK_SIZE];
	struct skcipher_walk walk;
	int err;

	if (req->cryptlen < AES_BLOCK_SIZE)
		return -EINVAL;

	if (unlikely(tail)) {
		skcipher_request_set_tfm(&subreq, tfm);
		skcipher_request_set_callback(&subreq,
					      skcipher_request_flags(req),
					      NULL, NULL);
		skcipher_request_set_crypt(&subreq, req->src, req->dst,
					   req->cryptlen - tail, req->iv);
		req = &subreq;
	}

	err = skcipher_walk_virt(&walk, req, true);
	if (err)
		return err;

	crypto_cipher_encrypt_one(ctx->tweak_tfm, walk.iv, walk.iv);

	while (walk.nbytes >= AES_BLOCK_SIZE) {
		unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
		int reorder_last_tweak = !encrypt && tail > 0;

		if (walk.nbytes < walk.total) {
			blocks = round_down(blocks,
					    walk.stride / AES_BLOCK_SIZE);
			reorder_last_tweak = 0;
		}

		kernel_neon_begin();
		fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk,
		   ctx->key.rounds, blocks, walk.iv, reorder_last_tweak);
		kernel_neon_end();
		err = skcipher_walk_done(&walk,
					 walk.nbytes - blocks * AES_BLOCK_SIZE);
	}

	if (err || likely(!tail))
		return err;

	/* handle ciphertext stealing */
	scatterwalk_map_and_copy(buf, req->dst, req->cryptlen - AES_BLOCK_SIZE,
				 AES_BLOCK_SIZE, 0);
	memcpy(buf + AES_BLOCK_SIZE, buf, tail);
	scatterwalk_map_and_copy(buf, req->src, req->cryptlen, tail, 0);

	crypto_xor(buf, req->iv, AES_BLOCK_SIZE);

	if (encrypt)
		crypto_cipher_encrypt_one(ctx->cts_tfm, buf, buf);
	else
		crypto_cipher_decrypt_one(ctx->cts_tfm, buf, buf);

	crypto_xor(buf, req->iv, AES_BLOCK_SIZE);

	scatterwalk_map_and_copy(buf, req->dst, req->cryptlen - AES_BLOCK_SIZE,
				 AES_BLOCK_SIZE + tail, 1);
	return 0;
}

static int xts_encrypt(struct skcipher_request *req)
{
	return __xts_crypt(req, true, aesbs_xts_encrypt);
}

static int xts_decrypt(struct skcipher_request *req)
{
	return __xts_crypt(req, false, aesbs_xts_decrypt);
}

static struct skcipher_alg aes_algs[] = { {
	.base.cra_name		= "__ecb(aes)",
	.base.cra_driver_name	= "__ecb-aes-neonbs",
	.base.cra_priority	= 250,
	.base.cra_blocksize	= AES_BLOCK_SIZE,
	.base.cra_ctxsize	= sizeof(struct aesbs_ctx),
	.base.cra_module	= THIS_MODULE,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.walksize		= 8 * AES_BLOCK_SIZE,
	.setkey			= aesbs_setkey,
	.encrypt		= ecb_encrypt,
	.decrypt		= ecb_decrypt,
}, {
	.base.cra_name		= "__cbc(aes)",
	.base.cra_driver_name	= "__cbc-aes-neonbs",
	.base.cra_priority	= 250,
	.base.cra_blocksize	= AES_BLOCK_SIZE,
	.base.cra_ctxsize	= sizeof(struct aesbs_cbc_ctx),
	.base.cra_module	= THIS_MODULE,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL |
				  CRYPTO_ALG_NEED_FALLBACK,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.walksize		= 8 * AES_BLOCK_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.setkey			= aesbs_cbc_setkey,
	.encrypt		= cbc_encrypt,
	.decrypt		= cbc_decrypt,
	.init			= cbc_init,
	.exit			= cbc_exit,
}, {
	.base.cra_name		= "__ctr(aes)",
	.base.cra_driver_name	= "__ctr-aes-neonbs",
	.base.cra_priority	= 250,
	.base.cra_blocksize	= 1,
	.base.cra_ctxsize	= sizeof(struct aesbs_ctx),
	.base.cra_module	= THIS_MODULE,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.chunksize		= AES_BLOCK_SIZE,
	.walksize		= 8 * AES_BLOCK_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.setkey			= aesbs_setkey,
	.encrypt		= ctr_encrypt,
	.decrypt		= ctr_encrypt,
}, {
	.base.cra_name		= "ctr(aes)",
	.base.cra_driver_name	= "ctr-aes-neonbs-sync",
	.base.cra_priority	= 250 - 1,
	.base.cra_blocksize	= 1,
	.base.cra_ctxsize	= sizeof(struct aesbs_ctr_ctx),
	.base.cra_module	= THIS_MODULE,

	.min_keysize		= AES_MIN_KEY_SIZE,
	.max_keysize		= AES_MAX_KEY_SIZE,
	.chunksize		= AES_BLOCK_SIZE,
	.walksize		= 8 * AES_BLOCK_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.setkey			= aesbs_ctr_setkey_sync,
	.encrypt		= ctr_encrypt_sync,
	.decrypt		= ctr_encrypt_sync,
}, {
	.base.cra_name		= "__xts(aes)",
	.base.cra_driver_name	= "__xts-aes-neonbs",
	.base.cra_priority	= 250,
	.base.cra_blocksize	= AES_BLOCK_SIZE,
	.base.cra_ctxsize	= sizeof(struct aesbs_xts_ctx),
	.base.cra_module	= THIS_MODULE,
	.base.cra_flags		= CRYPTO_ALG_INTERNAL,

	.min_keysize		= 2 * AES_MIN_KEY_SIZE,
	.max_keysize		= 2 * AES_MAX_KEY_SIZE,
	.walksize		= 8 * AES_BLOCK_SIZE,
	.ivsize			= AES_BLOCK_SIZE,
	.setkey			= aesbs_xts_setkey,
	.encrypt		= xts_encrypt,
	.decrypt		= xts_decrypt,
	.init			= xts_init,
	.exit			= xts_exit,
} };

static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];

static void aes_exit(void)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
		if (aes_simd_algs[i])
			simd_skcipher_free(aes_simd_algs[i]);

	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}

static int __init aes_init(void)
{
	struct simd_skcipher_alg *simd;
	const char *basename;
	const char *algname;
	const char *drvname;
	int err;
	int i;

	if (!(elf_hwcap & HWCAP_NEON))
		return -ENODEV;

	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
	if (err)
		return err;

	for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
		if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
			continue;

		algname = aes_algs[i].base.cra_name + 2;
		drvname = aes_algs[i].base.cra_driver_name + 2;
		basename = aes_algs[i].base.cra_driver_name;
		simd = simd_skcipher_create_compat(algname, drvname, basename);
		err = PTR_ERR(simd);
		if (IS_ERR(simd))
			goto unregister_simds;

		aes_simd_algs[i] = simd;
	}
	return 0;

unregister_simds:
	aes_exit();
	return err;
}

late_initcall(aes_init);
module_exit(aes_exit);


code:
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * BLAKE2b digest algorithm, NEON accelerated
 *
 * Copyright 2020 Google LLC
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>

	.text
	.fpu		neon

	// The arguments to blake2b_compress_neon()
	STATE		.req	r0
	BLOCK		.req	r1
	NBLOCKS		.req	r2
	INC		.req	r3

	// Pointers to the rotation tables
	ROR24_TABLE	.req	r4
	ROR16_TABLE	.req	r5

	// The original stack pointer
	ORIG_SP		.req	r6

	// NEON registers which contain the message words of the current block.
	// M_0-M_3 are occasionally used for other purposes too.
	M_0		.req	d16
	M_1		.req	d17
	M_2		.req	d18
	M_3		.req	d19
	M_4		.req	d20
	M_5		.req	d21
	M_6		.req	d22
	M_7		.req	d23
	M_8		.req	d24
	M_9		.req	d25
	M_10		.req	d26
	M_11		.req	d27
	M_12		.req	d28
	M_13		.req	d29
	M_14		.req	d30
	M_15		.req	d31

	.align		4
	// Tables for computing ror64(x, 24) and ror64(x, 16) using the vtbl.8
	// instruction.  This is the most efficient way to implement these
	// rotation amounts with NEON.  (On Cortex-A53 it's the same speed as
	// vshr.u64 + vsli.u64, while on Cortex-A7 it's faster.)
.Lror24_table:
	.byte		3, 4, 5, 6, 7, 0, 1, 2
.Lror16_table:
	.byte		2, 3, 4, 5, 6, 7, 0, 1
	// The BLAKE2b initialization vector
.Lblake2b_IV:
	.quad		0x6a09e667f3bcc908, 0xbb67ae8584caa73b
	.quad		0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1
	.quad		0x510e527fade682d1, 0x9b05688c2b3e6c1f
	.quad		0x1f83d9abfb41bd6b, 0x5be0cd19137e2179

// Execute one round of BLAKE2b by updating the state matrix v[0..15] in the
// NEON registers q0-q7.  The message block is in q8..q15 (M_0-M_15).  The stack
// pointer points to a 32-byte aligned buffer containing a copy of q8 and q9
// (M_0-M_3), so that they can be reloaded if they are used as temporary
// registers.  The macro arguments s0-s15 give the order in which the message
// words are used in this round.  'final' is 1 if this is the final round.
.macro	_blake2b_round	s0, s1, s2, s3, s4, s5, s6, s7, \
			s8, s9, s10, s11, s12, s13, s14, s15, final=0

	// Mix the columns:
	// (v[0], v[4], v[8], v[12]), (v[1], v[5], v[9], v[13]),
	// (v[2], v[6], v[10], v[14]), and (v[3], v[7], v[11], v[15]).

	// a += b + m[blake2b_sigma[r][2*i + 0]];
	vadd.u64	q0, q0, q2
	vadd.u64	q1, q1, q3
	vadd.u64	d0, d0, M_\s0
	vadd.u64	d1, d1, M_\s2
	vadd.u64	d2, d2, M_\s4
	vadd.u64	d3, d3, M_\s6

	// d = ror64(d ^ a, 32);
	veor		q6, q6, q0
	veor		q7, q7, q1
	vrev64.32	q6, q6
	vrev64.32	q7, q7

	// c += d;
	vadd.u64	q4, q4, q6
	vadd.u64	q5, q5, q7

	// b = ror64(b ^ c, 24);
	vld1.8		{M_0}, [ROR24_TABLE, :64]
	veor		q2, q2, q4
	veor		q3, q3, q5
	vtbl.8		d4, {d4}, M_0
	vtbl.8		d5, {d5}, M_0
	vtbl.8		d6, {d6}, M_0
	vtbl.8		d7, {d7}, M_0

	// a += b + m[blake2b_sigma[r][2*i + 1]];
	//
	// M_0 got clobbered above, so we have to reload it if any of the four
	// message words this step needs happens to be M_0.  Otherwise we don't
	// need to reload it here, as it will just get clobbered again below.
.if \s1 == 0 || \s3 == 0 || \s5 == 0 || \s7 == 0
	vld1.8		{M_0}, [sp, :64]
.endif
	vadd.u64	q0, q0, q2
	vadd.u64	q1, q1, q3
	vadd.u64	d0, d0, M_\s1
	vadd.u64	d1, d1, M_\s3
	vadd.u64	d2, d2, M_\s5
	vadd.u64	d3, d3, M_\s7

	// d = ror64(d ^ a, 16);
	vld1.8		{M_0}, [ROR16_TABLE, :64]
	veor		q6, q6, q0
	veor		q7, q7, q1
	vtbl.8		d12, {d12}, M_0
	vtbl.8		d13, {d13}, M_0
	vtbl.8		d14, {d14}, M_0
	vtbl.8		d15, {d15}, M_0

	// c += d;
	vadd.u64	q4, q4, q6
	vadd.u64	q5, q5, q7

	// b = ror64(b ^ c, 63);
	//
	// This rotation amount isn't a multiple of 8, so it has to be
	// implemented using a pair of shifts, which requires temporary
	// registers.  Use q8-q9 (M_0-M_3) for this, and reload them afterwards.
	veor		q8, q2, q4
	veor		q9, q3, q5
	vshr.u64	q2, q8, #63
	vshr.u64	q3, q9, #63
	vsli.u64	q2, q8, #1
	vsli.u64	q3, q9, #1
	vld1.8		{q8-q9}, [sp, :256]

	// Mix the diagonals:
	// (v[0], v[5], v[10], v[15]), (v[1], v[6], v[11], v[12]),
	// (v[2], v[7], v[8], v[13]), and (v[3], v[4], v[9], v[14]).
	//
	// There are two possible ways to do this: use 'vext' instructions to
	// shift the rows of the matrix so that the diagonals become columns,
	// and undo it afterwards; or just use 64-bit operations on 'd'
	// registers instead of 128-bit operations on 'q' registers.  We use the
	// latter approach, as it performs much better on Cortex-A7.

	// a += b + m[blake2b_sigma[r][2*i + 0]];
	vadd.u64	d0, d0, d5
	vadd.u64	d1, d1, d6
	vadd.u64	d2, d2, d7
	vadd.u64	d3, d3, d4
	vadd.u64	d0, d0, M_\s8
	vadd.u64	d1, d1, M_\s10
	vadd.u64	d2, d2, M_\s12
	vadd.u64	d3, d3, M_\s14

	// d = ror64(d ^ a, 32);
	veor		d15, d15, d0
	veor		d12, d12, d1
	veor		d13, d13, d2
	veor		d14, d14, d3
	vrev64.32	d15, d15
	vrev64.32	d12, d12
	vrev64.32	d13, d13
	vrev64.32	d14, d14

	// c += d;
	vadd.u64	d10, d10, d15
	vadd.u64	d11, d11, d12
	vadd.u64	d8, d8, d13
	vadd.u64	d9, d9, d14

	// b = ror64(b ^ c, 24);
	vld1.8		{M_0}, [ROR24_TABLE, :64]
	veor		d5, d5, d10
	veor		d6, d6, d11
	veor		d7, d7, d8
	veor		d4, d4, d9
	vtbl.8		d5, {d5}, M_0
	vtbl.8		d6, {d6}, M_0
	vtbl.8		d7, {d7}, M_0
	vtbl.8		d4, {d4}, M_0

	// a += b + m[blake2b_sigma[r][2*i + 1]];
.if \s9 == 0 || \s11 == 0 || \s13 == 0 || \s15 == 0
	vld1.8		{M_0}, [sp, :64]
.endif
	vadd.u64	d0, d0, d5
	vadd.u64	d1, d1, d6
	vadd.u64	d2, d2, d7
	vadd.u64	d3, d3, d4
	vadd.u64	d0, d0, M_\s9
	vadd.u64	d1, d1, M_\s11
	vadd.u64	d2, d2, M_\s13
	vadd.u64	d3, d3, M_\s15

	// d = ror64(d ^ a, 16);
	vld1.8		{M_0}, [ROR16_TABLE, :64]
	veor		d15, d15, d0
	veor		d12, d12, d1
	veor		d13, d13, d2
	veor		d14, d14, d3
	vtbl.8		d12, {d12}, M_0
	vtbl.8		d13, {d13}, M_0
	vtbl.8		d14, {d14}, M_0
	vtbl.8		d15, {d15}, M_0

	// c += d;
	vadd.u64	d10, d10, d15
	vadd.u64	d11, d11, d12
	vadd.u64	d8, d8, d13
	vadd.u64	d9, d9, d14

	// b = ror64(b ^ c, 63);
	veor		d16, d4, d9
	veor		d17, d5, d10
	veor		d18, d6, d11
	veor		d19, d7, d8
	vshr.u64	q2, q8, #63
	vshr.u64	q3, q9, #63
	vsli.u64	q2, q8, #1
	vsli.u64	q3, q9, #1
	// Reloading q8-q9 can be skipped on the final round.
.if ! \final
	vld1.8		{q8-q9}, [sp, :256]
.endif
.endm

//
// void blake2b_compress_neon(struct blake2b_state *state,
//			      const u8 *block, size_t nblocks, u32 inc);
//
// Only the first three fields of struct blake2b_state are used:
//	u64 h[8];	(inout)
//	u64 t[2];	(inout)
//	u64 f[2];	(in)
//
	.align		5
ENTRY(blake2b_compress_neon)
	push		{r4-r10}

	// Allocate a 32-byte stack buffer that is 32-byte aligned.
	mov		ORIG_SP, sp
	sub		ip, sp, #32
	bic		ip, ip, #31
	mov		sp, ip

	adr		ROR24_TABLE, .Lror24_table
	adr		ROR16_TABLE, .Lror16_table

	mov		ip, STATE
	vld1.64		{q0-q1}, [ip]!		// Load h[0..3]
	vld1.64		{q2-q3}, [ip]!		// Load h[4..7]
.Lnext_block:
	  adr		r10, .Lblake2b_IV
	vld1.64		{q14-q15}, [ip]		// Load t[0..1] and f[0..1]
	vld1.64		{q4-q5}, [r10]!		// Load IV[0..3]
	  vmov		r7, r8, d28		// Copy t[0] to (r7, r8)
	vld1.64		{q6-q7}, [r10]		// Load IV[4..7]
	  adds		r7, r7, INC		// Increment counter
	bcs		.Lslow_inc_ctr
	vmov.i32	d28[0], r7
	vst1.64		{d28}, [ip]		// Update t[0]
.Linc_ctr_done:

	// Load the next message block and finish initializing the state matrix
	// 'v'.  Fortunately, there are exactly enough NEON registers to fit the
	// entire state matrix in q0-q7 and the entire message block in q8-15.
	//
	// However, _blake2b_round also needs some extra registers for rotates,
	// so we have to spill some registers.  It's better to spill the message
	// registers than the state registers, as the message doesn't change.
	// Therefore we store a copy of the first 32 bytes of the message block
	// (q8-q9) in an aligned buffer on the stack so that they can be
	// reloaded when needed.  (We could just reload directly from the
	// message buffer, but it's faster to use aligned loads.)
	vld1.8		{q8-q9}, [BLOCK]!
	  veor		q6, q6, q14	// v[12..13] = IV[4..5] ^ t[0..1]
	vld1.8		{q10-q11}, [BLOCK]!
	  veor		q7, q7, q15	// v[14..15] = IV[6..7] ^ f[0..1]
	vld1.8		{q12-q13}, [BLOCK]!
	vst1.8		{q8-q9}, [sp, :256]
	  mov		ip, STATE
	vld1.8		{q14-q15}, [BLOCK]!

	// Execute the rounds.  Each round is provided the order in which it
	// needs to use the message words.
	_blake2b_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	_blake2b_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3
	_blake2b_round	11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4
	_blake2b_round	7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8
	_blake2b_round	9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13
	_blake2b_round	2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9
	_blake2b_round	12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11
	_blake2b_round	13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10
	_blake2b_round	6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5
	_blake2b_round	10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0
	_blake2b_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	_blake2b_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 \
			final=1

	// Fold the final state matrix into the hash chaining value:
	//
	//	for (i = 0; i < 8; i++)
	//		h[i] ^= v[i] ^ v[i + 8];
	//
	  vld1.64	{q8-q9}, [ip]!		// Load old h[0..3]
	veor		q0, q0, q4		// v[0..1] ^= v[8..9]
	veor		q1, q1, q5		// v[2..3] ^= v[10..11]
	  vld1.64	{q10-q11}, [ip]		// Load old h[4..7]
	veor		q2, q2, q6		// v[4..5] ^= v[12..13]
	veor		q3, q3, q7		// v[6..7] ^= v[14..15]
	veor		q0, q0, q8		// v[0..1] ^= h[0..1]
	veor		q1, q1, q9		// v[2..3] ^= h[2..3]
	  mov		ip, STATE
	  subs		NBLOCKS, NBLOCKS, #1	// nblocks--
	  vst1.64	{q0-q1}, [ip]!		// Store new h[0..3]
	veor		q2, q2, q10		// v[4..5] ^= h[4..5]
	veor		q3, q3, q11		// v[6..7] ^= h[6..7]
	  vst1.64	{q2-q3}, [ip]!		// Store new h[4..7]

	// Advance to the next block, if there is one.
	bne		.Lnext_block		// nblocks != 0?

	mov		sp, ORIG_SP
	pop		{r4-r10}
	mov		pc, lr

.Lslow_inc_ctr:
	// Handle the case where the counter overflowed its low 32 bits, by
	// carrying the overflow bit into the full 128-bit counter.
	vmov		r9, r10, d29
	adcs		r8, r8, #0
	adcs		r9, r9, #0
	adc		r10, r10, #0
	vmov		d28, r7, r8
	vmov		d29, r9, r10
	vst1.64		{q14}, [ip]		// Update t[0] and t[1]
	b		.Linc_ctr_done
ENDPROC(blake2b_compress_neon)


code:
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * BLAKE2b digest algorithm, NEON accelerated
 *
 * Copyright 2020 Google LLC
 */

#include <crypto/internal/blake2b.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>

#include <linux/module.h>
#include <linux/sizes.h>

#include <asm/neon.h>
#include <asm/simd.h>

asmlinkage void blake2b_compress_neon(struct blake2b_state *state,
				      const u8 *block, size_t nblocks, u32 inc);

static void blake2b_compress_arch(struct blake2b_state *state,
				  const u8 *block, size_t nblocks, u32 inc)
{
	if (!crypto_simd_usable()) {
		blake2b_compress_generic(state, block, nblocks, inc);
		return;
	}

	do {
		const size_t blocks = min_t(size_t, nblocks,
					    SZ_4K / BLAKE2B_BLOCK_SIZE);

		kernel_neon_begin();
		blake2b_compress_neon(state, block, blocks, inc);
		kernel_neon_end();

		nblocks -= blocks;
		block += blocks * BLAKE2B_BLOCK_SIZE;
	} while (nblocks);
}

static int crypto_blake2b_update_neon(struct shash_desc *desc,
				      const u8 *in, unsigned int inlen)
{
	return crypto_blake2b_update(desc, in, inlen, blake2b_compress_arch);
}

static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out)
{
	return crypto_blake2b_final(desc, out, blake2b_compress_arch);
}

#define BLAKE2B_ALG(name, driver_name, digest_size)			\
	{								\
		.base.cra_name		= name,				\
		.base.cra_driver_name	= driver_name,			\
		.base.cra_priority	= 200,				\
		.base.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,	\
		.base.cra_blocksize	= BLAKE2B_BLOCK_SIZE,		\
		.base.cra_ctxsize	= sizeof(struct blake2b_tfm_ctx), \
		.base.cra_module	= THIS_MODULE,			\
		.digestsize		= digest_size,			\
		.setkey			= crypto_blake2b_setkey,	\
		.init			= crypto_blake2b_init,		\
		.update			= crypto_blake2b_update_neon,	\
		.final			= crypto_blake2b_final_neon,	\
		.descsize		= sizeof(struct blake2b_state),	\
	}

static struct shash_alg blake2b_neon_algs[] = {
	BLAKE2B_ALG("blake2b-160", "blake2b-160-neon", BLAKE2B_160_HASH_SIZE),
	BLAKE2B_ALG("blake2b-256", "blake2b-256-neon", BLAKE2B_256_HASH_SIZE),
	BLAKE2B_ALG("blake2b-384", "blake2b-384-neon", BLAKE2B_384_HASH_SIZE),
	BLAKE2B_ALG("blake2b-512", "blake2b-512-neon", BLAKE2B_512_HASH_SIZE),
};

static int __init blake2b_neon_mod_init(void)
{
	if (!(elf_hwcap & HWCAP_NEON))
		return -ENODEV;

	return crypto_register_shashes(blake2b_neon_algs,
				       ARRAY_SIZE(blake2b_neon_algs));
}

static void __exit blake2b_neon_mod_exit(void)
{
	crypto_unregister_shashes(blake2b_neon_algs,
				  ARRAY_SIZE(blake2b_neon_algs));
}

module_init(blake2b_neon_mod_init);
module_exit(blake2b_neon_mod_exit);

MODULE_DESCRIPTION("BLAKE2b digest algorithm, NEON accelerated");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
MODULE_ALIAS_CRYPTO("blake2b-160");
MODULE_ALIAS_CRYPTO("blake2b-160-neon");
MODULE_ALIAS_CRYPTO("blake2b-256");
MODULE_ALIAS_CRYPTO("blake2b-256-neon");
MODULE_ALIAS_CRYPTO("blake2b-384");
MODULE_ALIAS_CRYPTO("blake2b-384-neon");
MODULE_ALIAS_CRYPTO("blake2b-512");
MODULE_ALIAS_CRYPTO("blake2b-512-neon");


code:
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * BLAKE2s digest algorithm, ARM scalar implementation
 *
 * Copyright 2020 Google LLC
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

	// Registers used to hold message words temporarily.  There aren't
	// enough ARM registers to hold the whole message block, so we have to
	// load the words on-demand.
	M_0		.req	r12
	M_1		.req	r14

// The BLAKE2s initialization vector
.Lblake2s_IV:
	.word	0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A
	.word	0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19

.macro __ldrd		a, b, src, offset
#if __LINUX_ARM_ARCH__ >= 6
	ldrd		\a, \b, [\src, #\offset]
#else
	ldr		\a, [\src, #\offset]
	ldr		\b, [\src, #\offset + 4]
#endif
.endm

.macro __strd		a, b, dst, offset
#if __LINUX_ARM_ARCH__ >= 6
	strd		\a, \b, [\dst, #\offset]
#else
	str		\a, [\dst, #\offset]
	str		\b, [\dst, #\offset + 4]
#endif
.endm

.macro _le32_bswap	a, tmp
#ifdef __ARMEB__
	rev_l		\a, \tmp
#endif
.endm

.macro _le32_bswap_8x	a, b, c, d, e, f, g, h,  tmp
	_le32_bswap	\a, \tmp
	_le32_bswap	\b, \tmp
	_le32_bswap	\c, \tmp
	_le32_bswap	\d, \tmp
	_le32_bswap	\e, \tmp
	_le32_bswap	\f, \tmp
	_le32_bswap	\g, \tmp
	_le32_bswap	\h, \tmp
.endm

// Execute a quarter-round of BLAKE2s by mixing two columns or two diagonals.
// (a0, b0, c0, d0) and (a1, b1, c1, d1) give the registers containing the two
// columns/diagonals.  s0-s1 are the word offsets to the message words the first
// column/diagonal needs, and likewise s2-s3 for the second column/diagonal.
// M_0 and M_1 are free to use, and the message block can be found at sp + 32.
//
// Note that to save instructions, the rotations don't happen when the
// pseudocode says they should, but rather they are delayed until the values are
// used.  See the comment above _blake2s_round().
.macro _blake2s_quarterround  a0, b0, c0, d0,  a1, b1, c1, d1,  s0, s1, s2, s3

	ldr		M_0, [sp, #32 + 4 * \s0]
	ldr		M_1, [sp, #32 + 4 * \s2]

	// a += b + m[blake2s_sigma[r][2*i + 0]];
	add		\a0, \a0, \b0, ror #brot
	add		\a1, \a1, \b1, ror #brot
	add		\a0, \a0, M_0
	add		\a1, \a1, M_1

	// d = ror32(d ^ a, 16);
	eor		\d0, \a0, \d0, ror #drot
	eor		\d1, \a1, \d1, ror #drot

	// c += d;
	add		\c0, \c0, \d0, ror #16
	add		\c1, \c1, \d1, ror #16

	// b = ror32(b ^ c, 12);
	eor		\b0, \c0, \b0, ror #brot
	eor		\b1, \c1, \b1, ror #brot

	ldr		M_0, [sp, #32 + 4 * \s1]
	ldr		M_1, [sp, #32 + 4 * \s3]

	// a += b + m[blake2s_sigma[r][2*i + 1]];
	add		\a0, \a0, \b0, ror #12
	add		\a1, \a1, \b1, ror #12
	add		\a0, \a0, M_0
	add		\a1, \a1, M_1

	// d = ror32(d ^ a, 8);
	eor		\d0, \a0, \d0, ror#16
	eor		\d1, \a1, \d1, ror#16

	// c += d;
	add		\c0, \c0, \d0, ror#8
	add		\c1, \c1, \d1, ror#8

	// b = ror32(b ^ c, 7);
	eor		\b0, \c0, \b0, ror#12
	eor		\b1, \c1, \b1, ror#12
.endm

// Execute one round of BLAKE2s by updating the state matrix v[0..15].  v[0..9]
// are in r0..r9.  The stack pointer points to 8 bytes of scratch space for
// spilling v[8..9], then to v[9..15], then to the message block.  r10-r12 and
// r14 are free to use.  The macro arguments s0-s15 give the order in which the
// message words are used in this round.
//
// All rotates are performed using the implicit rotate operand accepted by the
// 'add' and 'eor' instructions.  This is faster than using explicit rotate
// instructions.  To make this work, we allow the values in the second and last
// rows of the BLAKE2s state matrix (rows 'b' and 'd') to temporarily have the
// wrong rotation amount.  The rotation amount is then fixed up just in time
// when the values are used.  'brot' is the number of bits the values in row 'b'
// need to be rotated right to arrive at the correct values, and 'drot'
// similarly for row 'd'.  (brot, drot) start out as (0, 0) but we make it such
// that they end up as (7, 8) after every round.
.macro	_blake2s_round	s0, s1, s2, s3, s4, s5, s6, s7, \
			s8, s9, s10, s11, s12, s13, s14, s15

	// Mix first two columns:
	// (v[0], v[4], v[8], v[12]) and (v[1], v[5], v[9], v[13]).
	__ldrd		r10, r11, sp, 16	// load v[12] and v[13]
	_blake2s_quarterround	r0, r4, r8, r10,  r1, r5, r9, r11, \
				\s0, \s1, \s2, \s3
	__strd		r8, r9, sp, 0
	__strd		r10, r11, sp, 16

	// Mix second two columns:
	// (v[2], v[6], v[10], v[14]) and (v[3], v[7], v[11], v[15]).
	__ldrd		r8, r9, sp, 8		// load v[10] and v[11]
	__ldrd		r10, r11, sp, 24	// load v[14] and v[15]
	_blake2s_quarterround	r2, r6, r8, r10,  r3, r7, r9, r11, \
				\s4, \s5, \s6, \s7
	str		r10, [sp, #24]		// store v[14]
	// v[10], v[11], and v[15] are used below, so no need to store them yet.

	.set brot, 7
	.set drot, 8

	// Mix first two diagonals:
	// (v[0], v[5], v[10], v[15]) and (v[1], v[6], v[11], v[12]).
	ldr		r10, [sp, #16]		// load v[12]
	_blake2s_quarterround	r0, r5, r8, r11,  r1, r6, r9, r10, \
				\s8, \s9, \s10, \s11
	__strd		r8, r9, sp, 8
	str		r11, [sp, #28]
	str		r10, [sp, #16]

	// Mix second two diagonals:
	// (v[2], v[7], v[8], v[13]) and (v[3], v[4], v[9], v[14]).
	__ldrd		r8, r9, sp, 0		// load v[8] and v[9]
	__ldrd		r10, r11, sp, 20	// load v[13] and v[14]
	_blake2s_quarterround	r2, r7, r8, r10,  r3, r4, r9, r11, \
				\s12, \s13, \s14, \s15
	__strd		r10, r11, sp, 20
.endm

//
// void blake2s_compress(struct blake2s_state *state,
//			 const u8 *block, size_t nblocks, u32 inc);
//
// Only the first three fields of struct blake2s_state are used:
//	u32 h[8];	(inout)
//	u32 t[2];	(inout)
//	u32 f[2];	(in)
//
	.align		5
ENTRY(blake2s_compress)
	push		{r0-r2,r4-r11,lr}	// keep this an even number

.Lnext_block:
	// r0 is 'state'
	// r1 is 'block'
	// r3 is 'inc'

	// Load and increment the counter t[0..1].
	__ldrd		r10, r11, r0, 32
	adds		r10, r10, r3
	adc		r11, r11, #0
	__strd		r10, r11, r0, 32

	// _blake2s_round is very short on registers, so copy the message block
	// to the stack to save a register during the rounds.  This also has the
	// advantage that misalignment only needs to be dealt with in one place.
	sub		sp, sp, #64
	mov		r12, sp
	tst		r1, #3
	bne		.Lcopy_block_misaligned
	ldmia		r1!, {r2-r9}
	_le32_bswap_8x	r2, r3, r4, r5, r6, r7, r8, r9,  r14
	stmia		r12!, {r2-r9}
	ldmia		r1!, {r2-r9}
	_le32_bswap_8x	r2, r3, r4, r5, r6, r7, r8, r9,  r14
	stmia		r12, {r2-r9}
.Lcopy_block_done:
	str		r1, [sp, #68]		// Update message pointer

	// Calculate v[8..15].  Push v[9..15] onto the stack, and leave space
	// for spilling v[8..9].  Leave v[8..9] in r8-r9.
	mov		r14, r0			// r14 = state
	adr		r12, .Lblake2s_IV
	ldmia		r12!, {r8-r9}		// load IV[0..1]
	__ldrd		r0, r1, r14, 40		// load f[0..1]
	ldm		r12, {r2-r7}		// load IV[3..7]
	eor		r4, r4, r10		// v[12] = IV[4] ^ t[0]
	eor		r5, r5, r11		// v[13] = IV[5] ^ t[1]
	eor		r6, r6, r0		// v[14] = IV[6] ^ f[0]
	eor		r7, r7, r1		// v[15] = IV[7] ^ f[1]
	push		{r2-r7}			// push v[9..15]
	sub		sp, sp, #8		// leave space for v[8..9]

	// Load h[0..7] == v[0..7].
	ldm		r14, {r0-r7}

	// Execute the rounds.  Each round is provided the order in which it
	// needs to use the message words.
	.set brot, 0
	.set drot, 0
	_blake2s_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	_blake2s_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3
	_blake2s_round	11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4
	_blake2s_round	7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8
	_blake2s_round	9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13
	_blake2s_round	2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9
	_blake2s_round	12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11
	_blake2s_round	13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10
	_blake2s_round	6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5
	_blake2s_round	10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0

	// Fold the final state matrix into the hash chaining value:
	//
	//	for (i = 0; i < 8; i++)
	//		h[i] ^= v[i] ^ v[i + 8];
	//
	ldr		r14, [sp, #96]		// r14 = &h[0]
	add		sp, sp, #8		// v[8..9] are already loaded.
	pop		{r10-r11}		// load v[10..11]
	eor		r0, r0, r8
	eor		r1, r1, r9
	eor		r2, r2, r10
	eor		r3, r3, r11
	ldm		r14, {r8-r11}		// load h[0..3]
	eor		r0, r0, r8
	eor		r1, r1, r9
	eor		r2, r2, r10
	eor		r3, r3, r11
	stmia		r14!, {r0-r3}		// store new h[0..3]
	ldm		r14, {r0-r3}		// load old h[4..7]
	pop		{r8-r11}		// load v[12..15]
	eor		r0, r0, r4, ror #brot
	eor		r1, r1, r5, ror #brot
	eor		r2, r2, r6, ror #brot
	eor		r3, r3, r7, ror #brot
	eor		r0, r0, r8, ror #drot
	eor		r1, r1, r9, ror #drot
	eor		r2, r2, r10, ror #drot
	eor		r3, r3, r11, ror #drot
	  add		sp, sp, #64		// skip copy of message block
	stm		r14, {r0-r3}		// store new h[4..7]

	// Advance to the next block, if there is one.  Note that if there are
	// multiple blocks, then 'inc' (the counter increment amount) must be
	// 64.  So we can simply set it to 64 without re-loading it.
	ldm		sp, {r0, r1, r2}	// load (state, block, nblocks)
	mov		r3, #64			// set 'inc'
	subs		r2, r2, #1		// nblocks--
	str		r2, [sp, #8]
	bne		.Lnext_block		// nblocks != 0?

	pop		{r0-r2,r4-r11,pc}

	// The next message block (pointed to by r1) isn't 4-byte aligned, so it
	// can't be loaded using ldmia.  Copy it to the stack buffer (pointed to
	// by r12) using an alternative method.  r2-r9 are free to use.
.Lcopy_block_misaligned:
	mov		r2, #64
1:
#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
	ldr		r3, [r1], #4
	_le32_bswap	r3, r4
#else
	ldrb		r3, [r1, #0]
	ldrb		r4, [r1, #1]
	ldrb		r5, [r1, #2]
	ldrb		r6, [r1, #3]
	add		r1, r1, #4
	orr		r3, r3, r4, lsl #8
	orr		r3, r3, r5, lsl #16
	orr		r3, r3, r6, lsl #24
#endif
	subs		r2, r2, #4
	str		r3, [r12], #4
	bne		1b
	b		.Lcopy_block_done
ENDPROC(blake2s_compress)


code:
// SPDX-License-Identifier: GPL-2.0-or-later

#include <crypto/internal/blake2s.h>
#include <linux/module.h>

/* defined in blake2s-core.S */
EXPORT_SYMBOL(blake2s_compress);


code:
// SPDX-License-Identifier: GPL-2.0
/*
 * ARM NEON accelerated ChaCha and XChaCha stream ciphers,
 * including ChaCha20 (RFC7539)
 *
 * Copyright (C) 2016-2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
 * Copyright (C) 2015 Martin Willi
 */

#include <crypto/algapi.h>
#include <crypto/internal/chacha.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>

#include <asm/cputype.h>
#include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h>

asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
				      int nrounds);
asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
				       int nrounds, unsigned int nbytes);
asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds);
asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);

asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
			     const u32 *state, int nrounds);

static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);

static inline bool neon_usable(void)
{
	return static_branch_likely(&use_neon) && crypto_simd_usable();
}

static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
			  unsigned int bytes, int nrounds)
{
	u8 buf[CHACHA_BLOCK_SIZE];

	while (bytes > CHACHA_BLOCK_SIZE) {
		unsigned int l = min(bytes, CHACHA_BLOCK_SIZE * 4U);

		chacha_4block_xor_neon(state, dst, src, nrounds, l);
		bytes -= l;
		src += l;
		dst += l;
		state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);
	}
	if (bytes) {
		const u8 *s = src;
		u8 *d = dst;

		if (bytes != CHACHA_BLOCK_SIZE)
			s = d = memcpy(buf, src, bytes);
		chacha_block_xor_neon(state, d, s, nrounds);
		if (d != dst)
			memcpy(dst, buf, bytes);
		state[12]++;
	}
}

void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
{
	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
		hchacha_block_arm(state, stream, nrounds);
	} else {
		kernel_neon_begin();
		hchacha_block_neon(state, stream, nrounds);
		kernel_neon_end();
	}
}
EXPORT_SYMBOL(hchacha_block_arch);

void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
{
	chacha_init_generic(state, key, iv);
}
EXPORT_SYMBOL(chacha_init_arch);

void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
		       int nrounds)
{
	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||
	    bytes <= CHACHA_BLOCK_SIZE) {
		chacha_doarm(dst, src, bytes, state, nrounds);
		state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
		return;
	}

	do {
		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);

		kernel_neon_begin();
		chacha_doneon(state, dst, src, todo, nrounds);
		kernel_neon_end();

		bytes -= todo;
		src += todo;
		dst += todo;
	} while (bytes);
}
EXPORT_SYMBOL(chacha_crypt_arch);

static int chacha_stream_xor(struct skcipher_request *req,
			     const struct chacha_ctx *ctx, const u8 *iv,
			     bool neon)
{
	struct skcipher_walk walk;
	u32 state[16];
	int err;

	err = skcipher_walk_virt(&walk, req, false);

	chacha_init_generic(state, ctx->key, iv);

	while (walk.nbytes > 0) {
		unsigned int nbytes = walk.nbytes;

		if (nbytes < walk.total)
			nbytes = round_down(nbytes, walk.stride);

		if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
			chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr,
				     nbytes, state, ctx->nrounds);
			state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE);
		} else {
			kernel_neon_begin();
			chacha_doneon(state, walk.dst.virt.addr,
				      walk.src.virt.addr, nbytes, ctx->nrounds);
			kernel_neon_end();
		}
		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
	}

	return err;
}

static int do_chacha(struct skcipher_request *req, bool neon)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);

	return chacha_stream_xor(req, ctx, req->iv, neon);
}

static int chacha_arm(struct skcipher_request *req)
{
	return do_chacha(req, false);
}

static int chacha_neon(struct skcipher_request *req)
{
	return do_chacha(req, neon_usable());
}

static int do_xchacha(struct skcipher_request *req, bool neon)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct chacha_ctx subctx;
	u32 state[16];
	u8 real_iv[16];

	chacha_init_generic(state, ctx->key, req->iv);

	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
		hchacha_block_arm(state, subctx.key, ctx->nrounds);
	} else {
		kernel_neon_begin();
		hchacha_block_neon(state, subctx.key, ctx->nrounds);
		kernel_neon_end();
	}
	subctx.nrounds = ctx->nrounds;

	memcpy(&real_iv[0], req->iv + 24, 8);
	memcpy(&real_iv[8], req->iv + 16, 8);
	return chacha_stream_xor(req, &subctx, real_iv, neon);
}

static int xchacha_arm(struct skcipher_request *req)
{
	return do_xchacha(req, false);
}

static int xchacha_neon(struct skcipher_request *req)
{
	return do_xchacha(req, neon_usable());
}

static struct skcipher_alg arm_algs[] = {
	{
		.base.cra_name		= "chacha20",
		.base.cra_driver_name	= "chacha20-arm",
		.base.cra_priority	= 200,
		.base.cra_blocksize	= 1,
		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
		.base.cra_module	= THIS_MODULE,

		.min_keysize		= CHACHA_KEY_SIZE,
		.max_keysize		= CHACHA_KEY_SIZE,
		.ivsize			= CHACHA_IV_SIZE,
		.chunksize		= CHACHA_BLOCK_SIZE,
		.setkey			= chacha20_setkey,
		.encrypt		= chacha_arm,
		.decrypt		= chacha_arm,
	}, {
		.base.cra_name		= "xchacha20",
		.base.cra_driver_name	= "xchacha20-arm",
		.base.cra_priority	= 200,
		.base.cra_blocksize	= 1,
		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
		.base.cra_module	= THIS_MODULE,

		.min_keysize		= CHACHA_KEY_SIZE,
		.max_keysize		= CHACHA_KEY_SIZE,
		.ivsize			= XCHACHA_IV_SIZE,
		.chunksize		= CHACHA_BLOCK_SIZE,
		.setkey			= chacha20_setkey,
		.encrypt		= xchacha_arm,
		.decrypt		= xchacha_arm,
	}, {
		.base.cra_name		= "xchacha12",
		.base.cra_driver_name	= "xchacha12-arm",
		.base.cra_priority	= 200,
		.base.cra_blocksize	= 1,
		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
		.base.cra_module	= THIS_MODULE,

		.min_keysize		= CHACHA_KEY_SIZE,
		.max_keysize		= CHACHA_KEY_SIZE,
		.ivsize			= XCHACHA_IV_SIZE,
		.chunksize		= CHACHA_BLOCK_SIZE,
		.setkey			= chacha12_setkey,
		.encrypt		= xchacha_arm,
		.decrypt		= xchacha_arm,
	},
};

static struct skcipher_alg neon_algs[] = {
	{
		.base.cra_name		= "chacha20",
		.base.cra_driver_name	= "chacha20-neon",
		.base.cra_priority	= 300,
		.base.cra_blocksize	= 1,
		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
		.base.cra_module	= THIS_MODULE,

		.min_keysize		= CHACHA_KEY_SIZE,
		.max_keysize		= CHACHA_KEY_SIZE,
		.ivsize			= CHACHA_IV_SIZE,
		.chunksize		= CHACHA_BLOCK_SIZE,
		.walksize		= 4 * CHACHA_BLOCK_SIZE,
		.setkey			= chacha20_setkey,
		.encrypt		= chacha_neon,
		.decrypt		= chacha_neon,
	}, {
		.base.cra_name		= "xchacha20",
		.base.cra_driver_name	= "xchacha20-neon",
		.base.cra_priority	= 300,
		.base.cra_blocksize	= 1,
		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
		.base.cra_module	= THIS_MODULE,

		.min_keysize		= CHACHA_KEY_SIZE,
		.max_keysize		= CHACHA_KEY_SIZE,
		.ivsize			= XCHACHA_IV_SIZE,
		.chunksize		= CHACHA_BLOCK_SIZE,
		.walksize		= 4 * CHACHA_BLOCK_SIZE,
		.setkey			= chacha20_setkey,
		.encrypt		= xchacha_neon,
		.decrypt		= xchacha_neon,
	}, {
		.base.cra_name		= "xchacha12",
		.base.cra_driver_name	= "xchacha12-neon",
		.base.cra_priority	= 300,
		.base.cra_blocksize	= 1,
		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
		.base.cra_module	= THIS_MODULE,

		.min_keysize		= CHACHA_KEY_SIZE,
		.max_keysize		= CHACHA_KEY_SIZE,
		.ivsize			= XCHACHA_IV_SIZE,
		.chunksize		= CHACHA_BLOCK_SIZE,
		.walksize		= 4 * CHACHA_BLOCK_SIZE,
		.setkey			= chacha12_setkey,
		.encrypt		= xchacha_neon,
		.decrypt		= xchacha_neon,
	}
};

static int __init chacha_simd_mod_init(void)
{
	int err = 0;

	if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
		err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
		if (err)
			return err;
	}

	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
		int i;

		switch (read_cpuid_part()) {
		case ARM_CPU_PART_CORTEX_A7:
		case ARM_CPU_PART_CORTEX_A5:
			/*
			 * The Cortex-A7 and Cortex-A5 do not perform well with
			 * the NEON implementation but do incredibly with the
			 * scalar one and use less power.
			 */
			for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
				neon_algs[i].base.cra_priority = 0;
			break;
		default:
			static_branch_enable(&use_neon);
		}

		if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
			err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
			if (err)
				crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
		}
	}
	return err;
}

static void __exit chacha_simd_mod_fini(void)
{
	if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
		crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
		if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON))
			crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
	}
}

module_init(chacha_simd_mod_init);
module_exit(chacha_simd_mod_fini);

MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_CRYPTO("chacha20");
MODULE_ALIAS_CRYPTO("chacha20-arm");
MODULE_ALIAS_CRYPTO("xchacha20");
MODULE_ALIAS_CRYPTO("xchacha20-arm");
MODULE_ALIAS_CRYPTO("xchacha12");
MODULE_ALIAS_CRYPTO("xchacha12-arm");
#ifdef CONFIG_KERNEL_MODE_NEON
MODULE_ALIAS_CRYPTO("chacha20-neon");
MODULE_ALIAS_CRYPTO("xchacha20-neon");
MODULE_ALIAS_CRYPTO("xchacha12-neon");
#endif