code:
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 *
 * Based on public domain code from Daniel J. Bernstein and Peter Schwabe. This
 * began from SUPERCOP's curve25519/neon2/scalarmult.s, but has subsequently been
 * manually reworked for use in kernel space.
 */

#include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/internal/kpp.h>
#include <crypto/internal/simd.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/jump_label.h>
#include <linux/scatterlist.h>
#include <crypto/curve25519.h>

asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE],
				const u8 secret[CURVE25519_KEY_SIZE],
				const u8 basepoint[CURVE25519_KEY_SIZE]);

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);

void curve25519_arch(u8 out[CURVE25519_KEY_SIZE],
		     const u8 scalar[CURVE25519_KEY_SIZE],
		     const u8 point[CURVE25519_KEY_SIZE])
{
	if (static_branch_likely(&have_neon) && crypto_simd_usable()) {
		kernel_neon_begin();
		curve25519_neon(out, scalar, point);
		kernel_neon_end();
	} else {
		curve25519_generic(out, scalar, point);
	}
}
EXPORT_SYMBOL(curve25519_arch);

void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
			  const u8 secret[CURVE25519_KEY_SIZE])
{
	return curve25519_arch(pub, secret, curve25519_base_point);
}
EXPORT_SYMBOL(curve25519_base_arch);

static int curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
				 unsigned int len)
{
	u8 *secret = kpp_tfm_ctx(tfm);

	if (!len)
		curve25519_generate_secret(secret);
	else if (len == CURVE25519_KEY_SIZE &&
		 crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE))
		memcpy(secret, buf, CURVE25519_KEY_SIZE);
	else
		return -EINVAL;
	return 0;
}

static int curve25519_compute_value(struct kpp_request *req)
{
	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
	const u8 *secret = kpp_tfm_ctx(tfm);
	u8 public_key[CURVE25519_KEY_SIZE];
	u8 buf[CURVE25519_KEY_SIZE];
	int copied, nbytes;
	u8 const *bp;

	if (req->src) {
		copied = sg_copy_to_buffer(req->src,
					   sg_nents_for_len(req->src,
							    CURVE25519_KEY_SIZE),
					   public_key, CURVE25519_KEY_SIZE);
		if (copied != CURVE25519_KEY_SIZE)
			return -EINVAL;
		bp = public_key;
	} else {
		bp = curve25519_base_point;
	}

	curve25519_arch(buf, secret, bp);

	/* might want less than we've got */
	nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len);
	copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst,
								nbytes),
				     buf, nbytes);
	if (copied != nbytes)
		return -EINVAL;
	return 0;
}

static unsigned int curve25519_max_size(struct crypto_kpp *tfm)
{
	return CURVE25519_KEY_SIZE;
}

static struct kpp_alg curve25519_alg = {
	.base.cra_name		= "curve25519",
	.base.cra_driver_name	= "curve25519-neon",
	.base.cra_priority	= 200,
	.base.cra_module	= THIS_MODULE,
	.base.cra_ctxsize	= CURVE25519_KEY_SIZE,

	.set_secret		= curve25519_set_secret,
	.generate_public_key	= curve25519_compute_value,
	.compute_shared_secret	= curve25519_compute_value,
	.max_size		= curve25519_max_size,
};

static int __init arm_curve25519_init(void)
{
	if (elf_hwcap & HWCAP_NEON) {
		static_branch_enable(&have_neon);
		return IS_REACHABLE(CONFIG_CRYPTO_KPP) ?
			crypto_register_kpp(&curve25519_alg) : 0;
	}
	return 0;
}

static void __exit arm_curve25519_exit(void)
{
	if (IS_REACHABLE(CONFIG_CRYPTO_KPP) && elf_hwcap & HWCAP_NEON)
		crypto_unregister_kpp(&curve25519_alg);
}

module_init(arm_curve25519_init);
module_exit(arm_curve25519_exit);

MODULE_ALIAS_CRYPTO("curve25519");
MODULE_ALIAS_CRYPTO("curve25519-neon");
MODULE_LICENSE("GPL v2");


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions.
 *
 * Copyright (C) 2015 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

	.arch		armv8-a
	.fpu		crypto-neon-fp-armv8

	SHASH		.req	q0
	T1		.req	q1
	XL		.req	q2
	XM		.req	q3
	XH		.req	q4
	IN1		.req	q4

	SHASH_L		.req	d0
	SHASH_H		.req	d1
	T1_L		.req	d2
	T1_H		.req	d3
	XL_L		.req	d4
	XL_H		.req	d5
	XM_L		.req	d6
	XM_H		.req	d7
	XH_L		.req	d8

	t0l		.req	d10
	t0h		.req	d11
	t1l		.req	d12
	t1h		.req	d13
	t2l		.req	d14
	t2h		.req	d15
	t3l		.req	d16
	t3h		.req	d17
	t4l		.req	d18
	t4h		.req	d19

	t0q		.req	q5
	t1q		.req	q6
	t2q		.req	q7
	t3q		.req	q8
	t4q		.req	q9
	T2		.req	q9

	s1l		.req	d20
	s1h		.req	d21
	s2l		.req	d22
	s2h		.req	d23
	s3l		.req	d24
	s3h		.req	d25
	s4l		.req	d26
	s4h		.req	d27

	MASK		.req	d28
	SHASH2_p8	.req	d28

	k16		.req	d29
	k32		.req	d30
	k48		.req	d31
	SHASH2_p64	.req	d31

	HH		.req	q10
	HH3		.req	q11
	HH4		.req	q12
	HH34		.req	q13

	HH_L		.req	d20
	HH_H		.req	d21
	HH3_L		.req	d22
	HH3_H		.req	d23
	HH4_L		.req	d24
	HH4_H		.req	d25
	HH34_L		.req	d26
	HH34_H		.req	d27
	SHASH2_H	.req	d29

	XL2		.req	q5
	XM2		.req	q6
	XH2		.req	q7
	T3		.req	q8

	XL2_L		.req	d10
	XL2_H		.req	d11
	XM2_L		.req	d12
	XM2_H		.req	d13
	T3_L		.req	d16
	T3_H		.req	d17

	.text

	.macro		__pmull_p64, rd, rn, rm, b1, b2, b3, b4
	vmull.p64	\rd, \rn, \rm
	.endm

	/*
	 * This implementation of 64x64 -> 128 bit polynomial multiplication
	 * using vmull.p8 instructions (8x8 -> 16) is taken from the paper
	 * "Fast Software Polynomial Multiplication on ARM Processors Using
	 * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and
	 * Ricardo Dahab (https://hal.inria.fr/hal-01506572)
	 *
	 * It has been slightly tweaked for in-order performance, and to allow
	 * 'rq' to overlap with 'ad' or 'bd'.
	 */
	.macro		__pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l
	vext.8		t0l, \ad, \ad, #1	@ A1
	.ifc		\b1, t4l
	vext.8		t4l, \bd, \bd, #1	@ B1
	.endif
	vmull.p8	t0q, t0l, \bd		@ F = A1*B
	vext.8		t1l, \ad, \ad, #2	@ A2
	vmull.p8	t4q, \ad, \b1		@ E = A*B1
	.ifc		\b2, t3l
	vext.8		t3l, \bd, \bd, #2	@ B2
	.endif
	vmull.p8	t1q, t1l, \bd		@ H = A2*B
	vext.8		t2l, \ad, \ad, #3	@ A3
	vmull.p8	t3q, \ad, \b2		@ G = A*B2
	veor		t0q, t0q, t4q		@ L = E + F
	.ifc		\b3, t4l
	vext.8		t4l, \bd, \bd, #3	@ B3
	.endif
	vmull.p8	t2q, t2l, \bd		@ J = A3*B
	veor		t0l, t0l, t0h		@ t0 = (L) (P0 + P1) << 8
	veor		t1q, t1q, t3q		@ M = G + H
	.ifc		\b4, t3l
	vext.8		t3l, \bd, \bd, #4	@ B4
	.endif
	vmull.p8	t4q, \ad, \b3		@ I = A*B3
	veor		t1l, t1l, t1h		@ t1 = (M) (P2 + P3) << 16
	vmull.p8	t3q, \ad, \b4		@ K = A*B4
	vand		t0h, t0h, k48
	vand		t1h, t1h, k32
	veor		t2q, t2q, t4q		@ N = I + J
	veor		t0l, t0l, t0h
	veor		t1l, t1l, t1h
	veor		t2l, t2l, t2h		@ t2 = (N) (P4 + P5) << 24
	vand		t2h, t2h, k16
	veor		t3l, t3l, t3h		@ t3 = (K) (P6 + P7) << 32
	vmov.i64	t3h, #0
	vext.8		t0q, t0q, t0q, #15
	veor		t2l, t2l, t2h
	vext.8		t1q, t1q, t1q, #14
	vmull.p8	\rq, \ad, \bd		@ D = A*B
	vext.8		t2q, t2q, t2q, #13
	vext.8		t3q, t3q, t3q, #12
	veor		t0q, t0q, t1q
	veor		t2q, t2q, t3q
	veor		\rq, \rq, t0q
	veor		\rq, \rq, t2q
	.endm

	//
	// PMULL (64x64->128) based reduction for CPUs that can do
	// it in a single instruction.
	//
	.macro		__pmull_reduce_p64
	vmull.p64	T1, XL_L, MASK

	veor		XH_L, XH_L, XM_H
	vext.8		T1, T1, T1, #8
	veor		XL_H, XL_H, XM_L
	veor		T1, T1, XL

	vmull.p64	XL, T1_H, MASK
	.endm

	//
	// Alternative reduction for CPUs that lack support for the
	// 64x64->128 PMULL instruction
	//
	.macro		__pmull_reduce_p8
	veor		XL_H, XL_H, XM_L
	veor		XH_L, XH_L, XM_H

	vshl.i64	T1, XL, #57
	vshl.i64	T2, XL, #62
	veor		T1, T1, T2
	vshl.i64	T2, XL, #63
	veor		T1, T1, T2
	veor		XL_H, XL_H, T1_L
	veor		XH_L, XH_L, T1_H

	vshr.u64	T1, XL, #1
	veor		XH, XH, XL
	veor		XL, XL, T1
	vshr.u64	T1, T1, #6
	vshr.u64	XL, XL, #1
	.endm

	.macro		ghash_update, pn
	vld1.64		{XL}, [r1]

	/* do the head block first, if supplied */
	ldr		ip, [sp]
	teq		ip, #0
	beq		0f
	vld1.64		{T1}, [ip]
	teq		r0, #0
	b		3f

0:	.ifc		\pn, p64
	tst		r0, #3			// skip until #blocks is a
	bne		2f			// round multiple of 4

	vld1.8		{XL2-XM2}, [r2]!
1:	vld1.8		{T3-T2}, [r2]!
	vrev64.8	XL2, XL2
	vrev64.8	XM2, XM2

	subs		r0, r0, #4

	vext.8		T1, XL2, XL2, #8
	veor		XL2_H, XL2_H, XL_L
	veor		XL, XL, T1

	vrev64.8	T3, T3
	vrev64.8	T1, T2

	vmull.p64	XH, HH4_H, XL_H			// a1 * b1
	veor		XL2_H, XL2_H, XL_H
	vmull.p64	XL, HH4_L, XL_L			// a0 * b0
	vmull.p64	XM, HH34_H, XL2_H		// (a1 + a0)(b1 + b0)

	vmull.p64	XH2, HH3_H, XM2_L		// a1 * b1
	veor		XM2_L, XM2_L, XM2_H
	vmull.p64	XL2, HH3_L, XM2_H		// a0 * b0
	vmull.p64	XM2, HH34_L, XM2_L		// (a1 + a0)(b1 + b0)

	veor		XH, XH, XH2
	veor		XL, XL, XL2
	veor		XM, XM, XM2

	vmull.p64	XH2, HH_H, T3_L			// a1 * b1
	veor		T3_L, T3_L, T3_H
	vmull.p64	XL2, HH_L, T3_H			// a0 * b0
	vmull.p64	XM2, SHASH2_H, T3_L		// (a1 + a0)(b1 + b0)

	veor		XH, XH, XH2
	veor		XL, XL, XL2
	veor		XM, XM, XM2

	vmull.p64	XH2, SHASH_H, T1_L		// a1 * b1
	veor		T1_L, T1_L, T1_H
	vmull.p64	XL2, SHASH_L, T1_H		// a0 * b0
	vmull.p64	XM2, SHASH2_p64, T1_L		// (a1 + a0)(b1 + b0)

	veor		XH, XH, XH2
	veor		XL, XL, XL2
	veor		XM, XM, XM2

	beq		4f

	vld1.8		{XL2-XM2}, [r2]!

	veor		T1, XL, XH
	veor		XM, XM, T1

	__pmull_reduce_p64

	veor		T1, T1, XH
	veor		XL, XL, T1

	b		1b
	.endif

2:	vld1.64		{T1}, [r2]!
	subs		r0, r0, #1

3:	/* multiply XL by SHASH in GF(2^128) */
#ifndef CONFIG_CPU_BIG_ENDIAN
	vrev64.8	T1, T1
#endif
	vext.8		IN1, T1, T1, #8
	veor		T1_L, T1_L, XL_H
	veor		XL, XL, IN1

	__pmull_\pn	XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h	@ a1 * b1
	veor		T1, T1, XL
	__pmull_\pn	XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l	@ a0 * b0
	__pmull_\pn	XM, T1_L, SHASH2_\pn			@ (a1+a0)(b1+b0)

4:	veor		T1, XL, XH
	veor		XM, XM, T1

	__pmull_reduce_\pn

	veor		T1, T1, XH
	veor		XL, XL, T1

	bne		0b

	vst1.64		{XL}, [r1]
	bx		lr
	.endm

	/*
	 * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
	 *			   struct ghash_key const *k, const char *head)
	 */
ENTRY(pmull_ghash_update_p64)
	vld1.64		{SHASH}, [r3]!
	vld1.64		{HH}, [r3]!
	vld1.64		{HH3-HH4}, [r3]

	veor		SHASH2_p64, SHASH_L, SHASH_H
	veor		SHASH2_H, HH_L, HH_H
	veor		HH34_L, HH3_L, HH3_H
	veor		HH34_H, HH4_L, HH4_H

	vmov.i8		MASK, #0xe1
	vshl.u64	MASK, MASK, #57

	ghash_update	p64
ENDPROC(pmull_ghash_update_p64)

ENTRY(pmull_ghash_update_p8)
	vld1.64		{SHASH}, [r3]
	veor		SHASH2_p8, SHASH_L, SHASH_H

	vext.8		s1l, SHASH_L, SHASH_L, #1
	vext.8		s2l, SHASH_L, SHASH_L, #2
	vext.8		s3l, SHASH_L, SHASH_L, #3
	vext.8		s4l, SHASH_L, SHASH_L, #4
	vext.8		s1h, SHASH_H, SHASH_H, #1
	vext.8		s2h, SHASH_H, SHASH_H, #2
	vext.8		s3h, SHASH_H, SHASH_H, #3
	vext.8		s4h, SHASH_H, SHASH_H, #4

	vmov.i64	k16, #0xffff
	vmov.i64	k32, #0xffffffff
	vmov.i64	k48, #0xffffffffffff

	ghash_update	p8
ENDPROC(pmull_ghash_update_p8)

repo_name: angery/linux


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
 *
 * Copyright (C) 2015 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
 */

#include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h>
#include <asm/unaligned.h>
#include <crypto/b128ops.h>
#include <crypto/cryptd.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <crypto/gf128mul.h>
#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/jump_label.h>
#include <linux/module.h>

MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_CRYPTO("ghash");

#define GHASH_BLOCK_SIZE	16
#define GHASH_DIGEST_SIZE	16

struct ghash_key {
	be128	k;
	u64	h[][2];
};

struct ghash_desc_ctx {
	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
	u8 buf[GHASH_BLOCK_SIZE];
	u32 count;
};

struct ghash_async_ctx {
	struct cryptd_ahash *cryptd_tfm;
};

asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
				       u64 const h[][2], const char *head);

asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
				      u64 const h[][2], const char *head);

static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64);

static int ghash_init(struct shash_desc *desc)
{
	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);

	*ctx = (struct ghash_desc_ctx){};
	return 0;
}

static void ghash_do_update(int blocks, u64 dg[], const char *src,
			    struct ghash_key *key, const char *head)
{
	if (likely(crypto_simd_usable())) {
		kernel_neon_begin();
		if (static_branch_likely(&use_p64))
			pmull_ghash_update_p64(blocks, dg, src, key->h, head);
		else
			pmull_ghash_update_p8(blocks, dg, src, key->h, head);
		kernel_neon_end();
	} else {
		be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };

		do {
			const u8 *in = src;

			if (head) {
				in = head;
				blocks++;
				head = NULL;
			} else {
				src += GHASH_BLOCK_SIZE;
			}

			crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
			gf128mul_lle(&dst, &key->k);
		} while (--blocks);

		dg[0] = be64_to_cpu(dst.b);
		dg[1] = be64_to_cpu(dst.a);
	}
}

static int ghash_update(struct shash_desc *desc, const u8 *src,
			unsigned int len)
{
	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;

	ctx->count += len;

	if ((partial + len) >= GHASH_BLOCK_SIZE) {
		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
		int blocks;

		if (partial) {
			int p = GHASH_BLOCK_SIZE - partial;

			memcpy(ctx->buf + partial, src, p);
			src += p;
			len -= p;
		}

		blocks = len / GHASH_BLOCK_SIZE;
		len %= GHASH_BLOCK_SIZE;

		ghash_do_update(blocks, ctx->digest, src, key,
				partial ? ctx->buf : NULL);
		src += blocks * GHASH_BLOCK_SIZE;
		partial = 0;
	}
	if (len)
		memcpy(ctx->buf + partial, src, len);
	return 0;
}

static int ghash_final(struct shash_desc *desc, u8 *dst)
{
	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;

	if (partial) {
		struct ghash_key *key = crypto_shash_ctx(desc->tfm);

		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
		ghash_do_update(1, ctx->digest, ctx->buf, key, NULL);
	}
	put_unaligned_be64(ctx->digest[1], dst);
	put_unaligned_be64(ctx->digest[0], dst + 8);

	*ctx = (struct ghash_desc_ctx){};
	return 0;
}

static void ghash_reflect(u64 h[], const be128 *k)
{
	u64 carry = be64_to_cpu(k->a) >> 63;

	h[0] = (be64_to_cpu(k->b) << 1) | carry;
	h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);

	if (carry)
		h[1] ^= 0xc200000000000000UL;
}

static int ghash_setkey(struct crypto_shash *tfm,
			const u8 *inkey, unsigned int keylen)
{
	struct ghash_key *key = crypto_shash_ctx(tfm);

	if (keylen != GHASH_BLOCK_SIZE)
		return -EINVAL;

	/* needed for the fallback */
	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
	ghash_reflect(key->h[0], &key->k);

	if (static_branch_likely(&use_p64)) {
		be128 h = key->k;

		gf128mul_lle(&h, &key->k);
		ghash_reflect(key->h[1], &h);

		gf128mul_lle(&h, &key->k);
		ghash_reflect(key->h[2], &h);

		gf128mul_lle(&h, &key->k);
		ghash_reflect(key->h[3], &h);
	}
	return 0;
}

static struct shash_alg ghash_alg = {
	.digestsize		= GHASH_DIGEST_SIZE,
	.init			= ghash_init,
	.update			= ghash_update,
	.final			= ghash_final,
	.setkey			= ghash_setkey,
	.descsize		= sizeof(struct ghash_desc_ctx),

	.base.cra_name		= "ghash",
	.base.cra_driver_name	= "ghash-ce-sync",
	.base.cra_priority	= 300 - 1,
	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
	.base.cra_ctxsize	= sizeof(struct ghash_key) + sizeof(u64[2]),
	.base.cra_module	= THIS_MODULE,
};

static int ghash_async_init(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);

	desc->tfm = child;
	return crypto_shash_init(desc);
}

static int ghash_async_update(struct ahash_request *req)
{
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

	if (!crypto_simd_usable() ||
	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
		memcpy(cryptd_req, req, sizeof(*req));
		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
		return crypto_ahash_update(cryptd_req);
	} else {
		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
		return shash_ahash_update(req, desc);
	}
}

static int ghash_async_final(struct ahash_request *req)
{
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

	if (!crypto_simd_usable() ||
	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
		memcpy(cryptd_req, req, sizeof(*req));
		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
		return crypto_ahash_final(cryptd_req);
	} else {
		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
		return crypto_shash_final(desc, req->result);
	}
}

static int ghash_async_digest(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

	if (!crypto_simd_usable() ||
	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
		memcpy(cryptd_req, req, sizeof(*req));
		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
		return crypto_ahash_digest(cryptd_req);
	} else {
		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);

		desc->tfm = child;
		return shash_ahash_digest(req, desc);
	}
}

static int ghash_async_import(struct ahash_request *req, const void *in)
{
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);

	desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm);

	return crypto_shash_import(desc, in);
}

static int ghash_async_export(struct ahash_request *req, void *out)
{
	struct ahash_request *cryptd_req = ahash_request_ctx(req);
	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);

	return crypto_shash_export(desc, out);
}

static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
	struct crypto_ahash *child = &ctx->cryptd_tfm->base;

	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
			       & CRYPTO_TFM_REQ_MASK);
	return crypto_ahash_setkey(child, key, keylen);
}

static int ghash_async_init_tfm(struct crypto_tfm *tfm)
{
	struct cryptd_ahash *cryptd_tfm;
	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);

	cryptd_tfm = cryptd_alloc_ahash("ghash-ce-sync", 0, 0);
	if (IS_ERR(cryptd_tfm))
		return PTR_ERR(cryptd_tfm);
	ctx->cryptd_tfm = cryptd_tfm;
	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
				 sizeof(struct ahash_request) +
				 crypto_ahash_reqsize(&cryptd_tfm->base));

	return 0;
}

static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
{
	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);

	cryptd_free_ahash(ctx->cryptd_tfm);
}

static struct ahash_alg ghash_async_alg = {
	.init			= ghash_async_init,
	.update			= ghash_async_update,
	.final			= ghash_async_final,
	.setkey			= ghash_async_setkey,
	.digest			= ghash_async_digest,
	.import			= ghash_async_import,
	.export			= ghash_async_export,
	.halg.digestsize	= GHASH_DIGEST_SIZE,
	.halg.statesize		= sizeof(struct ghash_desc_ctx),
	.halg.base		= {
		.cra_name	= "ghash",
		.cra_driver_name = "ghash-ce",
		.cra_priority	= 300,
		.cra_flags	= CRYPTO_ALG_ASYNC,
		.cra_blocksize	= GHASH_BLOCK_SIZE,
		.cra_ctxsize	= sizeof(struct ghash_async_ctx),
		.cra_module	= THIS_MODULE,
		.cra_init	= ghash_async_init_tfm,
		.cra_exit	= ghash_async_exit_tfm,
	},
};

static int __init ghash_ce_mod_init(void)
{
	int err;

	if (!(elf_hwcap & HWCAP_NEON))
		return -ENODEV;

	if (elf_hwcap2 & HWCAP2_PMULL) {
		ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]);
		static_branch_enable(&use_p64);
	}

	err = crypto_register_shash(&ghash_alg);
	if (err)
		return err;
	err = crypto_register_ahash(&ghash_async_alg);
	if (err)
		goto err_shash;

	return 0;

err_shash:
	crypto_unregister_shash(&ghash_alg);
	return err;
}

static void __exit ghash_ce_mod_exit(void)
{
	crypto_unregister_ahash(&ghash_async_alg);
	crypto_unregister_shash(&ghash_alg);
}

module_init(ghash_ce_mod_init);
module_exit(ghash_ce_mod_exit);


code:
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * NH - ε-almost-universal hash function, NEON accelerated version
 *
 * Copyright 2018 Google LLC
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>

	.text
	.fpu		neon

	KEY		.req	r0
	MESSAGE		.req	r1
	MESSAGE_LEN	.req	r2
	HASH		.req	r3

	PASS0_SUMS	.req	q0
	PASS0_SUM_A	.req	d0
	PASS0_SUM_B	.req	d1
	PASS1_SUMS	.req	q1
	PASS1_SUM_A	.req	d2
	PASS1_SUM_B	.req	d3
	PASS2_SUMS	.req	q2
	PASS2_SUM_A	.req	d4
	PASS2_SUM_B	.req	d5
	PASS3_SUMS	.req	q3
	PASS3_SUM_A	.req	d6
	PASS3_SUM_B	.req	d7
	K0		.req	q4
	K1		.req	q5
	K2		.req	q6
	K3		.req	q7
	T0		.req	q8
	T0_L		.req	d16
	T0_H		.req	d17
	T1		.req	q9
	T1_L		.req	d18
	T1_H		.req	d19
	T2		.req	q10
	T2_L		.req	d20
	T2_H		.req	d21
	T3		.req	q11
	T3_L		.req	d22
	T3_H		.req	d23

.macro _nh_stride	k0, k1, k2, k3

	// Load next message stride
	vld1.8		{T3}, [MESSAGE]!

	// Load next key stride
	vld1.32		{\k3}, [KEY]!

	// Add message words to key words
	vadd.u32	T0, T3, \k0
	vadd.u32	T1, T3, \k1
	vadd.u32	T2, T3, \k2
	vadd.u32	T3, T3, \k3

	// Multiply 32x32 => 64 and accumulate
	vmlal.u32	PASS0_SUMS, T0_L, T0_H
	vmlal.u32	PASS1_SUMS, T1_L, T1_H
	vmlal.u32	PASS2_SUMS, T2_L, T2_H
	vmlal.u32	PASS3_SUMS, T3_L, T3_H
.endm

/*
 * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
 *		__le64 hash[NH_NUM_PASSES])
 *
 * It's guaranteed that message_len % 16 == 0.
 */
ENTRY(nh_neon)

	vld1.32		{K0,K1}, [KEY]!
	  vmov.u64	PASS0_SUMS, #0
	  vmov.u64	PASS1_SUMS, #0
	vld1.32		{K2}, [KEY]!
	  vmov.u64	PASS2_SUMS, #0
	  vmov.u64	PASS3_SUMS, #0

	subs		MESSAGE_LEN, MESSAGE_LEN, #64
	blt		.Lloop4_done
.Lloop4:
	_nh_stride	K0, K1, K2, K3
	_nh_stride	K1, K2, K3, K0
	_nh_stride	K2, K3, K0, K1
	_nh_stride	K3, K0, K1, K2
	subs		MESSAGE_LEN, MESSAGE_LEN, #64
	bge		.Lloop4

.Lloop4_done:
	ands		MESSAGE_LEN, MESSAGE_LEN, #63
	beq		.Ldone
	_nh_stride	K0, K1, K2, K3

	subs		MESSAGE_LEN, MESSAGE_LEN, #16
	beq		.Ldone
	_nh_stride	K1, K2, K3, K0

	subs		MESSAGE_LEN, MESSAGE_LEN, #16
	beq		.Ldone
	_nh_stride	K2, K3, K0, K1

.Ldone:
	// Sum the accumulators for each pass, then store the sums to 'hash'
	vadd.u64	T0_L, PASS0_SUM_A, PASS0_SUM_B
	vadd.u64	T0_H, PASS1_SUM_A, PASS1_SUM_B
	vadd.u64	T1_L, PASS2_SUM_A, PASS2_SUM_B
	vadd.u64	T1_H, PASS3_SUM_A, PASS3_SUM_B
	vst1.8		{T0-T1}, [HASH]
	bx		lr
ENDPROC(nh_neon)


code:
// SPDX-License-Identifier: GPL-2.0
/*
 * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
 * (NEON accelerated version)
 *
 * Copyright 2018 Google LLC
 */

#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <crypto/nhpoly1305.h>
#include <linux/module.h>

asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
			__le64 hash[NH_NUM_PASSES]);

static int nhpoly1305_neon_update(struct shash_desc *desc,
				  const u8 *src, unsigned int srclen)
{
	if (srclen < 64 || !crypto_simd_usable())
		return crypto_nhpoly1305_update(desc, src, srclen);

	do {
		unsigned int n = min_t(unsigned int, srclen, SZ_4K);

		kernel_neon_begin();
		crypto_nhpoly1305_update_helper(desc, src, n, nh_neon);
		kernel_neon_end();
		src += n;
		srclen -= n;
	} while (srclen);
	return 0;
}

static struct shash_alg nhpoly1305_alg = {
	.base.cra_name		= "nhpoly1305",
	.base.cra_driver_name	= "nhpoly1305-neon",
	.base.cra_priority	= 200,
	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
	.base.cra_module	= THIS_MODULE,
	.digestsize		= POLY1305_DIGEST_SIZE,
	.init			= crypto_nhpoly1305_init,
	.update			= nhpoly1305_neon_update,
	.final			= crypto_nhpoly1305_final,
	.setkey			= crypto_nhpoly1305_setkey,
	.descsize		= sizeof(struct nhpoly1305_state),
};

static int __init nhpoly1305_mod_init(void)
{
	if (!(elf_hwcap & HWCAP_NEON))
		return -ENODEV;

	return crypto_register_shash(&nhpoly1305_alg);
}

static void __exit nhpoly1305_mod_exit(void)
{
	crypto_unregister_shash(&nhpoly1305_alg);
}

module_init(nhpoly1305_mod_init);
module_exit(nhpoly1305_mod_exit);

MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
MODULE_ALIAS_CRYPTO("nhpoly1305");
MODULE_ALIAS_CRYPTO("nhpoly1305-neon");


code:
#!/usr/bin/env perl
# SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause
#
# ====================================================================
# Written by Andy Polyakov, @dot-asm, initially for the OpenSSL
# project.
# ====================================================================
#
#			IALU(*)/gcc-4.4		NEON
#
# ARM11xx(ARMv6)	7.78/+100%		-
# Cortex-A5		6.35/+130%		3.00
# Cortex-A8		6.25/+115%		2.36
# Cortex-A9		5.10/+95%		2.55
# Cortex-A15		3.85/+85%		1.25(**)
# Snapdragon S4		5.70/+100%		1.48(**)
#
# (*)	this is for -march=armv6, i.e. with bunch of ldrb loading data;
# (**)	these are trade-off results, they can be improved by ~8% but at
#	the cost of 15/12% regression on Cortex-A5/A7, it's even possible
#	to improve Cortex-A9 result, but then A5/A7 loose more than 20%;

$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
    open STDOUT,">$output";
}

($ctx,$inp,$len,$padbit)=map("r$_",(0..3));

$code.=<<___;
#ifndef	__KERNEL__
# include "arm_arch.h"
#else
# define __ARM_ARCH__ __LINUX_ARM_ARCH__
# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__
# define poly1305_init   poly1305_init_arm
# define poly1305_blocks poly1305_blocks_arm
# define poly1305_emit   poly1305_emit_arm
.globl	poly1305_blocks_neon
#endif

#if defined(__thumb2__)
.syntax	unified
.thumb
#else
.code	32
#endif

.text

.globl	poly1305_emit
.globl	poly1305_blocks
.globl	poly1305_init
.type	poly1305_init,%function
.align	5
poly1305_init:
.Lpoly1305_init:
	stmdb	sp!,{r4-r11}

	eor	r3,r3,r3
	cmp	$inp,#0
	str	r3,[$ctx,#0]		@ zero hash value
	str	r3,[$ctx,#4]
	str	r3,[$ctx,#8]
	str	r3,[$ctx,#12]
	str	r3,[$ctx,#16]
	str	r3,[$ctx,#36]		@ clear is_base2_26
	add	$ctx,$ctx,#20

#ifdef	__thumb2__
	it	eq
#endif
	moveq	r0,#0
	beq	.Lno_key

#if	__ARM_MAX_ARCH__>=7
	mov	r3,#-1
	str	r3,[$ctx,#28]		@ impossible key power value
# ifndef __KERNEL__
	adr	r11,.Lpoly1305_init
	ldr	r12,.LOPENSSL_armcap
# endif
#endif
	ldrb	r4,[$inp,#0]
	mov	r10,#0x0fffffff
	ldrb	r5,[$inp,#1]
	and	r3,r10,#-4		@ 0x0ffffffc
	ldrb	r6,[$inp,#2]
	ldrb	r7,[$inp,#3]
	orr	r4,r4,r5,lsl#8
	ldrb	r5,[$inp,#4]
	orr	r4,r4,r6,lsl#16
	ldrb	r6,[$inp,#5]
	orr	r4,r4,r7,lsl#24
	ldrb	r7,[$inp,#6]
	and	r4,r4,r10

#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
# if !defined(_WIN32)
	ldr	r12,[r11,r12]		@ OPENSSL_armcap_P
# endif
# if defined(__APPLE__) || defined(_WIN32)
	ldr	r12,[r12]
# endif
#endif
	ldrb	r8,[$inp,#7]
	orr	r5,r5,r6,lsl#8
	ldrb	r6,[$inp,#8]
	orr	r5,r5,r7,lsl#16
	ldrb	r7,[$inp,#9]
	orr	r5,r5,r8,lsl#24
	ldrb	r8,[$inp,#10]
	and	r5,r5,r3

#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
	tst	r12,#ARMV7_NEON		@ check for NEON
# ifdef	__thumb2__
	adr	r9,.Lpoly1305_blocks_neon
	adr	r11,.Lpoly1305_blocks
	it	ne
	movne	r11,r9
	adr	r12,.Lpoly1305_emit
	orr	r11,r11,#1		@ thumb-ify addresses
	orr	r12,r12,#1
# else
	add	r12,r11,#(.Lpoly1305_emit-.Lpoly1305_init)
	ite	eq
	addeq	r11,r11,#(.Lpoly1305_blocks-.Lpoly1305_init)
	addne	r11,r11,#(.Lpoly1305_blocks_neon-.Lpoly1305_init)
# endif
#endif
	ldrb	r9,[$inp,#11]
	orr	r6,r6,r7,lsl#8
	ldrb	r7,[$inp,#12]
	orr	r6,r6,r8,lsl#16
	ldrb	r8,[$inp,#13]
	orr	r6,r6,r9,lsl#24
	ldrb	r9,[$inp,#14]
	and	r6,r6,r3

	ldrb	r10,[$inp,#15]
	orr	r7,r7,r8,lsl#8
	str	r4,[$ctx,#0]
	orr	r7,r7,r9,lsl#16
	str	r5,[$ctx,#4]
	orr	r7,r7,r10,lsl#24
	str	r6,[$ctx,#8]
	and	r7,r7,r3
	str	r7,[$ctx,#12]
#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
	stmia	r2,{r11,r12}		@ fill functions table
	mov	r0,#1
#else
	mov	r0,#0
#endif
.Lno_key:
	ldmia	sp!,{r4-r11}
#if	__ARM_ARCH__>=5
	ret				@ bx	lr
#else
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	poly1305_init,.-poly1305_init
___
{
my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
my ($s1,$s2,$s3)=($r1,$r2,$r3);

$code.=<<___;
.type	poly1305_blocks,%function
.align	5
poly1305_blocks:
.Lpoly1305_blocks:
	stmdb	sp!,{r3-r11,lr}

	ands	$len,$len,#-16
	beq	.Lno_data

	add	$len,$len,$inp		@ end pointer
	sub	sp,sp,#32

#if __ARM_ARCH__<7
	ldmia	$ctx,{$h0-$r3}		@ load context
	add	$ctx,$ctx,#20
	str	$len,[sp,#16]		@ offload stuff
	str	$ctx,[sp,#12]
#else
	ldr	lr,[$ctx,#36]		@ is_base2_26
	ldmia	$ctx!,{$h0-$h4}		@ load hash value
	str	$len,[sp,#16]		@ offload stuff
	str	$ctx,[sp,#12]

	adds	$r0,$h0,$h1,lsl#26	@ base 2^26 -> base 2^32
	mov	$r1,$h1,lsr#6
	adcs	$r1,$r1,$h2,lsl#20
	mov	$r2,$h2,lsr#12
	adcs	$r2,$r2,$h3,lsl#14
	mov	$r3,$h3,lsr#18
	adcs	$r3,$r3,$h4,lsl#8
	mov	$len,#0
	teq	lr,#0
	str	$len,[$ctx,#16]		@ clear is_base2_26
	adc	$len,$len,$h4,lsr#24

	itttt	ne
	movne	$h0,$r0			@ choose between radixes
	movne	$h1,$r1
	movne	$h2,$r2
	movne	$h3,$r3
	ldmia	$ctx,{$r0-$r3}		@ load key
	it	ne
	movne	$h4,$len
#endif

	mov	lr,$inp
	cmp	$padbit,#0
	str	$r1,[sp,#20]
	str	$r2,[sp,#24]
	str	$r3,[sp,#28]
	b	.Loop

.align	4
.Loop:
#if __ARM_ARCH__<7
	ldrb	r0,[lr],#16		@ load input
# ifdef	__thumb2__
	it	hi
# endif
	addhi	$h4,$h4,#1		@ 1<<128
	ldrb	r1,[lr,#-15]
	ldrb	r2,[lr,#-14]
	ldrb	r3,[lr,#-13]
	orr	r1,r0,r1,lsl#8
	ldrb	r0,[lr,#-12]
	orr	r2,r1,r2,lsl#16
	ldrb	r1,[lr,#-11]
	orr	r3,r2,r3,lsl#24
	ldrb	r2,[lr,#-10]
	adds	$h0,$h0,r3		@ accumulate input

	ldrb	r3,[lr,#-9]
	orr	r1,r0,r1,lsl#8
	ldrb	r0,[lr,#-8]
	orr	r2,r1,r2,lsl#16
	ldrb	r1,[lr,#-7]
	orr	r3,r2,r3,lsl#24
	ldrb	r2,[lr,#-6]
	adcs	$h1,$h1,r3

	ldrb	r3,[lr,#-5]
	orr	r1,r0,r1,lsl#8
	ldrb	r0,[lr,#-4]
	orr	r2,r1,r2,lsl#16
	ldrb	r1,[lr,#-3]
	orr	r3,r2,r3,lsl#24
	ldrb	r2,[lr,#-2]
	adcs	$h2,$h2,r3

	ldrb	r3,[lr,#-1]
	orr	r1,r0,r1,lsl#8
	str	lr,[sp,#8]		@ offload input pointer
	orr	r2,r1,r2,lsl#16
	add	$s1,$r1,$r1,lsr#2
	orr	r3,r2,r3,lsl#24
#else
	ldr	r0,[lr],#16		@ load input
	it	hi
	addhi	$h4,$h4,#1		@ padbit
	ldr	r1,[lr,#-12]
	ldr	r2,[lr,#-8]
	ldr	r3,[lr,#-4]
# ifdef	__ARMEB__
	rev	r0,r0
	rev	r1,r1
	rev	r2,r2
	rev	r3,r3
# endif
	adds	$h0,$h0,r0		@ accumulate input
	str	lr,[sp,#8]		@ offload input pointer
	adcs	$h1,$h1,r1
	add	$s1,$r1,$r1,lsr#2
	adcs	$h2,$h2,r2
#endif
	add	$s2,$r2,$r2,lsr#2
	adcs	$h3,$h3,r3
	add	$s3,$r3,$r3,lsr#2

	umull	r2,r3,$h1,$r0
	 adc	$h4,$h4,#0
	umull	r0,r1,$h0,$r0
	umlal	r2,r3,$h4,$s1
	umlal	r0,r1,$h3,$s1
	ldr	$r1,[sp,#20]		@ reload $r1
	umlal	r2,r3,$h2,$s3
	umlal	r0,r1,$h1,$s3
	umlal	r2,r3,$h3,$s2
	umlal	r0,r1,$h2,$s2
	umlal	r2,r3,$h0,$r1
	str	r0,[sp,#0]		@ future $h0
	 mul	r0,$s2,$h4
	ldr	$r2,[sp,#24]		@ reload $r2
	adds	r2,r2,r1		@ d1+=d0>>32
	 eor	r1,r1,r1
	adc	lr,r3,#0		@ future $h2
	str	r2,[sp,#4]		@ future $h1

	mul	r2,$s3,$h4
	eor	r3,r3,r3
	umlal	r0,r1,$h3,$s3
	ldr	$r3,[sp,#28]		@ reload $r3
	umlal	r2,r3,$h3,$r0
	umlal	r0,r1,$h2,$r0
	umlal	r2,r3,$h2,$r1
	umlal	r0,r1,$h1,$r1
	umlal	r2,r3,$h1,$r2
	umlal	r0,r1,$h0,$r2
	umlal	r2,r3,$h0,$r3
	ldr	$h0,[sp,#0]
	mul	$h4,$r0,$h4
	ldr	$h1,[sp,#4]

	adds	$h2,lr,r0		@ d2+=d1>>32
	ldr	lr,[sp,#8]		@ reload input pointer
	adc	r1,r1,#0
	adds	$h3,r2,r1		@ d3+=d2>>32
	ldr	r0,[sp,#16]		@ reload end pointer
	adc	r3,r3,#0
	add	$h4,$h4,r3		@ h4+=d3>>32

	and	r1,$h4,#-4
	and	$h4,$h4,#3
	add	r1,r1,r1,lsr#2		@ *=5
	adds	$h0,$h0,r1
	adcs	$h1,$h1,#0
	adcs	$h2,$h2,#0
	adcs	$h3,$h3,#0
	adc	$h4,$h4,#0

	cmp	r0,lr			@ done yet?
	bhi	.Loop

	ldr	$ctx,[sp,#12]
	add	sp,sp,#32
	stmdb	$ctx,{$h0-$h4}		@ store the result

.Lno_data:
#if	__ARM_ARCH__>=5
	ldmia	sp!,{r3-r11,pc}
#else
	ldmia	sp!,{r3-r11,lr}
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	poly1305_blocks,.-poly1305_blocks
___
}
{
my ($ctx,$mac,$nonce)=map("r$_",(0..2));
my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
my $g4=$ctx;

$code.=<<___;
.type	poly1305_emit,%function
.align	5
poly1305_emit:
.Lpoly1305_emit:
	stmdb	sp!,{r4-r11}

	ldmia	$ctx,{$h0-$h4}

#if __ARM_ARCH__>=7
	ldr	ip,[$ctx,#36]		@ is_base2_26

	adds	$g0,$h0,$h1,lsl#26	@ base 2^26 -> base 2^32
	mov	$g1,$h1,lsr#6
	adcs	$g1,$g1,$h2,lsl#20
	mov	$g2,$h2,lsr#12
	adcs	$g2,$g2,$h3,lsl#14
	mov	$g3,$h3,lsr#18
	adcs	$g3,$g3,$h4,lsl#8
	mov	$g4,#0
	adc	$g4,$g4,$h4,lsr#24

	tst	ip,ip
	itttt	ne
	movne	$h0,$g0
	movne	$h1,$g1
	movne	$h2,$g2
	movne	$h3,$g3
	it	ne
	movne	$h4,$g4
#endif

	adds	$g0,$h0,#5		@ compare to modulus
	adcs	$g1,$h1,#0
	adcs	$g2,$h2,#0
	adcs	$g3,$h3,#0
	adc	$g4,$h4,#0
	tst	$g4,#4			@ did it carry/borrow?

#ifdef	__thumb2__
	it	ne
#endif
	movne	$h0,$g0
	ldr	$g0,[$nonce,#0]
#ifdef	__thumb2__
	it	ne
#endif
	movne	$h1,$g1
	ldr	$g1,[$nonce,#4]
#ifdef	__thumb2__
	it	ne
#endif
	movne	$h2,$g2
	ldr	$g2,[$nonce,#8]
#ifdef	__thumb2__
	it	ne
#endif
	movne	$h3,$g3
	ldr	$g3,[$nonce,#12]

	adds	$h0,$h0,$g0
	adcs	$h1,$h1,$g1
	adcs	$h2,$h2,$g2
	adc	$h3,$h3,$g3

#if __ARM_ARCH__>=7
# ifdef __ARMEB__
	rev	$h0,$h0
	rev	$h1,$h1
	rev	$h2,$h2
	rev	$h3,$h3
# endif
	str	$h0,[$mac,#0]
	str	$h1,[$mac,#4]
	str	$h2,[$mac,#8]
	str	$h3,[$mac,#12]
#else
	strb	$h0,[$mac,#0]
	mov	$h0,$h0,lsr#8
	strb	$h1,[$mac,#4]
	mov	$h1,$h1,lsr#8
	strb	$h2,[$mac,#8]
	mov	$h2,$h2,lsr#8
	strb	$h3,[$mac,#12]
	mov	$h3,$h3,lsr#8

	strb	$h0,[$mac,#1]
	mov	$h0,$h0,lsr#8
	strb	$h1,[$mac,#5]
	mov	$h1,$h1,lsr#8
	strb	$h2,[$mac,#9]
	mov	$h2,$h2,lsr#8
	strb	$h3,[$mac,#13]
	mov	$h3,$h3,lsr#8

	strb	$h0,[$mac,#2]
	mov	$h0,$h0,lsr#8
	strb	$h1,[$mac,#6]
	mov	$h1,$h1,lsr#8
	strb	$h2,[$mac,#10]
	mov	$h2,$h2,lsr#8
	strb	$h3,[$mac,#14]
	mov	$h3,$h3,lsr#8

	strb	$h0,[$mac,#3]
	strb	$h1,[$mac,#7]
	strb	$h2,[$mac,#11]
	strb	$h3,[$mac,#15]
#endif
	ldmia	sp!,{r4-r11}
#if	__ARM_ARCH__>=5
	ret				@ bx	lr
#else
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	poly1305_emit,.-poly1305_emit
___
{
my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
my ($T0,$T1,$MASK) = map("q$_",(15,4,0));

my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));

$code.=<<___;
#if	__ARM_MAX_ARCH__>=7
.fpu	neon

.type	poly1305_init_neon,%function
.align	5
poly1305_init_neon:
.Lpoly1305_init_neon:
	ldr	r3,[$ctx,#48]		@ first table element
	cmp	r3,#-1			@ is value impossible?
	bne	.Lno_init_neon

	ldr	r4,[$ctx,#20]		@ load key base 2^32
	ldr	r5,[$ctx,#24]
	ldr	r6,[$ctx,#28]
	ldr	r7,[$ctx,#32]

	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
	mov	r3,r4,lsr#26
	mov	r4,r5,lsr#20
	orr	r3,r3,r5,lsl#6
	mov	r5,r6,lsr#14
	orr	r4,r4,r6,lsl#12
	mov	r6,r7,lsr#8
	orr	r5,r5,r7,lsl#18
	and	r3,r3,#0x03ffffff
	and	r4,r4,#0x03ffffff
	and	r5,r5,#0x03ffffff

	vdup.32	$R0,r2			@ r^1 in both lanes
	add	r2,r3,r3,lsl#2		@ *5
	vdup.32	$R1,r3
	add	r3,r4,r4,lsl#2
	vdup.32	$S1,r2
	vdup.32	$R2,r4
	add	r4,r5,r5,lsl#2
	vdup.32	$S2,r3
	vdup.32	$R3,r5
	add	r5,r6,r6,lsl#2
	vdup.32	$S3,r4
	vdup.32	$R4,r6
	vdup.32	$S4,r5

	mov	$zeros,#2		@ counter

.Lsquare_neon:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4

	vmull.u32	$D0,$R0,${R0}[1]
	vmull.u32	$D1,$R1,${R0}[1]
	vmull.u32	$D2,$R2,${R0}[1]
	vmull.u32	$D3,$R3,${R0}[1]
	vmull.u32	$D4,$R4,${R0}[1]

	vmlal.u32	$D0,$R4,${S1}[1]
	vmlal.u32	$D1,$R0,${R1}[1]
	vmlal.u32	$D2,$R1,${R1}[1]
	vmlal.u32	$D3,$R2,${R1}[1]
	vmlal.u32	$D4,$R3,${R1}[1]

	vmlal.u32	$D0,$R3,${S2}[1]
	vmlal.u32	$D1,$R4,${S2}[1]
	vmlal.u32	$D3,$R1,${R2}[1]
	vmlal.u32	$D2,$R0,${R2}[1]
	vmlal.u32	$D4,$R2,${R2}[1]

	vmlal.u32	$D0,$R2,${S3}[1]
	vmlal.u32	$D3,$R0,${R3}[1]
	vmlal.u32	$D1,$R3,${S3}[1]
	vmlal.u32	$D2,$R4,${S3}[1]
	vmlal.u32	$D4,$R1,${R3}[1]

	vmlal.u32	$D3,$R4,${S4}[1]
	vmlal.u32	$D0,$R1,${S4}[1]
	vmlal.u32	$D1,$R2,${S4}[1]
	vmlal.u32	$D2,$R3,${S4}[1]
	vmlal.u32	$D4,$R0,${R4}[1]

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
	@ and P. Schwabe
	@
	@ H0>>+H1>>+H2>>+H3>>+H4
	@ H3>>+H4>>*5+H0>>+H1
	@
	@ Trivia.
	@
	@ Result of multiplication of n-bit number by m-bit number is
	@ n+m bits wide. However! Even though 2^n is a n+1-bit number,
	@ m-bit number multiplied by 2^n is still n+m bits wide.
	@
	@ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
	@ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
	@ one is n+1 bits wide.
	@
	@ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
	@ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
	@ can be 27. However! In cases when their width exceeds 26 bits
	@ they are limited by 2^26+2^6. This in turn means that *sum*
	@ of the products with these values can still be viewed as sum
	@ of 52-bit numbers as long as the amount of addends is not a
	@ power of 2. For example,
	@
	@ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
	@
	@ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
	@ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
	@ 8 * (2^52) or 2^55. However, the value is then multiplied by
	@ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
	@ which is less than 32 * (2^52) or 2^57. And when processing
	@ data we are looking at triple as many addends...
	@
	@ In key setup procedure pre-reduced H0 is limited by 5*4+1 and
	@ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
	@ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
	@ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
	@ instruction accepts 2x32-bit input and writes 2x64-bit result.
	@ This means that result of reduction have to be compressed upon
	@ loop wrap-around. This can be done in the process of reduction
	@ to minimize amount of instructions [as well as amount of
	@ 128-bit instructions, which benefits low-end processors], but
	@ one has to watch for H2 (which is narrower than H0) and 5*H4
	@ not being wider than 58 bits, so that result of right shift
	@ by 26 bits fits in 32 bits. This is also useful on x86,
	@ because it allows to use paddd in place for paddq, which
	@ benefits Atom, where paddq is ridiculously slow.

	vshr.u64	$T0,$D3,#26
	vmovn.i64	$D3#lo,$D3
	 vshr.u64	$T1,$D0,#26
	 vmovn.i64	$D0#lo,$D0
	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
	vbic.i32	$D3#lo,#0xfc000000	@ &=0x03ffffff
	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
	 vbic.i32	$D0#lo,#0xfc000000

	vshrn.u64	$T0#lo,$D4,#26
	vmovn.i64	$D4#lo,$D4
	 vshr.u64	$T1,$D1,#26
	 vmovn.i64	$D1#lo,$D1
	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
	vbic.i32	$D4#lo,#0xfc000000
	 vbic.i32	$D1#lo,#0xfc000000

	vadd.i32	$D0#lo,$D0#lo,$T0#lo
	vshl.u32	$T0#lo,$T0#lo,#2
	 vshrn.u64	$T1#lo,$D2,#26
	 vmovn.i64	$D2#lo,$D2
	vadd.i32	$D0#lo,$D0#lo,$T0#lo	@ h4 -> h0
	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
	 vbic.i32	$D2#lo,#0xfc000000

	vshr.u32	$T0#lo,$D0#lo,#26
	vbic.i32	$D0#lo,#0xfc000000
	 vshr.u32	$T1#lo,$D3#lo,#26
	 vbic.i32	$D3#lo,#0xfc000000
	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4

	subs		$zeros,$zeros,#1
	beq		.Lsquare_break_neon

	add		$tbl0,$ctx,#(48+0*9*4)
	add		$tbl1,$ctx,#(48+1*9*4)

	vtrn.32		$R0,$D0#lo		@ r^2:r^1
	vtrn.32		$R2,$D2#lo
	vtrn.32		$R3,$D3#lo
	vtrn.32		$R1,$D1#lo
	vtrn.32		$R4,$D4#lo

	vshl.u32	$S2,$R2,#2		@ *5
	vshl.u32	$S3,$R3,#2
	vshl.u32	$S1,$R1,#2
	vshl.u32	$S4,$R4,#2
	vadd.i32	$S2,$S2,$R2
	vadd.i32	$S1,$S1,$R1
	vadd.i32	$S3,$S3,$R3
	vadd.i32	$S4,$S4,$R4

	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vst1.32		{${S4}[0]},[$tbl0,:32]
	vst1.32		{${S4}[1]},[$tbl1,:32]

	b		.Lsquare_neon

.align	4
.Lsquare_break_neon:
	add		$tbl0,$ctx,#(48+2*4*9)
	add		$tbl1,$ctx,#(48+3*4*9)

	vmov		$R0,$D0#lo		@ r^4:r^3
	vshl.u32	$S1,$D1#lo,#2		@ *5
	vmov		$R1,$D1#lo
	vshl.u32	$S2,$D2#lo,#2
	vmov		$R2,$D2#lo
	vshl.u32	$S3,$D3#lo,#2
	vmov		$R3,$D3#lo
	vshl.u32	$S4,$D4#lo,#2
	vmov		$R4,$D4#lo
	vadd.i32	$S1,$S1,$D1#lo
	vadd.i32	$S2,$S2,$D2#lo
	vadd.i32	$S3,$S3,$D3#lo
	vadd.i32	$S4,$S4,$D4#lo

	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vst1.32		{${S4}[0]},[$tbl0]
	vst1.32		{${S4}[1]},[$tbl1]

.Lno_init_neon:
	ret				@ bx	lr
.size	poly1305_init_neon,.-poly1305_init_neon

.type	poly1305_blocks_neon,%function
.align	5
poly1305_blocks_neon:
.Lpoly1305_blocks_neon:
	ldr	ip,[$ctx,#36]		@ is_base2_26

	cmp	$len,#64
	blo	.Lpoly1305_blocks

	stmdb	sp!,{r4-r7}
	vstmdb	sp!,{d8-d15}		@ ABI specification says so

	tst	ip,ip			@ is_base2_26?
	bne	.Lbase2_26_neon

	stmdb	sp!,{r1-r3,lr}
	bl	.Lpoly1305_init_neon

	ldr	r4,[$ctx,#0]		@ load hash value base 2^32
	ldr	r5,[$ctx,#4]
	ldr	r6,[$ctx,#8]
	ldr	r7,[$ctx,#12]
	ldr	ip,[$ctx,#16]

	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
	mov	r3,r4,lsr#26
	 veor	$D0#lo,$D0#lo,$D0#lo
	mov	r4,r5,lsr#20
	orr	r3,r3,r5,lsl#6
	 veor	$D1#lo,$D1#lo,$D1#lo
	mov	r5,r6,lsr#14
	orr	r4,r4,r6,lsl#12
	 veor	$D2#lo,$D2#lo,$D2#lo
	mov	r6,r7,lsr#8
	orr	r5,r5,r7,lsl#18
	 veor	$D3#lo,$D3#lo,$D3#lo
	and	r3,r3,#0x03ffffff
	orr	r6,r6,ip,lsl#24
	 veor	$D4#lo,$D4#lo,$D4#lo
	and	r4,r4,#0x03ffffff
	mov	r1,#1
	and	r5,r5,#0x03ffffff
	str	r1,[$ctx,#36]		@ set is_base2_26

	vmov.32	$D0#lo[0],r2
	vmov.32	$D1#lo[0],r3
	vmov.32	$D2#lo[0],r4
	vmov.32	$D3#lo[0],r5
	vmov.32	$D4#lo[0],r6
	adr	$zeros,.Lzeros

	ldmia	sp!,{r1-r3,lr}
	b	.Lhash_loaded

.align	4
.Lbase2_26_neon:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ load hash value

	veor		$D0#lo,$D0#lo,$D0#lo
	veor		$D1#lo,$D1#lo,$D1#lo
	veor		$D2#lo,$D2#lo,$D2#lo
	veor		$D3#lo,$D3#lo,$D3#lo
	veor		$D4#lo,$D4#lo,$D4#lo
	vld4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
	adr		$zeros,.Lzeros
	vld1.32		{$D4#lo[0]},[$ctx]
	sub		$ctx,$ctx,#16		@ rewind

.Lhash_loaded:
	add		$in2,$inp,#32
	mov		$padbit,$padbit,lsl#24
	tst		$len,#31
	beq		.Leven

	vld4.32		{$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
	vmov.32		$H4#lo[0],$padbit
	sub		$len,$len,#16
	add		$in2,$inp,#32

# ifdef	__ARMEB__
	vrev32.8	$H0,$H0
	vrev32.8	$H3,$H3
	vrev32.8	$H1,$H1
	vrev32.8	$H2,$H2
# endif
	vsri.u32	$H4#lo,$H3#lo,#8	@ base 2^32 -> base 2^26
	vshl.u32	$H3#lo,$H3#lo,#18

	vsri.u32	$H3#lo,$H2#lo,#14
	vshl.u32	$H2#lo,$H2#lo,#12
	vadd.i32	$H4#hi,$H4#lo,$D4#lo	@ add hash value and move to #hi

	vbic.i32	$H3#lo,#0xfc000000
	vsri.u32	$H2#lo,$H1#lo,#20
	vshl.u32	$H1#lo,$H1#lo,#6

	vbic.i32	$H2#lo,#0xfc000000
	vsri.u32	$H1#lo,$H0#lo,#26
	vadd.i32	$H3#hi,$H3#lo,$D3#lo

	vbic.i32	$H0#lo,#0xfc000000
	vbic.i32	$H1#lo,#0xfc000000
	vadd.i32	$H2#hi,$H2#lo,$D2#lo

	vadd.i32	$H0#hi,$H0#lo,$D0#lo
	vadd.i32	$H1#hi,$H1#lo,$D1#lo

	mov		$tbl1,$zeros
	add		$tbl0,$ctx,#48

	cmp		$len,$len
	b		.Long_tail

.align	4
.Leven:
	subs		$len,$len,#64
	it		lo
	movlo		$in2,$zeros

	vmov.i32	$H4,#1<<24		@ padbit, yes, always
	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
	add		$inp,$inp,#64
	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
	add		$in2,$in2,#64
	itt		hi
	addhi		$tbl1,$ctx,#(48+1*9*4)
	addhi		$tbl0,$ctx,#(48+3*9*4)

# ifdef	__ARMEB__
	vrev32.8	$H0,$H0
	vrev32.8	$H3,$H3
	vrev32.8	$H1,$H1
	vrev32.8	$H2,$H2
# endif
	vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
	vshl.u32	$H3,$H3,#18

	vsri.u32	$H3,$H2,#14
	vshl.u32	$H2,$H2,#12

	vbic.i32	$H3,#0xfc000000
	vsri.u32	$H2,$H1,#20
	vshl.u32	$H1,$H1,#6

	vbic.i32	$H2,#0xfc000000
	vsri.u32	$H1,$H0,#26

	vbic.i32	$H0,#0xfc000000
	vbic.i32	$H1,#0xfc000000

	bls		.Lskip_loop

	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^2
	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4
	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	b		.Loop_neon

.align	5
.Loop_neon:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
	@   \___________________/
	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
	@   \___________________/ \____________________/
	@
	@ Note that we start with inp[2:3]*r^2. This is because it
	@ doesn't depend on reduction in previous iteration.
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ inp[2:3]*r^2

	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ accumulate inp[0:1]
	vmull.u32	$D2,$H2#hi,${R0}[1]
	vadd.i32	$H0#lo,$H0#lo,$D0#lo
	vmull.u32	$D0,$H0#hi,${R0}[1]
	vadd.i32	$H3#lo,$H3#lo,$D3#lo
	vmull.u32	$D3,$H3#hi,${R0}[1]
	vmlal.u32	$D2,$H1#hi,${R1}[1]
	vadd.i32	$H1#lo,$H1#lo,$D1#lo
	vmull.u32	$D1,$H1#hi,${R0}[1]

	vadd.i32	$H4#lo,$H4#lo,$D4#lo
	vmull.u32	$D4,$H4#hi,${R0}[1]
	subs		$len,$len,#64
	vmlal.u32	$D0,$H4#hi,${S1}[1]
	it		lo
	movlo		$in2,$zeros
	vmlal.u32	$D3,$H2#hi,${R1}[1]
	vld1.32		${S4}[1],[$tbl1,:32]
	vmlal.u32	$D1,$H0#hi,${R1}[1]
	vmlal.u32	$D4,$H3#hi,${R1}[1]

	vmlal.u32	$D0,$H3#hi,${S2}[1]
	vmlal.u32	$D3,$H1#hi,${R2}[1]
	vmlal.u32	$D4,$H2#hi,${R2}[1]
	vmlal.u32	$D1,$H4#hi,${S2}[1]
	vmlal.u32	$D2,$H0#hi,${R2}[1]

	vmlal.u32	$D3,$H0#hi,${R3}[1]
	vmlal.u32	$D0,$H2#hi,${S3}[1]
	vmlal.u32	$D4,$H1#hi,${R3}[1]
	vmlal.u32	$D1,$H3#hi,${S3}[1]
	vmlal.u32	$D2,$H4#hi,${S3}[1]

	vmlal.u32	$D3,$H4#hi,${S4}[1]
	vmlal.u32	$D0,$H1#hi,${S4}[1]
	vmlal.u32	$D4,$H0#hi,${R4}[1]
	vmlal.u32	$D1,$H2#hi,${S4}[1]
	vmlal.u32	$D2,$H3#hi,${S4}[1]

	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
	add		$in2,$in2,#64

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ (hash+inp[0:1])*r^4 and accumulate

	vmlal.u32	$D3,$H3#lo,${R0}[0]
	vmlal.u32	$D0,$H0#lo,${R0}[0]
	vmlal.u32	$D4,$H4#lo,${R0}[0]
	vmlal.u32	$D1,$H1#lo,${R0}[0]
	vmlal.u32	$D2,$H2#lo,${R0}[0]
	vld1.32		${S4}[0],[$tbl0,:32]

	vmlal.u32	$D3,$H2#lo,${R1}[0]
	vmlal.u32	$D0,$H4#lo,${S1}[0]
	vmlal.u32	$D4,$H3#lo,${R1}[0]
	vmlal.u32	$D1,$H0#lo,${R1}[0]
	vmlal.u32	$D2,$H1#lo,${R1}[0]

	vmlal.u32	$D3,$H1#lo,${R2}[0]
	vmlal.u32	$D0,$H3#lo,${S2}[0]
	vmlal.u32	$D4,$H2#lo,${R2}[0]
	vmlal.u32	$D1,$H4#lo,${S2}[0]
	vmlal.u32	$D2,$H0#lo,${R2}[0]

	vmlal.u32	$D3,$H0#lo,${R3}[0]
	vmlal.u32	$D0,$H2#lo,${S3}[0]
	vmlal.u32	$D4,$H1#lo,${R3}[0]
	vmlal.u32	$D1,$H3#lo,${S3}[0]
	vmlal.u32	$D3,$H4#lo,${S4}[0]

	vmlal.u32	$D2,$H4#lo,${S3}[0]
	vmlal.u32	$D0,$H1#lo,${S4}[0]
	vmlal.u32	$D4,$H0#lo,${R4}[0]
	vmov.i32	$H4,#1<<24		@ padbit, yes, always
	vmlal.u32	$D1,$H2#lo,${S4}[0]
	vmlal.u32	$D2,$H3#lo,${S4}[0]

	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
	add		$inp,$inp,#64
# ifdef	__ARMEB__
	vrev32.8	$H0,$H0
	vrev32.8	$H1,$H1
	vrev32.8	$H2,$H2
	vrev32.8	$H3,$H3
# endif

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ lazy reduction interleaved with base 2^32 -> base 2^26 of
	@ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4.

	vshr.u64	$T0,$D3,#26
	vmovn.i64	$D3#lo,$D3
	 vshr.u64	$T1,$D0,#26
	 vmovn.i64	$D0#lo,$D0
	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
	vbic.i32	$D3#lo,#0xfc000000
	  vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
	  vshl.u32	$H3,$H3,#18
	 vbic.i32	$D0#lo,#0xfc000000

	vshrn.u64	$T0#lo,$D4,#26
	vmovn.i64	$D4#lo,$D4
	 vshr.u64	$T1,$D1,#26
	 vmovn.i64	$D1#lo,$D1
	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
	  vsri.u32	$H3,$H2,#14
	vbic.i32	$D4#lo,#0xfc000000
	  vshl.u32	$H2,$H2,#12
	 vbic.i32	$D1#lo,#0xfc000000

	vadd.i32	$D0#lo,$D0#lo,$T0#lo
	vshl.u32	$T0#lo,$T0#lo,#2
	  vbic.i32	$H3,#0xfc000000
	 vshrn.u64	$T1#lo,$D2,#26
	 vmovn.i64	$D2#lo,$D2
	vaddl.u32	$D0,$D0#lo,$T0#lo	@ h4 -> h0 [widen for a sec]
	  vsri.u32	$H2,$H1,#20
	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
	  vshl.u32	$H1,$H1,#6
	 vbic.i32	$D2#lo,#0xfc000000
	  vbic.i32	$H2,#0xfc000000

	vshrn.u64	$T0#lo,$D0,#26		@ re-narrow
	vmovn.i64	$D0#lo,$D0
	  vsri.u32	$H1,$H0,#26
	  vbic.i32	$H0,#0xfc000000
	 vshr.u32	$T1#lo,$D3#lo,#26
	 vbic.i32	$D3#lo,#0xfc000000
	vbic.i32	$D0#lo,#0xfc000000
	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4
	  vbic.i32	$H1,#0xfc000000

	bhi		.Loop_neon

.Lskip_loop:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1

	add		$tbl1,$ctx,#(48+0*9*4)
	add		$tbl0,$ctx,#(48+1*9*4)
	adds		$len,$len,#32
	it		ne
	movne		$len,#0
	bne		.Long_tail

	vadd.i32	$H2#hi,$H2#lo,$D2#lo	@ add hash value and move to #hi
	vadd.i32	$H0#hi,$H0#lo,$D0#lo
	vadd.i32	$H3#hi,$H3#lo,$D3#lo
	vadd.i32	$H1#hi,$H1#lo,$D1#lo
	vadd.i32	$H4#hi,$H4#lo,$D4#lo

.Long_tail:
	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^1
	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^2

	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ can be redundant
	vmull.u32	$D2,$H2#hi,$R0
	vadd.i32	$H0#lo,$H0#lo,$D0#lo
	vmull.u32	$D0,$H0#hi,$R0
	vadd.i32	$H3#lo,$H3#lo,$D3#lo
	vmull.u32	$D3,$H3#hi,$R0
	vadd.i32	$H1#lo,$H1#lo,$D1#lo
	vmull.u32	$D1,$H1#hi,$R0
	vadd.i32	$H4#lo,$H4#lo,$D4#lo
	vmull.u32	$D4,$H4#hi,$R0

	vmlal.u32	$D0,$H4#hi,$S1
	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vmlal.u32	$D3,$H2#hi,$R1
	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	vmlal.u32	$D1,$H0#hi,$R1
	vmlal.u32	$D4,$H3#hi,$R1
	vmlal.u32	$D2,$H1#hi,$R1

	vmlal.u32	$D3,$H1#hi,$R2
	vld1.32		${S4}[1],[$tbl1,:32]
	vmlal.u32	$D0,$H3#hi,$S2
	vld1.32		${S4}[0],[$tbl0,:32]
	vmlal.u32	$D4,$H2#hi,$R2
	vmlal.u32	$D1,$H4#hi,$S2
	vmlal.u32	$D2,$H0#hi,$R2

	vmlal.u32	$D3,$H0#hi,$R3
	 it		ne
	 addne		$tbl1,$ctx,#(48+2*9*4)
	vmlal.u32	$D0,$H2#hi,$S3
	 it		ne
	 addne		$tbl0,$ctx,#(48+3*9*4)
	vmlal.u32	$D4,$H1#hi,$R3
	vmlal.u32	$D1,$H3#hi,$S3
	vmlal.u32	$D2,$H4#hi,$S3

	vmlal.u32	$D3,$H4#hi,$S4
	 vorn		$MASK,$MASK,$MASK	@ all-ones, can be redundant
	vmlal.u32	$D0,$H1#hi,$S4
	 vshr.u64	$MASK,$MASK,#38
	vmlal.u32	$D4,$H0#hi,$R4
	vmlal.u32	$D1,$H2#hi,$S4
	vmlal.u32	$D2,$H3#hi,$S4

	beq		.Lshort_tail

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ (hash+inp[0:1])*r^4:r^3 and accumulate

	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^3
	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4

	vmlal.u32	$D2,$H2#lo,$R0
	vmlal.u32	$D0,$H0#lo,$R0
	vmlal.u32	$D3,$H3#lo,$R0
	vmlal.u32	$D1,$H1#lo,$R0
	vmlal.u32	$D4,$H4#lo,$R0

	vmlal.u32	$D0,$H4#lo,$S1
	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vmlal.u32	$D3,$H2#lo,$R1
	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	vmlal.u32	$D1,$H0#lo,$R1
	vmlal.u32	$D4,$H3#lo,$R1
	vmlal.u32	$D2,$H1#lo,$R1

	vmlal.u32	$D3,$H1#lo,$R2
	vld1.32		${S4}[1],[$tbl1,:32]
	vmlal.u32	$D0,$H3#lo,$S2
	vld1.32		${S4}[0],[$tbl0,:32]
	vmlal.u32	$D4,$H2#lo,$R2
	vmlal.u32	$D1,$H4#lo,$S2
	vmlal.u32	$D2,$H0#lo,$R2

	vmlal.u32	$D3,$H0#lo,$R3
	vmlal.u32	$D0,$H2#lo,$S3
	vmlal.u32	$D4,$H1#lo,$R3
	vmlal.u32	$D1,$H3#lo,$S3
	vmlal.u32	$D2,$H4#lo,$S3

	vmlal.u32	$D3,$H4#lo,$S4
	 vorn		$MASK,$MASK,$MASK	@ all-ones
	vmlal.u32	$D0,$H1#lo,$S4
	 vshr.u64	$MASK,$MASK,#38
	vmlal.u32	$D4,$H0#lo,$R4
	vmlal.u32	$D1,$H2#lo,$S4
	vmlal.u32	$D2,$H3#lo,$S4

.Lshort_tail:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ horizontal addition

	vadd.i64	$D3#lo,$D3#lo,$D3#hi
	vadd.i64	$D0#lo,$D0#lo,$D0#hi
	vadd.i64	$D4#lo,$D4#lo,$D4#hi
	vadd.i64	$D1#lo,$D1#lo,$D1#hi
	vadd.i64	$D2#lo,$D2#lo,$D2#hi

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ lazy reduction, but without narrowing

	vshr.u64	$T0,$D3,#26
	vand.i64	$D3,$D3,$MASK
	 vshr.u64	$T1,$D0,#26
	 vand.i64	$D0,$D0,$MASK
	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1

	vshr.u64	$T0,$D4,#26
	vand.i64	$D4,$D4,$MASK
	 vshr.u64	$T1,$D1,#26
	 vand.i64	$D1,$D1,$MASK
	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2

	vadd.i64	$D0,$D0,$T0
	vshl.u64	$T0,$T0,#2
	 vshr.u64	$T1,$D2,#26
	 vand.i64	$D2,$D2,$MASK
	vadd.i64	$D0,$D0,$T0		@ h4 -> h0
	 vadd.i64	$D3,$D3,$T1		@ h2 -> h3

	vshr.u64	$T0,$D0,#26
	vand.i64	$D0,$D0,$MASK
	 vshr.u64	$T1,$D3,#26
	 vand.i64	$D3,$D3,$MASK
	vadd.i64	$D1,$D1,$T0		@ h0 -> h1
	 vadd.i64	$D4,$D4,$T1		@ h3 -> h4

	cmp		$len,#0
	bne		.Leven

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ store hash value

	vst4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
	vst1.32		{$D4#lo[0]},[$ctx]

	vldmia	sp!,{d8-d15}			@ epilogue
	ldmia	sp!,{r4-r7}
	ret					@ bx	lr
.size	poly1305_blocks_neon,.-poly1305_blocks_neon

.align	5
.Lzeros:
.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
#ifndef	__KERNEL__
.LOPENSSL_armcap:
# ifdef	_WIN32
.word	OPENSSL_armcap_P
# else
.word	OPENSSL_armcap_P-.Lpoly1305_init
# endif
.comm	OPENSSL_armcap_P,4,4
.hidden	OPENSSL_armcap_P
#endif
#endif
___
}	}
$code.=<<___;
.asciz	"Poly1305 for ARMv4/NEON, CRYPTOGAMS by \@dot-asm"
.align	2
___

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/geo;

	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
	s/\bret\b/bx	lr/go						or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;	# make it possible to compile with -march=armv4

	print $_,"\n";
}
close STDOUT; # enforce flush


code:
// SPDX-License-Identifier: GPL-2.0
/*
 * OpenSSL/Cryptogams accelerated Poly1305 transform for ARM
 *
 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
 */

#include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h>
#include <asm/unaligned.h>
#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/poly1305.h>
#include <crypto/internal/simd.h>
#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/jump_label.h>
#include <linux/module.h>

void poly1305_init_arm(void *state, const u8 *key);
void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit);
void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit);
void poly1305_emit_arm(void *state, u8 *digest, const u32 *nonce);

void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit)
{
}

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);

void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
{
	poly1305_init_arm(&dctx->h, key);
	dctx->s[0] = get_unaligned_le32(key + 16);
	dctx->s[1] = get_unaligned_le32(key + 20);
	dctx->s[2] = get_unaligned_le32(key + 24);
	dctx->s[3] = get_unaligned_le32(key + 28);
	dctx->buflen = 0;
}
EXPORT_SYMBOL(poly1305_init_arch);

static int arm_poly1305_init(struct shash_desc *desc)
{
	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);

	dctx->buflen = 0;
	dctx->rset = 0;
	dctx->sset = false;

	return 0;
}

static void arm_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src,
				 u32 len, u32 hibit, bool do_neon)
{
	if (unlikely(!dctx->sset)) {
		if (!dctx->rset) {
			poly1305_init_arm(&dctx->h, src);
			src += POLY1305_BLOCK_SIZE;
			len -= POLY1305_BLOCK_SIZE;
			dctx->rset = 1;
		}
		if (len >= POLY1305_BLOCK_SIZE) {
			dctx->s[0] = get_unaligned_le32(src +  0);
			dctx->s[1] = get_unaligned_le32(src +  4);
			dctx->s[2] = get_unaligned_le32(src +  8);
			dctx->s[3] = get_unaligned_le32(src + 12);
			src += POLY1305_BLOCK_SIZE;
			len -= POLY1305_BLOCK_SIZE;
			dctx->sset = true;
		}
		if (len < POLY1305_BLOCK_SIZE)
			return;
	}

	len &= ~(POLY1305_BLOCK_SIZE - 1);

	if (static_branch_likely(&have_neon) && likely(do_neon))
		poly1305_blocks_neon(&dctx->h, src, len, hibit);
	else
		poly1305_blocks_arm(&dctx->h, src, len, hibit);
}

static void arm_poly1305_do_update(struct poly1305_desc_ctx *dctx,
				    const u8 *src, u32 len, bool do_neon)
{
	if (unlikely(dctx->buflen)) {
		u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen);

		memcpy(dctx->buf + dctx->buflen, src, bytes);
		src += bytes;
		len -= bytes;
		dctx->buflen += bytes;

		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
			arm_poly1305_blocks(dctx, dctx->buf,
					    POLY1305_BLOCK_SIZE, 1, false);
			dctx->buflen = 0;
		}
	}

	if (likely(len >= POLY1305_BLOCK_SIZE)) {
		arm_poly1305_blocks(dctx, src, len, 1, do_neon);
		src += round_down(len, POLY1305_BLOCK_SIZE);
		len %= POLY1305_BLOCK_SIZE;
	}

	if (unlikely(len)) {
		dctx->buflen = len;
		memcpy(dctx->buf, src, len);
	}
}

static int arm_poly1305_update(struct shash_desc *desc,
			       const u8 *src, unsigned int srclen)
{
	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);

	arm_poly1305_do_update(dctx, src, srclen, false);
	return 0;
}

static int __maybe_unused arm_poly1305_update_neon(struct shash_desc *desc,
						   const u8 *src,
						   unsigned int srclen)
{
	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
	bool do_neon = crypto_simd_usable() && srclen > 128;

	if (static_branch_likely(&have_neon) && do_neon)
		kernel_neon_begin();
	arm_poly1305_do_update(dctx, src, srclen, do_neon);
	if (static_branch_likely(&have_neon) && do_neon)
		kernel_neon_end();
	return 0;
}

void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
			  unsigned int nbytes)
{
	bool do_neon = IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
		       crypto_simd_usable();

	if (unlikely(dctx->buflen)) {
		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);

		memcpy(dctx->buf + dctx->buflen, src, bytes);
		src += bytes;
		nbytes -= bytes;
		dctx->buflen += bytes;

		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
			poly1305_blocks_arm(&dctx->h, dctx->buf,
					    POLY1305_BLOCK_SIZE, 1);
			dctx->buflen = 0;
		}
	}

	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
		unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);

		if (static_branch_likely(&have_neon) && do_neon) {
			do {
				unsigned int todo = min_t(unsigned int, len, SZ_4K);

				kernel_neon_begin();
				poly1305_blocks_neon(&dctx->h, src, todo, 1);
				kernel_neon_end();

				len -= todo;
				src += todo;
			} while (len);
		} else {
			poly1305_blocks_arm(&dctx->h, src, len, 1);
			src += len;
		}
		nbytes %= POLY1305_BLOCK_SIZE;
	}

	if (unlikely(nbytes)) {
		dctx->buflen = nbytes;
		memcpy(dctx->buf, src, nbytes);
	}
}
EXPORT_SYMBOL(poly1305_update_arch);

void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
{
	if (unlikely(dctx->buflen)) {
		dctx->buf[dctx->buflen++] = 1;
		memset(dctx->buf + dctx->buflen, 0,
		       POLY1305_BLOCK_SIZE - dctx->buflen);
		poly1305_blocks_arm(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
	}

	poly1305_emit_arm(&dctx->h, dst, dctx->s);
	*dctx = (struct poly1305_desc_ctx){};
}
EXPORT_SYMBOL(poly1305_final_arch);

static int arm_poly1305_final(struct shash_desc *desc, u8 *dst)
{
	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);

	if (unlikely(!dctx->sset))
		return -ENOKEY;

	poly1305_final_arch(dctx, dst);
	return 0;
}

static struct shash_alg arm_poly1305_algs[] = {{
	.init			= arm_poly1305_init,
	.update			= arm_poly1305_update,
	.final			= arm_poly1305_final,
	.digestsize		= POLY1305_DIGEST_SIZE,
	.descsize		= sizeof(struct poly1305_desc_ctx),

	.base.cra_name		= "poly1305",
	.base.cra_driver_name	= "poly1305-arm",
	.base.cra_priority	= 150,
	.base.cra_blocksize	= POLY1305_BLOCK_SIZE,
	.base.cra_module	= THIS_MODULE,
#ifdef CONFIG_KERNEL_MODE_NEON
}, {
	.init			= arm_poly1305_init,
	.update			= arm_poly1305_update_neon,
	.final			= arm_poly1305_final,
	.digestsize		= POLY1305_DIGEST_SIZE,
	.descsize		= sizeof(struct poly1305_desc_ctx),

	.base.cra_name		= "poly1305",
	.base.cra_driver_name	= "poly1305-neon",
	.base.cra_priority	= 200,
	.base.cra_blocksize	= POLY1305_BLOCK_SIZE,
	.base.cra_module	= THIS_MODULE,
#endif
}};

static int __init arm_poly1305_mod_init(void)
{
	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
	    (elf_hwcap & HWCAP_NEON))
		static_branch_enable(&have_neon);
	else if (IS_REACHABLE(CONFIG_CRYPTO_HASH))
		/* register only the first entry */
		return crypto_register_shash(&arm_poly1305_algs[0]);

	return IS_REACHABLE(CONFIG_CRYPTO_HASH) ?
		crypto_register_shashes(arm_poly1305_algs,
					ARRAY_SIZE(arm_poly1305_algs)) : 0;
}

static void __exit arm_poly1305_mod_exit(void)
{
	if (!IS_REACHABLE(CONFIG_CRYPTO_HASH))
		return;
	if (!static_branch_likely(&have_neon)) {
		crypto_unregister_shash(&arm_poly1305_algs[0]);
		return;
	}
	crypto_unregister_shashes(arm_poly1305_algs,
				  ARRAY_SIZE(arm_poly1305_algs));
}

module_init(arm_poly1305_mod_init);
module_exit(arm_poly1305_mod_exit);

MODULE_LICENSE("GPL v2");
MODULE_ALIAS_CRYPTO("poly1305");
MODULE_ALIAS_CRYPTO("poly1305-arm");
MODULE_ALIAS_CRYPTO("poly1305-neon");


code:
#define __ARM_ARCH__ __LINUX_ARM_ARCH__
@ SPDX-License-Identifier: GPL-2.0

@ This code is taken from the OpenSSL project but the author (Andy Polyakov)
@ has relicensed it under the GPLv2. Therefore this program is free software;
@ you can redistribute it and/or modify it under the terms of the GNU General
@ Public License version 2 as published by the Free Software Foundation.
@
@ The original headers, including the original license headers, are
@ included below for completeness.

@ ====================================================================
@ Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
@ project. The module is, however, dual licensed under OpenSSL and
@ CRYPTOGAMS licenses depending on where you obtain it. For further
@ details see https://www.openssl.org/~appro/cryptogams/.
@ ====================================================================

@ sha1_block procedure for ARMv4.
@
@ January 2007.

@ Size/performance trade-off
@ ====================================================================
@ impl		size in bytes	comp cycles[*]	measured performance
@ ====================================================================
@ thumb		304		3212		4420
@ armv4-small	392/+29%	1958/+64%	2250/+96%
@ armv4-compact	740/+89%	1552/+26%	1840/+22%
@ armv4-large	1420/+92%	1307/+19%	1370/+34%[***]
@ full unroll	~5100/+260%	~1260/+4%	~1300/+5%
@ ====================================================================
@ thumb		= same as 'small' but in Thumb instructions[**] and
@		  with recurring code in two private functions;
@ small		= detached Xload/update, loops are folded;
@ compact	= detached Xload/update, 5x unroll;
@ large		= interleaved Xload/update, 5x unroll;
@ full unroll	= interleaved Xload/update, full unroll, estimated[!];
@
@ [*]	Manually counted instructions in "grand" loop body. Measured
@	performance is affected by prologue and epilogue overhead,
@	i-cache availability, branch penalties, etc.
@ [**]	While each Thumb instruction is twice smaller, they are not as
@	diverse as ARM ones: e.g., there are only two arithmetic
@	instructions with 3 arguments, no [fixed] rotate, addressing
@	modes are limited. As result it takes more instructions to do
@	the same job in Thumb, therefore the code is never twice as
@	small and always slower.
@ [***]	which is also ~35% better than compiler generated code. Dual-
@	issue Cortex A8 core was measured to process input block in
@	~990 cycles.

@ August 2010.
@
@ Rescheduling for dual-issue pipeline resulted in 13% improvement on
@ Cortex A8 core and in absolute terms ~870 cycles per input block
@ [or 13.6 cycles per byte].

@ February 2011.
@
@ Profiler-assisted and platform-specific optimization resulted in 10%
@ improvement on Cortex A8 core and 12.2 cycles per byte.

#include <linux/linkage.h>

.text

.align	2
ENTRY(sha1_block_data_order)
	stmdb	sp!,{r4-r12,lr}
	add	r2,r1,r2,lsl#6	@ r2 to point at the end of r1
	ldmia	r0,{r3,r4,r5,r6,r7}
.Lloop:
	ldr	r8,.LK_00_19
	mov	r14,sp
	sub	sp,sp,#15*4
	mov	r5,r5,ror#30
	mov	r6,r6,ror#30
	mov	r7,r7,ror#30		@ [6]
.L_00_15:
#if __ARM_ARCH__<7
	ldrb	r10,[r1,#2]
	ldrb	r9,[r1,#3]
	ldrb	r11,[r1,#1]
	add	r7,r8,r7,ror#2			@ E+=K_00_19
	ldrb	r12,[r1],#4
	orr	r9,r9,r10,lsl#8
	eor	r10,r5,r6			@ F_xx_xx
	orr	r9,r9,r11,lsl#16
	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
	orr	r9,r9,r12,lsl#24
#else
	ldr	r9,[r1],#4			@ handles unaligned
	add	r7,r8,r7,ror#2			@ E+=K_00_19
	eor	r10,r5,r6			@ F_xx_xx
	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
#ifdef __ARMEL__
	rev	r9,r9				@ byte swap
#endif
#endif
	and	r10,r4,r10,ror#2
	add	r7,r7,r9			@ E+=X[i]
	eor	r10,r10,r6,ror#2		@ F_00_19(B,C,D)
	str	r9,[r14,#-4]!
	add	r7,r7,r10			@ E+=F_00_19(B,C,D)
#if __ARM_ARCH__<7
	ldrb	r10,[r1,#2]
	ldrb	r9,[r1,#3]
	ldrb	r11,[r1,#1]
	add	r6,r8,r6,ror#2			@ E+=K_00_19
	ldrb	r12,[r1],#4
	orr	r9,r9,r10,lsl#8
	eor	r10,r4,r5			@ F_xx_xx
	orr	r9,r9,r11,lsl#16
	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
	orr	r9,r9,r12,lsl#24
#else
	ldr	r9,[r1],#4			@ handles unaligned
	add	r6,r8,r6,ror#2			@ E+=K_00_19
	eor	r10,r4,r5			@ F_xx_xx
	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
#ifdef __ARMEL__
	rev	r9,r9				@ byte swap
#endif
#endif
	and	r10,r3,r10,ror#2
	add	r6,r6,r9			@ E+=X[i]
	eor	r10,r10,r5,ror#2		@ F_00_19(B,C,D)
	str	r9,[r14,#-4]!
	add	r6,r6,r10			@ E+=F_00_19(B,C,D)
#if __ARM_ARCH__<7
	ldrb	r10,[r1,#2]
	ldrb	r9,[r1,#3]
	ldrb	r11,[r1,#1]
	add	r5,r8,r5,ror#2			@ E+=K_00_19
	ldrb	r12,[r1],#4
	orr	r9,r9,r10,lsl#8
	eor	r10,r3,r4			@ F_xx_xx
	orr	r9,r9,r11,lsl#16
	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
	orr	r9,r9,r12,lsl#24
#else
	ldr	r9,[r1],#4			@ handles unaligned
	add	r5,r8,r5,ror#2			@ E+=K_00_19
	eor	r10,r3,r4			@ F_xx_xx
	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
#ifdef __ARMEL__
	rev	r9,r9				@ byte swap
#endif
#endif
	and	r10,r7,r10,ror#2
	add	r5,r5,r9			@ E+=X[i]
	eor	r10,r10,r4,ror#2		@ F_00_19(B,C,D)
	str	r9,[r14,#-4]!
	add	r5,r5,r10			@ E+=F_00_19(B,C,D)
#if __ARM_ARCH__<7
	ldrb	r10,[r1,#2]
	ldrb	r9,[r1,#3]
	ldrb	r11,[r1,#1]
	add	r4,r8,r4,ror#2			@ E+=K_00_19
	ldrb	r12,[r1],#4
	orr	r9,r9,r10,lsl#8
	eor	r10,r7,r3			@ F_xx_xx
	orr	r9,r9,r11,lsl#16
	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
	orr	r9,r9,r12,lsl#24
#else
	ldr	r9,[r1],#4			@ handles unaligned
	add	r4,r8,r4,ror#2			@ E+=K_00_19
	eor	r10,r7,r3			@ F_xx_xx
	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
#ifdef __ARMEL__
	rev	r9,r9				@ byte swap
#endif
#endif
	and	r10,r6,r10,ror#2
	add	r4,r4,r9			@ E+=X[i]
	eor	r10,r10,r3,ror#2		@ F_00_19(B,C,D)
	str	r9,[r14,#-4]!
	add	r4,r4,r10			@ E+=F_00_19(B,C,D)
#if __ARM_ARCH__<7
	ldrb	r10,[r1,#2]
	ldrb	r9,[r1,#3]
	ldrb	r11,[r1,#1]
	add	r3,r8,r3,ror#2			@ E+=K_00_19
	ldrb	r12,[r1],#4
	orr	r9,r9,r10,lsl#8
	eor	r10,r6,r7			@ F_xx_xx
	orr	r9,r9,r11,lsl#16
	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
	orr	r9,r9,r12,lsl#24
#else
	ldr	r9,[r1],#4			@ handles unaligned
	add	r3,r8,r3,ror#2			@ E+=K_00_19
	eor	r10,r6,r7			@ F_xx_xx
	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
#ifdef __ARMEL__
	rev	r9,r9				@ byte swap
#endif
#endif
	and	r10,r5,r10,ror#2
	add	r3,r3,r9			@ E+=X[i]
	eor	r10,r10,r7,ror#2		@ F_00_19(B,C,D)
	str	r9,[r14,#-4]!
	add	r3,r3,r10			@ E+=F_00_19(B,C,D)
	cmp	r14,sp
	bne	.L_00_15		@ [((11+4)*5+2)*3]
	sub	sp,sp,#25*4
#if __ARM_ARCH__<7
	ldrb	r10,[r1,#2]
	ldrb	r9,[r1,#3]
	ldrb	r11,[r1,#1]
	add	r7,r8,r7,ror#2			@ E+=K_00_19
	ldrb	r12,[r1],#4
	orr	r9,r9,r10,lsl#8
	eor	r10,r5,r6			@ F_xx_xx
	orr	r9,r9,r11,lsl#16
	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
	orr	r9,r9,r12,lsl#24
#else
	ldr	r9,[r1],#4			@ handles unaligned
	add	r7,r8,r7,ror#2			@ E+=K_00_19
	eor	r10,r5,r6			@ F_xx_xx
	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
#ifdef __ARMEL__
	rev	r9,r9				@ byte swap
#endif
#endif
	and	r10,r4,r10,ror#2
	add	r7,r7,r9			@ E+=X[i]
	eor	r10,r10,r6,ror#2		@ F_00_19(B,C,D)
	str	r9,[r14,#-4]!
	add	r7,r7,r10			@ E+=F_00_19(B,C,D)
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r6,r8,r6,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r4,r5			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r3,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r6,r6,r9			@ E+=X[i]
	eor	r10,r10,r5,ror#2		@ F_00_19(B,C,D)
	add	r6,r6,r10			@ E+=F_00_19(B,C,D)
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r5,r8,r5,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r3,r4			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r7,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r5,r5,r9			@ E+=X[i]
	eor	r10,r10,r4,ror#2		@ F_00_19(B,C,D)
	add	r5,r5,r10			@ E+=F_00_19(B,C,D)
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r4,r8,r4,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r7,r3			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r6,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r4,r4,r9			@ E+=X[i]
	eor	r10,r10,r3,ror#2		@ F_00_19(B,C,D)
	add	r4,r4,r10			@ E+=F_00_19(B,C,D)
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r3,r8,r3,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r6,r7			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r5,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r3,r3,r9			@ E+=X[i]
	eor	r10,r10,r7,ror#2		@ F_00_19(B,C,D)
	add	r3,r3,r10			@ E+=F_00_19(B,C,D)

	ldr	r8,.LK_20_39		@ [+15+16*4]
	cmn	sp,#0			@ [+3], clear carry to denote 20_39
.L_20_39_or_60_79:
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r7,r8,r7,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r5,r6			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	eor r10,r4,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r7,r7,r9			@ E+=X[i]
	add	r7,r7,r10			@ E+=F_20_39(B,C,D)
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r6,r8,r6,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r4,r5			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	eor r10,r3,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r6,r6,r9			@ E+=X[i]
	add	r6,r6,r10			@ E+=F_20_39(B,C,D)
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r5,r8,r5,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r3,r4			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	eor r10,r7,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r5,r5,r9			@ E+=X[i]
	add	r5,r5,r10			@ E+=F_20_39(B,C,D)
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r4,r8,r4,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r7,r3			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	eor r10,r6,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r4,r4,r9			@ E+=X[i]
	add	r4,r4,r10			@ E+=F_20_39(B,C,D)
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r3,r8,r3,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r6,r7			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	eor r10,r5,r10,ror#2					@ F_xx_xx
						@ F_xx_xx
	add	r3,r3,r9			@ E+=X[i]
	add	r3,r3,r10			@ E+=F_20_39(B,C,D)
 ARM(	teq	r14,sp		)	@ preserve carry
 THUMB(	mov	r11,sp		)
 THUMB(	teq	r14,r11		)	@ preserve carry
	bne	.L_20_39_or_60_79	@ [+((12+3)*5+2)*4]
	bcs	.L_done			@ [+((12+3)*5+2)*4], spare 300 bytes

	ldr	r8,.LK_40_59
	sub	sp,sp,#20*4		@ [+2]
.L_40_59:
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r7,r8,r7,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r5,r6			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r4,r10,ror#2					@ F_xx_xx
	and r11,r5,r6					@ F_xx_xx
	add	r7,r7,r9			@ E+=X[i]
	add	r7,r7,r10			@ E+=F_40_59(B,C,D)
	add	r7,r7,r11,ror#2
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r6,r8,r6,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r4,r5			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r3,r10,ror#2					@ F_xx_xx
	and r11,r4,r5					@ F_xx_xx
	add	r6,r6,r9			@ E+=X[i]
	add	r6,r6,r10			@ E+=F_40_59(B,C,D)
	add	r6,r6,r11,ror#2
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r5,r8,r5,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r3,r4			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r7,r10,ror#2					@ F_xx_xx
	and r11,r3,r4					@ F_xx_xx
	add	r5,r5,r9			@ E+=X[i]
	add	r5,r5,r10			@ E+=F_40_59(B,C,D)
	add	r5,r5,r11,ror#2
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r4,r8,r4,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r7,r3			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r6,r10,ror#2					@ F_xx_xx
	and r11,r7,r3					@ F_xx_xx
	add	r4,r4,r9			@ E+=X[i]
	add	r4,r4,r10			@ E+=F_40_59(B,C,D)
	add	r4,r4,r11,ror#2
	ldr	r9,[r14,#15*4]
	ldr	r10,[r14,#13*4]
	ldr	r11,[r14,#7*4]
	add	r3,r8,r3,ror#2			@ E+=K_xx_xx
	ldr	r12,[r14,#2*4]
	eor	r9,r9,r10
	eor	r11,r11,r12			@ 1 cycle stall
	eor	r10,r6,r7			@ F_xx_xx
	mov	r9,r9,ror#31
	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
	eor	r9,r9,r11,ror#31
	str	r9,[r14,#-4]!
	and r10,r5,r10,ror#2					@ F_xx_xx
	and r11,r6,r7					@ F_xx_xx
	add	r3,r3,r9			@ E+=X[i]
	add	r3,r3,r10			@ E+=F_40_59(B,C,D)
	add	r3,r3,r11,ror#2
	cmp	r14,sp
	bne	.L_40_59		@ [+((12+5)*5+2)*4]

	ldr	r8,.LK_60_79
	sub	sp,sp,#20*4
	cmp	sp,#0			@ set carry to denote 60_79
	b	.L_20_39_or_60_79	@ [+4], spare 300 bytes
.L_done:
	add	sp,sp,#80*4		@ "deallocate" stack frame
	ldmia	r0,{r8,r9,r10,r11,r12}
	add	r3,r8,r3
	add	r4,r9,r4
	add	r5,r10,r5,ror#2
	add	r6,r11,r6,ror#2
	add	r7,r12,r7,ror#2
	stmia	r0,{r3,r4,r5,r6,r7}
	teq	r1,r2
	bne	.Lloop			@ [+18], total 1307

	ldmia	sp!,{r4-r12,pc}
.align	2
.LK_00_19:	.word	0x5a827999
.LK_20_39:	.word	0x6ed9eba1
.LK_40_59:	.word	0x8f1bbcdc
.LK_60_79:	.word	0xca62c1d6
ENDPROC(sha1_block_data_order)
.asciz	"SHA1 block transform for ARMv4, CRYPTOGAMS by <appro@openssl.org>"
.align	2


code:
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* sha1-armv7-neon.S - ARM/NEON accelerated SHA-1 transform function
 *
 * Copyright © 2013-2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

.syntax unified
.fpu neon

.text

/* Context structure */

#define state_h0 0
#define state_h1 4
#define state_h2 8
#define state_h3 12
#define state_h4 16

/* Constants */

#define K1  0x5A827999
#define K2  0x6ED9EBA1
#define K3  0x8F1BBCDC
#define K4  0xCA62C1D6
.align 4
.LK_VEC:
.LK1:	.long K1, K1, K1, K1
.LK2:	.long K2, K2, K2, K2
.LK3:	.long K3, K3, K3, K3
.LK4:	.long K4, K4, K4, K4

/* Register macros */

#define RSTATE r0
#define RDATA r1
#define RNBLKS r2
#define ROLDSTACK r3
#define RWK lr

#define _a r4
#define _b r5
#define _c r6
#define _d r7
#define _e r8

#define RT0 r9
#define RT1 r10
#define RT2 r11
#define RT3 r12

#define W0 q0
#define W1 q7
#define W2 q2
#define W3 q3
#define W4 q4
#define W5 q6
#define W6 q5
#define W7 q1

#define tmp0 q8
#define tmp1 q9
#define tmp2 q10
#define tmp3 q11

#define qK1 q12
#define qK2 q13
#define qK3 q14
#define qK4 q15

#ifdef CONFIG_CPU_BIG_ENDIAN
#define ARM_LE(code...)
#else
#define ARM_LE(code...)		code
#endif

/* Round function macros. */

#define WK_offs(i) (((i) & 15) * 4)

#define _R_F1(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
	      W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	ldr RT3, [sp, WK_offs(i)]; \
		pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	bic RT0, d, b; \
	add e, e, a, ror #(32 - 5); \
	and RT1, c, b; \
		pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	add RT0, RT0, RT3; \
	add e, e, RT1; \
	ror b, #(32 - 30); \
		pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	add e, e, RT0;

#define _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
	      W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	ldr RT3, [sp, WK_offs(i)]; \
		pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	eor RT0, d, b; \
	add e, e, a, ror #(32 - 5); \
	eor RT0, RT0, c; \
		pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	add e, e, RT3; \
	ror b, #(32 - 30); \
		pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	add e, e, RT0; \

#define _R_F3(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
	      W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	ldr RT3, [sp, WK_offs(i)]; \
		pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	eor RT0, b, c; \
	and RT1, b, c; \
	add e, e, a, ror #(32 - 5); \
		pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	and RT0, RT0, d; \
	add RT1, RT1, RT3; \
	add e, e, RT0; \
	ror b, #(32 - 30); \
		pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
	add e, e, RT1;

#define _R_F4(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
	      W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	_R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
	      W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28)

#define _R(a,b,c,d,e,f,i,pre1,pre2,pre3,i16,\
           W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	_R_##f(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
	       W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28)

#define R(a,b,c,d,e,f,i) \
	_R_##f(a,b,c,d,e,i,dummy,dummy,dummy,i16,\
	       W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28)

#define dummy(...)

/* Input expansion macros. */

/********* Precalc macros for rounds 0-15 *************************************/

#define W_PRECALC_00_15() \
	add       RWK, sp, #(WK_offs(0));			\
	\
	vld1.32   {W0, W7}, [RDATA]!;				\
 ARM_LE(vrev32.8  W0, W0;	)	/* big => little */	\
	vld1.32   {W6, W5}, [RDATA]!;				\
	vadd.u32  tmp0, W0, curK;				\
 ARM_LE(vrev32.8  W7, W7;	)	/* big => little */	\
 ARM_LE(vrev32.8  W6, W6;	)	/* big => little */	\
	vadd.u32  tmp1, W7, curK;				\
 ARM_LE(vrev32.8  W5, W5;	)	/* big => little */	\
	vadd.u32  tmp2, W6, curK;				\
	vst1.32   {tmp0, tmp1}, [RWK]!;				\
	vadd.u32  tmp3, W5, curK;				\
	vst1.32   {tmp2, tmp3}, [RWK];				\

#define WPRECALC_00_15_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vld1.32   {W0, W7}, [RDATA]!;				\

#define WPRECALC_00_15_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	add       RWK, sp, #(WK_offs(0));			\

#define WPRECALC_00_15_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 ARM_LE(vrev32.8  W0, W0;	)	/* big => little */	\

#define WPRECALC_00_15_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vld1.32   {W6, W5}, [RDATA]!;				\

#define WPRECALC_00_15_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vadd.u32  tmp0, W0, curK;				\

#define WPRECALC_00_15_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 ARM_LE(vrev32.8  W7, W7;	)	/* big => little */	\

#define WPRECALC_00_15_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 ARM_LE(vrev32.8  W6, W6;	)	/* big => little */	\

#define WPRECALC_00_15_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vadd.u32  tmp1, W7, curK;				\

#define WPRECALC_00_15_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 ARM_LE(vrev32.8  W5, W5;	)	/* big => little */	\

#define WPRECALC_00_15_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vadd.u32  tmp2, W6, curK;				\

#define WPRECALC_00_15_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vst1.32   {tmp0, tmp1}, [RWK]!;				\

#define WPRECALC_00_15_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vadd.u32  tmp3, W5, curK;				\

#define WPRECALC_00_15_12(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vst1.32   {tmp2, tmp3}, [RWK];				\

/********* Precalc macros for rounds 16-31 ************************************/

#define WPRECALC_16_31_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	veor      tmp0, tmp0;			\
	vext.8    W, W_m16, W_m12, #8;		\

#define WPRECALC_16_31_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	add       RWK, sp, #(WK_offs(i));	\
	vext.8    tmp0, W_m04, tmp0, #4;	\

#define WPRECALC_16_31_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	veor      tmp0, tmp0, W_m16;		\
	veor.32   W, W, W_m08;			\

#define WPRECALC_16_31_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	veor      tmp1, tmp1;			\
	veor      W, W, tmp0;			\

#define WPRECALC_16_31_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vshl.u32  tmp0, W, #1;			\

#define WPRECALC_16_31_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vext.8    tmp1, tmp1, W, #(16-12);	\
	vshr.u32  W, W, #31;			\

#define WPRECALC_16_31_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vorr      tmp0, tmp0, W;		\
	vshr.u32  W, tmp1, #30;			\

#define WPRECALC_16_31_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vshl.u32  tmp1, tmp1, #2;		\

#define WPRECALC_16_31_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	veor      tmp0, tmp0, W;		\

#define WPRECALC_16_31_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	veor      W, tmp0, tmp1;		\

#define WPRECALC_16_31_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vadd.u32  tmp0, W, curK;		\

#define WPRECALC_16_31_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vst1.32   {tmp0}, [RWK];

/********* Precalc macros for rounds 32-79 ************************************/

#define WPRECALC_32_79_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	veor W, W_m28; \

#define WPRECALC_32_79_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vext.8 tmp0, W_m08, W_m04, #8; \

#define WPRECALC_32_79_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	veor W, W_m16; \

#define WPRECALC_32_79_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	veor W, tmp0; \

#define WPRECALC_32_79_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	add RWK, sp, #(WK_offs(i&~3)); \

#define WPRECALC_32_79_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vshl.u32 tmp1, W, #2; \

#define WPRECALC_32_79_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vshr.u32 tmp0, W, #30; \

#define WPRECALC_32_79_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vorr W, tmp0, tmp1; \

#define WPRECALC_32_79_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vadd.u32 tmp0, W, curK; \

#define WPRECALC_32_79_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
	vst1.32 {tmp0}, [RWK];

/*
 * Transform nblks*64 bytes (nblks*16 32-bit words) at DATA.
 *
 * unsigned int
 * sha1_transform_neon (void *ctx, const unsigned char *data,
 *                      unsigned int nblks)
 */
.align 3
ENTRY(sha1_transform_neon)
  /* input:
   *	r0: ctx, CTX
   *	r1: data (64*nblks bytes)
   *	r2: nblks
   */

  cmp RNBLKS, #0;
  beq .Ldo_nothing;

  push {r4-r12, lr};
  /*vpush {q4-q7};*/

  adr RT3, .LK_VEC;

  mov ROLDSTACK, sp;

  /* Align stack. */
  sub RT0, sp, #(16*4);
  and RT0, #(~(16-1));
  mov sp, RT0;

  vld1.32 {qK1-qK2}, [RT3]!; /* Load K1,K2 */

  /* Get the values of the chaining variables. */
  ldm RSTATE, {_a-_e};

  vld1.32 {qK3-qK4}, [RT3]; /* Load K3,K4 */

#undef curK
#define curK qK1
  /* Precalc 0-15. */
  W_PRECALC_00_15();

.Loop:
  /* Transform 0-15 + Precalc 16-31. */
  _R( _a, _b, _c, _d, _e, F1,  0,
      WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 16,
      W4, W5, W6, W7, W0, _, _, _ );
  _R( _e, _a, _b, _c, _d, F1,  1,
      WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 16,
      W4, W5, W6, W7, W0, _, _, _ );
  _R( _d, _e, _a, _b, _c, F1,  2,
      WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 16,
      W4, W5, W6, W7, W0, _, _, _ );
  _R( _c, _d, _e, _a, _b, F1,  3,
      WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,16,
      W4, W5, W6, W7, W0, _, _, _ );

#undef curK
#define curK qK2
  _R( _b, _c, _d, _e, _a, F1,  4,
      WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 20,
      W3, W4, W5, W6, W7, _, _, _ );
  _R( _a, _b, _c, _d, _e, F1,  5,
      WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 20,
      W3, W4, W5, W6, W7, _, _, _ );
  _R( _e, _a, _b, _c, _d, F1,  6,
      WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 20,
      W3, W4, W5, W6, W7, _, _, _ );
  _R( _d, _e, _a, _b, _c, F1,  7,
      WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,20,
      W3, W4, W5, W6, W7, _, _, _ );

  _R( _c, _d, _e, _a, _b, F1,  8,
      WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 24,
      W2, W3, W4, W5, W6, _, _, _ );
  _R( _b, _c, _d, _e, _a, F1,  9,
      WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 24,
      W2, W3, W4, W5, W6, _, _, _ );
  _R( _a, _b, _c, _d, _e, F1, 10,
      WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 24,
      W2, W3, W4, W5, W6, _, _, _ );
  _R( _e, _a, _b, _c, _d, F1, 11,
      WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,24,
      W2, W3, W4, W5, W6, _, _, _ );

  _R( _d, _e, _a, _b, _c, F1, 12,
      WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 28,
      W1, W2, W3, W4, W5, _, _, _ );
  _R( _c, _d, _e, _a, _b, F1, 13,
      WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 28,
      W1, W2, W3, W4, W5, _, _, _ );
  _R( _b, _c, _d, _e, _a, F1, 14,
      WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 28,
      W1, W2, W3, W4, W5, _, _, _ );
  _R( _a, _b, _c, _d, _e, F1, 15,
      WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,28,
      W1, W2, W3, W4, W5, _, _, _ );

  /* Transform 16-63 + Precalc 32-79. */
  _R( _e, _a, _b, _c, _d, F1, 16,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 32,
      W0, W1, W2, W3, W4, W5, W6, W7);
  _R( _d, _e, _a, _b, _c, F1, 17,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 32,
      W0, W1, W2, W3, W4, W5, W6, W7);
  _R( _c, _d, _e, _a, _b, F1, 18,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 32,
      W0, W1, W2, W3, W4, W5, W6, W7);
  _R( _b, _c, _d, _e, _a, F1, 19,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 32,
      W0, W1, W2, W3, W4, W5, W6, W7);

  _R( _a, _b, _c, _d, _e, F2, 20,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 36,
      W7, W0, W1, W2, W3, W4, W5, W6);
  _R( _e, _a, _b, _c, _d, F2, 21,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 36,
      W7, W0, W1, W2, W3, W4, W5, W6);
  _R( _d, _e, _a, _b, _c, F2, 22,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 36,
      W7, W0, W1, W2, W3, W4, W5, W6);
  _R( _c, _d, _e, _a, _b, F2, 23,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 36,
      W7, W0, W1, W2, W3, W4, W5, W6);

#undef curK
#define curK qK3
  _R( _b, _c, _d, _e, _a, F2, 24,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 40,
      W6, W7, W0, W1, W2, W3, W4, W5);
  _R( _a, _b, _c, _d, _e, F2, 25,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 40,
      W6, W7, W0, W1, W2, W3, W4, W5);
  _R( _e, _a, _b, _c, _d, F2, 26,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 40,
      W6, W7, W0, W1, W2, W3, W4, W5);
  _R( _d, _e, _a, _b, _c, F2, 27,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 40,
      W6, W7, W0, W1, W2, W3, W4, W5);

  _R( _c, _d, _e, _a, _b, F2, 28,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 44,
      W5, W6, W7, W0, W1, W2, W3, W4);
  _R( _b, _c, _d, _e, _a, F2, 29,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 44,
      W5, W6, W7, W0, W1, W2, W3, W4);
  _R( _a, _b, _c, _d, _e, F2, 30,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 44,
      W5, W6, W7, W0, W1, W2, W3, W4);
  _R( _e, _a, _b, _c, _d, F2, 31,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 44,
      W5, W6, W7, W0, W1, W2, W3, W4);

  _R( _d, _e, _a, _b, _c, F2, 32,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 48,
      W4, W5, W6, W7, W0, W1, W2, W3);
  _R( _c, _d, _e, _a, _b, F2, 33,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 48,
      W4, W5, W6, W7, W0, W1, W2, W3);
  _R( _b, _c, _d, _e, _a, F2, 34,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 48,
      W4, W5, W6, W7, W0, W1, W2, W3);
  _R( _a, _b, _c, _d, _e, F2, 35,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 48,
      W4, W5, W6, W7, W0, W1, W2, W3);

  _R( _e, _a, _b, _c, _d, F2, 36,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 52,
      W3, W4, W5, W6, W7, W0, W1, W2);
  _R( _d, _e, _a, _b, _c, F2, 37,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 52,
      W3, W4, W5, W6, W7, W0, W1, W2);
  _R( _c, _d, _e, _a, _b, F2, 38,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 52,
      W3, W4, W5, W6, W7, W0, W1, W2);
  _R( _b, _c, _d, _e, _a, F2, 39,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 52,
      W3, W4, W5, W6, W7, W0, W1, W2);

  _R( _a, _b, _c, _d, _e, F3, 40,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 56,
      W2, W3, W4, W5, W6, W7, W0, W1);
  _R( _e, _a, _b, _c, _d, F3, 41,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 56,
      W2, W3, W4, W5, W6, W7, W0, W1);
  _R( _d, _e, _a, _b, _c, F3, 42,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 56,
      W2, W3, W4, W5, W6, W7, W0, W1);
  _R( _c, _d, _e, _a, _b, F3, 43,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 56,
      W2, W3, W4, W5, W6, W7, W0, W1);

#undef curK
#define curK qK4
  _R( _b, _c, _d, _e, _a, F3, 44,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 60,
      W1, W2, W3, W4, W5, W6, W7, W0);
  _R( _a, _b, _c, _d, _e, F3, 45,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 60,
      W1, W2, W3, W4, W5, W6, W7, W0);
  _R( _e, _a, _b, _c, _d, F3, 46,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 60,
      W1, W2, W3, W4, W5, W6, W7, W0);
  _R( _d, _e, _a, _b, _c, F3, 47,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 60,
      W1, W2, W3, W4, W5, W6, W7, W0);

  _R( _c, _d, _e, _a, _b, F3, 48,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 64,
      W0, W1, W2, W3, W4, W5, W6, W7);
  _R( _b, _c, _d, _e, _a, F3, 49,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 64,
      W0, W1, W2, W3, W4, W5, W6, W7);
  _R( _a, _b, _c, _d, _e, F3, 50,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 64,
      W0, W1, W2, W3, W4, W5, W6, W7);
  _R( _e, _a, _b, _c, _d, F3, 51,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 64,
      W0, W1, W2, W3, W4, W5, W6, W7);

  _R( _d, _e, _a, _b, _c, F3, 52,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 68,
      W7, W0, W1, W2, W3, W4, W5, W6);
  _R( _c, _d, _e, _a, _b, F3, 53,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 68,
      W7, W0, W1, W2, W3, W4, W5, W6);
  _R( _b, _c, _d, _e, _a, F3, 54,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 68,
      W7, W0, W1, W2, W3, W4, W5, W6);
  _R( _a, _b, _c, _d, _e, F3, 55,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 68,
      W7, W0, W1, W2, W3, W4, W5, W6);

  _R( _e, _a, _b, _c, _d, F3, 56,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 72,
      W6, W7, W0, W1, W2, W3, W4, W5);
  _R( _d, _e, _a, _b, _c, F3, 57,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 72,
      W6, W7, W0, W1, W2, W3, W4, W5);
  _R( _c, _d, _e, _a, _b, F3, 58,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 72,
      W6, W7, W0, W1, W2, W3, W4, W5);
  _R( _b, _c, _d, _e, _a, F3, 59,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 72,
      W6, W7, W0, W1, W2, W3, W4, W5);

  subs RNBLKS, #1;

  _R( _a, _b, _c, _d, _e, F4, 60,
      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 76,
      W5, W6, W7, W0, W1, W2, W3, W4);
  _R( _e, _a, _b, _c, _d, F4, 61,
      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 76,
      W5, W6, W7, W0, W1, W2, W3, W4);
  _R( _d, _e, _a, _b, _c, F4, 62,
      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 76,
      W5, W6, W7, W0, W1, W2, W3, W4);
  _R( _c, _d, _e, _a, _b, F4, 63,
      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 76,
      W5, W6, W7, W0, W1, W2, W3, W4);

  beq .Lend;

  /* Transform 64-79 + Precalc 0-15 of next block. */
#undef curK
#define curK qK1
  _R( _b, _c, _d, _e, _a, F4, 64,
      WPRECALC_00_15_0, dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _a, _b, _c, _d, _e, F4, 65,
      WPRECALC_00_15_1, dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _e, _a, _b, _c, _d, F4, 66,
      WPRECALC_00_15_2, dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _d, _e, _a, _b, _c, F4, 67,
      WPRECALC_00_15_3, dummy, dummy, _, _, _, _, _, _, _, _, _ );

  _R( _c, _d, _e, _a, _b, F4, 68,
      dummy,            dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _b, _c, _d, _e, _a, F4, 69,
      dummy,            dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _a, _b, _c, _d, _e, F4, 70,
      WPRECALC_00_15_4, dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _e, _a, _b, _c, _d, F4, 71,
      WPRECALC_00_15_5, dummy, dummy, _, _, _, _, _, _, _, _, _ );

  _R( _d, _e, _a, _b, _c, F4, 72,
      dummy,            dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _c, _d, _e, _a, _b, F4, 73,
      dummy,            dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _b, _c, _d, _e, _a, F4, 74,
      WPRECALC_00_15_6, dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _a, _b, _c, _d, _e, F4, 75,
      WPRECALC_00_15_7, dummy, dummy, _, _, _, _, _, _, _, _, _ );

  _R( _e, _a, _b, _c, _d, F4, 76,
      WPRECALC_00_15_8, dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _d, _e, _a, _b, _c, F4, 77,
      WPRECALC_00_15_9, dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _c, _d, _e, _a, _b, F4, 78,
      WPRECALC_00_15_10, dummy, dummy, _, _, _, _, _, _, _, _, _ );
  _R( _b, _c, _d, _e, _a, F4, 79,
      WPRECALC_00_15_11, dummy, WPRECALC_00_15_12, _, _, _, _, _, _, _, _, _ );

  /* Update the chaining variables. */
  ldm RSTATE, {RT0-RT3};
  add _a, RT0;
  ldr RT0, [RSTATE, #state_h4];
  add _b, RT1;
  add _c, RT2;
  add _d, RT3;
  add _e, RT0;
  stm RSTATE, {_a-_e};

  b .Loop;

.Lend:
  /* Transform 64-79 */
  R( _b, _c, _d, _e, _a, F4, 64 );
  R( _a, _b, _c, _d, _e, F4, 65 );
  R( _e, _a, _b, _c, _d, F4, 66 );
  R( _d, _e, _a, _b, _c, F4, 67 );
  R( _c, _d, _e, _a, _b, F4, 68 );
  R( _b, _c, _d, _e, _a, F4, 69 );
  R( _a, _b, _c, _d, _e, F4, 70 );
  R( _e, _a, _b, _c, _d, F4, 71 );
  R( _d, _e, _a, _b, _c, F4, 72 );
  R( _c, _d, _e, _a, _b, F4, 73 );
  R( _b, _c, _d, _e, _a, F4, 74 );
  R( _a, _b, _c, _d, _e, F4, 75 );
  R( _e, _a, _b, _c, _d, F4, 76 );
  R( _d, _e, _a, _b, _c, F4, 77 );
  R( _c, _d, _e, _a, _b, F4, 78 );
  R( _b, _c, _d, _e, _a, F4, 79 );

  mov sp, ROLDSTACK;

  /* Update the chaining variables. */
  ldm RSTATE, {RT0-RT3};
  add _a, RT0;
  ldr RT0, [RSTATE, #state_h4];
  add _b, RT1;
  add _c, RT2;
  add _d, RT3;
  /*vpop {q4-q7};*/
  add _e, RT0;
  stm RSTATE, {_a-_e};

  pop {r4-r12, pc};

.Ldo_nothing:
  bx lr
ENDPROC(sha1_transform_neon)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * sha1-ce-core.S - SHA-1 secure hash using ARMv8 Crypto Extensions
 *
 * Copyright (C) 2015 Linaro Ltd.
 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

	.text
	.arch		armv8-a
	.fpu		crypto-neon-fp-armv8

	k0		.req	q0
	k1		.req	q1
	k2		.req	q2
	k3		.req	q3

	ta0		.req	q4
	ta1		.req	q5
	tb0		.req	q5
	tb1		.req	q4

	dga		.req	q6
	dgb		.req	q7
	dgbs		.req	s28

	dg0		.req	q12
	dg1a0		.req	q13
	dg1a1		.req	q14
	dg1b0		.req	q14
	dg1b1		.req	q13

	.macro		add_only, op, ev, rc, s0, dg1
	.ifnb		\s0
	vadd.u32	tb\ev, q\s0, \rc
	.endif
	sha1h.32	dg1b\ev, dg0
	.ifb		\dg1
	sha1\op\().32	dg0, dg1a\ev, ta\ev
	.else
	sha1\op\().32	dg0, \dg1, ta\ev
	.endif
	.endm

	.macro		add_update, op, ev, rc, s0, s1, s2, s3, dg1
	sha1su0.32	q\s0, q\s1, q\s2
	add_only	\op, \ev, \rc, \s1, \dg1
	sha1su1.32	q\s0, q\s3
	.endm

	.align		6
.Lsha1_rcon:
	.word		0x5a827999, 0x5a827999, 0x5a827999, 0x5a827999
	.word		0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1
	.word		0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc
	.word		0xca62c1d6, 0xca62c1d6, 0xca62c1d6, 0xca62c1d6

	/*
	 * void sha1_ce_transform(struct sha1_state *sst, u8 const *src,
	 *			  int blocks);
	 */
ENTRY(sha1_ce_transform)
	/* load round constants */
	adr		ip, .Lsha1_rcon
	vld1.32		{k0-k1}, [ip, :128]!
	vld1.32		{k2-k3}, [ip, :128]

	/* load state */
	vld1.32		{dga}, [r0]
	vldr		dgbs, [r0, #16]

	/* load input */
0:	vld1.32		{q8-q9}, [r1]!
	vld1.32		{q10-q11}, [r1]!
	subs		r2, r2, #1

#ifndef CONFIG_CPU_BIG_ENDIAN
	vrev32.8	q8, q8
	vrev32.8	q9, q9
	vrev32.8	q10, q10
	vrev32.8	q11, q11
#endif

	vadd.u32	ta0, q8, k0
	vmov		dg0, dga

	add_update	c, 0, k0,  8,  9, 10, 11, dgb
	add_update	c, 1, k0,  9, 10, 11,  8
	add_update	c, 0, k0, 10, 11,  8,  9
	add_update	c, 1, k0, 11,  8,  9, 10
	add_update	c, 0, k1,  8,  9, 10, 11

	add_update	p, 1, k1,  9, 10, 11,  8
	add_update	p, 0, k1, 10, 11,  8,  9
	add_update	p, 1, k1, 11,  8,  9, 10
	add_update	p, 0, k1,  8,  9, 10, 11
	add_update	p, 1, k2,  9, 10, 11,  8

	add_update	m, 0, k2, 10, 11,  8,  9
	add_update	m, 1, k2, 11,  8,  9, 10
	add_update	m, 0, k2,  8,  9, 10, 11
	add_update	m, 1, k2,  9, 10, 11,  8
	add_update	m, 0, k3, 10, 11,  8,  9

	add_update	p, 1, k3, 11,  8,  9, 10
	add_only	p, 0, k3,  9
	add_only	p, 1, k3, 10
	add_only	p, 0, k3, 11
	add_only	p, 1

	/* update state */
	vadd.u32	dga, dga, dg0
	vadd.u32	dgb, dgb, dg1a0
	bne		0b

	/* store new state */
	vst1.32		{dga}, [r0]
	vstr		dgbs, [r0, #16]
	bx		lr
ENDPROC(sha1_ce_transform)