code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions
 *
 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <crypto/sha1.h>
#include <crypto/sha1_base.h>
#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/module.h>

#include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h>

#include "sha1.h"

MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");

asmlinkage void sha1_ce_transform(struct sha1_state *sst, u8 const *src,
				  int blocks);

static int sha1_ce_update(struct shash_desc *desc, const u8 *data,
			  unsigned int len)
{
	struct sha1_state *sctx = shash_desc_ctx(desc);

	if (!crypto_simd_usable() ||
	    (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
		return sha1_update_arm(desc, data, len);

	kernel_neon_begin();
	sha1_base_do_update(desc, data, len, sha1_ce_transform);
	kernel_neon_end();

	return 0;
}

static int sha1_ce_finup(struct shash_desc *desc, const u8 *data,
			 unsigned int len, u8 *out)
{
	if (!crypto_simd_usable())
		return sha1_finup_arm(desc, data, len, out);

	kernel_neon_begin();
	if (len)
		sha1_base_do_update(desc, data, len, sha1_ce_transform);
	sha1_base_do_finalize(desc, sha1_ce_transform);
	kernel_neon_end();

	return sha1_base_finish(desc, out);
}

static int sha1_ce_final(struct shash_desc *desc, u8 *out)
{
	return sha1_ce_finup(desc, NULL, 0, out);
}

static struct shash_alg alg = {
	.init			= sha1_base_init,
	.update			= sha1_ce_update,
	.final			= sha1_ce_final,
	.finup			= sha1_ce_finup,
	.descsize		= sizeof(struct sha1_state),
	.digestsize		= SHA1_DIGEST_SIZE,
	.base			= {
		.cra_name		= "sha1",
		.cra_driver_name	= "sha1-ce",
		.cra_priority		= 200,
		.cra_blocksize		= SHA1_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,
	}
};

static int __init sha1_ce_mod_init(void)
{
	return crypto_register_shash(&alg);
}

static void __exit sha1_ce_mod_fini(void)
{
	crypto_unregister_shash(&alg);
}

module_cpu_feature_match(SHA1, sha1_ce_mod_init);
module_exit(sha1_ce_mod_fini);


code:
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ASM_ARM_CRYPTO_SHA1_H
#define ASM_ARM_CRYPTO_SHA1_H

#include <linux/crypto.h>
#include <crypto/sha1.h>

extern int sha1_update_arm(struct shash_desc *desc, const u8 *data,
			   unsigned int len);

extern int sha1_finup_arm(struct shash_desc *desc, const u8 *data,
			   unsigned int len, u8 *out);

#endif

repo_name: angery/linux


code:
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Cryptographic API.
 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation
 *
 * This file is based on sha1_generic.c and sha1_ssse3_glue.c
 *
 * Copyright (c) Alan Smithee.
 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
 * Copyright (c) Mathias Krause <minipli@googlemail.com>
 */

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>
#include <crypto/sha1.h>
#include <crypto/sha1_base.h>
#include <asm/byteorder.h>

#include "sha1.h"

asmlinkage void sha1_block_data_order(u32 *digest,
		const unsigned char *data, unsigned int rounds);

int sha1_update_arm(struct shash_desc *desc, const u8 *data,
		    unsigned int len)
{
	/* make sure casting to sha1_block_fn() is safe */
	BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);

	return sha1_base_do_update(desc, data, len,
				   (sha1_block_fn *)sha1_block_data_order);
}
EXPORT_SYMBOL_GPL(sha1_update_arm);

static int sha1_final(struct shash_desc *desc, u8 *out)
{
	sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_block_data_order);
	return sha1_base_finish(desc, out);
}

int sha1_finup_arm(struct shash_desc *desc, const u8 *data,
		   unsigned int len, u8 *out)
{
	sha1_base_do_update(desc, data, len,
			    (sha1_block_fn *)sha1_block_data_order);
	return sha1_final(desc, out);
}
EXPORT_SYMBOL_GPL(sha1_finup_arm);

static struct shash_alg alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_update_arm,
	.final		=	sha1_final,
	.finup		=	sha1_finup_arm,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name=	"sha1-asm",
		.cra_priority	=	150,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static int __init sha1_mod_init(void)
{
	return crypto_register_shash(&alg);
}

static void __exit sha1_mod_fini(void)
{
	crypto_unregister_shash(&alg);
}

module_init(sha1_mod_init);
module_exit(sha1_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm (ARM)");
MODULE_ALIAS_CRYPTO("sha1");
MODULE_AUTHOR("David McCullough <ucdevel@gmail.com>");


code:
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
 * ARM NEON instructions.
 *
 * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This file is based on sha1_generic.c and sha1_ssse3_glue.c:
 *  Copyright (c) Alan Smithee.
 *  Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
 *  Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
 *  Copyright (c) Mathias Krause <minipli@googlemail.com>
 *  Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
 */

#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha1.h>
#include <crypto/sha1_base.h>
#include <asm/neon.h>
#include <asm/simd.h>

#include "sha1.h"

asmlinkage void sha1_transform_neon(void *state_h, const char *data,
				    unsigned int rounds);

static int sha1_neon_update(struct shash_desc *desc, const u8 *data,
			  unsigned int len)
{
	struct sha1_state *sctx = shash_desc_ctx(desc);

	if (!crypto_simd_usable() ||
	    (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
		return sha1_update_arm(desc, data, len);

	kernel_neon_begin();
	sha1_base_do_update(desc, data, len,
			    (sha1_block_fn *)sha1_transform_neon);
	kernel_neon_end();

	return 0;
}

static int sha1_neon_finup(struct shash_desc *desc, const u8 *data,
			   unsigned int len, u8 *out)
{
	if (!crypto_simd_usable())
		return sha1_finup_arm(desc, data, len, out);

	kernel_neon_begin();
	if (len)
		sha1_base_do_update(desc, data, len,
				    (sha1_block_fn *)sha1_transform_neon);
	sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_neon);
	kernel_neon_end();

	return sha1_base_finish(desc, out);
}

static int sha1_neon_final(struct shash_desc *desc, u8 *out)
{
	return sha1_neon_finup(desc, NULL, 0, out);
}

static struct shash_alg alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_neon_update,
	.final		=	sha1_neon_final,
	.finup		=	sha1_neon_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name		= "sha1",
		.cra_driver_name	= "sha1-neon",
		.cra_priority		= 250,
		.cra_blocksize		= SHA1_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,
	}
};

static int __init sha1_neon_mod_init(void)
{
	if (!cpu_has_neon())
		return -ENODEV;

	return crypto_register_shash(&alg);
}

static void __exit sha1_neon_mod_fini(void)
{
	crypto_unregister_shash(&alg);
}

module_init(sha1_neon_mod_init);
module_exit(sha1_neon_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, NEON accelerated");
MODULE_ALIAS_CRYPTO("sha1");


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * sha2-ce-core.S - SHA-224/256 secure hash using ARMv8 Crypto Extensions
 *
 * Copyright (C) 2015 Linaro Ltd.
 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

	.text
	.arch		armv8-a
	.fpu		crypto-neon-fp-armv8

	k0		.req	q7
	k1		.req	q8
	rk		.req	r3

	ta0		.req	q9
	ta1		.req	q10
	tb0		.req	q10
	tb1		.req	q9

	dga		.req	q11
	dgb		.req	q12

	dg0		.req	q13
	dg1		.req	q14
	dg2		.req	q15

	.macro		add_only, ev, s0
	vmov		dg2, dg0
	.ifnb		\s0
	vld1.32		{k\ev}, [rk, :128]!
	.endif
	sha256h.32	dg0, dg1, tb\ev
	sha256h2.32	dg1, dg2, tb\ev
	.ifnb		\s0
	vadd.u32	ta\ev, q\s0, k\ev
	.endif
	.endm

	.macro		add_update, ev, s0, s1, s2, s3
	sha256su0.32	q\s0, q\s1
	add_only	\ev, \s1
	sha256su1.32	q\s0, q\s2, q\s3
	.endm

	.align		6
.Lsha256_rcon:
	.word		0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
	.word		0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
	.word		0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
	.word		0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
	.word		0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
	.word		0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
	.word		0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
	.word		0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
	.word		0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
	.word		0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
	.word		0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
	.word		0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
	.word		0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
	.word		0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
	.word		0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
	.word		0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2

	/*
	 * void sha2_ce_transform(struct sha256_state *sst, u8 const *src,
				  int blocks);
	 */
ENTRY(sha2_ce_transform)
	/* load state */
	vld1.32		{dga-dgb}, [r0]

	/* load input */
0:	vld1.32		{q0-q1}, [r1]!
	vld1.32		{q2-q3}, [r1]!
	subs		r2, r2, #1

#ifndef CONFIG_CPU_BIG_ENDIAN
	vrev32.8	q0, q0
	vrev32.8	q1, q1
	vrev32.8	q2, q2
	vrev32.8	q3, q3
#endif

	/* load first round constant */
	adr		rk, .Lsha256_rcon
	vld1.32		{k0}, [rk, :128]!

	vadd.u32	ta0, q0, k0
	vmov		dg0, dga
	vmov		dg1, dgb

	add_update	1, 0, 1, 2, 3
	add_update	0, 1, 2, 3, 0
	add_update	1, 2, 3, 0, 1
	add_update	0, 3, 0, 1, 2
	add_update	1, 0, 1, 2, 3
	add_update	0, 1, 2, 3, 0
	add_update	1, 2, 3, 0, 1
	add_update	0, 3, 0, 1, 2
	add_update	1, 0, 1, 2, 3
	add_update	0, 1, 2, 3, 0
	add_update	1, 2, 3, 0, 1
	add_update	0, 3, 0, 1, 2

	add_only	1, 1
	add_only	0, 2
	add_only	1, 3
	add_only	0

	/* update state */
	vadd.u32	dga, dga, dg0
	vadd.u32	dgb, dgb, dg1
	bne		0b

	/* store new state */
	vst1.32		{dga-dgb}, [r0]
	bx		lr
ENDPROC(sha2_ce_transform)


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions
 *
 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <crypto/sha2.h>
#include <crypto/sha256_base.h>
#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/module.h>

#include <asm/hwcap.h>
#include <asm/simd.h>
#include <asm/neon.h>
#include <asm/unaligned.h>

#include "sha256_glue.h"

MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");

asmlinkage void sha2_ce_transform(struct sha256_state *sst, u8 const *src,
				  int blocks);

static int sha2_ce_update(struct shash_desc *desc, const u8 *data,
			  unsigned int len)
{
	struct sha256_state *sctx = shash_desc_ctx(desc);

	if (!crypto_simd_usable() ||
	    (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
		return crypto_sha256_arm_update(desc, data, len);

	kernel_neon_begin();
	sha256_base_do_update(desc, data, len,
			      (sha256_block_fn *)sha2_ce_transform);
	kernel_neon_end();

	return 0;
}

static int sha2_ce_finup(struct shash_desc *desc, const u8 *data,
			 unsigned int len, u8 *out)
{
	if (!crypto_simd_usable())
		return crypto_sha256_arm_finup(desc, data, len, out);

	kernel_neon_begin();
	if (len)
		sha256_base_do_update(desc, data, len,
				      (sha256_block_fn *)sha2_ce_transform);
	sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform);
	kernel_neon_end();

	return sha256_base_finish(desc, out);
}

static int sha2_ce_final(struct shash_desc *desc, u8 *out)
{
	return sha2_ce_finup(desc, NULL, 0, out);
}

static struct shash_alg algs[] = { {
	.init			= sha224_base_init,
	.update			= sha2_ce_update,
	.final			= sha2_ce_final,
	.finup			= sha2_ce_finup,
	.descsize		= sizeof(struct sha256_state),
	.digestsize		= SHA224_DIGEST_SIZE,
	.base			= {
		.cra_name		= "sha224",
		.cra_driver_name	= "sha224-ce",
		.cra_priority		= 300,
		.cra_blocksize		= SHA256_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,
	}
}, {
	.init			= sha256_base_init,
	.update			= sha2_ce_update,
	.final			= sha2_ce_final,
	.finup			= sha2_ce_finup,
	.descsize		= sizeof(struct sha256_state),
	.digestsize		= SHA256_DIGEST_SIZE,
	.base			= {
		.cra_name		= "sha256",
		.cra_driver_name	= "sha256-ce",
		.cra_priority		= 300,
		.cra_blocksize		= SHA256_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,
	}
} };

static int __init sha2_ce_mod_init(void)
{
	return crypto_register_shashes(algs, ARRAY_SIZE(algs));
}

static void __exit sha2_ce_mod_fini(void)
{
	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
}

module_cpu_feature_match(SHA2, sha2_ce_mod_init);
module_exit(sha2_ce_mod_fini);


code:
#!/usr/bin/env perl
# SPDX-License-Identifier: GPL-2.0

# This code is taken from the OpenSSL project but the author (Andy Polyakov)
# has relicensed it under the GPLv2. Therefore this program is free software;
# you can redistribute it and/or modify it under the terms of the GNU General
# Public License version 2 as published by the Free Software Foundation.
#
# The original headers, including the original license headers, are
# included below for completeness.

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see https://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# SHA256 block procedure for ARMv4. May 2007.

# Performance is ~2x better than gcc 3.4 generated code and in "abso-
# lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
# byte [on single-issue Xscale PXA250 core].

# July 2010.
#
# Rescheduling for dual-issue pipeline resulted in 22% improvement on
# Cortex A8 core and ~20 cycles per processed byte.

# February 2011.
#
# Profiler-assisted and platform-specific optimization resulted in 16%
# improvement on Cortex A8 core and ~15.4 cycles per processed byte.

# September 2013.
#
# Add NEON implementation. On Cortex A8 it was measured to process one
# byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
# S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
# code (meaning that latter performs sub-optimally, nothing was done
# about it).

# May 2014.
#
# Add ARMv8 code path performing at 2.0 cpb on Apple A7.

while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

$ctx="r0";	$t0="r0";
$inp="r1";	$t4="r1";
$len="r2";	$t1="r2";
$T1="r3";	$t3="r3";
$A="r4";
$B="r5";
$C="r6";
$D="r7";
$E="r8";
$F="r9";
$G="r10";
$H="r11";
@V=($A,$B,$C,$D,$E,$F,$G,$H);
$t2="r12";
$Ktbl="r14";

@Sigma0=( 2,13,22);
@Sigma1=( 6,11,25);
@sigma0=( 7,18, 3);
@sigma1=(17,19,10);

sub BODY_00_15 {
my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;

$code.=<<___ if ($i<16);
#if __ARM_ARCH__>=7
	@ ldr	$t1,[$inp],#4			@ $i
# if $i==15
	str	$inp,[sp,#17*4]			@ make room for $t4
# endif
	eor	$t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
	add	$a,$a,$t2			@ h+=Maj(a,b,c) from the past
	eor	$t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`	@ Sigma1(e)
# ifndef __ARMEB__
	rev	$t1,$t1
# endif
#else
	@ ldrb	$t1,[$inp,#3]			@ $i
	add	$a,$a,$t2			@ h+=Maj(a,b,c) from the past
	ldrb	$t2,[$inp,#2]
	ldrb	$t0,[$inp,#1]
	orr	$t1,$t1,$t2,lsl#8
	ldrb	$t2,[$inp],#4
	orr	$t1,$t1,$t0,lsl#16
# if $i==15
	str	$inp,[sp,#17*4]			@ make room for $t4
# endif
	eor	$t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
	orr	$t1,$t1,$t2,lsl#24
	eor	$t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`	@ Sigma1(e)
#endif
___
$code.=<<___;
	ldr	$t2,[$Ktbl],#4			@ *K256++
	add	$h,$h,$t1			@ h+=X[i]
	str	$t1,[sp,#`$i%16`*4]
	eor	$t1,$f,$g
	add	$h,$h,$t0,ror#$Sigma1[0]	@ h+=Sigma1(e)
	and	$t1,$t1,$e
	add	$h,$h,$t2			@ h+=K256[i]
	eor	$t1,$t1,$g			@ Ch(e,f,g)
	eor	$t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
	add	$h,$h,$t1			@ h+=Ch(e,f,g)
#if $i==31
	and	$t2,$t2,#0xff
	cmp	$t2,#0xf2			@ done?
#endif
#if $i<15
# if __ARM_ARCH__>=7
	ldr	$t1,[$inp],#4			@ prefetch
# else
	ldrb	$t1,[$inp,#3]
# endif
	eor	$t2,$a,$b			@ a^b, b^c in next round
#else
	ldr	$t1,[sp,#`($i+2)%16`*4]		@ from future BODY_16_xx
	eor	$t2,$a,$b			@ a^b, b^c in next round
	ldr	$t4,[sp,#`($i+15)%16`*4]	@ from future BODY_16_xx
#endif
	eor	$t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]`	@ Sigma0(a)
	and	$t3,$t3,$t2			@ (b^c)&=(a^b)
	add	$d,$d,$h			@ d+=h
	eor	$t3,$t3,$b			@ Maj(a,b,c)
	add	$h,$h,$t0,ror#$Sigma0[0]	@ h+=Sigma0(a)
	@ add	$h,$h,$t3			@ h+=Maj(a,b,c)
___
	($t2,$t3)=($t3,$t2);
}

sub BODY_16_XX {
my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;

$code.=<<___;
	@ ldr	$t1,[sp,#`($i+1)%16`*4]		@ $i
	@ ldr	$t4,[sp,#`($i+14)%16`*4]
	mov	$t0,$t1,ror#$sigma0[0]
	add	$a,$a,$t2			@ h+=Maj(a,b,c) from the past
	mov	$t2,$t4,ror#$sigma1[0]
	eor	$t0,$t0,$t1,ror#$sigma0[1]
	eor	$t2,$t2,$t4,ror#$sigma1[1]
	eor	$t0,$t0,$t1,lsr#$sigma0[2]	@ sigma0(X[i+1])
	ldr	$t1,[sp,#`($i+0)%16`*4]
	eor	$t2,$t2,$t4,lsr#$sigma1[2]	@ sigma1(X[i+14])
	ldr	$t4,[sp,#`($i+9)%16`*4]

	add	$t2,$t2,$t0
	eor	$t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`	@ from BODY_00_15
	add	$t1,$t1,$t2
	eor	$t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`	@ Sigma1(e)
	add	$t1,$t1,$t4			@ X[i]
___
	&BODY_00_15(@_);
}

$code=<<___;
#ifndef __KERNEL__
# include "arm_arch.h"
#else
# define __ARM_ARCH__ __LINUX_ARM_ARCH__
# define __ARM_MAX_ARCH__ 7
#endif

.text
#if __ARM_ARCH__<7
.code	32
#else
.syntax unified
# ifdef __thumb2__
.thumb
# else
.code   32
# endif
#endif

.type	K256,%object
.align	5
K256:
.word	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
.word	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
.word	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
.word	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
.word	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
.word	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
.word	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
.word	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
.word	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
.word	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
.word	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
.word	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
.word	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
.word	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
.word	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
.word	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
.size	K256,.-K256
.word	0				@ terminator
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
.LOPENSSL_armcap:
.word	OPENSSL_armcap_P-sha256_block_data_order
#endif
.align	5

.global	sha256_block_data_order
.type	sha256_block_data_order,%function
sha256_block_data_order:
.Lsha256_block_data_order:
#if __ARM_ARCH__<7
	sub	r3,pc,#8		@ sha256_block_data_order
#else
	adr	r3,.Lsha256_block_data_order
#endif
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
	ldr	r12,.LOPENSSL_armcap
	ldr	r12,[r3,r12]		@ OPENSSL_armcap_P
	tst	r12,#ARMV8_SHA256
	bne	.LARMv8
	tst	r12,#ARMV7_NEON
	bne	.LNEON
#endif
	add	$len,$inp,$len,lsl#6	@ len to point at the end of inp
	stmdb	sp!,{$ctx,$inp,$len,r4-r11,lr}
	ldmia	$ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
	sub	$Ktbl,r3,#256+32	@ K256
	sub	sp,sp,#16*4		@ alloca(X[16])
.Loop:
# if __ARM_ARCH__>=7
	ldr	$t1,[$inp],#4
# else
	ldrb	$t1,[$inp,#3]
# endif
	eor	$t3,$B,$C		@ magic
	eor	$t2,$t2,$t2
___
for($i=0;$i<16;$i++)	{ &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
$code.=".Lrounds_16_xx:\n";
for (;$i<32;$i++)	{ &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
#if __ARM_ARCH__>=7
	ite	eq			@ Thumb2 thing, sanity check in ARM
#endif
	ldreq	$t3,[sp,#16*4]		@ pull ctx
	bne	.Lrounds_16_xx

	add	$A,$A,$t2		@ h+=Maj(a,b,c) from the past
	ldr	$t0,[$t3,#0]
	ldr	$t1,[$t3,#4]
	ldr	$t2,[$t3,#8]
	add	$A,$A,$t0
	ldr	$t0,[$t3,#12]
	add	$B,$B,$t1
	ldr	$t1,[$t3,#16]
	add	$C,$C,$t2
	ldr	$t2,[$t3,#20]
	add	$D,$D,$t0
	ldr	$t0,[$t3,#24]
	add	$E,$E,$t1
	ldr	$t1,[$t3,#28]
	add	$F,$F,$t2
	ldr	$inp,[sp,#17*4]		@ pull inp
	ldr	$t2,[sp,#18*4]		@ pull inp+len
	add	$G,$G,$t0
	add	$H,$H,$t1
	stmia	$t3,{$A,$B,$C,$D,$E,$F,$G,$H}
	cmp	$inp,$t2
	sub	$Ktbl,$Ktbl,#256	@ rewind Ktbl
	bne	.Loop

	add	sp,sp,#`16+3`*4	@ destroy frame
#if __ARM_ARCH__>=5
	ldmia	sp!,{r4-r11,pc}
#else
	ldmia	sp!,{r4-r11,lr}
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	sha256_block_data_order,.-sha256_block_data_order
___
######################################################################
# NEON stuff
#
{{{
my @X=map("q$_",(0..3));
my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
my $Xfer=$t4;
my $j=0;

sub Dlo()   { shift=~m|q([1]?[0-9])|?"d".($1*2):"";     }
sub Dhi()   { shift=~m|q([1]?[0-9])|?"d".($1*2+1):"";   }

sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
  my $arg = pop;
    $arg = "#$arg" if ($arg*1 eq $arg);
    $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
}

sub Xupdate()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);
  my ($a,$b,$c,$d,$e,$f,$g,$h);

	&vext_8		($T0,@X[0],@X[1],4);	# X[1..4]
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vext_8		($T1,@X[2],@X[3],4);	# X[9..12]
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshr_u32	($T2,$T0,$sigma0[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(@X[0],@X[0],$T1);	# X[0..3] += X[9..12]
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshr_u32	($T1,$T0,$sigma0[2]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vsli_32	($T2,$T0,32-$sigma0[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshr_u32	($T3,$T0,$sigma0[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		($T1,$T1,$T2);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vsli_32	($T3,$T0,32-$sigma0[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T4,&Dhi(@X[3]),$sigma1[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		($T1,$T1,$T3);		# sigma0(X[1..4])
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vsli_32	($T4,&Dhi(@X[3]),32-$sigma1[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T5,&Dhi(@X[3]),$sigma1[2]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(@X[0],@X[0],$T1);	# X[0..3] += sigma0(X[1..4])
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &veor		($T5,$T5,$T4);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T4,&Dhi(@X[3]),$sigma1[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vsli_32	($T4,&Dhi(@X[3]),32-$sigma1[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &veor		($T5,$T5,$T4);		# sigma1(X[14..15])
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T4,&Dlo(@X[0]),$sigma1[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vsli_32	($T4,&Dlo(@X[0]),32-$sigma1[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T5,&Dlo(@X[0]),$sigma1[2]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &veor		($T5,$T5,$T4);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T4,&Dlo(@X[0]),$sigma1[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vld1_32	("{$T0}","[$Ktbl,:128]!");
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vsli_32	($T4,&Dlo(@X[0]),32-$sigma1[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &veor		($T5,$T5,$T4);		# sigma1(X[16..17])
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	($T0,$T0,@X[0]);
	 while($#insns>=2) { eval(shift(@insns)); }
	&vst1_32	("{$T0}","[$Xfer,:128]!");
	 eval(shift(@insns));
	 eval(shift(@insns));

	push(@X,shift(@X));		# "rotate" X[]
}

sub Xpreload()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);
  my ($a,$b,$c,$d,$e,$f,$g,$h);

	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vld1_32	("{$T0}","[$Ktbl,:128]!");
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vrev32_8	(@X[0],@X[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	($T0,$T0,@X[0]);
	 foreach (@insns) { eval; }	# remaining instructions
	&vst1_32	("{$T0}","[$Xfer,:128]!");

	push(@X,shift(@X));		# "rotate" X[]
}

sub body_00_15 () {
	(
	'($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
	'&add	($h,$h,$t1)',			# h+=X[i]+K[i]
	'&eor	($t1,$f,$g)',
	'&eor	($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
	'&add	($a,$a,$t2)',			# h+=Maj(a,b,c) from the past
	'&and	($t1,$t1,$e)',
	'&eor	($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))',	# Sigma1(e)
	'&eor	($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
	'&eor	($t1,$t1,$g)',			# Ch(e,f,g)
	'&add	($h,$h,$t2,"ror#$Sigma1[0]")',	# h+=Sigma1(e)
	'&eor	($t2,$a,$b)',			# a^b, b^c in next round
	'&eor	($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))',	# Sigma0(a)
	'&add	($h,$h,$t1)',			# h+=Ch(e,f,g)
	'&ldr	($t1,sprintf "[sp,#%d]",4*(($j+1)&15))	if (($j&15)!=15);'.
	'&ldr	($t1,"[$Ktbl]")				if ($j==15);'.
	'&ldr	($t1,"[sp,#64]")			if ($j==31)',
	'&and	($t3,$t3,$t2)',			# (b^c)&=(a^b)
	'&add	($d,$d,$h)',			# d+=h
	'&add	($h,$h,$t0,"ror#$Sigma0[0]");'.	# h+=Sigma0(a)
	'&eor	($t3,$t3,$b)',			# Maj(a,b,c)
	'$j++;	unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
	)
}

$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.arch	armv7-a
.fpu	neon

.global	sha256_block_data_order_neon
.type	sha256_block_data_order_neon,%function
.align	4
sha256_block_data_order_neon:
.LNEON:
	stmdb	sp!,{r4-r12,lr}

	sub	$H,sp,#16*4+16
	adr	$Ktbl,.Lsha256_block_data_order
	sub	$Ktbl,$Ktbl,#.Lsha256_block_data_order-K256
	bic	$H,$H,#15		@ align for 128-bit stores
	mov	$t2,sp
	mov	sp,$H			@ alloca
	add	$len,$inp,$len,lsl#6	@ len to point at the end of inp

	vld1.8		{@X[0]},[$inp]!
	vld1.8		{@X[1]},[$inp]!
	vld1.8		{@X[2]},[$inp]!
	vld1.8		{@X[3]},[$inp]!
	vld1.32		{$T0},[$Ktbl,:128]!
	vld1.32		{$T1},[$Ktbl,:128]!
	vld1.32		{$T2},[$Ktbl,:128]!
	vld1.32		{$T3},[$Ktbl,:128]!
	vrev32.8	@X[0],@X[0]		@ yes, even on
	str		$ctx,[sp,#64]
	vrev32.8	@X[1],@X[1]		@ big-endian
	str		$inp,[sp,#68]
	mov		$Xfer,sp
	vrev32.8	@X[2],@X[2]
	str		$len,[sp,#72]
	vrev32.8	@X[3],@X[3]
	str		$t2,[sp,#76]		@ save original sp
	vadd.i32	$T0,$T0,@X[0]
	vadd.i32	$T1,$T1,@X[1]
	vst1.32		{$T0},[$Xfer,:128]!
	vadd.i32	$T2,$T2,@X[2]
	vst1.32		{$T1},[$Xfer,:128]!
	vadd.i32	$T3,$T3,@X[3]
	vst1.32		{$T2},[$Xfer,:128]!
	vst1.32		{$T3},[$Xfer,:128]!

	ldmia		$ctx,{$A-$H}
	sub		$Xfer,$Xfer,#64
	ldr		$t1,[sp,#0]
	eor		$t2,$t2,$t2
	eor		$t3,$B,$C
	b		.L_00_48

.align	4
.L_00_48:
___
	&Xupdate(\&body_00_15);
	&Xupdate(\&body_00_15);
	&Xupdate(\&body_00_15);
	&Xupdate(\&body_00_15);
$code.=<<___;
	teq	$t1,#0				@ check for K256 terminator
	ldr	$t1,[sp,#0]
	sub	$Xfer,$Xfer,#64
	bne	.L_00_48

	ldr		$inp,[sp,#68]
	ldr		$t0,[sp,#72]
	sub		$Ktbl,$Ktbl,#256	@ rewind $Ktbl
	teq		$inp,$t0
	it		eq
	subeq		$inp,$inp,#64		@ avoid SEGV
	vld1.8		{@X[0]},[$inp]!		@ load next input block
	vld1.8		{@X[1]},[$inp]!
	vld1.8		{@X[2]},[$inp]!
	vld1.8		{@X[3]},[$inp]!
	it		ne
	strne		$inp,[sp,#68]
	mov		$Xfer,sp
___
	&Xpreload(\&body_00_15);
	&Xpreload(\&body_00_15);
	&Xpreload(\&body_00_15);
	&Xpreload(\&body_00_15);
$code.=<<___;
	ldr	$t0,[$t1,#0]
	add	$A,$A,$t2			@ h+=Maj(a,b,c) from the past
	ldr	$t2,[$t1,#4]
	ldr	$t3,[$t1,#8]
	ldr	$t4,[$t1,#12]
	add	$A,$A,$t0			@ accumulate
	ldr	$t0,[$t1,#16]
	add	$B,$B,$t2
	ldr	$t2,[$t1,#20]
	add	$C,$C,$t3
	ldr	$t3,[$t1,#24]
	add	$D,$D,$t4
	ldr	$t4,[$t1,#28]
	add	$E,$E,$t0
	str	$A,[$t1],#4
	add	$F,$F,$t2
	str	$B,[$t1],#4
	add	$G,$G,$t3
	str	$C,[$t1],#4
	add	$H,$H,$t4
	str	$D,[$t1],#4
	stmia	$t1,{$E-$H}

	ittte	ne
	movne	$Xfer,sp
	ldrne	$t1,[sp,#0]
	eorne	$t2,$t2,$t2
	ldreq	sp,[sp,#76]			@ restore original sp
	itt	ne
	eorne	$t3,$B,$C
	bne	.L_00_48

	ldmia	sp!,{r4-r12,pc}
.size	sha256_block_data_order_neon,.-sha256_block_data_order_neon
#endif
___
}}}
######################################################################
# ARMv8 stuff
#
{{{
my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
my @MSG=map("q$_",(8..11));
my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
my $Ktbl="r3";

$code.=<<___;
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)

# ifdef __thumb2__
#  define INST(a,b,c,d)	.byte	c,d|0xc,a,b
# else
#  define INST(a,b,c,d)	.byte	a,b,c,d
# endif

.type	sha256_block_data_order_armv8,%function
.align	5
sha256_block_data_order_armv8:
.LARMv8:
	vld1.32	{$ABCD,$EFGH},[$ctx]
# ifdef __thumb2__
	adr	$Ktbl,.LARMv8
	sub	$Ktbl,$Ktbl,#.LARMv8-K256
# else
	adrl	$Ktbl,K256
# endif
	add	$len,$inp,$len,lsl#6	@ len to point at the end of inp

.Loop_v8:
	vld1.8		{@MSG[0]-@MSG[1]},[$inp]!
	vld1.8		{@MSG[2]-@MSG[3]},[$inp]!
	vld1.32		{$W0},[$Ktbl]!
	vrev32.8	@MSG[0],@MSG[0]
	vrev32.8	@MSG[1],@MSG[1]
	vrev32.8	@MSG[2],@MSG[2]
	vrev32.8	@MSG[3],@MSG[3]
	vmov		$ABCD_SAVE,$ABCD	@ offload
	vmov		$EFGH_SAVE,$EFGH
	teq		$inp,$len
___
for($i=0;$i<12;$i++) {
$code.=<<___;
	vld1.32		{$W1},[$Ktbl]!
	vadd.i32	$W0,$W0,@MSG[0]
	sha256su0	@MSG[0],@MSG[1]
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W0
	sha256h2	$EFGH,$abcd,$W0
	sha256su1	@MSG[0],@MSG[2],@MSG[3]
___
	($W0,$W1)=($W1,$W0);	push(@MSG,shift(@MSG));
}
$code.=<<___;
	vld1.32		{$W1},[$Ktbl]!
	vadd.i32	$W0,$W0,@MSG[0]
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W0
	sha256h2	$EFGH,$abcd,$W0

	vld1.32		{$W0},[$Ktbl]!
	vadd.i32	$W1,$W1,@MSG[1]
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W1
	sha256h2	$EFGH,$abcd,$W1

	vld1.32		{$W1},[$Ktbl]
	vadd.i32	$W0,$W0,@MSG[2]
	sub		$Ktbl,$Ktbl,#256-16	@ rewind
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W0
	sha256h2	$EFGH,$abcd,$W0

	vadd.i32	$W1,$W1,@MSG[3]
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W1
	sha256h2	$EFGH,$abcd,$W1

	vadd.i32	$ABCD,$ABCD,$ABCD_SAVE
	vadd.i32	$EFGH,$EFGH,$EFGH_SAVE
	it		ne
	bne		.Loop_v8

	vst1.32		{$ABCD,$EFGH},[$ctx]

	ret		@ bx lr
.size	sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
#endif
___
}}}
$code.=<<___;
.asciz  "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
.comm   OPENSSL_armcap_P,4,4
#endif
___

open SELF,$0;
while(<SELF>) {
	next if (/^#!/);
	last if (!s/^#/@/ and !/^$/);
	print;
}
close SELF;

{   my  %opcode = (
	"sha256h"	=> 0xf3000c40,	"sha256h2"	=> 0xf3100c40,
	"sha256su0"	=> 0xf3ba03c0,	"sha256su1"	=> 0xf3200c40	);

    sub unsha256 {
	my ($mnemonic,$arg)=@_;

	if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
	    my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
					 |(($2&7)<<17)|(($2&8)<<4)
					 |(($3&7)<<1) |(($3&8)<<2);
	    # since ARMv7 instructions are always encoded little-endian.
	    # correct solution is to use .inst directive, but older
	    # assemblers don't implement it:-(
	    sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
			$word&0xff,($word>>8)&0xff,
			($word>>16)&0xff,($word>>24)&0xff,
			$mnemonic,$arg;
	}
    }
}

foreach (split($/,$code)) {

	s/\`([^\`]*)\`/eval $1/geo;

	s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;

	s/\bret\b/bx	lr/go		or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;	# make it possible to compile with -march=armv4

	print $_,"\n";
}

close STDOUT; # enforce flush


code:
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Glue code for the SHA256 Secure Hash Algorithm assembly implementation
 * using optimized ARM assembler and NEON instructions.
 *
 * Copyright © 2015 Google Inc.
 *
 * This file is based on sha256_ssse3_glue.c:
 *   Copyright (C) 2013 Intel Corporation
 *   Author: Tim Chen <tim.c.chen@linux.intel.com>
 */

#include <crypto/internal/hash.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <linux/string.h>
#include <crypto/sha2.h>
#include <crypto/sha256_base.h>
#include <asm/simd.h>
#include <asm/neon.h>

#include "sha256_glue.h"

asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
					unsigned int num_blks);

int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	/* make sure casting to sha256_block_fn() is safe */
	BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);

	return sha256_base_do_update(desc, data, len,
				(sha256_block_fn *)sha256_block_data_order);
}
EXPORT_SYMBOL(crypto_sha256_arm_update);

static int crypto_sha256_arm_final(struct shash_desc *desc, u8 *out)
{
	sha256_base_do_finalize(desc,
				(sha256_block_fn *)sha256_block_data_order);
	return sha256_base_finish(desc, out);
}

int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data,
			    unsigned int len, u8 *out)
{
	sha256_base_do_update(desc, data, len,
			      (sha256_block_fn *)sha256_block_data_order);
	return crypto_sha256_arm_final(desc, out);
}
EXPORT_SYMBOL(crypto_sha256_arm_finup);

static struct shash_alg algs[] = { {
	.digestsize	=	SHA256_DIGEST_SIZE,
	.init		=	sha256_base_init,
	.update		=	crypto_sha256_arm_update,
	.final		=	crypto_sha256_arm_final,
	.finup		=	crypto_sha256_arm_finup,
	.descsize	=	sizeof(struct sha256_state),
	.base		=	{
		.cra_name	=	"sha256",
		.cra_driver_name =	"sha256-asm",
		.cra_priority	=	150,
		.cra_blocksize	=	SHA256_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
}, {
	.digestsize	=	SHA224_DIGEST_SIZE,
	.init		=	sha224_base_init,
	.update		=	crypto_sha256_arm_update,
	.final		=	crypto_sha256_arm_final,
	.finup		=	crypto_sha256_arm_finup,
	.descsize	=	sizeof(struct sha256_state),
	.base		=	{
		.cra_name	=	"sha224",
		.cra_driver_name =	"sha224-asm",
		.cra_priority	=	150,
		.cra_blocksize	=	SHA224_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
} };

static int __init sha256_mod_init(void)
{
	int res = crypto_register_shashes(algs, ARRAY_SIZE(algs));

	if (res < 0)
		return res;

	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) {
		res = crypto_register_shashes(sha256_neon_algs,
					      ARRAY_SIZE(sha256_neon_algs));

		if (res < 0)
			crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
	}

	return res;
}

static void __exit sha256_mod_fini(void)
{
	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));

	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon())
		crypto_unregister_shashes(sha256_neon_algs,
					  ARRAY_SIZE(sha256_neon_algs));
}

module_init(sha256_mod_init);
module_exit(sha256_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm (ARM), including NEON");

MODULE_ALIAS_CRYPTO("sha256");


code:
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _CRYPTO_SHA256_GLUE_H
#define _CRYPTO_SHA256_GLUE_H

#include <linux/crypto.h>

extern struct shash_alg sha256_neon_algs[2];

int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len);

int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data,
			    unsigned int len, u8 *hash);

#endif /* _CRYPTO_SHA256_GLUE_H */


code:
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Glue code for the SHA256 Secure Hash Algorithm assembly implementation
 * using NEON instructions.
 *
 * Copyright © 2015 Google Inc.
 *
 * This file is based on sha512_neon_glue.c:
 *   Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 */

#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <linux/types.h>
#include <linux/string.h>
#include <crypto/sha2.h>
#include <crypto/sha256_base.h>
#include <asm/byteorder.h>
#include <asm/simd.h>
#include <asm/neon.h>

#include "sha256_glue.h"

asmlinkage void sha256_block_data_order_neon(u32 *digest, const void *data,
					     unsigned int num_blks);

static int crypto_sha256_neon_update(struct shash_desc *desc, const u8 *data,
				     unsigned int len)
{
	struct sha256_state *sctx = shash_desc_ctx(desc);

	if (!crypto_simd_usable() ||
	    (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
		return crypto_sha256_arm_update(desc, data, len);

	kernel_neon_begin();
	sha256_base_do_update(desc, data, len,
			(sha256_block_fn *)sha256_block_data_order_neon);
	kernel_neon_end();

	return 0;
}

static int crypto_sha256_neon_finup(struct shash_desc *desc, const u8 *data,
				    unsigned int len, u8 *out)
{
	if (!crypto_simd_usable())
		return crypto_sha256_arm_finup(desc, data, len, out);

	kernel_neon_begin();
	if (len)
		sha256_base_do_update(desc, data, len,
			(sha256_block_fn *)sha256_block_data_order_neon);
	sha256_base_do_finalize(desc,
			(sha256_block_fn *)sha256_block_data_order_neon);
	kernel_neon_end();

	return sha256_base_finish(desc, out);
}

static int crypto_sha256_neon_final(struct shash_desc *desc, u8 *out)
{
	return crypto_sha256_neon_finup(desc, NULL, 0, out);
}

struct shash_alg sha256_neon_algs[] = { {
	.digestsize	=	SHA256_DIGEST_SIZE,
	.init		=	sha256_base_init,
	.update		=	crypto_sha256_neon_update,
	.final		=	crypto_sha256_neon_final,
	.finup		=	crypto_sha256_neon_finup,
	.descsize	=	sizeof(struct sha256_state),
	.base		=	{
		.cra_name	=	"sha256",
		.cra_driver_name =	"sha256-neon",
		.cra_priority	=	250,
		.cra_blocksize	=	SHA256_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
}, {
	.digestsize	=	SHA224_DIGEST_SIZE,
	.init		=	sha224_base_init,
	.update		=	crypto_sha256_neon_update,
	.final		=	crypto_sha256_neon_final,
	.finup		=	crypto_sha256_neon_finup,
	.descsize	=	sizeof(struct sha256_state),
	.base		=	{
		.cra_name	=	"sha224",
		.cra_driver_name =	"sha224-neon",
		.cra_priority	=	250,
		.cra_blocksize	=	SHA224_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
} };


code:
#!/usr/bin/env perl
# SPDX-License-Identifier: GPL-2.0

# This code is taken from the OpenSSL project but the author (Andy Polyakov)
# has relicensed it under the GPLv2. Therefore this program is free software;
# you can redistribute it and/or modify it under the terms of the GNU General
# Public License version 2 as published by the Free Software Foundation.
#
# The original headers, including the original license headers, are
# included below for completeness.

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see https://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# SHA512 block procedure for ARMv4. September 2007.

# This code is ~4.5 (four and a half) times faster than code generated
# by gcc 3.4 and it spends ~72 clock cycles per byte [on single-issue
# Xscale PXA250 core].
#
# July 2010.
#
# Rescheduling for dual-issue pipeline resulted in 6% improvement on
# Cortex A8 core and ~40 cycles per processed byte.

# February 2011.
#
# Profiler-assisted and platform-specific optimization resulted in 7%
# improvement on Coxtex A8 core and ~38 cycles per byte.

# March 2011.
#
# Add NEON implementation. On Cortex A8 it was measured to process
# one byte in 23.3 cycles or ~60% faster than integer-only code.

# August 2012.
#
# Improve NEON performance by 12% on Snapdragon S4. In absolute
# terms it's 22.6 cycles per byte, which is disappointing result.
# Technical writers asserted that 3-way S4 pipeline can sustain
# multiple NEON instructions per cycle, but dual NEON issue could
# not be observed, see https://www.openssl.org/~appro/Snapdragon-S4.html
# for further details. On side note Cortex-A15 processes one byte in
# 16 cycles.

# Byte order [in]dependence. =========================================
#
# Originally caller was expected to maintain specific *dword* order in
# h[0-7], namely with most significant dword at *lower* address, which
# was reflected in below two parameters as 0 and 4. Now caller is
# expected to maintain native byte order for whole 64-bit values.
$hi="HI";
$lo="LO";
# ====================================================================

while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

$ctx="r0";	# parameter block
$inp="r1";
$len="r2";

$Tlo="r3";
$Thi="r4";
$Alo="r5";
$Ahi="r6";
$Elo="r7";
$Ehi="r8";
$t0="r9";
$t1="r10";
$t2="r11";
$t3="r12";
############	r13 is stack pointer
$Ktbl="r14";
############	r15 is program counter

$Aoff=8*0;
$Boff=8*1;
$Coff=8*2;
$Doff=8*3;
$Eoff=8*4;
$Foff=8*5;
$Goff=8*6;
$Hoff=8*7;
$Xoff=8*8;

sub BODY_00_15() {
my $magic = shift;
$code.=<<___;
	@ Sigma1(x)	(ROTR((x),14) ^ ROTR((x),18)  ^ ROTR((x),41))
	@ LO		lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23
	@ HI		hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23
	mov	$t0,$Elo,lsr#14
	str	$Tlo,[sp,#$Xoff+0]
	mov	$t1,$Ehi,lsr#14
	str	$Thi,[sp,#$Xoff+4]
	eor	$t0,$t0,$Ehi,lsl#18
	ldr	$t2,[sp,#$Hoff+0]	@ h.lo
	eor	$t1,$t1,$Elo,lsl#18
	ldr	$t3,[sp,#$Hoff+4]	@ h.hi
	eor	$t0,$t0,$Elo,lsr#18
	eor	$t1,$t1,$Ehi,lsr#18
	eor	$t0,$t0,$Ehi,lsl#14
	eor	$t1,$t1,$Elo,lsl#14
	eor	$t0,$t0,$Ehi,lsr#9
	eor	$t1,$t1,$Elo,lsr#9
	eor	$t0,$t0,$Elo,lsl#23
	eor	$t1,$t1,$Ehi,lsl#23	@ Sigma1(e)
	adds	$Tlo,$Tlo,$t0
	ldr	$t0,[sp,#$Foff+0]	@ f.lo
	adc	$Thi,$Thi,$t1		@ T += Sigma1(e)
	ldr	$t1,[sp,#$Foff+4]	@ f.hi
	adds	$Tlo,$Tlo,$t2
	ldr	$t2,[sp,#$Goff+0]	@ g.lo
	adc	$Thi,$Thi,$t3		@ T += h
	ldr	$t3,[sp,#$Goff+4]	@ g.hi

	eor	$t0,$t0,$t2
	str	$Elo,[sp,#$Eoff+0]
	eor	$t1,$t1,$t3
	str	$Ehi,[sp,#$Eoff+4]
	and	$t0,$t0,$Elo
	str	$Alo,[sp,#$Aoff+0]
	and	$t1,$t1,$Ehi
	str	$Ahi,[sp,#$Aoff+4]
	eor	$t0,$t0,$t2
	ldr	$t2,[$Ktbl,#$lo]	@ K[i].lo
	eor	$t1,$t1,$t3		@ Ch(e,f,g)
	ldr	$t3,[$Ktbl,#$hi]	@ K[i].hi

	adds	$Tlo,$Tlo,$t0
	ldr	$Elo,[sp,#$Doff+0]	@ d.lo
	adc	$Thi,$Thi,$t1		@ T += Ch(e,f,g)
	ldr	$Ehi,[sp,#$Doff+4]	@ d.hi
	adds	$Tlo,$Tlo,$t2
	and	$t0,$t2,#0xff
	adc	$Thi,$Thi,$t3		@ T += K[i]
	adds	$Elo,$Elo,$Tlo
	ldr	$t2,[sp,#$Boff+0]	@ b.lo
	adc	$Ehi,$Ehi,$Thi		@ d += T
	teq	$t0,#$magic

	ldr	$t3,[sp,#$Coff+0]	@ c.lo
#if __ARM_ARCH__>=7
	it	eq			@ Thumb2 thing, sanity check in ARM
#endif
	orreq	$Ktbl,$Ktbl,#1
	@ Sigma0(x)	(ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39))
	@ LO		lo>>28^hi<<4  ^ hi>>2^lo<<30 ^ hi>>7^lo<<25
	@ HI		hi>>28^lo<<4  ^ lo>>2^hi<<30 ^ lo>>7^hi<<25
	mov	$t0,$Alo,lsr#28
	mov	$t1,$Ahi,lsr#28
	eor	$t0,$t0,$Ahi,lsl#4
	eor	$t1,$t1,$Alo,lsl#4
	eor	$t0,$t0,$Ahi,lsr#2
	eor	$t1,$t1,$Alo,lsr#2
	eor	$t0,$t0,$Alo,lsl#30
	eor	$t1,$t1,$Ahi,lsl#30
	eor	$t0,$t0,$Ahi,lsr#7
	eor	$t1,$t1,$Alo,lsr#7
	eor	$t0,$t0,$Alo,lsl#25
	eor	$t1,$t1,$Ahi,lsl#25	@ Sigma0(a)
	adds	$Tlo,$Tlo,$t0
	and	$t0,$Alo,$t2
	adc	$Thi,$Thi,$t1		@ T += Sigma0(a)

	ldr	$t1,[sp,#$Boff+4]	@ b.hi
	orr	$Alo,$Alo,$t2
	ldr	$t2,[sp,#$Coff+4]	@ c.hi
	and	$Alo,$Alo,$t3
	and	$t3,$Ahi,$t1
	orr	$Ahi,$Ahi,$t1
	orr	$Alo,$Alo,$t0		@ Maj(a,b,c).lo
	and	$Ahi,$Ahi,$t2
	adds	$Alo,$Alo,$Tlo
	orr	$Ahi,$Ahi,$t3		@ Maj(a,b,c).hi
	sub	sp,sp,#8
	adc	$Ahi,$Ahi,$Thi		@ h += T
	tst	$Ktbl,#1
	add	$Ktbl,$Ktbl,#8
___
}
$code=<<___;
#ifndef __KERNEL__
# include "arm_arch.h"
# define VFP_ABI_PUSH	vstmdb	sp!,{d8-d15}
# define VFP_ABI_POP	vldmia	sp!,{d8-d15}
#else
# define __ARM_ARCH__ __LINUX_ARM_ARCH__
# define __ARM_MAX_ARCH__ 7
# define VFP_ABI_PUSH
# define VFP_ABI_POP
#endif

#ifdef __ARMEL__
# define LO 0
# define HI 4
# define WORD64(hi0,lo0,hi1,lo1)	.word	lo0,hi0, lo1,hi1
#else
# define HI 0
# define LO 4
# define WORD64(hi0,lo0,hi1,lo1)	.word	hi0,lo0, hi1,lo1
#endif

.text
#if __ARM_ARCH__<7
.code	32
#else
.syntax unified
# ifdef __thumb2__
.thumb
# else
.code   32
# endif
#endif

.type	K512,%object
.align	5
K512:
WORD64(0x428a2f98,0xd728ae22, 0x71374491,0x23ef65cd)
WORD64(0xb5c0fbcf,0xec4d3b2f, 0xe9b5dba5,0x8189dbbc)
WORD64(0x3956c25b,0xf348b538, 0x59f111f1,0xb605d019)
WORD64(0x923f82a4,0xaf194f9b, 0xab1c5ed5,0xda6d8118)
WORD64(0xd807aa98,0xa3030242, 0x12835b01,0x45706fbe)
WORD64(0x243185be,0x4ee4b28c, 0x550c7dc3,0xd5ffb4e2)
WORD64(0x72be5d74,0xf27b896f, 0x80deb1fe,0x3b1696b1)
WORD64(0x9bdc06a7,0x25c71235, 0xc19bf174,0xcf692694)
WORD64(0xe49b69c1,0x9ef14ad2, 0xefbe4786,0x384f25e3)
WORD64(0x0fc19dc6,0x8b8cd5b5, 0x240ca1cc,0x77ac9c65)
WORD64(0x2de92c6f,0x592b0275, 0x4a7484aa,0x6ea6e483)
WORD64(0x5cb0a9dc,0xbd41fbd4, 0x76f988da,0x831153b5)
WORD64(0x983e5152,0xee66dfab, 0xa831c66d,0x2db43210)
WORD64(0xb00327c8,0x98fb213f, 0xbf597fc7,0xbeef0ee4)
WORD64(0xc6e00bf3,0x3da88fc2, 0xd5a79147,0x930aa725)
WORD64(0x06ca6351,0xe003826f, 0x14292967,0x0a0e6e70)
WORD64(0x27b70a85,0x46d22ffc, 0x2e1b2138,0x5c26c926)
WORD64(0x4d2c6dfc,0x5ac42aed, 0x53380d13,0x9d95b3df)
WORD64(0x650a7354,0x8baf63de, 0x766a0abb,0x3c77b2a8)
WORD64(0x81c2c92e,0x47edaee6, 0x92722c85,0x1482353b)
WORD64(0xa2bfe8a1,0x4cf10364, 0xa81a664b,0xbc423001)
WORD64(0xc24b8b70,0xd0f89791, 0xc76c51a3,0x0654be30)
WORD64(0xd192e819,0xd6ef5218, 0xd6990624,0x5565a910)
WORD64(0xf40e3585,0x5771202a, 0x106aa070,0x32bbd1b8)
WORD64(0x19a4c116,0xb8d2d0c8, 0x1e376c08,0x5141ab53)
WORD64(0x2748774c,0xdf8eeb99, 0x34b0bcb5,0xe19b48a8)
WORD64(0x391c0cb3,0xc5c95a63, 0x4ed8aa4a,0xe3418acb)
WORD64(0x5b9cca4f,0x7763e373, 0x682e6ff3,0xd6b2b8a3)
WORD64(0x748f82ee,0x5defb2fc, 0x78a5636f,0x43172f60)
WORD64(0x84c87814,0xa1f0ab72, 0x8cc70208,0x1a6439ec)
WORD64(0x90befffa,0x23631e28, 0xa4506ceb,0xde82bde9)
WORD64(0xbef9a3f7,0xb2c67915, 0xc67178f2,0xe372532b)
WORD64(0xca273ece,0xea26619c, 0xd186b8c7,0x21c0c207)
WORD64(0xeada7dd6,0xcde0eb1e, 0xf57d4f7f,0xee6ed178)
WORD64(0x06f067aa,0x72176fba, 0x0a637dc5,0xa2c898a6)
WORD64(0x113f9804,0xbef90dae, 0x1b710b35,0x131c471b)
WORD64(0x28db77f5,0x23047d84, 0x32caab7b,0x40c72493)
WORD64(0x3c9ebe0a,0x15c9bebc, 0x431d67c4,0x9c100d4c)
WORD64(0x4cc5d4be,0xcb3e42b6, 0x597f299c,0xfc657e2a)
WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817)
.size	K512,.-K512
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
.LOPENSSL_armcap:
.word	OPENSSL_armcap_P-sha512_block_data_order
.skip	32-4
#else
.skip	32
#endif

.global	sha512_block_data_order
.type	sha512_block_data_order,%function
sha512_block_data_order:
.Lsha512_block_data_order:
#if __ARM_ARCH__<7
	sub	r3,pc,#8		@ sha512_block_data_order
#else
	adr	r3,.Lsha512_block_data_order
#endif
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
	ldr	r12,.LOPENSSL_armcap
	ldr	r12,[r3,r12]		@ OPENSSL_armcap_P
	tst	r12,#1
	bne	.LNEON
#endif
	add	$len,$inp,$len,lsl#7	@ len to point at the end of inp
	stmdb	sp!,{r4-r12,lr}
	sub	$Ktbl,r3,#672		@ K512
	sub	sp,sp,#9*8

	ldr	$Elo,[$ctx,#$Eoff+$lo]
	ldr	$Ehi,[$ctx,#$Eoff+$hi]
	ldr	$t0, [$ctx,#$Goff+$lo]
	ldr	$t1, [$ctx,#$Goff+$hi]
	ldr	$t2, [$ctx,#$Hoff+$lo]
	ldr	$t3, [$ctx,#$Hoff+$hi]
.Loop:
	str	$t0, [sp,#$Goff+0]
	str	$t1, [sp,#$Goff+4]
	str	$t2, [sp,#$Hoff+0]
	str	$t3, [sp,#$Hoff+4]
	ldr	$Alo,[$ctx,#$Aoff+$lo]
	ldr	$Ahi,[$ctx,#$Aoff+$hi]
	ldr	$Tlo,[$ctx,#$Boff+$lo]
	ldr	$Thi,[$ctx,#$Boff+$hi]
	ldr	$t0, [$ctx,#$Coff+$lo]
	ldr	$t1, [$ctx,#$Coff+$hi]
	ldr	$t2, [$ctx,#$Doff+$lo]
	ldr	$t3, [$ctx,#$Doff+$hi]
	str	$Tlo,[sp,#$Boff+0]
	str	$Thi,[sp,#$Boff+4]
	str	$t0, [sp,#$Coff+0]
	str	$t1, [sp,#$Coff+4]
	str	$t2, [sp,#$Doff+0]
	str	$t3, [sp,#$Doff+4]
	ldr	$Tlo,[$ctx,#$Foff+$lo]
	ldr	$Thi,[$ctx,#$Foff+$hi]
	str	$Tlo,[sp,#$Foff+0]
	str	$Thi,[sp,#$Foff+4]

.L00_15:
#if __ARM_ARCH__<7
	ldrb	$Tlo,[$inp,#7]
	ldrb	$t0, [$inp,#6]
	ldrb	$t1, [$inp,#5]
	ldrb	$t2, [$inp,#4]
	ldrb	$Thi,[$inp,#3]
	ldrb	$t3, [$inp,#2]
	orr	$Tlo,$Tlo,$t0,lsl#8
	ldrb	$t0, [$inp,#1]
	orr	$Tlo,$Tlo,$t1,lsl#16
	ldrb	$t1, [$inp],#8
	orr	$Tlo,$Tlo,$t2,lsl#24
	orr	$Thi,$Thi,$t3,lsl#8
	orr	$Thi,$Thi,$t0,lsl#16
	orr	$Thi,$Thi,$t1,lsl#24
#else
	ldr	$Tlo,[$inp,#4]
	ldr	$Thi,[$inp],#8
#ifdef __ARMEL__
	rev	$Tlo,$Tlo
	rev	$Thi,$Thi
#endif
#endif
___
	&BODY_00_15(0x94);
$code.=<<___;
	tst	$Ktbl,#1
	beq	.L00_15
	ldr	$t0,[sp,#`$Xoff+8*(16-1)`+0]
	ldr	$t1,[sp,#`$Xoff+8*(16-1)`+4]
	bic	$Ktbl,$Ktbl,#1
.L16_79:
	@ sigma0(x)	(ROTR((x),1)  ^ ROTR((x),8)  ^ ((x)>>7))
	@ LO		lo>>1^hi<<31  ^ lo>>8^hi<<24 ^ lo>>7^hi<<25
	@ HI		hi>>1^lo<<31  ^ hi>>8^lo<<24 ^ hi>>7
	mov	$Tlo,$t0,lsr#1
	ldr	$t2,[sp,#`$Xoff+8*(16-14)`+0]
	mov	$Thi,$t1,lsr#1
	ldr	$t3,[sp,#`$Xoff+8*(16-14)`+4]
	eor	$Tlo,$Tlo,$t1,lsl#31
	eor	$Thi,$Thi,$t0,lsl#31
	eor	$Tlo,$Tlo,$t0,lsr#8
	eor	$Thi,$Thi,$t1,lsr#8
	eor	$Tlo,$Tlo,$t1,lsl#24
	eor	$Thi,$Thi,$t0,lsl#24
	eor	$Tlo,$Tlo,$t0,lsr#7
	eor	$Thi,$Thi,$t1,lsr#7
	eor	$Tlo,$Tlo,$t1,lsl#25

	@ sigma1(x)	(ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6))
	@ LO		lo>>19^hi<<13 ^ hi>>29^lo<<3 ^ lo>>6^hi<<26
	@ HI		hi>>19^lo<<13 ^ lo>>29^hi<<3 ^ hi>>6
	mov	$t0,$t2,lsr#19
	mov	$t1,$t3,lsr#19
	eor	$t0,$t0,$t3,lsl#13
	eor	$t1,$t1,$t2,lsl#13
	eor	$t0,$t0,$t3,lsr#29
	eor	$t1,$t1,$t2,lsr#29
	eor	$t0,$t0,$t2,lsl#3
	eor	$t1,$t1,$t3,lsl#3
	eor	$t0,$t0,$t2,lsr#6
	eor	$t1,$t1,$t3,lsr#6
	ldr	$t2,[sp,#`$Xoff+8*(16-9)`+0]
	eor	$t0,$t0,$t3,lsl#26

	ldr	$t3,[sp,#`$Xoff+8*(16-9)`+4]
	adds	$Tlo,$Tlo,$t0
	ldr	$t0,[sp,#`$Xoff+8*16`+0]
	adc	$Thi,$Thi,$t1

	ldr	$t1,[sp,#`$Xoff+8*16`+4]
	adds	$Tlo,$Tlo,$t2
	adc	$Thi,$Thi,$t3
	adds	$Tlo,$Tlo,$t0
	adc	$Thi,$Thi,$t1
___
	&BODY_00_15(0x17);
$code.=<<___;
#if __ARM_ARCH__>=7
	ittt	eq			@ Thumb2 thing, sanity check in ARM
#endif
	ldreq	$t0,[sp,#`$Xoff+8*(16-1)`+0]
	ldreq	$t1,[sp,#`$Xoff+8*(16-1)`+4]
	beq	.L16_79
	bic	$Ktbl,$Ktbl,#1

	ldr	$Tlo,[sp,#$Boff+0]
	ldr	$Thi,[sp,#$Boff+4]
	ldr	$t0, [$ctx,#$Aoff+$lo]
	ldr	$t1, [$ctx,#$Aoff+$hi]
	ldr	$t2, [$ctx,#$Boff+$lo]
	ldr	$t3, [$ctx,#$Boff+$hi]
	adds	$t0,$Alo,$t0
	str	$t0, [$ctx,#$Aoff+$lo]
	adc	$t1,$Ahi,$t1
	str	$t1, [$ctx,#$Aoff+$hi]
	adds	$t2,$Tlo,$t2
	str	$t2, [$ctx,#$Boff+$lo]
	adc	$t3,$Thi,$t3
	str	$t3, [$ctx,#$Boff+$hi]

	ldr	$Alo,[sp,#$Coff+0]
	ldr	$Ahi,[sp,#$Coff+4]
	ldr	$Tlo,[sp,#$Doff+0]
	ldr	$Thi,[sp,#$Doff+4]
	ldr	$t0, [$ctx,#$Coff+$lo]
	ldr	$t1, [$ctx,#$Coff+$hi]
	ldr	$t2, [$ctx,#$Doff+$lo]
	ldr	$t3, [$ctx,#$Doff+$hi]
	adds	$t0,$Alo,$t0
	str	$t0, [$ctx,#$Coff+$lo]
	adc	$t1,$Ahi,$t1
	str	$t1, [$ctx,#$Coff+$hi]
	adds	$t2,$Tlo,$t2
	str	$t2, [$ctx,#$Doff+$lo]
	adc	$t3,$Thi,$t3
	str	$t3, [$ctx,#$Doff+$hi]

	ldr	$Tlo,[sp,#$Foff+0]
	ldr	$Thi,[sp,#$Foff+4]
	ldr	$t0, [$ctx,#$Eoff+$lo]
	ldr	$t1, [$ctx,#$Eoff+$hi]
	ldr	$t2, [$ctx,#$Foff+$lo]
	ldr	$t3, [$ctx,#$Foff+$hi]
	adds	$Elo,$Elo,$t0
	str	$Elo,[$ctx,#$Eoff+$lo]
	adc	$Ehi,$Ehi,$t1
	str	$Ehi,[$ctx,#$Eoff+$hi]
	adds	$t2,$Tlo,$t2
	str	$t2, [$ctx,#$Foff+$lo]
	adc	$t3,$Thi,$t3
	str	$t3, [$ctx,#$Foff+$hi]

	ldr	$Alo,[sp,#$Goff+0]
	ldr	$Ahi,[sp,#$Goff+4]
	ldr	$Tlo,[sp,#$Hoff+0]
	ldr	$Thi,[sp,#$Hoff+4]
	ldr	$t0, [$ctx,#$Goff+$lo]
	ldr	$t1, [$ctx,#$Goff+$hi]
	ldr	$t2, [$ctx,#$Hoff+$lo]
	ldr	$t3, [$ctx,#$Hoff+$hi]
	adds	$t0,$Alo,$t0
	str	$t0, [$ctx,#$Goff+$lo]
	adc	$t1,$Ahi,$t1
	str	$t1, [$ctx,#$Goff+$hi]
	adds	$t2,$Tlo,$t2
	str	$t2, [$ctx,#$Hoff+$lo]
	adc	$t3,$Thi,$t3
	str	$t3, [$ctx,#$Hoff+$hi]

	add	sp,sp,#640
	sub	$Ktbl,$Ktbl,#640

	teq	$inp,$len
	bne	.Loop

	add	sp,sp,#8*9		@ destroy frame
#if __ARM_ARCH__>=5
	ldmia	sp!,{r4-r12,pc}
#else
	ldmia	sp!,{r4-r12,lr}
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	sha512_block_data_order,.-sha512_block_data_order
___

{
my @Sigma0=(28,34,39);
my @Sigma1=(14,18,41);
my @sigma0=(1, 8, 7);
my @sigma1=(19,61,6);

my $Ktbl="r3";
my $cnt="r12";	# volatile register known as ip, intra-procedure-call scratch

my @X=map("d$_",(0..15));
my @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("d$_",(16..23));

sub NEON_00_15() {
my $i=shift;
my ($a,$b,$c,$d,$e,$f,$g,$h)=@_;
my ($t0,$t1,$t2,$T1,$K,$Ch,$Maj)=map("d$_",(24..31));	# temps

$code.=<<___ if ($i<16 || $i&1);
	vshr.u64	$t0,$e,#@Sigma1[0]	@ $i
#if $i<16
	vld1.64		{@X[$i%16]},[$inp]!	@ handles unaligned
#endif
	vshr.u64	$t1,$e,#@Sigma1[1]
#if $i>0
	 vadd.i64	$a,$Maj			@ h+=Maj from the past
#endif
	vshr.u64	$t2,$e,#@Sigma1[2]
___
$code.=<<___;
	vld1.64		{$K},[$Ktbl,:64]!	@ K[i++]
	vsli.64		$t0,$e,#`64-@Sigma1[0]`
	vsli.64		$t1,$e,#`64-@Sigma1[1]`
	vmov		$Ch,$e
	vsli.64		$t2,$e,#`64-@Sigma1[2]`
#if $i<16 && defined(__ARMEL__)
	vrev64.8	@X[$i],@X[$i]
#endif
	veor		$t1,$t0
	vbsl		$Ch,$f,$g		@ Ch(e,f,g)
	vshr.u64	$t0,$a,#@Sigma0[0]
	veor		$t2,$t1			@ Sigma1(e)
	vadd.i64	$T1,$Ch,$h
	vshr.u64	$t1,$a,#@Sigma0[1]
	vsli.64		$t0,$a,#`64-@Sigma0[0]`
	vadd.i64	$T1,$t2
	vshr.u64	$t2,$a,#@Sigma0[2]
	vadd.i64	$K,@X[$i%16]
	vsli.64		$t1,$a,#`64-@Sigma0[1]`
	veor		$Maj,$a,$b
	vsli.64		$t2,$a,#`64-@Sigma0[2]`
	veor		$h,$t0,$t1
	vadd.i64	$T1,$K
	vbsl		$Maj,$c,$b		@ Maj(a,b,c)
	veor		$h,$t2			@ Sigma0(a)
	vadd.i64	$d,$T1
	vadd.i64	$Maj,$T1
	@ vadd.i64	$h,$Maj
___
}

sub NEON_16_79() {
my $i=shift;

if ($i&1)	{ &NEON_00_15($i,@_); return; }

# 2x-vectorized, therefore runs every 2nd round
my @X=map("q$_",(0..7));			# view @X as 128-bit vector
my ($t0,$t1,$s0,$s1) = map("q$_",(12..15));	# temps
my ($d0,$d1,$d2) = map("d$_",(24..26));		# temps from NEON_00_15
my $e=@_[4];					# $e from NEON_00_15
$i /= 2;
$code.=<<___;
	vshr.u64	$t0,@X[($i+7)%8],#@sigma1[0]
	vshr.u64	$t1,@X[($i+7)%8],#@sigma1[1]
	 vadd.i64	@_[0],d30			@ h+=Maj from the past
	vshr.u64	$s1,@X[($i+7)%8],#@sigma1[2]
	vsli.64		$t0,@X[($i+7)%8],#`64-@sigma1[0]`
	vext.8		$s0,@X[$i%8],@X[($i+1)%8],#8	@ X[i+1]
	vsli.64		$t1,@X[($i+7)%8],#`64-@sigma1[1]`
	veor		$s1,$t0
	vshr.u64	$t0,$s0,#@sigma0[0]
	veor		$s1,$t1				@ sigma1(X[i+14])
	vshr.u64	$t1,$s0,#@sigma0[1]
	vadd.i64	@X[$i%8],$s1
	vshr.u64	$s1,$s0,#@sigma0[2]
	vsli.64		$t0,$s0,#`64-@sigma0[0]`
	vsli.64		$t1,$s0,#`64-@sigma0[1]`
	vext.8		$s0,@X[($i+4)%8],@X[($i+5)%8],#8	@ X[i+9]
	veor		$s1,$t0
	vshr.u64	$d0,$e,#@Sigma1[0]		@ from NEON_00_15
	vadd.i64	@X[$i%8],$s0
	vshr.u64	$d1,$e,#@Sigma1[1]		@ from NEON_00_15
	veor		$s1,$t1				@ sigma0(X[i+1])
	vshr.u64	$d2,$e,#@Sigma1[2]		@ from NEON_00_15
	vadd.i64	@X[$i%8],$s1
___
	&NEON_00_15(2*$i,@_);
}

$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.arch	armv7-a
.fpu	neon

.global	sha512_block_data_order_neon
.type	sha512_block_data_order_neon,%function
.align	4
sha512_block_data_order_neon:
.LNEON:
	dmb				@ errata #451034 on early Cortex A8
	add	$len,$inp,$len,lsl#7	@ len to point at the end of inp
	VFP_ABI_PUSH
	adr	$Ktbl,.Lsha512_block_data_order
	sub	$Ktbl,$Ktbl,.Lsha512_block_data_order-K512
	vldmia	$ctx,{$A-$H}		@ load context
.Loop_neon:
___
for($i=0;$i<16;$i++)	{ &NEON_00_15($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	mov		$cnt,#4
.L16_79_neon:
	subs		$cnt,#1
___
for(;$i<32;$i++)	{ &NEON_16_79($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	bne		.L16_79_neon

	 vadd.i64	$A,d30		@ h+=Maj from the past
	vldmia		$ctx,{d24-d31}	@ load context to temp
	vadd.i64	q8,q12		@ vectorized accumulate
	vadd.i64	q9,q13
	vadd.i64	q10,q14
	vadd.i64	q11,q15
	vstmia		$ctx,{$A-$H}	@ save context
	teq		$inp,$len
	sub		$Ktbl,#640	@ rewind K512
	bne		.Loop_neon

	VFP_ABI_POP
	ret				@ bx lr
.size	sha512_block_data_order_neon,.-sha512_block_data_order_neon
#endif
___
}
$code.=<<___;
.asciz	"SHA512 block transform for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
.comm	OPENSSL_armcap_P,4,4
#endif
___

$code =~ s/\`([^\`]*)\`/eval $1/gem;
$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm;	# make it possible to compile with -march=armv4
$code =~ s/\bret\b/bx	lr/gm;

open SELF,$0;
while(<SELF>) {
	next if (/^#!/);
	last if (!s/^#/@/ and !/^$/);
	print;
}
close SELF;

print $code;
close STDOUT; # enforce flush


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * sha512-glue.c - accelerated SHA-384/512 for ARM
 *
 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#include <crypto/internal/hash.h>
#include <crypto/sha2.h>
#include <crypto/sha512_base.h>
#include <linux/crypto.h>
#include <linux/module.h>

#include <asm/hwcap.h>
#include <asm/neon.h>

#include "sha512.h"

MODULE_DESCRIPTION("Accelerated SHA-384/SHA-512 secure hash for ARM");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");

MODULE_ALIAS_CRYPTO("sha384");
MODULE_ALIAS_CRYPTO("sha512");
MODULE_ALIAS_CRYPTO("sha384-arm");
MODULE_ALIAS_CRYPTO("sha512-arm");

asmlinkage void sha512_block_data_order(u64 *state, u8 const *src, int blocks);

int sha512_arm_update(struct shash_desc *desc, const u8 *data,
		      unsigned int len)
{
	return sha512_base_do_update(desc, data, len,
		(sha512_block_fn *)sha512_block_data_order);
}

static int sha512_arm_final(struct shash_desc *desc, u8 *out)
{
	sha512_base_do_finalize(desc,
		(sha512_block_fn *)sha512_block_data_order);
	return sha512_base_finish(desc, out);
}

int sha512_arm_finup(struct shash_desc *desc, const u8 *data,
		     unsigned int len, u8 *out)
{
	sha512_base_do_update(desc, data, len,
		(sha512_block_fn *)sha512_block_data_order);
	return sha512_arm_final(desc, out);
}

static struct shash_alg sha512_arm_algs[] = { {
	.init			= sha384_base_init,
	.update			= sha512_arm_update,
	.final			= sha512_arm_final,
	.finup			= sha512_arm_finup,
	.descsize		= sizeof(struct sha512_state),
	.digestsize		= SHA384_DIGEST_SIZE,
	.base			= {
		.cra_name		= "sha384",
		.cra_driver_name	= "sha384-arm",
		.cra_priority		= 250,
		.cra_blocksize		= SHA512_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,
	}
},  {
	.init			= sha512_base_init,
	.update			= sha512_arm_update,
	.final			= sha512_arm_final,
	.finup			= sha512_arm_finup,
	.descsize		= sizeof(struct sha512_state),
	.digestsize		= SHA512_DIGEST_SIZE,
	.base			= {
		.cra_name		= "sha512",
		.cra_driver_name	= "sha512-arm",
		.cra_priority		= 250,
		.cra_blocksize		= SHA512_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,
	}
} };

static int __init sha512_arm_mod_init(void)
{
	int err;

	err = crypto_register_shashes(sha512_arm_algs,
				      ARRAY_SIZE(sha512_arm_algs));
	if (err)
		return err;

	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) {
		err = crypto_register_shashes(sha512_neon_algs,
					      ARRAY_SIZE(sha512_neon_algs));
		if (err)
			goto err_unregister;
	}
	return 0;

err_unregister:
	crypto_unregister_shashes(sha512_arm_algs,
				  ARRAY_SIZE(sha512_arm_algs));

	return err;
}

static void __exit sha512_arm_mod_fini(void)
{
	crypto_unregister_shashes(sha512_arm_algs,
				  ARRAY_SIZE(sha512_arm_algs));
	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon())
		crypto_unregister_shashes(sha512_neon_algs,
					  ARRAY_SIZE(sha512_neon_algs));
}

module_init(sha512_arm_mod_init);
module_exit(sha512_arm_mod_fini);


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * sha512-neon-glue.c - accelerated SHA-384/512 for ARM NEON
 *
 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <crypto/sha2.h>
#include <crypto/sha512_base.h>
#include <linux/crypto.h>
#include <linux/module.h>

#include <asm/simd.h>
#include <asm/neon.h>

#include "sha512.h"

MODULE_ALIAS_CRYPTO("sha384-neon");
MODULE_ALIAS_CRYPTO("sha512-neon");

asmlinkage void sha512_block_data_order_neon(u64 *state, u8 const *src,
					     int blocks);

static int sha512_neon_update(struct shash_desc *desc, const u8 *data,
			      unsigned int len)
{
	struct sha512_state *sctx = shash_desc_ctx(desc);

	if (!crypto_simd_usable() ||
	    (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
		return sha512_arm_update(desc, data, len);

	kernel_neon_begin();
	sha512_base_do_update(desc, data, len,
		(sha512_block_fn *)sha512_block_data_order_neon);
	kernel_neon_end();

	return 0;
}

static int sha512_neon_finup(struct shash_desc *desc, const u8 *data,
			     unsigned int len, u8 *out)
{
	if (!crypto_simd_usable())
		return sha512_arm_finup(desc, data, len, out);

	kernel_neon_begin();
	if (len)
		sha512_base_do_update(desc, data, len,
			(sha512_block_fn *)sha512_block_data_order_neon);
	sha512_base_do_finalize(desc,
		(sha512_block_fn *)sha512_block_data_order_neon);
	kernel_neon_end();

	return sha512_base_finish(desc, out);
}

static int sha512_neon_final(struct shash_desc *desc, u8 *out)
{
	return sha512_neon_finup(desc, NULL, 0, out);
}

struct shash_alg sha512_neon_algs[] = { {
	.init			= sha384_base_init,
	.update			= sha512_neon_update,
	.final			= sha512_neon_final,
	.finup			= sha512_neon_finup,
	.descsize		= sizeof(struct sha512_state),
	.digestsize		= SHA384_DIGEST_SIZE,
	.base			= {
		.cra_name		= "sha384",
		.cra_driver_name	= "sha384-neon",
		.cra_priority		= 300,
		.cra_blocksize		= SHA384_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,

	}
},  {
	.init			= sha512_base_init,
	.update			= sha512_neon_update,
	.final			= sha512_neon_final,
	.finup			= sha512_neon_finup,
	.descsize		= sizeof(struct sha512_state),
	.digestsize		= SHA512_DIGEST_SIZE,
	.base			= {
		.cra_name		= "sha512",
		.cra_driver_name	= "sha512-neon",
		.cra_priority		= 300,
		.cra_blocksize		= SHA512_BLOCK_SIZE,
		.cra_module		= THIS_MODULE,
	}
} };


code:
/* SPDX-License-Identifier: GPL-2.0 */

int sha512_arm_update(struct shash_desc *desc, const u8 *data,
		      unsigned int len);

int sha512_arm_finup(struct shash_desc *desc, const u8 *data,
		     unsigned int len, u8 *out);

extern struct shash_alg sha512_neon_algs[2];


code:
# SPDX-License-Identifier: GPL-2.0
generic-y += early_ioremap.h
generic-y += extable.h
generic-y += flat.h
generic-y += parport.h

generated-y += mach-types.h
generated-y += unistd-nr.h

path: arch/arm/include/asm/Kbuild


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * arch/arm/include/asm/arch_gicv3.h
 *
 * Copyright (C) 2015 ARM Ltd.
 */
#ifndef __ASM_ARCH_GICV3_H
#define __ASM_ARCH_GICV3_H

#ifndef __ASSEMBLY__

#include <linux/io.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <asm/barrier.h>
#include <asm/cacheflush.h>
#include <asm/cp15.h>

#define ICC_EOIR1			__ACCESS_CP15(c12, 0, c12, 1)
#define ICC_DIR				__ACCESS_CP15(c12, 0, c11, 1)
#define ICC_IAR1			__ACCESS_CP15(c12, 0, c12, 0)
#define ICC_SGI1R			__ACCESS_CP15_64(0, c12)
#define ICC_PMR				__ACCESS_CP15(c4, 0, c6, 0)
#define ICC_CTLR			__ACCESS_CP15(c12, 0, c12, 4)
#define ICC_SRE				__ACCESS_CP15(c12, 0, c12, 5)
#define ICC_IGRPEN1			__ACCESS_CP15(c12, 0, c12, 7)
#define ICC_BPR1			__ACCESS_CP15(c12, 0, c12, 3)
#define ICC_RPR				__ACCESS_CP15(c12, 0, c11, 3)

#define __ICC_AP0Rx(x)			__ACCESS_CP15(c12, 0, c8, 4 | x)
#define ICC_AP0R0			__ICC_AP0Rx(0)
#define ICC_AP0R1			__ICC_AP0Rx(1)
#define ICC_AP0R2			__ICC_AP0Rx(2)
#define ICC_AP0R3			__ICC_AP0Rx(3)

#define __ICC_AP1Rx(x)			__ACCESS_CP15(c12, 0, c9, x)
#define ICC_AP1R0			__ICC_AP1Rx(0)
#define ICC_AP1R1			__ICC_AP1Rx(1)
#define ICC_AP1R2			__ICC_AP1Rx(2)
#define ICC_AP1R3			__ICC_AP1Rx(3)

#define CPUIF_MAP(a32, a64)			\
static inline void write_ ## a64(u32 val)	\
{						\
	write_sysreg(val, a32);			\
}						\
static inline u32 read_ ## a64(void)		\
{						\
	return read_sysreg(a32); 		\
}						\

CPUIF_MAP(ICC_EOIR1, ICC_EOIR1_EL1)
CPUIF_MAP(ICC_PMR, ICC_PMR_EL1)
CPUIF_MAP(ICC_AP0R0, ICC_AP0R0_EL1)
CPUIF_MAP(ICC_AP0R1, ICC_AP0R1_EL1)
CPUIF_MAP(ICC_AP0R2, ICC_AP0R2_EL1)
CPUIF_MAP(ICC_AP0R3, ICC_AP0R3_EL1)
CPUIF_MAP(ICC_AP1R0, ICC_AP1R0_EL1)
CPUIF_MAP(ICC_AP1R1, ICC_AP1R1_EL1)
CPUIF_MAP(ICC_AP1R2, ICC_AP1R2_EL1)
CPUIF_MAP(ICC_AP1R3, ICC_AP1R3_EL1)

#define read_gicreg(r)                 read_##r()
#define write_gicreg(v, r)             write_##r(v)

/* Low-level accessors */

static inline void gic_write_dir(u32 val)
{
	write_sysreg(val, ICC_DIR);
	isb();
}

static inline u32 gic_read_iar(void)
{
	u32 irqstat = read_sysreg(ICC_IAR1);

	dsb(sy);

	return irqstat;
}

static inline void gic_write_ctlr(u32 val)
{
	write_sysreg(val, ICC_CTLR);
	isb();
}

static inline u32 gic_read_ctlr(void)
{
	return read_sysreg(ICC_CTLR);
}

static inline void gic_write_grpen1(u32 val)
{
	write_sysreg(val, ICC_IGRPEN1);
	isb();
}

static inline void gic_write_sgi1r(u64 val)
{
	write_sysreg(val, ICC_SGI1R);
}

static inline u32 gic_read_sre(void)
{
	return read_sysreg(ICC_SRE);
}

static inline void gic_write_sre(u32 val)
{
	write_sysreg(val, ICC_SRE);
	isb();
}

static inline void gic_write_bpr1(u32 val)
{
	write_sysreg(val, ICC_BPR1);
}

static inline u32 gic_read_pmr(void)
{
	return read_sysreg(ICC_PMR);
}

static inline void gic_write_pmr(u32 val)
{
	write_sysreg(val, ICC_PMR);
}

static inline u32 gic_read_rpr(void)
{
	return read_sysreg(ICC_RPR);
}

/*
 * Even in 32bit systems that use LPAE, there is no guarantee that the I/O
 * interface provides true 64bit atomic accesses, so using strd/ldrd doesn't
 * make much sense.
 * Moreover, 64bit I/O emulation is extremely difficult to implement on
 * AArch32, since the syndrome register doesn't provide any information for
 * them.
 * Consequently, the following IO helpers use 32bit accesses.
 */
static inline void __gic_writeq_nonatomic(u64 val, volatile void __iomem *addr)
{
	writel_relaxed((u32)val, addr);
	writel_relaxed((u32)(val >> 32), addr + 4);
}

static inline u64 __gic_readq_nonatomic(const volatile void __iomem *addr)
{
	u64 val;

	val = readl_relaxed(addr);
	val |= (u64)readl_relaxed(addr + 4) << 32;
	return val;
}

#define gic_flush_dcache_to_poc(a,l)    __cpuc_flush_dcache_area((a), (l))

/*
 *  GICD_IROUTERn, contain the affinity values associated to each interrupt.
 *  The upper-word (aff3) will always be 0, so there is no need for a lock.
 */
#define gic_write_irouter(v, c)		__gic_writeq_nonatomic(v, c)

/*
 * GICR_TYPER is an ID register and doesn't need atomicity.
 */
#define gic_read_typer(c)		__gic_readq_nonatomic(c)

/*
 * GITS_BASER - hi and lo bits may be accessed independently.
 */
#define gits_read_baser(c)		__gic_readq_nonatomic(c)
#define gits_write_baser(v, c)		__gic_writeq_nonatomic(v, c)

/*
 * GICR_PENDBASER and GICR_PROPBASE are changed with LPIs disabled, so they
 * won't be being used during any updates and can be changed non-atomically
 */
#define gicr_read_propbaser(c)		__gic_readq_nonatomic(c)
#define gicr_write_propbaser(v, c)	__gic_writeq_nonatomic(v, c)
#define gicr_read_pendbaser(c)		__gic_readq_nonatomic(c)
#define gicr_write_pendbaser(v, c)	__gic_writeq_nonatomic(v, c)

/*
 * GICR_xLPIR - only the lower bits are significant
 */
#define gic_read_lpir(c)		readl_relaxed(c)
#define gic_write_lpir(v, c)		writel_relaxed(lower_32_bits(v), c)

/*
 * GITS_TYPER is an ID register and doesn't need atomicity.
 */
#define gits_read_typer(c)		__gic_readq_nonatomic(c)

/*
 * GITS_CBASER - hi and lo bits may be accessed independently.
 */
#define gits_read_cbaser(c)		__gic_readq_nonatomic(c)
#define gits_write_cbaser(v, c)		__gic_writeq_nonatomic(v, c)

/*
 * GITS_CWRITER - hi and lo bits may be accessed independently.
 */
#define gits_write_cwriter(v, c)	__gic_writeq_nonatomic(v, c)

/*
 * GICR_VPROPBASER - hi and lo bits may be accessed independently.
 */
#define gicr_read_vpropbaser(c)		__gic_readq_nonatomic(c)
#define gicr_write_vpropbaser(v, c)	__gic_writeq_nonatomic(v, c)

/*
 * GICR_VPENDBASER - the Valid bit must be cleared before changing
 * anything else.
 */
static inline void gicr_write_vpendbaser(u64 val, void __iomem *addr)
{
	u32 tmp;

	tmp = readl_relaxed(addr + 4);
	if (tmp & (GICR_VPENDBASER_Valid >> 32)) {
		tmp &= ~(GICR_VPENDBASER_Valid >> 32);
		writel_relaxed(tmp, addr + 4);
	}

	/*
	 * Use the fact that __gic_writeq_nonatomic writes the second
	 * half of the 64bit quantity after the first.
	 */
	__gic_writeq_nonatomic(val, addr);
}

#define gicr_read_vpendbaser(c)		__gic_readq_nonatomic(c)

static inline bool gic_prio_masking_enabled(void)
{
	return false;
}

static inline void gic_pmr_mask_irqs(void)
{
	/* Should not get called. */
	WARN_ON_ONCE(true);
}

static inline void gic_arch_enable_irqs(void)
{
	/* Should not get called. */
	WARN_ON_ONCE(true);
}

#endif /* !__ASSEMBLY__ */
#endif /* !__ASM_ARCH_GICV3_H */


code:
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASMARM_ARCH_TIMER_H
#define __ASMARM_ARCH_TIMER_H

#include <asm/barrier.h>
#include <asm/errno.h>
#include <asm/hwcap.h>
#include <linux/clocksource.h>
#include <linux/init.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/types.h>

#include <clocksource/arm_arch_timer.h>

#ifdef CONFIG_ARM_ARCH_TIMER
/* 32bit ARM doesn't know anything about timer errata... */
#define has_erratum_handler(h)		(false)
#define erratum_handler(h)		(arch_timer_##h)

int arch_timer_arch_init(void);

/*
 * These register accessors are marked inline so the compiler can
 * nicely work out which register we want, and chuck away the rest of
 * the code. At least it does so with a recent GCC (4.6.3).
 */
static __always_inline
void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u64 val)
{
	if (access == ARCH_TIMER_PHYS_ACCESS) {
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			asm volatile("mcr p15, 0, %0, c14, c2, 1" : : "r" ((u32)val));
			isb();
			break;
		case ARCH_TIMER_REG_CVAL:
			asm volatile("mcrr p15, 2, %Q0, %R0, c14" : : "r" (val));
			break;
		default:
			BUILD_BUG();
		}
	} else if (access == ARCH_TIMER_VIRT_ACCESS) {
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			asm volatile("mcr p15, 0, %0, c14, c3, 1" : : "r" ((u32)val));
			isb();
			break;
		case ARCH_TIMER_REG_CVAL:
			asm volatile("mcrr p15, 3, %Q0, %R0, c14" : : "r" (val));
			break;
		default:
			BUILD_BUG();
		}
	} else {
		BUILD_BUG();
	}
}

static __always_inline
u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg)
{
	u32 val = 0;

	if (access == ARCH_TIMER_PHYS_ACCESS) {
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			asm volatile("mrc p15, 0, %0, c14, c2, 1" : "=r" (val));
			break;
		default:
			BUILD_BUG();
		}
	} else if (access == ARCH_TIMER_VIRT_ACCESS) {
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			asm volatile("mrc p15, 0, %0, c14, c3, 1" : "=r" (val));
			break;
		default:
			BUILD_BUG();
		}
	} else {
		BUILD_BUG();
	}

	return val;
}

static inline u32 arch_timer_get_cntfrq(void)
{
	u32 val;
	asm volatile("mrc p15, 0, %0, c14, c0, 0" : "=r" (val));
	return val;
}

static inline u64 __arch_counter_get_cntpct(void)
{
	u64 cval;

	isb();
	asm volatile("mrrc p15, 0, %Q0, %R0, c14" : "=r" (cval));
	return cval;
}

static inline u64 __arch_counter_get_cntpct_stable(void)
{
	return __arch_counter_get_cntpct();
}

static inline u64 __arch_counter_get_cntvct(void)
{
	u64 cval;

	isb();
	asm volatile("mrrc p15, 1, %Q0, %R0, c14" : "=r" (cval));
	return cval;
}

static inline u64 __arch_counter_get_cntvct_stable(void)
{
	return __arch_counter_get_cntvct();
}

static inline u32 arch_timer_get_cntkctl(void)
{
	u32 cntkctl;
	asm volatile("mrc p15, 0, %0, c14, c1, 0" : "=r" (cntkctl));
	return cntkctl;
}

static inline void arch_timer_set_cntkctl(u32 cntkctl)
{
	asm volatile("mcr p15, 0, %0, c14, c1, 0" : : "r" (cntkctl));
	isb();
}

static inline void arch_timer_set_evtstrm_feature(void)
{
	elf_hwcap |= HWCAP_EVTSTRM;
}

static inline bool arch_timer_have_evtstrm_feature(void)
{
	return elf_hwcap & HWCAP_EVTSTRM;
}
#endif

#endif


code:
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_ARCHRANDOM_H
#define _ASM_ARCHRANDOM_H

static inline bool __init smccc_probe_trng(void)
{
	return false;
}

#include <asm-generic/archrandom.h>

#endif /* _ASM_ARCHRANDOM_H */


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * arch/arm/include/asm/arm-cci.h
 *
 * Copyright (C) 2015 ARM Ltd.
 */

#ifndef __ASM_ARM_CCI_H
#define __ASM_ARM_CCI_H

#ifdef CONFIG_MCPM
#include <asm/mcpm.h>

/*
 * We don't have a reliable way of detecting whether,
 * if we have access to secure-only registers, unless
 * mcpm is registered.
 */
static inline bool platform_has_secure_cci_access(void)
{
	return mcpm_is_available();
}

#else
static inline bool platform_has_secure_cci_access(void)
{
	return false;
}
#endif

#endif


code: #include <generated/asm-offsets.h>


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  arch/arm/include/asm/assembler.h
 *
 *  Copyright (C) 1996-2000 Russell King
 *
 *  This file contains arm architecture specific defines
 *  for the different processors.
 *
 *  Do not include any C declarations in this file - it is included by
 *  assembler source.
 */
#ifndef __ASM_ASSEMBLER_H__
#define __ASM_ASSEMBLER_H__

#ifndef __ASSEMBLY__
#error "Only include this from assembly code"
#endif

#include <asm/ptrace.h>
#include <asm/opcodes-virt.h>
#include <asm/asm-offsets.h>
#include <asm/page.h>
#include <asm/thread_info.h>
#include <asm/uaccess-asm.h>

#define IOMEM(x)	(x)

/*
 * Endian independent macros for shifting bytes within registers.
 */
#ifndef __ARMEB__
#define lspull          lsr
#define lspush          lsl
#define get_byte_0      lsl #0
#define get_byte_1	lsr #8
#define get_byte_2	lsr #16
#define get_byte_3	lsr #24
#define put_byte_0      lsl #0
#define put_byte_1	lsl #8
#define put_byte_2	lsl #16
#define put_byte_3	lsl #24
#else
#define lspull          lsl
#define lspush          lsr
#define get_byte_0	lsr #24
#define get_byte_1	lsr #16
#define get_byte_2	lsr #8
#define get_byte_3      lsl #0
#define put_byte_0	lsl #24
#define put_byte_1	lsl #16
#define put_byte_2	lsl #8
#define put_byte_3      lsl #0
#endif

/* Select code for any configuration running in BE8 mode */
#ifdef CONFIG_CPU_ENDIAN_BE8
#define ARM_BE8(code...) code
#else
#define ARM_BE8(code...)
#endif

/*
 * Data preload for architectures that support it
 */
#if __LINUX_ARM_ARCH__ >= 5
#define PLD(code...)	code
#else
#define PLD(code...)
#endif

/*
 * This can be used to enable code to cacheline align the destination
 * pointer when bulk writing to memory.  Experiments on StrongARM and
 * XScale didn't show this a worthwhile thing to do when the cache is not
 * set to write-allocate (this would need further testing on XScale when WA
 * is used).
 *
 * On Feroceon there is much to gain however, regardless of cache mode.
 */
#ifdef CONFIG_CPU_FEROCEON
#define CALGN(code...) code
#else
#define CALGN(code...)
#endif

#define IMM12_MASK 0xfff

/* the frame pointer used for stack unwinding */
ARM(	fpreg	.req	r11	)
THUMB(	fpreg	.req	r7	)

/*
 * Enable and disable interrupts
 */
#if __LINUX_ARM_ARCH__ >= 6
	.macro	disable_irq_notrace
	cpsid	i
	.endm

	.macro	enable_irq_notrace
	cpsie	i
	.endm
#else
	.macro	disable_irq_notrace
	msr	cpsr_c, #PSR_I_BIT | SVC_MODE
	.endm

	.macro	enable_irq_notrace
	msr	cpsr_c, #SVC_MODE
	.endm
#endif

#if __LINUX_ARM_ARCH__ < 7
	.macro	dsb, args
	mcr	p15, 0, r0, c7, c10, 4
	.endm

	.macro	isb, args
	mcr	p15, 0, r0, c7, c5, 4
	.endm
#endif

	.macro asm_trace_hardirqs_off, save=1
#if defined(CONFIG_TRACE_IRQFLAGS)
	.if \save
	stmdb   sp!, {r0-r3, ip, lr}
	.endif
	bl	trace_hardirqs_off
	.if \save
	ldmia	sp!, {r0-r3, ip, lr}
	.endif
#endif
	.endm

	.macro asm_trace_hardirqs_on, cond=al, save=1
#if defined(CONFIG_TRACE_IRQFLAGS)
	/*
	 * actually the registers should be pushed and pop'd conditionally, but
	 * after bl the flags are certainly clobbered
	 */
	.if \save
	stmdb   sp!, {r0-r3, ip, lr}
	.endif
	bl\cond	trace_hardirqs_on
	.if \save
	ldmia	sp!, {r0-r3, ip, lr}
	.endif
#endif
	.endm

	.macro disable_irq, save=1
	disable_irq_notrace
	asm_trace_hardirqs_off \save
	.endm

	.macro enable_irq
	asm_trace_hardirqs_on
	enable_irq_notrace
	.endm
/*
 * Save the current IRQ state and disable IRQs.  Note that this macro
 * assumes FIQs are enabled, and that the processor is in SVC mode.
 */
	.macro	save_and_disable_irqs, oldcpsr
#ifdef CONFIG_CPU_V7M
	mrs	\oldcpsr, primask
#else
	mrs	\oldcpsr, cpsr
#endif
	disable_irq
	.endm

	.macro	save_and_disable_irqs_notrace, oldcpsr
#ifdef CONFIG_CPU_V7M
	mrs	\oldcpsr, primask
#else
	mrs	\oldcpsr, cpsr
#endif
	disable_irq_notrace
	.endm

/*
 * Restore interrupt state previously stored in a register.  We don't
 * guarantee that this will preserve the flags.
 */
	.macro	restore_irqs_notrace, oldcpsr
#ifdef CONFIG_CPU_V7M
	msr	primask, \oldcpsr
#else
	msr	cpsr_c, \oldcpsr
#endif
	.endm

	.macro restore_irqs, oldcpsr
	tst	\oldcpsr, #PSR_I_BIT
	asm_trace_hardirqs_on cond=eq
	restore_irqs_notrace \oldcpsr
	.endm

/*
 * Assembly version of "adr rd, BSYM(sym)".  This should only be used to
 * reference local symbols in the same assembly file which are to be
 * resolved by the assembler.  Other usage is undefined.
 */
	.irp	c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo
	.macro	badr\c, rd, sym
#ifdef CONFIG_THUMB2_KERNEL
	adr\c	\rd, \sym + 1
#else
	adr\c	\rd, \sym
#endif
	.endm
	.endr

/*
 * Get current thread_info.
 */
	.macro	get_thread_info, rd
	/* thread_info is the first member of struct task_struct */
	get_current \rd
	.endm

/*
 * Increment/decrement the preempt count.
 */
#ifdef CONFIG_PREEMPT_COUNT
	.macro	inc_preempt_count, ti, tmp
	ldr	\tmp, [\ti, #TI_PREEMPT]	@ get preempt count
	add	\tmp, \tmp, #1			@ increment it
	str	\tmp, [\ti, #TI_PREEMPT]
	.endm

	.macro	dec_preempt_count, ti, tmp
	ldr	\tmp, [\ti, #TI_PREEMPT]	@ get preempt count
	sub	\tmp, \tmp, #1			@ decrement it
	str	\tmp, [\ti, #TI_PREEMPT]
	.endm

	.macro	dec_preempt_count_ti, ti, tmp
	get_thread_info \ti
	dec_preempt_count \ti, \tmp
	.endm
#else
	.macro	inc_preempt_count, ti, tmp
	.endm

	.macro	dec_preempt_count, ti, tmp
	.endm

	.macro	dec_preempt_count_ti, ti, tmp
	.endm
#endif

#define USERL(l, x...)				\
9999:	x;					\
	.pushsection __ex_table,"a";		\
	.align	3;				\
	.long	9999b,l;			\
	.popsection

#define USER(x...)	USERL(9001f, x)

#ifdef CONFIG_SMP
#define ALT_SMP(instr...)					\
9998:	instr
/*
 * Note: if you get assembler errors from ALT_UP() when building with
 * CONFIG_THUMB2_KERNEL, you almost certainly need to use
 * ALT_SMP( W(instr) ... )
 */
#define ALT_UP(instr...)					\
	.pushsection ".alt.smp.init", "a"			;\
	.align	2						;\
	.long	9998b - .					;\
9997:	instr							;\
	.if . - 9997b == 2					;\
		nop						;\
	.endif							;\
	.if . - 9997b != 4					;\
		.error "ALT_UP() content must assemble to exactly 4 bytes";\
	.endif							;\
	.popsection
#define ALT_UP_B(label)					\
	.pushsection ".alt.smp.init", "a"			;\
	.align	2						;\
	.long	9998b - .					;\
	W(b)	. + (label - 9998b)					;\
	.popsection
#else
#define ALT_SMP(instr...)
#define ALT_UP(instr...) instr
#define ALT_UP_B(label) b label
#endif

	/*
	 * this_cpu_offset - load the per-CPU offset of this CPU into
	 * 		     register 'rd'
	 */
	.macro		this_cpu_offset, rd:req
#ifdef CONFIG_SMP
ALT_SMP(mrc		p15, 0, \rd, c13, c0, 4)
#ifdef CONFIG_CPU_V6
ALT_UP_B(.L1_\@)
.L0_\@:
	.subsection	1
.L1_\@: ldr_va		\rd, __per_cpu_offset
	b		.L0_\@
	.previous
#endif
#else
	mov		\rd, #0
#endif
	.endm

	/*
	 * set_current - store the task pointer of this CPU's current task
	 */
	.macro		set_current, rn:req, tmp:req
#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP)
9998:	mcr		p15, 0, \rn, c13, c0, 3		@ set TPIDRURO register
#ifdef CONFIG_CPU_V6
ALT_UP_B(.L0_\@)
	.subsection	1
.L0_\@: str_va		\rn, __current, \tmp
	b		.L1_\@
	.previous
.L1_\@:
#endif
#else
	str_va		\rn, __current, \tmp
#endif
	.endm

	/*
	 * get_current - load the task pointer of this CPU's current task
	 */
	.macro		get_current, rd:req
#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP)
9998:	mrc		p15, 0, \rd, c13, c0, 3		@ get TPIDRURO register
#ifdef CONFIG_CPU_V6
ALT_UP_B(.L0_\@)
	.subsection	1
.L0_\@: ldr_va		\rd, __current
	b		.L1_\@
	.previous
.L1_\@:
#endif
#else
	ldr_va		\rd, __current
#endif
	.endm

	/*
	 * reload_current - reload the task pointer of this CPU's current task
	 *		    into the TLS register
	 */
	.macro		reload_current, t1:req, t2:req
#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP)
#ifdef CONFIG_CPU_V6
ALT_SMP(nop)
ALT_UP_B(.L0_\@)
#endif
	ldr_this_cpu	\t1, __entry_task, \t1, \t2
	mcr		p15, 0, \t1, c13, c0, 3		@ store in TPIDRURO
.L0_\@:
#endif
	.endm

/*
 * Instruction barrier
 */
	.macro	instr_sync
#if __LINUX_ARM_ARCH__ >= 7
	isb
#elif __LINUX_ARM_ARCH__ == 6
	mcr	p15, 0, r0, c7, c5, 4
#endif
	.endm

/*
 * SMP data memory barrier
 */
	.macro	smp_dmb mode
#ifdef CONFIG_SMP
#if __LINUX_ARM_ARCH__ >= 7
	.ifeqs "\mode","arm"
	ALT_SMP(dmb	ish)
	.else
	ALT_SMP(W(dmb)	ish)
	.endif
#elif __LINUX_ARM_ARCH__ == 6
	ALT_SMP(mcr	p15, 0, r0, c7, c10, 5)	@ dmb
#else
#error Incompatible SMP platform
#endif
	.ifeqs "\mode","arm"
	ALT_UP(nop)
	.else
	ALT_UP(W(nop))
	.endif
#endif
	.endm

#if defined(CONFIG_CPU_V7M)
	/*
	 * setmode is used to assert to be in svc mode during boot. For v7-M
	 * this is done in __v7m_setup, so setmode can be empty here.
	 */
	.macro	setmode, mode, reg
	.endm
#elif defined(CONFIG_THUMB2_KERNEL)
	.macro	setmode, mode, reg
	mov	\reg, #\mode
	msr	cpsr_c, \reg
	.endm
#else
	.macro	setmode, mode, reg
	msr	cpsr_c, #\mode
	.endm
#endif

/*
 * Helper macro to enter SVC mode cleanly and mask interrupts. reg is
 * a scratch register for the macro to overwrite.
 *
 * This macro is intended for forcing the CPU into SVC mode at boot time.
 * you cannot return to the original mode.
 */
.macro safe_svcmode_maskall reg:req
#if __LINUX_ARM_ARCH__ >= 6 && !defined(CONFIG_CPU_V7M)
	mrs	\reg , cpsr
	eor	\reg, \reg, #HYP_MODE
	tst	\reg, #MODE_MASK
	bic	\reg , \reg , #MODE_MASK
	orr	\reg , \reg , #PSR_I_BIT | PSR_F_BIT | SVC_MODE
THUMB(	orr	\reg , \reg , #PSR_T_BIT	)
	bne	1f
	orr	\reg, \reg, #PSR_A_BIT
	badr	lr, 2f
	msr	spsr_cxsf, \reg
	__MSR_ELR_HYP(14)
	__ERET
1:	msr	cpsr_c, \reg
2:
#else
/*
 * workaround for possibly broken pre-v6 hardware
 * (akita, Sharp Zaurus C-1000, PXA270-based)
 */
	setmode	PSR_F_BIT | PSR_I_BIT | SVC_MODE, \reg
#endif
.endm

/*
 * STRT/LDRT access macros with ARM and Thumb-2 variants
 */
#ifdef CONFIG_THUMB2_KERNEL

	.macro	usraccoff, instr, reg, ptr, inc, off, cond, abort, t=TUSER()
9999:
	.if	\inc == 1
	\instr\()b\t\cond\().w \reg, [\ptr, #\off]
	.elseif	\inc == 4
	\instr\t\cond\().w \reg, [\ptr, #\off]
	.else
	.error	"Unsupported inc macro argument"
	.endif

	.pushsection __ex_table,"a"
	.align	3
	.long	9999b, \abort
	.popsection
	.endm

	.macro	usracc, instr, reg, ptr, inc, cond, rept, abort
	@ explicit IT instruction needed because of the label
	@ introduced by the USER macro
	.ifnc	\cond,al
	.if	\rept == 1
	itt	\cond
	.elseif	\rept == 2
	ittt	\cond
	.else
	.error	"Unsupported rept macro argument"
	.endif
	.endif

	@ Slightly optimised to avoid incrementing the pointer twice
	usraccoff \instr, \reg, \ptr, \inc, 0, \cond, \abort
	.if	\rept == 2
	usraccoff \instr, \reg, \ptr, \inc, \inc, \cond, \abort
	.endif

	add\cond \ptr, #\rept * \inc
	.endm

#else	/* !CONFIG_THUMB2_KERNEL */

	.macro	usracc, instr, reg, ptr, inc, cond, rept, abort, t=TUSER()
	.rept	\rept
9999:
	.if	\inc == 1
	\instr\()b\t\cond \reg, [\ptr], #\inc
	.elseif	\inc == 4
	\instr\t\cond \reg, [\ptr], #\inc
	.else
	.error	"Unsupported inc macro argument"
	.endif

	.pushsection __ex_table,"a"
	.align	3
	.long	9999b, \abort
	.popsection
	.endr
	.endm

#endif	/* CONFIG_THUMB2_KERNEL */

	.macro	strusr, reg, ptr, inc, cond=al, rept=1, abort=9001f
	usracc	str, \reg, \ptr, \inc, \cond, \rept, \abort
	.endm

	.macro	ldrusr, reg, ptr, inc, cond=al, rept=1, abort=9001f
	usracc	ldr, \reg, \ptr, \inc, \cond, \rept, \abort
	.endm

/* Utility macro for declaring string literals */
	.macro	string name:req, string
	.type \name , #object
\name:
	.asciz "\string"
	.size \name , . - \name
	.endm

	.irp	c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo
	.macro	ret\c, reg
#if __LINUX_ARM_ARCH__ < 6
	mov\c	pc, \reg
#else
	.ifeqs	"\reg", "lr"
	bx\c	\reg
	.else
	mov\c	pc, \reg
	.endif
#endif
	.endm
	.endr

	.macro	ret.w, reg
	ret	\reg
#ifdef CONFIG_THUMB2_KERNEL
	nop
#endif
	.endm

	.macro	bug, msg, line
#ifdef CONFIG_THUMB2_KERNEL
1:	.inst	0xde02
#else
1:	.inst	0xe7f001f2
#endif
#ifdef CONFIG_DEBUG_BUGVERBOSE
	.pushsection .rodata.str, "aMS", %progbits, 1
2:	.asciz	"\msg"
	.popsection
	.pushsection __bug_table, "aw"
	.align	2
	.word	1b, 2b
	.hword	\line
	.popsection
#endif
	.endm

#ifdef CONFIG_KPROBES
#define _ASM_NOKPROBE(entry)				\
	.pushsection "_kprobe_blacklist", "aw" ;	\
	.balign 4 ;					\
	.long entry;					\
	.popsection
#else
#define _ASM_NOKPROBE(entry)
#endif

	.macro		__adldst_l, op, reg, sym, tmp, c
	.if		__LINUX_ARM_ARCH__ < 7
	ldr\c		\tmp, .La\@
	.subsection	1
	.align		2
.La\@:	.long		\sym - .Lpc\@
	.previous
	.else
	.ifnb		\c
 THUMB(	ittt		\c			)
	.endif
	movw\c		\tmp, #:lower16:\sym - .Lpc\@
	movt\c		\tmp, #:upper16:\sym - .Lpc\@
	.endif

#ifndef CONFIG_THUMB2_KERNEL
	.set		.Lpc\@, . + 8			// PC bias
	.ifc		\op, add
	add\c		\reg, \tmp, pc
	.else
	\op\c		\reg, [pc, \tmp]
	.endif
#else
.Lb\@:	add\c		\tmp, \tmp, pc
	/*
	 * In Thumb-2 builds, the PC bias depends on whether we are currently
	 * emitting into a .arm or a .thumb section. The size of the add opcode
	 * above will be 2 bytes when emitting in Thumb mode and 4 bytes when
	 * emitting in ARM mode, so let's use this to account for the bias.
	 */
	.set		.Lpc\@, . + (. - .Lb\@)

	.ifnc		\op, add
	\op\c		\reg, [\tmp]
	.endif
#endif
	.endm

	/*
	 * mov_l - move a constant value or [relocated] address into a register
	 */
	.macro		mov_l, dst:req, imm:req, cond
	.if		__LINUX_ARM_ARCH__ < 7
	ldr\cond	\dst, =\imm
	.else
	movw\cond	\dst, #:lower16:\imm
	movt\cond	\dst, #:upper16:\imm
	.endif
	.endm

	/*
	 * adr_l - adr pseudo-op with unlimited range
	 *
	 * @dst: destination register
	 * @sym: name of the symbol
	 * @cond: conditional opcode suffix
	 */
	.macro		adr_l, dst:req, sym:req, cond
	__adldst_l	add, \dst, \sym, \dst, \cond
	.endm

	/*
	 * ldr_l - ldr <literal> pseudo-op with unlimited range
	 *
	 * @dst: destination register
	 * @sym: name of the symbol
	 * @cond: conditional opcode suffix
	 */
	.macro		ldr_l, dst:req, sym:req, cond
	__adldst_l	ldr, \dst, \sym, \dst, \cond
	.endm

	/*
	 * str_l - str <literal> pseudo-op with unlimited range
	 *
	 * @src: source register
	 * @sym: name of the symbol
	 * @tmp: mandatory scratch register
	 * @cond: conditional opcode suffix
	 */
	.macro		str_l, src:req, sym:req, tmp:req, cond
	__adldst_l	str, \src, \sym, \tmp, \cond
	.endm

	.macro		__ldst_va, op, reg, tmp, sym, cond, offset
#if __LINUX_ARM_ARCH__ >= 7 || \
    !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \
    (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
	mov_l		\tmp, \sym, \cond
#else
	/*
	 * Avoid a literal load, by emitting a sequence of ADD/LDR instructions
	 * with the appropriate relocations. The combined sequence has a range
	 * of -/+ 256 MiB, which should be sufficient for the core kernel and
	 * for modules loaded into the module region.
	 */
	.globl		\sym
	.reloc		.L0_\@, R_ARM_ALU_PC_G0_NC, \sym
	.reloc		.L1_\@, R_ARM_ALU_PC_G1_NC, \sym
	.reloc		.L2_\@, R_ARM_LDR_PC_G2, \sym
.L0_\@: sub\cond	\tmp, pc, #8 - \offset
.L1_\@: sub\cond	\tmp, \tmp, #4 - \offset
.L2_\@:
#endif
	\op\cond	\reg, [\tmp, #\offset]
	.endm

	/*
	 * ldr_va - load a 32-bit word from the virtual address of \sym
	 */
	.macro		ldr_va, rd:req, sym:req, cond, tmp, offset=0
	.ifnb		\tmp
	__ldst_va	ldr, \rd, \tmp, \sym, \cond, \offset
	.else
	__ldst_va	ldr, \rd, \rd, \sym, \cond, \offset
	.endif
	.endm

	/*
	 * str_va - store a 32-bit word to the virtual address of \sym
	 */
	.macro		str_va, rn:req, sym:req, tmp:req, cond
	__ldst_va	str, \rn, \tmp, \sym, \cond, 0
	.endm

	/*
	 * ldr_this_cpu_armv6 - Load a 32-bit word from the per-CPU variable 'sym',
	 *			without using a temp register. Supported in ARM mode
	 *			only.
	 */
	.macro		ldr_this_cpu_armv6, rd:req, sym:req
	this_cpu_offset	\rd
	.globl		\sym
	.reloc		.L0_\@, R_ARM_ALU_PC_G0_NC, \sym
	.reloc		.L1_\@, R_ARM_ALU_PC_G1_NC, \sym
	.reloc		.L2_\@, R_ARM_LDR_PC_G2, \sym
	add		\rd, \rd, pc
.L0_\@: sub		\rd, \rd, #4
.L1_\@: sub		\rd, \rd, #0
.L2_\@: ldr		\rd, [\rd, #4]
	.endm

	/*
	 * ldr_this_cpu - Load a 32-bit word from the per-CPU variable 'sym'
	 *		  into register 'rd', which may be the stack pointer,
	 *		  using 't1' and 't2' as general temp registers. These
	 *		  are permitted to overlap with 'rd' if != sp
	 */
	.macro		ldr_this_cpu, rd:req, sym:req, t1:req, t2:req
#ifndef CONFIG_SMP
	ldr_va		\rd, \sym, tmp=\t1
#elif __LINUX_ARM_ARCH__ >= 7 || \
      !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \
      (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
	this_cpu_offset	\t1
	mov_l		\t2, \sym
	ldr		\rd, [\t1, \t2]
#else
	ldr_this_cpu_armv6 \rd, \sym
#endif
	.endm

	/*
	 * rev_l - byte-swap a 32-bit value
	 *
	 * @val: source/destination register
	 * @tmp: scratch register
	 */
	.macro		rev_l, val:req, tmp:req
	.if		__LINUX_ARM_ARCH__ < 6
	eor		\tmp, \val, \val, ror #16
	bic		\tmp, \tmp, #0x00ff0000
	mov		\val, \val, ror #8
	eor		\val, \val, \tmp, lsr #8
	.else
	rev		\val, \val
	.endif
	.endm

	.if		__LINUX_ARM_ARCH__ < 6
	.set		.Lrev_l_uses_tmp, 1
	.else
	.set		.Lrev_l_uses_tmp, 0
	.endif

	/*
	 * bl_r - branch and link to register
	 *
	 * @dst: target to branch to
	 * @c: conditional opcode suffix
	 */
	.macro		bl_r, dst:req, c
	.if		__LINUX_ARM_ARCH__ < 6
	mov\c		lr, pc
	mov\c		pc, \dst
	.else
	blx\c		\dst
	.endif
	.endm

#endif /* __ASM_ASSEMBLER_H__ */