code:
/* SPDX-License-Identifier: GPL-2.0 */
/* ld script to make ARM Linux kernel
 * taken from the i386 version by Russell King
 * Written by Martin Mares <mj@atrey.karlin.mff.cuni.cz>
 */

/* No __ro_after_init data in the .rodata section - which will always be ro */
#define RO_AFTER_INIT_DATA

#include <linux/sizes.h>

#include <asm/vmlinux.lds.h>
#include <asm/cache.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
#include <asm/mpu.h>
#include <asm/page.h>

OUTPUT_ARCH(arm)
ENTRY(stext)

#ifndef __ARMEB__
jiffies = jiffies_64;
#else
jiffies = jiffies_64 + 4;
#endif

SECTIONS
{
	/*
	 * XXX: The linker does not define how output sections are
	 * assigned to input sections when there are multiple statements
	 * matching the same input section name.  There is no documented
	 * order of matching.
	 *
	 * unwind exit sections must be discarded before the rest of the
	 * unwind sections get included.
	 */
	/DISCARD/ : {
		ARM_DISCARD
		*(.alt.smp.init)
		*(.pv_table)
#ifndef CONFIG_ARM_UNWIND
		*(.ARM.exidx) *(.ARM.exidx.*)
		*(.ARM.extab) *(.ARM.extab.*)
#endif
	}

	. = XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR);
	_xiprom = .;			/* XIP ROM area to be mapped */

	.head.text : {
		_text = .;
		HEAD_TEXT
	}

	.text : {			/* Real text segment		*/
		_stext = .;		/* Text and read-only data	*/
		ARM_TEXT
	}

	RO_DATA(PAGE_SIZE)

	. = ALIGN(4);
	__ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) {
		__start___ex_table = .;
		ARM_MMU_KEEP(*(__ex_table))
		__stop___ex_table = .;
	}

#ifdef CONFIG_ARM_UNWIND
	ARM_UNWIND_SECTIONS
#endif

	_etext = .;			/* End of text and rodata section */

	ARM_VECTORS
	INIT_TEXT_SECTION(8)
	.exit.text : {
		ARM_EXIT_KEEP(EXIT_TEXT)
	}
	.init.proc.info : {
		ARM_CPU_DISCARD(PROC_INFO)
	}
	.init.arch.info : {
		__arch_info_begin = .;
		*(.arch.info.init)
		__arch_info_end = .;
	}
	.init.tagtable : {
		__tagtable_begin = .;
		*(.taglist.init)
		__tagtable_end = .;
	}
	.init.rodata : {
		INIT_SETUP(16)
		INIT_CALLS
		CON_INITCALL
		INIT_RAM_FS
	}

#ifdef CONFIG_ARM_MPU
	. = ALIGN(SZ_128K);
#endif
	_exiprom = .;			/* End of XIP ROM area */

/*
 * From this point, stuff is considered writable and will be copied to RAM
 */
	__data_loc = ALIGN(4);		/* location in file */
	. = PAGE_OFFSET + TEXT_OFFSET;	/* location in memory */
#undef LOAD_OFFSET
#define LOAD_OFFSET (PAGE_OFFSET + TEXT_OFFSET - __data_loc)

	. = ALIGN(THREAD_SIZE);
	_sdata = .;
	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE)
	.data.ro_after_init : AT(ADDR(.data.ro_after_init) - LOAD_OFFSET) {
		*(.data..ro_after_init)
	}
	_edata = .;

	. = ALIGN(PAGE_SIZE);
	__init_begin = .;
	.init.data : AT(ADDR(.init.data) - LOAD_OFFSET) {
		INIT_DATA
	}
	.exit.data : AT(ADDR(.exit.data) - LOAD_OFFSET) {
		ARM_EXIT_KEEP(EXIT_DATA)
	}
#ifdef CONFIG_SMP
	PERCPU_SECTION(L1_CACHE_BYTES)
#endif

#ifdef CONFIG_HAVE_TCM
	ARM_TCM
#endif

	/*
	 * End of copied data. We need a dummy section to get its LMA.
	 * Also located before final ALIGN() as trailing padding is not stored
	 * in the resulting binary file and useless to copy.
	 */
	.data.endmark : AT(ADDR(.data.endmark) - LOAD_OFFSET) { }
	_edata_loc = LOADADDR(.data.endmark);

	. = ALIGN(PAGE_SIZE);
	__init_end = .;

	BSS_SECTION(0, 0, 8)
#ifdef CONFIG_ARM_MPU
	. = ALIGN(PMSAv8_MINALIGN);
#endif
	_end = .;

	STABS_DEBUG
	DWARF_DEBUG
	ARM_DETAILS

	ARM_ASSERTS
}

/*
 * These must never be empty
 * If you have to comment these two assert statements out, your
 * binutils is too old (for other reasons as well)
 */
ASSERT((__proc_info_end - __proc_info_begin), "missing CPU support")
#ifndef CONFIG_COMPILE_TEST
ASSERT((__arch_info_end - __arch_info_begin), "no machine record defined")
#endif

#ifdef CONFIG_XIP_DEFLATED_DATA
/*
 * The .bss is used as a stack area for __inflate_kernel_data() whose stack
 * frame is 9568 bytes. Make sure it has extra room left.
 */
ASSERT((_end - __bss_start) >= 12288, ".bss too small for CONFIG_XIP_DEFLATED_DATA")
#endif

#if defined(CONFIG_ARM_MPU) && !defined(CONFIG_COMPILE_TEST)
/*
 * Due to PMSAv7 restriction on base address and size we have to
 * enforce minimal alignment restrictions. It was seen that weaker
 * alignment restriction on _xiprom will likely force XIP address
 * space spawns multiple MPU regions thus it is likely we run in
 * situation when we are reprogramming MPU region we run on with
 * something which doesn't cover reprogramming code itself, so as soon
 * as we update MPU settings we'd immediately try to execute straight
 * from background region which is XN.
 * It seem that alignment in 1M should suit most users.
 * _exiprom is aligned as 1/8 of 1M so can be covered by subregion
 * disable
 */
ASSERT(!(_xiprom & (SZ_1M - 1)), "XIP start address may cause MPU programming issues")
ASSERT(!(_exiprom & (SZ_128K - 1)), "XIP end address may cause MPU programming issues")
#endif


code:
/* SPDX-License-Identifier: GPL-2.0 */
/* ld script to make ARM Linux kernel
 * taken from the i386 version by Russell King
 * Written by Martin Mares <mj@atrey.karlin.mff.cuni.cz>
 */

#ifdef CONFIG_XIP_KERNEL
#include "vmlinux-xip.lds.S"
#else

#include <linux/pgtable.h>
#include <asm/vmlinux.lds.h>
#include <asm/cache.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
#include <asm/mpu.h>
#include <asm/page.h>

OUTPUT_ARCH(arm)
ENTRY(stext)

#ifndef __ARMEB__
jiffies = jiffies_64;
#else
jiffies = jiffies_64 + 4;
#endif

SECTIONS
{
	/*
	 * XXX: The linker does not define how output sections are
	 * assigned to input sections when there are multiple statements
	 * matching the same input section name.  There is no documented
	 * order of matching.
	 *
	 * unwind exit sections must be discarded before the rest of the
	 * unwind sections get included.
	 */
	/DISCARD/ : {
		ARM_DISCARD
#ifndef CONFIG_SMP_ON_UP
		*(.alt.smp.init)
#endif
#ifndef CONFIG_ARM_UNWIND
		*(.ARM.exidx) *(.ARM.exidx.*)
		*(.ARM.extab) *(.ARM.extab.*)
#endif
	}

	. = KERNEL_OFFSET + TEXT_OFFSET;
	.head.text : {
		_text = .;
		HEAD_TEXT
	}

#ifdef CONFIG_STRICT_KERNEL_RWX
	. = ALIGN(1<<SECTION_SHIFT);
#endif

#ifdef CONFIG_ARM_MPU
	. = ALIGN(PMSAv8_MINALIGN);
#endif
	.text : {			/* Real text segment		*/
		_stext = .;		/* Text and read-only data	*/
		ARM_TEXT
	}

#ifdef CONFIG_DEBUG_ALIGN_RODATA
	. = ALIGN(1<<SECTION_SHIFT);
#endif
	_etext = .;			/* End of text section */

	RO_DATA(PAGE_SIZE)

	. = ALIGN(4);
	__ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) {
		__start___ex_table = .;
		ARM_MMU_KEEP(*(__ex_table))
		__stop___ex_table = .;
	}

#ifdef CONFIG_ARM_UNWIND
	ARM_UNWIND_SECTIONS
#endif

#ifdef CONFIG_STRICT_KERNEL_RWX
	. = ALIGN(1<<SECTION_SHIFT);
#else
	. = ALIGN(PAGE_SIZE);
#endif
	__init_begin = .;

	ARM_VECTORS
	INIT_TEXT_SECTION(8)
	.exit.text : {
		ARM_EXIT_KEEP(EXIT_TEXT)
	}
	.init.proc.info : {
		ARM_CPU_DISCARD(PROC_INFO)
	}
	.init.arch.info : {
		__arch_info_begin = .;
		*(.arch.info.init)
		__arch_info_end = .;
	}
	.init.tagtable : {
		__tagtable_begin = .;
		*(.taglist.init)
		__tagtable_end = .;
	}
#ifdef CONFIG_SMP_ON_UP
	.init.smpalt : {
		__smpalt_begin = .;
		*(.alt.smp.init)
		__smpalt_end = .;
	}
#endif
	.init.pv_table : {
		__pv_table_begin = .;
		*(.pv_table)
		__pv_table_end = .;
	}

	INIT_DATA_SECTION(16)

	.exit.data : {
		ARM_EXIT_KEEP(EXIT_DATA)
	}

#ifdef CONFIG_SMP
	PERCPU_SECTION(L1_CACHE_BYTES)
#endif

#ifdef CONFIG_HAVE_TCM
	ARM_TCM
#endif

#ifdef CONFIG_STRICT_KERNEL_RWX
	. = ALIGN(1<<SECTION_SHIFT);
#else
	. = ALIGN(THREAD_ALIGN);
#endif
	__init_end = .;

	_sdata = .;
	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
	_edata = .;

	BSS_SECTION(0, 0, 0)
#ifdef CONFIG_ARM_MPU
	. = ALIGN(PMSAv8_MINALIGN);
#endif
	_end = .;

	STABS_DEBUG
	DWARF_DEBUG
	ARM_DETAILS

	ARM_ASSERTS
}

#ifdef CONFIG_STRICT_KERNEL_RWX
/*
 * Without CONFIG_DEBUG_ALIGN_RODATA, __start_rodata_section_aligned will
 * be the first section-aligned location after __start_rodata. Otherwise,
 * it will be equal to __start_rodata.
 */
__start_rodata_section_aligned = ALIGN(__start_rodata, 1 << SECTION_SHIFT);
#endif

/*
 * These must never be empty
 * If you have to comment these two assert statements out, your
 * binutils is too old (for other reasons as well)
 */
ASSERT((__proc_info_end - __proc_info_begin), "missing CPU support")
#ifndef CONFIG_COMPILE_TEST
ASSERT((__arch_info_end - __arch_info_begin), "no machine record defined")
#endif

#endif /* CONFIG_XIP_KERNEL */

repo_name: angery/linux


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * linux/arch/arm/kernel/xscale-cp0.c
 *
 * XScale DSP and iWMMXt coprocessor context switching and handling
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/io.h>
#include <asm/thread_notify.h>
#include <asm/cputype.h>

asm("	.arch armv5te\n");

static inline void dsp_save_state(u32 *state)
{
	__asm__ __volatile__ (
		"mrrc	p0, 0, %0, %1, c0\n"
		: "=r" (state[0]), "=r" (state[1]));
}

static inline void dsp_load_state(u32 *state)
{
	__asm__ __volatile__ (
		"mcrr	p0, 0, %0, %1, c0\n"
		: : "r" (state[0]), "r" (state[1]));
}

static int dsp_do(struct notifier_block *self, unsigned long cmd, void *t)
{
	struct thread_info *thread = t;

	switch (cmd) {
	case THREAD_NOTIFY_FLUSH:
		thread->cpu_context.extra[0] = 0;
		thread->cpu_context.extra[1] = 0;
		break;

	case THREAD_NOTIFY_SWITCH:
		dsp_save_state(current_thread_info()->cpu_context.extra);
		dsp_load_state(thread->cpu_context.extra);
		break;
	}

	return NOTIFY_DONE;
}

static struct notifier_block dsp_notifier_block = {
	.notifier_call	= dsp_do,
};

#ifdef CONFIG_IWMMXT
static int iwmmxt_do(struct notifier_block *self, unsigned long cmd, void *t)
{
	struct thread_info *thread = t;

	switch (cmd) {
	case THREAD_NOTIFY_FLUSH:
		/*
		 * flush_thread() zeroes thread->fpstate, so no need
		 * to do anything here.
		 *
		 * FALLTHROUGH: Ensure we don't try to overwrite our newly
		 * initialised state information on the first fault.
		 */

	case THREAD_NOTIFY_EXIT:
		iwmmxt_task_release(thread);
		break;

	case THREAD_NOTIFY_SWITCH:
		iwmmxt_task_switch(thread);
		break;
	}

	return NOTIFY_DONE;
}

static struct notifier_block iwmmxt_notifier_block = {
	.notifier_call	= iwmmxt_do,
};
#endif

static u32 __init xscale_cp_access_read(void)
{
	u32 value;

	__asm__ __volatile__ (
		"mrc	p15, 0, %0, c15, c1, 0\n\t"
		: "=r" (value));

	return value;
}

static void __init xscale_cp_access_write(u32 value)
{
	u32 temp;

	__asm__ __volatile__ (
		"mcr	p15, 0, %1, c15, c1, 0\n\t"
		"mrc	p15, 0, %0, c15, c1, 0\n\t"
		"mov	%0, %0\n\t"
		"sub	pc, pc, #4\n\t"
		: "=r" (temp) : "r" (value));
}

/*
 * Detect whether we have a MAC coprocessor (40 bit register) or an
 * iWMMXt coprocessor (64 bit registers) by loading 00000100:00000000
 * into a coprocessor register and reading it back, and checking
 * whether the upper word survived intact.
 */
static int __init cpu_has_iwmmxt(void)
{
	u32 lo;
	u32 hi;

	/*
	 * This sequence is interpreted by the DSP coprocessor as:
	 *	mar	acc0, %2, %3
	 *	mra	%0, %1, acc0
	 *
	 * And by the iWMMXt coprocessor as:
	 *	tmcrr	wR0, %2, %3
	 *	tmrrc	%0, %1, wR0
	 */
	__asm__ __volatile__ (
		"mcrr	p0, 0, %2, %3, c0\n"
		"mrrc	p0, 0, %0, %1, c0\n"
		: "=r" (lo), "=r" (hi)
		: "r" (0), "r" (0x100));

	return !!hi;
}

/*
 * If we detect that the CPU has iWMMXt (and CONFIG_IWMMXT=y), we
 * disable CP0/CP1 on boot, and let call_fpe() and the iWMMXt lazy
 * switch code handle iWMMXt context switching.  If on the other
 * hand the CPU has a DSP coprocessor, we keep access to CP0 enabled
 * all the time, and save/restore acc0 on context switch in non-lazy
 * fashion.
 */
static int __init xscale_cp0_init(void)
{
	u32 cp_access;

	/* do not attempt to probe iwmmxt on non-xscale family CPUs */
	if (!cpu_is_xscale_family())
		return 0;

	cp_access = xscale_cp_access_read() & ~3;
	xscale_cp_access_write(cp_access | 1);

	if (cpu_has_iwmmxt()) {
#ifndef CONFIG_IWMMXT
		pr_warn("CAUTION: XScale iWMMXt coprocessor detected, but kernel support is missing.\n");
#else
		pr_info("XScale iWMMXt coprocessor detected.\n");
		elf_hwcap |= HWCAP_IWMMXT;
		thread_register_notifier(&iwmmxt_notifier_block);
#endif
	} else {
		pr_info("XScale DSP coprocessor detected.\n");
		thread_register_notifier(&dsp_notifier_block);
		cp_access |= 1;
	}

	xscale_cp_access_write(cp_access);

	return 0;
}

late_initcall(xscale_cp0_init);


code:
# SPDX-License-Identifier: GPL-2.0
#
# linux/arch/arm/lib/Makefile
#
# Copyright (C) 1995-2000 Russell King
#

lib-y		:= changebit.o csumipv6.o csumpartial.o               \
		   csumpartialcopy.o csumpartialcopyuser.o clearbit.o \
		   delay.o delay-loop.o findbit.o memchr.o memcpy.o   \
		   memmove.o memset.o setbit.o                        \
		   strchr.o strrchr.o                                 \
		   testchangebit.o testclearbit.o testsetbit.o        \
		   ashldi3.o ashrdi3.o lshrdi3.o muldi3.o             \
		   ucmpdi2.o lib1funcs.o div64.o                      \
		   io-readsb.o io-writesb.o io-readsl.o io-writesl.o  \
		   call_with_stack.o bswapsdi2.o

mmu-y		:= clear_user.o copy_page.o getuser.o putuser.o       \
		   copy_from_user.o copy_to_user.o

ifdef CONFIG_CC_IS_CLANG
  lib-y	+= backtrace-clang.o
else
  lib-y	+= backtrace.o
endif

# using lib_ here won't override already available weak symbols
obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o

lib-$(CONFIG_MMU) += $(mmu-y)

ifeq ($(CONFIG_CPU_32v3),y)
  lib-y	+= io-readsw-armv3.o io-writesw-armv3.o
else
  lib-y	+= io-readsw-armv4.o io-writesw-armv4.o
endif

$(obj)/csumpartialcopy.o:	$(obj)/csumpartialcopygeneric.S
$(obj)/csumpartialcopyuser.o:	$(obj)/csumpartialcopygeneric.S

ifeq ($(CONFIG_KERNEL_MODE_NEON),y)
  NEON_FLAGS			:= -march=armv7-a -mfloat-abi=softfp -mfpu=neon
  CFLAGS_xor-neon.o		+= $(NEON_FLAGS)
  obj-$(CONFIG_XOR_BLOCKS)	+= xor-neon.o
endif

obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o

path: arch/arm/lib/Makefile


code:
/* Copyright 1995, 1996, 1998, 1999, 2000, 2003, 2004, 2005
   Free Software Foundation, Inc.

This file is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2, or (at your option) any
later version.

In addition to the permissions in the GNU General Public License, the
Free Software Foundation gives you unlimited permission to link the
compiled version of this file into combinations with other programs,
and to distribute those combinations without any restriction coming
from the use of this file.  (The General Public License restrictions
do apply in other respects; for example, they cover modification of
the file, and distribution when not linked into a combine
executable.)

This file is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; see the file COPYING.  If not, write to
the Free Software Foundation, 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA.  */

#include <linux/linkage.h>
#include <asm/assembler.h>

#ifdef __ARMEB__
#define al r1
#define ah r0
#else
#define al r0
#define ah r1
#endif

ENTRY(__ashldi3)
ENTRY(__aeabi_llsl)

	subs	r3, r2, #32
	rsb	ip, r2, #32
	movmi	ah, ah, lsl r2
	movpl	ah, al, lsl r3
 ARM(	orrmi	ah, ah, al, lsr ip	)
 THUMB(	lsrmi	r3, al, ip		)
 THUMB(	orrmi	ah, ah, r3		)
	mov	al, al, lsl r2
	ret	lr

ENDPROC(__ashldi3)
ENDPROC(__aeabi_llsl)


code:
/* Copyright 1995, 1996, 1998, 1999, 2000, 2003, 2004, 2005
   Free Software Foundation, Inc.

This file is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2, or (at your option) any
later version.

In addition to the permissions in the GNU General Public License, the
Free Software Foundation gives you unlimited permission to link the
compiled version of this file into combinations with other programs,
and to distribute those combinations without any restriction coming
from the use of this file.  (The General Public License restrictions
do apply in other respects; for example, they cover modification of
the file, and distribution when not linked into a combine
executable.)

This file is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; see the file COPYING.  If not, write to
the Free Software Foundation, 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA.  */

#include <linux/linkage.h>
#include <asm/assembler.h>

#ifdef __ARMEB__
#define al r1
#define ah r0
#else
#define al r0
#define ah r1
#endif

ENTRY(__ashrdi3)
ENTRY(__aeabi_lasr)

	subs	r3, r2, #32
	rsb	ip, r2, #32
	movmi	al, al, lsr r2
	movpl	al, ah, asr r3
 ARM(	orrmi	al, al, ah, lsl ip	)
 THUMB(	lslmi	r3, ah, ip		)
 THUMB(	orrmi	al, al, r3		)
	mov	ah, ah, asr r2
	ret	lr

ENDPROC(__ashrdi3)
ENDPROC(__aeabi_lasr)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/backtrace-clang.S
 *
 *  Copyright (C) 2019 Nathan Huckleberry
 *
 */
#include <linux/kern_levels.h>
#include <linux/linkage.h>
#include <asm/assembler.h>
		.text

/* fp is 0 or stack frame */

#define frame	r4
#define sv_fp	r5
#define sv_pc	r6
#define mask	r7
#define sv_lr	r8
#define loglvl	r9

ENTRY(c_backtrace)

#if !defined(CONFIG_FRAME_POINTER) || !defined(CONFIG_PRINTK)
		ret	lr
ENDPROC(c_backtrace)
#else

/*
 * Clang does not store pc or sp in function prologues so we don't know exactly
 * where the function starts.
 *
 * We can treat the current frame's lr as the saved pc and the preceding
 * frame's lr as the current frame's lr, but we can't trace the most recent
 * call.  Inserting a false stack frame allows us to reference the function
 * called last in the stacktrace.
 *
 * If the call instruction was a bl we can look at the callers branch
 * instruction to calculate the saved pc.  We can recover the pc in most cases,
 * but in cases such as calling function pointers we cannot. In this case,
 * default to using the lr. This will be some address in the function, but will
 * not be the function start.
 *
 * Unfortunately due to the stack frame layout we can't dump r0 - r3, but these
 * are less frequently saved.
 *
 * Stack frame layout:
 * 		<larger addresses>
 * 		saved lr
 * 	frame=> saved fp
 * 		optionally saved caller registers (r4 - r10)
 * 		optionally saved arguments (r0 - r3)
 * 		<top of stack frame>
 * 		<smaller addresses>
 *
 * Functions start with the following code sequence:
 * corrected pc =>  stmfd sp!, {..., fp, lr}
 *		add fp, sp, #x
 *		stmfd sp!, {r0 - r3} (optional)
 *
 *
 *
 *
 *
 *
 * The diagram below shows an example stack setup for dump_stack.
 *
 * The frame for c_backtrace has pointers to the code of dump_stack. This is
 * why the frame of c_backtrace is used to for the pc calculation of
 * dump_stack. This is why we must move back a frame to print dump_stack.
 *
 * The stored locals for dump_stack are in dump_stack's frame. This means that
 * to fully print dump_stack's frame we need both the frame for dump_stack (for
 * locals) and the frame that was called by dump_stack (for pc).
 *
 * To print locals we must know where the function start is. If we read the
 * function prologue opcodes we can determine which variables are stored in the
 * stack frame.
 *
 * To find the function start of dump_stack we can look at the stored LR of
 * show_stack. It points at the instruction directly after the bl dump_stack.
 * We can then read the offset from the bl opcode to determine where the branch
 * takes us.  The address calculated must be the start of dump_stack.
 *
 * c_backtrace frame           dump_stack:
 * {[LR]    }  ============|   ...
 * {[FP]    }  =======|    |   bl c_backtrace
 *                    |    |=> ...
 * {[R4-R10]}         |
 * {[R0-R3] }         |        show_stack:
 * dump_stack frame   |        ...
 * {[LR]    } =============|   bl dump_stack
 * {[FP]    } <=======|    |=> ...
 * {[R4-R10]}
 * {[R0-R3] }
 */

		stmfd	sp!, {r4 - r9, fp, lr}	@ Save an extra register
						@ to ensure 8 byte alignment
		movs	frame, r0		@ if frame pointer is zero
		beq	no_frame		@ we have no stack frames
		mov	loglvl, r2
		tst	r1, #0x10		@ 26 or 32-bit mode?
		moveq	mask, #0xfc000003
		movne	mask, #0		@ mask for 32-bit

/*
 * Switches the current frame to be the frame for dump_stack.
 */
		add	frame, sp, #24		@ switch to false frame
for_each_frame:	tst	frame, mask		@ Check for address exceptions
		bne	no_frame

/*
 * sv_fp is the stack frame with the locals for the current considered
 * function.
 *
 * sv_pc is the saved lr frame the frame above. This is a pointer to a code
 * address within the current considered function, but it is not the function
 * start. This value gets updated to be the function start later if it is
 * possible.
 */
1001:		ldr	sv_pc, [frame, #4]	@ get saved 'pc'
1002:		ldr	sv_fp, [frame, #0]	@ get saved fp

		teq	sv_fp, mask		@ make sure next frame exists
		beq	no_frame

/*
 * sv_lr is the lr from the function that called the current function. This is
 * a pointer to a code address in the current function's caller.  sv_lr-4 is
 * the instruction used to call the current function.
 *
 * This sv_lr can be used to calculate the function start if the function was
 * called using a bl instruction. If the function start can be recovered sv_pc
 * is overwritten with the function start.
 *
 * If the current function was called using a function pointer we cannot
 * recover the function start and instead continue with sv_pc as an arbitrary
 * value within the current function. If this is the case we cannot print
 * registers for the current function, but the stacktrace is still printed
 * properly.
 */
1003:		ldr	sv_lr, [sv_fp, #4]	@ get saved lr from next frame

1004:		ldr	r0, [sv_lr, #-4]	@ get call instruction
		ldr	r3, .Lopcode+4
		and	r2, r3, r0		@ is this a bl call
		teq	r2, r3
		bne	finished_setup		@ give up if it's not
		and	r0, #0xffffff		@ get call offset 24-bit int
		lsl	r0, r0, #8		@ sign extend offset
		asr	r0, r0, #8
		ldr	sv_pc, [sv_fp, #4]	@ get lr address
		add	sv_pc, sv_pc, #-4	@ get call instruction address
		add	sv_pc, sv_pc, #8	@ take care of prefetch
		add	sv_pc, sv_pc, r0, lsl #2@ find function start

finished_setup:

		bic	sv_pc, sv_pc, mask	@ mask PC/LR for the mode

/*
 * Print the function (sv_pc) and where it was called from (sv_lr).
 */
		mov	r0, sv_pc

		mov	r1, sv_lr
		mov	r2, frame
		bic	r1, r1, mask		@ mask PC/LR for the mode
		mov	r3, loglvl
		bl	dump_backtrace_entry

/*
 * Test if the function start is a stmfd instruction to determine which
 * registers were stored in the function prologue.
 *
 * If we could not recover the sv_pc because we were called through a function
 * pointer the comparison will fail and no registers will print. Unwinding will
 * continue as if there had been no registers stored in this frame.
 */
1005:		ldr	r1, [sv_pc, #0]		@ if stmfd sp!, {..., fp, lr}
		ldr	r3, .Lopcode		@ instruction exists,
		teq	r3, r1, lsr #11
		ldr	r0, [frame]		@ locals are stored in
						@ the preceding frame
		subeq	r0, r0, #4
		mov	r2, loglvl
		bleq	dump_backtrace_stm	@ dump saved registers

/*
 * If we are out of frames or if the next frame is invalid.
 */
		teq	sv_fp, #0		@ zero saved fp means
		beq	no_frame		@ no further frames

		cmp	sv_fp, frame		@ next frame must be
		mov	frame, sv_fp		@ above the current frame
#ifdef CONFIG_IRQSTACKS
		@
		@ Kernel stacks may be discontiguous in memory. If the next
		@ frame is below the previous frame, accept it as long as it
		@ lives in kernel memory.
		@
		cmpls	sv_fp, #PAGE_OFFSET
#endif
		bhi	for_each_frame

1006:		adr	r0, .Lbad
		mov	r1, loglvl
		mov	r2, frame
		bl	_printk
no_frame:	ldmfd	sp!, {r4 - r9, fp, pc}
ENDPROC(c_backtrace)
		.pushsection __ex_table,"a"
		.align	3
		.long	1001b, 1006b
		.long	1002b, 1006b
		.long	1003b, 1006b
		.long	1004b, finished_setup
		.long   1005b, 1006b
		.popsection

.Lbad:		.asciz	"%sBacktrace aborted due to bad frame pointer <%p>\n"
		.align
.Lopcode:	.word	0xe92d4800 >> 11	@ stmfd sp!, {... fp, lr}
		.word	0x0b000000		@ bl if these bits are set

#endif


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/backtrace.S
 *
 *  Copyright (C) 1995, 1996 Russell King
 *
 * 27/03/03 Ian Molton Clean up CONFIG_CPU
 */
#include <linux/kern_levels.h>
#include <linux/linkage.h>
#include <asm/assembler.h>
		.text

@ fp is 0 or stack frame

#define frame	r4
#define sv_fp	r5
#define sv_pc	r6
#define mask	r7
#define offset	r8
#define loglvl	r9

ENTRY(c_backtrace)

#if !defined(CONFIG_FRAME_POINTER) || !defined(CONFIG_PRINTK)
		ret	lr
ENDPROC(c_backtrace)
#else
		stmfd	sp!, {r4 - r9, lr}	@ Save an extra register so we have a location...
		movs	frame, r0		@ if frame pointer is zero
		beq	no_frame		@ we have no stack frames
		mov	loglvl, r2

		tst	r1, #0x10		@ 26 or 32-bit mode?
 ARM(		moveq	mask, #0xfc000003	)
 THUMB(		moveq	mask, #0xfc000000	)
 THUMB(		orreq	mask, #0x03		)
		movne	mask, #0		@ mask for 32-bit

1:		stmfd	sp!, {pc}		@ calculate offset of PC stored
		ldr	r0, [sp], #4		@ by stmfd for this CPU
		adr	r1, 1b
		sub	offset, r0, r1

/*
 * Stack frame layout:
 *             optionally saved caller registers (r4 - r10)
 *             saved fp
 *             saved sp
 *             saved lr
 *    frame => saved pc
 *             optionally saved arguments (r0 - r3)
 * saved sp => <next word>
 *
 * Functions start with the following code sequence:
 *                  mov   ip, sp
 *                  stmfd sp!, {r0 - r3} (optional)
 * corrected pc =>  stmfd sp!, {..., fp, ip, lr, pc}
 */
for_each_frame:	tst	frame, mask		@ Check for address exceptions
		bne	no_frame

1001:		ldr	sv_pc, [frame, #0]	@ get saved pc
1002:		ldr	sv_fp, [frame, #-12]	@ get saved fp

		sub	sv_pc, sv_pc, offset	@ Correct PC for prefetching
		bic	sv_pc, sv_pc, mask	@ mask PC/LR for the mode

1003:		ldr	r2, [sv_pc, #-4]	@ if stmfd sp!, {args} exists,
		ldr	r3, .Ldsi+4		@ adjust saved 'pc' back one
		teq	r3, r2, lsr #11		@ instruction
		subne	r0, sv_pc, #4		@ allow for mov
		subeq	r0, sv_pc, #8		@ allow for mov + stmia

		ldr	r1, [frame, #-4]	@ get saved lr
		mov	r2, frame
		bic	r1, r1, mask		@ mask PC/LR for the mode
		mov	r3, loglvl
		bl	dump_backtrace_entry

		ldr	r1, [sv_pc, #-4]	@ if stmfd sp!, {args} exists,
		ldr	r3, .Ldsi+4
		teq	r3, r1, lsr #11
		ldreq	r0, [frame, #-8]	@ get sp
		subeq	r0, r0, #4		@ point at the last arg
		mov	r2, loglvl
		bleq	dump_backtrace_stm	@ dump saved registers

1004:		ldr	r1, [sv_pc, #0]		@ if stmfd sp!, {..., fp, ip, lr, pc}
		ldr	r3, .Ldsi		@ instruction exists,
		teq	r3, r1, lsr #11
		subeq	r0, frame, #16
		mov	r2, loglvl
		bleq	dump_backtrace_stm	@ dump saved registers

		teq	sv_fp, #0		@ zero saved fp means
		beq	no_frame		@ no further frames

		cmp	sv_fp, frame		@ next frame must be
		mov	frame, sv_fp		@ above the current frame
#ifdef CONFIG_IRQSTACKS
		@
		@ Kernel stacks may be discontiguous in memory. If the next
		@ frame is below the previous frame, accept it as long as it
		@ lives in kernel memory.
		@
		cmpls	sv_fp, #PAGE_OFFSET
#endif
		bhi	for_each_frame

1006:		adr	r0, .Lbad
		mov	r1, loglvl
		mov	r2, frame
		bl	_printk
no_frame:	ldmfd	sp!, {r4 - r9, pc}
ENDPROC(c_backtrace)

		.pushsection __ex_table,"a"
		.align	3
		.long	1001b, 1006b
		.long	1002b, 1006b
		.long	1003b, 1006b
		.long	1004b, 1006b
		.popsection

.Lbad:		.asciz	"%sBacktrace aborted due to bad frame pointer <%p>\n"
		.align
.Ldsi:		.word	0xe92dd800 >> 11	@ stmfd sp!, {... fp, ip, lr, pc}
		.word	0xe92d0000 >> 11	@ stmfd sp!, {}

#endif


code:
/* SPDX-License-Identifier: GPL-2.0 */
#include <asm/assembler.h>
#include <asm/unwind.h>

#if __LINUX_ARM_ARCH__ >= 6
	.macro	bitop, name, instr
ENTRY(	\name		)
UNWIND(	.fnstart	)
	ands	ip, r1, #3
	strbne	r1, [ip]		@ assert word-aligned
	mov	r2, #1
	and	r3, r0, #31		@ Get bit offset
	mov	r0, r0, lsr #5
	add	r1, r1, r0, lsl #2	@ Get word offset
#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
	.arch_extension	mp
	ALT_SMP(W(pldw)	[r1])
	ALT_UP(W(nop))
#endif
	mov	r3, r2, lsl r3
1:	ldrex	r2, [r1]
	\instr	r2, r2, r3
	strex	r0, r2, [r1]
	cmp	r0, #0
	bne	1b
	bx	lr
UNWIND(	.fnend		)
ENDPROC(\name		)
	.endm

	.macro	testop, name, instr, store
ENTRY(	\name		)
UNWIND(	.fnstart	)
	ands	ip, r1, #3
	strbne	r1, [ip]		@ assert word-aligned
	mov	r2, #1
	and	r3, r0, #31		@ Get bit offset
	mov	r0, r0, lsr #5
	add	r1, r1, r0, lsl #2	@ Get word offset
	mov	r3, r2, lsl r3		@ create mask
	smp_dmb
#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
	.arch_extension	mp
	ALT_SMP(W(pldw)	[r1])
	ALT_UP(W(nop))
#endif
1:	ldrex	r2, [r1]
	ands	r0, r2, r3		@ save old value of bit
	\instr	r2, r2, r3		@ toggle bit
	strex	ip, r2, [r1]
	cmp	ip, #0
	bne	1b
	smp_dmb
	cmp	r0, #0
	movne	r0, #1
2:	bx	lr
UNWIND(	.fnend		)
ENDPROC(\name		)
	.endm
#else
	.macro	bitop, name, instr
ENTRY(	\name		)
UNWIND(	.fnstart	)
	ands	ip, r1, #3
	strbne	r1, [ip]		@ assert word-aligned
	and	r2, r0, #31
	mov	r0, r0, lsr #5
	mov	r3, #1
	mov	r3, r3, lsl r2
	save_and_disable_irqs ip
	ldr	r2, [r1, r0, lsl #2]
	\instr	r2, r2, r3
	str	r2, [r1, r0, lsl #2]
	restore_irqs ip
	ret	lr
UNWIND(	.fnend		)
ENDPROC(\name		)
	.endm

/**
 * testop - implement a test_and_xxx_bit operation.
 * @instr: operational instruction
 * @store: store instruction
 *
 * Note: we can trivially conditionalise the store instruction
 * to avoid dirtying the data cache.
 */
	.macro	testop, name, instr, store
ENTRY(	\name		)
UNWIND(	.fnstart	)
	ands	ip, r1, #3
	strbne	r1, [ip]		@ assert word-aligned
	and	r3, r0, #31
	mov	r0, r0, lsr #5
	save_and_disable_irqs ip
	ldr	r2, [r1, r0, lsl #2]!
	mov	r0, #1
	tst	r2, r0, lsl r3
	\instr	r2, r2, r0, lsl r3
	\store	r2, [r1]
	moveq	r0, #0
	restore_irqs ip
	ret	lr
UNWIND(	.fnend		)
ENDPROC(\name		)
	.endm
#endif


code:
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/linkage.h>
#include <asm/assembler.h>

#if __LINUX_ARM_ARCH__ >= 6
ENTRY(__bswapsi2)
	rev r0, r0
	bx lr
ENDPROC(__bswapsi2)

ENTRY(__bswapdi2)
	rev r3, r0
	rev r0, r1
	mov r1, r3
	bx lr
ENDPROC(__bswapdi2)
#else
ENTRY(__bswapsi2)
	eor r3, r0, r0, ror #16
	mov r3, r3, lsr #8
	bic r3, r3, #0xff00
	eor r0, r3, r0, ror #8
	ret lr
ENDPROC(__bswapsi2)

ENTRY(__bswapdi2)
	mov ip, r1
	eor r3, ip, ip, ror #16
	eor r1, r0, r0, ror #16
	mov r1, r1, lsr #8
	mov r3, r3, lsr #8
	bic r3, r3, #0xff00
	bic r1, r1, #0xff00
	eor r1, r1, r0, ror #8
	eor r0, r3, ip, ror #8
	ret lr
ENDPROC(__bswapdi2)
#endif


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * arch/arm/lib/call_with_stack.S
 *
 * Copyright (C) 2011 ARM Ltd.
 * Written by Will Deacon <will.deacon@arm.com>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>

/*
 * void call_with_stack(void (*fn)(void *), void *arg, void *sp)
 *
 * Change the stack to that pointed at by sp, then invoke fn(arg) with
 * the new stack.
 *
 * The sequence below follows the APCS frame convention for frame pointer
 * unwinding, and implements the unwinder annotations needed by the EABI
 * unwinder.
 */

ENTRY(call_with_stack)
#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
	mov	ip, sp
	push	{fp, ip, lr, pc}
	sub	fp, ip, #4
#else
UNWIND( .fnstart		)
UNWIND( .save	{fpreg, lr}	)
	push	{fpreg, lr}
UNWIND( .setfp	fpreg, sp	)
	mov	fpreg, sp
#endif
	mov	sp, r2
	mov	r2, r0
	mov	r0, r1

	bl_r	r2

#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
	ldmdb	fp, {fp, sp, pc}
#else
	mov	sp, fpreg
	pop	{fpreg, pc}
UNWIND( .fnend			)
#endif
	.globl call_with_stack_end
call_with_stack_end:
ENDPROC(call_with_stack)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/changebit.S
 *
 *  Copyright (C) 1995-1996 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include "bitops.h"
                .text

bitop	_change_bit, eor


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/clear_user.S
 *
 *  Copyright (C) 1995, 1996,1997,1998 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>

		.text

/* Prototype: unsigned long arm_clear_user(void *addr, size_t sz)
 * Purpose  : clear some user memory
 * Params   : addr - user memory address to clear
 *          : sz   - number of bytes to clear
 * Returns  : number of bytes NOT cleared
 */
ENTRY(__clear_user_std)
WEAK(arm_clear_user)
UNWIND(.fnstart)
UNWIND(.save {r1, lr})
		stmfd	sp!, {r1, lr}
		mov	r2, #0
		cmp	r1, #4
		blt	2f
		ands	ip, r0, #3
		beq	1f
		cmp	ip, #2
		strusr	r2, r0, 1
		strusr	r2, r0, 1, le
		strusr	r2, r0, 1, lt
		rsb	ip, ip, #4
		sub	r1, r1, ip		@  7  6  5  4  3  2  1
1:		subs	r1, r1, #8		@ -1 -2 -3 -4 -5 -6 -7
		strusr	r2, r0, 4, pl, rept=2
		bpl	1b
		adds	r1, r1, #4		@  3  2  1  0 -1 -2 -3
		strusr	r2, r0, 4, pl
2:		tst	r1, #2			@ 1x 1x 0x 0x 1x 1x 0x
		strusr	r2, r0, 1, ne, rept=2
		tst	r1, #1			@ x1 x0 x1 x0 x1 x0 x1
		it	ne			@ explicit IT needed for the label
USER(		strbtne	r2, [r0])
		mov	r0, #0
		ldmfd	sp!, {r1, pc}
UNWIND(.fnend)
ENDPROC(arm_clear_user)
ENDPROC(__clear_user_std)

		.pushsection .text.fixup,"ax"
		.align	0
9001:		ldmfd	sp!, {r0, pc}
		.popsection


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/clearbit.S
 *
 *  Copyright (C) 1995-1996 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include "bitops.h"
                .text

bitop	_clear_bit, bic


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/copy_from_user.S
 *
 *  Author:	Nicolas Pitre
 *  Created:	Sep 29, 2005
 *  Copyright:	MontaVista Software, Inc.
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>

/*
 * Prototype:
 *
 *	size_t arm_copy_from_user(void *to, const void *from, size_t n)
 *
 * Purpose:
 *
 *	copy a block to kernel memory from user memory
 *
 * Params:
 *
 *	to = kernel memory
 *	from = user memory
 *	n = number of bytes to copy
 *
 * Return value:
 *
 *	Number of bytes NOT copied.
 */

#ifdef CONFIG_CPU_USE_DOMAINS

#ifndef CONFIG_THUMB2_KERNEL
#define LDR1W_SHIFT	0
#else
#define LDR1W_SHIFT	1
#endif

	.macro ldr1w ptr reg abort
	ldrusr	\reg, \ptr, 4, abort=\abort
	.endm

	.macro ldr4w ptr reg1 reg2 reg3 reg4 abort
	ldr1w \ptr, \reg1, \abort
	ldr1w \ptr, \reg2, \abort
	ldr1w \ptr, \reg3, \abort
	ldr1w \ptr, \reg4, \abort
	.endm

	.macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	ldr4w \ptr, \reg1, \reg2, \reg3, \reg4, \abort
	ldr4w \ptr, \reg5, \reg6, \reg7, \reg8, \abort
	.endm

#else

#define LDR1W_SHIFT	0

	.macro ldr1w ptr reg abort
	USERL(\abort, W(ldr) \reg, [\ptr], #4)
	.endm

	.macro ldr4w ptr reg1 reg2 reg3 reg4 abort
	USERL(\abort, ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4})
	.endm

	.macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	USERL(\abort, ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8})
	.endm

#endif /* CONFIG_CPU_USE_DOMAINS */

	.macro ldr1b ptr reg cond=al abort
	ldrusr	\reg, \ptr, 1, \cond, abort=\abort
	.endm

#define STR1W_SHIFT	0

	.macro str1w ptr reg abort
	W(str) \reg, [\ptr], #4
	.endm

	.macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
	.endm

	.macro str1b ptr reg cond=al abort
	strb\cond \reg, [\ptr], #1
	.endm

	.macro enter regs:vararg
	mov	r3, #0
UNWIND( .save	{r0, r2, r3, \regs}		)
	stmdb	sp!, {r0, r2, r3, \regs}
	.endm

	.macro exit regs:vararg
	add	sp, sp, #8
	ldmfd	sp!, {r0, \regs}
	.endm

	.text

ENTRY(arm_copy_from_user)
#ifdef CONFIG_CPU_SPECTRE
	ldr	r3, =TASK_SIZE
	uaccess_mask_range_ptr r1, r2, r3, ip
#endif

#include "copy_template.S"

ENDPROC(arm_copy_from_user)

	.pushsection .text.fixup,"ax"
	.align 0
	copy_abort_preamble
	ldmfd	sp!, {r1, r2, r3}
	sub	r0, r0, r1
	rsb	r0, r0, r2
	copy_abort_end
	.popsection


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/copypage.S
 *
 *  Copyright (C) 1995-1999 Russell King
 *
 *  ASM optimised string functions
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/cache.h>

#define COPY_COUNT (PAGE_SZ / (2 * L1_CACHE_BYTES) PLD( -1 ))

		.text
		.align	5
/*
 * StrongARM optimised copy_page routine
 * now 1.78bytes/cycle, was 1.60 bytes/cycle (50MHz bus -> 89MB/s)
 * Note that we probably achieve closer to the 100MB/s target with
 * the core clock switching.
 */
ENTRY(copy_page)
		stmfd	sp!, {r4, lr}			@	2
	PLD(	pld	[r1, #0]		)
	PLD(	pld	[r1, #L1_CACHE_BYTES]		)
		mov	r2, #COPY_COUNT			@	1
		ldmia	r1!, {r3, r4, ip, lr}		@	4+1
1:	PLD(	pld	[r1, #2 * L1_CACHE_BYTES])
	PLD(	pld	[r1, #3 * L1_CACHE_BYTES])
2:
	.rept	(2 * L1_CACHE_BYTES / 16 - 1)
		stmia	r0!, {r3, r4, ip, lr}		@	4
		ldmia	r1!, {r3, r4, ip, lr}		@	4
	.endr
		subs	r2, r2, #1			@	1
		stmia	r0!, {r3, r4, ip, lr}		@	4
		ldmiagt	r1!, {r3, r4, ip, lr}		@	4
		bgt	1b				@	1
	PLD(	ldmiaeq r1!, {r3, r4, ip, lr}	)
	PLD(	beq	2b			)
		ldmfd	sp!, {r4, pc}			@	3
ENDPROC(copy_page)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/copy_template.s
 *
 *  Code template for optimized memory copy functions
 *
 *  Author:	Nicolas Pitre
 *  Created:	Sep 28, 2005
 *  Copyright:	MontaVista Software, Inc.
 */

/*
 * Theory of operation
 * -------------------
 *
 * This file provides the core code for a forward memory copy used in
 * the implementation of memcopy(), copy_to_user() and copy_from_user().
 *
 * The including file must define the following accessor macros
 * according to the need of the given function:
 *
 * ldr1w ptr reg abort
 *
 *	This loads one word from 'ptr', stores it in 'reg' and increments
 *	'ptr' to the next word. The 'abort' argument is used for fixup tables.
 *
 * ldr4w ptr reg1 reg2 reg3 reg4 abort
 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
 *
 *	This loads four or eight words starting from 'ptr', stores them
 *	in provided registers and increments 'ptr' past those words.
 *	The'abort' argument is used for fixup tables.
 *
 * ldr1b ptr reg cond abort
 *
 *	Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
 *	It also must apply the condition code if provided, otherwise the
 *	"al" condition is assumed by default.
 *
 * str1w ptr reg abort
 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
 * str1b ptr reg cond abort
 *
 *	Same as their ldr* counterparts, but data is stored to 'ptr' location
 *	rather than being loaded.
 *
 * enter reg1 reg2
 *
 *	Preserve the provided registers on the stack plus any additional
 *	data as needed by the implementation including this code. Called
 *	upon code entry.
 *
 * usave reg1 reg2
 *
 *	Unwind annotation macro is corresponding for 'enter' macro.
 *	It tell unwinder that preserved some provided registers on the stack
 *	and additional data by a prior 'enter' macro.
 *
 * exit reg1 reg2
 *
 *	Restore registers with the values previously saved with the
 *	'preserv' macro. Called upon code termination.
 *
 * LDR1W_SHIFT
 * STR1W_SHIFT
 *
 *	Correction to be applied to the "ip" register when branching into
 *	the ldr1w or str1w instructions (some of these macros may expand to
 *	than one 32bit instruction in Thumb-2)
 */

	UNWIND(	.fnstart			)
		enter	r4, UNWIND(fpreg,) lr
	UNWIND(	.setfp	fpreg, sp		)
	UNWIND(	mov	fpreg, sp		)

		subs	r2, r2, #4
		blt	8f
		ands	ip, r0, #3
	PLD(	pld	[r1, #0]		)
		bne	9f
		ands	ip, r1, #3
		bne	10f

1:		subs	r2, r2, #(28)
		stmfd	sp!, {r5, r6, r8, r9}
		blt	5f

	CALGN(	ands	ip, r0, #31		)
	CALGN(	rsb	r3, ip, #32		)
	CALGN(	sbcsne	r4, r3, r2		)  @ C is always set here
	CALGN(	bcs	2f			)
	CALGN(	adr	r4, 6f			)
	CALGN(	subs	r2, r2, r3		)  @ C gets set
	CALGN(	add	pc, r4, ip		)

	PLD(	pld	[r1, #0]		)
2:	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #28]		)
	PLD(	blt	4f			)
	PLD(	pld	[r1, #60]		)
	PLD(	pld	[r1, #92]		)

3:	PLD(	pld	[r1, #124]		)
4:		ldr8w	r1, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
		subs	r2, r2, #32
		str8w	r0, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
		bge	3b
	PLD(	cmn	r2, #96			)
	PLD(	bge	4b			)

5:		ands	ip, r2, #28
		rsb	ip, ip, #32
#if LDR1W_SHIFT > 0
		lsl	ip, ip, #LDR1W_SHIFT
#endif
		addne	pc, pc, ip		@ C is always clear here
		b	7f
6:
		.rept	(1 << LDR1W_SHIFT)
		W(nop)
		.endr
		ldr1w	r1, r3, abort=20f
		ldr1w	r1, r4, abort=20f
		ldr1w	r1, r5, abort=20f
		ldr1w	r1, r6, abort=20f
		ldr1w	r1, r8, abort=20f
		ldr1w	r1, r9, abort=20f
		ldr1w	r1, lr, abort=20f

#if LDR1W_SHIFT < STR1W_SHIFT
		lsl	ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
#elif LDR1W_SHIFT > STR1W_SHIFT
		lsr	ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
#endif
		add	pc, pc, ip
		nop
		.rept	(1 << STR1W_SHIFT)
		W(nop)
		.endr
		str1w	r0, r3, abort=20f
		str1w	r0, r4, abort=20f
		str1w	r0, r5, abort=20f
		str1w	r0, r6, abort=20f
		str1w	r0, r8, abort=20f
		str1w	r0, r9, abort=20f
		str1w	r0, lr, abort=20f

	CALGN(	bcs	2b			)

7:		ldmfd	sp!, {r5, r6, r8, r9}

8:		movs	r2, r2, lsl #31
		ldr1b	r1, r3, ne, abort=21f
		ldr1b	r1, r4, cs, abort=21f
		ldr1b	r1, ip, cs, abort=21f
		str1b	r0, r3, ne, abort=21f
		str1b	r0, r4, cs, abort=21f
		str1b	r0, ip, cs, abort=21f

		exit	r4, UNWIND(fpreg,) pc

9:		rsb	ip, ip, #4
		cmp	ip, #2
		ldr1b	r1, r3, gt, abort=21f
		ldr1b	r1, r4, ge, abort=21f
		ldr1b	r1, lr, abort=21f
		str1b	r0, r3, gt, abort=21f
		str1b	r0, r4, ge, abort=21f
		subs	r2, r2, ip
		str1b	r0, lr, abort=21f
		blt	8b
		ands	ip, r1, #3
		beq	1b

10:		bic	r1, r1, #3
		cmp	ip, #2
		ldr1w	r1, lr, abort=21f
		beq	17f
		bgt	18f

		.macro	forward_copy_shift pull push

		subs	r2, r2, #28
		blt	14f

	CALGN(	ands	ip, r0, #31		)
	CALGN(	rsb	ip, ip, #32		)
	CALGN(	sbcsne	r4, ip, r2		)  @ C is always set here
	CALGN(	subcc	r2, r2, ip		)
	CALGN(	bcc	15f			)

11:		stmfd	sp!, {r5, r6, r8 - r10}

	PLD(	pld	[r1, #0]		)
	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #28]		)
	PLD(	blt	13f			)
	PLD(	pld	[r1, #60]		)
	PLD(	pld	[r1, #92]		)

12:	PLD(	pld	[r1, #124]		)
13:		ldr4w	r1, r4, r5, r6, r8, abort=19f
		mov	r3, lr, lspull #\pull
		subs	r2, r2, #32
		ldr4w	r1, r9, r10, ip, lr, abort=19f
		orr	r3, r3, r4, lspush #\push
		mov	r4, r4, lspull #\pull
		orr	r4, r4, r5, lspush #\push
		mov	r5, r5, lspull #\pull
		orr	r5, r5, r6, lspush #\push
		mov	r6, r6, lspull #\pull
		orr	r6, r6, r8, lspush #\push
		mov	r8, r8, lspull #\pull
		orr	r8, r8, r9, lspush #\push
		mov	r9, r9, lspull #\pull
		orr	r9, r9, r10, lspush #\push
		mov	r10, r10, lspull #\pull
		orr	r10, r10, ip, lspush #\push
		mov	ip, ip, lspull #\pull
		orr	ip, ip, lr, lspush #\push
		str8w	r0, r3, r4, r5, r6, r8, r9, r10, ip, abort=19f
		bge	12b
	PLD(	cmn	r2, #96			)
	PLD(	bge	13b			)

		ldmfd	sp!, {r5, r6, r8 - r10}

14:		ands	ip, r2, #28
		beq	16f

15:		mov	r3, lr, lspull #\pull
		ldr1w	r1, lr, abort=21f
		subs	ip, ip, #4
		orr	r3, r3, lr, lspush #\push
		str1w	r0, r3, abort=21f
		bgt	15b
	CALGN(	cmp	r2, #0			)
	CALGN(	bge	11b			)

16:		sub	r1, r1, #(\push / 8)
		b	8b

		.endm

		forward_copy_shift	pull=8	push=24

17:		forward_copy_shift	pull=16	push=16

18:		forward_copy_shift	pull=24	push=8

	UNWIND(	.fnend				)

/*
 * Abort preamble and completion macros.
 * If a fixup handler is required then those macros must surround it.
 * It is assumed that the fixup code will handle the private part of
 * the exit macro.
 */

	.macro	copy_abort_preamble
19:	ldmfd	sp!, {r5, r6, r8 - r10}
	b	21f
20:	ldmfd	sp!, {r5, r6, r8, r9}
21:
	.endm

	.macro	copy_abort_end
	ldmfd	sp!, {r4, UNWIND(fpreg,) pc}
	.endm


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/copy_to_user.S
 *
 *  Author:	Nicolas Pitre
 *  Created:	Sep 29, 2005
 *  Copyright:	MontaVista Software, Inc.
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>

/*
 * Prototype:
 *
 *	size_t arm_copy_to_user(void *to, const void *from, size_t n)
 *
 * Purpose:
 *
 *	copy a block to user memory from kernel memory
 *
 * Params:
 *
 *	to = user memory
 *	from = kernel memory
 *	n = number of bytes to copy
 *
 * Return value:
 *
 *	Number of bytes NOT copied.
 */

#define LDR1W_SHIFT	0

	.macro ldr1w ptr reg abort
	W(ldr) \reg, [\ptr], #4
	.endm

	.macro ldr4w ptr reg1 reg2 reg3 reg4 abort
	ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4}
	.endm

	.macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
	.endm

	.macro ldr1b ptr reg cond=al abort
	ldrb\cond \reg, [\ptr], #1
	.endm

#ifdef CONFIG_CPU_USE_DOMAINS

#ifndef CONFIG_THUMB2_KERNEL
#define STR1W_SHIFT	0
#else
#define STR1W_SHIFT	1
#endif

	.macro str1w ptr reg abort
	strusr	\reg, \ptr, 4, abort=\abort
	.endm

	.macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	str1w \ptr, \reg1, \abort
	str1w \ptr, \reg2, \abort
	str1w \ptr, \reg3, \abort
	str1w \ptr, \reg4, \abort
	str1w \ptr, \reg5, \abort
	str1w \ptr, \reg6, \abort
	str1w \ptr, \reg7, \abort
	str1w \ptr, \reg8, \abort
	.endm

#else

#define STR1W_SHIFT	0

	.macro str1w ptr reg abort
	USERL(\abort, W(str) \reg, [\ptr], #4)
	.endm

	.macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	USERL(\abort, stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8})
	.endm

#endif /* CONFIG_CPU_USE_DOMAINS */

	.macro str1b ptr reg cond=al abort
	strusr	\reg, \ptr, 1, \cond, abort=\abort
	.endm

	.macro enter regs:vararg
	mov	r3, #0
UNWIND( .save	{r0, r2, r3, \regs}		)
	stmdb	sp!, {r0, r2, r3, \regs}
	.endm

	.macro exit regs:vararg
	add	sp, sp, #8
	ldmfd	sp!, {r0, \regs}
	.endm

	.text

ENTRY(__copy_to_user_std)
WEAK(arm_copy_to_user)
#ifdef CONFIG_CPU_SPECTRE
	ldr	r3, =TASK_SIZE
	uaccess_mask_range_ptr r0, r2, r3, ip
#endif

#include "copy_template.S"

ENDPROC(arm_copy_to_user)
ENDPROC(__copy_to_user_std)

	.pushsection .text.fixup,"ax"
	.align 0
	copy_abort_preamble
	ldmfd	sp!, {r1, r2, r3}
	sub	r0, r0, r1
	rsb	r0, r0, r2
	copy_abort_end
	.popsection


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/csumipv6.S
 *
 *  Copyright (C) 1995-1998 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

		.text

ENTRY(__csum_ipv6_magic)
		str	lr, [sp, #-4]!
		adds	ip, r2, r3
		ldmia	r1, {r1 - r3, lr}
		adcs	ip, ip, r1
		adcs	ip, ip, r2
		adcs	ip, ip, r3
		adcs	ip, ip, lr
		ldmia	r0, {r0 - r3}
		adcs	r0, ip, r0
		adcs	r0, r0, r1
		adcs	r0, r0, r2
		ldr	r2, [sp, #4]
		adcs	r0, r0, r3
		adcs	r0, r0, r2
		adcs	r0, r0, #0
		ldmfd	sp!, {pc}
ENDPROC(__csum_ipv6_magic)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/csumpartial.S
 *
 *  Copyright (C) 1995-1998 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

		.text

/*
 * Function: __u32 csum_partial(const char *src, int len, __u32 sum)
 * Params  : r0 = buffer, r1 = len, r2 = checksum
 * Returns : r0 = new checksum
 */

buf	.req	r0
len	.req	r1
sum	.req	r2
td0	.req	r3
td1	.req	r4	@ save before use
td2	.req	r5	@ save before use
td3	.req	lr

.Lzero:		mov	r0, sum
		add	sp, sp, #4
		ldr	pc, [sp], #4

		/*
		 * Handle 0 to 7 bytes, with any alignment of source and
		 * destination pointers.  Note that when we get here, C = 0
		 */
.Lless8:		teq	len, #0			@ check for zero count
		beq	.Lzero

		/* we must have at least one byte. */
		tst	buf, #1			@ odd address?
		movne	sum, sum, ror #8
		ldrbne	td0, [buf], #1
		subne	len, len, #1
		adcsne	sum, sum, td0, put_byte_1

.Lless4:		tst	len, #6
		beq	.Lless8_byte

		/* we are now half-word aligned */

.Lless8_wordlp:
#if __LINUX_ARM_ARCH__ >= 4
		ldrh	td0, [buf], #2
		sub	len, len, #2
#else
		ldrb	td0, [buf], #1
		ldrb	td3, [buf], #1
		sub	len, len, #2
#ifndef __ARMEB__
		orr	td0, td0, td3, lsl #8
#else
		orr	td0, td3, td0, lsl #8
#endif
#endif
		adcs	sum, sum, td0
		tst	len, #6
		bne	.Lless8_wordlp

.Lless8_byte:	tst	len, #1			@ odd number of bytes
		ldrbne	td0, [buf], #1		@ include last byte
		adcsne	sum, sum, td0, put_byte_0	@ update checksum

.Ldone:		adc	r0, sum, #0		@ collect up the last carry
		ldr	td0, [sp], #4
		tst	td0, #1			@ check buffer alignment
		movne	r0, r0, ror #8		@ rotate checksum by 8 bits
		ldr	pc, [sp], #4		@ return

.Lnot_aligned:	tst	buf, #1			@ odd address
		ldrbne	td0, [buf], #1		@ make even
		subne	len, len, #1
		adcsne	sum, sum, td0, put_byte_1	@ update checksum

		tst	buf, #2			@ 32-bit aligned?
#if __LINUX_ARM_ARCH__ >= 4
		ldrhne	td0, [buf], #2		@ make 32-bit aligned
		subne	len, len, #2
#else
		ldrbne	td0, [buf], #1
		ldrbne	ip, [buf], #1
		subne	len, len, #2
#ifndef __ARMEB__
		orrne	td0, td0, ip, lsl #8
#else
		orrne	td0, ip, td0, lsl #8
#endif
#endif
		adcsne	sum, sum, td0		@ update checksum
		ret	lr

ENTRY(csum_partial)
		stmfd	sp!, {buf, lr}
		cmp	len, #8			@ Ensure that we have at least
		blo	.Lless8			@ 8 bytes to copy.

		tst	buf, #1
		movne	sum, sum, ror #8

		adds	sum, sum, #0		@ C = 0
		tst	buf, #3			@ Test destination alignment
		blne	.Lnot_aligned		@ align destination, return here

1:		bics	ip, len, #31
		beq	3f

		stmfd	sp!, {r4 - r5}
2:		ldmia	buf!, {td0, td1, td2, td3}
		adcs	sum, sum, td0
		adcs	sum, sum, td1
		adcs	sum, sum, td2
		adcs	sum, sum, td3
		ldmia	buf!, {td0, td1, td2, td3}
		adcs	sum, sum, td0
		adcs	sum, sum, td1
		adcs	sum, sum, td2
		adcs	sum, sum, td3
		sub	ip, ip, #32
		teq	ip, #0
		bne	2b
		ldmfd	sp!, {r4 - r5}

3:		tst	len, #0x1c		@ should not change C
		beq	.Lless4

4:		ldr	td0, [buf], #4
		sub	len, len, #4
		adcs	sum, sum, td0
		tst	len, #0x1c
		bne	4b
		b	.Lless4
ENDPROC(csum_partial)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/csumpartialcopy.S
 *
 *  Copyright (C) 1995-1998 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

		.text

/* Function: __u32 csum_partial_copy_nocheck(const char *src, char *dst, int len)
 * Params  : r0 = src, r1 = dst, r2 = len
 * Returns : r0 = new checksum
 */

		.macro	save_regs
		stmfd	sp!, {r1, r4 - r8, lr}
		.endm

		.macro	load_regs
		ldmfd	sp!, {r1, r4 - r8, pc}
		.endm

		.macro	load1b, reg1
		ldrb	\reg1, [r0], #1
		.endm

		.macro	load2b, reg1, reg2
		ldrb	\reg1, [r0], #1
		ldrb	\reg2, [r0], #1
		.endm

		.macro	load1l, reg1
		ldr	\reg1, [r0], #4
		.endm

		.macro	load2l, reg1, reg2
		ldr	\reg1, [r0], #4
		ldr	\reg2, [r0], #4
		.endm

		.macro	load4l, reg1, reg2, reg3, reg4
		ldmia	r0!, {\reg1, \reg2, \reg3, \reg4}
		.endm

#define FN_ENTRY	ENTRY(csum_partial_copy_nocheck)
#define FN_EXIT		ENDPROC(csum_partial_copy_nocheck)

#include "csumpartialcopygeneric.S"


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/csumpartialcopygeneric.S
 *
 *  Copyright (C) 1995-2001 Russell King
 */
#include <asm/assembler.h>

/*
 * unsigned int
 * csum_partial_copy_xxx(const char *src, char *dst, int len, int sum, )
 *  r0 = src, r1 = dst, r2 = len, r3 = sum
 *  Returns : r0 = checksum
 *
 * Note that 'tst' and 'teq' preserve the carry flag.
 */

src	.req	r0
dst	.req	r1
len	.req	r2
sum	.req	r3

.Lzero:		mov	r0, sum
		load_regs

		/*
		 * Align an unaligned destination pointer.  We know that
		 * we have >= 8 bytes here, so we don't need to check
		 * the length.  Note that the source pointer hasn't been
		 * aligned yet.
		 */
.Ldst_unaligned:
		tst	dst, #1
		beq	.Ldst_16bit

		load1b	ip
		sub	len, len, #1
		adcs	sum, sum, ip, put_byte_1	@ update checksum
		strb	ip, [dst], #1
		tst	dst, #2
		reteq	lr			@ dst is now 32bit aligned

.Ldst_16bit:	load2b	r8, ip
		sub	len, len, #2
		adcs	sum, sum, r8, put_byte_0
		strb	r8, [dst], #1
		adcs	sum, sum, ip, put_byte_1
		strb	ip, [dst], #1
		ret	lr			@ dst is now 32bit aligned

		/*
		 * Handle 0 to 7 bytes, with any alignment of source and
		 * destination pointers.  Note that when we get here, C = 0
		 */
.Lless8:	teq	len, #0			@ check for zero count
		beq	.Lzero

		/* we must have at least one byte. */
		tst	dst, #1			@ dst 16-bit aligned
		beq	.Lless8_aligned

		/* Align dst */
		load1b	ip
		sub	len, len, #1
		adcs	sum, sum, ip, put_byte_1	@ update checksum
		strb	ip, [dst], #1
		tst	len, #6
		beq	.Lless8_byteonly

1:		load2b	r8, ip
		sub	len, len, #2
		adcs	sum, sum, r8, put_byte_0
		strb	r8, [dst], #1
		adcs	sum, sum, ip, put_byte_1
		strb	ip, [dst], #1
.Lless8_aligned:
		tst	len, #6
		bne	1b
.Lless8_byteonly:
		tst	len, #1
		beq	.Ldone
		load1b	r8
		adcs	sum, sum, r8, put_byte_0	@ update checksum
		strb	r8, [dst], #1
		b	.Ldone

FN_ENTRY
		save_regs
		mov	sum, #-1

		cmp	len, #8			@ Ensure that we have at least
		blo	.Lless8			@ 8 bytes to copy.

		adds	sum, sum, #0		@ C = 0
		tst	dst, #3			@ Test destination alignment
		blne	.Ldst_unaligned		@ align destination, return here

		/*
		 * Ok, the dst pointer is now 32bit aligned, and we know
		 * that we must have more than 4 bytes to copy.  Note
		 * that C contains the carry from the dst alignment above.
		 */

		tst	src, #3			@ Test source alignment
		bne	.Lsrc_not_aligned

		/* Routine for src & dst aligned */

		bics	ip, len, #15
		beq	2f

1:		load4l	r4, r5, r6, r7
		stmia	dst!, {r4, r5, r6, r7}
		adcs	sum, sum, r4
		adcs	sum, sum, r5
		adcs	sum, sum, r6
		adcs	sum, sum, r7
		sub	ip, ip, #16
		teq	ip, #0
		bne	1b

2:		ands	ip, len, #12
		beq	4f
		tst	ip, #8
		beq	3f
		load2l	r4, r5
		stmia	dst!, {r4, r5}
		adcs	sum, sum, r4
		adcs	sum, sum, r5
		tst	ip, #4
		beq	4f

3:		load1l	r4
		str	r4, [dst], #4
		adcs	sum, sum, r4

4:		ands	len, len, #3
		beq	.Ldone
		load1l	r4
		tst	len, #2
		mov	r5, r4, get_byte_0
		beq	.Lexit
		adcs	sum, sum, r4, lspush #16
		strb	r5, [dst], #1
		mov	r5, r4, get_byte_1
		strb	r5, [dst], #1
		mov	r5, r4, get_byte_2
.Lexit:		tst	len, #1
		strbne	r5, [dst], #1
		andne	r5, r5, #255
		adcsne	sum, sum, r5, put_byte_0

		/*
		 * If the dst pointer was not 16-bit aligned, we
		 * need to rotate the checksum here to get around
		 * the inefficient byte manipulations in the
		 * architecture independent code.
		 */
.Ldone:		adc	r0, sum, #0
		ldr	sum, [sp, #0]		@ dst
		tst	sum, #1
		movne	r0, r0, ror #8
		load_regs

.Lsrc_not_aligned:
		adc	sum, sum, #0		@ include C from dst alignment
		and	ip, src, #3
		bic	src, src, #3
		load1l	r5
		cmp	ip, #2
		beq	.Lsrc2_aligned
		bhi	.Lsrc3_aligned
		mov	r4, r5, lspull #8		@ C = 0
		bics	ip, len, #15
		beq	2f
1:		load4l	r5, r6, r7, r8
		orr	r4, r4, r5, lspush #24
		mov	r5, r5, lspull #8
		orr	r5, r5, r6, lspush #24
		mov	r6, r6, lspull #8
		orr	r6, r6, r7, lspush #24
		mov	r7, r7, lspull #8
		orr	r7, r7, r8, lspush #24
		stmia	dst!, {r4, r5, r6, r7}
		adcs	sum, sum, r4
		adcs	sum, sum, r5
		adcs	sum, sum, r6
		adcs	sum, sum, r7
		mov	r4, r8, lspull #8
		sub	ip, ip, #16
		teq	ip, #0
		bne	1b
2:		ands	ip, len, #12
		beq	4f
		tst	ip, #8
		beq	3f
		load2l	r5, r6
		orr	r4, r4, r5, lspush #24
		mov	r5, r5, lspull #8
		orr	r5, r5, r6, lspush #24
		stmia	dst!, {r4, r5}
		adcs	sum, sum, r4
		adcs	sum, sum, r5
		mov	r4, r6, lspull #8
		tst	ip, #4
		beq	4f
3:		load1l	r5
		orr	r4, r4, r5, lspush #24
		str	r4, [dst], #4
		adcs	sum, sum, r4
		mov	r4, r5, lspull #8
4:		ands	len, len, #3
		beq	.Ldone
		mov	r5, r4, get_byte_0
		tst	len, #2
		beq	.Lexit
		adcs	sum, sum, r4, lspush #16
		strb	r5, [dst], #1
		mov	r5, r4, get_byte_1
		strb	r5, [dst], #1
		mov	r5, r4, get_byte_2
		b	.Lexit

.Lsrc2_aligned:	mov	r4, r5, lspull #16
		adds	sum, sum, #0
		bics	ip, len, #15
		beq	2f
1:		load4l	r5, r6, r7, r8
		orr	r4, r4, r5, lspush #16
		mov	r5, r5, lspull #16
		orr	r5, r5, r6, lspush #16
		mov	r6, r6, lspull #16
		orr	r6, r6, r7, lspush #16
		mov	r7, r7, lspull #16
		orr	r7, r7, r8, lspush #16
		stmia	dst!, {r4, r5, r6, r7}
		adcs	sum, sum, r4
		adcs	sum, sum, r5
		adcs	sum, sum, r6
		adcs	sum, sum, r7
		mov	r4, r8, lspull #16
		sub	ip, ip, #16
		teq	ip, #0
		bne	1b
2:		ands	ip, len, #12
		beq	4f
		tst	ip, #8
		beq	3f
		load2l	r5, r6
		orr	r4, r4, r5, lspush #16
		mov	r5, r5, lspull #16
		orr	r5, r5, r6, lspush #16
		stmia	dst!, {r4, r5}
		adcs	sum, sum, r4
		adcs	sum, sum, r5
		mov	r4, r6, lspull #16
		tst	ip, #4
		beq	4f
3:		load1l	r5
		orr	r4, r4, r5, lspush #16
		str	r4, [dst], #4
		adcs	sum, sum, r4
		mov	r4, r5, lspull #16
4:		ands	len, len, #3
		beq	.Ldone
		mov	r5, r4, get_byte_0
		tst	len, #2
		beq	.Lexit
		adcs	sum, sum, r4
		strb	r5, [dst], #1
		mov	r5, r4, get_byte_1
		strb	r5, [dst], #1
		tst	len, #1
		beq	.Ldone
		load1b	r5
		b	.Lexit

.Lsrc3_aligned:	mov	r4, r5, lspull #24
		adds	sum, sum, #0
		bics	ip, len, #15
		beq	2f
1:		load4l	r5, r6, r7, r8
		orr	r4, r4, r5, lspush #8
		mov	r5, r5, lspull #24
		orr	r5, r5, r6, lspush #8
		mov	r6, r6, lspull #24
		orr	r6, r6, r7, lspush #8
		mov	r7, r7, lspull #24
		orr	r7, r7, r8, lspush #8
		stmia	dst!, {r4, r5, r6, r7}
		adcs	sum, sum, r4
		adcs	sum, sum, r5
		adcs	sum, sum, r6
		adcs	sum, sum, r7
		mov	r4, r8, lspull #24
		sub	ip, ip, #16
		teq	ip, #0
		bne	1b
2:		ands	ip, len, #12
		beq	4f
		tst	ip, #8
		beq	3f
		load2l	r5, r6
		orr	r4, r4, r5, lspush #8
		mov	r5, r5, lspull #24
		orr	r5, r5, r6, lspush #8
		stmia	dst!, {r4, r5}
		adcs	sum, sum, r4
		adcs	sum, sum, r5
		mov	r4, r6, lspull #24
		tst	ip, #4
		beq	4f
3:		load1l	r5
		orr	r4, r4, r5, lspush #8
		str	r4, [dst], #4
		adcs	sum, sum, r4
		mov	r4, r5, lspull #24
4:		ands	len, len, #3
		beq	.Ldone
		mov	r5, r4, get_byte_0
		tst	len, #2
		beq	.Lexit
		strb	r5, [dst], #1
		adcs	sum, sum, r4
		load1l	r4
		mov	r5, r4, get_byte_0
		strb	r5, [dst], #1
		adcs	sum, sum, r4, lspush #24
		mov	r5, r4, get_byte_1
		b	.Lexit
FN_EXIT


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/csumpartialcopyuser.S
 *
 *  Copyright (C) 1995-1998 Russell King
 *
 * 27/03/03 Ian Molton Clean up CONFIG_CPU
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/errno.h>
#include <asm/asm-offsets.h>

		.text

#ifdef CONFIG_CPU_SW_DOMAIN_PAN
		.macro	save_regs
		mrc	p15, 0, ip, c3, c0, 0
		stmfd	sp!, {r1, r2, r4 - r8, ip, lr}
		uaccess_enable ip
		.endm

		.macro	load_regs
		ldmfd	sp!, {r1, r2, r4 - r8, ip, lr}
		mcr	p15, 0, ip, c3, c0, 0
		ret	lr
		.endm
#else
		.macro	save_regs
		stmfd	sp!, {r1, r2, r4 - r8, lr}
		.endm

		.macro	load_regs
		ldmfd	sp!, {r1, r2, r4 - r8, pc}
		.endm
#endif

		.macro	load1b,	reg1
		ldrusr	\reg1, r0, 1
		.endm

		.macro	load2b, reg1, reg2
		ldrusr	\reg1, r0, 1
		ldrusr	\reg2, r0, 1
		.endm

		.macro	load1l, reg1
		ldrusr	\reg1, r0, 4
		.endm

		.macro	load2l, reg1, reg2
		ldrusr	\reg1, r0, 4
		ldrusr	\reg2, r0, 4
		.endm

		.macro	load4l, reg1, reg2, reg3, reg4
		ldrusr	\reg1, r0, 4
		ldrusr	\reg2, r0, 4
		ldrusr	\reg3, r0, 4
		ldrusr	\reg4, r0, 4
		.endm

/*
 * unsigned int
 * csum_partial_copy_from_user(const char *src, char *dst, int len)
 *  r0 = src, r1 = dst, r2 = len
 *  Returns : r0 = checksum or 0
 */

#define FN_ENTRY	ENTRY(csum_partial_copy_from_user)
#define FN_EXIT		ENDPROC(csum_partial_copy_from_user)

#include "csumpartialcopygeneric.S"

/*
 * We report fault by returning 0 csum - impossible in normal case, since
 * we start with 0xffffffff for initial sum.
 */
		.pushsection .text.fixup,"ax"
		.align	4
9001:		mov	r0, #0
		load_regs
		.popsection


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/delay.S
 *
 *  Copyright (C) 1995, 1996 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/delay.h>

#ifdef CONFIG_ARCH_RPC
		.arch	armv4
#endif

		.text

.LC0:		.word	loops_per_jiffy
.LC1:		.word	UDELAY_MULT

/*
 * loops = r0 * HZ * loops_per_jiffy / 1000000
 *
 * r0  <= 2000
 * HZ  <= 1000
 */

ENTRY(__loop_udelay)
		ldr	r2, .LC1
		mul	r0, r2, r0		@ r0 = delay_us * UDELAY_MULT
ENTRY(__loop_const_udelay)			@ 0 <= r0 <= 0xfffffaf0
		ldr	r2, .LC0
		ldr	r2, [r2]
		umull	r1, r0, r2, r0		@ r0-r1 = r0 * loops_per_jiffy
		adds	r1, r1, #0xffffffff	@ rounding up ...
		adcs	r0, r0, r0		@ and right shift by 31
		reteq	lr

		.align 3

@ Delay routine
ENTRY(__loop_delay)
		subs	r0, r0, #1
#if 0
		retls	lr
		subs	r0, r0, #1
		retls	lr
		subs	r0, r0, #1
		retls	lr
		subs	r0, r0, #1
		retls	lr
		subs	r0, r0, #1
		retls	lr
		subs	r0, r0, #1
		retls	lr
		subs	r0, r0, #1
		retls	lr
		subs	r0, r0, #1
#endif
		bhi	__loop_delay
		ret	lr
ENDPROC(__loop_udelay)
ENDPROC(__loop_const_udelay)
ENDPROC(__loop_delay)


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Delay loops based on the OpenRISC implementation.
 *
 * Copyright (C) 2012 ARM Limited
 *
 * Author: Will Deacon <will.deacon@arm.com>
 */

#include <linux/clocksource.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/timex.h>

/*
 * Default to the loop-based delay implementation.
 */
struct arm_delay_ops arm_delay_ops __ro_after_init = {
	.delay		= __loop_delay,
	.const_udelay	= __loop_const_udelay,
	.udelay		= __loop_udelay,
};

static const struct delay_timer *delay_timer;
static bool delay_calibrated;
static u64 delay_res;

int read_current_timer(unsigned long *timer_val)
{
	if (!delay_timer)
		return -ENXIO;

	*timer_val = delay_timer->read_current_timer();
	return 0;
}
EXPORT_SYMBOL_GPL(read_current_timer);

static inline u64 cyc_to_ns(u64 cyc, u32 mult, u32 shift)
{
	return (cyc * mult) >> shift;
}

static void __timer_delay(unsigned long cycles)
{
	cycles_t start = get_cycles();

	while ((get_cycles() - start) < cycles)
		cpu_relax();
}

static void __timer_const_udelay(unsigned long xloops)
{
	unsigned long long loops = xloops;
	loops *= arm_delay_ops.ticks_per_jiffy;
	__timer_delay(loops >> UDELAY_SHIFT);
}

static void __timer_udelay(unsigned long usecs)
{
	__timer_const_udelay(usecs * UDELAY_MULT);
}

void __init register_current_timer_delay(const struct delay_timer *timer)
{
	u32 new_mult, new_shift;
	u64 res;

	clocks_calc_mult_shift(&new_mult, &new_shift, timer->freq,
			       NSEC_PER_SEC, 3600);
	res = cyc_to_ns(1ULL, new_mult, new_shift);

	if (res > 1000) {
		pr_err("Ignoring delay timer %ps, which has insufficient resolution of %lluns\n",
			timer, res);
		return;
	}

	if (!delay_calibrated && (!delay_res || (res < delay_res))) {
		pr_info("Switching to timer-based delay loop, resolution %lluns\n", res);
		delay_timer			= timer;
		lpj_fine			= timer->freq / HZ;
		delay_res			= res;

		/* cpufreq may scale loops_per_jiffy, so keep a private copy */
		arm_delay_ops.ticks_per_jiffy	= lpj_fine;
		arm_delay_ops.delay		= __timer_delay;
		arm_delay_ops.const_udelay	= __timer_const_udelay;
		arm_delay_ops.udelay		= __timer_udelay;
	} else {
		pr_info("Ignoring duplicate/late registration of read_current_timer delay\n");
	}
}

unsigned long calibrate_delay_is_known(void)
{
	delay_calibrated = true;
	return lpj_fine;
}

void calibration_delay_done(void)
{
	delay_calibrated = true;
}


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/div64.S
 *
 *  Optimized computation of 64-bit dividend / 32-bit divisor
 *
 *  Author:	Nicolas Pitre
 *  Created:	Oct 5, 2003
 *  Copyright:	Monta Vista Software, Inc.
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>

#ifdef __ARMEB__
#define xh r0
#define xl r1
#define yh r2
#define yl r3
#else
#define xl r0
#define xh r1
#define yl r2
#define yh r3
#endif

/*
 * __do_div64: perform a division with 64-bit dividend and 32-bit divisor.
 *
 * Note: Calling convention is totally non standard for optimal code.
 *       This is meant to be used by do_div() from include/asm/div64.h only.
 *
 * Input parameters:
 * 	xh-xl	= dividend (clobbered)
 * 	r4	= divisor (preserved)
 *
 * Output values:
 * 	yh-yl	= result
 * 	xh	= remainder
 *
 * Clobbered regs: xl, ip
 */

ENTRY(__do_div64)
UNWIND(.fnstart)

	@ Test for easy paths first.
	subs	ip, r4, #1
	bls	9f			@ divisor is 0 or 1
	tst	ip, r4
	beq	8f			@ divisor is power of 2

	@ See if we need to handle upper 32-bit result.
	cmp	xh, r4
	mov	yh, #0
	blo	3f

	@ Align divisor with upper part of dividend.
	@ The aligned divisor is stored in yl preserving the original.
	@ The bit position is stored in ip.

#if __LINUX_ARM_ARCH__ >= 5

	clz	yl, r4
	clz	ip, xh
	sub	yl, yl, ip
	mov	ip, #1
	mov	ip, ip, lsl yl
	mov	yl, r4, lsl yl

#else

	mov	yl, r4
	mov	ip, #1
1:	cmp	yl, #0x80000000
	cmpcc	yl, xh
	movcc	yl, yl, lsl #1
	movcc	ip, ip, lsl #1
	bcc	1b

#endif

	@ The division loop for needed upper bit positions.
 	@ Break out early if dividend reaches 0.
2:	cmp	xh, yl
	orrcs	yh, yh, ip
	subscs	xh, xh, yl
	movsne	ip, ip, lsr #1
	mov	yl, yl, lsr #1
	bne	2b

	@ See if we need to handle lower 32-bit result.
3:	cmp	xh, #0
	mov	yl, #0
	cmpeq	xl, r4
	movlo	xh, xl
	retlo	lr

	@ The division loop for lower bit positions.
	@ Here we shift remainer bits leftwards rather than moving the
	@ divisor for comparisons, considering the carry-out bit as well.
	mov	ip, #0x80000000
4:	movs	xl, xl, lsl #1
	adcs	xh, xh, xh
	beq	6f
	cmpcc	xh, r4
5:	orrcs	yl, yl, ip
	subcs	xh, xh, r4
	movs	ip, ip, lsr #1
	bne	4b
	ret	lr

	@ The top part of remainder became zero.  If carry is set
	@ (the 33th bit) this is a false positive so resume the loop.
	@ Otherwise, if lower part is also null then we are done.
6:	bcs	5b
	cmp	xl, #0
	reteq	lr

	@ We still have remainer bits in the low part.  Bring them up.

#if __LINUX_ARM_ARCH__ >= 5

	clz	xh, xl			@ we know xh is zero here so...
	add	xh, xh, #1
	mov	xl, xl, lsl xh
	mov	ip, ip, lsr xh

#else

7:	movs	xl, xl, lsl #1
	mov	ip, ip, lsr #1
	bcc	7b

#endif

	@ Current remainder is now 1.  It is worthless to compare with
	@ divisor at this point since divisor can not be smaller than 3 here.
	@ If possible, branch for another shift in the division loop.
	@ If no bit position left then we are done.
	movs	ip, ip, lsr #1
	mov	xh, #1
	bne	4b
	ret	lr

8:	@ Division by a power of 2: determine what that divisor order is
	@ then simply shift values around

#if __LINUX_ARM_ARCH__ >= 5

	clz	ip, r4
	rsb	ip, ip, #31

#else

	mov	yl, r4
	cmp	r4, #(1 << 16)
	mov	ip, #0
	movhs	yl, yl, lsr #16
	movhs	ip, #16

	cmp	yl, #(1 << 8)
	movhs	yl, yl, lsr #8
	addhs	ip, ip, #8

	cmp	yl, #(1 << 4)
	movhs	yl, yl, lsr #4
	addhs	ip, ip, #4

	cmp	yl, #(1 << 2)
	addhi	ip, ip, #3
	addls	ip, ip, yl, lsr #1

#endif

	mov	yh, xh, lsr ip
	mov	yl, xl, lsr ip
	rsb	ip, ip, #32
 ARM(	orr	yl, yl, xh, lsl ip	)
 THUMB(	lsl	xh, xh, ip		)
 THUMB(	orr	yl, yl, xh		)
	mov	xh, xl, lsl ip
	mov	xh, xh, lsr ip
	ret	lr

	@ eq -> division by 1: obvious enough...
9:	moveq	yl, xl
	moveq	yh, xh
	moveq	xh, #0
	reteq	lr
UNWIND(.fnend)

UNWIND(.fnstart)
UNWIND(.pad #4)
UNWIND(.save {lr})
Ldiv0_64:
	@ Division by 0:
	str	lr, [sp, #-8]!
	bl	__div0

	@ as wrong as it could be...
	mov	yl, #0
	mov	yh, #0
	mov	xh, #0
	ldr	pc, [sp], #8

UNWIND(.fnend)
ENDPROC(__do_div64)


code:
// SPDX-License-Identifier: GPL-2.0

#include <linux/error-injection.h>
#include <linux/kprobes.h>

void override_function_with_return(struct pt_regs *regs)
{
	instruction_pointer_set(regs, regs->ARM_lr);
}
NOKPROBE_SYMBOL(override_function_with_return);


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/findbit.S
 *
 *  Copyright (C) 1995-2000 Russell King
 *
 * 16th March 2001 - John Ripley <jripley@sonicblue.com>
 *   Fixed so that "size" is an exclusive not an inclusive quantity.
 *   All users of these functions expect exclusive sizes, and may
 *   also call with zero size.
 * Reworked by rmk.
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>
                .text

#ifdef __ARMEB__
#define SWAB_ENDIAN le
#else
#define SWAB_ENDIAN be
#endif

		.macro	find_first, endian, set, name
ENTRY(_find_first_\name\()bit_\endian)
	UNWIND(	.fnstart)
		teq	r1, #0
		beq	3f
		mov	r2, #0
1:		ldr	r3, [r0], #4
		.ifeq \set
		mvns	r3, r3			@ invert/test bits
		.else
		movs	r3, r3			@ test bits
		.endif
		.ifc \endian, SWAB_ENDIAN
		bne	.L_found_swab
		.else
		bne	.L_found		@ found the bit?
		.endif
		add	r2, r2, #32		@ next index
2:		cmp	r2, r1			@ any more?
		blo	1b
3:		mov	r0, r1			@ no more bits
		ret	lr
	UNWIND(	.fnend)
ENDPROC(_find_first_\name\()bit_\endian)
		.endm

		.macro	find_next, endian, set, name
ENTRY(_find_next_\name\()bit_\endian)
	UNWIND(	.fnstart)
		cmp	r2, r1
		bhs	3b
		mov	ip, r2, lsr #5		@ word index
		add	r0, r0, ip, lsl #2
		ands	ip, r2, #31		@ bit position
		beq	1b
		ldr	r3, [r0], #4
		.ifeq \set
		mvn	r3, r3			@ invert bits
		.endif
		.ifc \endian, SWAB_ENDIAN
		rev_l	r3, ip
		.if	.Lrev_l_uses_tmp
		@ we need to recompute ip because rev_l will have overwritten
		@ it.
		and	ip, r2, #31		@ bit position
		.endif
		.endif
		movs	r3, r3, lsr ip		@ shift off unused bits
		bne	.L_found
		orr	r2, r2, #31		@ no zero bits
		add	r2, r2, #1		@ align bit pointer
		b	2b			@ loop for next bit
	UNWIND(	.fnend)
ENDPROC(_find_next_\name\()bit_\endian)
		.endm

		.macro	find_bit, endian, set, name
		find_first \endian, \set, \name
		find_next  \endian, \set, \name
		.endm

/* _find_first_zero_bit_le and _find_next_zero_bit_le */
		find_bit le, 0, zero_

/* _find_first_bit_le and _find_next_bit_le */
		find_bit le, 1

#ifdef __ARMEB__

/* _find_first_zero_bit_be and _find_next_zero_bit_be */
		find_bit be, 0, zero_

/* _find_first_bit_be and _find_next_bit_be */
		find_bit be, 1

#endif

/*
 * One or more bits in the LSB of r3 are assumed to be set.
 */
.L_found_swab:
	UNWIND(	.fnstart)
		rev_l	r3, ip
.L_found:
#if __LINUX_ARM_ARCH__ >= 7
		rbit	r3, r3			@ reverse bits
		clz	r3, r3			@ count high zero bits
		add	r0, r2, r3		@ add offset of first set bit
#elif __LINUX_ARM_ARCH__ >= 5
		rsb	r0, r3, #0
		and	r3, r3, r0		@ mask out lowest bit set
		clz	r3, r3			@ count high zero bits
		rsb	r3, r3, #31		@ offset of first set bit
		add	r0, r2, r3		@ add offset of first set bit
#else
		mov	ip, #~0
		tst	r3, ip, lsr #16		@ test bits 0-15
		addeq	r2, r2, #16
		moveq	r3, r3, lsr #16
		tst	r3, #0x00ff
		addeq	r2, r2, #8
		moveq	r3, r3, lsr #8
		tst	r3, #0x000f
		addeq	r2, r2, #4
		moveq	r3, r3, lsr #4
		tst	r3, #0x0003
		addeq	r2, r2, #2
		moveq	r3, r3, lsr #2
		tst	r3, #0x0001
		addeq	r2, r2, #1
		mov	r0, r2
#endif
		cmp	r1, r0			@ Clamp to maxbit
		movlo	r0, r1
		ret	lr
	UNWIND(	.fnend)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/getuser.S
 *
 *  Copyright (C) 2001 Russell King
 *
 *  Idea from x86 version, (C) Copyright 1998 Linus Torvalds
 *
 * These functions have a non-standard call interface to make them more
 * efficient, especially as they return an error value in addition to
 * the "real" return value.
 *
 * __get_user_X
 *
 * Inputs:	r0 contains the address
 *		r1 contains the address limit, which must be preserved
 * Outputs:	r0 is the error code
 *		r2, r3 contains the zero-extended value
 *		lr corrupted
 *
 * No other registers must be altered.  (see <asm/uaccess.h>
 * for specific ASM register usage).
 *
 * Note that ADDR_LIMIT is either 0 or 0xc0000000.
 * Note also that it is intended that __get_user_bad is not global.
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/errno.h>
#include <asm/domain.h>

ENTRY(__get_user_1)
	check_uaccess r0, 1, r1, r2, __get_user_bad
1: TUSER(ldrb)	r2, [r0]
	mov	r0, #0
	ret	lr
ENDPROC(__get_user_1)
_ASM_NOKPROBE(__get_user_1)

ENTRY(__get_user_2)
	check_uaccess r0, 2, r1, r2, __get_user_bad
#if __LINUX_ARM_ARCH__ >= 6

2: TUSER(ldrh)	r2, [r0]

#else

#ifdef CONFIG_CPU_USE_DOMAINS
rb	.req	ip
2:	ldrbt	r2, [r0], #1
3:	ldrbt	rb, [r0], #0
#else
rb	.req	r0
2:	ldrb	r2, [r0]
3:	ldrb	rb, [r0, #1]
#endif
#ifndef __ARMEB__
	orr	r2, r2, rb, lsl #8
#else
	orr	r2, rb, r2, lsl #8
#endif

#endif /* __LINUX_ARM_ARCH__ >= 6 */

	mov	r0, #0
	ret	lr
ENDPROC(__get_user_2)
_ASM_NOKPROBE(__get_user_2)

ENTRY(__get_user_4)
	check_uaccess r0, 4, r1, r2, __get_user_bad
4: TUSER(ldr)	r2, [r0]
	mov	r0, #0
	ret	lr
ENDPROC(__get_user_4)
_ASM_NOKPROBE(__get_user_4)

ENTRY(__get_user_8)
	check_uaccess r0, 8, r1, r2, __get_user_bad8
#ifdef CONFIG_THUMB2_KERNEL
5: TUSER(ldr)	r2, [r0]
6: TUSER(ldr)	r3, [r0, #4]
#else
5: TUSER(ldr)	r2, [r0], #4
6: TUSER(ldr)	r3, [r0]
#endif
	mov	r0, #0
	ret	lr
ENDPROC(__get_user_8)
_ASM_NOKPROBE(__get_user_8)

#ifdef __ARMEB__
ENTRY(__get_user_32t_8)
	check_uaccess r0, 8, r1, r2, __get_user_bad
#ifdef CONFIG_CPU_USE_DOMAINS
	add	r0, r0, #4
7:	ldrt	r2, [r0]
#else
7:	ldr	r2, [r0, #4]
#endif
	mov	r0, #0
	ret	lr
ENDPROC(__get_user_32t_8)
_ASM_NOKPROBE(__get_user_32t_8)

ENTRY(__get_user_64t_1)
	check_uaccess r0, 1, r1, r2, __get_user_bad8
8: TUSER(ldrb)	r3, [r0]
	mov	r0, #0
	ret	lr
ENDPROC(__get_user_64t_1)
_ASM_NOKPROBE(__get_user_64t_1)

ENTRY(__get_user_64t_2)
	check_uaccess r0, 2, r1, r2, __get_user_bad8
#ifdef CONFIG_CPU_USE_DOMAINS
rb	.req	ip
9:	ldrbt	r3, [r0], #1
10:	ldrbt	rb, [r0], #0
#else
rb	.req	r0
9:	ldrb	r3, [r0]
10:	ldrb	rb, [r0, #1]
#endif
	orr	r3, rb, r3, lsl #8
	mov	r0, #0
	ret	lr
ENDPROC(__get_user_64t_2)
_ASM_NOKPROBE(__get_user_64t_2)

ENTRY(__get_user_64t_4)
	check_uaccess r0, 4, r1, r2, __get_user_bad8
11: TUSER(ldr)	r3, [r0]
	mov	r0, #0
	ret	lr
ENDPROC(__get_user_64t_4)
_ASM_NOKPROBE(__get_user_64t_4)
#endif

__get_user_bad8:
	mov	r3, #0
__get_user_bad:
	mov	r2, #0
	mov	r0, #-EFAULT
	ret	lr
ENDPROC(__get_user_bad)
ENDPROC(__get_user_bad8)
_ASM_NOKPROBE(__get_user_bad)
_ASM_NOKPROBE(__get_user_bad8)

.pushsection __ex_table, "a"
	.long	1b, __get_user_bad
	.long	2b, __get_user_bad
#if __LINUX_ARM_ARCH__ < 6
	.long	3b, __get_user_bad
#endif
	.long	4b, __get_user_bad
	.long	5b, __get_user_bad8
	.long	6b, __get_user_bad8
#ifdef __ARMEB__
	.long   7b, __get_user_bad
	.long	8b, __get_user_bad8
	.long	9b, __get_user_bad8
	.long	10b, __get_user_bad8
	.long	11b, __get_user_bad8
#endif
.popsection


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/io-readsb.S
 *
 *  Copyright (C) 1995-2000 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

.Linsb_align:	rsb	ip, ip, #4
		cmp	ip, r2
		movgt	ip, r2
		cmp	ip, #2
		ldrb	r3, [r0]
		strb	r3, [r1], #1
		ldrbge	r3, [r0]
		strbge	r3, [r1], #1
		ldrbgt	r3, [r0]
		strbgt	r3, [r1], #1
		subs	r2, r2, ip
		bne	.Linsb_aligned

ENTRY(__raw_readsb)
		teq	r2, #0		@ do we have to check for the zero len?
		reteq	lr
		ands	ip, r1, #3
		bne	.Linsb_align

.Linsb_aligned:	stmfd	sp!, {r4 - r6, lr}

		subs	r2, r2, #16
		bmi	.Linsb_no_16

.Linsb_16_lp:	ldrb	r3, [r0]
		ldrb	r4, [r0]
		ldrb	r5, [r0]
		mov	r3, r3,     put_byte_0
		ldrb	r6, [r0]
		orr	r3, r3, r4, put_byte_1
		ldrb	r4, [r0]
		orr	r3, r3, r5, put_byte_2
		ldrb	r5, [r0]
		orr	r3, r3, r6, put_byte_3
		ldrb	r6, [r0]
		mov	r4, r4,     put_byte_0
		ldrb	ip, [r0]
		orr	r4, r4, r5, put_byte_1
		ldrb	r5, [r0]
		orr	r4, r4, r6, put_byte_2
		ldrb	r6, [r0]
		orr	r4, r4, ip, put_byte_3
		ldrb	ip, [r0]
		mov	r5, r5,     put_byte_0
		ldrb	lr, [r0]
		orr	r5, r5, r6, put_byte_1
		ldrb	r6, [r0]
		orr	r5, r5, ip, put_byte_2
		ldrb	ip, [r0]
		orr	r5, r5, lr, put_byte_3
		ldrb	lr, [r0]
		mov	r6, r6,     put_byte_0
		orr	r6, r6, ip, put_byte_1
		ldrb	ip, [r0]
		orr	r6, r6, lr, put_byte_2
		orr	r6, r6, ip, put_byte_3
		stmia	r1!, {r3 - r6}

		subs	r2, r2, #16
		bpl	.Linsb_16_lp

		tst	r2, #15
		ldmfdeq	sp!, {r4 - r6, pc}

.Linsb_no_16:	tst	r2, #8
		beq	.Linsb_no_8

		ldrb	r3, [r0]
		ldrb	r4, [r0]
		ldrb	r5, [r0]
		mov	r3, r3,     put_byte_0
		ldrb	r6, [r0]
		orr	r3, r3, r4, put_byte_1
		ldrb	r4, [r0]
		orr	r3, r3, r5, put_byte_2
		ldrb	r5, [r0]
		orr	r3, r3, r6, put_byte_3
		ldrb	r6, [r0]
		mov	r4, r4,     put_byte_0
		ldrb	ip, [r0]
		orr	r4, r4, r5, put_byte_1
		orr	r4, r4, r6, put_byte_2
		orr	r4, r4, ip, put_byte_3
		stmia	r1!, {r3, r4}

.Linsb_no_8:	tst	r2, #4
		beq	.Linsb_no_4

		ldrb	r3, [r0]
		ldrb	r4, [r0]
		ldrb	r5, [r0]
		ldrb	r6, [r0]
		mov	r3, r3,     put_byte_0
		orr	r3, r3, r4, put_byte_1
		orr	r3, r3, r5, put_byte_2
		orr	r3, r3, r6, put_byte_3
		str	r3, [r1], #4

.Linsb_no_4:	ands	r2, r2, #3
		ldmfdeq	sp!, {r4 - r6, pc}

		cmp	r2, #2
		ldrb	r3, [r0]
		strb	r3, [r1], #1
		ldrbge	r3, [r0]
		strbge	r3, [r1], #1
		ldrbgt	r3, [r0]
		strbgt	r3, [r1]

		ldmfd	sp!, {r4 - r6, pc}
ENDPROC(__raw_readsb)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/io-readsl.S
 *
 *  Copyright (C) 1995-2000 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

ENTRY(__raw_readsl)
		teq	r2, #0		@ do we have to check for the zero len?
		reteq	lr
		ands	ip, r1, #3
		bne	3f

		subs	r2, r2, #4
		bmi	2f
		stmfd	sp!, {r4, lr}
1:		ldr	r3, [r0, #0]
		ldr	r4, [r0, #0]
		ldr	ip, [r0, #0]
		ldr	lr, [r0, #0]
		subs	r2, r2, #4
		stmia	r1!, {r3, r4, ip, lr}
		bpl	1b
		ldmfd	sp!, {r4, lr}
2:		movs	r2, r2, lsl #31
		ldrcs	r3, [r0, #0]
		ldrcs	ip, [r0, #0]
		stmiacs	r1!, {r3, ip}
		ldrne	r3, [r0, #0]
		strne	r3, [r1, #0]
		ret	lr

3:		ldr	r3, [r0]
		cmp	ip, #2
		mov	ip, r3, get_byte_0
		strb	ip, [r1], #1
		bgt	6f
		mov	ip, r3, get_byte_1
		strb	ip, [r1], #1
		beq	5f
		mov	ip, r3, get_byte_2
		strb	ip, [r1], #1

4:		subs	r2, r2, #1
		mov	ip, r3, lspull #24
		ldrne	r3, [r0]
		orrne	ip, ip, r3, lspush #8
		strne	ip, [r1], #4
		bne	4b
		b	8f

5:		subs	r2, r2, #1
		mov	ip, r3, lspull #16
		ldrne	r3, [r0]
		orrne	ip, ip, r3, lspush #16
		strne	ip, [r1], #4
		bne	5b
		b	7f

6:		subs	r2, r2, #1
		mov	ip, r3, lspull #8
		ldrne	r3, [r0]
		orrne	ip, ip, r3, lspush #24
		strne	ip, [r1], #4
		bne	6b

		mov	r3, ip, get_byte_2
		strb	r3, [r1, #2]
7:		mov	r3, ip, get_byte_1
		strb	r3, [r1, #1]
8:		mov	r3, ip, get_byte_0
		strb	r3, [r1, #0]
		ret	lr
ENDPROC(__raw_readsl)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/io-readsw-armv3.S
 *
 *  Copyright (C) 1995-2000 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

.Linsw_bad_alignment:
		adr	r0, .Linsw_bad_align_msg
		mov	r2, lr
		b	panic
.Linsw_bad_align_msg:
		.asciz	"insw: bad buffer alignment (0x%p, lr=0x%08lX)\n"
		.align

.Linsw_align:	tst	r1, #1
		bne	.Linsw_bad_alignment

		ldr	r3, [r0]
		strb	r3, [r1], #1
		mov	r3, r3, lsr #8
		strb	r3, [r1], #1

		subs	r2, r2, #1
		reteq	lr

ENTRY(__raw_readsw)
		teq	r2, #0		@ do we have to check for the zero len?
		reteq	lr
		tst	r1, #3
		bne	.Linsw_align

.Linsw_aligned:	mov	ip, #0xff
		orr	ip, ip, ip, lsl #8
		stmfd	sp!, {r4, r5, r6, lr}

		subs	r2, r2, #8
		bmi	.Lno_insw_8

.Linsw_8_lp:	ldr	r3, [r0]
		and	r3, r3, ip
		ldr	r4, [r0]
		orr	r3, r3, r4, lsl #16

		ldr	r4, [r0]
		and	r4, r4, ip
		ldr	r5, [r0]
		orr	r4, r4, r5, lsl #16

		ldr	r5, [r0]
		and	r5, r5, ip
		ldr	r6, [r0]
		orr	r5, r5, r6, lsl #16

		ldr	r6, [r0]
		and	r6, r6, ip
		ldr	lr, [r0]
		orr	r6, r6, lr, lsl #16

		stmia	r1!, {r3 - r6}

		subs	r2, r2, #8
		bpl	.Linsw_8_lp

		tst	r2, #7
		ldmfdeq	sp!, {r4, r5, r6, pc}

.Lno_insw_8:	tst	r2, #4
		beq	.Lno_insw_4

		ldr	r3, [r0]
		and	r3, r3, ip
		ldr	r4, [r0]
		orr	r3, r3, r4, lsl #16

		ldr	r4, [r0]
		and	r4, r4, ip
		ldr	r5, [r0]
		orr	r4, r4, r5, lsl #16

		stmia	r1!, {r3, r4}

.Lno_insw_4:	tst	r2, #2
		beq	.Lno_insw_2

		ldr	r3, [r0]
		and	r3, r3, ip
		ldr	r4, [r0]
		orr	r3, r3, r4, lsl #16

		str	r3, [r1], #4

.Lno_insw_2:	tst	r2, #1
		ldrne	r3, [r0]
		strbne	r3, [r1], #1
		movne	r3, r3, lsr #8
		strbne	r3, [r1]

		ldmfd	sp!, {r4, r5, r6, pc}


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/io-readsw-armv4.S
 *
 *  Copyright (C) 1995-2000 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

		.macro	pack, rd, hw1, hw2
#ifndef __ARMEB__
		orr	\rd, \hw1, \hw2, lsl #16
#else
		orr	\rd, \hw2, \hw1, lsl #16
#endif
		.endm

.Linsw_align:	movs	ip, r1, lsl #31
		bne	.Linsw_noalign
		ldrh	ip, [r0]
		sub	r2, r2, #1
		strh	ip, [r1], #2

ENTRY(__raw_readsw)
		teq	r2, #0
		reteq	lr
		tst	r1, #3
		bne	.Linsw_align

		stmfd	sp!, {r4, r5, lr}

		subs	r2, r2, #8
		bmi	.Lno_insw_8

.Linsw_8_lp:	ldrh	r3, [r0]
		ldrh	r4, [r0]
		pack	r3, r3, r4

		ldrh	r4, [r0]
		ldrh	r5, [r0]
		pack	r4, r4, r5

		ldrh	r5, [r0]
		ldrh	ip, [r0]
		pack	r5, r5, ip

		ldrh	ip, [r0]
		ldrh	lr, [r0]
		pack	ip, ip, lr

		subs	r2, r2, #8
		stmia	r1!, {r3 - r5, ip}
		bpl	.Linsw_8_lp

.Lno_insw_8:	tst	r2, #4
		beq	.Lno_insw_4

		ldrh	r3, [r0]
		ldrh	r4, [r0]
		pack	r3, r3, r4

		ldrh	r4, [r0]
		ldrh	ip, [r0]
		pack	r4, r4, ip

		stmia	r1!, {r3, r4}

.Lno_insw_4:	movs	r2, r2, lsl #31
		bcc	.Lno_insw_2

		ldrh	r3, [r0]
		ldrh	ip, [r0]
		pack	r3, r3, ip
		str	r3, [r1], #4

.Lno_insw_2:	ldrhne	r3, [r0]
		strhne	r3, [r1]

		ldmfd	sp!, {r4, r5, pc}

#ifdef __ARMEB__
#define _BE_ONLY_(code...)	code
#define _LE_ONLY_(code...)
#define push_hbyte0		lsr #8
#define pull_hbyte1		lsl #24
#else
#define _BE_ONLY_(code...)
#define _LE_ONLY_(code...) code
#define push_hbyte0		lsl #24
#define pull_hbyte1		lsr #8
#endif

.Linsw_noalign:	stmfd	sp!, {r4, lr}
		ldrbcc	ip, [r1, #-1]!
		bcc	1f

		ldrh	ip, [r0]
		sub	r2, r2, #1
   _BE_ONLY_(	mov	ip, ip, ror #8		)
		strb	ip, [r1], #1
   _LE_ONLY_(	mov	ip, ip, lsr #8		)
   _BE_ONLY_(	mov	ip, ip, lsr #24		)

1:		subs	r2, r2, #2
		bmi	3f
   _BE_ONLY_(	mov	ip, ip, lsl #24		)

2:		ldrh	r3, [r0]
		ldrh	r4, [r0]
		subs	r2, r2, #2
		orr	ip, ip, r3, lsl #8
		orr	ip, ip, r4, push_hbyte0
		str	ip, [r1], #4
		mov	ip, r4, pull_hbyte1
		bpl	2b

   _BE_ONLY_(	mov	ip, ip, lsr #24		)

3:		tst	r2, #1
		strb	ip, [r1], #1
		ldrhne	ip, [r0]
   _BE_ONLY_(	movne	ip, ip, ror #8		)
		strbne	ip, [r1], #1
   _LE_ONLY_(	movne	ip, ip, lsr #8		)
   _BE_ONLY_(	movne	ip, ip, lsr #24		)
		strbne	ip, [r1]
		ldmfd	sp!, {r4, pc}
ENDPROC(__raw_readsw)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/io-writesb.S
 *
 *  Copyright (C) 1995-2000 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

		.macro	outword, rd
#ifndef __ARMEB__
		strb	\rd, [r0]
		mov	\rd, \rd, lsr #8
		strb	\rd, [r0]
		mov	\rd, \rd, lsr #8
		strb	\rd, [r0]
		mov	\rd, \rd, lsr #8
		strb	\rd, [r0]
#else
		mov	lr, \rd, lsr #24
		strb	lr, [r0]
		mov	lr, \rd, lsr #16
		strb	lr, [r0]
		mov	lr, \rd, lsr #8
		strb	lr, [r0]
		strb	\rd, [r0]
#endif
		.endm

.Loutsb_align:	rsb	ip, ip, #4
		cmp	ip, r2
		movgt	ip, r2
		cmp	ip, #2
		ldrb	r3, [r1], #1
		strb	r3, [r0]
		ldrbge	r3, [r1], #1
		strbge	r3, [r0]
		ldrbgt	r3, [r1], #1
		strbgt	r3, [r0]
		subs	r2, r2, ip
		bne	.Loutsb_aligned

ENTRY(__raw_writesb)
		teq	r2, #0		@ do we have to check for the zero len?
		reteq	lr
		ands	ip, r1, #3
		bne	.Loutsb_align

.Loutsb_aligned:
		stmfd	sp!, {r4, r5, lr}

		subs	r2, r2, #16
		bmi	.Loutsb_no_16

.Loutsb_16_lp:	ldmia	r1!, {r3, r4, r5, ip}
		outword	r3
		outword	r4
		outword	r5
		outword	ip
		subs	r2, r2, #16
		bpl	.Loutsb_16_lp

		tst	r2, #15
		ldmfdeq	sp!, {r4, r5, pc}

.Loutsb_no_16:	tst	r2, #8
		beq	.Loutsb_no_8

		ldmia	r1!, {r3, r4}
		outword	r3
		outword	r4

.Loutsb_no_8:	tst	r2, #4
		beq	.Loutsb_no_4

		ldr	r3, [r1], #4
		outword	r3

.Loutsb_no_4:	ands	r2, r2, #3
		ldmfdeq	sp!, {r4, r5, pc}

		cmp	r2, #2
		ldrb	r3, [r1], #1
		strb	r3, [r0]
		ldrbge	r3, [r1], #1
		strbge	r3, [r0]
		ldrbgt	r3, [r1]
		strbgt	r3, [r0]

		ldmfd	sp!, {r4, r5, pc}
ENDPROC(__raw_writesb)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/io-writesl.S
 *
 *  Copyright (C) 1995-2000 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

ENTRY(__raw_writesl)
		teq	r2, #0		@ do we have to check for the zero len?
		reteq	lr
		ands	ip, r1, #3
		bne	3f

		subs	r2, r2, #4
		bmi	2f
		stmfd	sp!, {r4, lr}
1:		ldmia	r1!, {r3, r4, ip, lr}
		subs	r2, r2, #4
		str	r3, [r0, #0]
		str	r4, [r0, #0]
		str	ip, [r0, #0]
		str	lr, [r0, #0]
		bpl	1b
		ldmfd	sp!, {r4, lr}
2:		movs	r2, r2, lsl #31
		ldmiacs	r1!, {r3, ip}
		strcs	r3, [r0, #0]
		ldrne	r3, [r1, #0]
		strcs	ip, [r0, #0]
		strne	r3, [r0, #0]
		ret	lr

3:		bic	r1, r1, #3
		ldr	r3, [r1], #4
		cmp	ip, #2
		blt	5f
		bgt	6f

4:		mov	ip, r3, lspull #16
		ldr	r3, [r1], #4
		subs	r2, r2, #1
		orr	ip, ip, r3, lspush #16
		str	ip, [r0]
		bne	4b
		ret	lr

5:		mov	ip, r3, lspull #8
		ldr	r3, [r1], #4
		subs	r2, r2, #1
		orr	ip, ip, r3, lspush #24
		str	ip, [r0]
		bne	5b
		ret	lr

6:		mov	ip, r3, lspull #24
		ldr	r3, [r1], #4
		subs	r2, r2, #1
		orr	ip, ip, r3, lspush #8
		str	ip, [r0]
		bne	6b
		ret	lr
ENDPROC(__raw_writesl)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/io-writesw-armv3.S
 *
 *  Copyright (C) 1995-2000 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

.Loutsw_bad_alignment:
		adr	r0, .Loutsw_bad_align_msg
		mov	r2, lr
		b	panic
.Loutsw_bad_align_msg:
		.asciz	"outsw: bad buffer alignment (0x%p, lr=0x%08lX)\n"
		.align

.Loutsw_align:	tst	r1, #1
		bne	.Loutsw_bad_alignment

		add	r1, r1, #2

		ldr	r3, [r1, #-4]
		mov	r3, r3, lsr #16
		orr	r3, r3, r3, lsl #16
		str	r3, [r0]
		subs	r2, r2, #1
		reteq	lr

ENTRY(__raw_writesw)
		teq	r2, #0		@ do we have to check for the zero len?
		reteq	lr
		tst	r1, #3
		bne	.Loutsw_align

		stmfd	sp!, {r4, r5, r6, lr}

		subs	r2, r2, #8
		bmi	.Lno_outsw_8

.Loutsw_8_lp:	ldmia	r1!, {r3, r4, r5, r6}

		mov	ip, r3, lsl #16
		orr	ip, ip, ip, lsr #16
		str	ip, [r0]

		mov	ip, r3, lsr #16
		orr	ip, ip, ip, lsl #16
		str	ip, [r0]

		mov	ip, r4, lsl #16
		orr	ip, ip, ip, lsr #16
		str	ip, [r0]

		mov	ip, r4, lsr #16
		orr	ip, ip, ip, lsl #16
		str	ip, [r0]

		mov	ip, r5, lsl #16
		orr	ip, ip, ip, lsr #16
		str	ip, [r0]

		mov	ip, r5, lsr #16
		orr	ip, ip, ip, lsl #16
		str	ip, [r0]

		mov	ip, r6, lsl #16
		orr	ip, ip, ip, lsr #16
		str	ip, [r0]

		mov	ip, r6, lsr #16
		orr	ip, ip, ip, lsl #16
		str	ip, [r0]

		subs	r2, r2, #8
		bpl	.Loutsw_8_lp

		tst	r2, #7
		ldmfdeq	sp!, {r4, r5, r6, pc}

.Lno_outsw_8:	tst	r2, #4
		beq	.Lno_outsw_4

		ldmia	r1!, {r3, r4}

		mov	ip, r3, lsl #16
		orr	ip, ip, ip, lsr #16
		str	ip, [r0]

		mov	ip, r3, lsr #16
		orr	ip, ip, ip, lsl #16
		str	ip, [r0]

		mov	ip, r4, lsl #16
		orr	ip, ip, ip, lsr #16
		str	ip, [r0]

		mov	ip, r4, lsr #16
		orr	ip, ip, ip, lsl #16
		str	ip, [r0]

.Lno_outsw_4:	tst	r2, #2
		beq	.Lno_outsw_2

		ldr	r3, [r1], #4

		mov	ip, r3, lsl #16
		orr	ip, ip, ip, lsr #16
		str	ip, [r0]

		mov	ip, r3, lsr #16
		orr	ip, ip, ip, lsl #16
		str	ip, [r0]

.Lno_outsw_2:	tst	r2, #1

		ldrne	r3, [r1]

		movne	ip, r3, lsl #16
		orrne	ip, ip, ip, lsr #16
		strne	ip, [r0]

		ldmfd	sp!, {r4, r5, r6, pc}


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/io-writesw-armv4.S
 *
 *  Copyright (C) 1995-2000 Russell King
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

		.macro	outword, rd
#ifndef __ARMEB__
		strh	\rd, [r0]
		mov	\rd, \rd, lsr #16
		strh	\rd, [r0]
#else
		mov	lr, \rd, lsr #16
		strh	lr, [r0]
		strh	\rd, [r0]
#endif
		.endm

.Loutsw_align:	movs	ip, r1, lsl #31
		bne	.Loutsw_noalign

		ldrh	r3, [r1], #2
		sub	r2, r2, #1
		strh	r3, [r0]

ENTRY(__raw_writesw)
		teq	r2, #0
		reteq	lr
		ands	r3, r1, #3
		bne	.Loutsw_align

		stmfd	sp!, {r4, r5, lr}

		subs	r2, r2, #8
		bmi	.Lno_outsw_8

.Loutsw_8_lp:	ldmia	r1!, {r3, r4, r5, ip}
		subs	r2, r2, #8
		outword	r3
		outword	r4
		outword	r5
		outword	ip
		bpl	.Loutsw_8_lp

.Lno_outsw_8:	tst	r2, #4
		beq	.Lno_outsw_4

		ldmia	r1!, {r3, ip}
		outword	r3
		outword	ip

.Lno_outsw_4:	movs	r2, r2, lsl #31
		bcc	.Lno_outsw_2

		ldr	r3, [r1], #4
		outword	r3

.Lno_outsw_2:	ldrhne	r3, [r1]
		strhne	r3, [r0]

		ldmfd	sp!, {r4, r5, pc}

#ifdef __ARMEB__
#define pull_hbyte0	lsl #8
#define push_hbyte1	lsr #24
#else
#define pull_hbyte0	lsr #24
#define push_hbyte1	lsl #8
#endif

.Loutsw_noalign:
 ARM(		ldr	r3, [r1, -r3]!	)
 THUMB(		rsb	r3, r3, #0	)
 THUMB(		ldr	r3, [r1, r3]	)
 THUMB(		sub	r1, r3		)
		subcs	r2, r2, #1
		bcs	2f
		subs	r2, r2, #2
		bmi	3f

1:		mov	ip, r3, lsr #8
		strh	ip, [r0]
2:		mov	ip, r3, pull_hbyte0
		ldr	r3, [r1, #4]!
		subs	r2, r2, #2
		orr	ip, ip, r3, push_hbyte1
		strh	ip, [r0]
		bpl	1b

		tst	r2, #1
3:		movne	ip, r3, lsr #8
		strhne	ip, [r0]
		ret	lr
ENDPROC(__raw_writesw)


code:
/*
 * linux/arch/arm/lib/lib1funcs.S: Optimized ARM division routines
 *
 * Author: Nicolas Pitre <nico@fluxnic.net>
 *   - contributed to gcc-3.4 on Sep 30, 2003
 *   - adapted for the Linux kernel on Oct 2, 2003
 */

/* Copyright 1995, 1996, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.

This file is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2, or (at your option) any
later version.

In addition to the permissions in the GNU General Public License, the
Free Software Foundation gives you unlimited permission to link the
compiled version of this file into combinations with other programs,
and to distribute those combinations without any restriction coming
from the use of this file.  (The General Public License restrictions
do apply in other respects; for example, they cover modification of
the file, and distribution when not linked into a combine
executable.)

This file is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; see the file COPYING.  If not, write to
the Free Software Foundation, 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.  */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>

.macro ARM_DIV_BODY dividend, divisor, result, curbit

#if __LINUX_ARM_ARCH__ >= 5

	clz	\curbit, \divisor
	clz	\result, \dividend
	sub	\result, \curbit, \result
	mov	\curbit, #1
	mov	\divisor, \divisor, lsl \result
	mov	\curbit, \curbit, lsl \result
	mov	\result, #0

#else

	@ Initially shift the divisor left 3 bits if possible,
	@ set curbit accordingly.  This allows for curbit to be located
	@ at the left end of each 4 bit nibbles in the division loop
	@ to save one loop in most cases.
	tst	\divisor, #0xe0000000
	moveq	\divisor, \divisor, lsl #3
	moveq	\curbit, #8
	movne	\curbit, #1

	@ Unless the divisor is very big, shift it up in multiples of
	@ four bits, since this is the amount of unwinding in the main
	@ division loop.  Continue shifting until the divisor is
	@ larger than the dividend.
1:	cmp	\divisor, #0x10000000
	cmplo	\divisor, \dividend
	movlo	\divisor, \divisor, lsl #4
	movlo	\curbit, \curbit, lsl #4
	blo	1b

	@ For very big divisors, we must shift it a bit at a time, or
	@ we will be in danger of overflowing.
1:	cmp	\divisor, #0x80000000
	cmplo	\divisor, \dividend
	movlo	\divisor, \divisor, lsl #1
	movlo	\curbit, \curbit, lsl #1
	blo	1b

	mov	\result, #0

#endif

	@ Division loop
1:	cmp	\dividend, \divisor
	subhs	\dividend, \dividend, \divisor
	orrhs	\result,   \result,   \curbit
	cmp	\dividend, \divisor,  lsr #1
	subhs	\dividend, \dividend, \divisor, lsr #1
	orrhs	\result,   \result,   \curbit,  lsr #1
	cmp	\dividend, \divisor,  lsr #2
	subhs	\dividend, \dividend, \divisor, lsr #2
	orrhs	\result,   \result,   \curbit,  lsr #2
	cmp	\dividend, \divisor,  lsr #3
	subhs	\dividend, \dividend, \divisor, lsr #3
	orrhs	\result,   \result,   \curbit,  lsr #3
	cmp	\dividend, #0			@ Early termination?
	movsne	\curbit,   \curbit,  lsr #4	@ No, any more bits to do?
	movne	\divisor,  \divisor, lsr #4
	bne	1b

.endm

.macro ARM_DIV2_ORDER divisor, order

#if __LINUX_ARM_ARCH__ >= 5

	clz	\order, \divisor
	rsb	\order, \order, #31

#else

	cmp	\divisor, #(1 << 16)
	movhs	\divisor, \divisor, lsr #16
	movhs	\order, #16
	movlo	\order, #0

	cmp	\divisor, #(1 << 8)
	movhs	\divisor, \divisor, lsr #8
	addhs	\order, \order, #8

	cmp	\divisor, #(1 << 4)
	movhs	\divisor, \divisor, lsr #4
	addhs	\order, \order, #4

	cmp	\divisor, #(1 << 2)
	addhi	\order, \order, #3
	addls	\order, \order, \divisor, lsr #1

#endif

.endm

.macro ARM_MOD_BODY dividend, divisor, order, spare

#if __LINUX_ARM_ARCH__ >= 5

	clz	\order, \divisor
	clz	\spare, \dividend
	sub	\order, \order, \spare
	mov	\divisor, \divisor, lsl \order

#else

	mov	\order, #0

	@ Unless the divisor is very big, shift it up in multiples of
	@ four bits, since this is the amount of unwinding in the main
	@ division loop.  Continue shifting until the divisor is
	@ larger than the dividend.
1:	cmp	\divisor, #0x10000000
	cmplo	\divisor, \dividend
	movlo	\divisor, \divisor, lsl #4
	addlo	\order, \order, #4
	blo	1b

	@ For very big divisors, we must shift it a bit at a time, or
	@ we will be in danger of overflowing.
1:	cmp	\divisor, #0x80000000
	cmplo	\divisor, \dividend
	movlo	\divisor, \divisor, lsl #1
	addlo	\order, \order, #1
	blo	1b

#endif

	@ Perform all needed subtractions to keep only the reminder.
	@ Do comparisons in batch of 4 first.
	subs	\order, \order, #3		@ yes, 3 is intended here
	blt	2f

1:	cmp	\dividend, \divisor
	subhs	\dividend, \dividend, \divisor
	cmp	\dividend, \divisor,  lsr #1
	subhs	\dividend, \dividend, \divisor, lsr #1
	cmp	\dividend, \divisor,  lsr #2
	subhs	\dividend, \dividend, \divisor, lsr #2
	cmp	\dividend, \divisor,  lsr #3
	subhs	\dividend, \dividend, \divisor, lsr #3
	cmp	\dividend, #1
	mov	\divisor, \divisor, lsr #4
	subsge	\order, \order, #4
	bge	1b

	tst	\order, #3
	teqne	\dividend, #0
	beq	5f

	@ Either 1, 2 or 3 comparison/subtractions are left.
2:	cmn	\order, #2
	blt	4f
	beq	3f
	cmp	\dividend, \divisor
	subhs	\dividend, \dividend, \divisor
	mov	\divisor,  \divisor,  lsr #1
3:	cmp	\dividend, \divisor
	subhs	\dividend, \dividend, \divisor
	mov	\divisor,  \divisor,  lsr #1
4:	cmp	\dividend, \divisor
	subhs	\dividend, \dividend, \divisor
5:
.endm

#ifdef CONFIG_ARM_PATCH_IDIV
	.align	3
#endif

ENTRY(__udivsi3)
ENTRY(__aeabi_uidiv)
UNWIND(.fnstart)

	subs	r2, r1, #1
	reteq	lr
	bcc	Ldiv0
	cmp	r0, r1
	bls	11f
	tst	r1, r2
	beq	12f

	ARM_DIV_BODY r0, r1, r2, r3

	mov	r0, r2
	ret	lr

11:	moveq	r0, #1
	movne	r0, #0
	ret	lr

12:	ARM_DIV2_ORDER r1, r2

	mov	r0, r0, lsr r2
	ret	lr

UNWIND(.fnend)
ENDPROC(__udivsi3)
ENDPROC(__aeabi_uidiv)

ENTRY(__umodsi3)
UNWIND(.fnstart)

	subs	r2, r1, #1			@ compare divisor with 1
	bcc	Ldiv0
	cmpne	r0, r1				@ compare dividend with divisor
	moveq   r0, #0
	tsthi	r1, r2				@ see if divisor is power of 2
	andeq	r0, r0, r2
	retls	lr

	ARM_MOD_BODY r0, r1, r2, r3

	ret	lr

UNWIND(.fnend)
ENDPROC(__umodsi3)

#ifdef CONFIG_ARM_PATCH_IDIV
	.align 3
#endif

ENTRY(__divsi3)
ENTRY(__aeabi_idiv)
UNWIND(.fnstart)

	cmp	r1, #0
	eor	ip, r0, r1			@ save the sign of the result.
	beq	Ldiv0
	rsbmi	r1, r1, #0			@ loops below use unsigned.
	subs	r2, r1, #1			@ division by 1 or -1 ?
	beq	10f
	movs	r3, r0
	rsbmi	r3, r0, #0			@ positive dividend value
	cmp	r3, r1
	bls	11f
	tst	r1, r2				@ divisor is power of 2 ?
	beq	12f

	ARM_DIV_BODY r3, r1, r0, r2

	cmp	ip, #0
	rsbmi	r0, r0, #0
	ret	lr

10:	teq	ip, r0				@ same sign ?
	rsbmi	r0, r0, #0
	ret	lr

11:	movlo	r0, #0
	moveq	r0, ip, asr #31
	orreq	r0, r0, #1
	ret	lr

12:	ARM_DIV2_ORDER r1, r2

	cmp	ip, #0
	mov	r0, r3, lsr r2
	rsbmi	r0, r0, #0
	ret	lr

UNWIND(.fnend)
ENDPROC(__divsi3)
ENDPROC(__aeabi_idiv)

ENTRY(__modsi3)
UNWIND(.fnstart)

	cmp	r1, #0
	beq	Ldiv0
	rsbmi	r1, r1, #0			@ loops below use unsigned.
	movs	ip, r0				@ preserve sign of dividend
	rsbmi	r0, r0, #0			@ if negative make positive
	subs	r2, r1, #1			@ compare divisor with 1
	cmpne	r0, r1				@ compare dividend with divisor
	moveq	r0, #0
	tsthi	r1, r2				@ see if divisor is power of 2
	andeq	r0, r0, r2
	bls	10f

	ARM_MOD_BODY r0, r1, r2, r3

10:	cmp	ip, #0
	rsbmi	r0, r0, #0
	ret	lr

UNWIND(.fnend)
ENDPROC(__modsi3)

#ifdef CONFIG_AEABI

ENTRY(__aeabi_uidivmod)
UNWIND(.fnstart)
UNWIND(.save {r0, r1, ip, lr}	)

	stmfd	sp!, {r0, r1, ip, lr}
	bl	__aeabi_uidiv
	ldmfd	sp!, {r1, r2, ip, lr}
	mul	r3, r0, r2
	sub	r1, r1, r3
	ret	lr

UNWIND(.fnend)
ENDPROC(__aeabi_uidivmod)

ENTRY(__aeabi_idivmod)
UNWIND(.fnstart)
UNWIND(.save {r0, r1, ip, lr}	)
	stmfd	sp!, {r0, r1, ip, lr}
	bl	__aeabi_idiv
	ldmfd	sp!, {r1, r2, ip, lr}
	mul	r3, r0, r2
	sub	r1, r1, r3
	ret	lr

UNWIND(.fnend)
ENDPROC(__aeabi_idivmod)

#endif

Ldiv0:
UNWIND(.fnstart)
UNWIND(.pad #4)
UNWIND(.save {lr})
	str	lr, [sp, #-8]!
	bl	__div0
	mov	r0, #0			@ About as wrong as it could be.
	ldr	pc, [sp], #8
UNWIND(.fnend)
ENDPROC(Ldiv0)


code:
/* Copyright 1995, 1996, 1998, 1999, 2000, 2003, 2004, 2005
   Free Software Foundation, Inc.

This file is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2, or (at your option) any
later version.

In addition to the permissions in the GNU General Public License, the
Free Software Foundation gives you unlimited permission to link the
compiled version of this file into combinations with other programs,
and to distribute those combinations without any restriction coming
from the use of this file.  (The General Public License restrictions
do apply in other respects; for example, they cover modification of
the file, and distribution when not linked into a combine
executable.)

This file is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; see the file COPYING.  If not, write to
the Free Software Foundation, 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA.  */

#include <linux/linkage.h>
#include <asm/assembler.h>

#ifdef __ARMEB__
#define al r1
#define ah r0
#else
#define al r0
#define ah r1
#endif

ENTRY(__lshrdi3)
ENTRY(__aeabi_llsr)

	subs	r3, r2, #32
	rsb	ip, r2, #32
	movmi	al, al, lsr r2
	movpl	al, ah, lsr r3
 ARM(	orrmi	al, al, ah, lsl ip	)
 THUMB(	lslmi	r3, ah, ip		)
 THUMB(	orrmi	al, al, r3		)
	mov	ah, ah, lsr r2
	ret	lr

ENDPROC(__lshrdi3)
ENDPROC(__aeabi_llsr)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/memchr.S
 *
 *  Copyright (C) 1995-2000 Russell King
 *
 *  ASM optimised string functions
 */
#include <linux/linkage.h>
#include <asm/assembler.h>

	.text
	.align	5
ENTRY(memchr)
1:	subs	r2, r2, #1
	bmi	2f
	ldrb	r3, [r0], #1
	teq	r3, r1
	bne	1b
	sub	r0, r0, #1
2:	movne	r0, #0
	ret	lr
ENDPROC(memchr)


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/memcpy.S
 *
 *  Author:	Nicolas Pitre
 *  Created:	Sep 28, 2005
 *  Copyright:	MontaVista Software, Inc.
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>

#define LDR1W_SHIFT	0
#define STR1W_SHIFT	0

	.macro ldr1w ptr reg abort
	W(ldr) \reg, [\ptr], #4
	.endm

	.macro ldr4w ptr reg1 reg2 reg3 reg4 abort
	ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4}
	.endm

	.macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
	.endm

	.macro ldr1b ptr reg cond=al abort
	ldrb\cond \reg, [\ptr], #1
	.endm

	.macro str1w ptr reg abort
	W(str) \reg, [\ptr], #4
	.endm

	.macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
	.endm

	.macro str1b ptr reg cond=al abort
	strb\cond \reg, [\ptr], #1
	.endm

	.macro enter regs:vararg
UNWIND( .save	{r0, \regs}		)
	stmdb sp!, {r0, \regs}
	.endm

	.macro exit regs:vararg
	ldmfd sp!, {r0, \regs}
	.endm

	.text

/* Prototype: void *memcpy(void *dest, const void *src, size_t n); */

ENTRY(__memcpy)
ENTRY(mmiocpy)
WEAK(memcpy)

#include "copy_template.S"

ENDPROC(memcpy)
ENDPROC(mmiocpy)
ENDPROC(__memcpy)