code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 *
 * Copyright (C) 2009, 2010 ARM Limited
 *
 * Author: Will Deacon <will.deacon@arm.com>
 */

/*
 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
 * using the CPU's debug registers.
 */
#define pr_fmt(fmt) "hw-breakpoint: " fmt

#include <linux/errno.h>
#include <linux/hardirq.h>
#include <linux/perf_event.h>
#include <linux/hw_breakpoint.h>
#include <linux/smp.h>
#include <linux/cpu_pm.h>
#include <linux/coresight.h>

#include <asm/cacheflush.h>
#include <asm/cputype.h>
#include <asm/current.h>
#include <asm/hw_breakpoint.h>
#include <asm/traps.h>

/* Breakpoint currently in use for each BRP. */
static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);

/* Watchpoint currently in use for each WRP. */
static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);

/* Number of BRP/WRP registers on this CPU. */
static int core_num_brps __ro_after_init;
static int core_num_wrps __ro_after_init;

/* Debug architecture version. */
static u8 debug_arch __ro_after_init;

/* Does debug architecture support OS Save and Restore? */
static bool has_ossr __ro_after_init;

/* Maximum supported watchpoint length. */
static u8 max_watchpoint_len __ro_after_init;

#define READ_WB_REG_CASE(OP2, M, VAL)			\
	case ((OP2 << 4) + M):				\
		ARM_DBG_READ(c0, c ## M, OP2, VAL);	\
		break

#define WRITE_WB_REG_CASE(OP2, M, VAL)			\
	case ((OP2 << 4) + M):				\
		ARM_DBG_WRITE(c0, c ## M, OP2, VAL);	\
		break

#define GEN_READ_WB_REG_CASES(OP2, VAL)		\
	READ_WB_REG_CASE(OP2, 0, VAL);		\
	READ_WB_REG_CASE(OP2, 1, VAL);		\
	READ_WB_REG_CASE(OP2, 2, VAL);		\
	READ_WB_REG_CASE(OP2, 3, VAL);		\
	READ_WB_REG_CASE(OP2, 4, VAL);		\
	READ_WB_REG_CASE(OP2, 5, VAL);		\
	READ_WB_REG_CASE(OP2, 6, VAL);		\
	READ_WB_REG_CASE(OP2, 7, VAL);		\
	READ_WB_REG_CASE(OP2, 8, VAL);		\
	READ_WB_REG_CASE(OP2, 9, VAL);		\
	READ_WB_REG_CASE(OP2, 10, VAL);		\
	READ_WB_REG_CASE(OP2, 11, VAL);		\
	READ_WB_REG_CASE(OP2, 12, VAL);		\
	READ_WB_REG_CASE(OP2, 13, VAL);		\
	READ_WB_REG_CASE(OP2, 14, VAL);		\
	READ_WB_REG_CASE(OP2, 15, VAL)

#define GEN_WRITE_WB_REG_CASES(OP2, VAL)	\
	WRITE_WB_REG_CASE(OP2, 0, VAL);		\
	WRITE_WB_REG_CASE(OP2, 1, VAL);		\
	WRITE_WB_REG_CASE(OP2, 2, VAL);		\
	WRITE_WB_REG_CASE(OP2, 3, VAL);		\
	WRITE_WB_REG_CASE(OP2, 4, VAL);		\
	WRITE_WB_REG_CASE(OP2, 5, VAL);		\
	WRITE_WB_REG_CASE(OP2, 6, VAL);		\
	WRITE_WB_REG_CASE(OP2, 7, VAL);		\
	WRITE_WB_REG_CASE(OP2, 8, VAL);		\
	WRITE_WB_REG_CASE(OP2, 9, VAL);		\
	WRITE_WB_REG_CASE(OP2, 10, VAL);	\
	WRITE_WB_REG_CASE(OP2, 11, VAL);	\
	WRITE_WB_REG_CASE(OP2, 12, VAL);	\
	WRITE_WB_REG_CASE(OP2, 13, VAL);	\
	WRITE_WB_REG_CASE(OP2, 14, VAL);	\
	WRITE_WB_REG_CASE(OP2, 15, VAL)

static u32 read_wb_reg(int n)
{
	u32 val = 0;

	switch (n) {
	GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val);
	GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val);
	GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val);
	GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val);
	default:
		pr_warn("attempt to read from unknown breakpoint register %d\n",
			n);
	}

	return val;
}

static void write_wb_reg(int n, u32 val)
{
	switch (n) {
	GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val);
	GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val);
	GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val);
	GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val);
	default:
		pr_warn("attempt to write to unknown breakpoint register %d\n",
			n);
	}
	isb();
}

/* Determine debug architecture. */
static u8 get_debug_arch(void)
{
	u32 didr;

	/* Do we implement the extended CPUID interface? */
	if (((read_cpuid_id() >> 16) & 0xf) != 0xf) {
		pr_warn_once("CPUID feature registers not supported. "
			     "Assuming v6 debug is present.\n");
		return ARM_DEBUG_ARCH_V6;
	}

	ARM_DBG_READ(c0, c0, 0, didr);
	return (didr >> 16) & 0xf;
}

u8 arch_get_debug_arch(void)
{
	return debug_arch;
}

static int debug_arch_supported(void)
{
	u8 arch = get_debug_arch();

	/* We don't support the memory-mapped interface. */
	return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) ||
		arch >= ARM_DEBUG_ARCH_V7_1;
}

/* Can we determine the watchpoint access type from the fsr? */
static int debug_exception_updates_fsr(void)
{
	return get_debug_arch() >= ARM_DEBUG_ARCH_V8;
}

/* Determine number of WRP registers available. */
static int get_num_wrp_resources(void)
{
	u32 didr;
	ARM_DBG_READ(c0, c0, 0, didr);
	return ((didr >> 28) & 0xf) + 1;
}

/* Determine number of BRP registers available. */
static int get_num_brp_resources(void)
{
	u32 didr;
	ARM_DBG_READ(c0, c0, 0, didr);
	return ((didr >> 24) & 0xf) + 1;
}

/* Does this core support mismatch breakpoints? */
static int core_has_mismatch_brps(void)
{
	return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 &&
		get_num_brp_resources() > 1);
}

/* Determine number of usable WRPs available. */
static int get_num_wrps(void)
{
	/*
	 * On debug architectures prior to 7.1, when a watchpoint fires, the
	 * only way to work out which watchpoint it was is by disassembling
	 * the faulting instruction and working out the address of the memory
	 * access.
	 *
	 * Furthermore, we can only do this if the watchpoint was precise
	 * since imprecise watchpoints prevent us from calculating register
	 * based addresses.
	 *
	 * Providing we have more than 1 breakpoint register, we only report
	 * a single watchpoint register for the time being. This way, we always
	 * know which watchpoint fired. In the future we can either add a
	 * disassembler and address generation emulator, or we can insert a
	 * check to see if the DFAR is set on watchpoint exception entry
	 * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows
	 * that it is set on some implementations].
	 */
	if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1)
		return 1;

	return get_num_wrp_resources();
}

/* Determine number of usable BRPs available. */
static int get_num_brps(void)
{
	int brps = get_num_brp_resources();
	return core_has_mismatch_brps() ? brps - 1 : brps;
}

/*
 * In order to access the breakpoint/watchpoint control registers,
 * we must be running in debug monitor mode. Unfortunately, we can
 * be put into halting debug mode at any time by an external debugger
 * but there is nothing we can do to prevent that.
 */
static int monitor_mode_enabled(void)
{
	u32 dscr;
	ARM_DBG_READ(c0, c1, 0, dscr);
	return !!(dscr & ARM_DSCR_MDBGEN);
}

static int enable_monitor_mode(void)
{
	u32 dscr;
	ARM_DBG_READ(c0, c1, 0, dscr);

	/* If monitor mode is already enabled, just return. */
	if (dscr & ARM_DSCR_MDBGEN)
		goto out;

	/* Write to the corresponding DSCR. */
	switch (get_debug_arch()) {
	case ARM_DEBUG_ARCH_V6:
	case ARM_DEBUG_ARCH_V6_1:
		ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN));
		break;
	case ARM_DEBUG_ARCH_V7_ECP14:
	case ARM_DEBUG_ARCH_V7_1:
	case ARM_DEBUG_ARCH_V8:
	case ARM_DEBUG_ARCH_V8_1:
	case ARM_DEBUG_ARCH_V8_2:
	case ARM_DEBUG_ARCH_V8_4:
		ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN));
		isb();
		break;
	default:
		return -ENODEV;
	}

	/* Check that the write made it through. */
	ARM_DBG_READ(c0, c1, 0, dscr);
	if (!(dscr & ARM_DSCR_MDBGEN)) {
		pr_warn_once("Failed to enable monitor mode on CPU %d.\n",
				smp_processor_id());
		return -EPERM;
	}

out:
	return 0;
}

int hw_breakpoint_slots(int type)
{
	if (!debug_arch_supported())
		return 0;

	/*
	 * We can be called early, so don't rely on
	 * our static variables being initialised.
	 */
	switch (type) {
	case TYPE_INST:
		return get_num_brps();
	case TYPE_DATA:
		return get_num_wrps();
	default:
		pr_warn("unknown slot type: %d\n", type);
		return 0;
	}
}

/*
 * Check if 8-bit byte-address select is available.
 * This clobbers WRP 0.
 */
static u8 get_max_wp_len(void)
{
	u32 ctrl_reg;
	struct arch_hw_breakpoint_ctrl ctrl;
	u8 size = 4;

	if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14)
		goto out;

	memset(&ctrl, 0, sizeof(ctrl));
	ctrl.len = ARM_BREAKPOINT_LEN_8;
	ctrl_reg = encode_ctrl_reg(ctrl);

	write_wb_reg(ARM_BASE_WVR, 0);
	write_wb_reg(ARM_BASE_WCR, ctrl_reg);
	if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)
		size = 8;

out:
	return size;
}

u8 arch_get_max_wp_len(void)
{
	return max_watchpoint_len;
}

/*
 * Install a perf counter breakpoint.
 */
int arch_install_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	struct perf_event **slot, **slots;
	int i, max_slots, ctrl_base, val_base;
	u32 addr, ctrl;

	addr = info->address;
	ctrl = encode_ctrl_reg(info->ctrl) | 0x1;

	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
		/* Breakpoint */
		ctrl_base = ARM_BASE_BCR;
		val_base = ARM_BASE_BVR;
		slots = this_cpu_ptr(bp_on_reg);
		max_slots = core_num_brps;
	} else {
		/* Watchpoint */
		ctrl_base = ARM_BASE_WCR;
		val_base = ARM_BASE_WVR;
		slots = this_cpu_ptr(wp_on_reg);
		max_slots = core_num_wrps;
	}

	for (i = 0; i < max_slots; ++i) {
		slot = &slots[i];

		if (!*slot) {
			*slot = bp;
			break;
		}
	}

	if (i == max_slots) {
		pr_warn("Can't find any breakpoint slot\n");
		return -EBUSY;
	}

	/* Override the breakpoint data with the step data. */
	if (info->step_ctrl.enabled) {
		addr = info->trigger & ~0x3;
		ctrl = encode_ctrl_reg(info->step_ctrl);
		if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) {
			i = 0;
			ctrl_base = ARM_BASE_BCR + core_num_brps;
			val_base = ARM_BASE_BVR + core_num_brps;
		}
	}

	/* Setup the address register. */
	write_wb_reg(val_base + i, addr);

	/* Setup the control register. */
	write_wb_reg(ctrl_base + i, ctrl);
	return 0;
}

void arch_uninstall_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	struct perf_event **slot, **slots;
	int i, max_slots, base;

	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
		/* Breakpoint */
		base = ARM_BASE_BCR;
		slots = this_cpu_ptr(bp_on_reg);
		max_slots = core_num_brps;
	} else {
		/* Watchpoint */
		base = ARM_BASE_WCR;
		slots = this_cpu_ptr(wp_on_reg);
		max_slots = core_num_wrps;
	}

	/* Remove the breakpoint. */
	for (i = 0; i < max_slots; ++i) {
		slot = &slots[i];

		if (*slot == bp) {
			*slot = NULL;
			break;
		}
	}

	if (i == max_slots) {
		pr_warn("Can't find any breakpoint slot\n");
		return;
	}

	/* Ensure that we disable the mismatch breakpoint. */
	if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE &&
	    info->step_ctrl.enabled) {
		i = 0;
		base = ARM_BASE_BCR + core_num_brps;
	}

	/* Reset the control register. */
	write_wb_reg(base + i, 0);
}

static int get_hbp_len(u8 hbp_len)
{
	unsigned int len_in_bytes = 0;

	switch (hbp_len) {
	case ARM_BREAKPOINT_LEN_1:
		len_in_bytes = 1;
		break;
	case ARM_BREAKPOINT_LEN_2:
		len_in_bytes = 2;
		break;
	case ARM_BREAKPOINT_LEN_4:
		len_in_bytes = 4;
		break;
	case ARM_BREAKPOINT_LEN_8:
		len_in_bytes = 8;
		break;
	}

	return len_in_bytes;
}

/*
 * Check whether bp virtual address is in kernel space.
 */
int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
{
	unsigned int len;
	unsigned long va;

	va = hw->address;
	len = get_hbp_len(hw->ctrl.len);

	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
}

/*
 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
 * Hopefully this will disappear when ptrace can bypass the conversion
 * to generic breakpoint descriptions.
 */
int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
			   int *gen_len, int *gen_type)
{
	/* Type */
	switch (ctrl.type) {
	case ARM_BREAKPOINT_EXECUTE:
		*gen_type = HW_BREAKPOINT_X;
		break;
	case ARM_BREAKPOINT_LOAD:
		*gen_type = HW_BREAKPOINT_R;
		break;
	case ARM_BREAKPOINT_STORE:
		*gen_type = HW_BREAKPOINT_W;
		break;
	case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
		*gen_type = HW_BREAKPOINT_RW;
		break;
	default:
		return -EINVAL;
	}

	/* Len */
	switch (ctrl.len) {
	case ARM_BREAKPOINT_LEN_1:
		*gen_len = HW_BREAKPOINT_LEN_1;
		break;
	case ARM_BREAKPOINT_LEN_2:
		*gen_len = HW_BREAKPOINT_LEN_2;
		break;
	case ARM_BREAKPOINT_LEN_4:
		*gen_len = HW_BREAKPOINT_LEN_4;
		break;
	case ARM_BREAKPOINT_LEN_8:
		*gen_len = HW_BREAKPOINT_LEN_8;
		break;
	default:
		return -EINVAL;
	}

	return 0;
}

/*
 * Construct an arch_hw_breakpoint from a perf_event.
 */
static int arch_build_bp_info(struct perf_event *bp,
			      const struct perf_event_attr *attr,
			      struct arch_hw_breakpoint *hw)
{
	/* Type */
	switch (attr->bp_type) {
	case HW_BREAKPOINT_X:
		hw->ctrl.type = ARM_BREAKPOINT_EXECUTE;
		break;
	case HW_BREAKPOINT_R:
		hw->ctrl.type = ARM_BREAKPOINT_LOAD;
		break;
	case HW_BREAKPOINT_W:
		hw->ctrl.type = ARM_BREAKPOINT_STORE;
		break;
	case HW_BREAKPOINT_RW:
		hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
		break;
	default:
		return -EINVAL;
	}

	/* Len */
	switch (attr->bp_len) {
	case HW_BREAKPOINT_LEN_1:
		hw->ctrl.len = ARM_BREAKPOINT_LEN_1;
		break;
	case HW_BREAKPOINT_LEN_2:
		hw->ctrl.len = ARM_BREAKPOINT_LEN_2;
		break;
	case HW_BREAKPOINT_LEN_4:
		hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
		break;
	case HW_BREAKPOINT_LEN_8:
		hw->ctrl.len = ARM_BREAKPOINT_LEN_8;
		if ((hw->ctrl.type != ARM_BREAKPOINT_EXECUTE)
			&& max_watchpoint_len >= 8)
			break;
		fallthrough;
	default:
		return -EINVAL;
	}

	/*
	 * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes.
	 * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported
	 * by the hardware and must be aligned to the appropriate number of
	 * bytes.
	 */
	if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE &&
	    hw->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
	    hw->ctrl.len != ARM_BREAKPOINT_LEN_4)
		return -EINVAL;

	/* Address */
	hw->address = attr->bp_addr;

	/* Privilege */
	hw->ctrl.privilege = ARM_BREAKPOINT_USER;
	if (arch_check_bp_in_kernelspace(hw))
		hw->ctrl.privilege |= ARM_BREAKPOINT_PRIV;

	/* Enabled? */
	hw->ctrl.enabled = !attr->disabled;

	/* Mismatch */
	hw->ctrl.mismatch = 0;

	return 0;
}

/*
 * Validate the arch-specific HW Breakpoint register settings.
 */
int hw_breakpoint_arch_parse(struct perf_event *bp,
			     const struct perf_event_attr *attr,
			     struct arch_hw_breakpoint *hw)
{
	int ret = 0;
	u32 offset, alignment_mask = 0x3;

	/* Ensure that we are in monitor debug mode. */
	if (!monitor_mode_enabled())
		return -ENODEV;

	/* Build the arch_hw_breakpoint. */
	ret = arch_build_bp_info(bp, attr, hw);
	if (ret)
		goto out;

	/* Check address alignment. */
	if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8)
		alignment_mask = 0x7;
	offset = hw->address & alignment_mask;
	switch (offset) {
	case 0:
		/* Aligned */
		break;
	case 1:
	case 2:
		/* Allow halfword watchpoints and breakpoints. */
		if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2)
			break;
		fallthrough;
	case 3:
		/* Allow single byte watchpoint. */
		if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1)
			break;
		fallthrough;
	default:
		ret = -EINVAL;
		goto out;
	}

	hw->address &= ~alignment_mask;
	hw->ctrl.len <<= offset;

	if (is_default_overflow_handler(bp)) {
		/*
		 * Mismatch breakpoints are required for single-stepping
		 * breakpoints.
		 */
		if (!core_has_mismatch_brps())
			return -EINVAL;

		/* We don't allow mismatch breakpoints in kernel space. */
		if (arch_check_bp_in_kernelspace(hw))
			return -EPERM;

		/*
		 * Per-cpu breakpoints are not supported by our stepping
		 * mechanism.
		 */
		if (!bp->hw.target)
			return -EINVAL;

		/*
		 * We only support specific access types if the fsr
		 * reports them.
		 */
		if (!debug_exception_updates_fsr() &&
		    (hw->ctrl.type == ARM_BREAKPOINT_LOAD ||
		     hw->ctrl.type == ARM_BREAKPOINT_STORE))
			return -EINVAL;
	}

out:
	return ret;
}

/*
 * Enable/disable single-stepping over the breakpoint bp at address addr.
 */
static void enable_single_step(struct perf_event *bp, u32 addr)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);

	arch_uninstall_hw_breakpoint(bp);
	info->step_ctrl.mismatch  = 1;
	info->step_ctrl.len	  = ARM_BREAKPOINT_LEN_4;
	info->step_ctrl.type	  = ARM_BREAKPOINT_EXECUTE;
	info->step_ctrl.privilege = info->ctrl.privilege;
	info->step_ctrl.enabled	  = 1;
	info->trigger		  = addr;
	arch_install_hw_breakpoint(bp);
}

static void disable_single_step(struct perf_event *bp)
{
	arch_uninstall_hw_breakpoint(bp);
	counter_arch_bp(bp)->step_ctrl.enabled = 0;
	arch_install_hw_breakpoint(bp);
}

/*
 * Arm32 hardware does not always report a watchpoint hit address that matches
 * one of the watchpoints set. It can also report an address "near" the
 * watchpoint if a single instruction access both watched and unwatched
 * addresses. There is no straight-forward way, short of disassembling the
 * offending instruction, to map that address back to the watchpoint. This
 * function computes the distance of the memory access from the watchpoint as a
 * heuristic for the likelyhood that a given access triggered the watchpoint.
 *
 * See this same function in the arm64 platform code, which has the same
 * problem.
 *
 * The function returns the distance of the address from the bytes watched by
 * the watchpoint. In case of an exact match, it returns 0.
 */
static u32 get_distance_from_watchpoint(unsigned long addr, u32 val,
					struct arch_hw_breakpoint_ctrl *ctrl)
{
	u32 wp_low, wp_high;
	u32 lens, lene;

	lens = __ffs(ctrl->len);
	lene = __fls(ctrl->len);

	wp_low = val + lens;
	wp_high = val + lene;
	if (addr < wp_low)
		return wp_low - addr;
	else if (addr > wp_high)
		return addr - wp_high;
	else
		return 0;
}

static int watchpoint_fault_on_uaccess(struct pt_regs *regs,
				       struct arch_hw_breakpoint *info)
{
	return !user_mode(regs) && info->ctrl.privilege == ARM_BREAKPOINT_USER;
}

static void watchpoint_handler(unsigned long addr, unsigned int fsr,
			       struct pt_regs *regs)
{
	int i, access, closest_match = 0;
	u32 min_dist = -1, dist;
	u32 val, ctrl_reg;
	struct perf_event *wp, **slots;
	struct arch_hw_breakpoint *info;
	struct arch_hw_breakpoint_ctrl ctrl;

	slots = this_cpu_ptr(wp_on_reg);

	/*
	 * Find all watchpoints that match the reported address. If no exact
	 * match is found. Attribute the hit to the closest watchpoint.
	 */
	rcu_read_lock();
	for (i = 0; i < core_num_wrps; ++i) {
		wp = slots[i];
		if (wp == NULL)
			continue;

		/*
		 * The DFAR is an unknown value on debug architectures prior
		 * to 7.1. Since we only allow a single watchpoint on these
		 * older CPUs, we can set the trigger to the lowest possible
		 * faulting address.
		 */
		if (debug_arch < ARM_DEBUG_ARCH_V7_1) {
			BUG_ON(i > 0);
			info = counter_arch_bp(wp);
			info->trigger = wp->attr.bp_addr;
		} else {
			/* Check that the access type matches. */
			if (debug_exception_updates_fsr()) {
				access = (fsr & ARM_FSR_ACCESS_MASK) ?
					  HW_BREAKPOINT_W : HW_BREAKPOINT_R;
				if (!(access & hw_breakpoint_type(wp)))
					continue;
			}

			val = read_wb_reg(ARM_BASE_WVR + i);
			ctrl_reg = read_wb_reg(ARM_BASE_WCR + i);
			decode_ctrl_reg(ctrl_reg, &ctrl);
			dist = get_distance_from_watchpoint(addr, val, &ctrl);
			if (dist < min_dist) {
				min_dist = dist;
				closest_match = i;
			}
			/* Is this an exact match? */
			if (dist != 0)
				continue;

			/* We have a winner. */
			info = counter_arch_bp(wp);
			info->trigger = addr;
		}

		pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);

		/*
		 * If we triggered a user watchpoint from a uaccess routine,
		 * then handle the stepping ourselves since userspace really
		 * can't help us with this.
		 */
		if (watchpoint_fault_on_uaccess(regs, info))
			goto step;

		perf_bp_event(wp, regs);

		/*
		 * Defer stepping to the overflow handler if one is installed.
		 * Otherwise, insert a temporary mismatch breakpoint so that
		 * we can single-step over the watchpoint trigger.
		 */
		if (!is_default_overflow_handler(wp))
			continue;
step:
		enable_single_step(wp, instruction_pointer(regs));
	}

	if (min_dist > 0 && min_dist != -1) {
		/* No exact match found. */
		wp = slots[closest_match];
		info = counter_arch_bp(wp);
		info->trigger = addr;
		pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
		perf_bp_event(wp, regs);
		if (is_default_overflow_handler(wp))
			enable_single_step(wp, instruction_pointer(regs));
	}

	rcu_read_unlock();
}

static void watchpoint_single_step_handler(unsigned long pc)
{
	int i;
	struct perf_event *wp, **slots;
	struct arch_hw_breakpoint *info;

	slots = this_cpu_ptr(wp_on_reg);

	for (i = 0; i < core_num_wrps; ++i) {
		rcu_read_lock();

		wp = slots[i];

		if (wp == NULL)
			goto unlock;

		info = counter_arch_bp(wp);
		if (!info->step_ctrl.enabled)
			goto unlock;

		/*
		 * Restore the original watchpoint if we've completed the
		 * single-step.
		 */
		if (info->trigger != pc)
			disable_single_step(wp);

unlock:
		rcu_read_unlock();
	}
}

static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs)
{
	int i;
	u32 ctrl_reg, val, addr;
	struct perf_event *bp, **slots;
	struct arch_hw_breakpoint *info;
	struct arch_hw_breakpoint_ctrl ctrl;

	slots = this_cpu_ptr(bp_on_reg);

	/* The exception entry code places the amended lr in the PC. */
	addr = regs->ARM_pc;

	/* Check the currently installed breakpoints first. */
	for (i = 0; i < core_num_brps; ++i) {
		rcu_read_lock();

		bp = slots[i];

		if (bp == NULL)
			goto unlock;

		info = counter_arch_bp(bp);

		/* Check if the breakpoint value matches. */
		val = read_wb_reg(ARM_BASE_BVR + i);
		if (val != (addr & ~0x3))
			goto mismatch;

		/* Possible match, check the byte address select to confirm. */
		ctrl_reg = read_wb_reg(ARM_BASE_BCR + i);
		decode_ctrl_reg(ctrl_reg, &ctrl);
		if ((1 << (addr & 0x3)) & ctrl.len) {
			info->trigger = addr;
			pr_debug("breakpoint fired: address = 0x%x\n", addr);
			perf_bp_event(bp, regs);
			if (is_default_overflow_handler(bp))
				enable_single_step(bp, addr);
			goto unlock;
		}

mismatch:
		/* If we're stepping a breakpoint, it can now be restored. */
		if (info->step_ctrl.enabled)
			disable_single_step(bp);
unlock:
		rcu_read_unlock();
	}

	/* Handle any pending watchpoint single-step breakpoints. */
	watchpoint_single_step_handler(addr);
}

/*
 * Called from either the Data Abort Handler [watchpoint] or the
 * Prefetch Abort Handler [breakpoint] with interrupts disabled.
 */
static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr,
				 struct pt_regs *regs)
{
	int ret = 0;
	u32 dscr;

	preempt_disable();

	if (interrupts_enabled(regs))
		local_irq_enable();

	/* We only handle watchpoints and hardware breakpoints. */
	ARM_DBG_READ(c0, c1, 0, dscr);

	/* Perform perf callbacks. */
	switch (ARM_DSCR_MOE(dscr)) {
	case ARM_ENTRY_BREAKPOINT:
		breakpoint_handler(addr, regs);
		break;
	case ARM_ENTRY_ASYNC_WATCHPOINT:
		WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n");
		fallthrough;
	case ARM_ENTRY_SYNC_WATCHPOINT:
		watchpoint_handler(addr, fsr, regs);
		break;
	default:
		ret = 1; /* Unhandled fault. */
	}

	preempt_enable();

	return ret;
}

#ifdef CONFIG_ARM_ERRATA_764319
static int oslsr_fault;

static int debug_oslsr_trap(struct pt_regs *regs, unsigned int instr)
{
	oslsr_fault = 1;
	instruction_pointer(regs) += 4;
	return 0;
}

static struct undef_hook debug_oslsr_hook = {
	.instr_mask  = 0xffffffff,
	.instr_val = 0xee115e91,
	.fn = debug_oslsr_trap,
};
#endif

/*
 * One-time initialisation.
 */
static cpumask_t debug_err_mask;

static int debug_reg_trap(struct pt_regs *regs, unsigned int instr)
{
	int cpu = smp_processor_id();

	pr_warn("Debug register access (0x%x) caused undefined instruction on CPU %d\n",
		instr, cpu);

	/* Set the error flag for this CPU and skip the faulting instruction. */
	cpumask_set_cpu(cpu, &debug_err_mask);
	instruction_pointer(regs) += 4;
	return 0;
}

static struct undef_hook debug_reg_hook = {
	.instr_mask	= 0x0fe80f10,
	.instr_val	= 0x0e000e10,
	.fn		= debug_reg_trap,
};

/* Does this core support OS Save and Restore? */
static bool core_has_os_save_restore(void)
{
	u32 oslsr;

	switch (get_debug_arch()) {
	case ARM_DEBUG_ARCH_V7_1:
		return true;
	case ARM_DEBUG_ARCH_V7_ECP14:
#ifdef CONFIG_ARM_ERRATA_764319
		oslsr_fault = 0;
		register_undef_hook(&debug_oslsr_hook);
		ARM_DBG_READ(c1, c1, 4, oslsr);
		unregister_undef_hook(&debug_oslsr_hook);
		if (oslsr_fault)
			return false;
#else
		ARM_DBG_READ(c1, c1, 4, oslsr);
#endif
		if (oslsr & ARM_OSLSR_OSLM0)
			return true;
		fallthrough;
	default:
		return false;
	}
}

static void reset_ctrl_regs(unsigned int cpu)
{
	int i, raw_num_brps, err = 0;
	u32 val;

	/*
	 * v7 debug contains save and restore registers so that debug state
	 * can be maintained across low-power modes without leaving the debug
	 * logic powered up. It is IMPLEMENTATION DEFINED whether we can access
	 * the debug registers out of reset, so we must unlock the OS Lock
	 * Access Register to avoid taking undefined instruction exceptions
	 * later on.
	 */
	switch (debug_arch) {
	case ARM_DEBUG_ARCH_V6:
	case ARM_DEBUG_ARCH_V6_1:
		/* ARMv6 cores clear the registers out of reset. */
		goto out_mdbgen;
	case ARM_DEBUG_ARCH_V7_ECP14:
		/*
		 * Ensure sticky power-down is clear (i.e. debug logic is
		 * powered up).
		 */
		ARM_DBG_READ(c1, c5, 4, val);
		if ((val & 0x1) == 0)
			err = -EPERM;

		if (!has_ossr)
			goto clear_vcr;
		break;
	case ARM_DEBUG_ARCH_V7_1:
		/*
		 * Ensure the OS double lock is clear.
		 */
		ARM_DBG_READ(c1, c3, 4, val);
		if ((val & 0x1) == 1)
			err = -EPERM;
		break;
	}

	if (err) {
		pr_warn_once("CPU %d debug is powered down!\n", cpu);
		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
		return;
	}

	/*
	 * Unconditionally clear the OS lock by writing a value
	 * other than CS_LAR_KEY to the access register.
	 */
	ARM_DBG_WRITE(c1, c0, 4, ~CORESIGHT_UNLOCK);
	isb();

	/*
	 * Clear any configured vector-catch events before
	 * enabling monitor mode.
	 */
clear_vcr:
	ARM_DBG_WRITE(c0, c7, 0, 0);
	isb();

	if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
		pr_warn_once("CPU %d failed to disable vector catch\n", cpu);
		return;
	}

	/*
	 * The control/value register pairs are UNKNOWN out of reset so
	 * clear them to avoid spurious debug events.
	 */
	raw_num_brps = get_num_brp_resources();
	for (i = 0; i < raw_num_brps; ++i) {
		write_wb_reg(ARM_BASE_BCR + i, 0UL);
		write_wb_reg(ARM_BASE_BVR + i, 0UL);
	}

	for (i = 0; i < core_num_wrps; ++i) {
		write_wb_reg(ARM_BASE_WCR + i, 0UL);
		write_wb_reg(ARM_BASE_WVR + i, 0UL);
	}

	if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
		pr_warn_once("CPU %d failed to clear debug register pairs\n", cpu);
		return;
	}

	/*
	 * Have a crack at enabling monitor mode. We don't actually need
	 * it yet, but reporting an error early is useful if it fails.
	 */
out_mdbgen:
	if (enable_monitor_mode())
		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
}

static int dbg_reset_online(unsigned int cpu)
{
	local_irq_disable();
	reset_ctrl_regs(cpu);
	local_irq_enable();
	return 0;
}

#ifdef CONFIG_CPU_PM
static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action,
			     void *v)
{
	if (action == CPU_PM_EXIT)
		reset_ctrl_regs(smp_processor_id());

	return NOTIFY_OK;
}

static struct notifier_block dbg_cpu_pm_nb = {
	.notifier_call = dbg_cpu_pm_notify,
};

static void __init pm_init(void)
{
	cpu_pm_register_notifier(&dbg_cpu_pm_nb);
}
#else
static inline void pm_init(void)
{
}
#endif

static int __init arch_hw_breakpoint_init(void)
{
	int ret;

	debug_arch = get_debug_arch();

	if (!debug_arch_supported()) {
		pr_info("debug architecture 0x%x unsupported.\n", debug_arch);
		return 0;
	}

	/*
	 * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD
	 * whenever a WFI is issued, even if the core is not powered down, in
	 * violation of the architecture.  When DBGPRSR.SPD is set, accesses to
	 * breakpoint and watchpoint registers are treated as undefined, so
	 * this results in boot time and runtime failures when these are
	 * accessed and we unexpectedly take a trap.
	 *
	 * It's not clear if/how this can be worked around, so we blacklist
	 * Scorpion CPUs to avoid these issues.
	*/
	if (read_cpuid_part() == ARM_CPU_PART_SCORPION) {
		pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n");
		return 0;
	}

	has_ossr = core_has_os_save_restore();

	/* Determine how many BRPs/WRPs are available. */
	core_num_brps = get_num_brps();
	core_num_wrps = get_num_wrps();

	/*
	 * We need to tread carefully here because DBGSWENABLE may be
	 * driven low on this core and there isn't an architected way to
	 * determine that.
	 */
	cpus_read_lock();
	register_undef_hook(&debug_reg_hook);

	/*
	 * Register CPU notifier which resets the breakpoint resources. We
	 * assume that a halting debugger will leave the world in a nice state
	 * for us.
	 */
	ret = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN,
					   "arm/hw_breakpoint:online",
					   dbg_reset_online, NULL);
	unregister_undef_hook(&debug_reg_hook);
	if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) {
		core_num_brps = 0;
		core_num_wrps = 0;
		if (ret > 0)
			cpuhp_remove_state_nocalls_cpuslocked(ret);
		cpus_read_unlock();
		return 0;
	}

	pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n",
		core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " :
		"", core_num_wrps);

	/* Work out the maximum supported watchpoint length. */
	max_watchpoint_len = get_max_wp_len();
	pr_info("maximum watchpoint size is %u bytes.\n",
			max_watchpoint_len);

	/* Register debug fault handler. */
	hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
			TRAP_HWBKPT, "watchpoint debug exception");
	hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
			TRAP_HWBKPT, "breakpoint debug exception");
	cpus_read_unlock();

	/* Register PM notifiers. */
	pm_init();
	return 0;
}
arch_initcall(arch_hw_breakpoint_init);

void hw_breakpoint_pmu_read(struct perf_event *bp)
{
}

/*
 * Dummy function to register with die_notifier.
 */
int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
					unsigned long val, void *data)
{
	return NOTIFY_DONE;
}


code:
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Copyright (c) 2012 Linaro Limited.
 */

#include <linux/init.h>
#include <linux/irqchip/arm-gic-v3.h>
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/virt.h>

.arch armv7-a

#ifndef ZIMAGE
/*
 * For the kernel proper, we need to find out the CPU boot mode long after
 * boot, so we need to store it in a writable variable.
 *
 * This is not in .bss, because we set it sufficiently early that the boot-time
 * zeroing of .bss would clobber it.
 */
.data
	.align	2
ENTRY(__boot_cpu_mode)
	.long	0
.text

	/*
	 * Save the primary CPU boot mode. Requires 2 scratch registers.
	 */
	.macro	store_primary_cpu_mode	reg1, reg2
	mrs	\reg1, cpsr
	and	\reg1, \reg1, #MODE_MASK
	str_l	\reg1, __boot_cpu_mode, \reg2
	.endm

	/*
	 * Compare the current mode with the one saved on the primary CPU.
	 * If they don't match, record that fact. The Z bit indicates
	 * if there's a match or not.
	 * Requires 2 additional scratch registers.
	 */
	.macro	compare_cpu_mode_with_primary mode, reg1, reg2
	adr_l	\reg2, __boot_cpu_mode
	ldr	\reg1, [\reg2]
	cmp	\mode, \reg1		@ matches primary CPU boot mode?
	orrne	\reg1, \reg1, #BOOT_CPU_MODE_MISMATCH
	strne	\reg1, [\reg2]		@ record what happened and give up
	.endm

#else	/* ZIMAGE */

	.macro	store_primary_cpu_mode	reg1:req, reg2:req
	.endm

/*
 * The zImage loader only runs on one CPU, so we don't bother with mult-CPU
 * consistency checking:
 */
	.macro	compare_cpu_mode_with_primary mode, reg1, reg2
	cmp	\mode, \mode
	.endm

#endif /* ZIMAGE */

/*
 * Hypervisor stub installation functions.
 *
 * These must be called with the MMU and D-cache off.
 * They are not ABI compliant and are only intended to be called from the kernel
 * entry points in head.S.
 */
@ Call this from the primary CPU
ENTRY(__hyp_stub_install)
	store_primary_cpu_mode	r4, r5
ENDPROC(__hyp_stub_install)

	@ fall through...

@ Secondary CPUs should call here
ENTRY(__hyp_stub_install_secondary)
	mrs	r4, cpsr
	and	r4, r4, #MODE_MASK

	/*
	 * If the secondary has booted with a different mode, give up
	 * immediately.
	 */
	compare_cpu_mode_with_primary	r4, r5, r6
	retne	lr

	/*
	 * Once we have given up on one CPU, we do not try to install the
	 * stub hypervisor on the remaining ones: because the saved boot mode
	 * is modified, it can't compare equal to the CPSR mode field any
	 * more.
	 *
	 * Otherwise...
	 */

	cmp	r4, #HYP_MODE
	retne	lr			@ give up if the CPU is not in HYP mode

/*
 * Configure HSCTLR to set correct exception endianness/instruction set
 * state etc.
 * Turn off all traps
 * Eventually, CPU-specific code might be needed -- assume not for now
 *
 * This code relies on the "eret" instruction to synchronize the
 * various coprocessor accesses. This is done when we switch to SVC
 * (see safe_svcmode_maskall).
 */
	@ Now install the hypervisor stub:
	W(adr)	r7, __hyp_stub_vectors
	mcr	p15, 4, r7, c12, c0, 0	@ set hypervisor vector base (HVBAR)

	@ Disable all traps, so we don't get any nasty surprise
	mov	r7, #0
	mcr	p15, 4, r7, c1, c1, 0	@ HCR
	mcr	p15, 4, r7, c1, c1, 2	@ HCPTR
	mcr	p15, 4, r7, c1, c1, 3	@ HSTR

THUMB(	orr	r7, #(1 << 30)	)	@ HSCTLR.TE
ARM_BE8(orr	r7, r7, #(1 << 25))     @ HSCTLR.EE
	mcr	p15, 4, r7, c1, c0, 0	@ HSCTLR

	mrc	p15, 4, r7, c1, c1, 1	@ HDCR
	and	r7, #0x1f		@ Preserve HPMN
	mcr	p15, 4, r7, c1, c1, 1	@ HDCR

	@ Make sure NS-SVC is initialised appropriately
	mrc	p15, 0, r7, c1, c0, 0	@ SCTLR
	orr	r7, #(1 << 5)		@ CP15 barriers enabled
	bic	r7, #(3 << 7)		@ Clear SED/ITD for v8 (RES0 for v7)
	bic	r7, #(3 << 19)		@ WXN and UWXN disabled
	mcr	p15, 0, r7, c1, c0, 0	@ SCTLR

	mrc	p15, 0, r7, c0, c0, 0	@ MIDR
	mcr	p15, 4, r7, c0, c0, 0	@ VPIDR

	mrc	p15, 0, r7, c0, c0, 5	@ MPIDR
	mcr	p15, 4, r7, c0, c0, 5	@ VMPIDR

#if !defined(ZIMAGE) && defined(CONFIG_ARM_ARCH_TIMER)
	@ make CNTP_* and CNTPCT accessible from PL1
	mrc	p15, 0, r7, c0, c1, 1	@ ID_PFR1
	ubfx	r7, r7, #16, #4
	teq	r7, #0
	beq	1f
	mrc	p15, 4, r7, c14, c1, 0	@ CNTHCTL
	orr	r7, r7, #3		@ PL1PCEN | PL1PCTEN
	mcr	p15, 4, r7, c14, c1, 0	@ CNTHCTL
	mov	r7, #0
	mcrr	p15, 4, r7, r7, c14	@ CNTVOFF

	@ Disable virtual timer in case it was counting
	mrc	p15, 0, r7, c14, c3, 1	@ CNTV_CTL
	bic	r7, #1			@ Clear ENABLE
	mcr	p15, 0, r7, c14, c3, 1	@ CNTV_CTL
1:
#endif

#ifdef CONFIG_ARM_GIC_V3
	@ Check whether GICv3 system registers are available
	mrc	p15, 0, r7, c0, c1, 1	@ ID_PFR1
	ubfx	r7, r7, #28, #4
	teq	r7, #0
	beq	2f

	@ Enable system register accesses
	mrc	p15, 4, r7, c12, c9, 5	@ ICC_HSRE
	orr	r7, r7, #(ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE)
	mcr	p15, 4, r7, c12, c9, 5	@ ICC_HSRE
	isb

	@ SRE bit could be forced to 0 by firmware.
	@ Check whether it sticks before accessing any other sysreg
	mrc	p15, 4, r7, c12, c9, 5	@ ICC_HSRE
	tst	r7, #ICC_SRE_EL2_SRE
	beq	2f
	mov	r7, #0
	mcr	p15, 4, r7, c12, c11, 0	@ ICH_HCR
2:
#endif

	bx	lr			@ The boot CPU mode is left in r4.
ENDPROC(__hyp_stub_install_secondary)

__hyp_stub_do_trap:
#ifdef ZIMAGE
	teq	r0, #HVC_SET_VECTORS
	bne	1f
	/* Only the ZIMAGE stubs can change the HYP vectors */
	mcr	p15, 4, r1, c12, c0, 0	@ set HVBAR
	b	__hyp_stub_exit
#endif

1:	teq	r0, #HVC_SOFT_RESTART
	bne	2f
	bx	r1

2:	ldr	r0, =HVC_STUB_ERR
	__ERET

__hyp_stub_exit:
	mov	r0, #0
	__ERET
ENDPROC(__hyp_stub_do_trap)

/*
 * __hyp_set_vectors is only used when ZIMAGE must bounce between HYP
 * and SVC. For the kernel itself, the vectors are set once and for
 * all by the stubs.
 */
ENTRY(__hyp_set_vectors)
	mov	r1, r0
	mov	r0, #HVC_SET_VECTORS
	__HVC(0)
	ret	lr
ENDPROC(__hyp_set_vectors)

ENTRY(__hyp_soft_restart)
	mov	r1, r0
	mov	r0, #HVC_SOFT_RESTART
	__HVC(0)
	ret	lr
ENDPROC(__hyp_soft_restart)

.align 5
ENTRY(__hyp_stub_vectors)
__hyp_stub_reset:	W(b)	.
__hyp_stub_und:		W(b)	.
__hyp_stub_svc:		W(b)	.
__hyp_stub_pabort:	W(b)	.
__hyp_stub_dabort:	W(b)	.
__hyp_stub_trap:	W(b)	__hyp_stub_do_trap
__hyp_stub_irq:		W(b)	.
__hyp_stub_fiq:		W(b)	.
ENDPROC(__hyp_stub_vectors)

repo_name: angery/linux


code:
// SPDX-License-Identifier: GPL-2.0
#include <linux/bug.h>
#include <linux/kernel.h>
#include <asm/opcodes.h>

static unsigned long __arm_gen_branch_thumb2(unsigned long pc,
					     unsigned long addr, bool link,
					     bool warn)
{
	unsigned long s, j1, j2, i1, i2, imm10, imm11;
	unsigned long first, second;
	long offset;

	offset = (long)addr - (long)(pc + 4);
	if (offset < -16777216 || offset > 16777214) {
		WARN_ON_ONCE(warn);
		return 0;
	}

	s	= (offset >> 24) & 0x1;
	i1	= (offset >> 23) & 0x1;
	i2	= (offset >> 22) & 0x1;
	imm10	= (offset >> 12) & 0x3ff;
	imm11	= (offset >>  1) & 0x7ff;

	j1 = (!i1) ^ s;
	j2 = (!i2) ^ s;

	first = 0xf000 | (s << 10) | imm10;
	second = 0x9000 | (j1 << 13) | (j2 << 11) | imm11;
	if (link)
		second |= 1 << 14;

	return __opcode_thumb32_compose(first, second);
}

static unsigned long __arm_gen_branch_arm(unsigned long pc, unsigned long addr,
					  bool link, bool warn)
{
	unsigned long opcode = 0xea000000;
	long offset;

	if (link)
		opcode |= 1 << 24;

	offset = (long)addr - (long)(pc + 8);
	if (unlikely(offset < -33554432 || offset > 33554428)) {
		WARN_ON_ONCE(warn);
		return 0;
	}

	offset = (offset >> 2) & 0x00ffffff;

	return opcode | offset;
}

unsigned long
__arm_gen_branch(unsigned long pc, unsigned long addr, bool link, bool warn)
{
	if (IS_ENABLED(CONFIG_THUMB2_KERNEL))
		return __arm_gen_branch_thumb2(pc, addr, link, warn);
	else
		return __arm_gen_branch_arm(pc, addr, link, warn);
}


code:
// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/types.h>
#include <linux/io.h>
#include <linux/spinlock.h>

static DEFINE_RAW_SPINLOCK(__io_lock);

/*
 * Generic atomic MMIO modify.
 *
 * Allows thread-safe access to registers shared by unrelated subsystems.
 * The access is protected by a single MMIO-wide lock.
 */
void atomic_io_modify_relaxed(void __iomem *reg, u32 mask, u32 set)
{
	unsigned long flags;
	u32 value;

	raw_spin_lock_irqsave(&__io_lock, flags);
	value = readl_relaxed(reg) & ~mask;
	value |= (set & mask);
	writel_relaxed(value, reg);
	raw_spin_unlock_irqrestore(&__io_lock, flags);
}
EXPORT_SYMBOL(atomic_io_modify_relaxed);

void atomic_io_modify(void __iomem *reg, u32 mask, u32 set)
{
	unsigned long flags;
	u32 value;

	raw_spin_lock_irqsave(&__io_lock, flags);
	value = readl_relaxed(reg) & ~mask;
	value |= (set & mask);
	writel(value, reg);
	raw_spin_unlock_irqrestore(&__io_lock, flags);
}
EXPORT_SYMBOL(atomic_io_modify);

/*
 * Copy data from IO memory space to "real" memory space.
 * This needs to be optimized.
 */
void _memcpy_fromio(void *to, const volatile void __iomem *from, size_t count)
{
	unsigned char *t = to;
	while (count) {
		count--;
		*t = readb(from);
		t++;
		from++;
	}
}
EXPORT_SYMBOL(_memcpy_fromio);

/*
 * Copy data from "real" memory space to IO memory space.
 * This needs to be optimized.
 */
void _memcpy_toio(volatile void __iomem *to, const void *from, size_t count)
{
	const unsigned char *f = from;
	while (count) {
		count--;
		writeb(*f, to);
		f++;
		to++;
	}
}
EXPORT_SYMBOL(_memcpy_toio);

/*
 * "memset" on IO memory space.
 * This needs to be optimized.
 */
void _memset_io(volatile void __iomem *dst, int c, size_t count)
{
	while (count) {
		count--;
		writeb(c, dst);
		dst++;
	}
}
EXPORT_SYMBOL(_memset_io);


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/arch/arm/kernel/irq.c
 *
 *  Copyright (C) 1992 Linus Torvalds
 *  Modifications for ARM processor Copyright (C) 1995-2000 Russell King.
 *
 *  Support for Dynamic Tick Timer Copyright (C) 2004-2005 Nokia Corporation.
 *  Dynamic Tick Timer written by Tony Lindgren <tony@atomide.com> and
 *  Tuukka Tikkanen <tuukka.tikkanen@elektrobit.com>.
 *
 *  This file contains the code used by various IRQ handling routines:
 *  asking for different IRQ's should be done through these routines
 *  instead of just grabbing them. Thus setups with different IRQ numbers
 *  shouldn't result in any weird surprises, and installing new handlers
 *  should be easier.
 *
 *  IRQ's are in fact implemented a bit like signal handlers for the kernel.
 *  Naturally it's not a 1:1 relation, but there are similarities.
 */
#include <linux/signal.h>
#include <linux/ioport.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irqchip.h>
#include <linux/random.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/seq_file.h>
#include <linux/errno.h>
#include <linux/list.h>
#include <linux/kallsyms.h>
#include <linux/proc_fs.h>
#include <linux/export.h>

#include <asm/hardware/cache-l2x0.h>
#include <asm/hardware/cache-uniphier.h>
#include <asm/outercache.h>
#include <asm/softirq_stack.h>
#include <asm/exception.h>
#include <asm/mach/arch.h>
#include <asm/mach/irq.h>
#include <asm/mach/time.h>

#include "reboot.h"

unsigned long irq_err_count;

#ifdef CONFIG_IRQSTACKS

asmlinkage DEFINE_PER_CPU_READ_MOSTLY(u8 *, irq_stack_ptr);

static void __init init_irq_stacks(void)
{
	u8 *stack;
	int cpu;

	for_each_possible_cpu(cpu) {
		if (!IS_ENABLED(CONFIG_VMAP_STACK))
			stack = (u8 *)__get_free_pages(GFP_KERNEL,
						       THREAD_SIZE_ORDER);
		else
			stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,
					       THREADINFO_GFP, NUMA_NO_NODE,
					       __builtin_return_address(0));

		if (WARN_ON(!stack))
			break;
		per_cpu(irq_stack_ptr, cpu) = &stack[THREAD_SIZE];
	}
}

#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
static void ____do_softirq(void *arg)
{
	__do_softirq();
}

void do_softirq_own_stack(void)
{
	call_with_stack(____do_softirq, NULL,
			__this_cpu_read(irq_stack_ptr));
}
#endif
#endif

int arch_show_interrupts(struct seq_file *p, int prec)
{
#ifdef CONFIG_FIQ
	show_fiq_list(p, prec);
#endif
#ifdef CONFIG_SMP
	show_ipi_list(p, prec);
#endif
	seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
	return 0;
}

/*
 * handle_IRQ handles all hardware IRQ's.  Decoded IRQs should
 * not come via this function.  Instead, they should provide their
 * own 'handler'.  Used by platform code implementing C-based 1st
 * level decoding.
 */
void handle_IRQ(unsigned int irq, struct pt_regs *regs)
{
	struct irq_desc *desc;

	/*
	 * Some hardware gives randomly wrong interrupts.  Rather
	 * than crashing, do something sensible.
	 */
	if (unlikely(!irq || irq >= nr_irqs))
		desc = NULL;
	else
		desc = irq_to_desc(irq);

	if (likely(desc))
		handle_irq_desc(desc);
	else
		ack_bad_irq(irq);
}

void __init init_IRQ(void)
{
	int ret;

#ifdef CONFIG_IRQSTACKS
	init_irq_stacks();
#endif

	if (IS_ENABLED(CONFIG_OF) && !machine_desc->init_irq)
		irqchip_init();
	else
		machine_desc->init_irq();

	if (IS_ENABLED(CONFIG_OF) && IS_ENABLED(CONFIG_CACHE_L2X0) &&
	    (machine_desc->l2c_aux_mask || machine_desc->l2c_aux_val)) {
		if (!outer_cache.write_sec)
			outer_cache.write_sec = machine_desc->l2c_write_sec;
		ret = l2x0_of_init(machine_desc->l2c_aux_val,
				   machine_desc->l2c_aux_mask);
		if (ret && ret != -ENODEV)
			pr_err("L2C: failed to init: %d\n", ret);
	}

	uniphier_cache_init();
}

#ifdef CONFIG_SPARSE_IRQ
int __init arch_probe_nr_irqs(void)
{
	nr_irqs = machine_desc->nr_irqs ? machine_desc->nr_irqs : NR_IRQS;
	return nr_irqs;
}
#endif


code:
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  linux/arch/arm/kernel/isa.c
 *
 *  Copyright (C) 1999 Phil Blundell
 *
 *  ISA shared memory and I/O port support, and is required to support
 *  iopl, inb, outb and friends in userspace via glibc emulation.
 */
#include <linux/stddef.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/sysctl.h>
#include <linux/init.h>
#include <linux/io.h>

static unsigned int isa_membase, isa_portbase, isa_portshift;

static struct ctl_table ctl_isa_vars[4] = {
	{
		.procname	= "membase",
		.data		= &isa_membase,
		.maxlen		= sizeof(isa_membase),
		.mode		= 0444,
		.proc_handler	= proc_dointvec,
	}, {
		.procname	= "portbase",
		.data		= &isa_portbase,
		.maxlen		= sizeof(isa_portbase),
		.mode		= 0444,
		.proc_handler	= proc_dointvec,
	}, {
		.procname	= "portshift",
		.data		= &isa_portshift,
		.maxlen		= sizeof(isa_portshift),
		.mode		= 0444,
		.proc_handler	= proc_dointvec,
	}, {}
};

static struct ctl_table_header *isa_sysctl_header;

static struct ctl_table ctl_isa[2] = {
	{
		.procname	= "isa",
		.mode		= 0555,
		.child		= ctl_isa_vars,
	}, {}
};

static struct ctl_table ctl_bus[2] = {
	{
		.procname	= "bus",
		.mode		= 0555,
		.child		= ctl_isa,
	}, {}
};

void __init
register_isa_ports(unsigned int membase, unsigned int portbase, unsigned int portshift)
{
	isa_membase = membase;
	isa_portbase = portbase;
	isa_portshift = portshift;
	isa_sysctl_header = register_sysctl_table(ctl_bus);
}


code:
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/kernel/iwmmxt.S
 *
 *  XScale iWMMXt (Concan) context switching and handling
 *
 *  Initial code:
 *  Copyright (c) 2003, Intel Corporation
 *
 *  Full lazy switching support, optimizations and more, by Nicolas Pitre
*   Copyright (c) 2003-2004, MontaVista Software, Inc.
 */

#include <linux/linkage.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
#include <asm/asm-offsets.h>
#include <asm/assembler.h>
#include "iwmmxt.h"

#if defined(CONFIG_CPU_PJ4) || defined(CONFIG_CPU_PJ4B)
#define PJ4(code...)		code
#define XSC(code...)
#elif defined(CONFIG_CPU_MOHAWK) || \
	defined(CONFIG_CPU_XSC3) || \
	defined(CONFIG_CPU_XSCALE)
#define PJ4(code...)
#define XSC(code...)		code
#else
#error "Unsupported iWMMXt architecture"
#endif

#define MMX_WR0		 	(0x00)
#define MMX_WR1		 	(0x08)
#define MMX_WR2		 	(0x10)
#define MMX_WR3			(0x18)
#define MMX_WR4		 	(0x20)
#define MMX_WR5		 	(0x28)
#define MMX_WR6		 	(0x30)
#define MMX_WR7		 	(0x38)
#define MMX_WR8		 	(0x40)
#define MMX_WR9		 	(0x48)
#define MMX_WR10		(0x50)
#define MMX_WR11		(0x58)
#define MMX_WR12		(0x60)
#define MMX_WR13		(0x68)
#define MMX_WR14		(0x70)
#define MMX_WR15		(0x78)
#define MMX_WCSSF		(0x80)
#define MMX_WCASF		(0x84)
#define MMX_WCGR0		(0x88)
#define MMX_WCGR1		(0x8C)
#define MMX_WCGR2		(0x90)
#define MMX_WCGR3		(0x94)

#define MMX_SIZE		(0x98)

	.text
	.arm

/*
 * Lazy switching of Concan coprocessor context
 *
 * r10 = struct thread_info pointer
 * r9  = ret_from_exception
 * lr  = undefined instr exit
 *
 * called from prefetch exception handler with interrupts enabled
 */

ENTRY(iwmmxt_task_enable)
	inc_preempt_count r10, r3

	XSC(mrc	p15, 0, r2, c15, c1, 0)
	PJ4(mrc p15, 0, r2, c1, c0, 2)
	@ CP0 and CP1 accessible?
	XSC(tst	r2, #0x3)
	PJ4(tst	r2, #0xf)
	bne	4f				@ if so no business here
	@ enable access to CP0 and CP1
	XSC(orr	r2, r2, #0x3)
	XSC(mcr	p15, 0, r2, c15, c1, 0)
	PJ4(orr	r2, r2, #0xf)
	PJ4(mcr	p15, 0, r2, c1, c0, 2)

	ldr	r3, =concan_owner
	add	r0, r10, #TI_IWMMXT_STATE	@ get task Concan save area
	ldr	r2, [sp, #60]			@ current task pc value
	ldr	r1, [r3]			@ get current Concan owner
	str	r0, [r3]			@ this task now owns Concan regs
	sub	r2, r2, #4			@ adjust pc back
	str	r2, [sp, #60]

	mrc	p15, 0, r2, c2, c0, 0
	mov	r2, r2				@ cpwait
	bl	concan_save

#ifdef CONFIG_PREEMPT_COUNT
	get_thread_info r10
#endif
4:	dec_preempt_count r10, r3
	ret	r9				@ normal exit from exception

concan_save:

	teq	r1, #0				@ test for last ownership
	beq	concan_load			@ no owner, skip save

	tmrc	r2, wCon

	@ CUP? wCx
	tst	r2, #0x1
	beq 	1f

concan_dump:

	wstrw	wCSSF, r1, MMX_WCSSF
	wstrw	wCASF, r1, MMX_WCASF
	wstrw	wCGR0, r1, MMX_WCGR0
	wstrw	wCGR1, r1, MMX_WCGR1
	wstrw	wCGR2, r1, MMX_WCGR2
	wstrw	wCGR3, r1, MMX_WCGR3

1:	@ MUP? wRn
	tst	r2, #0x2
	beq	2f

	wstrd	wR0,  r1, MMX_WR0
	wstrd	wR1,  r1, MMX_WR1
	wstrd	wR2,  r1, MMX_WR2
	wstrd	wR3,  r1, MMX_WR3
	wstrd	wR4,  r1, MMX_WR4
	wstrd	wR5,  r1, MMX_WR5
	wstrd	wR6,  r1, MMX_WR6
	wstrd	wR7,  r1, MMX_WR7
	wstrd	wR8,  r1, MMX_WR8
	wstrd	wR9,  r1, MMX_WR9
	wstrd	wR10, r1, MMX_WR10
	wstrd	wR11, r1, MMX_WR11
	wstrd	wR12, r1, MMX_WR12
	wstrd	wR13, r1, MMX_WR13
	wstrd	wR14, r1, MMX_WR14
	wstrd	wR15, r1, MMX_WR15

2:	teq	r0, #0				@ anything to load?
	reteq	lr				@ if not, return

concan_load:

	@ Load wRn
	wldrd	wR0,  r0, MMX_WR0
	wldrd	wR1,  r0, MMX_WR1
	wldrd	wR2,  r0, MMX_WR2
	wldrd	wR3,  r0, MMX_WR3
	wldrd	wR4,  r0, MMX_WR4
	wldrd	wR5,  r0, MMX_WR5
	wldrd	wR6,  r0, MMX_WR6
	wldrd	wR7,  r0, MMX_WR7
	wldrd	wR8,  r0, MMX_WR8
	wldrd	wR9,  r0, MMX_WR9
	wldrd	wR10, r0, MMX_WR10
	wldrd	wR11, r0, MMX_WR11
	wldrd	wR12, r0, MMX_WR12
	wldrd	wR13, r0, MMX_WR13
	wldrd	wR14, r0, MMX_WR14
	wldrd	wR15, r0, MMX_WR15

	@ Load wCx
	wldrw	wCSSF, r0, MMX_WCSSF
	wldrw	wCASF, r0, MMX_WCASF
	wldrw	wCGR0, r0, MMX_WCGR0
	wldrw	wCGR1, r0, MMX_WCGR1
	wldrw	wCGR2, r0, MMX_WCGR2
	wldrw	wCGR3, r0, MMX_WCGR3

	@ clear CUP/MUP (only if r1 != 0)
	teq	r1, #0
	mov 	r2, #0
	reteq	lr

	tmcr	wCon, r2
	ret	lr

ENDPROC(iwmmxt_task_enable)

/*
 * Back up Concan regs to save area and disable access to them
 * (mainly for gdb or sleep mode usage)
 *
 * r0 = struct thread_info pointer of target task or NULL for any
 */

ENTRY(iwmmxt_task_disable)

	stmfd	sp!, {r4, lr}

	mrs	ip, cpsr
	orr	r2, ip, #PSR_I_BIT		@ disable interrupts
	msr	cpsr_c, r2

	ldr	r3, =concan_owner
	add	r2, r0, #TI_IWMMXT_STATE	@ get task Concan save area
	ldr	r1, [r3]			@ get current Concan owner
	teq	r1, #0				@ any current owner?
	beq	1f				@ no: quit
	teq	r0, #0				@ any owner?
	teqne	r1, r2				@ or specified one?
	bne	1f				@ no: quit

	@ enable access to CP0 and CP1
	XSC(mrc	p15, 0, r4, c15, c1, 0)
	XSC(orr	r4, r4, #0x3)
	XSC(mcr	p15, 0, r4, c15, c1, 0)
	PJ4(mrc p15, 0, r4, c1, c0, 2)
	PJ4(orr	r4, r4, #0xf)
	PJ4(mcr	p15, 0, r4, c1, c0, 2)

	mov	r0, #0				@ nothing to load
	str	r0, [r3]			@ no more current owner
	mrc	p15, 0, r2, c2, c0, 0
	mov	r2, r2				@ cpwait
	bl	concan_save

	@ disable access to CP0 and CP1
	XSC(bic	r4, r4, #0x3)
	XSC(mcr	p15, 0, r4, c15, c1, 0)
	PJ4(bic	r4, r4, #0xf)
	PJ4(mcr	p15, 0, r4, c1, c0, 2)

	mrc	p15, 0, r2, c2, c0, 0
	mov	r2, r2				@ cpwait

1:	msr	cpsr_c, ip			@ restore interrupt mode
	ldmfd	sp!, {r4, pc}

ENDPROC(iwmmxt_task_disable)

/*
 * Copy Concan state to given memory address
 *
 * r0 = struct thread_info pointer of target task
 * r1 = memory address where to store Concan state
 *
 * this is called mainly in the creation of signal stack frames
 */

ENTRY(iwmmxt_task_copy)

	mrs	ip, cpsr
	orr	r2, ip, #PSR_I_BIT		@ disable interrupts
	msr	cpsr_c, r2

	ldr	r3, =concan_owner
	add	r2, r0, #TI_IWMMXT_STATE	@ get task Concan save area
	ldr	r3, [r3]			@ get current Concan owner
	teq	r2, r3				@ does this task own it...
	beq	1f

	@ current Concan values are in the task save area
	msr	cpsr_c, ip			@ restore interrupt mode
	mov	r0, r1
	mov	r1, r2
	mov	r2, #MMX_SIZE
	b	memcpy

1:	@ this task owns Concan regs -- grab a copy from there
	mov	r0, #0				@ nothing to load
	mov	r2, #3				@ save all regs
	mov	r3, lr				@ preserve return address
	bl	concan_dump
	msr	cpsr_c, ip			@ restore interrupt mode
	ret	r3

ENDPROC(iwmmxt_task_copy)

/*
 * Restore Concan state from given memory address
 *
 * r0 = struct thread_info pointer of target task
 * r1 = memory address where to get Concan state from
 *
 * this is used to restore Concan state when unwinding a signal stack frame
 */

ENTRY(iwmmxt_task_restore)

	mrs	ip, cpsr
	orr	r2, ip, #PSR_I_BIT		@ disable interrupts
	msr	cpsr_c, r2

	ldr	r3, =concan_owner
	add	r2, r0, #TI_IWMMXT_STATE	@ get task Concan save area
	ldr	r3, [r3]			@ get current Concan owner
	bic	r2, r2, #0x7			@ 64-bit alignment
	teq	r2, r3				@ does this task own it...
	beq	1f

	@ this task doesn't own Concan regs -- use its save area
	msr	cpsr_c, ip			@ restore interrupt mode
	mov	r0, r2
	mov	r2, #MMX_SIZE
	b	memcpy

1:	@ this task owns Concan regs -- load them directly
	mov	r0, r1
	mov	r1, #0				@ don't clear CUP/MUP
	mov	r3, lr				@ preserve return address
	bl	concan_load
	msr	cpsr_c, ip			@ restore interrupt mode
	ret	r3

ENDPROC(iwmmxt_task_restore)

/*
 * Concan handling on task switch
 *
 * r0 = next thread_info pointer
 *
 * Called only from the iwmmxt notifier with task preemption disabled.
 */
ENTRY(iwmmxt_task_switch)

	XSC(mrc	p15, 0, r1, c15, c1, 0)
	PJ4(mrc	p15, 0, r1, c1, c0, 2)
	@ CP0 and CP1 accessible?
	XSC(tst	r1, #0x3)
	PJ4(tst	r1, #0xf)
	bne	1f				@ yes: block them for next task

	ldr	r2, =concan_owner
	add	r3, r0, #TI_IWMMXT_STATE	@ get next task Concan save area
	ldr	r2, [r2]			@ get current Concan owner
	teq	r2, r3				@ next task owns it?
	retne	lr				@ no: leave Concan disabled

1:	@ flip Concan access
	XSC(eor	r1, r1, #0x3)
	XSC(mcr	p15, 0, r1, c15, c1, 0)
	PJ4(eor r1, r1, #0xf)
	PJ4(mcr	p15, 0, r1, c1, c0, 2)

	mrc	p15, 0, r1, c2, c0, 0
	sub	pc, lr, r1, lsr #32		@ cpwait and return

ENDPROC(iwmmxt_task_switch)

/*
 * Remove Concan ownership of given task
 *
 * r0 = struct thread_info pointer
 */
ENTRY(iwmmxt_task_release)

	mrs	r2, cpsr
	orr	ip, r2, #PSR_I_BIT		@ disable interrupts
	msr	cpsr_c, ip
	ldr	r3, =concan_owner
	add	r0, r0, #TI_IWMMXT_STATE	@ get task Concan save area
	ldr	r1, [r3]			@ get current Concan owner
	eors	r0, r0, r1			@ if equal...
	streq	r0, [r3]			@ then clear ownership
	msr	cpsr_c, r2			@ restore interrupts
	ret	lr

ENDPROC(iwmmxt_task_release)

	.data
	.align	2
concan_owner:
	.word	0


code:
/* SPDX-License-Identifier: GPL-2.0 */

#ifndef __IWMMXT_H__
#define __IWMMXT_H__

.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
.set .LwR\b, \b
.set .Lr\b, \b
.endr

.set .LwCSSF, 0x2
.set .LwCASF, 0x3
.set .LwCGR0, 0x8
.set .LwCGR1, 0x9
.set .LwCGR2, 0xa
.set .LwCGR3, 0xb

.macro wldrd, reg:req, base:req, offset:req
.inst 0xedd00100 | (.L\reg << 12) | (.L\base << 16) | (\offset >> 2)
.endm

.macro wldrw, reg:req, base:req, offset:req
.inst 0xfd900100 | (.L\reg << 12) | (.L\base << 16) | (\offset >> 2)
.endm

.macro wstrd, reg:req, base:req, offset:req
.inst 0xedc00100 | (.L\reg << 12) | (.L\base << 16) | (\offset >> 2)
.endm

.macro wstrw, reg:req, base:req, offset:req
.inst 0xfd800100 | (.L\reg << 12) | (.L\base << 16) | (\offset >> 2)
.endm

#ifdef __clang__

#define wCon c1

.macro tmrc, dest:req, control:req
mrc p1, 0, \dest, \control, c0, 0
.endm

.macro tmcr, control:req, src:req
mcr p1, 0, \src, \control, c0, 0
.endm
#endif

#endif


code:
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/jump_label.h>
#include <asm/patch.h>
#include <asm/insn.h>

static void __arch_jump_label_transform(struct jump_entry *entry,
					enum jump_label_type type,
					bool is_static)
{
	void *addr = (void *)entry->code;
	unsigned int insn;

	if (type == JUMP_LABEL_JMP)
		insn = arm_gen_branch(entry->code, entry->target);
	else
		insn = arm_gen_nop();

	if (is_static)
		__patch_text_early(addr, insn);
	else
		patch_text(addr, insn);
}

void arch_jump_label_transform(struct jump_entry *entry,
			       enum jump_label_type type)
{
	__arch_jump_label_transform(entry, type, false);
}


code:
// SPDX-License-Identifier: GPL-2.0
/*
 * arch/arm/kernel/kgdb.c
 *
 * ARM KGDB support
 *
 * Copyright (c) 2002-2004 MontaVista Software, Inc
 * Copyright (c) 2008 Wind River Systems, Inc.
 *
 * Authors:  George Davis <davis_g@mvista.com>
 *           Deepak Saxena <dsaxena@plexity.net>
 */
#include <linux/irq.h>
#include <linux/kdebug.h>
#include <linux/kgdb.h>
#include <linux/uaccess.h>

#include <asm/patch.h>
#include <asm/traps.h>

struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] =
{
	{ "r0", 4, offsetof(struct pt_regs, ARM_r0)},
	{ "r1", 4, offsetof(struct pt_regs, ARM_r1)},
	{ "r2", 4, offsetof(struct pt_regs, ARM_r2)},
	{ "r3", 4, offsetof(struct pt_regs, ARM_r3)},
	{ "r4", 4, offsetof(struct pt_regs, ARM_r4)},
	{ "r5", 4, offsetof(struct pt_regs, ARM_r5)},
	{ "r6", 4, offsetof(struct pt_regs, ARM_r6)},
	{ "r7", 4, offsetof(struct pt_regs, ARM_r7)},
	{ "r8", 4, offsetof(struct pt_regs, ARM_r8)},
	{ "r9", 4, offsetof(struct pt_regs, ARM_r9)},
	{ "r10", 4, offsetof(struct pt_regs, ARM_r10)},
	{ "fp", 4, offsetof(struct pt_regs, ARM_fp)},
	{ "ip", 4, offsetof(struct pt_regs, ARM_ip)},
	{ "sp", 4, offsetof(struct pt_regs, ARM_sp)},
	{ "lr", 4, offsetof(struct pt_regs, ARM_lr)},
	{ "pc", 4, offsetof(struct pt_regs, ARM_pc)},
	{ "f0", 12, -1 },
	{ "f1", 12, -1 },
	{ "f2", 12, -1 },
	{ "f3", 12, -1 },
	{ "f4", 12, -1 },
	{ "f5", 12, -1 },
	{ "f6", 12, -1 },
	{ "f7", 12, -1 },
	{ "fps", 4, -1 },
	{ "cpsr", 4, offsetof(struct pt_regs, ARM_cpsr)},
};

char *dbg_get_reg(int regno, void *mem, struct pt_regs *regs)
{
	if (regno >= DBG_MAX_REG_NUM || regno < 0)
		return NULL;

	if (dbg_reg_def[regno].offset != -1)
		memcpy(mem, (void *)regs + dbg_reg_def[regno].offset,
		       dbg_reg_def[regno].size);
	else
		memset(mem, 0, dbg_reg_def[regno].size);
	return dbg_reg_def[regno].name;
}

int dbg_set_reg(int regno, void *mem, struct pt_regs *regs)
{
	if (regno >= DBG_MAX_REG_NUM || regno < 0)
		return -EINVAL;

	if (dbg_reg_def[regno].offset != -1)
		memcpy((void *)regs + dbg_reg_def[regno].offset, mem,
		       dbg_reg_def[regno].size);
	return 0;
}

void
sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task)
{
	struct thread_info *ti;
	int regno;

	/* Just making sure... */
	if (task == NULL)
		return;

	/* Initialize to zero */
	for (regno = 0; regno < GDB_MAX_REGS; regno++)
		gdb_regs[regno] = 0;

	/* Otherwise, we have only some registers from switch_to() */
	ti			= task_thread_info(task);
	gdb_regs[_R4]		= ti->cpu_context.r4;
	gdb_regs[_R5]		= ti->cpu_context.r5;
	gdb_regs[_R6]		= ti->cpu_context.r6;
	gdb_regs[_R7]		= ti->cpu_context.r7;
	gdb_regs[_R8]		= ti->cpu_context.r8;
	gdb_regs[_R9]		= ti->cpu_context.r9;
	gdb_regs[_R10]		= ti->cpu_context.sl;
	gdb_regs[_FP]		= ti->cpu_context.fp;
	gdb_regs[_SPT]		= ti->cpu_context.sp;
	gdb_regs[_PC]		= ti->cpu_context.pc;
}

void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc)
{
	regs->ARM_pc = pc;
}

static int compiled_break;

int kgdb_arch_handle_exception(int exception_vector, int signo,
			       int err_code, char *remcom_in_buffer,
			       char *remcom_out_buffer,
			       struct pt_regs *linux_regs)
{
	unsigned long addr;
	char *ptr;

	switch (remcom_in_buffer[0]) {
	case 'D':
	case 'k':
	case 'c':
		/*
		 * Try to read optional parameter, pc unchanged if no parm.
		 * If this was a compiled breakpoint, we need to move
		 * to the next instruction or we will just breakpoint
		 * over and over again.
		 */
		ptr = &remcom_in_buffer[1];
		if (kgdb_hex2long(&ptr, &addr))
			linux_regs->ARM_pc = addr;
		else if (compiled_break == 1)
			linux_regs->ARM_pc += 4;

		compiled_break = 0;

		return 0;
	}

	return -1;
}

static int kgdb_brk_fn(struct pt_regs *regs, unsigned int instr)
{
	kgdb_handle_exception(1, SIGTRAP, 0, regs);

	return 0;
}

static int kgdb_compiled_brk_fn(struct pt_regs *regs, unsigned int instr)
{
	compiled_break = 1;
	kgdb_handle_exception(1, SIGTRAP, 0, regs);

	return 0;
}

static struct undef_hook kgdb_brkpt_arm_hook = {
	.instr_mask		= 0xffffffff,
	.instr_val		= KGDB_BREAKINST,
	.cpsr_mask		= PSR_T_BIT | MODE_MASK,
	.cpsr_val		= SVC_MODE,
	.fn			= kgdb_brk_fn
};

static struct undef_hook kgdb_brkpt_thumb_hook = {
	.instr_mask		= 0xffff,
	.instr_val		= KGDB_BREAKINST & 0xffff,
	.cpsr_mask		= PSR_T_BIT | MODE_MASK,
	.cpsr_val		= PSR_T_BIT | SVC_MODE,
	.fn			= kgdb_brk_fn
};

static struct undef_hook kgdb_compiled_brkpt_arm_hook = {
	.instr_mask		= 0xffffffff,
	.instr_val		= KGDB_COMPILED_BREAK,
	.cpsr_mask		= PSR_T_BIT | MODE_MASK,
	.cpsr_val		= SVC_MODE,
	.fn			= kgdb_compiled_brk_fn
};

static struct undef_hook kgdb_compiled_brkpt_thumb_hook = {
	.instr_mask		= 0xffff,
	.instr_val		= KGDB_COMPILED_BREAK & 0xffff,
	.cpsr_mask		= PSR_T_BIT | MODE_MASK,
	.cpsr_val		= PSR_T_BIT | SVC_MODE,
	.fn			= kgdb_compiled_brk_fn
};

static int __kgdb_notify(struct die_args *args, unsigned long cmd)
{
	struct pt_regs *regs = args->regs;

	if (kgdb_handle_exception(1, args->signr, cmd, regs))
		return NOTIFY_DONE;
	return NOTIFY_STOP;
}
static int
kgdb_notify(struct notifier_block *self, unsigned long cmd, void *ptr)
{
	unsigned long flags;
	int ret;

	local_irq_save(flags);
	ret = __kgdb_notify(ptr, cmd);
	local_irq_restore(flags);

	return ret;
}

static struct notifier_block kgdb_notifier = {
	.notifier_call	= kgdb_notify,
	.priority	= -INT_MAX,
};

/**
 *	kgdb_arch_init - Perform any architecture specific initalization.
 *
 *	This function will handle the initalization of any architecture
 *	specific callbacks.
 */
int kgdb_arch_init(void)
{
	int ret = register_die_notifier(&kgdb_notifier);

	if (ret != 0)
		return ret;

	register_undef_hook(&kgdb_brkpt_arm_hook);
	register_undef_hook(&kgdb_brkpt_thumb_hook);
	register_undef_hook(&kgdb_compiled_brkpt_arm_hook);
	register_undef_hook(&kgdb_compiled_brkpt_thumb_hook);

	return 0;
}

/**
 *	kgdb_arch_exit - Perform any architecture specific uninitalization.
 *
 *	This function will handle the uninitalization of any architecture
 *	specific callbacks, for dynamic registration and unregistration.
 */
void kgdb_arch_exit(void)
{
	unregister_undef_hook(&kgdb_brkpt_arm_hook);
	unregister_undef_hook(&kgdb_brkpt_thumb_hook);
	unregister_undef_hook(&kgdb_compiled_brkpt_arm_hook);
	unregister_undef_hook(&kgdb_compiled_brkpt_thumb_hook);
	unregister_die_notifier(&kgdb_notifier);
}

int kgdb_arch_set_breakpoint(struct kgdb_bkpt *bpt)
{
	int err;

	/* patch_text() only supports int-sized breakpoints */
	BUILD_BUG_ON(sizeof(int) != BREAK_INSTR_SIZE);

	err = copy_from_kernel_nofault(bpt->saved_instr, (char *)bpt->bpt_addr,
				BREAK_INSTR_SIZE);
	if (err)
		return err;

	/* Machine is already stopped, so we can use __patch_text() directly */
	__patch_text((void *)bpt->bpt_addr,
		     *(unsigned int *)arch_kgdb_ops.gdb_bpt_instr);

	return err;
}

int kgdb_arch_remove_breakpoint(struct kgdb_bkpt *bpt)
{
	/* Machine is already stopped, so we can use __patch_text() directly */
	__patch_text((void *)bpt->bpt_addr, *(unsigned int *)bpt->saved_instr);

	return 0;
}

/*
 * Register our undef instruction hooks with ARM undef core.
 * We register a hook specifically looking for the KGB break inst
 * and we handle the normal undef case within the do_undefinstr
 * handler.
 */
const struct kgdb_arch arch_kgdb_ops = {
#ifndef __ARMEB__
	.gdb_bpt_instr		= {0xfe, 0xde, 0xff, 0xe7}
#else /* ! __ARMEB__ */
	.gdb_bpt_instr		= {0xe7, 0xff, 0xde, 0xfe}
#endif
};


code:
// SPDX-License-Identifier: GPL-2.0
/*
 * machine_kexec.c - handle transition of Linux booting another kernel
 */

#include <linux/mm.h>
#include <linux/kexec.h>
#include <linux/delay.h>
#include <linux/reboot.h>
#include <linux/io.h>
#include <linux/irq.h>
#include <linux/memblock.h>
#include <linux/of_fdt.h>
#include <asm/mmu_context.h>
#include <asm/cacheflush.h>
#include <asm/kexec-internal.h>
#include <asm/fncpy.h>
#include <asm/mach-types.h>
#include <asm/smp_plat.h>
#include <asm/system_misc.h>
#include <asm/set_memory.h>

extern void relocate_new_kernel(void);
extern const unsigned int relocate_new_kernel_size;

static atomic_t waiting_for_crash_ipi;

/*
 * Provide a dummy crash_notes definition while crash dump arrives to arm.
 * This prevents breakage of crash_notes attribute in kernel/ksysfs.c.
 */

int machine_kexec_prepare(struct kimage *image)
{
	struct kexec_segment *current_segment;
	__be32 header;
	int i, err;

	image->arch.kernel_r2 = image->start - KEXEC_ARM_ZIMAGE_OFFSET
				     + KEXEC_ARM_ATAGS_OFFSET;

	/*
	 * Validate that if the current HW supports SMP, then the SW supports
	 * and implements CPU hotplug for the current HW. If not, we won't be
	 * able to kexec reliably, so fail the prepare operation.
	 */
	if (num_possible_cpus() > 1 && platform_can_secondary_boot() &&
	    !platform_can_cpu_hotplug())
		return -EINVAL;

	/*
	 * No segment at default ATAGs address. try to locate
	 * a dtb using magic.
	 */
	for (i = 0; i < image->nr_segments; i++) {
		current_segment = &image->segment[i];

		if (!memblock_is_region_memory(idmap_to_phys(current_segment->mem),
					       current_segment->memsz))
			return -EINVAL;

		err = get_user(header, (__be32*)current_segment->buf);
		if (err)
			return err;

		if (header == cpu_to_be32(OF_DT_HEADER))
			image->arch.kernel_r2 = current_segment->mem;
	}
	return 0;
}

void machine_kexec_cleanup(struct kimage *image)
{
}

static void machine_crash_nonpanic_core(void *unused)
{
	struct pt_regs regs;

	local_fiq_disable();

	crash_setup_regs(&regs, get_irq_regs());
	printk(KERN_DEBUG "CPU %u will stop doing anything useful since another CPU has crashed\n",
	       smp_processor_id());
	crash_save_cpu(&regs, smp_processor_id());
	flush_cache_all();

	set_cpu_online(smp_processor_id(), false);
	atomic_dec(&waiting_for_crash_ipi);

	while (1) {
		cpu_relax();
		wfe();
	}
}

void crash_smp_send_stop(void)
{
	static int cpus_stopped;
	unsigned long msecs;

	if (cpus_stopped)
		return;

	atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1);
	smp_call_function(machine_crash_nonpanic_core, NULL, false);
	msecs = 1000; /* Wait at most a second for the other cpus to stop */
	while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) {
		mdelay(1);
		msecs--;
	}
	if (atomic_read(&waiting_for_crash_ipi) > 0)
		pr_warn("Non-crashing CPUs did not react to IPI\n");

	cpus_stopped = 1;
}

static void machine_kexec_mask_interrupts(void)
{
	unsigned int i;
	struct irq_desc *desc;

	for_each_irq_desc(i, desc) {
		struct irq_chip *chip;

		chip = irq_desc_get_chip(desc);
		if (!chip)
			continue;

		if (chip->irq_eoi && irqd_irq_inprogress(&desc->irq_data))
			chip->irq_eoi(&desc->irq_data);

		if (chip->irq_mask)
			chip->irq_mask(&desc->irq_data);

		if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data))
			chip->irq_disable(&desc->irq_data);
	}
}

void machine_crash_shutdown(struct pt_regs *regs)
{
	local_irq_disable();
	crash_smp_send_stop();

	crash_save_cpu(regs, smp_processor_id());
	machine_kexec_mask_interrupts();

	pr_info("Loading crashdump kernel...\n");
}

void machine_kexec(struct kimage *image)
{
	unsigned long page_list, reboot_entry_phys;
	struct kexec_relocate_data *data;
	void (*reboot_entry)(void);
	void *reboot_code_buffer;

	/*
	 * This can only happen if machine_shutdown() failed to disable some
	 * CPU, and that can only happen if the checks in
	 * machine_kexec_prepare() were not correct. If this fails, we can't
	 * reliably kexec anyway, so BUG_ON is appropriate.
	 */
	BUG_ON(num_online_cpus() > 1);

	page_list = image->head & PAGE_MASK;

	reboot_code_buffer = page_address(image->control_code_page);

	/* copy our kernel relocation code to the control code page */
	reboot_entry = fncpy(reboot_code_buffer,
			     &relocate_new_kernel,
			     relocate_new_kernel_size);

	data = reboot_code_buffer + relocate_new_kernel_size;
	data->kexec_start_address = image->start;
	data->kexec_indirection_page = page_list;
	data->kexec_mach_type = machine_arch_type;
	data->kexec_r2 = image->arch.kernel_r2;

	/* get the identity mapping physical address for the reboot code */
	reboot_entry_phys = virt_to_idmap(reboot_entry);

	pr_info("Bye!\n");

	soft_restart(reboot_entry_phys);
}

void arch_crash_save_vmcoreinfo(void)
{
#ifdef CONFIG_ARM_LPAE
	VMCOREINFO_CONFIG(ARM_LPAE);
#endif
}


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
 */

#include <linux/elf.h>
#include <linux/ftrace.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sort.h>
#include <linux/moduleloader.h>

#include <asm/cache.h>
#include <asm/opcodes.h>

#ifdef CONFIG_THUMB2_KERNEL
#define PLT_ENT_LDR		__opcode_to_mem_thumb32(0xf8dff000 | \
							(PLT_ENT_STRIDE - 4))
#else
#define PLT_ENT_LDR		__opcode_to_mem_arm(0xe59ff000 | \
						    (PLT_ENT_STRIDE - 8))
#endif

static const u32 fixed_plts[] = {
#ifdef CONFIG_DYNAMIC_FTRACE
	FTRACE_ADDR,
	MCOUNT_ADDR,
#endif
};

static bool in_init(const struct module *mod, unsigned long loc)
{
	return loc - (u32)mod->init_layout.base < mod->init_layout.size;
}

static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt)
{
	int i;

	if (!ARRAY_SIZE(fixed_plts) || pltsec->plt_count)
		return;
	pltsec->plt_count = ARRAY_SIZE(fixed_plts);

	for (i = 0; i < ARRAY_SIZE(plt->ldr); ++i)
		plt->ldr[i] = PLT_ENT_LDR;

	BUILD_BUG_ON(sizeof(fixed_plts) > sizeof(plt->lit));
	memcpy(plt->lit, fixed_plts, sizeof(fixed_plts));
}

u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val)
{
	struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
							  &mod->arch.init;
	struct plt_entries *plt;
	int idx;

	/* cache the address, ELF header is available only during module load */
	if (!pltsec->plt_ent)
		pltsec->plt_ent = (struct plt_entries *)pltsec->plt->sh_addr;
	plt = pltsec->plt_ent;

	prealloc_fixed(pltsec, plt);

	for (idx = 0; idx < ARRAY_SIZE(fixed_plts); ++idx)
		if (plt->lit[idx] == val)
			return (u32)&plt->ldr[idx];

	idx = 0;
	/*
	 * Look for an existing entry pointing to 'val'. Given that the
	 * relocations are sorted, this will be the last entry we allocated.
	 * (if one exists).
	 */
	if (pltsec->plt_count > 0) {
		plt += (pltsec->plt_count - 1) / PLT_ENT_COUNT;
		idx = (pltsec->plt_count - 1) % PLT_ENT_COUNT;

		if (plt->lit[idx] == val)
			return (u32)&plt->ldr[idx];

		idx = (idx + 1) % PLT_ENT_COUNT;
		if (!idx)
			plt++;
	}

	pltsec->plt_count++;
	BUG_ON(pltsec->plt_count * PLT_ENT_SIZE > pltsec->plt->sh_size);

	if (!idx)
		/* Populate a new set of entries */
		*plt = (struct plt_entries){
			{ [0 ... PLT_ENT_COUNT - 1] = PLT_ENT_LDR, },
			{ val, }
		};
	else
		plt->lit[idx] = val;

	return (u32)&plt->ldr[idx];
}

#define cmp_3way(a,b)	((a) < (b) ? -1 : (a) > (b))

static int cmp_rel(const void *a, const void *b)
{
	const Elf32_Rel *x = a, *y = b;
	int i;

	/* sort by type and symbol index */
	i = cmp_3way(ELF32_R_TYPE(x->r_info), ELF32_R_TYPE(y->r_info));
	if (i == 0)
		i = cmp_3way(ELF32_R_SYM(x->r_info), ELF32_R_SYM(y->r_info));
	return i;
}

static bool is_zero_addend_relocation(Elf32_Addr base, const Elf32_Rel *rel)
{
	u32 *tval = (u32 *)(base + rel->r_offset);

	/*
	 * Do a bitwise compare on the raw addend rather than fully decoding
	 * the offset and doing an arithmetic comparison.
	 * Note that a zero-addend jump/call relocation is encoded taking the
	 * PC bias into account, i.e., -8 for ARM and -4 for Thumb2.
	 */
	switch (ELF32_R_TYPE(rel->r_info)) {
		u16 upper, lower;

	case R_ARM_THM_CALL:
	case R_ARM_THM_JUMP24:
		upper = __mem_to_opcode_thumb16(((u16 *)tval)[0]);
		lower = __mem_to_opcode_thumb16(((u16 *)tval)[1]);

		return (upper & 0x7ff) == 0x7ff && (lower & 0x2fff) == 0x2ffe;

	case R_ARM_CALL:
	case R_ARM_PC24:
	case R_ARM_JUMP24:
		return (__mem_to_opcode_arm(*tval) & 0xffffff) == 0xfffffe;
	}
	BUG();
}

static bool duplicate_rel(Elf32_Addr base, const Elf32_Rel *rel, int num)
{
	const Elf32_Rel *prev;

	/*
	 * Entries are sorted by type and symbol index. That means that,
	 * if a duplicate entry exists, it must be in the preceding
	 * slot.
	 */
	if (!num)
		return false;

	prev = rel + num - 1;
	return cmp_rel(rel + num, prev) == 0 &&
	       is_zero_addend_relocation(base, prev);
}

/* Count how many PLT entries we may need */
static unsigned int count_plts(const Elf32_Sym *syms, Elf32_Addr base,
			       const Elf32_Rel *rel, int num, Elf32_Word dstidx)
{
	unsigned int ret = 0;
	const Elf32_Sym *s;
	int i;

	for (i = 0; i < num; i++) {
		switch (ELF32_R_TYPE(rel[i].r_info)) {
		case R_ARM_CALL:
		case R_ARM_PC24:
		case R_ARM_JUMP24:
		case R_ARM_THM_CALL:
		case R_ARM_THM_JUMP24:
			/*
			 * We only have to consider branch targets that resolve
			 * to symbols that are defined in a different section.
			 * This is not simply a heuristic, it is a fundamental
			 * limitation, since there is no guaranteed way to emit
			 * PLT entries sufficiently close to the branch if the
			 * section size exceeds the range of a branch
			 * instruction. So ignore relocations against defined
			 * symbols if they live in the same section as the
			 * relocation target.
			 */
			s = syms + ELF32_R_SYM(rel[i].r_info);
			if (s->st_shndx == dstidx)
				break;

			/*
			 * Jump relocations with non-zero addends against
			 * undefined symbols are supported by the ELF spec, but
			 * do not occur in practice (e.g., 'jump n bytes past
			 * the entry point of undefined function symbol f').
			 * So we need to support them, but there is no need to
			 * take them into consideration when trying to optimize
			 * this code. So let's only check for duplicates when
			 * the addend is zero. (Note that calls into the core
			 * module via init PLT entries could involve section
			 * relative symbol references with non-zero addends, for
			 * which we may end up emitting duplicates, but the init
			 * PLT is released along with the rest of the .init
			 * region as soon as module loading completes.)
			 */
			if (!is_zero_addend_relocation(base, rel + i) ||
			    !duplicate_rel(base, rel, i))
				ret++;
		}
	}
	return ret;
}

int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
			      char *secstrings, struct module *mod)
{
	unsigned long core_plts = ARRAY_SIZE(fixed_plts);
	unsigned long init_plts = ARRAY_SIZE(fixed_plts);
	Elf32_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum;
	Elf32_Sym *syms = NULL;

	/*
	 * To store the PLTs, we expand the .text section for core module code
	 * and for initialization code.
	 */
	for (s = sechdrs; s < sechdrs_end; ++s) {
		if (strcmp(".plt", secstrings + s->sh_name) == 0)
			mod->arch.core.plt = s;
		else if (strcmp(".init.plt", secstrings + s->sh_name) == 0)
			mod->arch.init.plt = s;
		else if (s->sh_type == SHT_SYMTAB)
			syms = (Elf32_Sym *)s->sh_addr;
	}

	if (!mod->arch.core.plt || !mod->arch.init.plt) {
		pr_err("%s: module PLT section(s) missing\n", mod->name);
		return -ENOEXEC;
	}
	if (!syms) {
		pr_err("%s: module symtab section missing\n", mod->name);
		return -ENOEXEC;
	}

	for (s = sechdrs + 1; s < sechdrs_end; ++s) {
		Elf32_Rel *rels = (void *)ehdr + s->sh_offset;
		int numrels = s->sh_size / sizeof(Elf32_Rel);
		Elf32_Shdr *dstsec = sechdrs + s->sh_info;

		if (s->sh_type != SHT_REL)
			continue;

		/* ignore relocations that operate on non-exec sections */
		if (!(dstsec->sh_flags & SHF_EXECINSTR))
			continue;

		/* sort by type and symbol index */
		sort(rels, numrels, sizeof(Elf32_Rel), cmp_rel, NULL);

		if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0)
			core_plts += count_plts(syms, dstsec->sh_addr, rels,
						numrels, s->sh_info);
		else
			init_plts += count_plts(syms, dstsec->sh_addr, rels,
						numrels, s->sh_info);
	}

	mod->arch.core.plt->sh_type = SHT_NOBITS;
	mod->arch.core.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
	mod->arch.core.plt->sh_addralign = L1_CACHE_BYTES;
	mod->arch.core.plt->sh_size = round_up(core_plts * PLT_ENT_SIZE,
					       sizeof(struct plt_entries));
	mod->arch.core.plt_count = 0;
	mod->arch.core.plt_ent = NULL;

	mod->arch.init.plt->sh_type = SHT_NOBITS;
	mod->arch.init.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
	mod->arch.init.plt->sh_addralign = L1_CACHE_BYTES;
	mod->arch.init.plt->sh_size = round_up(init_plts * PLT_ENT_SIZE,
					       sizeof(struct plt_entries));
	mod->arch.init.plt_count = 0;
	mod->arch.init.plt_ent = NULL;

	pr_debug("%s: plt=%x, init.plt=%x\n", __func__,
		 mod->arch.core.plt->sh_size, mod->arch.init.plt->sh_size);
	return 0;
}

bool in_module_plt(unsigned long loc)
{
	struct module *mod;
	bool ret;

	preempt_disable();
	mod = __module_text_address(loc);
	ret = mod && (loc - (u32)mod->arch.core.plt_ent < mod->arch.core.plt_count * PLT_ENT_SIZE ||
		      loc - (u32)mod->arch.init.plt_ent < mod->arch.init.plt_count * PLT_ENT_SIZE);
	preempt_enable();

	return ret;
}


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/arch/arm/kernel/module.c
 *
 *  Copyright (C) 2002 Russell King.
 *  Modified for nommu by Hyok S. Choi
 *
 * Module allocation method suggested by Andi Kleen.
 */
#include <linux/module.h>
#include <linux/moduleloader.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/elf.h>
#include <linux/vmalloc.h>
#include <linux/fs.h>
#include <linux/string.h>
#include <linux/gfp.h>

#include <asm/sections.h>
#include <asm/smp_plat.h>
#include <asm/unwind.h>
#include <asm/opcodes.h>

#ifdef CONFIG_XIP_KERNEL
/*
 * The XIP kernel text is mapped in the module area for modules and
 * some other stuff to work without any indirect relocations.
 * MODULES_VADDR is redefined here and not in asm/memory.h to avoid
 * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off.
 */
#undef MODULES_VADDR
#define MODULES_VADDR	(((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK)
#endif

#ifdef CONFIG_MMU
void *module_alloc(unsigned long size)
{
	gfp_t gfp_mask = GFP_KERNEL;
	void *p;

	/* Silence the initial allocation */
	if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS))
		gfp_mask |= __GFP_NOWARN;

	p = __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
				gfp_mask, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
				__builtin_return_address(0));
	if (!IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || p)
		return p;
	return __vmalloc_node_range(size, 1,  VMALLOC_START, VMALLOC_END,
				GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
				__builtin_return_address(0));
}
#endif

bool module_init_section(const char *name)
{
	return strstarts(name, ".init") ||
		strstarts(name, ".ARM.extab.init") ||
		strstarts(name, ".ARM.exidx.init");
}

bool module_exit_section(const char *name)
{
	return strstarts(name, ".exit") ||
		strstarts(name, ".ARM.extab.exit") ||
		strstarts(name, ".ARM.exidx.exit");
}

#ifdef CONFIG_ARM_HAS_GROUP_RELOCS
/*
 * This implements the partitioning algorithm for group relocations as
 * documented in the ARM AArch32 ELF psABI (IHI 0044).
 *
 * A single PC-relative symbol reference is divided in up to 3 add or subtract
 * operations, where the final one could be incorporated into a load/store
 * instruction with immediate offset. E.g.,
 *
 *   ADD	Rd, PC, #...		or	ADD	Rd, PC, #...
 *   ADD	Rd, Rd, #...			ADD	Rd, Rd, #...
 *   LDR	Rd, [Rd, #...]			ADD	Rd, Rd, #...
 *
 * The latter has a guaranteed range of only 16 MiB (3x8 == 24 bits), so it is
 * of limited use in the kernel. However, the ADD/ADD/LDR combo has a range of
 * -/+ 256 MiB, (2x8 + 12 == 28 bits), which means it has sufficient range for
 * any in-kernel symbol reference (unless module PLTs are being used).
 *
 * The main advantage of this approach over the typical pattern using a literal
 * load is that literal loads may miss in the D-cache, and generally lead to
 * lower cache efficiency for variables that are referenced often from many
 * different places in the code.
 */
static u32 get_group_rem(u32 group, u32 *offset)
{
	u32 val = *offset;
	u32 shift;
	do {
		shift = val ? (31 - __fls(val)) & ~1 : 32;
		*offset = val;
		if (!val)
			break;
		val &= 0xffffff >> shift;
	} while (group--);
	return shift;
}
#endif

int
apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
	       unsigned int relindex, struct module *module)
{
	Elf32_Shdr *symsec = sechdrs + symindex;
	Elf32_Shdr *relsec = sechdrs + relindex;
	Elf32_Shdr *dstsec = sechdrs + relsec->sh_info;
	Elf32_Rel *rel = (void *)relsec->sh_addr;
	unsigned int i;

	for (i = 0; i < relsec->sh_size / sizeof(Elf32_Rel); i++, rel++) {
		unsigned long loc;
		Elf32_Sym *sym;
		const char *symname;
#ifdef CONFIG_ARM_HAS_GROUP_RELOCS
		u32 shift, group = 1;
#endif
		s32 offset;
		u32 tmp;
#ifdef CONFIG_THUMB2_KERNEL
		u32 upper, lower, sign, j1, j2;
#endif

		offset = ELF32_R_SYM(rel->r_info);
		if (offset < 0 || offset > (symsec->sh_size / sizeof(Elf32_Sym))) {
			pr_err("%s: section %u reloc %u: bad relocation sym offset\n",
				module->name, relindex, i);
			return -ENOEXEC;
		}

		sym = ((Elf32_Sym *)symsec->sh_addr) + offset;
		symname = strtab + sym->st_name;

		if (rel->r_offset < 0 || rel->r_offset > dstsec->sh_size - sizeof(u32)) {
			pr_err("%s: section %u reloc %u sym '%s': out of bounds relocation, offset %d size %u\n",
			       module->name, relindex, i, symname,
			       rel->r_offset, dstsec->sh_size);
			return -ENOEXEC;
		}

		loc = dstsec->sh_addr + rel->r_offset;

		switch (ELF32_R_TYPE(rel->r_info)) {
		case R_ARM_NONE:
			/* ignore */
			break;

		case R_ARM_ABS32:
		case R_ARM_TARGET1:
			*(u32 *)loc += sym->st_value;
			break;

		case R_ARM_PC24:
		case R_ARM_CALL:
		case R_ARM_JUMP24:
			if (sym->st_value & 3) {
				pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (ARM -> Thumb)\n",
				       module->name, relindex, i, symname);
				return -ENOEXEC;
			}

			offset = __mem_to_opcode_arm(*(u32 *)loc);
			offset = (offset & 0x00ffffff) << 2;
			if (offset & 0x02000000)
				offset -= 0x04000000;

			offset += sym->st_value - loc;

			/*
			 * Route through a PLT entry if 'offset' exceeds the
			 * supported range. Note that 'offset + loc + 8'
			 * contains the absolute jump target, i.e.,
			 * @sym + addend, corrected for the +8 PC bias.
			 */
			if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) &&
			    (offset <= (s32)0xfe000000 ||
			     offset >= (s32)0x02000000))
				offset = get_module_plt(module, loc,
							offset + loc + 8)
					 - loc - 8;

			if (offset <= (s32)0xfe000000 ||
			    offset >= (s32)0x02000000) {
				pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
				       module->name, relindex, i, symname,
				       ELF32_R_TYPE(rel->r_info), loc,
				       sym->st_value);
				return -ENOEXEC;
			}

			offset >>= 2;
			offset &= 0x00ffffff;

			*(u32 *)loc &= __opcode_to_mem_arm(0xff000000);
			*(u32 *)loc |= __opcode_to_mem_arm(offset);
			break;

	       case R_ARM_V4BX:
		       /* Preserve Rm and the condition code. Alter
			* other bits to re-code instruction as
			* MOV PC,Rm.
			*/
		       *(u32 *)loc &= __opcode_to_mem_arm(0xf000000f);
		       *(u32 *)loc |= __opcode_to_mem_arm(0x01a0f000);
		       break;

		case R_ARM_PREL31:
			offset = (*(s32 *)loc << 1) >> 1; /* sign extend */
			offset += sym->st_value - loc;
			if (offset >= 0x40000000 || offset < -0x40000000) {
				pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
				       module->name, relindex, i, symname,
				       ELF32_R_TYPE(rel->r_info), loc,
				       sym->st_value);
				return -ENOEXEC;
			}
			*(u32 *)loc &= 0x80000000;
			*(u32 *)loc |= offset & 0x7fffffff;
			break;

		case R_ARM_REL32:
			*(u32 *)loc += sym->st_value - loc;
			break;

		case R_ARM_MOVW_ABS_NC:
		case R_ARM_MOVT_ABS:
		case R_ARM_MOVW_PREL_NC:
		case R_ARM_MOVT_PREL:
			offset = tmp = __mem_to_opcode_arm(*(u32 *)loc);
			offset = ((offset & 0xf0000) >> 4) | (offset & 0xfff);
			offset = (offset ^ 0x8000) - 0x8000;

			offset += sym->st_value;
			if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL ||
			    ELF32_R_TYPE(rel->r_info) == R_ARM_MOVW_PREL_NC)
				offset -= loc;
			if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_ABS ||
			    ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL)
				offset >>= 16;

			tmp &= 0xfff0f000;
			tmp |= ((offset & 0xf000) << 4) |
				(offset & 0x0fff);

			*(u32 *)loc = __opcode_to_mem_arm(tmp);
			break;

#ifdef CONFIG_ARM_HAS_GROUP_RELOCS
		case R_ARM_ALU_PC_G0_NC:
			group = 0;
			fallthrough;
		case R_ARM_ALU_PC_G1_NC:
			tmp = __mem_to_opcode_arm(*(u32 *)loc);
			offset = ror32(tmp & 0xff, (tmp & 0xf00) >> 7);
			if (tmp & BIT(22))
				offset = -offset;
			offset += sym->st_value - loc;
			if (offset < 0) {
				offset = -offset;
				tmp = (tmp & ~BIT(23)) | BIT(22); // SUB opcode
			} else {
				tmp = (tmp & ~BIT(22)) | BIT(23); // ADD opcode
			}

			shift = get_group_rem(group, &offset);
			if (shift < 24) {
				offset >>= 24 - shift;
				offset |= (shift + 8) << 7;
			}
			*(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset);
			break;

		case R_ARM_LDR_PC_G2:
			tmp = __mem_to_opcode_arm(*(u32 *)loc);
			offset = tmp & 0xfff;
			if (~tmp & BIT(23))		// U bit cleared?
				offset = -offset;
			offset += sym->st_value - loc;
			if (offset < 0) {
				offset = -offset;
				tmp &= ~BIT(23);	// clear U bit
			} else {
				tmp |= BIT(23);		// set U bit
			}
			get_group_rem(2, &offset);

			if (offset > 0xfff) {
				pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
				       module->name, relindex, i, symname,
				       ELF32_R_TYPE(rel->r_info), loc,
				       sym->st_value);
				return -ENOEXEC;
			}
			*(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset);
			break;
#endif
#ifdef CONFIG_THUMB2_KERNEL
		case R_ARM_THM_CALL:
		case R_ARM_THM_JUMP24:
			/*
			 * For function symbols, only Thumb addresses are
			 * allowed (no interworking).
			 *
			 * For non-function symbols, the destination
			 * has no specific ARM/Thumb disposition, so
			 * the branch is resolved under the assumption
			 * that interworking is not required.
			 */
			if (ELF32_ST_TYPE(sym->st_info) == STT_FUNC &&
			    !(sym->st_value & 1)) {
				pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (Thumb -> ARM)\n",
				       module->name, relindex, i, symname);
				return -ENOEXEC;
			}

			upper = __mem_to_opcode_thumb16(*(u16 *)loc);
			lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2));

			/*
			 * 25 bit signed address range (Thumb-2 BL and B.W
			 * instructions):
			 *   S:I1:I2:imm10:imm11:0
			 * where:
			 *   S     = upper[10]   = offset[24]
			 *   I1    = ~(J1 ^ S)   = offset[23]
			 *   I2    = ~(J2 ^ S)   = offset[22]
			 *   imm10 = upper[9:0]  = offset[21:12]
			 *   imm11 = lower[10:0] = offset[11:1]
			 *   J1    = lower[13]
			 *   J2    = lower[11]
			 */
			sign = (upper >> 10) & 1;
			j1 = (lower >> 13) & 1;
			j2 = (lower >> 11) & 1;
			offset = (sign << 24) | ((~(j1 ^ sign) & 1) << 23) |
				((~(j2 ^ sign) & 1) << 22) |
				((upper & 0x03ff) << 12) |
				((lower & 0x07ff) << 1);
			if (offset & 0x01000000)
				offset -= 0x02000000;
			offset += sym->st_value - loc;

			/*
			 * Route through a PLT entry if 'offset' exceeds the
			 * supported range.
			 */
			if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) &&
			    (offset <= (s32)0xff000000 ||
			     offset >= (s32)0x01000000))
				offset = get_module_plt(module, loc,
							offset + loc + 4)
					 - loc - 4;

			if (offset <= (s32)0xff000000 ||
			    offset >= (s32)0x01000000) {
				pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
				       module->name, relindex, i, symname,
				       ELF32_R_TYPE(rel->r_info), loc,
				       sym->st_value);
				return -ENOEXEC;
			}

			sign = (offset >> 24) & 1;
			j1 = sign ^ (~(offset >> 23) & 1);
			j2 = sign ^ (~(offset >> 22) & 1);
			upper = (u16)((upper & 0xf800) | (sign << 10) |
					    ((offset >> 12) & 0x03ff));
			lower = (u16)((lower & 0xd000) |
				      (j1 << 13) | (j2 << 11) |
				      ((offset >> 1) & 0x07ff));

			*(u16 *)loc = __opcode_to_mem_thumb16(upper);
			*(u16 *)(loc + 2) = __opcode_to_mem_thumb16(lower);
			break;

		case R_ARM_THM_MOVW_ABS_NC:
		case R_ARM_THM_MOVT_ABS:
		case R_ARM_THM_MOVW_PREL_NC:
		case R_ARM_THM_MOVT_PREL:
			upper = __mem_to_opcode_thumb16(*(u16 *)loc);
			lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2));

			/*
			 * MOVT/MOVW instructions encoding in Thumb-2:
			 *
			 * i	= upper[10]
			 * imm4	= upper[3:0]
			 * imm3	= lower[14:12]
			 * imm8	= lower[7:0]
			 *
			 * imm16 = imm4:i:imm3:imm8
			 */
			offset = ((upper & 0x000f) << 12) |
				((upper & 0x0400) << 1) |
				((lower & 0x7000) >> 4) | (lower & 0x00ff);
			offset = (offset ^ 0x8000) - 0x8000;
			offset += sym->st_value;

			if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL ||
			    ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVW_PREL_NC)
				offset -= loc;
			if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_ABS ||
			    ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL)
				offset >>= 16;

			upper = (u16)((upper & 0xfbf0) |
				      ((offset & 0xf000) >> 12) |
				      ((offset & 0x0800) >> 1));
			lower = (u16)((lower & 0x8f00) |
				      ((offset & 0x0700) << 4) |
				      (offset & 0x00ff));
			*(u16 *)loc = __opcode_to_mem_thumb16(upper);
			*(u16 *)(loc + 2) = __opcode_to_mem_thumb16(lower);
			break;
#endif

		default:
			pr_err("%s: unknown relocation: %u\n",
			       module->name, ELF32_R_TYPE(rel->r_info));
			return -ENOEXEC;
		}
	}
	return 0;
}

struct mod_unwind_map {
	const Elf_Shdr *unw_sec;
	const Elf_Shdr *txt_sec;
};

static const Elf_Shdr *find_mod_section(const Elf32_Ehdr *hdr,
	const Elf_Shdr *sechdrs, const char *name)
{
	const Elf_Shdr *s, *se;
	const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;

	for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++)
		if (strcmp(name, secstrs + s->sh_name) == 0)
			return s;

	return NULL;
}

extern void fixup_pv_table(const void *, unsigned long);
extern void fixup_smp(const void *, unsigned long);

int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs,
		    struct module *mod)
{
	const Elf_Shdr *s = NULL;
#ifdef CONFIG_ARM_UNWIND
	const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
	const Elf_Shdr *sechdrs_end = sechdrs + hdr->e_shnum;
	struct list_head *unwind_list = &mod->arch.unwind_list;

	INIT_LIST_HEAD(unwind_list);
	mod->arch.init_table = NULL;

	for (s = sechdrs; s < sechdrs_end; s++) {
		const char *secname = secstrs + s->sh_name;
		const char *txtname;
		const Elf_Shdr *txt_sec;

		if (!(s->sh_flags & SHF_ALLOC) ||
		    s->sh_type != ELF_SECTION_UNWIND)
			continue;

		if (!strcmp(".ARM.exidx", secname))
			txtname = ".text";
		else
			txtname = secname + strlen(".ARM.exidx");
		txt_sec = find_mod_section(hdr, sechdrs, txtname);

		if (txt_sec) {
			struct unwind_table *table =
				unwind_table_add(s->sh_addr,
						s->sh_size,
						txt_sec->sh_addr,
						txt_sec->sh_size);

			list_add(&table->mod_list, unwind_list);

			/* save init table for module_arch_freeing_init */
			if (strcmp(".ARM.exidx.init.text", secname) == 0)
				mod->arch.init_table = table;
		}
	}
#endif
#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
	s = find_mod_section(hdr, sechdrs, ".pv_table");
	if (s)
		fixup_pv_table((void *)s->sh_addr, s->sh_size);
#endif
	s = find_mod_section(hdr, sechdrs, ".alt.smp.init");
	if (s && !is_smp())
#ifdef CONFIG_SMP_ON_UP
		fixup_smp((void *)s->sh_addr, s->sh_size);
#else
		return -EINVAL;
#endif
	return 0;
}

void
module_arch_cleanup(struct module *mod)
{
#ifdef CONFIG_ARM_UNWIND
	struct unwind_table *tmp;
	struct unwind_table *n;

	list_for_each_entry_safe(tmp, n,
			&mod->arch.unwind_list, mod_list) {
		list_del(&tmp->mod_list);
		unwind_table_del(tmp);
	}
	mod->arch.init_table = NULL;
#endif
}

void __weak module_arch_freeing_init(struct module *mod)
{
#ifdef CONFIG_ARM_UNWIND
	struct unwind_table *init = mod->arch.init_table;

	if (init) {
		mod->arch.init_table = NULL;
		list_del(&init->mod_list);
		unwind_table_del(init);
	}
#endif
}


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/arch/arm/kernel/opcodes.c
 *
 *  A32 condition code lookup feature moved from nwfpe/fpopcode.c
 */

#include <linux/module.h>
#include <asm/opcodes.h>

#define ARM_OPCODE_CONDITION_UNCOND 0xf

/*
 * condition code lookup table
 * index into the table is test code: EQ, NE, ... LT, GT, AL, NV
 *
 * bit position in short is condition code: NZCV
 */
static const unsigned short cc_map[16] = {
	0xF0F0,			/* EQ == Z set            */
	0x0F0F,			/* NE                     */
	0xCCCC,			/* CS == C set            */
	0x3333,			/* CC                     */
	0xFF00,			/* MI == N set            */
	0x00FF,			/* PL                     */
	0xAAAA,			/* VS == V set            */
	0x5555,			/* VC                     */
	0x0C0C,			/* HI == C set && Z clear */
	0xF3F3,			/* LS == C clear || Z set */
	0xAA55,			/* GE == (N==V)           */
	0x55AA,			/* LT == (N!=V)           */
	0x0A05,			/* GT == (!Z && (N==V))   */
	0xF5FA,			/* LE == (Z || (N!=V))    */
	0xFFFF,			/* AL always              */
	0			/* NV                     */
};

/*
 * Returns:
 * ARM_OPCODE_CONDTEST_FAIL   - if condition fails
 * ARM_OPCODE_CONDTEST_PASS   - if condition passes (including AL)
 * ARM_OPCODE_CONDTEST_UNCOND - if NV condition, or separate unconditional
 *                              opcode space from v5 onwards
 *
 * Code that tests whether a conditional instruction would pass its condition
 * check should check that return value == ARM_OPCODE_CONDTEST_PASS.
 *
 * Code that tests if a condition means that the instruction would be executed
 * (regardless of conditional or unconditional) should instead check that the
 * return value != ARM_OPCODE_CONDTEST_FAIL.
 */
asmlinkage unsigned int arm_check_condition(u32 opcode, u32 psr)
{
	u32 cc_bits  = opcode >> 28;
	u32 psr_cond = psr >> 28;
	unsigned int ret;

	if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
		if ((cc_map[cc_bits] >> (psr_cond)) & 1)
			ret = ARM_OPCODE_CONDTEST_PASS;
		else
			ret = ARM_OPCODE_CONDTEST_FAIL;
	} else {
		ret = ARM_OPCODE_CONDTEST_UNCOND;
	}

	return ret;
}
EXPORT_SYMBOL_GPL(arm_check_condition);


code:
// SPDX-License-Identifier: GPL-2.0-only
/*
 *
 * Copyright (C) 2013 Citrix Systems
 *
 * Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
 */

#include <linux/export.h>
#include <linux/jump_label.h>
#include <linux/types.h>
#include <linux/static_call.h>
#include <asm/paravirt.h>

struct static_key paravirt_steal_enabled;
struct static_key paravirt_steal_rq_enabled;

static u64 native_steal_clock(int cpu)
{
	return 0;
}

DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);


code:
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/spinlock.h>
#include <linux/kprobes.h>
#include <linux/mm.h>
#include <linux/stop_machine.h>

#include <asm/cacheflush.h>
#include <asm/fixmap.h>
#include <asm/smp_plat.h>
#include <asm/opcodes.h>
#include <asm/patch.h>

struct patch {
	void *addr;
	unsigned int insn;
};

#ifdef CONFIG_MMU
static DEFINE_RAW_SPINLOCK(patch_lock);

static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags)
{
	unsigned int uintaddr = (uintptr_t) addr;
	bool module = !core_kernel_text(uintaddr);
	struct page *page;

	if (module && IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
		page = vmalloc_to_page(addr);
	else if (!module && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))
		page = virt_to_page(addr);
	else
		return addr;

	if (flags)
		raw_spin_lock_irqsave(&patch_lock, *flags);

	set_fixmap(fixmap, page_to_phys(page));

	return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
}

static void __kprobes patch_unmap(int fixmap, unsigned long *flags)
{
	clear_fixmap(fixmap);

	if (flags)
		raw_spin_unlock_irqrestore(&patch_lock, *flags);
}
#else
static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags)
{
	return addr;
}
static void __kprobes patch_unmap(int fixmap, unsigned long *flags) { }
#endif

void __kprobes __patch_text_real(void *addr, unsigned int insn, bool remap)
{
	bool thumb2 = IS_ENABLED(CONFIG_THUMB2_KERNEL);
	unsigned int uintaddr = (uintptr_t) addr;
	bool twopage = false;
	unsigned long flags;
	void *waddr = addr;
	int size;

	if (remap)
		waddr = patch_map(addr, FIX_TEXT_POKE0, &flags);

	if (thumb2 && __opcode_is_thumb16(insn)) {
		*(u16 *)waddr = __opcode_to_mem_thumb16(insn);
		size = sizeof(u16);
	} else if (thumb2 && (uintaddr & 2)) {
		u16 first = __opcode_thumb32_first(insn);
		u16 second = __opcode_thumb32_second(insn);
		u16 *addrh0 = waddr;
		u16 *addrh1 = waddr + 2;

		twopage = (uintaddr & ~PAGE_MASK) == PAGE_SIZE - 2;
		if (twopage && remap)
			addrh1 = patch_map(addr + 2, FIX_TEXT_POKE1, NULL);

		*addrh0 = __opcode_to_mem_thumb16(first);
		*addrh1 = __opcode_to_mem_thumb16(second);

		if (twopage && addrh1 != addr + 2) {
			flush_kernel_vmap_range(addrh1, 2);
			patch_unmap(FIX_TEXT_POKE1, NULL);
		}

		size = sizeof(u32);
	} else {
		if (thumb2)
			insn = __opcode_to_mem_thumb32(insn);
		else
			insn = __opcode_to_mem_arm(insn);

		*(u32 *)waddr = insn;
		size = sizeof(u32);
	}

	if (waddr != addr) {
		flush_kernel_vmap_range(waddr, twopage ? size / 2 : size);
		patch_unmap(FIX_TEXT_POKE0, &flags);
	}

	flush_icache_range((uintptr_t)(addr),
			   (uintptr_t)(addr) + size);
}

static int __kprobes patch_text_stop_machine(void *data)
{
	struct patch *patch = data;

	__patch_text(patch->addr, patch->insn);

	return 0;
}

void __kprobes patch_text(void *addr, unsigned int insn)
{
	struct patch patch = {
		.addr = addr,
		.insn = insn,
	};

	stop_machine_cpuslocked(patch_text_stop_machine, &patch, NULL);
}


code:
// SPDX-License-Identifier: GPL-2.0
/*
 * ARM callchain support
 *
 * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
 * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
 *
 * This code is based on the ARM OProfile backtrace code.
 */
#include <linux/perf_event.h>
#include <linux/uaccess.h>

#include <asm/stacktrace.h>

/*
 * The registers we're interested in are at the end of the variable
 * length saved register structure. The fp points at the end of this
 * structure so the address of this struct is:
 * (struct frame_tail *)(xxx->fp)-1
 *
 * This code has been adapted from the ARM OProfile support.
 */
struct frame_tail {
	struct frame_tail __user *fp;
	unsigned long sp;
	unsigned long lr;
} __attribute__((packed));

/*
 * Get the return address for a single stackframe and return a pointer to the
 * next frame tail.
 */
static struct frame_tail __user *
user_backtrace(struct frame_tail __user *tail,
	       struct perf_callchain_entry_ctx *entry)
{
	struct frame_tail buftail;
	unsigned long err;

	if (!access_ok(tail, sizeof(buftail)))
		return NULL;

	pagefault_disable();
	err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
	pagefault_enable();

	if (err)
		return NULL;

	perf_callchain_store(entry, buftail.lr);

	/*
	 * Frame pointers should strictly progress back up the stack
	 * (towards higher addresses).
	 */
	if (tail + 1 >= buftail.fp)
		return NULL;

	return buftail.fp - 1;
}

void
perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
{
	struct frame_tail __user *tail;

	perf_callchain_store(entry, regs->ARM_pc);

	if (!current->mm)
		return;

	tail = (struct frame_tail __user *)regs->ARM_fp - 1;

	while ((entry->nr < entry->max_stack) &&
	       tail && !((unsigned long)tail & 0x3))
		tail = user_backtrace(tail, entry);
}

/*
 * Gets called by walk_stackframe() for every stackframe. This will be called
 * whist unwinding the stackframe and is like a subroutine return so we use
 * the PC.
 */
static bool
callchain_trace(void *data, unsigned long pc)
{
	struct perf_callchain_entry_ctx *entry = data;
	perf_callchain_store(entry, pc);
	return true;
}

void
perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
{
	struct stackframe fr;

	arm_get_current_stackframe(regs, &fr);
	walk_stackframe(&fr, callchain_trace, entry);
}

unsigned long perf_instruction_pointer(struct pt_regs *regs)
{
	return instruction_pointer(regs);
}

unsigned long perf_misc_flags(struct pt_regs *regs)
{
	int misc = 0;

	if (user_mode(regs))
		misc |= PERF_RECORD_MISC_USER;
	else
		misc |= PERF_RECORD_MISC_KERNEL;

	return misc;
}