1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2020 SiFive
  4 */
  5
  6#include <linux/spinlock.h>
  7#include <linux/mm.h>
  8#include <linux/memory.h>
  9#include <linux/uaccess.h>
 10#include <linux/stop_machine.h>
 11#include <asm/kprobes.h>
 12#include <asm/cacheflush.h>
 13#include <asm/fixmap.h>
 14#include <asm/patch.h>
 15
 16struct patch_insn {
 17	void *addr;
 18	u32 insn;
 19	atomic_t cpu_count;
 20};
 21
 22#ifdef CONFIG_MMU
 23static void *patch_map(void *addr, int fixmap)
 24{
 25	uintptr_t uintaddr = (uintptr_t) addr;
 26	struct page *page;
 27
 28	if (core_kernel_text(uintaddr))
 29		page = phys_to_page(__pa_symbol(addr));
 30	else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
 31		page = vmalloc_to_page(addr);
 32	else
 33		return addr;
 34
 35	BUG_ON(!page);
 36
 37	return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
 38					 (uintaddr & ~PAGE_MASK));
 39}
 40NOKPROBE_SYMBOL(patch_map);
 41
 42static void patch_unmap(int fixmap)
 43{
 44	clear_fixmap(fixmap);
 45}
 46NOKPROBE_SYMBOL(patch_unmap);
 47
 48static int patch_insn_write(void *addr, const void *insn, size_t len)
 49{
 50	void *waddr = addr;
 51	bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
 52	int ret;
 53
 54	/*
 55	 * Before reaching here, it was expected to lock the text_mutex
 56	 * already, so we don't need to give another lock here and could
 57	 * ensure that it was safe between each cores.
 58	 */
 59	lockdep_assert_held(&text_mutex);
 60
 61	if (across_pages)
 62		patch_map(addr + len, FIX_TEXT_POKE1);
 63
 64	waddr = patch_map(addr, FIX_TEXT_POKE0);
 65
 66	ret = copy_to_kernel_nofault(waddr, insn, len);
 67
 68	patch_unmap(FIX_TEXT_POKE0);
 69
 70	if (across_pages)
 71		patch_unmap(FIX_TEXT_POKE1);
 72
 73	return ret;
 74}
 75NOKPROBE_SYMBOL(patch_insn_write);
 76#else
 77static int patch_insn_write(void *addr, const void *insn, size_t len)
 78{
 79	return copy_to_kernel_nofault(addr, insn, len);
 80}
 81NOKPROBE_SYMBOL(patch_insn_write);
 82#endif /* CONFIG_MMU */
 83
 84int patch_text_nosync(void *addr, const void *insns, size_t len)
 85{
 86	u32 *tp = addr;
 87	int ret;
 88
 89	ret = patch_insn_write(tp, insns, len);
 90
 91	if (!ret)
 92		flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
 93
 94	return ret;
 95}
 96NOKPROBE_SYMBOL(patch_text_nosync);
 97
 98static int patch_text_cb(void *data)
 99{
100	struct patch_insn *patch = data;
101	int ret = 0;
102
103	if (atomic_inc_return(&patch->cpu_count) == 1) {
104		ret =
105		    patch_text_nosync(patch->addr, &patch->insn,
106					    GET_INSN_LENGTH(patch->insn));
107		atomic_inc(&patch->cpu_count);
108	} else {
109		while (atomic_read(&patch->cpu_count) <= num_online_cpus())
110			cpu_relax();
111		smp_mb();
112	}
113
114	return ret;
115}
116NOKPROBE_SYMBOL(patch_text_cb);
117
118int patch_text(void *addr, u32 insn)
119{
120	struct patch_insn patch = {
121		.addr = addr,
122		.insn = insn,
123		.cpu_count = ATOMIC_INIT(0),
124	};
125
126	return stop_machine_cpuslocked(patch_text_cb,
127				       &patch, cpu_online_mask);
128}
129NOKPROBE_SYMBOL(patch_text);