1// SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * functions to patch RO kernel text during runtime
  4  *
  5  * Copyright (c) 2019 Sven Schnelle <svens@stackframe.org>
  6  */
  7
  8#include <linux/kernel.h>
  9#include <linux/spinlock.h>
 10#include <linux/kprobes.h>
 11#include <linux/mm.h>
 12#include <linux/stop_machine.h>
 13
 14#include <asm/cacheflush.h>
 15#include <asm/fixmap.h>
 16#include <asm/patch.h>
 17
 18struct patch {
 19	void *addr;
 20	u32 *insn;
 21	unsigned int len;
 22};
 23
 24static DEFINE_RAW_SPINLOCK(patch_lock);
 25
 26static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags,
 27				 int *need_unmap)
 28{
 29	unsigned long uintaddr = (uintptr_t) addr;
 30	bool module = !core_kernel_text(uintaddr);
 31	struct page *page;
 32
 33	*need_unmap = 0;
 34	if (module && IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
 35		page = vmalloc_to_page(addr);
 36	else if (!module && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))
 37		page = virt_to_page(addr);
 38	else
 39		return addr;
 40
 41	*need_unmap = 1;
 42	set_fixmap(fixmap, page_to_phys(page));
 43	if (flags)
 44		raw_spin_lock_irqsave(&patch_lock, *flags);
 45	else
 46		__acquire(&patch_lock);
 47
 48	return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
 49}
 50
 51static void __kprobes patch_unmap(int fixmap, unsigned long *flags)
 52{
 53	clear_fixmap(fixmap);
 54
 55	if (flags)
 56		raw_spin_unlock_irqrestore(&patch_lock, *flags);
 57	else
 58		__release(&patch_lock);
 59}
 60
 61void __kprobes __patch_text_multiple(void *addr, u32 *insn, unsigned int len)
 62{
 63	unsigned long start = (unsigned long)addr;
 64	unsigned long end = (unsigned long)addr + len;
 65	unsigned long flags;
 66	u32 *p, *fixmap;
 67	int mapped;
 68
 69	/* Make sure we don't have any aliases in cache */
 70	flush_kernel_vmap_range(addr, len);
 71	flush_icache_range(start, end);
 72
 73	p = fixmap = patch_map(addr, FIX_TEXT_POKE0, &flags, &mapped);
 74
 75	while (len >= 4) {
 76		*p++ = *insn++;
 77		addr += sizeof(u32);
 78		len -= sizeof(u32);
 79		if (len && offset_in_page(addr) == 0) {
 80			/*
 81			 * We're crossing a page boundary, so
 82			 * need to remap
 83			 */
 84			flush_kernel_vmap_range((void *)fixmap,
 85						(p-fixmap) * sizeof(*p));
 86			if (mapped)
 87				patch_unmap(FIX_TEXT_POKE0, &flags);
 88			p = fixmap = patch_map(addr, FIX_TEXT_POKE0, &flags,
 89						&mapped);
 90		}
 91	}
 92
 93	flush_kernel_vmap_range((void *)fixmap, (p-fixmap) * sizeof(*p));
 94	if (mapped)
 95		patch_unmap(FIX_TEXT_POKE0, &flags);
 96	flush_icache_range(start, end);
 97}
 98
 99void __kprobes __patch_text(void *addr, u32 insn)
100{
101	__patch_text_multiple(addr, &insn, sizeof(insn));
102}
103
104static int __kprobes patch_text_stop_machine(void *data)
105{
106	struct patch *patch = data;
107
108	__patch_text_multiple(patch->addr, patch->insn, patch->len);
109	return 0;
110}
111
112void __kprobes patch_text(void *addr, unsigned int insn)
113{
114	struct patch patch = {
115		.addr = addr,
116		.insn = &insn,
117		.len = sizeof(insn),
118	};
119
120	stop_machine_cpuslocked(patch_text_stop_machine, &patch, NULL);
121}
122
123void __kprobes patch_text_multiple(void *addr, u32 *insn, unsigned int len)
124{
125
126	struct patch patch = {
127		.addr = addr,
128		.insn = insn,
129		.len = len
130	};
131
132	stop_machine_cpuslocked(patch_text_stop_machine, &patch, NULL);
133}