1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Shadow Call Stack support.
  4 *
  5 * Copyright (C) 2019 Google LLC
  6 */
  7
  8#include <linux/kasan.h>
  9#include <linux/mm.h>
 10#include <linux/scs.h>
 11#include <linux/slab.h>
 12#include <linux/vmstat.h>
 13
 14static struct kmem_cache *scs_cache;
 15
 16static void __scs_account(void *s, int account)
 17{
 18	struct page *scs_page = virt_to_page(s);
 19
 20	mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB,
 21			    account * (SCS_SIZE / SZ_1K));
 22}
 23
 24static void *scs_alloc(int node)
 25{
 26	void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
 27
 28	if (!s)
 29		return NULL;
 30
 31	*__scs_magic(s) = SCS_END_MAGIC;
 32
 33	/*
 34	 * Poison the allocation to catch unintentional accesses to
 35	 * the shadow stack when KASAN is enabled.
 36	 */
 37	kasan_poison_object_data(scs_cache, s);
 38	__scs_account(s, 1);
 39	return s;
 40}
 41
 42static void scs_free(void *s)
 43{
 44	__scs_account(s, -1);
 45	kasan_unpoison_object_data(scs_cache, s);
 46	kmem_cache_free(scs_cache, s);
 47}
 48
 49void __init scs_init(void)
 50{
 51	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL);
 52}
 53
 54int scs_prepare(struct task_struct *tsk, int node)
 55{
 56	void *s = scs_alloc(node);
 57
 58	if (!s)
 59		return -ENOMEM;
 60
 61	task_scs(tsk) = task_scs_sp(tsk) = s;
 62	return 0;
 63}
 64
 65static void scs_check_usage(struct task_struct *tsk)
 66{
 67	static unsigned long highest;
 68
 69	unsigned long *p, prev, curr = highest, used = 0;
 70
 71	if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
 72		return;
 73
 74	for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
 75		if (!READ_ONCE_NOCHECK(*p))
 76			break;
 77		used += sizeof(*p);
 78	}
 79
 80	while (used > curr) {
 81		prev = cmpxchg_relaxed(&highest, curr, used);
 82
 83		if (prev == curr) {
 84			pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
 85				tsk->comm, task_pid_nr(tsk), used);
 86			break;
 87		}
 88
 89		curr = prev;
 90	}
 91}
 92
 93void scs_release(struct task_struct *tsk)
 94{
 95	void *s = task_scs(tsk);
 96
 97	if (!s)
 98		return;
 99
100	WARN(task_scs_end_corrupted(tsk),
101	     "corrupted shadow stack detected when freeing task\n");
102	scs_check_usage(tsk);
103	scs_free(s);
104}