1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2022 - Google LLC
  4 * Author: Ard Biesheuvel <ardb@google.com>
  5 */
  6
  7#include <linux/bug.h>
  8#include <linux/errno.h>
  9#include <linux/init.h>
 10#include <linux/linkage.h>
 11#include <linux/printk.h>
 12#include <linux/types.h>
 13
 14#include <asm/cacheflush.h>
 15#include <asm/scs.h>
 16
 17//
 18// This minimal DWARF CFI parser is partially based on the code in
 19// arch/arc/kernel/unwind.c, and on the document below:
 20// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
 21//
 22
 23#define DW_CFA_nop                          0x00
 24#define DW_CFA_set_loc                      0x01
 25#define DW_CFA_advance_loc1                 0x02
 26#define DW_CFA_advance_loc2                 0x03
 27#define DW_CFA_advance_loc4                 0x04
 28#define DW_CFA_offset_extended              0x05
 29#define DW_CFA_restore_extended             0x06
 30#define DW_CFA_undefined                    0x07
 31#define DW_CFA_same_value                   0x08
 32#define DW_CFA_register                     0x09
 33#define DW_CFA_remember_state               0x0a
 34#define DW_CFA_restore_state                0x0b
 35#define DW_CFA_def_cfa                      0x0c
 36#define DW_CFA_def_cfa_register             0x0d
 37#define DW_CFA_def_cfa_offset               0x0e
 38#define DW_CFA_def_cfa_expression           0x0f
 39#define DW_CFA_expression                   0x10
 40#define DW_CFA_offset_extended_sf           0x11
 41#define DW_CFA_def_cfa_sf                   0x12
 42#define DW_CFA_def_cfa_offset_sf            0x13
 43#define DW_CFA_val_offset                   0x14
 44#define DW_CFA_val_offset_sf                0x15
 45#define DW_CFA_val_expression               0x16
 46#define DW_CFA_lo_user                      0x1c
 47#define DW_CFA_negate_ra_state              0x2d
 48#define DW_CFA_GNU_args_size                0x2e
 49#define DW_CFA_GNU_negative_offset_extended 0x2f
 50#define DW_CFA_hi_user                      0x3f
 51
 52extern const u8 __eh_frame_start[], __eh_frame_end[];
 53
 54enum {
 55	PACIASP		= 0xd503233f,
 56	AUTIASP		= 0xd50323bf,
 57	SCS_PUSH	= 0xf800865e,
 58	SCS_POP		= 0xf85f8e5e,
 59};
 60
 61static void __always_inline scs_patch_loc(u64 loc)
 62{
 63	u32 insn = le32_to_cpup((void *)loc);
 64
 65	switch (insn) {
 66	case PACIASP:
 67		*(u32 *)loc = cpu_to_le32(SCS_PUSH);
 68		break;
 69	case AUTIASP:
 70		*(u32 *)loc = cpu_to_le32(SCS_POP);
 71		break;
 72	default:
 73		/*
 74		 * While the DW_CFA_negate_ra_state directive is guaranteed to
 75		 * appear right after a PACIASP/AUTIASP instruction, it may
 76		 * also appear after a DW_CFA_restore_state directive that
 77		 * restores a state that is only partially accurate, and is
 78		 * followed by DW_CFA_negate_ra_state directive to toggle the
 79		 * PAC bit again. So we permit other instructions here, and ignore
 80		 * them.
 81		 */
 82		return;
 83	}
 84	dcache_clean_pou(loc, loc + sizeof(u32));
 85}
 86
 87/*
 88 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
 89 * except the last one have bit #7 set.
 90 */
 91static int __always_inline skip_xleb128(const u8 **opcode, int size)
 92{
 93	u8 c;
 94
 95	do {
 96		c = *(*opcode)++;
 97		size--;
 98	} while (c & BIT(7));
 99
100	return size;
101}
102
103struct eh_frame {
104	/*
105	 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
106	 */
107	u32	size;
108
109	/*
110	 * The first frame is a Common Information Entry (CIE) frame, followed
111	 * by one or more Frame Description Entry (FDE) frames. In the former
112	 * case, this field is 0, otherwise it is the negated offset relative
113	 * to the associated CIE frame.
114	 */
115	u32	cie_id_or_pointer;
116
117	union {
118		struct { // CIE
119			u8	version;
120			u8	augmentation_string[];
121		};
122
123		struct { // FDE
124			s32	initial_loc;
125			s32	range;
126			u8	opcodes[];
127		};
128	};
129};
130
131static int noinstr scs_handle_fde_frame(const struct eh_frame *frame,
132					bool fde_has_augmentation_data,
133					int code_alignment_factor,
134					bool dry_run)
135{
136	int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
137	u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
138	const u8 *opcode = frame->opcodes;
139
140	if (fde_has_augmentation_data) {
141		int l;
142
143		// assume single byte uleb128_t
144		if (WARN_ON(*opcode & BIT(7)))
145			return -ENOEXEC;
146
147		l = *opcode++;
148		opcode += l;
149		size -= l + 1;
150	}
151
152	/*
153	 * Starting from 'loc', apply the CFA opcodes that advance the location
154	 * pointer, and identify the locations of the PAC instructions.
155	 */
156	while (size-- > 0) {
157		switch (*opcode++) {
158		case DW_CFA_nop:
159		case DW_CFA_remember_state:
160		case DW_CFA_restore_state:
161			break;
162
163		case DW_CFA_advance_loc1:
164			loc += *opcode++ * code_alignment_factor;
165			size--;
166			break;
167
168		case DW_CFA_advance_loc2:
169			loc += *opcode++ * code_alignment_factor;
170			loc += (*opcode++ << 8) * code_alignment_factor;
171			size -= 2;
172			break;
173
174		case DW_CFA_def_cfa:
175		case DW_CFA_offset_extended:
176			size = skip_xleb128(&opcode, size);
177			fallthrough;
178		case DW_CFA_def_cfa_offset:
179		case DW_CFA_def_cfa_offset_sf:
180		case DW_CFA_def_cfa_register:
181		case DW_CFA_same_value:
182		case DW_CFA_restore_extended:
183		case 0x80 ... 0xbf:
184			size = skip_xleb128(&opcode, size);
185			break;
186
187		case DW_CFA_negate_ra_state:
188			if (!dry_run)
189				scs_patch_loc(loc - 4);
190			break;
191
192		case 0x40 ... 0x7f:
193			// advance loc
194			loc += (opcode[-1] & 0x3f) * code_alignment_factor;
195			break;
196
197		case 0xc0 ... 0xff:
198			break;
199
200		default:
201			pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame);
202			return -ENOEXEC;
203		}
204	}
205	return 0;
206}
207
208int noinstr scs_patch(const u8 eh_frame[], int size)
209{
210	const u8 *p = eh_frame;
211
212	while (size > 4) {
213		const struct eh_frame *frame = (const void *)p;
214		bool fde_has_augmentation_data = true;
215		int code_alignment_factor = 1;
216		int ret;
217
218		if (frame->size == 0 ||
219		    frame->size == U32_MAX ||
220		    frame->size > size)
221			break;
222
223		if (frame->cie_id_or_pointer == 0) {
224			const u8 *p = frame->augmentation_string;
225
226			/* a 'z' in the augmentation string must come first */
227			fde_has_augmentation_data = *p == 'z';
228
229			/*
230			 * The code alignment factor is a uleb128 encoded field
231			 * but given that the only sensible values are 1 or 4,
232			 * there is no point in decoding the whole thing.
233			 */
234			p += strlen(p) + 1;
235			if (!WARN_ON(*p & BIT(7)))
236				code_alignment_factor = *p;
237		} else {
238			ret = scs_handle_fde_frame(frame,
239						   fde_has_augmentation_data,
240						   code_alignment_factor,
241						   true);
242			if (ret)
243				return ret;
244			scs_handle_fde_frame(frame, fde_has_augmentation_data,
245					     code_alignment_factor, false);
246		}
247
248		p += sizeof(frame->size) + frame->size;
249		size -= sizeof(frame->size) + frame->size;
250	}
251	return 0;
252}
253
254asmlinkage void __init scs_patch_vmlinux(void)
255{
256	if (!should_patch_pac_into_scs())
257		return;
258
259	WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start));
260	icache_inval_all_pou();
261	isb();
262}