Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2022 - Google LLC
4 * Author: Ard Biesheuvel <ardb@google.com>
5 */
6
7#include <linux/bug.h>
8#include <linux/errno.h>
9#include <linux/init.h>
10#include <linux/linkage.h>
11#include <linux/printk.h>
12#include <linux/types.h>
13
14#include <asm/cacheflush.h>
15#include <asm/scs.h>
16
17//
18// This minimal DWARF CFI parser is partially based on the code in
19// arch/arc/kernel/unwind.c, and on the document below:
20// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
21//
22
23#define DW_CFA_nop 0x00
24#define DW_CFA_set_loc 0x01
25#define DW_CFA_advance_loc1 0x02
26#define DW_CFA_advance_loc2 0x03
27#define DW_CFA_advance_loc4 0x04
28#define DW_CFA_offset_extended 0x05
29#define DW_CFA_restore_extended 0x06
30#define DW_CFA_undefined 0x07
31#define DW_CFA_same_value 0x08
32#define DW_CFA_register 0x09
33#define DW_CFA_remember_state 0x0a
34#define DW_CFA_restore_state 0x0b
35#define DW_CFA_def_cfa 0x0c
36#define DW_CFA_def_cfa_register 0x0d
37#define DW_CFA_def_cfa_offset 0x0e
38#define DW_CFA_def_cfa_expression 0x0f
39#define DW_CFA_expression 0x10
40#define DW_CFA_offset_extended_sf 0x11
41#define DW_CFA_def_cfa_sf 0x12
42#define DW_CFA_def_cfa_offset_sf 0x13
43#define DW_CFA_val_offset 0x14
44#define DW_CFA_val_offset_sf 0x15
45#define DW_CFA_val_expression 0x16
46#define DW_CFA_lo_user 0x1c
47#define DW_CFA_negate_ra_state 0x2d
48#define DW_CFA_GNU_args_size 0x2e
49#define DW_CFA_GNU_negative_offset_extended 0x2f
50#define DW_CFA_hi_user 0x3f
51
52extern const u8 __eh_frame_start[], __eh_frame_end[];
53
54enum {
55 PACIASP = 0xd503233f,
56 AUTIASP = 0xd50323bf,
57 SCS_PUSH = 0xf800865e,
58 SCS_POP = 0xf85f8e5e,
59};
60
61static void __always_inline scs_patch_loc(u64 loc)
62{
63 u32 insn = le32_to_cpup((void *)loc);
64
65 switch (insn) {
66 case PACIASP:
67 *(u32 *)loc = cpu_to_le32(SCS_PUSH);
68 break;
69 case AUTIASP:
70 *(u32 *)loc = cpu_to_le32(SCS_POP);
71 break;
72 default:
73 /*
74 * While the DW_CFA_negate_ra_state directive is guaranteed to
75 * appear right after a PACIASP/AUTIASP instruction, it may
76 * also appear after a DW_CFA_restore_state directive that
77 * restores a state that is only partially accurate, and is
78 * followed by DW_CFA_negate_ra_state directive to toggle the
79 * PAC bit again. So we permit other instructions here, and ignore
80 * them.
81 */
82 return;
83 }
84 dcache_clean_pou(loc, loc + sizeof(u32));
85}
86
87/*
88 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
89 * except the last one have bit #7 set.
90 */
91static int __always_inline skip_xleb128(const u8 **opcode, int size)
92{
93 u8 c;
94
95 do {
96 c = *(*opcode)++;
97 size--;
98 } while (c & BIT(7));
99
100 return size;
101}
102
103struct eh_frame {
104 /*
105 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
106 */
107 u32 size;
108
109 /*
110 * The first frame is a Common Information Entry (CIE) frame, followed
111 * by one or more Frame Description Entry (FDE) frames. In the former
112 * case, this field is 0, otherwise it is the negated offset relative
113 * to the associated CIE frame.
114 */
115 u32 cie_id_or_pointer;
116
117 union {
118 struct { // CIE
119 u8 version;
120 u8 augmentation_string[];
121 };
122
123 struct { // FDE
124 s32 initial_loc;
125 s32 range;
126 u8 opcodes[];
127 };
128 };
129};
130
131static int noinstr scs_handle_fde_frame(const struct eh_frame *frame,
132 bool fde_has_augmentation_data,
133 int code_alignment_factor)
134{
135 int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
136 u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
137 const u8 *opcode = frame->opcodes;
138
139 if (fde_has_augmentation_data) {
140 int l;
141
142 // assume single byte uleb128_t
143 if (WARN_ON(*opcode & BIT(7)))
144 return -ENOEXEC;
145
146 l = *opcode++;
147 opcode += l;
148 size -= l + 1;
149 }
150
151 /*
152 * Starting from 'loc', apply the CFA opcodes that advance the location
153 * pointer, and identify the locations of the PAC instructions.
154 */
155 while (size-- > 0) {
156 switch (*opcode++) {
157 case DW_CFA_nop:
158 case DW_CFA_remember_state:
159 case DW_CFA_restore_state:
160 break;
161
162 case DW_CFA_advance_loc1:
163 loc += *opcode++ * code_alignment_factor;
164 size--;
165 break;
166
167 case DW_CFA_advance_loc2:
168 loc += *opcode++ * code_alignment_factor;
169 loc += (*opcode++ << 8) * code_alignment_factor;
170 size -= 2;
171 break;
172
173 case DW_CFA_def_cfa:
174 case DW_CFA_offset_extended:
175 size = skip_xleb128(&opcode, size);
176 fallthrough;
177 case DW_CFA_def_cfa_offset:
178 case DW_CFA_def_cfa_offset_sf:
179 case DW_CFA_def_cfa_register:
180 case DW_CFA_same_value:
181 case DW_CFA_restore_extended:
182 case 0x80 ... 0xbf:
183 size = skip_xleb128(&opcode, size);
184 break;
185
186 case DW_CFA_negate_ra_state:
187 scs_patch_loc(loc - 4);
188 break;
189
190 case 0x40 ... 0x7f:
191 // advance loc
192 loc += (opcode[-1] & 0x3f) * code_alignment_factor;
193 break;
194
195 case 0xc0 ... 0xff:
196 break;
197
198 default:
199 pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame);
200 return -ENOEXEC;
201 }
202 }
203 return 0;
204}
205
206int noinstr scs_patch(const u8 eh_frame[], int size)
207{
208 const u8 *p = eh_frame;
209
210 while (size > 4) {
211 const struct eh_frame *frame = (const void *)p;
212 bool fde_has_augmentation_data = true;
213 int code_alignment_factor = 1;
214 int ret;
215
216 if (frame->size == 0 ||
217 frame->size == U32_MAX ||
218 frame->size > size)
219 break;
220
221 if (frame->cie_id_or_pointer == 0) {
222 const u8 *p = frame->augmentation_string;
223
224 /* a 'z' in the augmentation string must come first */
225 fde_has_augmentation_data = *p == 'z';
226
227 /*
228 * The code alignment factor is a uleb128 encoded field
229 * but given that the only sensible values are 1 or 4,
230 * there is no point in decoding the whole thing.
231 */
232 p += strlen(p) + 1;
233 if (!WARN_ON(*p & BIT(7)))
234 code_alignment_factor = *p;
235 } else {
236 ret = scs_handle_fde_frame(frame,
237 fde_has_augmentation_data,
238 code_alignment_factor);
239 if (ret)
240 return ret;
241 }
242
243 p += sizeof(frame->size) + frame->size;
244 size -= sizeof(frame->size) + frame->size;
245 }
246 return 0;
247}
248
249asmlinkage void __init scs_patch_vmlinux(void)
250{
251 if (!should_patch_pac_into_scs())
252 return;
253
254 WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start));
255 icache_inval_all_pou();
256 isb();
257}