Loading...
Note: File does not exist in v4.6.
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * sorttable.h
4 *
5 * Added ORC unwind tables sort support and other updates:
6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
7 * Shile Zhang <shile.zhang@linux.alibaba.com>
8 *
9 * Copyright 2011 - 2012 Cavium, Inc.
10 *
11 * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by:
12 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
13 *
14 * Some of this code was taken out of recordmcount.h written by:
15 *
16 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved.
17 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
18 */
19
20#undef extable_ent_size
21#undef compare_extable
22#undef get_mcount_loc
23#undef sort_mcount_loc
24#undef elf_mcount_loc
25#undef do_sort
26#undef Elf_Addr
27#undef Elf_Ehdr
28#undef Elf_Shdr
29#undef Elf_Rel
30#undef Elf_Rela
31#undef Elf_Sym
32#undef ELF_R_SYM
33#undef Elf_r_sym
34#undef ELF_R_INFO
35#undef Elf_r_info
36#undef ELF_ST_BIND
37#undef ELF_ST_TYPE
38#undef fn_ELF_R_SYM
39#undef fn_ELF_R_INFO
40#undef uint_t
41#undef _r
42#undef _w
43
44#ifdef SORTTABLE_64
45# define extable_ent_size 16
46# define compare_extable compare_extable_64
47# define get_mcount_loc get_mcount_loc_64
48# define sort_mcount_loc sort_mcount_loc_64
49# define elf_mcount_loc elf_mcount_loc_64
50# define do_sort do_sort_64
51# define Elf_Addr Elf64_Addr
52# define Elf_Ehdr Elf64_Ehdr
53# define Elf_Shdr Elf64_Shdr
54# define Elf_Rel Elf64_Rel
55# define Elf_Rela Elf64_Rela
56# define Elf_Sym Elf64_Sym
57# define ELF_R_SYM ELF64_R_SYM
58# define Elf_r_sym Elf64_r_sym
59# define ELF_R_INFO ELF64_R_INFO
60# define Elf_r_info Elf64_r_info
61# define ELF_ST_BIND ELF64_ST_BIND
62# define ELF_ST_TYPE ELF64_ST_TYPE
63# define fn_ELF_R_SYM fn_ELF64_R_SYM
64# define fn_ELF_R_INFO fn_ELF64_R_INFO
65# define uint_t uint64_t
66# define _r r8
67# define _w w8
68#else
69# define extable_ent_size 8
70# define compare_extable compare_extable_32
71# define get_mcount_loc get_mcount_loc_32
72# define sort_mcount_loc sort_mcount_loc_32
73# define elf_mcount_loc elf_mcount_loc_32
74# define do_sort do_sort_32
75# define Elf_Addr Elf32_Addr
76# define Elf_Ehdr Elf32_Ehdr
77# define Elf_Shdr Elf32_Shdr
78# define Elf_Rel Elf32_Rel
79# define Elf_Rela Elf32_Rela
80# define Elf_Sym Elf32_Sym
81# define ELF_R_SYM ELF32_R_SYM
82# define Elf_r_sym Elf32_r_sym
83# define ELF_R_INFO ELF32_R_INFO
84# define Elf_r_info Elf32_r_info
85# define ELF_ST_BIND ELF32_ST_BIND
86# define ELF_ST_TYPE ELF32_ST_TYPE
87# define fn_ELF_R_SYM fn_ELF32_R_SYM
88# define fn_ELF_R_INFO fn_ELF32_R_INFO
89# define uint_t uint32_t
90# define _r r
91# define _w w
92#endif
93
94#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
95/* ORC unwinder only support X86_64 */
96#include <asm/orc_types.h>
97
98#define ERRSTR_MAXSZ 256
99
100char g_err[ERRSTR_MAXSZ];
101int *g_orc_ip_table;
102struct orc_entry *g_orc_table;
103
104pthread_t orc_sort_thread;
105
106static inline unsigned long orc_ip(const int *ip)
107{
108 return (unsigned long)ip + *ip;
109}
110
111static int orc_sort_cmp(const void *_a, const void *_b)
112{
113 struct orc_entry *orc_a, *orc_b;
114 const int *a = g_orc_ip_table + *(int *)_a;
115 const int *b = g_orc_ip_table + *(int *)_b;
116 unsigned long a_val = orc_ip(a);
117 unsigned long b_val = orc_ip(b);
118
119 if (a_val > b_val)
120 return 1;
121 if (a_val < b_val)
122 return -1;
123
124 /*
125 * The "weak" section terminator entries need to always be on the left
126 * to ensure the lookup code skips them in favor of real entries.
127 * These terminator entries exist to handle any gaps created by
128 * whitelisted .o files which didn't get objtool generation.
129 */
130 orc_a = g_orc_table + (a - g_orc_ip_table);
131 orc_b = g_orc_table + (b - g_orc_ip_table);
132 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED)
133 return 0;
134 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
135}
136
137static void *sort_orctable(void *arg)
138{
139 int i;
140 int *idxs = NULL;
141 int *tmp_orc_ip_table = NULL;
142 struct orc_entry *tmp_orc_table = NULL;
143 unsigned int *orc_ip_size = (unsigned int *)arg;
144 unsigned int num_entries = *orc_ip_size / sizeof(int);
145 unsigned int orc_size = num_entries * sizeof(struct orc_entry);
146
147 idxs = (int *)malloc(*orc_ip_size);
148 if (!idxs) {
149 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
150 strerror(errno));
151 pthread_exit(g_err);
152 }
153
154 tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
155 if (!tmp_orc_ip_table) {
156 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
157 strerror(errno));
158 pthread_exit(g_err);
159 }
160
161 tmp_orc_table = (struct orc_entry *)malloc(orc_size);
162 if (!tmp_orc_table) {
163 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
164 strerror(errno));
165 pthread_exit(g_err);
166 }
167
168 /* initialize indices array, convert ip_table to absolute address */
169 for (i = 0; i < num_entries; i++) {
170 idxs[i] = i;
171 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
172 }
173 memcpy(tmp_orc_table, g_orc_table, orc_size);
174
175 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
176
177 for (i = 0; i < num_entries; i++) {
178 if (idxs[i] == i)
179 continue;
180
181 /* convert back to relative address */
182 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
183 g_orc_table[i] = tmp_orc_table[idxs[i]];
184 }
185
186 free(idxs);
187 free(tmp_orc_ip_table);
188 free(tmp_orc_table);
189 pthread_exit(NULL);
190}
191#endif
192
193static int compare_extable(const void *a, const void *b)
194{
195 Elf_Addr av = _r(a);
196 Elf_Addr bv = _r(b);
197
198 if (av < bv)
199 return -1;
200 if (av > bv)
201 return 1;
202 return 0;
203}
204#ifdef MCOUNT_SORT_ENABLED
205pthread_t mcount_sort_thread;
206
207struct elf_mcount_loc {
208 Elf_Ehdr *ehdr;
209 Elf_Shdr *init_data_sec;
210 uint_t start_mcount_loc;
211 uint_t stop_mcount_loc;
212};
213
214/* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */
215static void *sort_mcount_loc(void *arg)
216{
217 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg;
218 uint_t offset = emloc->start_mcount_loc - _r(&(emloc->init_data_sec)->sh_addr)
219 + _r(&(emloc->init_data_sec)->sh_offset);
220 uint_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc;
221 unsigned char *start_loc = (void *)emloc->ehdr + offset;
222
223 qsort(start_loc, count/sizeof(uint_t), sizeof(uint_t), compare_extable);
224 return NULL;
225}
226
227/* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */
228static void get_mcount_loc(uint_t *_start, uint_t *_stop)
229{
230 FILE *file_start, *file_stop;
231 char start_buff[20];
232 char stop_buff[20];
233 int len = 0;
234
235 file_start = popen(" grep start_mcount System.map | awk '{print $1}' ", "r");
236 if (!file_start) {
237 fprintf(stderr, "get start_mcount_loc error!");
238 return;
239 }
240
241 file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' ", "r");
242 if (!file_stop) {
243 fprintf(stderr, "get stop_mcount_loc error!");
244 pclose(file_start);
245 return;
246 }
247
248 while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) {
249 len = strlen(start_buff);
250 start_buff[len - 1] = '\0';
251 }
252 *_start = strtoul(start_buff, NULL, 16);
253
254 while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) {
255 len = strlen(stop_buff);
256 stop_buff[len - 1] = '\0';
257 }
258 *_stop = strtoul(stop_buff, NULL, 16);
259
260 pclose(file_start);
261 pclose(file_stop);
262}
263#endif
264static int do_sort(Elf_Ehdr *ehdr,
265 char const *const fname,
266 table_sort_t custom_sort)
267{
268 int rc = -1;
269 Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff));
270 Elf_Shdr *strtab_sec = NULL;
271 Elf_Shdr *symtab_sec = NULL;
272 Elf_Shdr *extab_sec = NULL;
273 Elf_Sym *sym;
274 const Elf_Sym *symtab;
275 Elf32_Word *symtab_shndx = NULL;
276 Elf_Sym *sort_needed_sym = NULL;
277 Elf_Shdr *sort_needed_sec;
278 Elf_Rel *relocs = NULL;
279 int relocs_size = 0;
280 uint32_t *sort_needed_loc;
281 const char *secstrings;
282 const char *strtab;
283 char *extab_image;
284 int extab_index = 0;
285 int i;
286 int idx;
287 unsigned int shnum;
288 unsigned int shstrndx;
289#ifdef MCOUNT_SORT_ENABLED
290 struct elf_mcount_loc mstruct = {0};
291 uint_t _start_mcount_loc = 0;
292 uint_t _stop_mcount_loc = 0;
293#endif
294#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
295 unsigned int orc_ip_size = 0;
296 unsigned int orc_size = 0;
297 unsigned int orc_num_entries = 0;
298#endif
299
300 shstrndx = r2(&ehdr->e_shstrndx);
301 if (shstrndx == SHN_XINDEX)
302 shstrndx = r(&shdr[0].sh_link);
303 secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset);
304
305 shnum = r2(&ehdr->e_shnum);
306 if (shnum == SHN_UNDEF)
307 shnum = _r(&shdr[0].sh_size);
308
309 for (i = 0, s = shdr; s < shdr + shnum; i++, s++) {
310 idx = r(&s->sh_name);
311 if (!strcmp(secstrings + idx, "__ex_table")) {
312 extab_sec = s;
313 extab_index = i;
314 }
315 if (!strcmp(secstrings + idx, ".symtab"))
316 symtab_sec = s;
317 if (!strcmp(secstrings + idx, ".strtab"))
318 strtab_sec = s;
319
320 if ((r(&s->sh_type) == SHT_REL ||
321 r(&s->sh_type) == SHT_RELA) &&
322 r(&s->sh_info) == extab_index) {
323 relocs = (void *)ehdr + _r(&s->sh_offset);
324 relocs_size = _r(&s->sh_size);
325 }
326 if (r(&s->sh_type) == SHT_SYMTAB_SHNDX)
327 symtab_shndx = (Elf32_Word *)((const char *)ehdr +
328 _r(&s->sh_offset));
329
330#ifdef MCOUNT_SORT_ENABLED
331 /* locate the .init.data section in vmlinux */
332 if (!strcmp(secstrings + idx, ".init.data")) {
333 get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc);
334 mstruct.ehdr = ehdr;
335 mstruct.init_data_sec = s;
336 mstruct.start_mcount_loc = _start_mcount_loc;
337 mstruct.stop_mcount_loc = _stop_mcount_loc;
338 }
339#endif
340
341#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
342 /* locate the ORC unwind tables */
343 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
344 orc_ip_size = s->sh_size;
345 g_orc_ip_table = (int *)((void *)ehdr +
346 s->sh_offset);
347 }
348 if (!strcmp(secstrings + idx, ".orc_unwind")) {
349 orc_size = s->sh_size;
350 g_orc_table = (struct orc_entry *)((void *)ehdr +
351 s->sh_offset);
352 }
353#endif
354 } /* for loop */
355
356#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
357 if (!g_orc_ip_table || !g_orc_table) {
358 fprintf(stderr,
359 "incomplete ORC unwind tables in file: %s\n", fname);
360 goto out;
361 }
362
363 orc_num_entries = orc_ip_size / sizeof(int);
364 if (orc_ip_size % sizeof(int) != 0 ||
365 orc_size % sizeof(struct orc_entry) != 0 ||
366 orc_num_entries != orc_size / sizeof(struct orc_entry)) {
367 fprintf(stderr,
368 "inconsistent ORC unwind table entries in file: %s\n",
369 fname);
370 goto out;
371 }
372
373 /* create thread to sort ORC unwind tables concurrently */
374 if (pthread_create(&orc_sort_thread, NULL,
375 sort_orctable, &orc_ip_size)) {
376 fprintf(stderr,
377 "pthread_create orc_sort_thread failed '%s': %s\n",
378 strerror(errno), fname);
379 goto out;
380 }
381#endif
382
383#ifdef MCOUNT_SORT_ENABLED
384 if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) {
385 fprintf(stderr,
386 "incomplete mcount's sort in file: %s\n",
387 fname);
388 goto out;
389 }
390
391 /* create thread to sort mcount_loc concurrently */
392 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) {
393 fprintf(stderr,
394 "pthread_create mcount_sort_thread failed '%s': %s\n",
395 strerror(errno), fname);
396 goto out;
397 }
398#endif
399 if (!extab_sec) {
400 fprintf(stderr, "no __ex_table in file: %s\n", fname);
401 goto out;
402 }
403
404 if (!symtab_sec) {
405 fprintf(stderr, "no .symtab in file: %s\n", fname);
406 goto out;
407 }
408
409 if (!strtab_sec) {
410 fprintf(stderr, "no .strtab in file: %s\n", fname);
411 goto out;
412 }
413
414 extab_image = (void *)ehdr + _r(&extab_sec->sh_offset);
415 strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset);
416 symtab = (const Elf_Sym *)((const char *)ehdr +
417 _r(&symtab_sec->sh_offset));
418
419 if (custom_sort) {
420 custom_sort(extab_image, _r(&extab_sec->sh_size));
421 } else {
422 int num_entries = _r(&extab_sec->sh_size) / extable_ent_size;
423 qsort(extab_image, num_entries,
424 extable_ent_size, compare_extable);
425 }
426
427 /* If there were relocations, we no longer need them. */
428 if (relocs)
429 memset(relocs, 0, relocs_size);
430
431 /* find the flag main_extable_sort_needed */
432 for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset);
433 sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym);
434 sym++) {
435 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT)
436 continue;
437 if (!strcmp(strtab + r(&sym->st_name),
438 "main_extable_sort_needed")) {
439 sort_needed_sym = sym;
440 break;
441 }
442 }
443
444 if (!sort_needed_sym) {
445 fprintf(stderr,
446 "no main_extable_sort_needed symbol in file: %s\n",
447 fname);
448 goto out;
449 }
450
451 sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx),
452 sort_needed_sym - symtab,
453 symtab_shndx)];
454 sort_needed_loc = (void *)ehdr +
455 _r(&sort_needed_sec->sh_offset) +
456 _r(&sort_needed_sym->st_value) -
457 _r(&sort_needed_sec->sh_addr);
458
459 /* extable has been sorted, clear the flag */
460 w(0, sort_needed_loc);
461 rc = 0;
462
463out:
464#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
465 if (orc_sort_thread) {
466 void *retval = NULL;
467 /* wait for ORC tables sort done */
468 rc = pthread_join(orc_sort_thread, &retval);
469 if (rc) {
470 fprintf(stderr,
471 "pthread_join failed '%s': %s\n",
472 strerror(errno), fname);
473 } else if (retval) {
474 rc = -1;
475 fprintf(stderr,
476 "failed to sort ORC tables '%s': %s\n",
477 (char *)retval, fname);
478 }
479 }
480#endif
481
482#ifdef MCOUNT_SORT_ENABLED
483 if (mcount_sort_thread) {
484 void *retval = NULL;
485 /* wait for mcount sort done */
486 rc = pthread_join(mcount_sort_thread, &retval);
487 if (rc) {
488 fprintf(stderr,
489 "pthread_join failed '%s': %s\n",
490 strerror(errno), fname);
491 } else if (retval) {
492 rc = -1;
493 fprintf(stderr,
494 "failed to sort mcount '%s': %s\n",
495 (char *)retval, fname);
496 }
497 }
498#endif
499 return rc;
500}