Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2021, Microsoft Corporation.
4 *
5 * Authors:
6 * Beau Belgrave <beaub@linux.microsoft.com>
7 */
8
9#include <linux/bitmap.h>
10#include <linux/cdev.h>
11#include <linux/hashtable.h>
12#include <linux/list.h>
13#include <linux/io.h>
14#include <linux/uio.h>
15#include <linux/ioctl.h>
16#include <linux/jhash.h>
17#include <linux/refcount.h>
18#include <linux/trace_events.h>
19#include <linux/tracefs.h>
20#include <linux/types.h>
21#include <linux/uaccess.h>
22/* Reminder to move to uapi when everything works */
23#ifdef CONFIG_COMPILE_TEST
24#include <linux/user_events.h>
25#else
26#include <uapi/linux/user_events.h>
27#endif
28#include "trace.h"
29#include "trace_dynevent.h"
30
31#define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1)
32
33#define FIELD_DEPTH_TYPE 0
34#define FIELD_DEPTH_NAME 1
35#define FIELD_DEPTH_SIZE 2
36
37/*
38 * Limits how many trace_event calls user processes can create:
39 * Must be a power of two of PAGE_SIZE.
40 */
41#define MAX_PAGE_ORDER 0
42#define MAX_PAGES (1 << MAX_PAGE_ORDER)
43#define MAX_BYTES (MAX_PAGES * PAGE_SIZE)
44#define MAX_EVENTS (MAX_BYTES * 8)
45
46/* Limit how long of an event name plus args within the subsystem. */
47#define MAX_EVENT_DESC 512
48#define EVENT_NAME(user_event) ((user_event)->tracepoint.name)
49#define MAX_FIELD_ARRAY_SIZE 1024
50
51/*
52 * The MAP_STATUS_* macros are used for taking a index and determining the
53 * appropriate byte and the bit in the byte to set/reset for an event.
54 *
55 * The lower 3 bits of the index decide which bit to set.
56 * The remaining upper bits of the index decide which byte to use for the bit.
57 *
58 * This is used when an event has a probe attached/removed to reflect live
59 * status of the event wanting tracing or not to user-programs via shared
60 * memory maps.
61 */
62#define MAP_STATUS_BYTE(index) ((index) >> 3)
63#define MAP_STATUS_MASK(index) BIT((index) & 7)
64
65/*
66 * Internal bits (kernel side only) to keep track of connected probes:
67 * These are used when status is requested in text form about an event. These
68 * bits are compared against an internal byte on the event to determine which
69 * probes to print out to the user.
70 *
71 * These do not reflect the mapped bytes between the user and kernel space.
72 */
73#define EVENT_STATUS_FTRACE BIT(0)
74#define EVENT_STATUS_PERF BIT(1)
75#define EVENT_STATUS_OTHER BIT(7)
76
77/*
78 * Stores the pages, tables, and locks for a group of events.
79 * Each logical grouping of events has its own group, with a
80 * matching page for status checks within user programs. This
81 * allows for isolation of events to user programs by various
82 * means.
83 */
84struct user_event_group {
85 struct page *pages;
86 char *register_page_data;
87 char *system_name;
88 struct hlist_node node;
89 struct mutex reg_mutex;
90 DECLARE_HASHTABLE(register_table, 8);
91 DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
92};
93
94/* Group for init_user_ns mapping, top-most group */
95static struct user_event_group *init_group;
96
97/*
98 * Stores per-event properties, as users register events
99 * within a file a user_event might be created if it does not
100 * already exist. These are globally used and their lifetime
101 * is tied to the refcnt member. These cannot go away until the
102 * refcnt reaches one.
103 */
104struct user_event {
105 struct user_event_group *group;
106 struct tracepoint tracepoint;
107 struct trace_event_call call;
108 struct trace_event_class class;
109 struct dyn_event devent;
110 struct hlist_node node;
111 struct list_head fields;
112 struct list_head validators;
113 refcount_t refcnt;
114 int index;
115 int flags;
116 int min_size;
117 char status;
118};
119
120/*
121 * Stores per-file events references, as users register events
122 * within a file this structure is modified and freed via RCU.
123 * The lifetime of this struct is tied to the lifetime of the file.
124 * These are not shared and only accessible by the file that created it.
125 */
126struct user_event_refs {
127 struct rcu_head rcu;
128 int count;
129 struct user_event *events[];
130};
131
132struct user_event_file_info {
133 struct user_event_group *group;
134 struct user_event_refs *refs;
135};
136
137#define VALIDATOR_ENSURE_NULL (1 << 0)
138#define VALIDATOR_REL (1 << 1)
139
140struct user_event_validator {
141 struct list_head link;
142 int offset;
143 int flags;
144};
145
146typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i,
147 void *tpdata, bool *faulted);
148
149static int user_event_parse(struct user_event_group *group, char *name,
150 char *args, char *flags,
151 struct user_event **newuser);
152
153static u32 user_event_key(char *name)
154{
155 return jhash(name, strlen(name), 0);
156}
157
158static void set_page_reservations(char *pages, bool set)
159{
160 int page;
161
162 for (page = 0; page < MAX_PAGES; ++page) {
163 void *addr = pages + (PAGE_SIZE * page);
164
165 if (set)
166 SetPageReserved(virt_to_page(addr));
167 else
168 ClearPageReserved(virt_to_page(addr));
169 }
170}
171
172static void user_event_group_destroy(struct user_event_group *group)
173{
174 if (group->register_page_data)
175 set_page_reservations(group->register_page_data, false);
176
177 if (group->pages)
178 __free_pages(group->pages, MAX_PAGE_ORDER);
179
180 kfree(group->system_name);
181 kfree(group);
182}
183
184static char *user_event_group_system_name(struct user_namespace *user_ns)
185{
186 char *system_name;
187 int len = sizeof(USER_EVENTS_SYSTEM) + 1;
188
189 if (user_ns != &init_user_ns) {
190 /*
191 * Unexpected at this point:
192 * We only currently support init_user_ns.
193 * When we enable more, this will trigger a failure so log.
194 */
195 pr_warn("user_events: Namespace other than init_user_ns!\n");
196 return NULL;
197 }
198
199 system_name = kmalloc(len, GFP_KERNEL);
200
201 if (!system_name)
202 return NULL;
203
204 snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM);
205
206 return system_name;
207}
208
209static inline struct user_event_group
210*user_event_group_from_user_ns(struct user_namespace *user_ns)
211{
212 if (user_ns == &init_user_ns)
213 return init_group;
214
215 return NULL;
216}
217
218static struct user_event_group *current_user_event_group(void)
219{
220 struct user_namespace *user_ns = current_user_ns();
221 struct user_event_group *group = NULL;
222
223 while (user_ns) {
224 group = user_event_group_from_user_ns(user_ns);
225
226 if (group)
227 break;
228
229 user_ns = user_ns->parent;
230 }
231
232 return group;
233}
234
235static struct user_event_group
236*user_event_group_create(struct user_namespace *user_ns)
237{
238 struct user_event_group *group;
239
240 group = kzalloc(sizeof(*group), GFP_KERNEL);
241
242 if (!group)
243 return NULL;
244
245 group->system_name = user_event_group_system_name(user_ns);
246
247 if (!group->system_name)
248 goto error;
249
250 group->pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER);
251
252 if (!group->pages)
253 goto error;
254
255 group->register_page_data = page_address(group->pages);
256
257 set_page_reservations(group->register_page_data, true);
258
259 /* Zero all bits beside 0 (which is reserved for failures) */
260 bitmap_zero(group->page_bitmap, MAX_EVENTS);
261 set_bit(0, group->page_bitmap);
262
263 mutex_init(&group->reg_mutex);
264 hash_init(group->register_table);
265
266 return group;
267error:
268 if (group)
269 user_event_group_destroy(group);
270
271 return NULL;
272};
273
274static __always_inline
275void user_event_register_set(struct user_event *user)
276{
277 int i = user->index;
278
279 user->group->register_page_data[MAP_STATUS_BYTE(i)] |= MAP_STATUS_MASK(i);
280}
281
282static __always_inline
283void user_event_register_clear(struct user_event *user)
284{
285 int i = user->index;
286
287 user->group->register_page_data[MAP_STATUS_BYTE(i)] &= ~MAP_STATUS_MASK(i);
288}
289
290static __always_inline __must_check
291bool user_event_last_ref(struct user_event *user)
292{
293 return refcount_read(&user->refcnt) == 1;
294}
295
296static __always_inline __must_check
297size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i)
298{
299 size_t ret;
300
301 pagefault_disable();
302
303 ret = copy_from_iter_nocache(addr, bytes, i);
304
305 pagefault_enable();
306
307 return ret;
308}
309
310static struct list_head *user_event_get_fields(struct trace_event_call *call)
311{
312 struct user_event *user = (struct user_event *)call->data;
313
314 return &user->fields;
315}
316
317/*
318 * Parses a register command for user_events
319 * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]]
320 *
321 * Example event named 'test' with a 20 char 'msg' field with an unsigned int
322 * 'id' field after:
323 * test char[20] msg;unsigned int id
324 *
325 * NOTE: Offsets are from the user data perspective, they are not from the
326 * trace_entry/buffer perspective. We automatically add the common properties
327 * sizes to the offset for the user.
328 *
329 * Upon success user_event has its ref count increased by 1.
330 */
331static int user_event_parse_cmd(struct user_event_group *group,
332 char *raw_command, struct user_event **newuser)
333{
334 char *name = raw_command;
335 char *args = strpbrk(name, " ");
336 char *flags;
337
338 if (args)
339 *args++ = '\0';
340
341 flags = strpbrk(name, ":");
342
343 if (flags)
344 *flags++ = '\0';
345
346 return user_event_parse(group, name, args, flags, newuser);
347}
348
349static int user_field_array_size(const char *type)
350{
351 const char *start = strchr(type, '[');
352 char val[8];
353 char *bracket;
354 int size = 0;
355
356 if (start == NULL)
357 return -EINVAL;
358
359 if (strscpy(val, start + 1, sizeof(val)) <= 0)
360 return -EINVAL;
361
362 bracket = strchr(val, ']');
363
364 if (!bracket)
365 return -EINVAL;
366
367 *bracket = '\0';
368
369 if (kstrtouint(val, 0, &size))
370 return -EINVAL;
371
372 if (size > MAX_FIELD_ARRAY_SIZE)
373 return -EINVAL;
374
375 return size;
376}
377
378static int user_field_size(const char *type)
379{
380 /* long is not allowed from a user, since it's ambigious in size */
381 if (strcmp(type, "s64") == 0)
382 return sizeof(s64);
383 if (strcmp(type, "u64") == 0)
384 return sizeof(u64);
385 if (strcmp(type, "s32") == 0)
386 return sizeof(s32);
387 if (strcmp(type, "u32") == 0)
388 return sizeof(u32);
389 if (strcmp(type, "int") == 0)
390 return sizeof(int);
391 if (strcmp(type, "unsigned int") == 0)
392 return sizeof(unsigned int);
393 if (strcmp(type, "s16") == 0)
394 return sizeof(s16);
395 if (strcmp(type, "u16") == 0)
396 return sizeof(u16);
397 if (strcmp(type, "short") == 0)
398 return sizeof(short);
399 if (strcmp(type, "unsigned short") == 0)
400 return sizeof(unsigned short);
401 if (strcmp(type, "s8") == 0)
402 return sizeof(s8);
403 if (strcmp(type, "u8") == 0)
404 return sizeof(u8);
405 if (strcmp(type, "char") == 0)
406 return sizeof(char);
407 if (strcmp(type, "unsigned char") == 0)
408 return sizeof(unsigned char);
409 if (str_has_prefix(type, "char["))
410 return user_field_array_size(type);
411 if (str_has_prefix(type, "unsigned char["))
412 return user_field_array_size(type);
413 if (str_has_prefix(type, "__data_loc "))
414 return sizeof(u32);
415 if (str_has_prefix(type, "__rel_loc "))
416 return sizeof(u32);
417
418 /* Uknown basic type, error */
419 return -EINVAL;
420}
421
422static void user_event_destroy_validators(struct user_event *user)
423{
424 struct user_event_validator *validator, *next;
425 struct list_head *head = &user->validators;
426
427 list_for_each_entry_safe(validator, next, head, link) {
428 list_del(&validator->link);
429 kfree(validator);
430 }
431}
432
433static void user_event_destroy_fields(struct user_event *user)
434{
435 struct ftrace_event_field *field, *next;
436 struct list_head *head = &user->fields;
437
438 list_for_each_entry_safe(field, next, head, link) {
439 list_del(&field->link);
440 kfree(field);
441 }
442}
443
444static int user_event_add_field(struct user_event *user, const char *type,
445 const char *name, int offset, int size,
446 int is_signed, int filter_type)
447{
448 struct user_event_validator *validator;
449 struct ftrace_event_field *field;
450 int validator_flags = 0;
451
452 field = kmalloc(sizeof(*field), GFP_KERNEL);
453
454 if (!field)
455 return -ENOMEM;
456
457 if (str_has_prefix(type, "__data_loc "))
458 goto add_validator;
459
460 if (str_has_prefix(type, "__rel_loc ")) {
461 validator_flags |= VALIDATOR_REL;
462 goto add_validator;
463 }
464
465 goto add_field;
466
467add_validator:
468 if (strstr(type, "char") != NULL)
469 validator_flags |= VALIDATOR_ENSURE_NULL;
470
471 validator = kmalloc(sizeof(*validator), GFP_KERNEL);
472
473 if (!validator) {
474 kfree(field);
475 return -ENOMEM;
476 }
477
478 validator->flags = validator_flags;
479 validator->offset = offset;
480
481 /* Want sequential access when validating */
482 list_add_tail(&validator->link, &user->validators);
483
484add_field:
485 field->type = type;
486 field->name = name;
487 field->offset = offset;
488 field->size = size;
489 field->is_signed = is_signed;
490 field->filter_type = filter_type;
491
492 list_add(&field->link, &user->fields);
493
494 /*
495 * Min size from user writes that are required, this does not include
496 * the size of trace_entry (common fields).
497 */
498 user->min_size = (offset + size) - sizeof(struct trace_entry);
499
500 return 0;
501}
502
503/*
504 * Parses the values of a field within the description
505 * Format: type name [size]
506 */
507static int user_event_parse_field(char *field, struct user_event *user,
508 u32 *offset)
509{
510 char *part, *type, *name;
511 u32 depth = 0, saved_offset = *offset;
512 int len, size = -EINVAL;
513 bool is_struct = false;
514
515 field = skip_spaces(field);
516
517 if (*field == '\0')
518 return 0;
519
520 /* Handle types that have a space within */
521 len = str_has_prefix(field, "unsigned ");
522 if (len)
523 goto skip_next;
524
525 len = str_has_prefix(field, "struct ");
526 if (len) {
527 is_struct = true;
528 goto skip_next;
529 }
530
531 len = str_has_prefix(field, "__data_loc unsigned ");
532 if (len)
533 goto skip_next;
534
535 len = str_has_prefix(field, "__data_loc ");
536 if (len)
537 goto skip_next;
538
539 len = str_has_prefix(field, "__rel_loc unsigned ");
540 if (len)
541 goto skip_next;
542
543 len = str_has_prefix(field, "__rel_loc ");
544 if (len)
545 goto skip_next;
546
547 goto parse;
548skip_next:
549 type = field;
550 field = strpbrk(field + len, " ");
551
552 if (field == NULL)
553 return -EINVAL;
554
555 *field++ = '\0';
556 depth++;
557parse:
558 name = NULL;
559
560 while ((part = strsep(&field, " ")) != NULL) {
561 switch (depth++) {
562 case FIELD_DEPTH_TYPE:
563 type = part;
564 break;
565 case FIELD_DEPTH_NAME:
566 name = part;
567 break;
568 case FIELD_DEPTH_SIZE:
569 if (!is_struct)
570 return -EINVAL;
571
572 if (kstrtou32(part, 10, &size))
573 return -EINVAL;
574 break;
575 default:
576 return -EINVAL;
577 }
578 }
579
580 if (depth < FIELD_DEPTH_SIZE || !name)
581 return -EINVAL;
582
583 if (depth == FIELD_DEPTH_SIZE)
584 size = user_field_size(type);
585
586 if (size == 0)
587 return -EINVAL;
588
589 if (size < 0)
590 return size;
591
592 *offset = saved_offset + size;
593
594 return user_event_add_field(user, type, name, saved_offset, size,
595 type[0] != 'u', FILTER_OTHER);
596}
597
598static int user_event_parse_fields(struct user_event *user, char *args)
599{
600 char *field;
601 u32 offset = sizeof(struct trace_entry);
602 int ret = -EINVAL;
603
604 if (args == NULL)
605 return 0;
606
607 while ((field = strsep(&args, ";")) != NULL) {
608 ret = user_event_parse_field(field, user, &offset);
609
610 if (ret)
611 break;
612 }
613
614 return ret;
615}
616
617static struct trace_event_fields user_event_fields_array[1];
618
619static const char *user_field_format(const char *type)
620{
621 if (strcmp(type, "s64") == 0)
622 return "%lld";
623 if (strcmp(type, "u64") == 0)
624 return "%llu";
625 if (strcmp(type, "s32") == 0)
626 return "%d";
627 if (strcmp(type, "u32") == 0)
628 return "%u";
629 if (strcmp(type, "int") == 0)
630 return "%d";
631 if (strcmp(type, "unsigned int") == 0)
632 return "%u";
633 if (strcmp(type, "s16") == 0)
634 return "%d";
635 if (strcmp(type, "u16") == 0)
636 return "%u";
637 if (strcmp(type, "short") == 0)
638 return "%d";
639 if (strcmp(type, "unsigned short") == 0)
640 return "%u";
641 if (strcmp(type, "s8") == 0)
642 return "%d";
643 if (strcmp(type, "u8") == 0)
644 return "%u";
645 if (strcmp(type, "char") == 0)
646 return "%d";
647 if (strcmp(type, "unsigned char") == 0)
648 return "%u";
649 if (strstr(type, "char[") != NULL)
650 return "%s";
651
652 /* Unknown, likely struct, allowed treat as 64-bit */
653 return "%llu";
654}
655
656static bool user_field_is_dyn_string(const char *type, const char **str_func)
657{
658 if (str_has_prefix(type, "__data_loc ")) {
659 *str_func = "__get_str";
660 goto check;
661 }
662
663 if (str_has_prefix(type, "__rel_loc ")) {
664 *str_func = "__get_rel_str";
665 goto check;
666 }
667
668 return false;
669check:
670 return strstr(type, "char") != NULL;
671}
672
673#define LEN_OR_ZERO (len ? len - pos : 0)
674static int user_dyn_field_set_string(int argc, const char **argv, int *iout,
675 char *buf, int len, bool *colon)
676{
677 int pos = 0, i = *iout;
678
679 *colon = false;
680
681 for (; i < argc; ++i) {
682 if (i != *iout)
683 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
684
685 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", argv[i]);
686
687 if (strchr(argv[i], ';')) {
688 ++i;
689 *colon = true;
690 break;
691 }
692 }
693
694 /* Actual set, advance i */
695 if (len != 0)
696 *iout = i;
697
698 return pos + 1;
699}
700
701static int user_field_set_string(struct ftrace_event_field *field,
702 char *buf, int len, bool colon)
703{
704 int pos = 0;
705
706 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->type);
707 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
708 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->name);
709
710 if (colon)
711 pos += snprintf(buf + pos, LEN_OR_ZERO, ";");
712
713 return pos + 1;
714}
715
716static int user_event_set_print_fmt(struct user_event *user, char *buf, int len)
717{
718 struct ftrace_event_field *field, *next;
719 struct list_head *head = &user->fields;
720 int pos = 0, depth = 0;
721 const char *str_func;
722
723 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
724
725 list_for_each_entry_safe_reverse(field, next, head, link) {
726 if (depth != 0)
727 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
728
729 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s",
730 field->name, user_field_format(field->type));
731
732 depth++;
733 }
734
735 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
736
737 list_for_each_entry_safe_reverse(field, next, head, link) {
738 if (user_field_is_dyn_string(field->type, &str_func))
739 pos += snprintf(buf + pos, LEN_OR_ZERO,
740 ", %s(%s)", str_func, field->name);
741 else
742 pos += snprintf(buf + pos, LEN_OR_ZERO,
743 ", REC->%s", field->name);
744 }
745
746 return pos + 1;
747}
748#undef LEN_OR_ZERO
749
750static int user_event_create_print_fmt(struct user_event *user)
751{
752 char *print_fmt;
753 int len;
754
755 len = user_event_set_print_fmt(user, NULL, 0);
756
757 print_fmt = kmalloc(len, GFP_KERNEL);
758
759 if (!print_fmt)
760 return -ENOMEM;
761
762 user_event_set_print_fmt(user, print_fmt, len);
763
764 user->call.print_fmt = print_fmt;
765
766 return 0;
767}
768
769static enum print_line_t user_event_print_trace(struct trace_iterator *iter,
770 int flags,
771 struct trace_event *event)
772{
773 /* Unsafe to try to decode user provided print_fmt, use hex */
774 trace_print_hex_dump_seq(&iter->seq, "", DUMP_PREFIX_OFFSET, 16,
775 1, iter->ent, iter->ent_size, true);
776
777 return trace_handle_return(&iter->seq);
778}
779
780static struct trace_event_functions user_event_funcs = {
781 .trace = user_event_print_trace,
782};
783
784static int user_event_set_call_visible(struct user_event *user, bool visible)
785{
786 int ret;
787 const struct cred *old_cred;
788 struct cred *cred;
789
790 cred = prepare_creds();
791
792 if (!cred)
793 return -ENOMEM;
794
795 /*
796 * While by default tracefs is locked down, systems can be configured
797 * to allow user_event files to be less locked down. The extreme case
798 * being "other" has read/write access to user_events_data/status.
799 *
800 * When not locked down, processes may not have permissions to
801 * add/remove calls themselves to tracefs. We need to temporarily
802 * switch to root file permission to allow for this scenario.
803 */
804 cred->fsuid = GLOBAL_ROOT_UID;
805
806 old_cred = override_creds(cred);
807
808 if (visible)
809 ret = trace_add_event_call(&user->call);
810 else
811 ret = trace_remove_event_call(&user->call);
812
813 revert_creds(old_cred);
814 put_cred(cred);
815
816 return ret;
817}
818
819static int destroy_user_event(struct user_event *user)
820{
821 int ret = 0;
822
823 /* Must destroy fields before call removal */
824 user_event_destroy_fields(user);
825
826 ret = user_event_set_call_visible(user, false);
827
828 if (ret)
829 return ret;
830
831 dyn_event_remove(&user->devent);
832
833 user_event_register_clear(user);
834 clear_bit(user->index, user->group->page_bitmap);
835 hash_del(&user->node);
836
837 user_event_destroy_validators(user);
838 kfree(user->call.print_fmt);
839 kfree(EVENT_NAME(user));
840 kfree(user);
841
842 return ret;
843}
844
845static struct user_event *find_user_event(struct user_event_group *group,
846 char *name, u32 *outkey)
847{
848 struct user_event *user;
849 u32 key = user_event_key(name);
850
851 *outkey = key;
852
853 hash_for_each_possible(group->register_table, user, node, key)
854 if (!strcmp(EVENT_NAME(user), name)) {
855 refcount_inc(&user->refcnt);
856 return user;
857 }
858
859 return NULL;
860}
861
862static int user_event_validate(struct user_event *user, void *data, int len)
863{
864 struct list_head *head = &user->validators;
865 struct user_event_validator *validator;
866 void *pos, *end = data + len;
867 u32 loc, offset, size;
868
869 list_for_each_entry(validator, head, link) {
870 pos = data + validator->offset;
871
872 /* Already done min_size check, no bounds check here */
873 loc = *(u32 *)pos;
874 offset = loc & 0xffff;
875 size = loc >> 16;
876
877 if (likely(validator->flags & VALIDATOR_REL))
878 pos += offset + sizeof(loc);
879 else
880 pos = data + offset;
881
882 pos += size;
883
884 if (unlikely(pos > end))
885 return -EFAULT;
886
887 if (likely(validator->flags & VALIDATOR_ENSURE_NULL))
888 if (unlikely(*(char *)(pos - 1) != '\0'))
889 return -EFAULT;
890 }
891
892 return 0;
893}
894
895/*
896 * Writes the user supplied payload out to a trace file.
897 */
898static void user_event_ftrace(struct user_event *user, struct iov_iter *i,
899 void *tpdata, bool *faulted)
900{
901 struct trace_event_file *file;
902 struct trace_entry *entry;
903 struct trace_event_buffer event_buffer;
904 size_t size = sizeof(*entry) + i->count;
905
906 file = (struct trace_event_file *)tpdata;
907
908 if (!file ||
909 !(file->flags & EVENT_FILE_FL_ENABLED) ||
910 trace_trigger_soft_disabled(file))
911 return;
912
913 /* Allocates and fills trace_entry, + 1 of this is data payload */
914 entry = trace_event_buffer_reserve(&event_buffer, file, size);
915
916 if (unlikely(!entry))
917 return;
918
919 if (unlikely(!copy_nofault(entry + 1, i->count, i)))
920 goto discard;
921
922 if (!list_empty(&user->validators) &&
923 unlikely(user_event_validate(user, entry, size)))
924 goto discard;
925
926 trace_event_buffer_commit(&event_buffer);
927
928 return;
929discard:
930 *faulted = true;
931 __trace_event_discard_commit(event_buffer.buffer,
932 event_buffer.event);
933}
934
935#ifdef CONFIG_PERF_EVENTS
936/*
937 * Writes the user supplied payload out to perf ring buffer.
938 */
939static void user_event_perf(struct user_event *user, struct iov_iter *i,
940 void *tpdata, bool *faulted)
941{
942 struct hlist_head *perf_head;
943
944 perf_head = this_cpu_ptr(user->call.perf_events);
945
946 if (perf_head && !hlist_empty(perf_head)) {
947 struct trace_entry *perf_entry;
948 struct pt_regs *regs;
949 size_t size = sizeof(*perf_entry) + i->count;
950 int context;
951
952 perf_entry = perf_trace_buf_alloc(ALIGN(size, 8),
953 ®s, &context);
954
955 if (unlikely(!perf_entry))
956 return;
957
958 perf_fetch_caller_regs(regs);
959
960 if (unlikely(!copy_nofault(perf_entry + 1, i->count, i)))
961 goto discard;
962
963 if (!list_empty(&user->validators) &&
964 unlikely(user_event_validate(user, perf_entry, size)))
965 goto discard;
966
967 perf_trace_buf_submit(perf_entry, size, context,
968 user->call.event.type, 1, regs,
969 perf_head, NULL);
970
971 return;
972discard:
973 *faulted = true;
974 perf_swevent_put_recursion_context(context);
975 }
976}
977#endif
978
979/*
980 * Update the register page that is shared between user processes.
981 */
982static void update_reg_page_for(struct user_event *user)
983{
984 struct tracepoint *tp = &user->tracepoint;
985 char status = 0;
986
987 if (atomic_read(&tp->key.enabled) > 0) {
988 struct tracepoint_func *probe_func_ptr;
989 user_event_func_t probe_func;
990
991 rcu_read_lock_sched();
992
993 probe_func_ptr = rcu_dereference_sched(tp->funcs);
994
995 if (probe_func_ptr) {
996 do {
997 probe_func = probe_func_ptr->func;
998
999 if (probe_func == user_event_ftrace)
1000 status |= EVENT_STATUS_FTRACE;
1001#ifdef CONFIG_PERF_EVENTS
1002 else if (probe_func == user_event_perf)
1003 status |= EVENT_STATUS_PERF;
1004#endif
1005 else
1006 status |= EVENT_STATUS_OTHER;
1007 } while ((++probe_func_ptr)->func);
1008 }
1009
1010 rcu_read_unlock_sched();
1011 }
1012
1013 if (status)
1014 user_event_register_set(user);
1015 else
1016 user_event_register_clear(user);
1017
1018 user->status = status;
1019}
1020
1021/*
1022 * Register callback for our events from tracing sub-systems.
1023 */
1024static int user_event_reg(struct trace_event_call *call,
1025 enum trace_reg type,
1026 void *data)
1027{
1028 struct user_event *user = (struct user_event *)call->data;
1029 int ret = 0;
1030
1031 if (!user)
1032 return -ENOENT;
1033
1034 switch (type) {
1035 case TRACE_REG_REGISTER:
1036 ret = tracepoint_probe_register(call->tp,
1037 call->class->probe,
1038 data);
1039 if (!ret)
1040 goto inc;
1041 break;
1042
1043 case TRACE_REG_UNREGISTER:
1044 tracepoint_probe_unregister(call->tp,
1045 call->class->probe,
1046 data);
1047 goto dec;
1048
1049#ifdef CONFIG_PERF_EVENTS
1050 case TRACE_REG_PERF_REGISTER:
1051 ret = tracepoint_probe_register(call->tp,
1052 call->class->perf_probe,
1053 data);
1054 if (!ret)
1055 goto inc;
1056 break;
1057
1058 case TRACE_REG_PERF_UNREGISTER:
1059 tracepoint_probe_unregister(call->tp,
1060 call->class->perf_probe,
1061 data);
1062 goto dec;
1063
1064 case TRACE_REG_PERF_OPEN:
1065 case TRACE_REG_PERF_CLOSE:
1066 case TRACE_REG_PERF_ADD:
1067 case TRACE_REG_PERF_DEL:
1068 break;
1069#endif
1070 }
1071
1072 return ret;
1073inc:
1074 refcount_inc(&user->refcnt);
1075 update_reg_page_for(user);
1076 return 0;
1077dec:
1078 update_reg_page_for(user);
1079 refcount_dec(&user->refcnt);
1080 return 0;
1081}
1082
1083static int user_event_create(const char *raw_command)
1084{
1085 struct user_event_group *group;
1086 struct user_event *user;
1087 char *name;
1088 int ret;
1089
1090 if (!str_has_prefix(raw_command, USER_EVENTS_PREFIX))
1091 return -ECANCELED;
1092
1093 raw_command += USER_EVENTS_PREFIX_LEN;
1094 raw_command = skip_spaces(raw_command);
1095
1096 name = kstrdup(raw_command, GFP_KERNEL);
1097
1098 if (!name)
1099 return -ENOMEM;
1100
1101 group = current_user_event_group();
1102
1103 if (!group) {
1104 kfree(name);
1105 return -ENOENT;
1106 }
1107
1108 mutex_lock(&group->reg_mutex);
1109
1110 ret = user_event_parse_cmd(group, name, &user);
1111
1112 if (!ret)
1113 refcount_dec(&user->refcnt);
1114
1115 mutex_unlock(&group->reg_mutex);
1116
1117 if (ret)
1118 kfree(name);
1119
1120 return ret;
1121}
1122
1123static int user_event_show(struct seq_file *m, struct dyn_event *ev)
1124{
1125 struct user_event *user = container_of(ev, struct user_event, devent);
1126 struct ftrace_event_field *field, *next;
1127 struct list_head *head;
1128 int depth = 0;
1129
1130 seq_printf(m, "%s%s", USER_EVENTS_PREFIX, EVENT_NAME(user));
1131
1132 head = trace_get_fields(&user->call);
1133
1134 list_for_each_entry_safe_reverse(field, next, head, link) {
1135 if (depth == 0)
1136 seq_puts(m, " ");
1137 else
1138 seq_puts(m, "; ");
1139
1140 seq_printf(m, "%s %s", field->type, field->name);
1141
1142 if (str_has_prefix(field->type, "struct "))
1143 seq_printf(m, " %d", field->size);
1144
1145 depth++;
1146 }
1147
1148 seq_puts(m, "\n");
1149
1150 return 0;
1151}
1152
1153static bool user_event_is_busy(struct dyn_event *ev)
1154{
1155 struct user_event *user = container_of(ev, struct user_event, devent);
1156
1157 return !user_event_last_ref(user);
1158}
1159
1160static int user_event_free(struct dyn_event *ev)
1161{
1162 struct user_event *user = container_of(ev, struct user_event, devent);
1163
1164 if (!user_event_last_ref(user))
1165 return -EBUSY;
1166
1167 return destroy_user_event(user);
1168}
1169
1170static bool user_field_match(struct ftrace_event_field *field, int argc,
1171 const char **argv, int *iout)
1172{
1173 char *field_name = NULL, *dyn_field_name = NULL;
1174 bool colon = false, match = false;
1175 int dyn_len, len;
1176
1177 if (*iout >= argc)
1178 return false;
1179
1180 dyn_len = user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1181 0, &colon);
1182
1183 len = user_field_set_string(field, field_name, 0, colon);
1184
1185 if (dyn_len != len)
1186 return false;
1187
1188 dyn_field_name = kmalloc(dyn_len, GFP_KERNEL);
1189 field_name = kmalloc(len, GFP_KERNEL);
1190
1191 if (!dyn_field_name || !field_name)
1192 goto out;
1193
1194 user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1195 dyn_len, &colon);
1196
1197 user_field_set_string(field, field_name, len, colon);
1198
1199 match = strcmp(dyn_field_name, field_name) == 0;
1200out:
1201 kfree(dyn_field_name);
1202 kfree(field_name);
1203
1204 return match;
1205}
1206
1207static bool user_fields_match(struct user_event *user, int argc,
1208 const char **argv)
1209{
1210 struct ftrace_event_field *field, *next;
1211 struct list_head *head = &user->fields;
1212 int i = 0;
1213
1214 list_for_each_entry_safe_reverse(field, next, head, link)
1215 if (!user_field_match(field, argc, argv, &i))
1216 return false;
1217
1218 if (i != argc)
1219 return false;
1220
1221 return true;
1222}
1223
1224static bool user_event_match(const char *system, const char *event,
1225 int argc, const char **argv, struct dyn_event *ev)
1226{
1227 struct user_event *user = container_of(ev, struct user_event, devent);
1228 bool match;
1229
1230 match = strcmp(EVENT_NAME(user), event) == 0 &&
1231 (!system || strcmp(system, USER_EVENTS_SYSTEM) == 0);
1232
1233 if (match && argc > 0)
1234 match = user_fields_match(user, argc, argv);
1235
1236 return match;
1237}
1238
1239static struct dyn_event_operations user_event_dops = {
1240 .create = user_event_create,
1241 .show = user_event_show,
1242 .is_busy = user_event_is_busy,
1243 .free = user_event_free,
1244 .match = user_event_match,
1245};
1246
1247static int user_event_trace_register(struct user_event *user)
1248{
1249 int ret;
1250
1251 ret = register_trace_event(&user->call.event);
1252
1253 if (!ret)
1254 return -ENODEV;
1255
1256 ret = user_event_set_call_visible(user, true);
1257
1258 if (ret)
1259 unregister_trace_event(&user->call.event);
1260
1261 return ret;
1262}
1263
1264/*
1265 * Parses the event name, arguments and flags then registers if successful.
1266 * The name buffer lifetime is owned by this method for success cases only.
1267 * Upon success the returned user_event has its ref count increased by 1.
1268 */
1269static int user_event_parse(struct user_event_group *group, char *name,
1270 char *args, char *flags,
1271 struct user_event **newuser)
1272{
1273 int ret;
1274 int index;
1275 u32 key;
1276 struct user_event *user;
1277
1278 /* Prevent dyn_event from racing */
1279 mutex_lock(&event_mutex);
1280 user = find_user_event(group, name, &key);
1281 mutex_unlock(&event_mutex);
1282
1283 if (user) {
1284 *newuser = user;
1285 /*
1286 * Name is allocated by caller, free it since it already exists.
1287 * Caller only worries about failure cases for freeing.
1288 */
1289 kfree(name);
1290 return 0;
1291 }
1292
1293 index = find_first_zero_bit(group->page_bitmap, MAX_EVENTS);
1294
1295 if (index == MAX_EVENTS)
1296 return -EMFILE;
1297
1298 user = kzalloc(sizeof(*user), GFP_KERNEL);
1299
1300 if (!user)
1301 return -ENOMEM;
1302
1303 INIT_LIST_HEAD(&user->class.fields);
1304 INIT_LIST_HEAD(&user->fields);
1305 INIT_LIST_HEAD(&user->validators);
1306
1307 user->group = group;
1308 user->tracepoint.name = name;
1309
1310 ret = user_event_parse_fields(user, args);
1311
1312 if (ret)
1313 goto put_user;
1314
1315 ret = user_event_create_print_fmt(user);
1316
1317 if (ret)
1318 goto put_user;
1319
1320 user->call.data = user;
1321 user->call.class = &user->class;
1322 user->call.name = name;
1323 user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
1324 user->call.tp = &user->tracepoint;
1325 user->call.event.funcs = &user_event_funcs;
1326 user->class.system = group->system_name;
1327
1328 user->class.fields_array = user_event_fields_array;
1329 user->class.get_fields = user_event_get_fields;
1330 user->class.reg = user_event_reg;
1331 user->class.probe = user_event_ftrace;
1332#ifdef CONFIG_PERF_EVENTS
1333 user->class.perf_probe = user_event_perf;
1334#endif
1335
1336 mutex_lock(&event_mutex);
1337
1338 ret = user_event_trace_register(user);
1339
1340 if (ret)
1341 goto put_user_lock;
1342
1343 user->index = index;
1344
1345 /* Ensure we track self ref and caller ref (2) */
1346 refcount_set(&user->refcnt, 2);
1347
1348 dyn_event_init(&user->devent, &user_event_dops);
1349 dyn_event_add(&user->devent, &user->call);
1350 set_bit(user->index, group->page_bitmap);
1351 hash_add(group->register_table, &user->node, key);
1352
1353 mutex_unlock(&event_mutex);
1354
1355 *newuser = user;
1356 return 0;
1357put_user_lock:
1358 mutex_unlock(&event_mutex);
1359put_user:
1360 user_event_destroy_fields(user);
1361 user_event_destroy_validators(user);
1362 kfree(user->call.print_fmt);
1363 kfree(user);
1364 return ret;
1365}
1366
1367/*
1368 * Deletes a previously created event if it is no longer being used.
1369 */
1370static int delete_user_event(struct user_event_group *group, char *name)
1371{
1372 u32 key;
1373 struct user_event *user = find_user_event(group, name, &key);
1374
1375 if (!user)
1376 return -ENOENT;
1377
1378 refcount_dec(&user->refcnt);
1379
1380 if (!user_event_last_ref(user))
1381 return -EBUSY;
1382
1383 return destroy_user_event(user);
1384}
1385
1386/*
1387 * Validates the user payload and writes via iterator.
1388 */
1389static ssize_t user_events_write_core(struct file *file, struct iov_iter *i)
1390{
1391 struct user_event_file_info *info = file->private_data;
1392 struct user_event_refs *refs;
1393 struct user_event *user = NULL;
1394 struct tracepoint *tp;
1395 ssize_t ret = i->count;
1396 int idx;
1397
1398 if (unlikely(copy_from_iter(&idx, sizeof(idx), i) != sizeof(idx)))
1399 return -EFAULT;
1400
1401 rcu_read_lock_sched();
1402
1403 refs = rcu_dereference_sched(info->refs);
1404
1405 /*
1406 * The refs->events array is protected by RCU, and new items may be
1407 * added. But the user retrieved from indexing into the events array
1408 * shall be immutable while the file is opened.
1409 */
1410 if (likely(refs && idx < refs->count))
1411 user = refs->events[idx];
1412
1413 rcu_read_unlock_sched();
1414
1415 if (unlikely(user == NULL))
1416 return -ENOENT;
1417
1418 if (unlikely(i->count < user->min_size))
1419 return -EINVAL;
1420
1421 tp = &user->tracepoint;
1422
1423 /*
1424 * It's possible key.enabled disables after this check, however
1425 * we don't mind if a few events are included in this condition.
1426 */
1427 if (likely(atomic_read(&tp->key.enabled) > 0)) {
1428 struct tracepoint_func *probe_func_ptr;
1429 user_event_func_t probe_func;
1430 struct iov_iter copy;
1431 void *tpdata;
1432 bool faulted;
1433
1434 if (unlikely(fault_in_iov_iter_readable(i, i->count)))
1435 return -EFAULT;
1436
1437 faulted = false;
1438
1439 rcu_read_lock_sched();
1440
1441 probe_func_ptr = rcu_dereference_sched(tp->funcs);
1442
1443 if (probe_func_ptr) {
1444 do {
1445 copy = *i;
1446 probe_func = probe_func_ptr->func;
1447 tpdata = probe_func_ptr->data;
1448 probe_func(user, ©, tpdata, &faulted);
1449 } while ((++probe_func_ptr)->func);
1450 }
1451
1452 rcu_read_unlock_sched();
1453
1454 if (unlikely(faulted))
1455 return -EFAULT;
1456 }
1457
1458 return ret;
1459}
1460
1461static int user_events_open(struct inode *node, struct file *file)
1462{
1463 struct user_event_group *group;
1464 struct user_event_file_info *info;
1465
1466 group = current_user_event_group();
1467
1468 if (!group)
1469 return -ENOENT;
1470
1471 info = kzalloc(sizeof(*info), GFP_KERNEL);
1472
1473 if (!info)
1474 return -ENOMEM;
1475
1476 info->group = group;
1477
1478 file->private_data = info;
1479
1480 return 0;
1481}
1482
1483static ssize_t user_events_write(struct file *file, const char __user *ubuf,
1484 size_t count, loff_t *ppos)
1485{
1486 struct iovec iov;
1487 struct iov_iter i;
1488
1489 if (unlikely(*ppos != 0))
1490 return -EFAULT;
1491
1492 if (unlikely(import_single_range(ITER_SOURCE, (char __user *)ubuf,
1493 count, &iov, &i)))
1494 return -EFAULT;
1495
1496 return user_events_write_core(file, &i);
1497}
1498
1499static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i)
1500{
1501 return user_events_write_core(kp->ki_filp, i);
1502}
1503
1504static int user_events_ref_add(struct user_event_file_info *info,
1505 struct user_event *user)
1506{
1507 struct user_event_group *group = info->group;
1508 struct user_event_refs *refs, *new_refs;
1509 int i, size, count = 0;
1510
1511 refs = rcu_dereference_protected(info->refs,
1512 lockdep_is_held(&group->reg_mutex));
1513
1514 if (refs) {
1515 count = refs->count;
1516
1517 for (i = 0; i < count; ++i)
1518 if (refs->events[i] == user)
1519 return i;
1520 }
1521
1522 size = struct_size(refs, events, count + 1);
1523
1524 new_refs = kzalloc(size, GFP_KERNEL);
1525
1526 if (!new_refs)
1527 return -ENOMEM;
1528
1529 new_refs->count = count + 1;
1530
1531 for (i = 0; i < count; ++i)
1532 new_refs->events[i] = refs->events[i];
1533
1534 new_refs->events[i] = user;
1535
1536 refcount_inc(&user->refcnt);
1537
1538 rcu_assign_pointer(info->refs, new_refs);
1539
1540 if (refs)
1541 kfree_rcu(refs, rcu);
1542
1543 return i;
1544}
1545
1546static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg)
1547{
1548 u32 size;
1549 long ret;
1550
1551 ret = get_user(size, &ureg->size);
1552
1553 if (ret)
1554 return ret;
1555
1556 if (size > PAGE_SIZE)
1557 return -E2BIG;
1558
1559 if (size < offsetofend(struct user_reg, write_index))
1560 return -EINVAL;
1561
1562 ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
1563
1564 if (ret)
1565 return ret;
1566
1567 kreg->size = size;
1568
1569 return 0;
1570}
1571
1572/*
1573 * Registers a user_event on behalf of a user process.
1574 */
1575static long user_events_ioctl_reg(struct user_event_file_info *info,
1576 unsigned long uarg)
1577{
1578 struct user_reg __user *ureg = (struct user_reg __user *)uarg;
1579 struct user_reg reg;
1580 struct user_event *user;
1581 char *name;
1582 long ret;
1583
1584 ret = user_reg_get(ureg, ®);
1585
1586 if (ret)
1587 return ret;
1588
1589 name = strndup_user((const char __user *)(uintptr_t)reg.name_args,
1590 MAX_EVENT_DESC);
1591
1592 if (IS_ERR(name)) {
1593 ret = PTR_ERR(name);
1594 return ret;
1595 }
1596
1597 ret = user_event_parse_cmd(info->group, name, &user);
1598
1599 if (ret) {
1600 kfree(name);
1601 return ret;
1602 }
1603
1604 ret = user_events_ref_add(info, user);
1605
1606 /* No longer need parse ref, ref_add either worked or not */
1607 refcount_dec(&user->refcnt);
1608
1609 /* Positive number is index and valid */
1610 if (ret < 0)
1611 return ret;
1612
1613 put_user((u32)ret, &ureg->write_index);
1614 put_user(user->index, &ureg->status_bit);
1615
1616 return 0;
1617}
1618
1619/*
1620 * Deletes a user_event on behalf of a user process.
1621 */
1622static long user_events_ioctl_del(struct user_event_file_info *info,
1623 unsigned long uarg)
1624{
1625 void __user *ubuf = (void __user *)uarg;
1626 char *name;
1627 long ret;
1628
1629 name = strndup_user(ubuf, MAX_EVENT_DESC);
1630
1631 if (IS_ERR(name))
1632 return PTR_ERR(name);
1633
1634 /* event_mutex prevents dyn_event from racing */
1635 mutex_lock(&event_mutex);
1636 ret = delete_user_event(info->group, name);
1637 mutex_unlock(&event_mutex);
1638
1639 kfree(name);
1640
1641 return ret;
1642}
1643
1644/*
1645 * Handles the ioctl from user mode to register or alter operations.
1646 */
1647static long user_events_ioctl(struct file *file, unsigned int cmd,
1648 unsigned long uarg)
1649{
1650 struct user_event_file_info *info = file->private_data;
1651 struct user_event_group *group = info->group;
1652 long ret = -ENOTTY;
1653
1654 switch (cmd) {
1655 case DIAG_IOCSREG:
1656 mutex_lock(&group->reg_mutex);
1657 ret = user_events_ioctl_reg(info, uarg);
1658 mutex_unlock(&group->reg_mutex);
1659 break;
1660
1661 case DIAG_IOCSDEL:
1662 mutex_lock(&group->reg_mutex);
1663 ret = user_events_ioctl_del(info, uarg);
1664 mutex_unlock(&group->reg_mutex);
1665 break;
1666 }
1667
1668 return ret;
1669}
1670
1671/*
1672 * Handles the final close of the file from user mode.
1673 */
1674static int user_events_release(struct inode *node, struct file *file)
1675{
1676 struct user_event_file_info *info = file->private_data;
1677 struct user_event_group *group;
1678 struct user_event_refs *refs;
1679 struct user_event *user;
1680 int i;
1681
1682 if (!info)
1683 return -EINVAL;
1684
1685 group = info->group;
1686
1687 /*
1688 * Ensure refs cannot change under any situation by taking the
1689 * register mutex during the final freeing of the references.
1690 */
1691 mutex_lock(&group->reg_mutex);
1692
1693 refs = info->refs;
1694
1695 if (!refs)
1696 goto out;
1697
1698 /*
1699 * The lifetime of refs has reached an end, it's tied to this file.
1700 * The underlying user_events are ref counted, and cannot be freed.
1701 * After this decrement, the user_events may be freed elsewhere.
1702 */
1703 for (i = 0; i < refs->count; ++i) {
1704 user = refs->events[i];
1705
1706 if (user)
1707 refcount_dec(&user->refcnt);
1708 }
1709out:
1710 file->private_data = NULL;
1711
1712 mutex_unlock(&group->reg_mutex);
1713
1714 kfree(refs);
1715 kfree(info);
1716
1717 return 0;
1718}
1719
1720static const struct file_operations user_data_fops = {
1721 .open = user_events_open,
1722 .write = user_events_write,
1723 .write_iter = user_events_write_iter,
1724 .unlocked_ioctl = user_events_ioctl,
1725 .release = user_events_release,
1726};
1727
1728static struct user_event_group *user_status_group(struct file *file)
1729{
1730 struct seq_file *m = file->private_data;
1731
1732 if (!m)
1733 return NULL;
1734
1735 return m->private;
1736}
1737
1738/*
1739 * Maps the shared page into the user process for checking if event is enabled.
1740 */
1741static int user_status_mmap(struct file *file, struct vm_area_struct *vma)
1742{
1743 char *pages;
1744 struct user_event_group *group = user_status_group(file);
1745 unsigned long size = vma->vm_end - vma->vm_start;
1746
1747 if (size != MAX_BYTES)
1748 return -EINVAL;
1749
1750 if (!group)
1751 return -EINVAL;
1752
1753 pages = group->register_page_data;
1754
1755 return remap_pfn_range(vma, vma->vm_start,
1756 virt_to_phys(pages) >> PAGE_SHIFT,
1757 size, vm_get_page_prot(VM_READ));
1758}
1759
1760static void *user_seq_start(struct seq_file *m, loff_t *pos)
1761{
1762 if (*pos)
1763 return NULL;
1764
1765 return (void *)1;
1766}
1767
1768static void *user_seq_next(struct seq_file *m, void *p, loff_t *pos)
1769{
1770 ++*pos;
1771 return NULL;
1772}
1773
1774static void user_seq_stop(struct seq_file *m, void *p)
1775{
1776}
1777
1778static int user_seq_show(struct seq_file *m, void *p)
1779{
1780 struct user_event_group *group = m->private;
1781 struct user_event *user;
1782 char status;
1783 int i, active = 0, busy = 0, flags;
1784
1785 if (!group)
1786 return -EINVAL;
1787
1788 mutex_lock(&group->reg_mutex);
1789
1790 hash_for_each(group->register_table, i, user, node) {
1791 status = user->status;
1792 flags = user->flags;
1793
1794 seq_printf(m, "%d:%s", user->index, EVENT_NAME(user));
1795
1796 if (flags != 0 || status != 0)
1797 seq_puts(m, " #");
1798
1799 if (status != 0) {
1800 seq_puts(m, " Used by");
1801 if (status & EVENT_STATUS_FTRACE)
1802 seq_puts(m, " ftrace");
1803 if (status & EVENT_STATUS_PERF)
1804 seq_puts(m, " perf");
1805 if (status & EVENT_STATUS_OTHER)
1806 seq_puts(m, " other");
1807 busy++;
1808 }
1809
1810 seq_puts(m, "\n");
1811 active++;
1812 }
1813
1814 mutex_unlock(&group->reg_mutex);
1815
1816 seq_puts(m, "\n");
1817 seq_printf(m, "Active: %d\n", active);
1818 seq_printf(m, "Busy: %d\n", busy);
1819 seq_printf(m, "Max: %ld\n", MAX_EVENTS);
1820
1821 return 0;
1822}
1823
1824static const struct seq_operations user_seq_ops = {
1825 .start = user_seq_start,
1826 .next = user_seq_next,
1827 .stop = user_seq_stop,
1828 .show = user_seq_show,
1829};
1830
1831static int user_status_open(struct inode *node, struct file *file)
1832{
1833 struct user_event_group *group;
1834 int ret;
1835
1836 group = current_user_event_group();
1837
1838 if (!group)
1839 return -ENOENT;
1840
1841 ret = seq_open(file, &user_seq_ops);
1842
1843 if (!ret) {
1844 /* Chain group to seq_file */
1845 struct seq_file *m = file->private_data;
1846
1847 m->private = group;
1848 }
1849
1850 return ret;
1851}
1852
1853static const struct file_operations user_status_fops = {
1854 .open = user_status_open,
1855 .mmap = user_status_mmap,
1856 .read = seq_read,
1857 .llseek = seq_lseek,
1858 .release = seq_release,
1859};
1860
1861/*
1862 * Creates a set of tracefs files to allow user mode interactions.
1863 */
1864static int create_user_tracefs(void)
1865{
1866 struct dentry *edata, *emmap;
1867
1868 edata = tracefs_create_file("user_events_data", TRACE_MODE_WRITE,
1869 NULL, NULL, &user_data_fops);
1870
1871 if (!edata) {
1872 pr_warn("Could not create tracefs 'user_events_data' entry\n");
1873 goto err;
1874 }
1875
1876 /* mmap with MAP_SHARED requires writable fd */
1877 emmap = tracefs_create_file("user_events_status", TRACE_MODE_WRITE,
1878 NULL, NULL, &user_status_fops);
1879
1880 if (!emmap) {
1881 tracefs_remove(edata);
1882 pr_warn("Could not create tracefs 'user_events_mmap' entry\n");
1883 goto err;
1884 }
1885
1886 return 0;
1887err:
1888 return -ENODEV;
1889}
1890
1891static int __init trace_events_user_init(void)
1892{
1893 int ret;
1894
1895 init_group = user_event_group_create(&init_user_ns);
1896
1897 if (!init_group)
1898 return -ENOMEM;
1899
1900 ret = create_user_tracefs();
1901
1902 if (ret) {
1903 pr_warn("user_events could not register with tracefs\n");
1904 user_event_group_destroy(init_group);
1905 init_group = NULL;
1906 return ret;
1907 }
1908
1909 if (dyn_event_register(&user_event_dops))
1910 pr_warn("user_events could not register with dyn_events\n");
1911
1912 return 0;
1913}
1914
1915fs_initcall(trace_events_user_init);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2021, Microsoft Corporation.
4 *
5 * Authors:
6 * Beau Belgrave <beaub@linux.microsoft.com>
7 */
8
9#include <linux/bitmap.h>
10#include <linux/cdev.h>
11#include <linux/hashtable.h>
12#include <linux/list.h>
13#include <linux/io.h>
14#include <linux/uio.h>
15#include <linux/ioctl.h>
16#include <linux/jhash.h>
17#include <linux/refcount.h>
18#include <linux/trace_events.h>
19#include <linux/tracefs.h>
20#include <linux/types.h>
21#include <linux/uaccess.h>
22#include <linux/highmem.h>
23#include <linux/init.h>
24#include <linux/user_events.h>
25#include "trace_dynevent.h"
26#include "trace_output.h"
27#include "trace.h"
28
29#define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1)
30
31#define FIELD_DEPTH_TYPE 0
32#define FIELD_DEPTH_NAME 1
33#define FIELD_DEPTH_SIZE 2
34
35/* Limit how long of an event name plus args within the subsystem. */
36#define MAX_EVENT_DESC 512
37#define EVENT_NAME(user_event) ((user_event)->reg_name)
38#define EVENT_TP_NAME(user_event) ((user_event)->tracepoint.name)
39#define MAX_FIELD_ARRAY_SIZE 1024
40
41/*
42 * Internal bits (kernel side only) to keep track of connected probes:
43 * These are used when status is requested in text form about an event. These
44 * bits are compared against an internal byte on the event to determine which
45 * probes to print out to the user.
46 *
47 * These do not reflect the mapped bytes between the user and kernel space.
48 */
49#define EVENT_STATUS_FTRACE BIT(0)
50#define EVENT_STATUS_PERF BIT(1)
51#define EVENT_STATUS_OTHER BIT(7)
52
53/*
54 * Stores the system name, tables, and locks for a group of events. This
55 * allows isolation for events by various means.
56 */
57struct user_event_group {
58 char *system_name;
59 char *system_multi_name;
60 struct hlist_node node;
61 struct mutex reg_mutex;
62 DECLARE_HASHTABLE(register_table, 8);
63 /* ID that moves forward within the group for multi-event names */
64 u64 multi_id;
65};
66
67/* Group for init_user_ns mapping, top-most group */
68static struct user_event_group *init_group;
69
70/* Max allowed events for the whole system */
71static unsigned int max_user_events = 32768;
72
73/* Current number of events on the whole system */
74static unsigned int current_user_events;
75
76/*
77 * Stores per-event properties, as users register events
78 * within a file a user_event might be created if it does not
79 * already exist. These are globally used and their lifetime
80 * is tied to the refcnt member. These cannot go away until the
81 * refcnt reaches one.
82 */
83struct user_event {
84 struct user_event_group *group;
85 char *reg_name;
86 struct tracepoint tracepoint;
87 struct trace_event_call call;
88 struct trace_event_class class;
89 struct dyn_event devent;
90 struct hlist_node node;
91 struct list_head fields;
92 struct list_head validators;
93 struct work_struct put_work;
94 refcount_t refcnt;
95 int min_size;
96 int reg_flags;
97 char status;
98};
99
100/*
101 * Stores per-mm/event properties that enable an address to be
102 * updated properly for each task. As tasks are forked, we use
103 * these to track enablement sites that are tied to an event.
104 */
105struct user_event_enabler {
106 struct list_head mm_enablers_link;
107 struct user_event *event;
108 unsigned long addr;
109
110 /* Track enable bit, flags, etc. Aligned for bitops. */
111 unsigned long values;
112};
113
114/* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */
115#define ENABLE_VAL_BIT_MASK 0x3F
116
117/* Bit 6 is for faulting status of enablement */
118#define ENABLE_VAL_FAULTING_BIT 6
119
120/* Bit 7 is for freeing status of enablement */
121#define ENABLE_VAL_FREEING_BIT 7
122
123/* Bit 8 is for marking 32-bit on 64-bit */
124#define ENABLE_VAL_32_ON_64_BIT 8
125
126#define ENABLE_VAL_COMPAT_MASK (1 << ENABLE_VAL_32_ON_64_BIT)
127
128/* Only duplicate the bit and compat values */
129#define ENABLE_VAL_DUP_MASK (ENABLE_VAL_BIT_MASK | ENABLE_VAL_COMPAT_MASK)
130
131#define ENABLE_BITOPS(e) (&(e)->values)
132
133#define ENABLE_BIT(e) ((int)((e)->values & ENABLE_VAL_BIT_MASK))
134
135#define EVENT_MULTI_FORMAT(f) ((f) & USER_EVENT_REG_MULTI_FORMAT)
136
137/* Used for asynchronous faulting in of pages */
138struct user_event_enabler_fault {
139 struct work_struct work;
140 struct user_event_mm *mm;
141 struct user_event_enabler *enabler;
142 int attempt;
143};
144
145static struct kmem_cache *fault_cache;
146
147/* Global list of memory descriptors using user_events */
148static LIST_HEAD(user_event_mms);
149static DEFINE_SPINLOCK(user_event_mms_lock);
150
151/*
152 * Stores per-file events references, as users register events
153 * within a file this structure is modified and freed via RCU.
154 * The lifetime of this struct is tied to the lifetime of the file.
155 * These are not shared and only accessible by the file that created it.
156 */
157struct user_event_refs {
158 struct rcu_head rcu;
159 int count;
160 struct user_event *events[];
161};
162
163struct user_event_file_info {
164 struct user_event_group *group;
165 struct user_event_refs *refs;
166};
167
168#define VALIDATOR_ENSURE_NULL (1 << 0)
169#define VALIDATOR_REL (1 << 1)
170
171struct user_event_validator {
172 struct list_head user_event_link;
173 int offset;
174 int flags;
175};
176
177static inline void align_addr_bit(unsigned long *addr, int *bit,
178 unsigned long *flags)
179{
180 if (IS_ALIGNED(*addr, sizeof(long))) {
181#ifdef __BIG_ENDIAN
182 /* 32 bit on BE 64 bit requires a 32 bit offset when aligned. */
183 if (test_bit(ENABLE_VAL_32_ON_64_BIT, flags))
184 *bit += 32;
185#endif
186 return;
187 }
188
189 *addr = ALIGN_DOWN(*addr, sizeof(long));
190
191 /*
192 * We only support 32 and 64 bit values. The only time we need
193 * to align is a 32 bit value on a 64 bit kernel, which on LE
194 * is always 32 bits, and on BE requires no change when unaligned.
195 */
196#ifdef __LITTLE_ENDIAN
197 *bit += 32;
198#endif
199}
200
201typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i,
202 void *tpdata, bool *faulted);
203
204static int user_event_parse(struct user_event_group *group, char *name,
205 char *args, char *flags,
206 struct user_event **newuser, int reg_flags);
207
208static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm);
209static struct user_event_mm *user_event_mm_get_all(struct user_event *user);
210static void user_event_mm_put(struct user_event_mm *mm);
211static int destroy_user_event(struct user_event *user);
212static bool user_fields_match(struct user_event *user, int argc,
213 const char **argv);
214
215static u32 user_event_key(char *name)
216{
217 return jhash(name, strlen(name), 0);
218}
219
220static bool user_event_capable(u16 reg_flags)
221{
222 /* Persistent events require CAP_PERFMON / CAP_SYS_ADMIN */
223 if (reg_flags & USER_EVENT_REG_PERSIST) {
224 if (!perfmon_capable())
225 return false;
226 }
227
228 return true;
229}
230
231static struct user_event *user_event_get(struct user_event *user)
232{
233 refcount_inc(&user->refcnt);
234
235 return user;
236}
237
238static void delayed_destroy_user_event(struct work_struct *work)
239{
240 struct user_event *user = container_of(
241 work, struct user_event, put_work);
242
243 mutex_lock(&event_mutex);
244
245 if (!refcount_dec_and_test(&user->refcnt))
246 goto out;
247
248 if (destroy_user_event(user)) {
249 /*
250 * The only reason this would fail here is if we cannot
251 * update the visibility of the event. In this case the
252 * event stays in the hashtable, waiting for someone to
253 * attempt to delete it later.
254 */
255 pr_warn("user_events: Unable to delete event\n");
256 refcount_set(&user->refcnt, 1);
257 }
258out:
259 mutex_unlock(&event_mutex);
260}
261
262static void user_event_put(struct user_event *user, bool locked)
263{
264 bool delete;
265
266 if (unlikely(!user))
267 return;
268
269 /*
270 * When the event is not enabled for auto-delete there will always
271 * be at least 1 reference to the event. During the event creation
272 * we initially set the refcnt to 2 to achieve this. In those cases
273 * the caller must acquire event_mutex and after decrement check if
274 * the refcnt is 1, meaning this is the last reference. When auto
275 * delete is enabled, there will only be 1 ref, IE: refcnt will be
276 * only set to 1 during creation to allow the below checks to go
277 * through upon the last put. The last put must always be done with
278 * the event mutex held.
279 */
280 if (!locked) {
281 lockdep_assert_not_held(&event_mutex);
282 delete = refcount_dec_and_mutex_lock(&user->refcnt, &event_mutex);
283 } else {
284 lockdep_assert_held(&event_mutex);
285 delete = refcount_dec_and_test(&user->refcnt);
286 }
287
288 if (!delete)
289 return;
290
291 /*
292 * We now have the event_mutex in all cases, which ensures that
293 * no new references will be taken until event_mutex is released.
294 * New references come through find_user_event(), which requires
295 * the event_mutex to be held.
296 */
297
298 if (user->reg_flags & USER_EVENT_REG_PERSIST) {
299 /* We should not get here when persist flag is set */
300 pr_alert("BUG: Auto-delete engaged on persistent event\n");
301 goto out;
302 }
303
304 /*
305 * Unfortunately we have to attempt the actual destroy in a work
306 * queue. This is because not all cases handle a trace_event_call
307 * being removed within the class->reg() operation for unregister.
308 */
309 INIT_WORK(&user->put_work, delayed_destroy_user_event);
310
311 /*
312 * Since the event is still in the hashtable, we have to re-inc
313 * the ref count to 1. This count will be decremented and checked
314 * in the work queue to ensure it's still the last ref. This is
315 * needed because a user-process could register the same event in
316 * between the time of event_mutex release and the work queue
317 * running the delayed destroy. If we removed the item now from
318 * the hashtable, this would result in a timing window where a
319 * user process would fail a register because the trace_event_call
320 * register would fail in the tracing layers.
321 */
322 refcount_set(&user->refcnt, 1);
323
324 if (WARN_ON_ONCE(!schedule_work(&user->put_work))) {
325 /*
326 * If we fail we must wait for an admin to attempt delete or
327 * another register/close of the event, whichever is first.
328 */
329 pr_warn("user_events: Unable to queue delayed destroy\n");
330 }
331out:
332 /* Ensure if we didn't have event_mutex before we unlock it */
333 if (!locked)
334 mutex_unlock(&event_mutex);
335}
336
337static void user_event_group_destroy(struct user_event_group *group)
338{
339 kfree(group->system_name);
340 kfree(group->system_multi_name);
341 kfree(group);
342}
343
344static char *user_event_group_system_name(void)
345{
346 char *system_name;
347 int len = sizeof(USER_EVENTS_SYSTEM) + 1;
348
349 system_name = kmalloc(len, GFP_KERNEL);
350
351 if (!system_name)
352 return NULL;
353
354 snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM);
355
356 return system_name;
357}
358
359static char *user_event_group_system_multi_name(void)
360{
361 return kstrdup(USER_EVENTS_MULTI_SYSTEM, GFP_KERNEL);
362}
363
364static struct user_event_group *current_user_event_group(void)
365{
366 return init_group;
367}
368
369static struct user_event_group *user_event_group_create(void)
370{
371 struct user_event_group *group;
372
373 group = kzalloc(sizeof(*group), GFP_KERNEL);
374
375 if (!group)
376 return NULL;
377
378 group->system_name = user_event_group_system_name();
379
380 if (!group->system_name)
381 goto error;
382
383 group->system_multi_name = user_event_group_system_multi_name();
384
385 if (!group->system_multi_name)
386 goto error;
387
388 mutex_init(&group->reg_mutex);
389 hash_init(group->register_table);
390
391 return group;
392error:
393 if (group)
394 user_event_group_destroy(group);
395
396 return NULL;
397};
398
399static void user_event_enabler_destroy(struct user_event_enabler *enabler,
400 bool locked)
401{
402 list_del_rcu(&enabler->mm_enablers_link);
403
404 /* No longer tracking the event via the enabler */
405 user_event_put(enabler->event, locked);
406
407 kfree(enabler);
408}
409
410static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr,
411 int attempt)
412{
413 bool unlocked;
414 int ret;
415
416 /*
417 * Normally this is low, ensure that it cannot be taken advantage of by
418 * bad user processes to cause excessive looping.
419 */
420 if (attempt > 10)
421 return -EFAULT;
422
423 mmap_read_lock(mm->mm);
424
425 /* Ensure MM has tasks, cannot use after exit_mm() */
426 if (refcount_read(&mm->tasks) == 0) {
427 ret = -ENOENT;
428 goto out;
429 }
430
431 ret = fixup_user_fault(mm->mm, uaddr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE,
432 &unlocked);
433out:
434 mmap_read_unlock(mm->mm);
435
436 return ret;
437}
438
439static int user_event_enabler_write(struct user_event_mm *mm,
440 struct user_event_enabler *enabler,
441 bool fixup_fault, int *attempt);
442
443static void user_event_enabler_fault_fixup(struct work_struct *work)
444{
445 struct user_event_enabler_fault *fault = container_of(
446 work, struct user_event_enabler_fault, work);
447 struct user_event_enabler *enabler = fault->enabler;
448 struct user_event_mm *mm = fault->mm;
449 unsigned long uaddr = enabler->addr;
450 int attempt = fault->attempt;
451 int ret;
452
453 ret = user_event_mm_fault_in(mm, uaddr, attempt);
454
455 if (ret && ret != -ENOENT) {
456 struct user_event *user = enabler->event;
457
458 pr_warn("user_events: Fault for mm: 0x%pK @ 0x%llx event: %s\n",
459 mm->mm, (unsigned long long)uaddr, EVENT_NAME(user));
460 }
461
462 /* Prevent state changes from racing */
463 mutex_lock(&event_mutex);
464
465 /* User asked for enabler to be removed during fault */
466 if (test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))) {
467 user_event_enabler_destroy(enabler, true);
468 goto out;
469 }
470
471 /*
472 * If we managed to get the page, re-issue the write. We do not
473 * want to get into a possible infinite loop, which is why we only
474 * attempt again directly if the page came in. If we couldn't get
475 * the page here, then we will try again the next time the event is
476 * enabled/disabled.
477 */
478 clear_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
479
480 if (!ret) {
481 mmap_read_lock(mm->mm);
482 user_event_enabler_write(mm, enabler, true, &attempt);
483 mmap_read_unlock(mm->mm);
484 }
485out:
486 mutex_unlock(&event_mutex);
487
488 /* In all cases we no longer need the mm or fault */
489 user_event_mm_put(mm);
490 kmem_cache_free(fault_cache, fault);
491}
492
493static bool user_event_enabler_queue_fault(struct user_event_mm *mm,
494 struct user_event_enabler *enabler,
495 int attempt)
496{
497 struct user_event_enabler_fault *fault;
498
499 fault = kmem_cache_zalloc(fault_cache, GFP_NOWAIT | __GFP_NOWARN);
500
501 if (!fault)
502 return false;
503
504 INIT_WORK(&fault->work, user_event_enabler_fault_fixup);
505 fault->mm = user_event_mm_get(mm);
506 fault->enabler = enabler;
507 fault->attempt = attempt;
508
509 /* Don't try to queue in again while we have a pending fault */
510 set_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
511
512 if (!schedule_work(&fault->work)) {
513 /* Allow another attempt later */
514 clear_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
515
516 user_event_mm_put(mm);
517 kmem_cache_free(fault_cache, fault);
518
519 return false;
520 }
521
522 return true;
523}
524
525static int user_event_enabler_write(struct user_event_mm *mm,
526 struct user_event_enabler *enabler,
527 bool fixup_fault, int *attempt)
528{
529 unsigned long uaddr = enabler->addr;
530 unsigned long *ptr;
531 struct page *page;
532 void *kaddr;
533 int bit = ENABLE_BIT(enabler);
534 int ret;
535
536 lockdep_assert_held(&event_mutex);
537 mmap_assert_locked(mm->mm);
538
539 *attempt += 1;
540
541 /* Ensure MM has tasks, cannot use after exit_mm() */
542 if (refcount_read(&mm->tasks) == 0)
543 return -ENOENT;
544
545 if (unlikely(test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler)) ||
546 test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))))
547 return -EBUSY;
548
549 align_addr_bit(&uaddr, &bit, ENABLE_BITOPS(enabler));
550
551 ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT,
552 &page, NULL);
553
554 if (unlikely(ret <= 0)) {
555 if (!fixup_fault)
556 return -EFAULT;
557
558 if (!user_event_enabler_queue_fault(mm, enabler, *attempt))
559 pr_warn("user_events: Unable to queue fault handler\n");
560
561 return -EFAULT;
562 }
563
564 kaddr = kmap_local_page(page);
565 ptr = kaddr + (uaddr & ~PAGE_MASK);
566
567 /* Update bit atomically, user tracers must be atomic as well */
568 if (enabler->event && enabler->event->status)
569 set_bit(bit, ptr);
570 else
571 clear_bit(bit, ptr);
572
573 kunmap_local(kaddr);
574 unpin_user_pages_dirty_lock(&page, 1, true);
575
576 return 0;
577}
578
579static bool user_event_enabler_exists(struct user_event_mm *mm,
580 unsigned long uaddr, unsigned char bit)
581{
582 struct user_event_enabler *enabler;
583
584 list_for_each_entry(enabler, &mm->enablers, mm_enablers_link) {
585 if (enabler->addr == uaddr && ENABLE_BIT(enabler) == bit)
586 return true;
587 }
588
589 return false;
590}
591
592static void user_event_enabler_update(struct user_event *user)
593{
594 struct user_event_enabler *enabler;
595 struct user_event_mm *next;
596 struct user_event_mm *mm;
597 int attempt;
598
599 lockdep_assert_held(&event_mutex);
600
601 /*
602 * We need to build a one-shot list of all the mms that have an
603 * enabler for the user_event passed in. This list is only valid
604 * while holding the event_mutex. The only reason for this is due
605 * to the global mm list being RCU protected and we use methods
606 * which can wait (mmap_read_lock and pin_user_pages_remote).
607 *
608 * NOTE: user_event_mm_get_all() increments the ref count of each
609 * mm that is added to the list to prevent removal timing windows.
610 * We must always put each mm after they are used, which may wait.
611 */
612 mm = user_event_mm_get_all(user);
613
614 while (mm) {
615 next = mm->next;
616 mmap_read_lock(mm->mm);
617
618 list_for_each_entry(enabler, &mm->enablers, mm_enablers_link) {
619 if (enabler->event == user) {
620 attempt = 0;
621 user_event_enabler_write(mm, enabler, true, &attempt);
622 }
623 }
624
625 mmap_read_unlock(mm->mm);
626 user_event_mm_put(mm);
627 mm = next;
628 }
629}
630
631static bool user_event_enabler_dup(struct user_event_enabler *orig,
632 struct user_event_mm *mm)
633{
634 struct user_event_enabler *enabler;
635
636 /* Skip pending frees */
637 if (unlikely(test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(orig))))
638 return true;
639
640 enabler = kzalloc(sizeof(*enabler), GFP_NOWAIT | __GFP_ACCOUNT);
641
642 if (!enabler)
643 return false;
644
645 enabler->event = user_event_get(orig->event);
646 enabler->addr = orig->addr;
647
648 /* Only dup part of value (ignore future flags, etc) */
649 enabler->values = orig->values & ENABLE_VAL_DUP_MASK;
650
651 /* Enablers not exposed yet, RCU not required */
652 list_add(&enabler->mm_enablers_link, &mm->enablers);
653
654 return true;
655}
656
657static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm)
658{
659 refcount_inc(&mm->refcnt);
660
661 return mm;
662}
663
664static struct user_event_mm *user_event_mm_get_all(struct user_event *user)
665{
666 struct user_event_mm *found = NULL;
667 struct user_event_enabler *enabler;
668 struct user_event_mm *mm;
669
670 /*
671 * We use the mm->next field to build a one-shot list from the global
672 * RCU protected list. To build this list the event_mutex must be held.
673 * This lets us build a list without requiring allocs that could fail
674 * when user based events are most wanted for diagnostics.
675 */
676 lockdep_assert_held(&event_mutex);
677
678 /*
679 * We do not want to block fork/exec while enablements are being
680 * updated, so we use RCU to walk the current tasks that have used
681 * user_events ABI for 1 or more events. Each enabler found in each
682 * task that matches the event being updated has a write to reflect
683 * the kernel state back into the process. Waits/faults must not occur
684 * during this. So we scan the list under RCU for all the mm that have
685 * the event within it. This is needed because mm_read_lock() can wait.
686 * Each user mm returned has a ref inc to handle remove RCU races.
687 */
688 rcu_read_lock();
689
690 list_for_each_entry_rcu(mm, &user_event_mms, mms_link) {
691 list_for_each_entry_rcu(enabler, &mm->enablers, mm_enablers_link) {
692 if (enabler->event == user) {
693 mm->next = found;
694 found = user_event_mm_get(mm);
695 break;
696 }
697 }
698 }
699
700 rcu_read_unlock();
701
702 return found;
703}
704
705static struct user_event_mm *user_event_mm_alloc(struct task_struct *t)
706{
707 struct user_event_mm *user_mm;
708
709 user_mm = kzalloc(sizeof(*user_mm), GFP_KERNEL_ACCOUNT);
710
711 if (!user_mm)
712 return NULL;
713
714 user_mm->mm = t->mm;
715 INIT_LIST_HEAD(&user_mm->enablers);
716 refcount_set(&user_mm->refcnt, 1);
717 refcount_set(&user_mm->tasks, 1);
718
719 /*
720 * The lifetime of the memory descriptor can slightly outlast
721 * the task lifetime if a ref to the user_event_mm is taken
722 * between list_del_rcu() and call_rcu(). Therefore we need
723 * to take a reference to it to ensure it can live this long
724 * under this corner case. This can also occur in clones that
725 * outlast the parent.
726 */
727 mmgrab(user_mm->mm);
728
729 return user_mm;
730}
731
732static void user_event_mm_attach(struct user_event_mm *user_mm, struct task_struct *t)
733{
734 unsigned long flags;
735
736 spin_lock_irqsave(&user_event_mms_lock, flags);
737 list_add_rcu(&user_mm->mms_link, &user_event_mms);
738 spin_unlock_irqrestore(&user_event_mms_lock, flags);
739
740 t->user_event_mm = user_mm;
741}
742
743static struct user_event_mm *current_user_event_mm(void)
744{
745 struct user_event_mm *user_mm = current->user_event_mm;
746
747 if (user_mm)
748 goto inc;
749
750 user_mm = user_event_mm_alloc(current);
751
752 if (!user_mm)
753 goto error;
754
755 user_event_mm_attach(user_mm, current);
756inc:
757 refcount_inc(&user_mm->refcnt);
758error:
759 return user_mm;
760}
761
762static void user_event_mm_destroy(struct user_event_mm *mm)
763{
764 struct user_event_enabler *enabler, *next;
765
766 list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link)
767 user_event_enabler_destroy(enabler, false);
768
769 mmdrop(mm->mm);
770 kfree(mm);
771}
772
773static void user_event_mm_put(struct user_event_mm *mm)
774{
775 if (mm && refcount_dec_and_test(&mm->refcnt))
776 user_event_mm_destroy(mm);
777}
778
779static void delayed_user_event_mm_put(struct work_struct *work)
780{
781 struct user_event_mm *mm;
782
783 mm = container_of(to_rcu_work(work), struct user_event_mm, put_rwork);
784 user_event_mm_put(mm);
785}
786
787void user_event_mm_remove(struct task_struct *t)
788{
789 struct user_event_mm *mm;
790 unsigned long flags;
791
792 might_sleep();
793
794 mm = t->user_event_mm;
795 t->user_event_mm = NULL;
796
797 /* Clone will increment the tasks, only remove if last clone */
798 if (!refcount_dec_and_test(&mm->tasks))
799 return;
800
801 /* Remove the mm from the list, so it can no longer be enabled */
802 spin_lock_irqsave(&user_event_mms_lock, flags);
803 list_del_rcu(&mm->mms_link);
804 spin_unlock_irqrestore(&user_event_mms_lock, flags);
805
806 /*
807 * We need to wait for currently occurring writes to stop within
808 * the mm. This is required since exit_mm() snaps the current rss
809 * stats and clears them. On the final mmdrop(), check_mm() will
810 * report a bug if these increment.
811 *
812 * All writes/pins are done under mmap_read lock, take the write
813 * lock to ensure in-progress faults have completed. Faults that
814 * are pending but yet to run will check the task count and skip
815 * the fault since the mm is going away.
816 */
817 mmap_write_lock(mm->mm);
818 mmap_write_unlock(mm->mm);
819
820 /*
821 * Put for mm must be done after RCU delay to handle new refs in
822 * between the list_del_rcu() and now. This ensures any get refs
823 * during rcu_read_lock() are accounted for during list removal.
824 *
825 * CPU A | CPU B
826 * ---------------------------------------------------------------
827 * user_event_mm_remove() | rcu_read_lock();
828 * list_del_rcu() | list_for_each_entry_rcu();
829 * call_rcu() | refcount_inc();
830 * . | rcu_read_unlock();
831 * schedule_work() | .
832 * user_event_mm_put() | .
833 *
834 * mmdrop() cannot be called in the softirq context of call_rcu()
835 * so we use a work queue after call_rcu() to run within.
836 */
837 INIT_RCU_WORK(&mm->put_rwork, delayed_user_event_mm_put);
838 queue_rcu_work(system_wq, &mm->put_rwork);
839}
840
841void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm)
842{
843 struct user_event_mm *mm = user_event_mm_alloc(t);
844 struct user_event_enabler *enabler;
845
846 if (!mm)
847 return;
848
849 rcu_read_lock();
850
851 list_for_each_entry_rcu(enabler, &old_mm->enablers, mm_enablers_link) {
852 if (!user_event_enabler_dup(enabler, mm))
853 goto error;
854 }
855
856 rcu_read_unlock();
857
858 user_event_mm_attach(mm, t);
859 return;
860error:
861 rcu_read_unlock();
862 user_event_mm_destroy(mm);
863}
864
865static bool current_user_event_enabler_exists(unsigned long uaddr,
866 unsigned char bit)
867{
868 struct user_event_mm *user_mm = current_user_event_mm();
869 bool exists;
870
871 if (!user_mm)
872 return false;
873
874 exists = user_event_enabler_exists(user_mm, uaddr, bit);
875
876 user_event_mm_put(user_mm);
877
878 return exists;
879}
880
881static struct user_event_enabler
882*user_event_enabler_create(struct user_reg *reg, struct user_event *user,
883 int *write_result)
884{
885 struct user_event_enabler *enabler;
886 struct user_event_mm *user_mm;
887 unsigned long uaddr = (unsigned long)reg->enable_addr;
888 int attempt = 0;
889
890 user_mm = current_user_event_mm();
891
892 if (!user_mm)
893 return NULL;
894
895 enabler = kzalloc(sizeof(*enabler), GFP_KERNEL_ACCOUNT);
896
897 if (!enabler)
898 goto out;
899
900 enabler->event = user;
901 enabler->addr = uaddr;
902 enabler->values = reg->enable_bit;
903
904#if BITS_PER_LONG >= 64
905 if (reg->enable_size == 4)
906 set_bit(ENABLE_VAL_32_ON_64_BIT, ENABLE_BITOPS(enabler));
907#endif
908
909retry:
910 /* Prevents state changes from racing with new enablers */
911 mutex_lock(&event_mutex);
912
913 /* Attempt to reflect the current state within the process */
914 mmap_read_lock(user_mm->mm);
915 *write_result = user_event_enabler_write(user_mm, enabler, false,
916 &attempt);
917 mmap_read_unlock(user_mm->mm);
918
919 /*
920 * If the write works, then we will track the enabler. A ref to the
921 * underlying user_event is held by the enabler to prevent it going
922 * away while the enabler is still in use by a process. The ref is
923 * removed when the enabler is destroyed. This means a event cannot
924 * be forcefully deleted from the system until all tasks using it
925 * exit or run exec(), which includes forks and clones.
926 */
927 if (!*write_result) {
928 user_event_get(user);
929 list_add_rcu(&enabler->mm_enablers_link, &user_mm->enablers);
930 }
931
932 mutex_unlock(&event_mutex);
933
934 if (*write_result) {
935 /* Attempt to fault-in and retry if it worked */
936 if (!user_event_mm_fault_in(user_mm, uaddr, attempt))
937 goto retry;
938
939 kfree(enabler);
940 enabler = NULL;
941 }
942out:
943 user_event_mm_put(user_mm);
944
945 return enabler;
946}
947
948static __always_inline __must_check
949bool user_event_last_ref(struct user_event *user)
950{
951 int last = 0;
952
953 if (user->reg_flags & USER_EVENT_REG_PERSIST)
954 last = 1;
955
956 return refcount_read(&user->refcnt) == last;
957}
958
959static __always_inline __must_check
960size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i)
961{
962 size_t ret;
963
964 pagefault_disable();
965
966 ret = copy_from_iter_nocache(addr, bytes, i);
967
968 pagefault_enable();
969
970 return ret;
971}
972
973static struct list_head *user_event_get_fields(struct trace_event_call *call)
974{
975 struct user_event *user = (struct user_event *)call->data;
976
977 return &user->fields;
978}
979
980/*
981 * Parses a register command for user_events
982 * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]]
983 *
984 * Example event named 'test' with a 20 char 'msg' field with an unsigned int
985 * 'id' field after:
986 * test char[20] msg;unsigned int id
987 *
988 * NOTE: Offsets are from the user data perspective, they are not from the
989 * trace_entry/buffer perspective. We automatically add the common properties
990 * sizes to the offset for the user.
991 *
992 * Upon success user_event has its ref count increased by 1.
993 */
994static int user_event_parse_cmd(struct user_event_group *group,
995 char *raw_command, struct user_event **newuser,
996 int reg_flags)
997{
998 char *name = raw_command;
999 char *args = strpbrk(name, " ");
1000 char *flags;
1001
1002 if (args)
1003 *args++ = '\0';
1004
1005 flags = strpbrk(name, ":");
1006
1007 if (flags)
1008 *flags++ = '\0';
1009
1010 return user_event_parse(group, name, args, flags, newuser, reg_flags);
1011}
1012
1013static int user_field_array_size(const char *type)
1014{
1015 const char *start = strchr(type, '[');
1016 char val[8];
1017 char *bracket;
1018 int size = 0;
1019
1020 if (start == NULL)
1021 return -EINVAL;
1022
1023 if (strscpy(val, start + 1, sizeof(val)) <= 0)
1024 return -EINVAL;
1025
1026 bracket = strchr(val, ']');
1027
1028 if (!bracket)
1029 return -EINVAL;
1030
1031 *bracket = '\0';
1032
1033 if (kstrtouint(val, 0, &size))
1034 return -EINVAL;
1035
1036 if (size > MAX_FIELD_ARRAY_SIZE)
1037 return -EINVAL;
1038
1039 return size;
1040}
1041
1042static int user_field_size(const char *type)
1043{
1044 /* long is not allowed from a user, since it's ambigious in size */
1045 if (strcmp(type, "s64") == 0)
1046 return sizeof(s64);
1047 if (strcmp(type, "u64") == 0)
1048 return sizeof(u64);
1049 if (strcmp(type, "s32") == 0)
1050 return sizeof(s32);
1051 if (strcmp(type, "u32") == 0)
1052 return sizeof(u32);
1053 if (strcmp(type, "int") == 0)
1054 return sizeof(int);
1055 if (strcmp(type, "unsigned int") == 0)
1056 return sizeof(unsigned int);
1057 if (strcmp(type, "s16") == 0)
1058 return sizeof(s16);
1059 if (strcmp(type, "u16") == 0)
1060 return sizeof(u16);
1061 if (strcmp(type, "short") == 0)
1062 return sizeof(short);
1063 if (strcmp(type, "unsigned short") == 0)
1064 return sizeof(unsigned short);
1065 if (strcmp(type, "s8") == 0)
1066 return sizeof(s8);
1067 if (strcmp(type, "u8") == 0)
1068 return sizeof(u8);
1069 if (strcmp(type, "char") == 0)
1070 return sizeof(char);
1071 if (strcmp(type, "unsigned char") == 0)
1072 return sizeof(unsigned char);
1073 if (str_has_prefix(type, "char["))
1074 return user_field_array_size(type);
1075 if (str_has_prefix(type, "unsigned char["))
1076 return user_field_array_size(type);
1077 if (str_has_prefix(type, "__data_loc "))
1078 return sizeof(u32);
1079 if (str_has_prefix(type, "__rel_loc "))
1080 return sizeof(u32);
1081
1082 /* Uknown basic type, error */
1083 return -EINVAL;
1084}
1085
1086static void user_event_destroy_validators(struct user_event *user)
1087{
1088 struct user_event_validator *validator, *next;
1089 struct list_head *head = &user->validators;
1090
1091 list_for_each_entry_safe(validator, next, head, user_event_link) {
1092 list_del(&validator->user_event_link);
1093 kfree(validator);
1094 }
1095}
1096
1097static void user_event_destroy_fields(struct user_event *user)
1098{
1099 struct ftrace_event_field *field, *next;
1100 struct list_head *head = &user->fields;
1101
1102 list_for_each_entry_safe(field, next, head, link) {
1103 list_del(&field->link);
1104 kfree(field);
1105 }
1106}
1107
1108static int user_event_add_field(struct user_event *user, const char *type,
1109 const char *name, int offset, int size,
1110 int is_signed, int filter_type)
1111{
1112 struct user_event_validator *validator;
1113 struct ftrace_event_field *field;
1114 int validator_flags = 0;
1115
1116 field = kmalloc(sizeof(*field), GFP_KERNEL_ACCOUNT);
1117
1118 if (!field)
1119 return -ENOMEM;
1120
1121 if (str_has_prefix(type, "__data_loc "))
1122 goto add_validator;
1123
1124 if (str_has_prefix(type, "__rel_loc ")) {
1125 validator_flags |= VALIDATOR_REL;
1126 goto add_validator;
1127 }
1128
1129 goto add_field;
1130
1131add_validator:
1132 if (strstr(type, "char") != NULL)
1133 validator_flags |= VALIDATOR_ENSURE_NULL;
1134
1135 validator = kmalloc(sizeof(*validator), GFP_KERNEL_ACCOUNT);
1136
1137 if (!validator) {
1138 kfree(field);
1139 return -ENOMEM;
1140 }
1141
1142 validator->flags = validator_flags;
1143 validator->offset = offset;
1144
1145 /* Want sequential access when validating */
1146 list_add_tail(&validator->user_event_link, &user->validators);
1147
1148add_field:
1149 field->type = type;
1150 field->name = name;
1151 field->offset = offset;
1152 field->size = size;
1153 field->is_signed = is_signed;
1154 field->filter_type = filter_type;
1155
1156 if (filter_type == FILTER_OTHER)
1157 field->filter_type = filter_assign_type(type);
1158
1159 list_add(&field->link, &user->fields);
1160
1161 /*
1162 * Min size from user writes that are required, this does not include
1163 * the size of trace_entry (common fields).
1164 */
1165 user->min_size = (offset + size) - sizeof(struct trace_entry);
1166
1167 return 0;
1168}
1169
1170/*
1171 * Parses the values of a field within the description
1172 * Format: type name [size]
1173 */
1174static int user_event_parse_field(char *field, struct user_event *user,
1175 u32 *offset)
1176{
1177 char *part, *type, *name;
1178 u32 depth = 0, saved_offset = *offset;
1179 int len, size = -EINVAL;
1180 bool is_struct = false;
1181
1182 field = skip_spaces(field);
1183
1184 if (*field == '\0')
1185 return 0;
1186
1187 /* Handle types that have a space within */
1188 len = str_has_prefix(field, "unsigned ");
1189 if (len)
1190 goto skip_next;
1191
1192 len = str_has_prefix(field, "struct ");
1193 if (len) {
1194 is_struct = true;
1195 goto skip_next;
1196 }
1197
1198 len = str_has_prefix(field, "__data_loc unsigned ");
1199 if (len)
1200 goto skip_next;
1201
1202 len = str_has_prefix(field, "__data_loc ");
1203 if (len)
1204 goto skip_next;
1205
1206 len = str_has_prefix(field, "__rel_loc unsigned ");
1207 if (len)
1208 goto skip_next;
1209
1210 len = str_has_prefix(field, "__rel_loc ");
1211 if (len)
1212 goto skip_next;
1213
1214 goto parse;
1215skip_next:
1216 type = field;
1217 field = strpbrk(field + len, " ");
1218
1219 if (field == NULL)
1220 return -EINVAL;
1221
1222 *field++ = '\0';
1223 depth++;
1224parse:
1225 name = NULL;
1226
1227 while ((part = strsep(&field, " ")) != NULL) {
1228 switch (depth++) {
1229 case FIELD_DEPTH_TYPE:
1230 type = part;
1231 break;
1232 case FIELD_DEPTH_NAME:
1233 name = part;
1234 break;
1235 case FIELD_DEPTH_SIZE:
1236 if (!is_struct)
1237 return -EINVAL;
1238
1239 if (kstrtou32(part, 10, &size))
1240 return -EINVAL;
1241 break;
1242 default:
1243 return -EINVAL;
1244 }
1245 }
1246
1247 if (depth < FIELD_DEPTH_SIZE || !name)
1248 return -EINVAL;
1249
1250 if (depth == FIELD_DEPTH_SIZE)
1251 size = user_field_size(type);
1252
1253 if (size == 0)
1254 return -EINVAL;
1255
1256 if (size < 0)
1257 return size;
1258
1259 *offset = saved_offset + size;
1260
1261 return user_event_add_field(user, type, name, saved_offset, size,
1262 type[0] != 'u', FILTER_OTHER);
1263}
1264
1265static int user_event_parse_fields(struct user_event *user, char *args)
1266{
1267 char *field;
1268 u32 offset = sizeof(struct trace_entry);
1269 int ret = -EINVAL;
1270
1271 if (args == NULL)
1272 return 0;
1273
1274 while ((field = strsep(&args, ";")) != NULL) {
1275 ret = user_event_parse_field(field, user, &offset);
1276
1277 if (ret)
1278 break;
1279 }
1280
1281 return ret;
1282}
1283
1284static struct trace_event_fields user_event_fields_array[1];
1285
1286static const char *user_field_format(const char *type)
1287{
1288 if (strcmp(type, "s64") == 0)
1289 return "%lld";
1290 if (strcmp(type, "u64") == 0)
1291 return "%llu";
1292 if (strcmp(type, "s32") == 0)
1293 return "%d";
1294 if (strcmp(type, "u32") == 0)
1295 return "%u";
1296 if (strcmp(type, "int") == 0)
1297 return "%d";
1298 if (strcmp(type, "unsigned int") == 0)
1299 return "%u";
1300 if (strcmp(type, "s16") == 0)
1301 return "%d";
1302 if (strcmp(type, "u16") == 0)
1303 return "%u";
1304 if (strcmp(type, "short") == 0)
1305 return "%d";
1306 if (strcmp(type, "unsigned short") == 0)
1307 return "%u";
1308 if (strcmp(type, "s8") == 0)
1309 return "%d";
1310 if (strcmp(type, "u8") == 0)
1311 return "%u";
1312 if (strcmp(type, "char") == 0)
1313 return "%d";
1314 if (strcmp(type, "unsigned char") == 0)
1315 return "%u";
1316 if (strstr(type, "char[") != NULL)
1317 return "%s";
1318
1319 /* Unknown, likely struct, allowed treat as 64-bit */
1320 return "%llu";
1321}
1322
1323static bool user_field_is_dyn_string(const char *type, const char **str_func)
1324{
1325 if (str_has_prefix(type, "__data_loc ")) {
1326 *str_func = "__get_str";
1327 goto check;
1328 }
1329
1330 if (str_has_prefix(type, "__rel_loc ")) {
1331 *str_func = "__get_rel_str";
1332 goto check;
1333 }
1334
1335 return false;
1336check:
1337 return strstr(type, "char") != NULL;
1338}
1339
1340#define LEN_OR_ZERO (len ? len - pos : 0)
1341static int user_dyn_field_set_string(int argc, const char **argv, int *iout,
1342 char *buf, int len, bool *colon)
1343{
1344 int pos = 0, i = *iout;
1345
1346 *colon = false;
1347
1348 for (; i < argc; ++i) {
1349 if (i != *iout)
1350 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1351
1352 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", argv[i]);
1353
1354 if (strchr(argv[i], ';')) {
1355 ++i;
1356 *colon = true;
1357 break;
1358 }
1359 }
1360
1361 /* Actual set, advance i */
1362 if (len != 0)
1363 *iout = i;
1364
1365 return pos + 1;
1366}
1367
1368static int user_field_set_string(struct ftrace_event_field *field,
1369 char *buf, int len, bool colon)
1370{
1371 int pos = 0;
1372
1373 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->type);
1374 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1375 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->name);
1376
1377 if (str_has_prefix(field->type, "struct "))
1378 pos += snprintf(buf + pos, LEN_OR_ZERO, " %d", field->size);
1379
1380 if (colon)
1381 pos += snprintf(buf + pos, LEN_OR_ZERO, ";");
1382
1383 return pos + 1;
1384}
1385
1386static int user_event_set_print_fmt(struct user_event *user, char *buf, int len)
1387{
1388 struct ftrace_event_field *field;
1389 struct list_head *head = &user->fields;
1390 int pos = 0, depth = 0;
1391 const char *str_func;
1392
1393 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1394
1395 list_for_each_entry_reverse(field, head, link) {
1396 if (depth != 0)
1397 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1398
1399 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s",
1400 field->name, user_field_format(field->type));
1401
1402 depth++;
1403 }
1404
1405 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1406
1407 list_for_each_entry_reverse(field, head, link) {
1408 if (user_field_is_dyn_string(field->type, &str_func))
1409 pos += snprintf(buf + pos, LEN_OR_ZERO,
1410 ", %s(%s)", str_func, field->name);
1411 else
1412 pos += snprintf(buf + pos, LEN_OR_ZERO,
1413 ", REC->%s", field->name);
1414 }
1415
1416 return pos + 1;
1417}
1418#undef LEN_OR_ZERO
1419
1420static int user_event_create_print_fmt(struct user_event *user)
1421{
1422 char *print_fmt;
1423 int len;
1424
1425 len = user_event_set_print_fmt(user, NULL, 0);
1426
1427 print_fmt = kmalloc(len, GFP_KERNEL_ACCOUNT);
1428
1429 if (!print_fmt)
1430 return -ENOMEM;
1431
1432 user_event_set_print_fmt(user, print_fmt, len);
1433
1434 user->call.print_fmt = print_fmt;
1435
1436 return 0;
1437}
1438
1439static enum print_line_t user_event_print_trace(struct trace_iterator *iter,
1440 int flags,
1441 struct trace_event *event)
1442{
1443 return print_event_fields(iter, event);
1444}
1445
1446static struct trace_event_functions user_event_funcs = {
1447 .trace = user_event_print_trace,
1448};
1449
1450static int user_event_set_call_visible(struct user_event *user, bool visible)
1451{
1452 int ret;
1453 const struct cred *old_cred;
1454 struct cred *cred;
1455
1456 cred = prepare_creds();
1457
1458 if (!cred)
1459 return -ENOMEM;
1460
1461 /*
1462 * While by default tracefs is locked down, systems can be configured
1463 * to allow user_event files to be less locked down. The extreme case
1464 * being "other" has read/write access to user_events_data/status.
1465 *
1466 * When not locked down, processes may not have permissions to
1467 * add/remove calls themselves to tracefs. We need to temporarily
1468 * switch to root file permission to allow for this scenario.
1469 */
1470 cred->fsuid = GLOBAL_ROOT_UID;
1471
1472 old_cred = override_creds(cred);
1473
1474 if (visible)
1475 ret = trace_add_event_call(&user->call);
1476 else
1477 ret = trace_remove_event_call(&user->call);
1478
1479 revert_creds(old_cred);
1480 put_cred(cred);
1481
1482 return ret;
1483}
1484
1485static int destroy_user_event(struct user_event *user)
1486{
1487 int ret = 0;
1488
1489 lockdep_assert_held(&event_mutex);
1490
1491 /* Must destroy fields before call removal */
1492 user_event_destroy_fields(user);
1493
1494 ret = user_event_set_call_visible(user, false);
1495
1496 if (ret)
1497 return ret;
1498
1499 dyn_event_remove(&user->devent);
1500 hash_del(&user->node);
1501
1502 user_event_destroy_validators(user);
1503
1504 /* If we have different names, both must be freed */
1505 if (EVENT_NAME(user) != EVENT_TP_NAME(user))
1506 kfree(EVENT_TP_NAME(user));
1507
1508 kfree(user->call.print_fmt);
1509 kfree(EVENT_NAME(user));
1510 kfree(user);
1511
1512 if (current_user_events > 0)
1513 current_user_events--;
1514 else
1515 pr_alert("BUG: Bad current_user_events\n");
1516
1517 return ret;
1518}
1519
1520static struct user_event *find_user_event(struct user_event_group *group,
1521 char *name, int argc, const char **argv,
1522 u32 flags, u32 *outkey)
1523{
1524 struct user_event *user;
1525 u32 key = user_event_key(name);
1526
1527 *outkey = key;
1528
1529 hash_for_each_possible(group->register_table, user, node, key) {
1530 /*
1531 * Single-format events shouldn't return multi-format
1532 * events. Callers expect the underlying tracepoint to match
1533 * the name exactly in these cases. Only check like-formats.
1534 */
1535 if (EVENT_MULTI_FORMAT(flags) != EVENT_MULTI_FORMAT(user->reg_flags))
1536 continue;
1537
1538 if (strcmp(EVENT_NAME(user), name))
1539 continue;
1540
1541 if (user_fields_match(user, argc, argv))
1542 return user_event_get(user);
1543
1544 /* Scan others if this is a multi-format event */
1545 if (EVENT_MULTI_FORMAT(flags))
1546 continue;
1547
1548 return ERR_PTR(-EADDRINUSE);
1549 }
1550
1551 return NULL;
1552}
1553
1554static int user_event_validate(struct user_event *user, void *data, int len)
1555{
1556 struct list_head *head = &user->validators;
1557 struct user_event_validator *validator;
1558 void *pos, *end = data + len;
1559 u32 loc, offset, size;
1560
1561 list_for_each_entry(validator, head, user_event_link) {
1562 pos = data + validator->offset;
1563
1564 /* Already done min_size check, no bounds check here */
1565 loc = *(u32 *)pos;
1566 offset = loc & 0xffff;
1567 size = loc >> 16;
1568
1569 if (likely(validator->flags & VALIDATOR_REL))
1570 pos += offset + sizeof(loc);
1571 else
1572 pos = data + offset;
1573
1574 pos += size;
1575
1576 if (unlikely(pos > end))
1577 return -EFAULT;
1578
1579 if (likely(validator->flags & VALIDATOR_ENSURE_NULL))
1580 if (unlikely(*(char *)(pos - 1) != '\0'))
1581 return -EFAULT;
1582 }
1583
1584 return 0;
1585}
1586
1587/*
1588 * Writes the user supplied payload out to a trace file.
1589 */
1590static void user_event_ftrace(struct user_event *user, struct iov_iter *i,
1591 void *tpdata, bool *faulted)
1592{
1593 struct trace_event_file *file;
1594 struct trace_entry *entry;
1595 struct trace_event_buffer event_buffer;
1596 size_t size = sizeof(*entry) + i->count;
1597
1598 file = (struct trace_event_file *)tpdata;
1599
1600 if (!file ||
1601 !(file->flags & EVENT_FILE_FL_ENABLED) ||
1602 trace_trigger_soft_disabled(file))
1603 return;
1604
1605 /* Allocates and fills trace_entry, + 1 of this is data payload */
1606 entry = trace_event_buffer_reserve(&event_buffer, file, size);
1607
1608 if (unlikely(!entry))
1609 return;
1610
1611 if (unlikely(i->count != 0 && !copy_nofault(entry + 1, i->count, i)))
1612 goto discard;
1613
1614 if (!list_empty(&user->validators) &&
1615 unlikely(user_event_validate(user, entry, size)))
1616 goto discard;
1617
1618 trace_event_buffer_commit(&event_buffer);
1619
1620 return;
1621discard:
1622 *faulted = true;
1623 __trace_event_discard_commit(event_buffer.buffer,
1624 event_buffer.event);
1625}
1626
1627#ifdef CONFIG_PERF_EVENTS
1628/*
1629 * Writes the user supplied payload out to perf ring buffer.
1630 */
1631static void user_event_perf(struct user_event *user, struct iov_iter *i,
1632 void *tpdata, bool *faulted)
1633{
1634 struct hlist_head *perf_head;
1635
1636 perf_head = this_cpu_ptr(user->call.perf_events);
1637
1638 if (perf_head && !hlist_empty(perf_head)) {
1639 struct trace_entry *perf_entry;
1640 struct pt_regs *regs;
1641 size_t size = sizeof(*perf_entry) + i->count;
1642 int context;
1643
1644 perf_entry = perf_trace_buf_alloc(ALIGN(size, 8),
1645 ®s, &context);
1646
1647 if (unlikely(!perf_entry))
1648 return;
1649
1650 perf_fetch_caller_regs(regs);
1651
1652 if (unlikely(i->count != 0 && !copy_nofault(perf_entry + 1, i->count, i)))
1653 goto discard;
1654
1655 if (!list_empty(&user->validators) &&
1656 unlikely(user_event_validate(user, perf_entry, size)))
1657 goto discard;
1658
1659 perf_trace_buf_submit(perf_entry, size, context,
1660 user->call.event.type, 1, regs,
1661 perf_head, NULL);
1662
1663 return;
1664discard:
1665 *faulted = true;
1666 perf_swevent_put_recursion_context(context);
1667 }
1668}
1669#endif
1670
1671/*
1672 * Update the enabled bit among all user processes.
1673 */
1674static void update_enable_bit_for(struct user_event *user)
1675{
1676 struct tracepoint *tp = &user->tracepoint;
1677 char status = 0;
1678
1679 if (static_key_enabled(&tp->key)) {
1680 struct tracepoint_func *probe_func_ptr;
1681 user_event_func_t probe_func;
1682
1683 rcu_read_lock_sched();
1684
1685 probe_func_ptr = rcu_dereference_sched(tp->funcs);
1686
1687 if (probe_func_ptr) {
1688 do {
1689 probe_func = probe_func_ptr->func;
1690
1691 if (probe_func == user_event_ftrace)
1692 status |= EVENT_STATUS_FTRACE;
1693#ifdef CONFIG_PERF_EVENTS
1694 else if (probe_func == user_event_perf)
1695 status |= EVENT_STATUS_PERF;
1696#endif
1697 else
1698 status |= EVENT_STATUS_OTHER;
1699 } while ((++probe_func_ptr)->func);
1700 }
1701
1702 rcu_read_unlock_sched();
1703 }
1704
1705 user->status = status;
1706
1707 user_event_enabler_update(user);
1708}
1709
1710/*
1711 * Register callback for our events from tracing sub-systems.
1712 */
1713static int user_event_reg(struct trace_event_call *call,
1714 enum trace_reg type,
1715 void *data)
1716{
1717 struct user_event *user = (struct user_event *)call->data;
1718 int ret = 0;
1719
1720 if (!user)
1721 return -ENOENT;
1722
1723 switch (type) {
1724 case TRACE_REG_REGISTER:
1725 ret = tracepoint_probe_register(call->tp,
1726 call->class->probe,
1727 data);
1728 if (!ret)
1729 goto inc;
1730 break;
1731
1732 case TRACE_REG_UNREGISTER:
1733 tracepoint_probe_unregister(call->tp,
1734 call->class->probe,
1735 data);
1736 goto dec;
1737
1738#ifdef CONFIG_PERF_EVENTS
1739 case TRACE_REG_PERF_REGISTER:
1740 ret = tracepoint_probe_register(call->tp,
1741 call->class->perf_probe,
1742 data);
1743 if (!ret)
1744 goto inc;
1745 break;
1746
1747 case TRACE_REG_PERF_UNREGISTER:
1748 tracepoint_probe_unregister(call->tp,
1749 call->class->perf_probe,
1750 data);
1751 goto dec;
1752
1753 case TRACE_REG_PERF_OPEN:
1754 case TRACE_REG_PERF_CLOSE:
1755 case TRACE_REG_PERF_ADD:
1756 case TRACE_REG_PERF_DEL:
1757 break;
1758#endif
1759 }
1760
1761 return ret;
1762inc:
1763 user_event_get(user);
1764 update_enable_bit_for(user);
1765 return 0;
1766dec:
1767 update_enable_bit_for(user);
1768 user_event_put(user, true);
1769 return 0;
1770}
1771
1772static int user_event_create(const char *raw_command)
1773{
1774 struct user_event_group *group;
1775 struct user_event *user;
1776 char *name;
1777 int ret;
1778
1779 if (!str_has_prefix(raw_command, USER_EVENTS_PREFIX))
1780 return -ECANCELED;
1781
1782 raw_command += USER_EVENTS_PREFIX_LEN;
1783 raw_command = skip_spaces(raw_command);
1784
1785 name = kstrdup(raw_command, GFP_KERNEL_ACCOUNT);
1786
1787 if (!name)
1788 return -ENOMEM;
1789
1790 group = current_user_event_group();
1791
1792 if (!group) {
1793 kfree(name);
1794 return -ENOENT;
1795 }
1796
1797 mutex_lock(&group->reg_mutex);
1798
1799 /* Dyn events persist, otherwise they would cleanup immediately */
1800 ret = user_event_parse_cmd(group, name, &user, USER_EVENT_REG_PERSIST);
1801
1802 if (!ret)
1803 user_event_put(user, false);
1804
1805 mutex_unlock(&group->reg_mutex);
1806
1807 if (ret)
1808 kfree(name);
1809
1810 return ret;
1811}
1812
1813static int user_event_show(struct seq_file *m, struct dyn_event *ev)
1814{
1815 struct user_event *user = container_of(ev, struct user_event, devent);
1816 struct ftrace_event_field *field;
1817 struct list_head *head;
1818 int depth = 0;
1819
1820 seq_printf(m, "%s%s", USER_EVENTS_PREFIX, EVENT_NAME(user));
1821
1822 head = trace_get_fields(&user->call);
1823
1824 list_for_each_entry_reverse(field, head, link) {
1825 if (depth == 0)
1826 seq_puts(m, " ");
1827 else
1828 seq_puts(m, "; ");
1829
1830 seq_printf(m, "%s %s", field->type, field->name);
1831
1832 if (str_has_prefix(field->type, "struct "))
1833 seq_printf(m, " %d", field->size);
1834
1835 depth++;
1836 }
1837
1838 seq_puts(m, "\n");
1839
1840 return 0;
1841}
1842
1843static bool user_event_is_busy(struct dyn_event *ev)
1844{
1845 struct user_event *user = container_of(ev, struct user_event, devent);
1846
1847 return !user_event_last_ref(user);
1848}
1849
1850static int user_event_free(struct dyn_event *ev)
1851{
1852 struct user_event *user = container_of(ev, struct user_event, devent);
1853
1854 if (!user_event_last_ref(user))
1855 return -EBUSY;
1856
1857 if (!user_event_capable(user->reg_flags))
1858 return -EPERM;
1859
1860 return destroy_user_event(user);
1861}
1862
1863static bool user_field_match(struct ftrace_event_field *field, int argc,
1864 const char **argv, int *iout)
1865{
1866 char *field_name = NULL, *dyn_field_name = NULL;
1867 bool colon = false, match = false;
1868 int dyn_len, len;
1869
1870 if (*iout >= argc)
1871 return false;
1872
1873 dyn_len = user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1874 0, &colon);
1875
1876 len = user_field_set_string(field, field_name, 0, colon);
1877
1878 if (dyn_len != len)
1879 return false;
1880
1881 dyn_field_name = kmalloc(dyn_len, GFP_KERNEL);
1882 field_name = kmalloc(len, GFP_KERNEL);
1883
1884 if (!dyn_field_name || !field_name)
1885 goto out;
1886
1887 user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1888 dyn_len, &colon);
1889
1890 user_field_set_string(field, field_name, len, colon);
1891
1892 match = strcmp(dyn_field_name, field_name) == 0;
1893out:
1894 kfree(dyn_field_name);
1895 kfree(field_name);
1896
1897 return match;
1898}
1899
1900static bool user_fields_match(struct user_event *user, int argc,
1901 const char **argv)
1902{
1903 struct ftrace_event_field *field;
1904 struct list_head *head = &user->fields;
1905 int i = 0;
1906
1907 if (argc == 0)
1908 return list_empty(head);
1909
1910 list_for_each_entry_reverse(field, head, link) {
1911 if (!user_field_match(field, argc, argv, &i))
1912 return false;
1913 }
1914
1915 if (i != argc)
1916 return false;
1917
1918 return true;
1919}
1920
1921static bool user_event_match(const char *system, const char *event,
1922 int argc, const char **argv, struct dyn_event *ev)
1923{
1924 struct user_event *user = container_of(ev, struct user_event, devent);
1925 bool match;
1926
1927 match = strcmp(EVENT_NAME(user), event) == 0;
1928
1929 if (match && system) {
1930 match = strcmp(system, user->group->system_name) == 0 ||
1931 strcmp(system, user->group->system_multi_name) == 0;
1932 }
1933
1934 if (match)
1935 match = user_fields_match(user, argc, argv);
1936
1937 return match;
1938}
1939
1940static struct dyn_event_operations user_event_dops = {
1941 .create = user_event_create,
1942 .show = user_event_show,
1943 .is_busy = user_event_is_busy,
1944 .free = user_event_free,
1945 .match = user_event_match,
1946};
1947
1948static int user_event_trace_register(struct user_event *user)
1949{
1950 int ret;
1951
1952 ret = register_trace_event(&user->call.event);
1953
1954 if (!ret)
1955 return -ENODEV;
1956
1957 ret = user_event_set_call_visible(user, true);
1958
1959 if (ret)
1960 unregister_trace_event(&user->call.event);
1961
1962 return ret;
1963}
1964
1965static int user_event_set_tp_name(struct user_event *user)
1966{
1967 lockdep_assert_held(&user->group->reg_mutex);
1968
1969 if (EVENT_MULTI_FORMAT(user->reg_flags)) {
1970 char *multi_name;
1971
1972 multi_name = kasprintf(GFP_KERNEL_ACCOUNT, "%s.%llx",
1973 user->reg_name, user->group->multi_id);
1974
1975 if (!multi_name)
1976 return -ENOMEM;
1977
1978 user->call.name = multi_name;
1979 user->tracepoint.name = multi_name;
1980
1981 /* Inc to ensure unique multi-event name next time */
1982 user->group->multi_id++;
1983 } else {
1984 /* Non Multi-format uses register name */
1985 user->call.name = user->reg_name;
1986 user->tracepoint.name = user->reg_name;
1987 }
1988
1989 return 0;
1990}
1991
1992/*
1993 * Counts how many ';' without a trailing space are in the args.
1994 */
1995static int count_semis_no_space(char *args)
1996{
1997 int count = 0;
1998
1999 while ((args = strchr(args, ';'))) {
2000 args++;
2001
2002 if (!isspace(*args))
2003 count++;
2004 }
2005
2006 return count;
2007}
2008
2009/*
2010 * Copies the arguments while ensuring all ';' have a trailing space.
2011 */
2012static char *insert_space_after_semis(char *args, int count)
2013{
2014 char *fixed, *pos;
2015 int len;
2016
2017 len = strlen(args) + count;
2018 fixed = kmalloc(len + 1, GFP_KERNEL);
2019
2020 if (!fixed)
2021 return NULL;
2022
2023 pos = fixed;
2024
2025 /* Insert a space after ';' if there is no trailing space. */
2026 while (*args) {
2027 *pos = *args++;
2028
2029 if (*pos++ == ';' && !isspace(*args))
2030 *pos++ = ' ';
2031 }
2032
2033 *pos = '\0';
2034
2035 return fixed;
2036}
2037
2038static char **user_event_argv_split(char *args, int *argc)
2039{
2040 char **split;
2041 char *fixed;
2042 int count;
2043
2044 /* Count how many ';' without a trailing space */
2045 count = count_semis_no_space(args);
2046
2047 /* No fixup is required */
2048 if (!count)
2049 return argv_split(GFP_KERNEL, args, argc);
2050
2051 /* We must fixup 'field;field' to 'field; field' */
2052 fixed = insert_space_after_semis(args, count);
2053
2054 if (!fixed)
2055 return NULL;
2056
2057 /* We do a normal split afterwards */
2058 split = argv_split(GFP_KERNEL, fixed, argc);
2059
2060 /* We can free since argv_split makes a copy */
2061 kfree(fixed);
2062
2063 return split;
2064}
2065
2066/*
2067 * Parses the event name, arguments and flags then registers if successful.
2068 * The name buffer lifetime is owned by this method for success cases only.
2069 * Upon success the returned user_event has its ref count increased by 1.
2070 */
2071static int user_event_parse(struct user_event_group *group, char *name,
2072 char *args, char *flags,
2073 struct user_event **newuser, int reg_flags)
2074{
2075 struct user_event *user;
2076 char **argv = NULL;
2077 int argc = 0;
2078 int ret;
2079 u32 key;
2080
2081 /* Currently don't support any text based flags */
2082 if (flags != NULL)
2083 return -EINVAL;
2084
2085 if (!user_event_capable(reg_flags))
2086 return -EPERM;
2087
2088 if (args) {
2089 argv = user_event_argv_split(args, &argc);
2090
2091 if (!argv)
2092 return -ENOMEM;
2093 }
2094
2095 /* Prevent dyn_event from racing */
2096 mutex_lock(&event_mutex);
2097 user = find_user_event(group, name, argc, (const char **)argv,
2098 reg_flags, &key);
2099 mutex_unlock(&event_mutex);
2100
2101 if (argv)
2102 argv_free(argv);
2103
2104 if (IS_ERR(user))
2105 return PTR_ERR(user);
2106
2107 if (user) {
2108 *newuser = user;
2109 /*
2110 * Name is allocated by caller, free it since it already exists.
2111 * Caller only worries about failure cases for freeing.
2112 */
2113 kfree(name);
2114
2115 return 0;
2116 }
2117
2118 user = kzalloc(sizeof(*user), GFP_KERNEL_ACCOUNT);
2119
2120 if (!user)
2121 return -ENOMEM;
2122
2123 INIT_LIST_HEAD(&user->class.fields);
2124 INIT_LIST_HEAD(&user->fields);
2125 INIT_LIST_HEAD(&user->validators);
2126
2127 user->group = group;
2128 user->reg_name = name;
2129 user->reg_flags = reg_flags;
2130
2131 ret = user_event_set_tp_name(user);
2132
2133 if (ret)
2134 goto put_user;
2135
2136 ret = user_event_parse_fields(user, args);
2137
2138 if (ret)
2139 goto put_user;
2140
2141 ret = user_event_create_print_fmt(user);
2142
2143 if (ret)
2144 goto put_user;
2145
2146 user->call.data = user;
2147 user->call.class = &user->class;
2148 user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
2149 user->call.tp = &user->tracepoint;
2150 user->call.event.funcs = &user_event_funcs;
2151
2152 if (EVENT_MULTI_FORMAT(user->reg_flags))
2153 user->class.system = group->system_multi_name;
2154 else
2155 user->class.system = group->system_name;
2156
2157 user->class.fields_array = user_event_fields_array;
2158 user->class.get_fields = user_event_get_fields;
2159 user->class.reg = user_event_reg;
2160 user->class.probe = user_event_ftrace;
2161#ifdef CONFIG_PERF_EVENTS
2162 user->class.perf_probe = user_event_perf;
2163#endif
2164
2165 mutex_lock(&event_mutex);
2166
2167 if (current_user_events >= max_user_events) {
2168 ret = -EMFILE;
2169 goto put_user_lock;
2170 }
2171
2172 ret = user_event_trace_register(user);
2173
2174 if (ret)
2175 goto put_user_lock;
2176
2177 if (user->reg_flags & USER_EVENT_REG_PERSIST) {
2178 /* Ensure we track self ref and caller ref (2) */
2179 refcount_set(&user->refcnt, 2);
2180 } else {
2181 /* Ensure we track only caller ref (1) */
2182 refcount_set(&user->refcnt, 1);
2183 }
2184
2185 dyn_event_init(&user->devent, &user_event_dops);
2186 dyn_event_add(&user->devent, &user->call);
2187 hash_add(group->register_table, &user->node, key);
2188 current_user_events++;
2189
2190 mutex_unlock(&event_mutex);
2191
2192 *newuser = user;
2193 return 0;
2194put_user_lock:
2195 mutex_unlock(&event_mutex);
2196put_user:
2197 user_event_destroy_fields(user);
2198 user_event_destroy_validators(user);
2199 kfree(user->call.print_fmt);
2200
2201 /* Caller frees reg_name on error, but not multi-name */
2202 if (EVENT_NAME(user) != EVENT_TP_NAME(user))
2203 kfree(EVENT_TP_NAME(user));
2204
2205 kfree(user);
2206 return ret;
2207}
2208
2209/*
2210 * Deletes previously created events if they are no longer being used.
2211 */
2212static int delete_user_event(struct user_event_group *group, char *name)
2213{
2214 struct user_event *user;
2215 struct hlist_node *tmp;
2216 u32 key = user_event_key(name);
2217 int ret = -ENOENT;
2218
2219 /* Attempt to delete all event(s) with the name passed in */
2220 hash_for_each_possible_safe(group->register_table, user, tmp, node, key) {
2221 if (strcmp(EVENT_NAME(user), name))
2222 continue;
2223
2224 if (!user_event_last_ref(user))
2225 return -EBUSY;
2226
2227 if (!user_event_capable(user->reg_flags))
2228 return -EPERM;
2229
2230 ret = destroy_user_event(user);
2231
2232 if (ret)
2233 goto out;
2234 }
2235out:
2236 return ret;
2237}
2238
2239/*
2240 * Validates the user payload and writes via iterator.
2241 */
2242static ssize_t user_events_write_core(struct file *file, struct iov_iter *i)
2243{
2244 struct user_event_file_info *info = file->private_data;
2245 struct user_event_refs *refs;
2246 struct user_event *user = NULL;
2247 struct tracepoint *tp;
2248 ssize_t ret = i->count;
2249 int idx;
2250
2251 if (unlikely(copy_from_iter(&idx, sizeof(idx), i) != sizeof(idx)))
2252 return -EFAULT;
2253
2254 if (idx < 0)
2255 return -EINVAL;
2256
2257 rcu_read_lock_sched();
2258
2259 refs = rcu_dereference_sched(info->refs);
2260
2261 /*
2262 * The refs->events array is protected by RCU, and new items may be
2263 * added. But the user retrieved from indexing into the events array
2264 * shall be immutable while the file is opened.
2265 */
2266 if (likely(refs && idx < refs->count))
2267 user = refs->events[idx];
2268
2269 rcu_read_unlock_sched();
2270
2271 if (unlikely(user == NULL))
2272 return -ENOENT;
2273
2274 if (unlikely(i->count < user->min_size))
2275 return -EINVAL;
2276
2277 tp = &user->tracepoint;
2278
2279 /*
2280 * It's possible key.enabled disables after this check, however
2281 * we don't mind if a few events are included in this condition.
2282 */
2283 if (likely(static_key_enabled(&tp->key))) {
2284 struct tracepoint_func *probe_func_ptr;
2285 user_event_func_t probe_func;
2286 struct iov_iter copy;
2287 void *tpdata;
2288 bool faulted;
2289
2290 if (unlikely(fault_in_iov_iter_readable(i, i->count)))
2291 return -EFAULT;
2292
2293 faulted = false;
2294
2295 rcu_read_lock_sched();
2296
2297 probe_func_ptr = rcu_dereference_sched(tp->funcs);
2298
2299 if (probe_func_ptr) {
2300 do {
2301 copy = *i;
2302 probe_func = probe_func_ptr->func;
2303 tpdata = probe_func_ptr->data;
2304 probe_func(user, ©, tpdata, &faulted);
2305 } while ((++probe_func_ptr)->func);
2306 }
2307
2308 rcu_read_unlock_sched();
2309
2310 if (unlikely(faulted))
2311 return -EFAULT;
2312 } else
2313 return -EBADF;
2314
2315 return ret;
2316}
2317
2318static int user_events_open(struct inode *node, struct file *file)
2319{
2320 struct user_event_group *group;
2321 struct user_event_file_info *info;
2322
2323 group = current_user_event_group();
2324
2325 if (!group)
2326 return -ENOENT;
2327
2328 info = kzalloc(sizeof(*info), GFP_KERNEL_ACCOUNT);
2329
2330 if (!info)
2331 return -ENOMEM;
2332
2333 info->group = group;
2334
2335 file->private_data = info;
2336
2337 return 0;
2338}
2339
2340static ssize_t user_events_write(struct file *file, const char __user *ubuf,
2341 size_t count, loff_t *ppos)
2342{
2343 struct iov_iter i;
2344
2345 if (unlikely(*ppos != 0))
2346 return -EFAULT;
2347
2348 if (unlikely(import_ubuf(ITER_SOURCE, (char __user *)ubuf, count, &i)))
2349 return -EFAULT;
2350
2351 return user_events_write_core(file, &i);
2352}
2353
2354static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i)
2355{
2356 return user_events_write_core(kp->ki_filp, i);
2357}
2358
2359static int user_events_ref_add(struct user_event_file_info *info,
2360 struct user_event *user)
2361{
2362 struct user_event_group *group = info->group;
2363 struct user_event_refs *refs, *new_refs;
2364 int i, size, count = 0;
2365
2366 refs = rcu_dereference_protected(info->refs,
2367 lockdep_is_held(&group->reg_mutex));
2368
2369 if (refs) {
2370 count = refs->count;
2371
2372 for (i = 0; i < count; ++i)
2373 if (refs->events[i] == user)
2374 return i;
2375 }
2376
2377 size = struct_size(refs, events, count + 1);
2378
2379 new_refs = kzalloc(size, GFP_KERNEL_ACCOUNT);
2380
2381 if (!new_refs)
2382 return -ENOMEM;
2383
2384 new_refs->count = count + 1;
2385
2386 for (i = 0; i < count; ++i)
2387 new_refs->events[i] = refs->events[i];
2388
2389 new_refs->events[i] = user_event_get(user);
2390
2391 rcu_assign_pointer(info->refs, new_refs);
2392
2393 if (refs)
2394 kfree_rcu(refs, rcu);
2395
2396 return i;
2397}
2398
2399static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg)
2400{
2401 u32 size;
2402 long ret;
2403
2404 ret = get_user(size, &ureg->size);
2405
2406 if (ret)
2407 return ret;
2408
2409 if (size > PAGE_SIZE)
2410 return -E2BIG;
2411
2412 if (size < offsetofend(struct user_reg, write_index))
2413 return -EINVAL;
2414
2415 ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
2416
2417 if (ret)
2418 return ret;
2419
2420 /* Ensure only valid flags */
2421 if (kreg->flags & ~(USER_EVENT_REG_MAX-1))
2422 return -EINVAL;
2423
2424 /* Ensure supported size */
2425 switch (kreg->enable_size) {
2426 case 4:
2427 /* 32-bit */
2428 break;
2429#if BITS_PER_LONG >= 64
2430 case 8:
2431 /* 64-bit */
2432 break;
2433#endif
2434 default:
2435 return -EINVAL;
2436 }
2437
2438 /* Ensure natural alignment */
2439 if (kreg->enable_addr % kreg->enable_size)
2440 return -EINVAL;
2441
2442 /* Ensure bit range for size */
2443 if (kreg->enable_bit > (kreg->enable_size * BITS_PER_BYTE) - 1)
2444 return -EINVAL;
2445
2446 /* Ensure accessible */
2447 if (!access_ok((const void __user *)(uintptr_t)kreg->enable_addr,
2448 kreg->enable_size))
2449 return -EFAULT;
2450
2451 kreg->size = size;
2452
2453 return 0;
2454}
2455
2456/*
2457 * Registers a user_event on behalf of a user process.
2458 */
2459static long user_events_ioctl_reg(struct user_event_file_info *info,
2460 unsigned long uarg)
2461{
2462 struct user_reg __user *ureg = (struct user_reg __user *)uarg;
2463 struct user_reg reg;
2464 struct user_event *user;
2465 struct user_event_enabler *enabler;
2466 char *name;
2467 long ret;
2468 int write_result;
2469
2470 ret = user_reg_get(ureg, ®);
2471
2472 if (ret)
2473 return ret;
2474
2475 /*
2476 * Prevent users from using the same address and bit multiple times
2477 * within the same mm address space. This can cause unexpected behavior
2478 * for user processes that is far easier to debug if this is explictly
2479 * an error upon registering.
2480 */
2481 if (current_user_event_enabler_exists((unsigned long)reg.enable_addr,
2482 reg.enable_bit))
2483 return -EADDRINUSE;
2484
2485 name = strndup_user((const char __user *)(uintptr_t)reg.name_args,
2486 MAX_EVENT_DESC);
2487
2488 if (IS_ERR(name)) {
2489 ret = PTR_ERR(name);
2490 return ret;
2491 }
2492
2493 ret = user_event_parse_cmd(info->group, name, &user, reg.flags);
2494
2495 if (ret) {
2496 kfree(name);
2497 return ret;
2498 }
2499
2500 ret = user_events_ref_add(info, user);
2501
2502 /* No longer need parse ref, ref_add either worked or not */
2503 user_event_put(user, false);
2504
2505 /* Positive number is index and valid */
2506 if (ret < 0)
2507 return ret;
2508
2509 /*
2510 * user_events_ref_add succeeded:
2511 * At this point we have a user_event, it's lifetime is bound by the
2512 * reference count, not this file. If anything fails, the user_event
2513 * still has a reference until the file is released. During release
2514 * any remaining references (from user_events_ref_add) are decremented.
2515 *
2516 * Attempt to create an enabler, which too has a lifetime tied in the
2517 * same way for the event. Once the task that caused the enabler to be
2518 * created exits or issues exec() then the enablers it has created
2519 * will be destroyed and the ref to the event will be decremented.
2520 */
2521 enabler = user_event_enabler_create(®, user, &write_result);
2522
2523 if (!enabler)
2524 return -ENOMEM;
2525
2526 /* Write failed/faulted, give error back to caller */
2527 if (write_result)
2528 return write_result;
2529
2530 put_user((u32)ret, &ureg->write_index);
2531
2532 return 0;
2533}
2534
2535/*
2536 * Deletes a user_event on behalf of a user process.
2537 */
2538static long user_events_ioctl_del(struct user_event_file_info *info,
2539 unsigned long uarg)
2540{
2541 void __user *ubuf = (void __user *)uarg;
2542 char *name;
2543 long ret;
2544
2545 name = strndup_user(ubuf, MAX_EVENT_DESC);
2546
2547 if (IS_ERR(name))
2548 return PTR_ERR(name);
2549
2550 /* event_mutex prevents dyn_event from racing */
2551 mutex_lock(&event_mutex);
2552 ret = delete_user_event(info->group, name);
2553 mutex_unlock(&event_mutex);
2554
2555 kfree(name);
2556
2557 return ret;
2558}
2559
2560static long user_unreg_get(struct user_unreg __user *ureg,
2561 struct user_unreg *kreg)
2562{
2563 u32 size;
2564 long ret;
2565
2566 ret = get_user(size, &ureg->size);
2567
2568 if (ret)
2569 return ret;
2570
2571 if (size > PAGE_SIZE)
2572 return -E2BIG;
2573
2574 if (size < offsetofend(struct user_unreg, disable_addr))
2575 return -EINVAL;
2576
2577 ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
2578
2579 /* Ensure no reserved values, since we don't support any yet */
2580 if (kreg->__reserved || kreg->__reserved2)
2581 return -EINVAL;
2582
2583 return ret;
2584}
2585
2586static int user_event_mm_clear_bit(struct user_event_mm *user_mm,
2587 unsigned long uaddr, unsigned char bit,
2588 unsigned long flags)
2589{
2590 struct user_event_enabler enabler;
2591 int result;
2592 int attempt = 0;
2593
2594 memset(&enabler, 0, sizeof(enabler));
2595 enabler.addr = uaddr;
2596 enabler.values = bit | flags;
2597retry:
2598 /* Prevents state changes from racing with new enablers */
2599 mutex_lock(&event_mutex);
2600
2601 /* Force the bit to be cleared, since no event is attached */
2602 mmap_read_lock(user_mm->mm);
2603 result = user_event_enabler_write(user_mm, &enabler, false, &attempt);
2604 mmap_read_unlock(user_mm->mm);
2605
2606 mutex_unlock(&event_mutex);
2607
2608 if (result) {
2609 /* Attempt to fault-in and retry if it worked */
2610 if (!user_event_mm_fault_in(user_mm, uaddr, attempt))
2611 goto retry;
2612 }
2613
2614 return result;
2615}
2616
2617/*
2618 * Unregisters an enablement address/bit within a task/user mm.
2619 */
2620static long user_events_ioctl_unreg(unsigned long uarg)
2621{
2622 struct user_unreg __user *ureg = (struct user_unreg __user *)uarg;
2623 struct user_event_mm *mm = current->user_event_mm;
2624 struct user_event_enabler *enabler, *next;
2625 struct user_unreg reg;
2626 unsigned long flags;
2627 long ret;
2628
2629 ret = user_unreg_get(ureg, ®);
2630
2631 if (ret)
2632 return ret;
2633
2634 if (!mm)
2635 return -ENOENT;
2636
2637 flags = 0;
2638 ret = -ENOENT;
2639
2640 /*
2641 * Flags freeing and faulting are used to indicate if the enabler is in
2642 * use at all. When faulting is set a page-fault is occurring asyncly.
2643 * During async fault if freeing is set, the enabler will be destroyed.
2644 * If no async fault is happening, we can destroy it now since we hold
2645 * the event_mutex during these checks.
2646 */
2647 mutex_lock(&event_mutex);
2648
2649 list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link) {
2650 if (enabler->addr == reg.disable_addr &&
2651 ENABLE_BIT(enabler) == reg.disable_bit) {
2652 set_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler));
2653
2654 /* We must keep compat flags for the clear */
2655 flags |= enabler->values & ENABLE_VAL_COMPAT_MASK;
2656
2657 if (!test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler)))
2658 user_event_enabler_destroy(enabler, true);
2659
2660 /* Removed at least one */
2661 ret = 0;
2662 }
2663 }
2664
2665 mutex_unlock(&event_mutex);
2666
2667 /* Ensure bit is now cleared for user, regardless of event status */
2668 if (!ret)
2669 ret = user_event_mm_clear_bit(mm, reg.disable_addr,
2670 reg.disable_bit, flags);
2671
2672 return ret;
2673}
2674
2675/*
2676 * Handles the ioctl from user mode to register or alter operations.
2677 */
2678static long user_events_ioctl(struct file *file, unsigned int cmd,
2679 unsigned long uarg)
2680{
2681 struct user_event_file_info *info = file->private_data;
2682 struct user_event_group *group = info->group;
2683 long ret = -ENOTTY;
2684
2685 switch (cmd) {
2686 case DIAG_IOCSREG:
2687 mutex_lock(&group->reg_mutex);
2688 ret = user_events_ioctl_reg(info, uarg);
2689 mutex_unlock(&group->reg_mutex);
2690 break;
2691
2692 case DIAG_IOCSDEL:
2693 mutex_lock(&group->reg_mutex);
2694 ret = user_events_ioctl_del(info, uarg);
2695 mutex_unlock(&group->reg_mutex);
2696 break;
2697
2698 case DIAG_IOCSUNREG:
2699 mutex_lock(&group->reg_mutex);
2700 ret = user_events_ioctl_unreg(uarg);
2701 mutex_unlock(&group->reg_mutex);
2702 break;
2703 }
2704
2705 return ret;
2706}
2707
2708/*
2709 * Handles the final close of the file from user mode.
2710 */
2711static int user_events_release(struct inode *node, struct file *file)
2712{
2713 struct user_event_file_info *info = file->private_data;
2714 struct user_event_group *group;
2715 struct user_event_refs *refs;
2716 int i;
2717
2718 if (!info)
2719 return -EINVAL;
2720
2721 group = info->group;
2722
2723 /*
2724 * Ensure refs cannot change under any situation by taking the
2725 * register mutex during the final freeing of the references.
2726 */
2727 mutex_lock(&group->reg_mutex);
2728
2729 refs = info->refs;
2730
2731 if (!refs)
2732 goto out;
2733
2734 /*
2735 * The lifetime of refs has reached an end, it's tied to this file.
2736 * The underlying user_events are ref counted, and cannot be freed.
2737 * After this decrement, the user_events may be freed elsewhere.
2738 */
2739 for (i = 0; i < refs->count; ++i)
2740 user_event_put(refs->events[i], false);
2741
2742out:
2743 file->private_data = NULL;
2744
2745 mutex_unlock(&group->reg_mutex);
2746
2747 kfree(refs);
2748 kfree(info);
2749
2750 return 0;
2751}
2752
2753static const struct file_operations user_data_fops = {
2754 .open = user_events_open,
2755 .write = user_events_write,
2756 .write_iter = user_events_write_iter,
2757 .unlocked_ioctl = user_events_ioctl,
2758 .release = user_events_release,
2759};
2760
2761static void *user_seq_start(struct seq_file *m, loff_t *pos)
2762{
2763 if (*pos)
2764 return NULL;
2765
2766 return (void *)1;
2767}
2768
2769static void *user_seq_next(struct seq_file *m, void *p, loff_t *pos)
2770{
2771 ++*pos;
2772 return NULL;
2773}
2774
2775static void user_seq_stop(struct seq_file *m, void *p)
2776{
2777}
2778
2779static int user_seq_show(struct seq_file *m, void *p)
2780{
2781 struct user_event_group *group = m->private;
2782 struct user_event *user;
2783 char status;
2784 int i, active = 0, busy = 0;
2785
2786 if (!group)
2787 return -EINVAL;
2788
2789 mutex_lock(&group->reg_mutex);
2790
2791 hash_for_each(group->register_table, i, user, node) {
2792 status = user->status;
2793
2794 seq_printf(m, "%s", EVENT_TP_NAME(user));
2795
2796 if (status != 0)
2797 seq_puts(m, " #");
2798
2799 if (status != 0) {
2800 seq_puts(m, " Used by");
2801 if (status & EVENT_STATUS_FTRACE)
2802 seq_puts(m, " ftrace");
2803 if (status & EVENT_STATUS_PERF)
2804 seq_puts(m, " perf");
2805 if (status & EVENT_STATUS_OTHER)
2806 seq_puts(m, " other");
2807 busy++;
2808 }
2809
2810 seq_puts(m, "\n");
2811 active++;
2812 }
2813
2814 mutex_unlock(&group->reg_mutex);
2815
2816 seq_puts(m, "\n");
2817 seq_printf(m, "Active: %d\n", active);
2818 seq_printf(m, "Busy: %d\n", busy);
2819
2820 return 0;
2821}
2822
2823static const struct seq_operations user_seq_ops = {
2824 .start = user_seq_start,
2825 .next = user_seq_next,
2826 .stop = user_seq_stop,
2827 .show = user_seq_show,
2828};
2829
2830static int user_status_open(struct inode *node, struct file *file)
2831{
2832 struct user_event_group *group;
2833 int ret;
2834
2835 group = current_user_event_group();
2836
2837 if (!group)
2838 return -ENOENT;
2839
2840 ret = seq_open(file, &user_seq_ops);
2841
2842 if (!ret) {
2843 /* Chain group to seq_file */
2844 struct seq_file *m = file->private_data;
2845
2846 m->private = group;
2847 }
2848
2849 return ret;
2850}
2851
2852static const struct file_operations user_status_fops = {
2853 .open = user_status_open,
2854 .read = seq_read,
2855 .llseek = seq_lseek,
2856 .release = seq_release,
2857};
2858
2859/*
2860 * Creates a set of tracefs files to allow user mode interactions.
2861 */
2862static int create_user_tracefs(void)
2863{
2864 struct dentry *edata, *emmap;
2865
2866 edata = tracefs_create_file("user_events_data", TRACE_MODE_WRITE,
2867 NULL, NULL, &user_data_fops);
2868
2869 if (!edata) {
2870 pr_warn("Could not create tracefs 'user_events_data' entry\n");
2871 goto err;
2872 }
2873
2874 emmap = tracefs_create_file("user_events_status", TRACE_MODE_READ,
2875 NULL, NULL, &user_status_fops);
2876
2877 if (!emmap) {
2878 tracefs_remove(edata);
2879 pr_warn("Could not create tracefs 'user_events_mmap' entry\n");
2880 goto err;
2881 }
2882
2883 return 0;
2884err:
2885 return -ENODEV;
2886}
2887
2888static int set_max_user_events_sysctl(const struct ctl_table *table, int write,
2889 void *buffer, size_t *lenp, loff_t *ppos)
2890{
2891 int ret;
2892
2893 mutex_lock(&event_mutex);
2894
2895 ret = proc_douintvec(table, write, buffer, lenp, ppos);
2896
2897 mutex_unlock(&event_mutex);
2898
2899 return ret;
2900}
2901
2902static struct ctl_table user_event_sysctls[] = {
2903 {
2904 .procname = "user_events_max",
2905 .data = &max_user_events,
2906 .maxlen = sizeof(unsigned int),
2907 .mode = 0644,
2908 .proc_handler = set_max_user_events_sysctl,
2909 },
2910};
2911
2912static int __init trace_events_user_init(void)
2913{
2914 int ret;
2915
2916 fault_cache = KMEM_CACHE(user_event_enabler_fault, 0);
2917
2918 if (!fault_cache)
2919 return -ENOMEM;
2920
2921 init_group = user_event_group_create();
2922
2923 if (!init_group) {
2924 kmem_cache_destroy(fault_cache);
2925 return -ENOMEM;
2926 }
2927
2928 ret = create_user_tracefs();
2929
2930 if (ret) {
2931 pr_warn("user_events could not register with tracefs\n");
2932 user_event_group_destroy(init_group);
2933 kmem_cache_destroy(fault_cache);
2934 init_group = NULL;
2935 return ret;
2936 }
2937
2938 if (dyn_event_register(&user_event_dops))
2939 pr_warn("user_events could not register with dyn_events\n");
2940
2941 register_sysctl_init("kernel", user_event_sysctls);
2942
2943 return 0;
2944}
2945
2946fs_initcall(trace_events_user_init);