Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
5 *
6 * Based on "hwlat_detector" tracer by:
7 * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8 * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9 * With feedback from Clark Williams <williams@redhat.com>
10 *
11 * And also based on the rtsl tracer presented on:
12 * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13 * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14 * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
15 *
16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
17 */
18
19#include <linux/kthread.h>
20#include <linux/tracefs.h>
21#include <linux/uaccess.h>
22#include <linux/cpumask.h>
23#include <linux/delay.h>
24#include <linux/sched/clock.h>
25#include <uapi/linux/sched/types.h>
26#include <linux/sched.h>
27#include "trace.h"
28
29#ifdef CONFIG_X86_LOCAL_APIC
30#include <asm/trace/irq_vectors.h>
31#undef TRACE_INCLUDE_PATH
32#undef TRACE_INCLUDE_FILE
33#endif /* CONFIG_X86_LOCAL_APIC */
34
35#include <trace/events/irq.h>
36#include <trace/events/sched.h>
37
38#define CREATE_TRACE_POINTS
39#include <trace/events/osnoise.h>
40
41/*
42 * Default values.
43 */
44#define BANNER "osnoise: "
45#define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
46#define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
47
48#define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
49#define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
50
51/*
52 * osnoise/options entries.
53 */
54enum osnoise_options_index {
55 OSN_DEFAULTS = 0,
56 OSN_WORKLOAD,
57 OSN_PANIC_ON_STOP,
58 OSN_PREEMPT_DISABLE,
59 OSN_IRQ_DISABLE,
60 OSN_MAX
61};
62
63static const char * const osnoise_options_str[OSN_MAX] = {
64 "DEFAULTS",
65 "OSNOISE_WORKLOAD",
66 "PANIC_ON_STOP",
67 "OSNOISE_PREEMPT_DISABLE",
68 "OSNOISE_IRQ_DISABLE" };
69
70#define OSN_DEFAULT_OPTIONS 0x2
71static unsigned long osnoise_options = OSN_DEFAULT_OPTIONS;
72
73/*
74 * trace_array of the enabled osnoise/timerlat instances.
75 */
76struct osnoise_instance {
77 struct list_head list;
78 struct trace_array *tr;
79};
80
81static struct list_head osnoise_instances;
82
83static bool osnoise_has_registered_instances(void)
84{
85 return !!list_first_or_null_rcu(&osnoise_instances,
86 struct osnoise_instance,
87 list);
88}
89
90/*
91 * osnoise_instance_registered - check if a tr is already registered
92 */
93static int osnoise_instance_registered(struct trace_array *tr)
94{
95 struct osnoise_instance *inst;
96 int found = 0;
97
98 rcu_read_lock();
99 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
100 if (inst->tr == tr)
101 found = 1;
102 }
103 rcu_read_unlock();
104
105 return found;
106}
107
108/*
109 * osnoise_register_instance - register a new trace instance
110 *
111 * Register a trace_array *tr in the list of instances running
112 * osnoise/timerlat tracers.
113 */
114static int osnoise_register_instance(struct trace_array *tr)
115{
116 struct osnoise_instance *inst;
117
118 /*
119 * register/unregister serialization is provided by trace's
120 * trace_types_lock.
121 */
122 lockdep_assert_held(&trace_types_lock);
123
124 inst = kmalloc(sizeof(*inst), GFP_KERNEL);
125 if (!inst)
126 return -ENOMEM;
127
128 INIT_LIST_HEAD_RCU(&inst->list);
129 inst->tr = tr;
130 list_add_tail_rcu(&inst->list, &osnoise_instances);
131
132 return 0;
133}
134
135/*
136 * osnoise_unregister_instance - unregister a registered trace instance
137 *
138 * Remove the trace_array *tr from the list of instances running
139 * osnoise/timerlat tracers.
140 */
141static void osnoise_unregister_instance(struct trace_array *tr)
142{
143 struct osnoise_instance *inst;
144 int found = 0;
145
146 /*
147 * register/unregister serialization is provided by trace's
148 * trace_types_lock.
149 */
150 list_for_each_entry_rcu(inst, &osnoise_instances, list,
151 lockdep_is_held(&trace_types_lock)) {
152 if (inst->tr == tr) {
153 list_del_rcu(&inst->list);
154 found = 1;
155 break;
156 }
157 }
158
159 if (!found)
160 return;
161
162 kvfree_rcu_mightsleep(inst);
163}
164
165/*
166 * NMI runtime info.
167 */
168struct osn_nmi {
169 u64 count;
170 u64 delta_start;
171};
172
173/*
174 * IRQ runtime info.
175 */
176struct osn_irq {
177 u64 count;
178 u64 arrival_time;
179 u64 delta_start;
180};
181
182#define IRQ_CONTEXT 0
183#define THREAD_CONTEXT 1
184#define THREAD_URET 2
185/*
186 * sofirq runtime info.
187 */
188struct osn_softirq {
189 u64 count;
190 u64 arrival_time;
191 u64 delta_start;
192};
193
194/*
195 * thread runtime info.
196 */
197struct osn_thread {
198 u64 count;
199 u64 arrival_time;
200 u64 delta_start;
201};
202
203/*
204 * Runtime information: this structure saves the runtime information used by
205 * one sampling thread.
206 */
207struct osnoise_variables {
208 struct task_struct *kthread;
209 bool sampling;
210 pid_t pid;
211 struct osn_nmi nmi;
212 struct osn_irq irq;
213 struct osn_softirq softirq;
214 struct osn_thread thread;
215 local_t int_counter;
216};
217
218/*
219 * Per-cpu runtime information.
220 */
221static DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
222
223/*
224 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
225 */
226static inline struct osnoise_variables *this_cpu_osn_var(void)
227{
228 return this_cpu_ptr(&per_cpu_osnoise_var);
229}
230
231#ifdef CONFIG_TIMERLAT_TRACER
232/*
233 * Runtime information for the timer mode.
234 */
235struct timerlat_variables {
236 struct task_struct *kthread;
237 struct hrtimer timer;
238 u64 rel_period;
239 u64 abs_period;
240 bool tracing_thread;
241 u64 count;
242 bool uthread_migrate;
243};
244
245static DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
246
247/*
248 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
249 */
250static inline struct timerlat_variables *this_cpu_tmr_var(void)
251{
252 return this_cpu_ptr(&per_cpu_timerlat_var);
253}
254
255/*
256 * tlat_var_reset - Reset the values of the given timerlat_variables
257 */
258static inline void tlat_var_reset(void)
259{
260 struct timerlat_variables *tlat_var;
261 int cpu;
262 /*
263 * So far, all the values are initialized as 0, so
264 * zeroing the structure is perfect.
265 */
266 for_each_cpu(cpu, cpu_online_mask) {
267 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
268 memset(tlat_var, 0, sizeof(*tlat_var));
269 }
270}
271#else /* CONFIG_TIMERLAT_TRACER */
272#define tlat_var_reset() do {} while (0)
273#endif /* CONFIG_TIMERLAT_TRACER */
274
275/*
276 * osn_var_reset - Reset the values of the given osnoise_variables
277 */
278static inline void osn_var_reset(void)
279{
280 struct osnoise_variables *osn_var;
281 int cpu;
282
283 /*
284 * So far, all the values are initialized as 0, so
285 * zeroing the structure is perfect.
286 */
287 for_each_cpu(cpu, cpu_online_mask) {
288 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
289 memset(osn_var, 0, sizeof(*osn_var));
290 }
291}
292
293/*
294 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
295 */
296static inline void osn_var_reset_all(void)
297{
298 osn_var_reset();
299 tlat_var_reset();
300}
301
302/*
303 * Tells NMIs to call back to the osnoise tracer to record timestamps.
304 */
305bool trace_osnoise_callback_enabled;
306
307/*
308 * osnoise sample structure definition. Used to store the statistics of a
309 * sample run.
310 */
311struct osnoise_sample {
312 u64 runtime; /* runtime */
313 u64 noise; /* noise */
314 u64 max_sample; /* max single noise sample */
315 int hw_count; /* # HW (incl. hypervisor) interference */
316 int nmi_count; /* # NMIs during this sample */
317 int irq_count; /* # IRQs during this sample */
318 int softirq_count; /* # softirqs during this sample */
319 int thread_count; /* # threads during this sample */
320};
321
322#ifdef CONFIG_TIMERLAT_TRACER
323/*
324 * timerlat sample structure definition. Used to store the statistics of
325 * a sample run.
326 */
327struct timerlat_sample {
328 u64 timer_latency; /* timer_latency */
329 unsigned int seqnum; /* unique sequence */
330 int context; /* timer context */
331};
332#endif
333
334/*
335 * Protect the interface.
336 */
337static struct mutex interface_lock;
338
339/*
340 * Tracer data.
341 */
342static struct osnoise_data {
343 u64 sample_period; /* total sampling period */
344 u64 sample_runtime; /* active sampling portion of period */
345 u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
346 u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
347#ifdef CONFIG_TIMERLAT_TRACER
348 u64 timerlat_period; /* timerlat period */
349 u64 print_stack; /* print IRQ stack if total > */
350 int timerlat_tracer; /* timerlat tracer */
351#endif
352 bool tainted; /* infor users and developers about a problem */
353} osnoise_data = {
354 .sample_period = DEFAULT_SAMPLE_PERIOD,
355 .sample_runtime = DEFAULT_SAMPLE_RUNTIME,
356 .stop_tracing = 0,
357 .stop_tracing_total = 0,
358#ifdef CONFIG_TIMERLAT_TRACER
359 .print_stack = 0,
360 .timerlat_period = DEFAULT_TIMERLAT_PERIOD,
361 .timerlat_tracer = 0,
362#endif
363};
364
365#ifdef CONFIG_TIMERLAT_TRACER
366static inline bool timerlat_enabled(void)
367{
368 return osnoise_data.timerlat_tracer;
369}
370
371static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
372{
373 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
374 /*
375 * If the timerlat is enabled, but the irq handler did
376 * not run yet enabling timerlat_tracer, do not trace.
377 */
378 if (!tlat_var->tracing_thread) {
379 osn_var->softirq.arrival_time = 0;
380 osn_var->softirq.delta_start = 0;
381 return 0;
382 }
383 return 1;
384}
385
386static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
387{
388 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
389 /*
390 * If the timerlat is enabled, but the irq handler did
391 * not run yet enabling timerlat_tracer, do not trace.
392 */
393 if (!tlat_var->tracing_thread) {
394 osn_var->thread.delta_start = 0;
395 osn_var->thread.arrival_time = 0;
396 return 0;
397 }
398 return 1;
399}
400#else /* CONFIG_TIMERLAT_TRACER */
401static inline bool timerlat_enabled(void)
402{
403 return false;
404}
405
406static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
407{
408 return 1;
409}
410static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
411{
412 return 1;
413}
414#endif
415
416#ifdef CONFIG_PREEMPT_RT
417/*
418 * Print the osnoise header info.
419 */
420static void print_osnoise_headers(struct seq_file *s)
421{
422 if (osnoise_data.tainted)
423 seq_puts(s, "# osnoise is tainted!\n");
424
425 seq_puts(s, "# _-------=> irqs-off\n");
426 seq_puts(s, "# / _------=> need-resched\n");
427 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
428 seq_puts(s, "# || / _----=> hardirq/softirq\n");
429 seq_puts(s, "# ||| / _---=> preempt-depth\n");
430 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
431 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
432
433 seq_puts(s, "# |||||| / ");
434 seq_puts(s, " MAX\n");
435
436 seq_puts(s, "# ||||| / ");
437 seq_puts(s, " SINGLE Interference counters:\n");
438
439 seq_puts(s, "# ||||||| RUNTIME ");
440 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
441
442 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
443 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
444
445 seq_puts(s, "# | | | ||||||| | | ");
446 seq_puts(s, " | | | | | | | |\n");
447}
448#else /* CONFIG_PREEMPT_RT */
449static void print_osnoise_headers(struct seq_file *s)
450{
451 if (osnoise_data.tainted)
452 seq_puts(s, "# osnoise is tainted!\n");
453
454 seq_puts(s, "# _-----=> irqs-off\n");
455 seq_puts(s, "# / _----=> need-resched\n");
456 seq_puts(s, "# | / _---=> hardirq/softirq\n");
457 seq_puts(s, "# || / _--=> preempt-depth\n");
458 seq_puts(s, "# ||| / _-=> migrate-disable ");
459 seq_puts(s, " MAX\n");
460 seq_puts(s, "# |||| / delay ");
461 seq_puts(s, " SINGLE Interference counters:\n");
462
463 seq_puts(s, "# ||||| RUNTIME ");
464 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
465
466 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
467 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
468
469 seq_puts(s, "# | | | ||||| | | ");
470 seq_puts(s, " | | | | | | | |\n");
471}
472#endif /* CONFIG_PREEMPT_RT */
473
474/*
475 * osnoise_taint - report an osnoise error.
476 */
477#define osnoise_taint(msg) ({ \
478 struct osnoise_instance *inst; \
479 struct trace_buffer *buffer; \
480 \
481 rcu_read_lock(); \
482 list_for_each_entry_rcu(inst, &osnoise_instances, list) { \
483 buffer = inst->tr->array_buffer.buffer; \
484 trace_array_printk_buf(buffer, _THIS_IP_, msg); \
485 } \
486 rcu_read_unlock(); \
487 osnoise_data.tainted = true; \
488})
489
490/*
491 * Record an osnoise_sample into the tracer buffer.
492 */
493static void
494__trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer)
495{
496 struct trace_event_call *call = &event_osnoise;
497 struct ring_buffer_event *event;
498 struct osnoise_entry *entry;
499
500 event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
501 tracing_gen_ctx());
502 if (!event)
503 return;
504 entry = ring_buffer_event_data(event);
505 entry->runtime = sample->runtime;
506 entry->noise = sample->noise;
507 entry->max_sample = sample->max_sample;
508 entry->hw_count = sample->hw_count;
509 entry->nmi_count = sample->nmi_count;
510 entry->irq_count = sample->irq_count;
511 entry->softirq_count = sample->softirq_count;
512 entry->thread_count = sample->thread_count;
513
514 if (!call_filter_check_discard(call, entry, buffer, event))
515 trace_buffer_unlock_commit_nostack(buffer, event);
516}
517
518/*
519 * Record an osnoise_sample on all osnoise instances.
520 */
521static void trace_osnoise_sample(struct osnoise_sample *sample)
522{
523 struct osnoise_instance *inst;
524 struct trace_buffer *buffer;
525
526 rcu_read_lock();
527 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
528 buffer = inst->tr->array_buffer.buffer;
529 __trace_osnoise_sample(sample, buffer);
530 }
531 rcu_read_unlock();
532}
533
534#ifdef CONFIG_TIMERLAT_TRACER
535/*
536 * Print the timerlat header info.
537 */
538#ifdef CONFIG_PREEMPT_RT
539static void print_timerlat_headers(struct seq_file *s)
540{
541 seq_puts(s, "# _-------=> irqs-off\n");
542 seq_puts(s, "# / _------=> need-resched\n");
543 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
544 seq_puts(s, "# || / _----=> hardirq/softirq\n");
545 seq_puts(s, "# ||| / _---=> preempt-depth\n");
546 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
547 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
548 seq_puts(s, "# |||||| /\n");
549 seq_puts(s, "# ||||||| ACTIVATION\n");
550 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
551 seq_puts(s, " CONTEXT LATENCY\n");
552 seq_puts(s, "# | | | ||||||| | | ");
553 seq_puts(s, " | |\n");
554}
555#else /* CONFIG_PREEMPT_RT */
556static void print_timerlat_headers(struct seq_file *s)
557{
558 seq_puts(s, "# _-----=> irqs-off\n");
559 seq_puts(s, "# / _----=> need-resched\n");
560 seq_puts(s, "# | / _---=> hardirq/softirq\n");
561 seq_puts(s, "# || / _--=> preempt-depth\n");
562 seq_puts(s, "# ||| / _-=> migrate-disable\n");
563 seq_puts(s, "# |||| / delay\n");
564 seq_puts(s, "# ||||| ACTIVATION\n");
565 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
566 seq_puts(s, " CONTEXT LATENCY\n");
567 seq_puts(s, "# | | | ||||| | | ");
568 seq_puts(s, " | |\n");
569}
570#endif /* CONFIG_PREEMPT_RT */
571
572static void
573__trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer)
574{
575 struct trace_event_call *call = &event_osnoise;
576 struct ring_buffer_event *event;
577 struct timerlat_entry *entry;
578
579 event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
580 tracing_gen_ctx());
581 if (!event)
582 return;
583 entry = ring_buffer_event_data(event);
584 entry->seqnum = sample->seqnum;
585 entry->context = sample->context;
586 entry->timer_latency = sample->timer_latency;
587
588 if (!call_filter_check_discard(call, entry, buffer, event))
589 trace_buffer_unlock_commit_nostack(buffer, event);
590}
591
592/*
593 * Record an timerlat_sample into the tracer buffer.
594 */
595static void trace_timerlat_sample(struct timerlat_sample *sample)
596{
597 struct osnoise_instance *inst;
598 struct trace_buffer *buffer;
599
600 rcu_read_lock();
601 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
602 buffer = inst->tr->array_buffer.buffer;
603 __trace_timerlat_sample(sample, buffer);
604 }
605 rcu_read_unlock();
606}
607
608#ifdef CONFIG_STACKTRACE
609
610#define MAX_CALLS 256
611
612/*
613 * Stack trace will take place only at IRQ level, so, no need
614 * to control nesting here.
615 */
616struct trace_stack {
617 int stack_size;
618 int nr_entries;
619 unsigned long calls[MAX_CALLS];
620};
621
622static DEFINE_PER_CPU(struct trace_stack, trace_stack);
623
624/*
625 * timerlat_save_stack - save a stack trace without printing
626 *
627 * Save the current stack trace without printing. The
628 * stack will be printed later, after the end of the measurement.
629 */
630static void timerlat_save_stack(int skip)
631{
632 unsigned int size, nr_entries;
633 struct trace_stack *fstack;
634
635 fstack = this_cpu_ptr(&trace_stack);
636
637 size = ARRAY_SIZE(fstack->calls);
638
639 nr_entries = stack_trace_save(fstack->calls, size, skip);
640
641 fstack->stack_size = nr_entries * sizeof(unsigned long);
642 fstack->nr_entries = nr_entries;
643
644 return;
645
646}
647
648static void
649__timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size)
650{
651 struct trace_event_call *call = &event_osnoise;
652 struct ring_buffer_event *event;
653 struct stack_entry *entry;
654
655 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
656 tracing_gen_ctx());
657 if (!event)
658 return;
659
660 entry = ring_buffer_event_data(event);
661
662 memcpy(&entry->caller, fstack->calls, size);
663 entry->size = fstack->nr_entries;
664
665 if (!call_filter_check_discard(call, entry, buffer, event))
666 trace_buffer_unlock_commit_nostack(buffer, event);
667}
668
669/*
670 * timerlat_dump_stack - dump a stack trace previously saved
671 */
672static void timerlat_dump_stack(u64 latency)
673{
674 struct osnoise_instance *inst;
675 struct trace_buffer *buffer;
676 struct trace_stack *fstack;
677 unsigned int size;
678
679 /*
680 * trace only if latency > print_stack config, if enabled.
681 */
682 if (!osnoise_data.print_stack || osnoise_data.print_stack > latency)
683 return;
684
685 preempt_disable_notrace();
686 fstack = this_cpu_ptr(&trace_stack);
687 size = fstack->stack_size;
688
689 rcu_read_lock();
690 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
691 buffer = inst->tr->array_buffer.buffer;
692 __timerlat_dump_stack(buffer, fstack, size);
693
694 }
695 rcu_read_unlock();
696 preempt_enable_notrace();
697}
698#else /* CONFIG_STACKTRACE */
699#define timerlat_dump_stack(u64 latency) do {} while (0)
700#define timerlat_save_stack(a) do {} while (0)
701#endif /* CONFIG_STACKTRACE */
702#endif /* CONFIG_TIMERLAT_TRACER */
703
704/*
705 * Macros to encapsulate the time capturing infrastructure.
706 */
707#define time_get() trace_clock_local()
708#define time_to_us(x) div_u64(x, 1000)
709#define time_sub(a, b) ((a) - (b))
710
711/*
712 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
713 *
714 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
715 * to discount the NMI interference.
716 *
717 * See get_int_safe_duration().
718 */
719static inline void
720cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
721{
722 if (osn_var->irq.delta_start)
723 osn_var->irq.delta_start += duration;
724}
725
726#ifndef CONFIG_PREEMPT_RT
727/*
728 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
729 *
730 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
731 * forward to discount the interference.
732 *
733 * See get_int_safe_duration().
734 */
735static inline void
736cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
737{
738 if (osn_var->softirq.delta_start)
739 osn_var->softirq.delta_start += duration;
740}
741#else /* CONFIG_PREEMPT_RT */
742#define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
743#endif
744
745/*
746 * cond_move_thread_delta_start - Forward the delta_start of a running thread
747 *
748 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
749 * is pushed forward to discount the interference.
750 *
751 * See get_int_safe_duration().
752 */
753static inline void
754cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
755{
756 if (osn_var->thread.delta_start)
757 osn_var->thread.delta_start += duration;
758}
759
760/*
761 * get_int_safe_duration - Get the duration of a window
762 *
763 * The irq, softirq and thread varaibles need to have its duration without
764 * the interference from higher priority interrupts. Instead of keeping a
765 * variable to discount the interrupt interference from these variables, the
766 * starting time of these variables are pushed forward with the interrupt's
767 * duration. In this way, a single variable is used to:
768 *
769 * - Know if a given window is being measured.
770 * - Account its duration.
771 * - Discount the interference.
772 *
773 * To avoid getting inconsistent values, e.g.,:
774 *
775 * now = time_get()
776 * ---> interrupt!
777 * delta_start -= int duration;
778 * <---
779 * duration = now - delta_start;
780 *
781 * result: negative duration if the variable duration before the
782 * interrupt was smaller than the interrupt execution.
783 *
784 * A counter of interrupts is used. If the counter increased, try
785 * to capture an interference safe duration.
786 */
787static inline s64
788get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
789{
790 u64 int_counter, now;
791 s64 duration;
792
793 do {
794 int_counter = local_read(&osn_var->int_counter);
795 /* synchronize with interrupts */
796 barrier();
797
798 now = time_get();
799 duration = (now - *delta_start);
800
801 /* synchronize with interrupts */
802 barrier();
803 } while (int_counter != local_read(&osn_var->int_counter));
804
805 /*
806 * This is an evidence of race conditions that cause
807 * a value to be "discounted" too much.
808 */
809 if (duration < 0)
810 osnoise_taint("Negative duration!\n");
811
812 *delta_start = 0;
813
814 return duration;
815}
816
817/*
818 *
819 * set_int_safe_time - Save the current time on *time, aware of interference
820 *
821 * Get the time, taking into consideration a possible interference from
822 * higher priority interrupts.
823 *
824 * See get_int_safe_duration() for an explanation.
825 */
826static u64
827set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
828{
829 u64 int_counter;
830
831 do {
832 int_counter = local_read(&osn_var->int_counter);
833 /* synchronize with interrupts */
834 barrier();
835
836 *time = time_get();
837
838 /* synchronize with interrupts */
839 barrier();
840 } while (int_counter != local_read(&osn_var->int_counter));
841
842 return int_counter;
843}
844
845#ifdef CONFIG_TIMERLAT_TRACER
846/*
847 * copy_int_safe_time - Copy *src into *desc aware of interference
848 */
849static u64
850copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
851{
852 u64 int_counter;
853
854 do {
855 int_counter = local_read(&osn_var->int_counter);
856 /* synchronize with interrupts */
857 barrier();
858
859 *dst = *src;
860
861 /* synchronize with interrupts */
862 barrier();
863 } while (int_counter != local_read(&osn_var->int_counter));
864
865 return int_counter;
866}
867#endif /* CONFIG_TIMERLAT_TRACER */
868
869/*
870 * trace_osnoise_callback - NMI entry/exit callback
871 *
872 * This function is called at the entry and exit NMI code. The bool enter
873 * distinguishes between either case. This function is used to note a NMI
874 * occurrence, compute the noise caused by the NMI, and to remove the noise
875 * it is potentially causing on other interference variables.
876 */
877void trace_osnoise_callback(bool enter)
878{
879 struct osnoise_variables *osn_var = this_cpu_osn_var();
880 u64 duration;
881
882 if (!osn_var->sampling)
883 return;
884
885 /*
886 * Currently trace_clock_local() calls sched_clock() and the
887 * generic version is not NMI safe.
888 */
889 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
890 if (enter) {
891 osn_var->nmi.delta_start = time_get();
892 local_inc(&osn_var->int_counter);
893 } else {
894 duration = time_get() - osn_var->nmi.delta_start;
895
896 trace_nmi_noise(osn_var->nmi.delta_start, duration);
897
898 cond_move_irq_delta_start(osn_var, duration);
899 cond_move_softirq_delta_start(osn_var, duration);
900 cond_move_thread_delta_start(osn_var, duration);
901 }
902 }
903
904 if (enter)
905 osn_var->nmi.count++;
906}
907
908/*
909 * osnoise_trace_irq_entry - Note the starting of an IRQ
910 *
911 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
912 * it is safe to use a single variable (ons_var->irq) to save the statistics.
913 * The arrival_time is used to report... the arrival time. The delta_start
914 * is used to compute the duration at the IRQ exit handler. See
915 * cond_move_irq_delta_start().
916 */
917void osnoise_trace_irq_entry(int id)
918{
919 struct osnoise_variables *osn_var = this_cpu_osn_var();
920
921 if (!osn_var->sampling)
922 return;
923 /*
924 * This value will be used in the report, but not to compute
925 * the execution time, so it is safe to get it unsafe.
926 */
927 osn_var->irq.arrival_time = time_get();
928 set_int_safe_time(osn_var, &osn_var->irq.delta_start);
929 osn_var->irq.count++;
930
931 local_inc(&osn_var->int_counter);
932}
933
934/*
935 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
936 *
937 * Computes the duration of the IRQ noise, and trace it. Also discounts the
938 * interference from other sources of noise could be currently being accounted.
939 */
940void osnoise_trace_irq_exit(int id, const char *desc)
941{
942 struct osnoise_variables *osn_var = this_cpu_osn_var();
943 s64 duration;
944
945 if (!osn_var->sampling)
946 return;
947
948 duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
949 trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
950 osn_var->irq.arrival_time = 0;
951 cond_move_softirq_delta_start(osn_var, duration);
952 cond_move_thread_delta_start(osn_var, duration);
953}
954
955/*
956 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
957 *
958 * Used to note the starting of an IRQ occurece.
959 */
960static void trace_irqentry_callback(void *data, int irq,
961 struct irqaction *action)
962{
963 osnoise_trace_irq_entry(irq);
964}
965
966/*
967 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
968 *
969 * Used to note the end of an IRQ occurece.
970 */
971static void trace_irqexit_callback(void *data, int irq,
972 struct irqaction *action, int ret)
973{
974 osnoise_trace_irq_exit(irq, action->name);
975}
976
977/*
978 * arch specific register function.
979 */
980int __weak osnoise_arch_register(void)
981{
982 return 0;
983}
984
985/*
986 * arch specific unregister function.
987 */
988void __weak osnoise_arch_unregister(void)
989{
990 return;
991}
992
993/*
994 * hook_irq_events - Hook IRQ handling events
995 *
996 * This function hooks the IRQ related callbacks to the respective trace
997 * events.
998 */
999static int hook_irq_events(void)
1000{
1001 int ret;
1002
1003 ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1004 if (ret)
1005 goto out_err;
1006
1007 ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1008 if (ret)
1009 goto out_unregister_entry;
1010
1011 ret = osnoise_arch_register();
1012 if (ret)
1013 goto out_irq_exit;
1014
1015 return 0;
1016
1017out_irq_exit:
1018 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1019out_unregister_entry:
1020 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1021out_err:
1022 return -EINVAL;
1023}
1024
1025/*
1026 * unhook_irq_events - Unhook IRQ handling events
1027 *
1028 * This function unhooks the IRQ related callbacks to the respective trace
1029 * events.
1030 */
1031static void unhook_irq_events(void)
1032{
1033 osnoise_arch_unregister();
1034 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1035 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1036}
1037
1038#ifndef CONFIG_PREEMPT_RT
1039/*
1040 * trace_softirq_entry_callback - Note the starting of a softirq
1041 *
1042 * Save the starting time of a softirq. As softirqs are non-preemptive to
1043 * other softirqs, it is safe to use a single variable (ons_var->softirq)
1044 * to save the statistics. The arrival_time is used to report... the
1045 * arrival time. The delta_start is used to compute the duration at the
1046 * softirq exit handler. See cond_move_softirq_delta_start().
1047 */
1048static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1049{
1050 struct osnoise_variables *osn_var = this_cpu_osn_var();
1051
1052 if (!osn_var->sampling)
1053 return;
1054 /*
1055 * This value will be used in the report, but not to compute
1056 * the execution time, so it is safe to get it unsafe.
1057 */
1058 osn_var->softirq.arrival_time = time_get();
1059 set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1060 osn_var->softirq.count++;
1061
1062 local_inc(&osn_var->int_counter);
1063}
1064
1065/*
1066 * trace_softirq_exit_callback - Note the end of an softirq
1067 *
1068 * Computes the duration of the softirq noise, and trace it. Also discounts the
1069 * interference from other sources of noise could be currently being accounted.
1070 */
1071static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1072{
1073 struct osnoise_variables *osn_var = this_cpu_osn_var();
1074 s64 duration;
1075
1076 if (!osn_var->sampling)
1077 return;
1078
1079 if (unlikely(timerlat_enabled()))
1080 if (!timerlat_softirq_exit(osn_var))
1081 return;
1082
1083 duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1084 trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1085 cond_move_thread_delta_start(osn_var, duration);
1086 osn_var->softirq.arrival_time = 0;
1087}
1088
1089/*
1090 * hook_softirq_events - Hook softirq handling events
1091 *
1092 * This function hooks the softirq related callbacks to the respective trace
1093 * events.
1094 */
1095static int hook_softirq_events(void)
1096{
1097 int ret;
1098
1099 ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1100 if (ret)
1101 goto out_err;
1102
1103 ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1104 if (ret)
1105 goto out_unreg_entry;
1106
1107 return 0;
1108
1109out_unreg_entry:
1110 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1111out_err:
1112 return -EINVAL;
1113}
1114
1115/*
1116 * unhook_softirq_events - Unhook softirq handling events
1117 *
1118 * This function hooks the softirq related callbacks to the respective trace
1119 * events.
1120 */
1121static void unhook_softirq_events(void)
1122{
1123 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1124 unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1125}
1126#else /* CONFIG_PREEMPT_RT */
1127/*
1128 * softirq are threads on the PREEMPT_RT mode.
1129 */
1130static int hook_softirq_events(void)
1131{
1132 return 0;
1133}
1134static void unhook_softirq_events(void)
1135{
1136}
1137#endif
1138
1139/*
1140 * thread_entry - Record the starting of a thread noise window
1141 *
1142 * It saves the context switch time for a noisy thread, and increments
1143 * the interference counters.
1144 */
1145static void
1146thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
1147{
1148 if (!osn_var->sampling)
1149 return;
1150 /*
1151 * The arrival time will be used in the report, but not to compute
1152 * the execution time, so it is safe to get it unsafe.
1153 */
1154 osn_var->thread.arrival_time = time_get();
1155
1156 set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1157
1158 osn_var->thread.count++;
1159 local_inc(&osn_var->int_counter);
1160}
1161
1162/*
1163 * thread_exit - Report the end of a thread noise window
1164 *
1165 * It computes the total noise from a thread, tracing if needed.
1166 */
1167static void
1168thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
1169{
1170 s64 duration;
1171
1172 if (!osn_var->sampling)
1173 return;
1174
1175 if (unlikely(timerlat_enabled()))
1176 if (!timerlat_thread_exit(osn_var))
1177 return;
1178
1179 duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1180
1181 trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1182
1183 osn_var->thread.arrival_time = 0;
1184}
1185
1186#ifdef CONFIG_TIMERLAT_TRACER
1187/*
1188 * osnoise_stop_exception - Stop tracing and the tracer.
1189 */
1190static __always_inline void osnoise_stop_exception(char *msg, int cpu)
1191{
1192 struct osnoise_instance *inst;
1193 struct trace_array *tr;
1194
1195 rcu_read_lock();
1196 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1197 tr = inst->tr;
1198 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1199 "stop tracing hit on cpu %d due to exception: %s\n",
1200 smp_processor_id(),
1201 msg);
1202
1203 if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1204 panic("tracer hit on cpu %d due to exception: %s\n",
1205 smp_processor_id(),
1206 msg);
1207
1208 tracer_tracing_off(tr);
1209 }
1210 rcu_read_unlock();
1211}
1212
1213/*
1214 * trace_sched_migrate_callback - sched:sched_migrate_task trace event handler
1215 *
1216 * his function is hooked to the sched:sched_migrate_task trace event, and monitors
1217 * timerlat user-space thread migration.
1218 */
1219static void trace_sched_migrate_callback(void *data, struct task_struct *p, int dest_cpu)
1220{
1221 struct osnoise_variables *osn_var;
1222 long cpu = task_cpu(p);
1223
1224 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
1225 if (osn_var->pid == p->pid && dest_cpu != cpu) {
1226 per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
1227 osnoise_taint("timerlat user-thread migrated\n");
1228 osnoise_stop_exception("timerlat user-thread migrated", cpu);
1229 }
1230}
1231
1232static int register_migration_monitor(void)
1233{
1234 int ret = 0;
1235
1236 /*
1237 * Timerlat thread migration check is only required when running timerlat in user-space.
1238 * Thus, enable callback only if timerlat is set with no workload.
1239 */
1240 if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options))
1241 ret = register_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1242
1243 return ret;
1244}
1245
1246static void unregister_migration_monitor(void)
1247{
1248 if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options))
1249 unregister_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1250}
1251#else
1252static int register_migration_monitor(void)
1253{
1254 return 0;
1255}
1256static void unregister_migration_monitor(void) {}
1257#endif
1258/*
1259 * trace_sched_switch - sched:sched_switch trace event handler
1260 *
1261 * This function is hooked to the sched:sched_switch trace event, and it is
1262 * used to record the beginning and to report the end of a thread noise window.
1263 */
1264static void
1265trace_sched_switch_callback(void *data, bool preempt,
1266 struct task_struct *p,
1267 struct task_struct *n,
1268 unsigned int prev_state)
1269{
1270 struct osnoise_variables *osn_var = this_cpu_osn_var();
1271 int workload = test_bit(OSN_WORKLOAD, &osnoise_options);
1272
1273 if ((p->pid != osn_var->pid) || !workload)
1274 thread_exit(osn_var, p);
1275
1276 if ((n->pid != osn_var->pid) || !workload)
1277 thread_entry(osn_var, n);
1278}
1279
1280/*
1281 * hook_thread_events - Hook the instrumentation for thread noise
1282 *
1283 * Hook the osnoise tracer callbacks to handle the noise from other
1284 * threads on the necessary kernel events.
1285 */
1286static int hook_thread_events(void)
1287{
1288 int ret;
1289
1290 ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1291 if (ret)
1292 return -EINVAL;
1293
1294 ret = register_migration_monitor();
1295 if (ret)
1296 goto out_unreg;
1297
1298 return 0;
1299
1300out_unreg:
1301 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1302 return -EINVAL;
1303}
1304
1305/*
1306 * unhook_thread_events - unhook the instrumentation for thread noise
1307 *
1308 * Unook the osnoise tracer callbacks to handle the noise from other
1309 * threads on the necessary kernel events.
1310 */
1311static void unhook_thread_events(void)
1312{
1313 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1314 unregister_migration_monitor();
1315}
1316
1317/*
1318 * save_osn_sample_stats - Save the osnoise_sample statistics
1319 *
1320 * Save the osnoise_sample statistics before the sampling phase. These
1321 * values will be used later to compute the diff betwneen the statistics
1322 * before and after the osnoise sampling.
1323 */
1324static void
1325save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1326{
1327 s->nmi_count = osn_var->nmi.count;
1328 s->irq_count = osn_var->irq.count;
1329 s->softirq_count = osn_var->softirq.count;
1330 s->thread_count = osn_var->thread.count;
1331}
1332
1333/*
1334 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1335 *
1336 * After a sample period, compute the difference on the osnoise_sample
1337 * statistics. The struct osnoise_sample *s contains the statistics saved via
1338 * save_osn_sample_stats() before the osnoise sampling.
1339 */
1340static void
1341diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1342{
1343 s->nmi_count = osn_var->nmi.count - s->nmi_count;
1344 s->irq_count = osn_var->irq.count - s->irq_count;
1345 s->softirq_count = osn_var->softirq.count - s->softirq_count;
1346 s->thread_count = osn_var->thread.count - s->thread_count;
1347}
1348
1349/*
1350 * osnoise_stop_tracing - Stop tracing and the tracer.
1351 */
1352static __always_inline void osnoise_stop_tracing(void)
1353{
1354 struct osnoise_instance *inst;
1355 struct trace_array *tr;
1356
1357 rcu_read_lock();
1358 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1359 tr = inst->tr;
1360 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1361 "stop tracing hit on cpu %d\n", smp_processor_id());
1362
1363 if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1364 panic("tracer hit stop condition on CPU %d\n", smp_processor_id());
1365
1366 tracer_tracing_off(tr);
1367 }
1368 rcu_read_unlock();
1369}
1370
1371/*
1372 * osnoise_has_tracing_on - Check if there is at least one instance on
1373 */
1374static __always_inline int osnoise_has_tracing_on(void)
1375{
1376 struct osnoise_instance *inst;
1377 int trace_is_on = 0;
1378
1379 rcu_read_lock();
1380 list_for_each_entry_rcu(inst, &osnoise_instances, list)
1381 trace_is_on += tracer_tracing_is_on(inst->tr);
1382 rcu_read_unlock();
1383
1384 return trace_is_on;
1385}
1386
1387/*
1388 * notify_new_max_latency - Notify a new max latency via fsnotify interface.
1389 */
1390static void notify_new_max_latency(u64 latency)
1391{
1392 struct osnoise_instance *inst;
1393 struct trace_array *tr;
1394
1395 rcu_read_lock();
1396 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1397 tr = inst->tr;
1398 if (tracer_tracing_is_on(tr) && tr->max_latency < latency) {
1399 tr->max_latency = latency;
1400 latency_fsnotify(tr);
1401 }
1402 }
1403 rcu_read_unlock();
1404}
1405
1406/*
1407 * run_osnoise - Sample the time and look for osnoise
1408 *
1409 * Used to capture the time, looking for potential osnoise latency repeatedly.
1410 * Different from hwlat_detector, it is called with preemption and interrupts
1411 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1412 * osnoise sampling thread, as they would do with a regular thread.
1413 */
1414static int run_osnoise(void)
1415{
1416 bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options);
1417 struct osnoise_variables *osn_var = this_cpu_osn_var();
1418 u64 start, sample, last_sample;
1419 u64 last_int_count, int_count;
1420 s64 noise = 0, max_noise = 0;
1421 s64 total, last_total = 0;
1422 struct osnoise_sample s;
1423 bool disable_preemption;
1424 unsigned int threshold;
1425 u64 runtime, stop_in;
1426 u64 sum_noise = 0;
1427 int hw_count = 0;
1428 int ret = -1;
1429
1430 /*
1431 * Disabling preemption is only required if IRQs are enabled,
1432 * and the options is set on.
1433 */
1434 disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options);
1435
1436 /*
1437 * Considers the current thread as the workload.
1438 */
1439 osn_var->pid = current->pid;
1440
1441 /*
1442 * Save the current stats for the diff
1443 */
1444 save_osn_sample_stats(osn_var, &s);
1445
1446 /*
1447 * if threshold is 0, use the default value of 5 us.
1448 */
1449 threshold = tracing_thresh ? : 5000;
1450
1451 /*
1452 * Apply PREEMPT and IRQ disabled options.
1453 */
1454 if (disable_irq)
1455 local_irq_disable();
1456
1457 if (disable_preemption)
1458 preempt_disable();
1459
1460 /*
1461 * Make sure NMIs see sampling first
1462 */
1463 osn_var->sampling = true;
1464 barrier();
1465
1466 /*
1467 * Transform the *_us config to nanoseconds to avoid the
1468 * division on the main loop.
1469 */
1470 runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1471 stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1472
1473 /*
1474 * Start timestemp
1475 */
1476 start = time_get();
1477
1478 /*
1479 * "previous" loop.
1480 */
1481 last_int_count = set_int_safe_time(osn_var, &last_sample);
1482
1483 do {
1484 /*
1485 * Get sample!
1486 */
1487 int_count = set_int_safe_time(osn_var, &sample);
1488
1489 noise = time_sub(sample, last_sample);
1490
1491 /*
1492 * This shouldn't happen.
1493 */
1494 if (noise < 0) {
1495 osnoise_taint("negative noise!");
1496 goto out;
1497 }
1498
1499 /*
1500 * Sample runtime.
1501 */
1502 total = time_sub(sample, start);
1503
1504 /*
1505 * Check for possible overflows.
1506 */
1507 if (total < last_total) {
1508 osnoise_taint("total overflow!");
1509 break;
1510 }
1511
1512 last_total = total;
1513
1514 if (noise >= threshold) {
1515 int interference = int_count - last_int_count;
1516
1517 if (noise > max_noise)
1518 max_noise = noise;
1519
1520 if (!interference)
1521 hw_count++;
1522
1523 sum_noise += noise;
1524
1525 trace_sample_threshold(last_sample, noise, interference);
1526
1527 if (osnoise_data.stop_tracing)
1528 if (noise > stop_in)
1529 osnoise_stop_tracing();
1530 }
1531
1532 /*
1533 * In some cases, notably when running on a nohz_full CPU with
1534 * a stopped tick PREEMPT_RCU has no way to account for QSs.
1535 * This will eventually cause unwarranted noise as PREEMPT_RCU
1536 * will force preemption as the means of ending the current
1537 * grace period. We avoid this problem by calling
1538 * rcu_momentary_dyntick_idle(), which performs a zero duration
1539 * EQS allowing PREEMPT_RCU to end the current grace period.
1540 * This call shouldn't be wrapped inside an RCU critical
1541 * section.
1542 *
1543 * Note that in non PREEMPT_RCU kernels QSs are handled through
1544 * cond_resched()
1545 */
1546 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
1547 if (!disable_irq)
1548 local_irq_disable();
1549
1550 rcu_momentary_dyntick_idle();
1551
1552 if (!disable_irq)
1553 local_irq_enable();
1554 }
1555
1556 /*
1557 * For the non-preemptive kernel config: let threads runs, if
1558 * they so wish, unless set not do to so.
1559 */
1560 if (!disable_irq && !disable_preemption)
1561 cond_resched();
1562
1563 last_sample = sample;
1564 last_int_count = int_count;
1565
1566 } while (total < runtime && !kthread_should_stop());
1567
1568 /*
1569 * Finish the above in the view for interrupts.
1570 */
1571 barrier();
1572
1573 osn_var->sampling = false;
1574
1575 /*
1576 * Make sure sampling data is no longer updated.
1577 */
1578 barrier();
1579
1580 /*
1581 * Return to the preemptive state.
1582 */
1583 if (disable_preemption)
1584 preempt_enable();
1585
1586 if (disable_irq)
1587 local_irq_enable();
1588
1589 /*
1590 * Save noise info.
1591 */
1592 s.noise = time_to_us(sum_noise);
1593 s.runtime = time_to_us(total);
1594 s.max_sample = time_to_us(max_noise);
1595 s.hw_count = hw_count;
1596
1597 /* Save interference stats info */
1598 diff_osn_sample_stats(osn_var, &s);
1599
1600 trace_osnoise_sample(&s);
1601
1602 notify_new_max_latency(max_noise);
1603
1604 if (osnoise_data.stop_tracing_total)
1605 if (s.noise > osnoise_data.stop_tracing_total)
1606 osnoise_stop_tracing();
1607
1608 return 0;
1609out:
1610 return ret;
1611}
1612
1613static struct cpumask osnoise_cpumask;
1614static struct cpumask save_cpumask;
1615
1616/*
1617 * osnoise_sleep - sleep until the next period
1618 */
1619static void osnoise_sleep(bool skip_period)
1620{
1621 u64 interval;
1622 ktime_t wake_time;
1623
1624 mutex_lock(&interface_lock);
1625 if (skip_period)
1626 interval = osnoise_data.sample_period;
1627 else
1628 interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1629 mutex_unlock(&interface_lock);
1630
1631 /*
1632 * differently from hwlat_detector, the osnoise tracer can run
1633 * without a pause because preemption is on.
1634 */
1635 if (!interval) {
1636 /* Let synchronize_rcu_tasks() make progress */
1637 cond_resched_tasks_rcu_qs();
1638 return;
1639 }
1640
1641 wake_time = ktime_add_us(ktime_get(), interval);
1642 __set_current_state(TASK_INTERRUPTIBLE);
1643
1644 while (schedule_hrtimeout(&wake_time, HRTIMER_MODE_ABS)) {
1645 if (kthread_should_stop())
1646 break;
1647 }
1648}
1649
1650/*
1651 * osnoise_migration_pending - checks if the task needs to migrate
1652 *
1653 * osnoise/timerlat threads are per-cpu. If there is a pending request to
1654 * migrate the thread away from the current CPU, something bad has happened.
1655 * Play the good citizen and leave.
1656 *
1657 * Returns 0 if it is safe to continue, 1 otherwise.
1658 */
1659static inline int osnoise_migration_pending(void)
1660{
1661 if (!current->migration_pending)
1662 return 0;
1663
1664 /*
1665 * If migration is pending, there is a task waiting for the
1666 * tracer to enable migration. The tracer does not allow migration,
1667 * thus: taint and leave to unblock the blocked thread.
1668 */
1669 osnoise_taint("migration requested to osnoise threads, leaving.");
1670
1671 /*
1672 * Unset this thread from the threads managed by the interface.
1673 * The tracers are responsible for cleaning their env before
1674 * exiting.
1675 */
1676 mutex_lock(&interface_lock);
1677 this_cpu_osn_var()->kthread = NULL;
1678 mutex_unlock(&interface_lock);
1679
1680 return 1;
1681}
1682
1683/*
1684 * osnoise_main - The osnoise detection kernel thread
1685 *
1686 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1687 * every period.
1688 */
1689static int osnoise_main(void *data)
1690{
1691 unsigned long flags;
1692
1693 /*
1694 * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1695 * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1696 *
1697 * To work around this limitation, disable migration and remove the
1698 * flag.
1699 */
1700 migrate_disable();
1701 raw_spin_lock_irqsave(¤t->pi_lock, flags);
1702 current->flags &= ~(PF_NO_SETAFFINITY);
1703 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
1704
1705 while (!kthread_should_stop()) {
1706 if (osnoise_migration_pending())
1707 break;
1708
1709 /* skip a period if tracing is off on all instances */
1710 if (!osnoise_has_tracing_on()) {
1711 osnoise_sleep(true);
1712 continue;
1713 }
1714
1715 run_osnoise();
1716 osnoise_sleep(false);
1717 }
1718
1719 migrate_enable();
1720 return 0;
1721}
1722
1723#ifdef CONFIG_TIMERLAT_TRACER
1724/*
1725 * timerlat_irq - hrtimer handler for timerlat.
1726 */
1727static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1728{
1729 struct osnoise_variables *osn_var = this_cpu_osn_var();
1730 struct timerlat_variables *tlat;
1731 struct timerlat_sample s;
1732 u64 now;
1733 u64 diff;
1734
1735 /*
1736 * I am not sure if the timer was armed for this CPU. So, get
1737 * the timerlat struct from the timer itself, not from this
1738 * CPU.
1739 */
1740 tlat = container_of(timer, struct timerlat_variables, timer);
1741
1742 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1743
1744 /*
1745 * Enable the osnoise: events for thread an softirq.
1746 */
1747 tlat->tracing_thread = true;
1748
1749 osn_var->thread.arrival_time = time_get();
1750
1751 /*
1752 * A hardirq is running: the timer IRQ. It is for sure preempting
1753 * a thread, and potentially preempting a softirq.
1754 *
1755 * At this point, it is not interesting to know the duration of the
1756 * preempted thread (and maybe softirq), but how much time they will
1757 * delay the beginning of the execution of the timer thread.
1758 *
1759 * To get the correct (net) delay added by the softirq, its delta_start
1760 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1761 * start of the sofitrq will be zeroed, accounting then only the time
1762 * after that.
1763 *
1764 * The thread follows the same principle. However, if a softirq is
1765 * running, the thread needs to receive the softirq delta_start. The
1766 * reason being is that the softirq will be the last to be unfolded,
1767 * resseting the thread delay to zero.
1768 *
1769 * The PREEMPT_RT is a special case, though. As softirqs run as threads
1770 * on RT, moving the thread is enough.
1771 */
1772 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1773 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1774 &osn_var->softirq.delta_start);
1775
1776 copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1777 &osn_var->irq.delta_start);
1778 } else {
1779 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1780 &osn_var->irq.delta_start);
1781 }
1782
1783 /*
1784 * Compute the current time with the expected time.
1785 */
1786 diff = now - tlat->abs_period;
1787
1788 tlat->count++;
1789 s.seqnum = tlat->count;
1790 s.timer_latency = diff;
1791 s.context = IRQ_CONTEXT;
1792
1793 trace_timerlat_sample(&s);
1794
1795 if (osnoise_data.stop_tracing) {
1796 if (time_to_us(diff) >= osnoise_data.stop_tracing) {
1797
1798 /*
1799 * At this point, if stop_tracing is set and <= print_stack,
1800 * print_stack is set and would be printed in the thread handler.
1801 *
1802 * Thus, print the stack trace as it is helpful to define the
1803 * root cause of an IRQ latency.
1804 */
1805 if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
1806 timerlat_save_stack(0);
1807 timerlat_dump_stack(time_to_us(diff));
1808 }
1809
1810 osnoise_stop_tracing();
1811 notify_new_max_latency(diff);
1812
1813 wake_up_process(tlat->kthread);
1814
1815 return HRTIMER_NORESTART;
1816 }
1817 }
1818
1819 wake_up_process(tlat->kthread);
1820
1821 if (osnoise_data.print_stack)
1822 timerlat_save_stack(0);
1823
1824 return HRTIMER_NORESTART;
1825}
1826
1827/*
1828 * wait_next_period - Wait for the next period for timerlat
1829 */
1830static int wait_next_period(struct timerlat_variables *tlat)
1831{
1832 ktime_t next_abs_period, now;
1833 u64 rel_period = osnoise_data.timerlat_period * 1000;
1834
1835 now = hrtimer_cb_get_time(&tlat->timer);
1836 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1837
1838 /*
1839 * Save the next abs_period.
1840 */
1841 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1842
1843 /*
1844 * If the new abs_period is in the past, skip the activation.
1845 */
1846 while (ktime_compare(now, next_abs_period) > 0) {
1847 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1848 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1849 }
1850
1851 set_current_state(TASK_INTERRUPTIBLE);
1852
1853 hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1854 schedule();
1855 return 1;
1856}
1857
1858/*
1859 * timerlat_main- Timerlat main
1860 */
1861static int timerlat_main(void *data)
1862{
1863 struct osnoise_variables *osn_var = this_cpu_osn_var();
1864 struct timerlat_variables *tlat = this_cpu_tmr_var();
1865 struct timerlat_sample s;
1866 struct sched_param sp;
1867 unsigned long flags;
1868 u64 now, diff;
1869
1870 /*
1871 * Make the thread RT, that is how cyclictest is usually used.
1872 */
1873 sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1874 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1875
1876 /*
1877 * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1878 * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1879 *
1880 * To work around this limitation, disable migration and remove the
1881 * flag.
1882 */
1883 migrate_disable();
1884 raw_spin_lock_irqsave(¤t->pi_lock, flags);
1885 current->flags &= ~(PF_NO_SETAFFINITY);
1886 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
1887
1888 tlat->count = 0;
1889 tlat->tracing_thread = false;
1890
1891 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1892 tlat->timer.function = timerlat_irq;
1893 tlat->kthread = current;
1894 osn_var->pid = current->pid;
1895 /*
1896 * Anotate the arrival time.
1897 */
1898 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1899
1900 wait_next_period(tlat);
1901
1902 osn_var->sampling = 1;
1903
1904 while (!kthread_should_stop()) {
1905
1906 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1907 diff = now - tlat->abs_period;
1908
1909 s.seqnum = tlat->count;
1910 s.timer_latency = diff;
1911 s.context = THREAD_CONTEXT;
1912
1913 trace_timerlat_sample(&s);
1914
1915 notify_new_max_latency(diff);
1916
1917 timerlat_dump_stack(time_to_us(diff));
1918
1919 tlat->tracing_thread = false;
1920 if (osnoise_data.stop_tracing_total)
1921 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1922 osnoise_stop_tracing();
1923
1924 if (osnoise_migration_pending())
1925 break;
1926
1927 wait_next_period(tlat);
1928 }
1929
1930 hrtimer_cancel(&tlat->timer);
1931 migrate_enable();
1932 return 0;
1933}
1934#else /* CONFIG_TIMERLAT_TRACER */
1935static int timerlat_main(void *data)
1936{
1937 return 0;
1938}
1939#endif /* CONFIG_TIMERLAT_TRACER */
1940
1941/*
1942 * stop_kthread - stop a workload thread
1943 */
1944static void stop_kthread(unsigned int cpu)
1945{
1946 struct task_struct *kthread;
1947
1948 kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread;
1949 if (kthread) {
1950 if (test_bit(OSN_WORKLOAD, &osnoise_options)) {
1951 kthread_stop(kthread);
1952 } else {
1953 /*
1954 * This is a user thread waiting on the timerlat_fd. We need
1955 * to close all users, and the best way to guarantee this is
1956 * by killing the thread. NOTE: this is a purpose specific file.
1957 */
1958 kill_pid(kthread->thread_pid, SIGKILL, 1);
1959 put_task_struct(kthread);
1960 }
1961 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1962 } else {
1963 /* if no workload, just return */
1964 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1965 /*
1966 * This is set in the osnoise tracer case.
1967 */
1968 per_cpu(per_cpu_osnoise_var, cpu).sampling = false;
1969 barrier();
1970 return;
1971 }
1972 }
1973}
1974
1975/*
1976 * stop_per_cpu_kthread - Stop per-cpu threads
1977 *
1978 * Stop the osnoise sampling htread. Use this on unload and at system
1979 * shutdown.
1980 */
1981static void stop_per_cpu_kthreads(void)
1982{
1983 int cpu;
1984
1985 cpus_read_lock();
1986
1987 for_each_online_cpu(cpu)
1988 stop_kthread(cpu);
1989
1990 cpus_read_unlock();
1991}
1992
1993/*
1994 * start_kthread - Start a workload tread
1995 */
1996static int start_kthread(unsigned int cpu)
1997{
1998 struct task_struct *kthread;
1999 void *main = osnoise_main;
2000 char comm[24];
2001
2002 if (timerlat_enabled()) {
2003 snprintf(comm, 24, "timerlat/%d", cpu);
2004 main = timerlat_main;
2005 } else {
2006 /* if no workload, just return */
2007 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2008 per_cpu(per_cpu_osnoise_var, cpu).sampling = true;
2009 barrier();
2010 return 0;
2011 }
2012 snprintf(comm, 24, "osnoise/%d", cpu);
2013 }
2014
2015 kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
2016
2017 if (IS_ERR(kthread)) {
2018 pr_err(BANNER "could not start sampling thread\n");
2019 stop_per_cpu_kthreads();
2020 return -ENOMEM;
2021 }
2022
2023 per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
2024
2025 return 0;
2026}
2027
2028/*
2029 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
2030 *
2031 * This starts the kernel thread that will look for osnoise on many
2032 * cpus.
2033 */
2034static int start_per_cpu_kthreads(void)
2035{
2036 struct cpumask *current_mask = &save_cpumask;
2037 int retval = 0;
2038 int cpu;
2039
2040 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2041 if (timerlat_enabled())
2042 return 0;
2043 }
2044
2045 cpus_read_lock();
2046 /*
2047 * Run only on online CPUs in which osnoise is allowed to run.
2048 */
2049 cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
2050
2051 for_each_possible_cpu(cpu)
2052 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
2053
2054 for_each_cpu(cpu, current_mask) {
2055 retval = start_kthread(cpu);
2056 if (retval) {
2057 cpus_read_unlock();
2058 stop_per_cpu_kthreads();
2059 return retval;
2060 }
2061 }
2062
2063 cpus_read_unlock();
2064
2065 return retval;
2066}
2067
2068#ifdef CONFIG_HOTPLUG_CPU
2069static void osnoise_hotplug_workfn(struct work_struct *dummy)
2070{
2071 unsigned int cpu = smp_processor_id();
2072
2073 mutex_lock(&trace_types_lock);
2074
2075 if (!osnoise_has_registered_instances())
2076 goto out_unlock_trace;
2077
2078 mutex_lock(&interface_lock);
2079 cpus_read_lock();
2080
2081 if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
2082 goto out_unlock;
2083
2084 start_kthread(cpu);
2085
2086out_unlock:
2087 cpus_read_unlock();
2088 mutex_unlock(&interface_lock);
2089out_unlock_trace:
2090 mutex_unlock(&trace_types_lock);
2091}
2092
2093static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
2094
2095/*
2096 * osnoise_cpu_init - CPU hotplug online callback function
2097 */
2098static int osnoise_cpu_init(unsigned int cpu)
2099{
2100 schedule_work_on(cpu, &osnoise_hotplug_work);
2101 return 0;
2102}
2103
2104/*
2105 * osnoise_cpu_die - CPU hotplug offline callback function
2106 */
2107static int osnoise_cpu_die(unsigned int cpu)
2108{
2109 stop_kthread(cpu);
2110 return 0;
2111}
2112
2113static void osnoise_init_hotplug_support(void)
2114{
2115 int ret;
2116
2117 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
2118 osnoise_cpu_init, osnoise_cpu_die);
2119 if (ret < 0)
2120 pr_warn(BANNER "Error to init cpu hotplug support\n");
2121
2122 return;
2123}
2124#else /* CONFIG_HOTPLUG_CPU */
2125static void osnoise_init_hotplug_support(void)
2126{
2127 return;
2128}
2129#endif /* CONFIG_HOTPLUG_CPU */
2130
2131/*
2132 * seq file functions for the osnoise/options file.
2133 */
2134static void *s_options_start(struct seq_file *s, loff_t *pos)
2135{
2136 int option = *pos;
2137
2138 mutex_lock(&interface_lock);
2139
2140 if (option >= OSN_MAX)
2141 return NULL;
2142
2143 return pos;
2144}
2145
2146static void *s_options_next(struct seq_file *s, void *v, loff_t *pos)
2147{
2148 int option = ++(*pos);
2149
2150 if (option >= OSN_MAX)
2151 return NULL;
2152
2153 return pos;
2154}
2155
2156static int s_options_show(struct seq_file *s, void *v)
2157{
2158 loff_t *pos = v;
2159 int option = *pos;
2160
2161 if (option == OSN_DEFAULTS) {
2162 if (osnoise_options == OSN_DEFAULT_OPTIONS)
2163 seq_printf(s, "%s", osnoise_options_str[option]);
2164 else
2165 seq_printf(s, "NO_%s", osnoise_options_str[option]);
2166 goto out;
2167 }
2168
2169 if (test_bit(option, &osnoise_options))
2170 seq_printf(s, "%s", osnoise_options_str[option]);
2171 else
2172 seq_printf(s, "NO_%s", osnoise_options_str[option]);
2173
2174out:
2175 if (option != OSN_MAX)
2176 seq_puts(s, " ");
2177
2178 return 0;
2179}
2180
2181static void s_options_stop(struct seq_file *s, void *v)
2182{
2183 seq_puts(s, "\n");
2184 mutex_unlock(&interface_lock);
2185}
2186
2187static const struct seq_operations osnoise_options_seq_ops = {
2188 .start = s_options_start,
2189 .next = s_options_next,
2190 .show = s_options_show,
2191 .stop = s_options_stop
2192};
2193
2194static int osnoise_options_open(struct inode *inode, struct file *file)
2195{
2196 return seq_open(file, &osnoise_options_seq_ops);
2197};
2198
2199/**
2200 * osnoise_options_write - Write function for "options" entry
2201 * @filp: The active open file structure
2202 * @ubuf: The user buffer that contains the value to write
2203 * @cnt: The maximum number of bytes to write to "file"
2204 * @ppos: The current position in @file
2205 *
2206 * Writing the option name sets the option, writing the "NO_"
2207 * prefix in front of the option name disables it.
2208 *
2209 * Writing "DEFAULTS" resets the option values to the default ones.
2210 */
2211static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf,
2212 size_t cnt, loff_t *ppos)
2213{
2214 int running, option, enable, retval;
2215 char buf[256], *option_str;
2216
2217 if (cnt >= 256)
2218 return -EINVAL;
2219
2220 if (copy_from_user(buf, ubuf, cnt))
2221 return -EFAULT;
2222
2223 buf[cnt] = 0;
2224
2225 if (strncmp(buf, "NO_", 3)) {
2226 option_str = strstrip(buf);
2227 enable = true;
2228 } else {
2229 option_str = strstrip(&buf[3]);
2230 enable = false;
2231 }
2232
2233 option = match_string(osnoise_options_str, OSN_MAX, option_str);
2234 if (option < 0)
2235 return -EINVAL;
2236
2237 /*
2238 * trace_types_lock is taken to avoid concurrency on start/stop.
2239 */
2240 mutex_lock(&trace_types_lock);
2241 running = osnoise_has_registered_instances();
2242 if (running)
2243 stop_per_cpu_kthreads();
2244
2245 mutex_lock(&interface_lock);
2246 /*
2247 * avoid CPU hotplug operations that might read options.
2248 */
2249 cpus_read_lock();
2250
2251 retval = cnt;
2252
2253 if (enable) {
2254 if (option == OSN_DEFAULTS)
2255 osnoise_options = OSN_DEFAULT_OPTIONS;
2256 else
2257 set_bit(option, &osnoise_options);
2258 } else {
2259 if (option == OSN_DEFAULTS)
2260 retval = -EINVAL;
2261 else
2262 clear_bit(option, &osnoise_options);
2263 }
2264
2265 cpus_read_unlock();
2266 mutex_unlock(&interface_lock);
2267
2268 if (running)
2269 start_per_cpu_kthreads();
2270 mutex_unlock(&trace_types_lock);
2271
2272 return retval;
2273}
2274
2275/*
2276 * osnoise_cpus_read - Read function for reading the "cpus" file
2277 * @filp: The active open file structure
2278 * @ubuf: The userspace provided buffer to read value into
2279 * @cnt: The maximum number of bytes to read
2280 * @ppos: The current "file" position
2281 *
2282 * Prints the "cpus" output into the user-provided buffer.
2283 */
2284static ssize_t
2285osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
2286 loff_t *ppos)
2287{
2288 char *mask_str;
2289 int len;
2290
2291 mutex_lock(&interface_lock);
2292
2293 len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
2294 mask_str = kmalloc(len, GFP_KERNEL);
2295 if (!mask_str) {
2296 count = -ENOMEM;
2297 goto out_unlock;
2298 }
2299
2300 len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
2301 if (len >= count) {
2302 count = -EINVAL;
2303 goto out_free;
2304 }
2305
2306 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
2307
2308out_free:
2309 kfree(mask_str);
2310out_unlock:
2311 mutex_unlock(&interface_lock);
2312
2313 return count;
2314}
2315
2316/*
2317 * osnoise_cpus_write - Write function for "cpus" entry
2318 * @filp: The active open file structure
2319 * @ubuf: The user buffer that contains the value to write
2320 * @cnt: The maximum number of bytes to write to "file"
2321 * @ppos: The current position in @file
2322 *
2323 * This function provides a write implementation for the "cpus"
2324 * interface to the osnoise trace. By default, it lists all CPUs,
2325 * in this way, allowing osnoise threads to run on any online CPU
2326 * of the system. It serves to restrict the execution of osnoise to the
2327 * set of CPUs writing via this interface. Why not use "tracing_cpumask"?
2328 * Because the user might be interested in tracing what is running on
2329 * other CPUs. For instance, one might run osnoise in one HT CPU
2330 * while observing what is running on the sibling HT CPU.
2331 */
2332static ssize_t
2333osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
2334 loff_t *ppos)
2335{
2336 cpumask_var_t osnoise_cpumask_new;
2337 int running, err;
2338 char buf[256];
2339
2340 if (count >= 256)
2341 return -EINVAL;
2342
2343 if (copy_from_user(buf, ubuf, count))
2344 return -EFAULT;
2345
2346 if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
2347 return -ENOMEM;
2348
2349 err = cpulist_parse(buf, osnoise_cpumask_new);
2350 if (err)
2351 goto err_free;
2352
2353 /*
2354 * trace_types_lock is taken to avoid concurrency on start/stop.
2355 */
2356 mutex_lock(&trace_types_lock);
2357 running = osnoise_has_registered_instances();
2358 if (running)
2359 stop_per_cpu_kthreads();
2360
2361 mutex_lock(&interface_lock);
2362 /*
2363 * osnoise_cpumask is read by CPU hotplug operations.
2364 */
2365 cpus_read_lock();
2366
2367 cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
2368
2369 cpus_read_unlock();
2370 mutex_unlock(&interface_lock);
2371
2372 if (running)
2373 start_per_cpu_kthreads();
2374 mutex_unlock(&trace_types_lock);
2375
2376 free_cpumask_var(osnoise_cpumask_new);
2377 return count;
2378
2379err_free:
2380 free_cpumask_var(osnoise_cpumask_new);
2381
2382 return err;
2383}
2384
2385#ifdef CONFIG_TIMERLAT_TRACER
2386static int timerlat_fd_open(struct inode *inode, struct file *file)
2387{
2388 struct osnoise_variables *osn_var;
2389 struct timerlat_variables *tlat;
2390 long cpu = (long) inode->i_cdev;
2391
2392 mutex_lock(&interface_lock);
2393
2394 /*
2395 * This file is accessible only if timerlat is enabled, and
2396 * NO_OSNOISE_WORKLOAD is set.
2397 */
2398 if (!timerlat_enabled() || test_bit(OSN_WORKLOAD, &osnoise_options)) {
2399 mutex_unlock(&interface_lock);
2400 return -EINVAL;
2401 }
2402
2403 migrate_disable();
2404
2405 osn_var = this_cpu_osn_var();
2406
2407 /*
2408 * The osn_var->pid holds the single access to this file.
2409 */
2410 if (osn_var->pid) {
2411 mutex_unlock(&interface_lock);
2412 migrate_enable();
2413 return -EBUSY;
2414 }
2415
2416 /*
2417 * timerlat tracer is a per-cpu tracer. Check if the user-space too
2418 * is pinned to a single CPU. The tracer laters monitor if the task
2419 * migrates and then disables tracer if it does. However, it is
2420 * worth doing this basic acceptance test to avoid obviusly wrong
2421 * setup.
2422 */
2423 if (current->nr_cpus_allowed > 1 || cpu != smp_processor_id()) {
2424 mutex_unlock(&interface_lock);
2425 migrate_enable();
2426 return -EPERM;
2427 }
2428
2429 /*
2430 * From now on, it is good to go.
2431 */
2432 file->private_data = inode->i_cdev;
2433
2434 get_task_struct(current);
2435
2436 osn_var->kthread = current;
2437 osn_var->pid = current->pid;
2438
2439 /*
2440 * Setup is done.
2441 */
2442 mutex_unlock(&interface_lock);
2443
2444 tlat = this_cpu_tmr_var();
2445 tlat->count = 0;
2446
2447 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
2448 tlat->timer.function = timerlat_irq;
2449
2450 migrate_enable();
2451 return 0;
2452};
2453
2454/*
2455 * timerlat_fd_read - Read function for "timerlat_fd" file
2456 * @file: The active open file structure
2457 * @ubuf: The userspace provided buffer to read value into
2458 * @cnt: The maximum number of bytes to read
2459 * @ppos: The current "file" position
2460 *
2461 * Prints 1 on timerlat, the number of interferences on osnoise, -1 on error.
2462 */
2463static ssize_t
2464timerlat_fd_read(struct file *file, char __user *ubuf, size_t count,
2465 loff_t *ppos)
2466{
2467 long cpu = (long) file->private_data;
2468 struct osnoise_variables *osn_var;
2469 struct timerlat_variables *tlat;
2470 struct timerlat_sample s;
2471 s64 diff;
2472 u64 now;
2473
2474 migrate_disable();
2475
2476 tlat = this_cpu_tmr_var();
2477
2478 /*
2479 * While in user-space, the thread is migratable. There is nothing
2480 * we can do about it.
2481 * So, if the thread is running on another CPU, stop the machinery.
2482 */
2483 if (cpu == smp_processor_id()) {
2484 if (tlat->uthread_migrate) {
2485 migrate_enable();
2486 return -EINVAL;
2487 }
2488 } else {
2489 per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
2490 osnoise_taint("timerlat user thread migrate\n");
2491 osnoise_stop_tracing();
2492 migrate_enable();
2493 return -EINVAL;
2494 }
2495
2496 osn_var = this_cpu_osn_var();
2497
2498 /*
2499 * The timerlat in user-space runs in a different order:
2500 * the read() starts from the execution of the previous occurrence,
2501 * sleeping for the next occurrence.
2502 *
2503 * So, skip if we are entering on read() before the first wakeup
2504 * from timerlat IRQ:
2505 */
2506 if (likely(osn_var->sampling)) {
2507 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2508 diff = now - tlat->abs_period;
2509
2510 /*
2511 * it was not a timer firing, but some other signal?
2512 */
2513 if (diff < 0)
2514 goto out;
2515
2516 s.seqnum = tlat->count;
2517 s.timer_latency = diff;
2518 s.context = THREAD_URET;
2519
2520 trace_timerlat_sample(&s);
2521
2522 notify_new_max_latency(diff);
2523
2524 tlat->tracing_thread = false;
2525 if (osnoise_data.stop_tracing_total)
2526 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
2527 osnoise_stop_tracing();
2528 } else {
2529 tlat->tracing_thread = false;
2530 tlat->kthread = current;
2531
2532 /* Annotate now to drift new period */
2533 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
2534
2535 osn_var->sampling = 1;
2536 }
2537
2538 /* wait for the next period */
2539 wait_next_period(tlat);
2540
2541 /* This is the wakeup from this cycle */
2542 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2543 diff = now - tlat->abs_period;
2544
2545 /*
2546 * it was not a timer firing, but some other signal?
2547 */
2548 if (diff < 0)
2549 goto out;
2550
2551 s.seqnum = tlat->count;
2552 s.timer_latency = diff;
2553 s.context = THREAD_CONTEXT;
2554
2555 trace_timerlat_sample(&s);
2556
2557 if (osnoise_data.stop_tracing_total) {
2558 if (time_to_us(diff) >= osnoise_data.stop_tracing_total) {
2559 timerlat_dump_stack(time_to_us(diff));
2560 notify_new_max_latency(diff);
2561 osnoise_stop_tracing();
2562 }
2563 }
2564
2565out:
2566 migrate_enable();
2567 return 0;
2568}
2569
2570static int timerlat_fd_release(struct inode *inode, struct file *file)
2571{
2572 struct osnoise_variables *osn_var;
2573 struct timerlat_variables *tlat_var;
2574 long cpu = (long) file->private_data;
2575
2576 migrate_disable();
2577 mutex_lock(&interface_lock);
2578
2579 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
2580 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
2581
2582 hrtimer_cancel(&tlat_var->timer);
2583 memset(tlat_var, 0, sizeof(*tlat_var));
2584
2585 osn_var->sampling = 0;
2586 osn_var->pid = 0;
2587
2588 /*
2589 * We are leaving, not being stopped... see stop_kthread();
2590 */
2591 if (osn_var->kthread) {
2592 put_task_struct(osn_var->kthread);
2593 osn_var->kthread = NULL;
2594 }
2595
2596 mutex_unlock(&interface_lock);
2597 migrate_enable();
2598 return 0;
2599}
2600#endif
2601
2602/*
2603 * osnoise/runtime_us: cannot be greater than the period.
2604 */
2605static struct trace_min_max_param osnoise_runtime = {
2606 .lock = &interface_lock,
2607 .val = &osnoise_data.sample_runtime,
2608 .max = &osnoise_data.sample_period,
2609 .min = NULL,
2610};
2611
2612/*
2613 * osnoise/period_us: cannot be smaller than the runtime.
2614 */
2615static struct trace_min_max_param osnoise_period = {
2616 .lock = &interface_lock,
2617 .val = &osnoise_data.sample_period,
2618 .max = NULL,
2619 .min = &osnoise_data.sample_runtime,
2620};
2621
2622/*
2623 * osnoise/stop_tracing_us: no limit.
2624 */
2625static struct trace_min_max_param osnoise_stop_tracing_in = {
2626 .lock = &interface_lock,
2627 .val = &osnoise_data.stop_tracing,
2628 .max = NULL,
2629 .min = NULL,
2630};
2631
2632/*
2633 * osnoise/stop_tracing_total_us: no limit.
2634 */
2635static struct trace_min_max_param osnoise_stop_tracing_total = {
2636 .lock = &interface_lock,
2637 .val = &osnoise_data.stop_tracing_total,
2638 .max = NULL,
2639 .min = NULL,
2640};
2641
2642#ifdef CONFIG_TIMERLAT_TRACER
2643/*
2644 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2645 * latency is higher than val.
2646 */
2647static struct trace_min_max_param osnoise_print_stack = {
2648 .lock = &interface_lock,
2649 .val = &osnoise_data.print_stack,
2650 .max = NULL,
2651 .min = NULL,
2652};
2653
2654/*
2655 * osnoise/timerlat_period: min 100 us, max 1 s
2656 */
2657static u64 timerlat_min_period = 100;
2658static u64 timerlat_max_period = 1000000;
2659static struct trace_min_max_param timerlat_period = {
2660 .lock = &interface_lock,
2661 .val = &osnoise_data.timerlat_period,
2662 .max = &timerlat_max_period,
2663 .min = &timerlat_min_period,
2664};
2665
2666static const struct file_operations timerlat_fd_fops = {
2667 .open = timerlat_fd_open,
2668 .read = timerlat_fd_read,
2669 .release = timerlat_fd_release,
2670 .llseek = generic_file_llseek,
2671};
2672#endif
2673
2674static const struct file_operations cpus_fops = {
2675 .open = tracing_open_generic,
2676 .read = osnoise_cpus_read,
2677 .write = osnoise_cpus_write,
2678 .llseek = generic_file_llseek,
2679};
2680
2681static const struct file_operations osnoise_options_fops = {
2682 .open = osnoise_options_open,
2683 .read = seq_read,
2684 .llseek = seq_lseek,
2685 .release = seq_release,
2686 .write = osnoise_options_write
2687};
2688
2689#ifdef CONFIG_TIMERLAT_TRACER
2690#ifdef CONFIG_STACKTRACE
2691static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2692{
2693 struct dentry *tmp;
2694
2695 tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2696 &osnoise_print_stack, &trace_min_max_fops);
2697 if (!tmp)
2698 return -ENOMEM;
2699
2700 return 0;
2701}
2702#else /* CONFIG_STACKTRACE */
2703static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2704{
2705 return 0;
2706}
2707#endif /* CONFIG_STACKTRACE */
2708
2709static int osnoise_create_cpu_timerlat_fd(struct dentry *top_dir)
2710{
2711 struct dentry *timerlat_fd;
2712 struct dentry *per_cpu;
2713 struct dentry *cpu_dir;
2714 char cpu_str[30]; /* see trace.c: tracing_init_tracefs_percpu() */
2715 long cpu;
2716
2717 /*
2718 * Why not using tracing instance per_cpu/ dir?
2719 *
2720 * Because osnoise/timerlat have a single workload, having
2721 * multiple files like these are wast of memory.
2722 */
2723 per_cpu = tracefs_create_dir("per_cpu", top_dir);
2724 if (!per_cpu)
2725 return -ENOMEM;
2726
2727 for_each_possible_cpu(cpu) {
2728 snprintf(cpu_str, 30, "cpu%ld", cpu);
2729 cpu_dir = tracefs_create_dir(cpu_str, per_cpu);
2730 if (!cpu_dir)
2731 goto out_clean;
2732
2733 timerlat_fd = trace_create_file("timerlat_fd", TRACE_MODE_READ,
2734 cpu_dir, NULL, &timerlat_fd_fops);
2735 if (!timerlat_fd)
2736 goto out_clean;
2737
2738 /* Record the CPU */
2739 d_inode(timerlat_fd)->i_cdev = (void *)(cpu);
2740 }
2741
2742 return 0;
2743
2744out_clean:
2745 tracefs_remove(per_cpu);
2746 return -ENOMEM;
2747}
2748
2749/*
2750 * init_timerlat_tracefs - A function to initialize the timerlat interface files
2751 */
2752static int init_timerlat_tracefs(struct dentry *top_dir)
2753{
2754 struct dentry *tmp;
2755 int retval;
2756
2757 tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2758 &timerlat_period, &trace_min_max_fops);
2759 if (!tmp)
2760 return -ENOMEM;
2761
2762 retval = osnoise_create_cpu_timerlat_fd(top_dir);
2763 if (retval)
2764 return retval;
2765
2766 return init_timerlat_stack_tracefs(top_dir);
2767}
2768#else /* CONFIG_TIMERLAT_TRACER */
2769static int init_timerlat_tracefs(struct dentry *top_dir)
2770{
2771 return 0;
2772}
2773#endif /* CONFIG_TIMERLAT_TRACER */
2774
2775/*
2776 * init_tracefs - A function to initialize the tracefs interface files
2777 *
2778 * This function creates entries in tracefs for "osnoise" and "timerlat".
2779 * It creates these directories in the tracing directory, and within that
2780 * directory the use can change and view the configs.
2781 */
2782static int init_tracefs(void)
2783{
2784 struct dentry *top_dir;
2785 struct dentry *tmp;
2786 int ret;
2787
2788 ret = tracing_init_dentry();
2789 if (ret)
2790 return -ENOMEM;
2791
2792 top_dir = tracefs_create_dir("osnoise", NULL);
2793 if (!top_dir)
2794 return 0;
2795
2796 tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2797 &osnoise_period, &trace_min_max_fops);
2798 if (!tmp)
2799 goto err;
2800
2801 tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2802 &osnoise_runtime, &trace_min_max_fops);
2803 if (!tmp)
2804 goto err;
2805
2806 tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2807 &osnoise_stop_tracing_in, &trace_min_max_fops);
2808 if (!tmp)
2809 goto err;
2810
2811 tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2812 &osnoise_stop_tracing_total, &trace_min_max_fops);
2813 if (!tmp)
2814 goto err;
2815
2816 tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2817 if (!tmp)
2818 goto err;
2819
2820 tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL,
2821 &osnoise_options_fops);
2822 if (!tmp)
2823 goto err;
2824
2825 ret = init_timerlat_tracefs(top_dir);
2826 if (ret)
2827 goto err;
2828
2829 return 0;
2830
2831err:
2832 tracefs_remove(top_dir);
2833 return -ENOMEM;
2834}
2835
2836static int osnoise_hook_events(void)
2837{
2838 int retval;
2839
2840 /*
2841 * Trace is already hooked, we are re-enabling from
2842 * a stop_tracing_*.
2843 */
2844 if (trace_osnoise_callback_enabled)
2845 return 0;
2846
2847 retval = hook_irq_events();
2848 if (retval)
2849 return -EINVAL;
2850
2851 retval = hook_softirq_events();
2852 if (retval)
2853 goto out_unhook_irq;
2854
2855 retval = hook_thread_events();
2856 /*
2857 * All fine!
2858 */
2859 if (!retval)
2860 return 0;
2861
2862 unhook_softirq_events();
2863out_unhook_irq:
2864 unhook_irq_events();
2865 return -EINVAL;
2866}
2867
2868static void osnoise_unhook_events(void)
2869{
2870 unhook_thread_events();
2871 unhook_softirq_events();
2872 unhook_irq_events();
2873}
2874
2875/*
2876 * osnoise_workload_start - start the workload and hook to events
2877 */
2878static int osnoise_workload_start(void)
2879{
2880 int retval;
2881
2882 /*
2883 * Instances need to be registered after calling workload
2884 * start. Hence, if there is already an instance, the
2885 * workload was already registered. Otherwise, this
2886 * code is on the way to register the first instance,
2887 * and the workload will start.
2888 */
2889 if (osnoise_has_registered_instances())
2890 return 0;
2891
2892 osn_var_reset_all();
2893
2894 retval = osnoise_hook_events();
2895 if (retval)
2896 return retval;
2897
2898 /*
2899 * Make sure that ftrace_nmi_enter/exit() see reset values
2900 * before enabling trace_osnoise_callback_enabled.
2901 */
2902 barrier();
2903 trace_osnoise_callback_enabled = true;
2904
2905 retval = start_per_cpu_kthreads();
2906 if (retval) {
2907 trace_osnoise_callback_enabled = false;
2908 /*
2909 * Make sure that ftrace_nmi_enter/exit() see
2910 * trace_osnoise_callback_enabled as false before continuing.
2911 */
2912 barrier();
2913
2914 osnoise_unhook_events();
2915 return retval;
2916 }
2917
2918 return 0;
2919}
2920
2921/*
2922 * osnoise_workload_stop - stop the workload and unhook the events
2923 */
2924static void osnoise_workload_stop(void)
2925{
2926 /*
2927 * Instances need to be unregistered before calling
2928 * stop. Hence, if there is a registered instance, more
2929 * than one instance is running, and the workload will not
2930 * yet stop. Otherwise, this code is on the way to disable
2931 * the last instance, and the workload can stop.
2932 */
2933 if (osnoise_has_registered_instances())
2934 return;
2935
2936 /*
2937 * If callbacks were already disabled in a previous stop
2938 * call, there is no need to disable then again.
2939 *
2940 * For instance, this happens when tracing is stopped via:
2941 * echo 0 > tracing_on
2942 * echo nop > current_tracer.
2943 */
2944 if (!trace_osnoise_callback_enabled)
2945 return;
2946
2947 trace_osnoise_callback_enabled = false;
2948 /*
2949 * Make sure that ftrace_nmi_enter/exit() see
2950 * trace_osnoise_callback_enabled as false before continuing.
2951 */
2952 barrier();
2953
2954 stop_per_cpu_kthreads();
2955
2956 osnoise_unhook_events();
2957}
2958
2959static void osnoise_tracer_start(struct trace_array *tr)
2960{
2961 int retval;
2962
2963 /*
2964 * If the instance is already registered, there is no need to
2965 * register it again.
2966 */
2967 if (osnoise_instance_registered(tr))
2968 return;
2969
2970 retval = osnoise_workload_start();
2971 if (retval)
2972 pr_err(BANNER "Error starting osnoise tracer\n");
2973
2974 osnoise_register_instance(tr);
2975}
2976
2977static void osnoise_tracer_stop(struct trace_array *tr)
2978{
2979 osnoise_unregister_instance(tr);
2980 osnoise_workload_stop();
2981}
2982
2983static int osnoise_tracer_init(struct trace_array *tr)
2984{
2985 /*
2986 * Only allow osnoise tracer if timerlat tracer is not running
2987 * already.
2988 */
2989 if (timerlat_enabled())
2990 return -EBUSY;
2991
2992 tr->max_latency = 0;
2993
2994 osnoise_tracer_start(tr);
2995 return 0;
2996}
2997
2998static void osnoise_tracer_reset(struct trace_array *tr)
2999{
3000 osnoise_tracer_stop(tr);
3001}
3002
3003static struct tracer osnoise_tracer __read_mostly = {
3004 .name = "osnoise",
3005 .init = osnoise_tracer_init,
3006 .reset = osnoise_tracer_reset,
3007 .start = osnoise_tracer_start,
3008 .stop = osnoise_tracer_stop,
3009 .print_header = print_osnoise_headers,
3010 .allow_instances = true,
3011};
3012
3013#ifdef CONFIG_TIMERLAT_TRACER
3014static void timerlat_tracer_start(struct trace_array *tr)
3015{
3016 int retval;
3017
3018 /*
3019 * If the instance is already registered, there is no need to
3020 * register it again.
3021 */
3022 if (osnoise_instance_registered(tr))
3023 return;
3024
3025 retval = osnoise_workload_start();
3026 if (retval)
3027 pr_err(BANNER "Error starting timerlat tracer\n");
3028
3029 osnoise_register_instance(tr);
3030
3031 return;
3032}
3033
3034static void timerlat_tracer_stop(struct trace_array *tr)
3035{
3036 int cpu;
3037
3038 osnoise_unregister_instance(tr);
3039
3040 /*
3041 * Instruct the threads to stop only if this is the last instance.
3042 */
3043 if (!osnoise_has_registered_instances()) {
3044 for_each_online_cpu(cpu)
3045 per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
3046 }
3047
3048 osnoise_workload_stop();
3049}
3050
3051static int timerlat_tracer_init(struct trace_array *tr)
3052{
3053 /*
3054 * Only allow timerlat tracer if osnoise tracer is not running already.
3055 */
3056 if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
3057 return -EBUSY;
3058
3059 /*
3060 * If this is the first instance, set timerlat_tracer to block
3061 * osnoise tracer start.
3062 */
3063 if (!osnoise_has_registered_instances())
3064 osnoise_data.timerlat_tracer = 1;
3065
3066 tr->max_latency = 0;
3067 timerlat_tracer_start(tr);
3068
3069 return 0;
3070}
3071
3072static void timerlat_tracer_reset(struct trace_array *tr)
3073{
3074 timerlat_tracer_stop(tr);
3075
3076 /*
3077 * If this is the last instance, reset timerlat_tracer allowing
3078 * osnoise to be started.
3079 */
3080 if (!osnoise_has_registered_instances())
3081 osnoise_data.timerlat_tracer = 0;
3082}
3083
3084static struct tracer timerlat_tracer __read_mostly = {
3085 .name = "timerlat",
3086 .init = timerlat_tracer_init,
3087 .reset = timerlat_tracer_reset,
3088 .start = timerlat_tracer_start,
3089 .stop = timerlat_tracer_stop,
3090 .print_header = print_timerlat_headers,
3091 .allow_instances = true,
3092};
3093
3094__init static int init_timerlat_tracer(void)
3095{
3096 return register_tracer(&timerlat_tracer);
3097}
3098#else /* CONFIG_TIMERLAT_TRACER */
3099__init static int init_timerlat_tracer(void)
3100{
3101 return 0;
3102}
3103#endif /* CONFIG_TIMERLAT_TRACER */
3104
3105__init static int init_osnoise_tracer(void)
3106{
3107 int ret;
3108
3109 mutex_init(&interface_lock);
3110
3111 cpumask_copy(&osnoise_cpumask, cpu_all_mask);
3112
3113 ret = register_tracer(&osnoise_tracer);
3114 if (ret) {
3115 pr_err(BANNER "Error registering osnoise!\n");
3116 return ret;
3117 }
3118
3119 ret = init_timerlat_tracer();
3120 if (ret) {
3121 pr_err(BANNER "Error registering timerlat!\n");
3122 return ret;
3123 }
3124
3125 osnoise_init_hotplug_support();
3126
3127 INIT_LIST_HEAD_RCU(&osnoise_instances);
3128
3129 init_tracefs();
3130
3131 return 0;
3132}
3133late_initcall(init_osnoise_tracer);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
5 *
6 * Based on "hwlat_detector" tracer by:
7 * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8 * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9 * With feedback from Clark Williams <williams@redhat.com>
10 *
11 * And also based on the rtsl tracer presented on:
12 * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13 * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14 * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
15 *
16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
17 */
18
19#include <linux/kthread.h>
20#include <linux/tracefs.h>
21#include <linux/uaccess.h>
22#include <linux/cpumask.h>
23#include <linux/delay.h>
24#include <linux/sched/clock.h>
25#include <uapi/linux/sched/types.h>
26#include <linux/sched.h>
27#include "trace.h"
28
29#ifdef CONFIG_X86_LOCAL_APIC
30#include <asm/trace/irq_vectors.h>
31#undef TRACE_INCLUDE_PATH
32#undef TRACE_INCLUDE_FILE
33#endif /* CONFIG_X86_LOCAL_APIC */
34
35#include <trace/events/irq.h>
36#include <trace/events/sched.h>
37
38#define CREATE_TRACE_POINTS
39#include <trace/events/osnoise.h>
40
41/*
42 * Default values.
43 */
44#define BANNER "osnoise: "
45#define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
46#define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
47
48#define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
49#define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
50
51/*
52 * osnoise/options entries.
53 */
54enum osnoise_options_index {
55 OSN_DEFAULTS = 0,
56 OSN_WORKLOAD,
57 OSN_PANIC_ON_STOP,
58 OSN_PREEMPT_DISABLE,
59 OSN_IRQ_DISABLE,
60 OSN_MAX
61};
62
63static const char * const osnoise_options_str[OSN_MAX] = {
64 "DEFAULTS",
65 "OSNOISE_WORKLOAD",
66 "PANIC_ON_STOP",
67 "OSNOISE_PREEMPT_DISABLE",
68 "OSNOISE_IRQ_DISABLE" };
69
70#define OSN_DEFAULT_OPTIONS 0x2
71static unsigned long osnoise_options = OSN_DEFAULT_OPTIONS;
72
73/*
74 * trace_array of the enabled osnoise/timerlat instances.
75 */
76struct osnoise_instance {
77 struct list_head list;
78 struct trace_array *tr;
79};
80
81static struct list_head osnoise_instances;
82
83static bool osnoise_has_registered_instances(void)
84{
85 return !!list_first_or_null_rcu(&osnoise_instances,
86 struct osnoise_instance,
87 list);
88}
89
90/*
91 * osnoise_instance_registered - check if a tr is already registered
92 */
93static int osnoise_instance_registered(struct trace_array *tr)
94{
95 struct osnoise_instance *inst;
96 int found = 0;
97
98 rcu_read_lock();
99 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
100 if (inst->tr == tr)
101 found = 1;
102 }
103 rcu_read_unlock();
104
105 return found;
106}
107
108/*
109 * osnoise_register_instance - register a new trace instance
110 *
111 * Register a trace_array *tr in the list of instances running
112 * osnoise/timerlat tracers.
113 */
114static int osnoise_register_instance(struct trace_array *tr)
115{
116 struct osnoise_instance *inst;
117
118 /*
119 * register/unregister serialization is provided by trace's
120 * trace_types_lock.
121 */
122 lockdep_assert_held(&trace_types_lock);
123
124 inst = kmalloc(sizeof(*inst), GFP_KERNEL);
125 if (!inst)
126 return -ENOMEM;
127
128 INIT_LIST_HEAD_RCU(&inst->list);
129 inst->tr = tr;
130 list_add_tail_rcu(&inst->list, &osnoise_instances);
131
132 return 0;
133}
134
135/*
136 * osnoise_unregister_instance - unregister a registered trace instance
137 *
138 * Remove the trace_array *tr from the list of instances running
139 * osnoise/timerlat tracers.
140 */
141static void osnoise_unregister_instance(struct trace_array *tr)
142{
143 struct osnoise_instance *inst;
144 int found = 0;
145
146 /*
147 * register/unregister serialization is provided by trace's
148 * trace_types_lock.
149 */
150 list_for_each_entry_rcu(inst, &osnoise_instances, list,
151 lockdep_is_held(&trace_types_lock)) {
152 if (inst->tr == tr) {
153 list_del_rcu(&inst->list);
154 found = 1;
155 break;
156 }
157 }
158
159 if (!found)
160 return;
161
162 kvfree_rcu_mightsleep(inst);
163}
164
165/*
166 * NMI runtime info.
167 */
168struct osn_nmi {
169 u64 count;
170 u64 delta_start;
171};
172
173/*
174 * IRQ runtime info.
175 */
176struct osn_irq {
177 u64 count;
178 u64 arrival_time;
179 u64 delta_start;
180};
181
182#define IRQ_CONTEXT 0
183#define THREAD_CONTEXT 1
184#define THREAD_URET 2
185/*
186 * sofirq runtime info.
187 */
188struct osn_softirq {
189 u64 count;
190 u64 arrival_time;
191 u64 delta_start;
192};
193
194/*
195 * thread runtime info.
196 */
197struct osn_thread {
198 u64 count;
199 u64 arrival_time;
200 u64 delta_start;
201};
202
203/*
204 * Runtime information: this structure saves the runtime information used by
205 * one sampling thread.
206 */
207struct osnoise_variables {
208 struct task_struct *kthread;
209 bool sampling;
210 pid_t pid;
211 struct osn_nmi nmi;
212 struct osn_irq irq;
213 struct osn_softirq softirq;
214 struct osn_thread thread;
215 local_t int_counter;
216};
217
218/*
219 * Per-cpu runtime information.
220 */
221static DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
222
223/*
224 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
225 */
226static inline struct osnoise_variables *this_cpu_osn_var(void)
227{
228 return this_cpu_ptr(&per_cpu_osnoise_var);
229}
230
231/*
232 * Protect the interface.
233 */
234static struct mutex interface_lock;
235
236#ifdef CONFIG_TIMERLAT_TRACER
237/*
238 * Runtime information for the timer mode.
239 */
240struct timerlat_variables {
241 struct task_struct *kthread;
242 struct hrtimer timer;
243 u64 rel_period;
244 u64 abs_period;
245 bool tracing_thread;
246 u64 count;
247 bool uthread_migrate;
248};
249
250static DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
251
252/*
253 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
254 */
255static inline struct timerlat_variables *this_cpu_tmr_var(void)
256{
257 return this_cpu_ptr(&per_cpu_timerlat_var);
258}
259
260/*
261 * tlat_var_reset - Reset the values of the given timerlat_variables
262 */
263static inline void tlat_var_reset(void)
264{
265 struct timerlat_variables *tlat_var;
266 int cpu;
267
268 /* Synchronize with the timerlat interfaces */
269 mutex_lock(&interface_lock);
270 /*
271 * So far, all the values are initialized as 0, so
272 * zeroing the structure is perfect.
273 */
274 for_each_cpu(cpu, cpu_online_mask) {
275 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
276 if (tlat_var->kthread)
277 hrtimer_cancel(&tlat_var->timer);
278 memset(tlat_var, 0, sizeof(*tlat_var));
279 }
280 mutex_unlock(&interface_lock);
281}
282#else /* CONFIG_TIMERLAT_TRACER */
283#define tlat_var_reset() do {} while (0)
284#endif /* CONFIG_TIMERLAT_TRACER */
285
286/*
287 * osn_var_reset - Reset the values of the given osnoise_variables
288 */
289static inline void osn_var_reset(void)
290{
291 struct osnoise_variables *osn_var;
292 int cpu;
293
294 /*
295 * So far, all the values are initialized as 0, so
296 * zeroing the structure is perfect.
297 */
298 for_each_cpu(cpu, cpu_online_mask) {
299 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
300 memset(osn_var, 0, sizeof(*osn_var));
301 }
302}
303
304/*
305 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
306 */
307static inline void osn_var_reset_all(void)
308{
309 osn_var_reset();
310 tlat_var_reset();
311}
312
313/*
314 * Tells NMIs to call back to the osnoise tracer to record timestamps.
315 */
316bool trace_osnoise_callback_enabled;
317
318/*
319 * osnoise sample structure definition. Used to store the statistics of a
320 * sample run.
321 */
322struct osnoise_sample {
323 u64 runtime; /* runtime */
324 u64 noise; /* noise */
325 u64 max_sample; /* max single noise sample */
326 int hw_count; /* # HW (incl. hypervisor) interference */
327 int nmi_count; /* # NMIs during this sample */
328 int irq_count; /* # IRQs during this sample */
329 int softirq_count; /* # softirqs during this sample */
330 int thread_count; /* # threads during this sample */
331};
332
333#ifdef CONFIG_TIMERLAT_TRACER
334/*
335 * timerlat sample structure definition. Used to store the statistics of
336 * a sample run.
337 */
338struct timerlat_sample {
339 u64 timer_latency; /* timer_latency */
340 unsigned int seqnum; /* unique sequence */
341 int context; /* timer context */
342};
343#endif
344
345/*
346 * Tracer data.
347 */
348static struct osnoise_data {
349 u64 sample_period; /* total sampling period */
350 u64 sample_runtime; /* active sampling portion of period */
351 u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
352 u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
353#ifdef CONFIG_TIMERLAT_TRACER
354 u64 timerlat_period; /* timerlat period */
355 u64 print_stack; /* print IRQ stack if total > */
356 int timerlat_tracer; /* timerlat tracer */
357#endif
358 bool tainted; /* infor users and developers about a problem */
359} osnoise_data = {
360 .sample_period = DEFAULT_SAMPLE_PERIOD,
361 .sample_runtime = DEFAULT_SAMPLE_RUNTIME,
362 .stop_tracing = 0,
363 .stop_tracing_total = 0,
364#ifdef CONFIG_TIMERLAT_TRACER
365 .print_stack = 0,
366 .timerlat_period = DEFAULT_TIMERLAT_PERIOD,
367 .timerlat_tracer = 0,
368#endif
369};
370
371#ifdef CONFIG_TIMERLAT_TRACER
372static inline bool timerlat_enabled(void)
373{
374 return osnoise_data.timerlat_tracer;
375}
376
377static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
378{
379 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
380 /*
381 * If the timerlat is enabled, but the irq handler did
382 * not run yet enabling timerlat_tracer, do not trace.
383 */
384 if (!tlat_var->tracing_thread) {
385 osn_var->softirq.arrival_time = 0;
386 osn_var->softirq.delta_start = 0;
387 return 0;
388 }
389 return 1;
390}
391
392static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
393{
394 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
395 /*
396 * If the timerlat is enabled, but the irq handler did
397 * not run yet enabling timerlat_tracer, do not trace.
398 */
399 if (!tlat_var->tracing_thread) {
400 osn_var->thread.delta_start = 0;
401 osn_var->thread.arrival_time = 0;
402 return 0;
403 }
404 return 1;
405}
406#else /* CONFIG_TIMERLAT_TRACER */
407static inline bool timerlat_enabled(void)
408{
409 return false;
410}
411
412static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
413{
414 return 1;
415}
416static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
417{
418 return 1;
419}
420#endif
421
422#ifdef CONFIG_PREEMPT_RT
423/*
424 * Print the osnoise header info.
425 */
426static void print_osnoise_headers(struct seq_file *s)
427{
428 if (osnoise_data.tainted)
429 seq_puts(s, "# osnoise is tainted!\n");
430
431 seq_puts(s, "# _-------=> irqs-off\n");
432 seq_puts(s, "# / _------=> need-resched\n");
433 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
434 seq_puts(s, "# || / _----=> hardirq/softirq\n");
435 seq_puts(s, "# ||| / _---=> preempt-depth\n");
436 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
437 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
438
439 seq_puts(s, "# |||||| / ");
440 seq_puts(s, " MAX\n");
441
442 seq_puts(s, "# ||||| / ");
443 seq_puts(s, " SINGLE Interference counters:\n");
444
445 seq_puts(s, "# ||||||| RUNTIME ");
446 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
447
448 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
449 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
450
451 seq_puts(s, "# | | | ||||||| | | ");
452 seq_puts(s, " | | | | | | | |\n");
453}
454#else /* CONFIG_PREEMPT_RT */
455static void print_osnoise_headers(struct seq_file *s)
456{
457 if (osnoise_data.tainted)
458 seq_puts(s, "# osnoise is tainted!\n");
459
460 seq_puts(s, "# _-----=> irqs-off\n");
461 seq_puts(s, "# / _----=> need-resched\n");
462 seq_puts(s, "# | / _---=> hardirq/softirq\n");
463 seq_puts(s, "# || / _--=> preempt-depth\n");
464 seq_puts(s, "# ||| / _-=> migrate-disable ");
465 seq_puts(s, " MAX\n");
466 seq_puts(s, "# |||| / delay ");
467 seq_puts(s, " SINGLE Interference counters:\n");
468
469 seq_puts(s, "# ||||| RUNTIME ");
470 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
471
472 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
473 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
474
475 seq_puts(s, "# | | | ||||| | | ");
476 seq_puts(s, " | | | | | | | |\n");
477}
478#endif /* CONFIG_PREEMPT_RT */
479
480/*
481 * osnoise_taint - report an osnoise error.
482 */
483#define osnoise_taint(msg) ({ \
484 struct osnoise_instance *inst; \
485 struct trace_buffer *buffer; \
486 \
487 rcu_read_lock(); \
488 list_for_each_entry_rcu(inst, &osnoise_instances, list) { \
489 buffer = inst->tr->array_buffer.buffer; \
490 trace_array_printk_buf(buffer, _THIS_IP_, msg); \
491 } \
492 rcu_read_unlock(); \
493 osnoise_data.tainted = true; \
494})
495
496/*
497 * Record an osnoise_sample into the tracer buffer.
498 */
499static void
500__trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer)
501{
502 struct ring_buffer_event *event;
503 struct osnoise_entry *entry;
504
505 event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
506 tracing_gen_ctx());
507 if (!event)
508 return;
509 entry = ring_buffer_event_data(event);
510 entry->runtime = sample->runtime;
511 entry->noise = sample->noise;
512 entry->max_sample = sample->max_sample;
513 entry->hw_count = sample->hw_count;
514 entry->nmi_count = sample->nmi_count;
515 entry->irq_count = sample->irq_count;
516 entry->softirq_count = sample->softirq_count;
517 entry->thread_count = sample->thread_count;
518
519 trace_buffer_unlock_commit_nostack(buffer, event);
520}
521
522/*
523 * Record an osnoise_sample on all osnoise instances.
524 */
525static void trace_osnoise_sample(struct osnoise_sample *sample)
526{
527 struct osnoise_instance *inst;
528 struct trace_buffer *buffer;
529
530 rcu_read_lock();
531 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
532 buffer = inst->tr->array_buffer.buffer;
533 __trace_osnoise_sample(sample, buffer);
534 }
535 rcu_read_unlock();
536}
537
538#ifdef CONFIG_TIMERLAT_TRACER
539/*
540 * Print the timerlat header info.
541 */
542#ifdef CONFIG_PREEMPT_RT
543static void print_timerlat_headers(struct seq_file *s)
544{
545 seq_puts(s, "# _-------=> irqs-off\n");
546 seq_puts(s, "# / _------=> need-resched\n");
547 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
548 seq_puts(s, "# || / _----=> hardirq/softirq\n");
549 seq_puts(s, "# ||| / _---=> preempt-depth\n");
550 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
551 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
552 seq_puts(s, "# |||||| /\n");
553 seq_puts(s, "# ||||||| ACTIVATION\n");
554 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
555 seq_puts(s, " CONTEXT LATENCY\n");
556 seq_puts(s, "# | | | ||||||| | | ");
557 seq_puts(s, " | |\n");
558}
559#else /* CONFIG_PREEMPT_RT */
560static void print_timerlat_headers(struct seq_file *s)
561{
562 seq_puts(s, "# _-----=> irqs-off\n");
563 seq_puts(s, "# / _----=> need-resched\n");
564 seq_puts(s, "# | / _---=> hardirq/softirq\n");
565 seq_puts(s, "# || / _--=> preempt-depth\n");
566 seq_puts(s, "# ||| / _-=> migrate-disable\n");
567 seq_puts(s, "# |||| / delay\n");
568 seq_puts(s, "# ||||| ACTIVATION\n");
569 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
570 seq_puts(s, " CONTEXT LATENCY\n");
571 seq_puts(s, "# | | | ||||| | | ");
572 seq_puts(s, " | |\n");
573}
574#endif /* CONFIG_PREEMPT_RT */
575
576static void
577__trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer)
578{
579 struct ring_buffer_event *event;
580 struct timerlat_entry *entry;
581
582 event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
583 tracing_gen_ctx());
584 if (!event)
585 return;
586 entry = ring_buffer_event_data(event);
587 entry->seqnum = sample->seqnum;
588 entry->context = sample->context;
589 entry->timer_latency = sample->timer_latency;
590
591 trace_buffer_unlock_commit_nostack(buffer, event);
592}
593
594/*
595 * Record an timerlat_sample into the tracer buffer.
596 */
597static void trace_timerlat_sample(struct timerlat_sample *sample)
598{
599 struct osnoise_instance *inst;
600 struct trace_buffer *buffer;
601
602 rcu_read_lock();
603 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
604 buffer = inst->tr->array_buffer.buffer;
605 __trace_timerlat_sample(sample, buffer);
606 }
607 rcu_read_unlock();
608}
609
610#ifdef CONFIG_STACKTRACE
611
612#define MAX_CALLS 256
613
614/*
615 * Stack trace will take place only at IRQ level, so, no need
616 * to control nesting here.
617 */
618struct trace_stack {
619 int stack_size;
620 int nr_entries;
621 unsigned long calls[MAX_CALLS];
622};
623
624static DEFINE_PER_CPU(struct trace_stack, trace_stack);
625
626/*
627 * timerlat_save_stack - save a stack trace without printing
628 *
629 * Save the current stack trace without printing. The
630 * stack will be printed later, after the end of the measurement.
631 */
632static void timerlat_save_stack(int skip)
633{
634 unsigned int size, nr_entries;
635 struct trace_stack *fstack;
636
637 fstack = this_cpu_ptr(&trace_stack);
638
639 size = ARRAY_SIZE(fstack->calls);
640
641 nr_entries = stack_trace_save(fstack->calls, size, skip);
642
643 fstack->stack_size = nr_entries * sizeof(unsigned long);
644 fstack->nr_entries = nr_entries;
645
646 return;
647
648}
649
650static void
651__timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size)
652{
653 struct ring_buffer_event *event;
654 struct stack_entry *entry;
655
656 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
657 tracing_gen_ctx());
658 if (!event)
659 return;
660
661 entry = ring_buffer_event_data(event);
662
663 memcpy(&entry->caller, fstack->calls, size);
664 entry->size = fstack->nr_entries;
665
666 trace_buffer_unlock_commit_nostack(buffer, event);
667}
668
669/*
670 * timerlat_dump_stack - dump a stack trace previously saved
671 */
672static void timerlat_dump_stack(u64 latency)
673{
674 struct osnoise_instance *inst;
675 struct trace_buffer *buffer;
676 struct trace_stack *fstack;
677 unsigned int size;
678
679 /*
680 * trace only if latency > print_stack config, if enabled.
681 */
682 if (!osnoise_data.print_stack || osnoise_data.print_stack > latency)
683 return;
684
685 preempt_disable_notrace();
686 fstack = this_cpu_ptr(&trace_stack);
687 size = fstack->stack_size;
688
689 rcu_read_lock();
690 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
691 buffer = inst->tr->array_buffer.buffer;
692 __timerlat_dump_stack(buffer, fstack, size);
693
694 }
695 rcu_read_unlock();
696 preempt_enable_notrace();
697}
698#else /* CONFIG_STACKTRACE */
699#define timerlat_dump_stack(u64 latency) do {} while (0)
700#define timerlat_save_stack(a) do {} while (0)
701#endif /* CONFIG_STACKTRACE */
702#endif /* CONFIG_TIMERLAT_TRACER */
703
704/*
705 * Macros to encapsulate the time capturing infrastructure.
706 */
707#define time_get() trace_clock_local()
708#define time_to_us(x) div_u64(x, 1000)
709#define time_sub(a, b) ((a) - (b))
710
711/*
712 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
713 *
714 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
715 * to discount the NMI interference.
716 *
717 * See get_int_safe_duration().
718 */
719static inline void
720cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
721{
722 if (osn_var->irq.delta_start)
723 osn_var->irq.delta_start += duration;
724}
725
726#ifndef CONFIG_PREEMPT_RT
727/*
728 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
729 *
730 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
731 * forward to discount the interference.
732 *
733 * See get_int_safe_duration().
734 */
735static inline void
736cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
737{
738 if (osn_var->softirq.delta_start)
739 osn_var->softirq.delta_start += duration;
740}
741#else /* CONFIG_PREEMPT_RT */
742#define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
743#endif
744
745/*
746 * cond_move_thread_delta_start - Forward the delta_start of a running thread
747 *
748 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
749 * is pushed forward to discount the interference.
750 *
751 * See get_int_safe_duration().
752 */
753static inline void
754cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
755{
756 if (osn_var->thread.delta_start)
757 osn_var->thread.delta_start += duration;
758}
759
760/*
761 * get_int_safe_duration - Get the duration of a window
762 *
763 * The irq, softirq and thread varaibles need to have its duration without
764 * the interference from higher priority interrupts. Instead of keeping a
765 * variable to discount the interrupt interference from these variables, the
766 * starting time of these variables are pushed forward with the interrupt's
767 * duration. In this way, a single variable is used to:
768 *
769 * - Know if a given window is being measured.
770 * - Account its duration.
771 * - Discount the interference.
772 *
773 * To avoid getting inconsistent values, e.g.,:
774 *
775 * now = time_get()
776 * ---> interrupt!
777 * delta_start -= int duration;
778 * <---
779 * duration = now - delta_start;
780 *
781 * result: negative duration if the variable duration before the
782 * interrupt was smaller than the interrupt execution.
783 *
784 * A counter of interrupts is used. If the counter increased, try
785 * to capture an interference safe duration.
786 */
787static inline s64
788get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
789{
790 u64 int_counter, now;
791 s64 duration;
792
793 do {
794 int_counter = local_read(&osn_var->int_counter);
795 /* synchronize with interrupts */
796 barrier();
797
798 now = time_get();
799 duration = (now - *delta_start);
800
801 /* synchronize with interrupts */
802 barrier();
803 } while (int_counter != local_read(&osn_var->int_counter));
804
805 /*
806 * This is an evidence of race conditions that cause
807 * a value to be "discounted" too much.
808 */
809 if (duration < 0)
810 osnoise_taint("Negative duration!\n");
811
812 *delta_start = 0;
813
814 return duration;
815}
816
817/*
818 *
819 * set_int_safe_time - Save the current time on *time, aware of interference
820 *
821 * Get the time, taking into consideration a possible interference from
822 * higher priority interrupts.
823 *
824 * See get_int_safe_duration() for an explanation.
825 */
826static u64
827set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
828{
829 u64 int_counter;
830
831 do {
832 int_counter = local_read(&osn_var->int_counter);
833 /* synchronize with interrupts */
834 barrier();
835
836 *time = time_get();
837
838 /* synchronize with interrupts */
839 barrier();
840 } while (int_counter != local_read(&osn_var->int_counter));
841
842 return int_counter;
843}
844
845#ifdef CONFIG_TIMERLAT_TRACER
846/*
847 * copy_int_safe_time - Copy *src into *desc aware of interference
848 */
849static u64
850copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
851{
852 u64 int_counter;
853
854 do {
855 int_counter = local_read(&osn_var->int_counter);
856 /* synchronize with interrupts */
857 barrier();
858
859 *dst = *src;
860
861 /* synchronize with interrupts */
862 barrier();
863 } while (int_counter != local_read(&osn_var->int_counter));
864
865 return int_counter;
866}
867#endif /* CONFIG_TIMERLAT_TRACER */
868
869/*
870 * trace_osnoise_callback - NMI entry/exit callback
871 *
872 * This function is called at the entry and exit NMI code. The bool enter
873 * distinguishes between either case. This function is used to note a NMI
874 * occurrence, compute the noise caused by the NMI, and to remove the noise
875 * it is potentially causing on other interference variables.
876 */
877void trace_osnoise_callback(bool enter)
878{
879 struct osnoise_variables *osn_var = this_cpu_osn_var();
880 u64 duration;
881
882 if (!osn_var->sampling)
883 return;
884
885 /*
886 * Currently trace_clock_local() calls sched_clock() and the
887 * generic version is not NMI safe.
888 */
889 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
890 if (enter) {
891 osn_var->nmi.delta_start = time_get();
892 local_inc(&osn_var->int_counter);
893 } else {
894 duration = time_get() - osn_var->nmi.delta_start;
895
896 trace_nmi_noise(osn_var->nmi.delta_start, duration);
897
898 cond_move_irq_delta_start(osn_var, duration);
899 cond_move_softirq_delta_start(osn_var, duration);
900 cond_move_thread_delta_start(osn_var, duration);
901 }
902 }
903
904 if (enter)
905 osn_var->nmi.count++;
906}
907
908/*
909 * osnoise_trace_irq_entry - Note the starting of an IRQ
910 *
911 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
912 * it is safe to use a single variable (ons_var->irq) to save the statistics.
913 * The arrival_time is used to report... the arrival time. The delta_start
914 * is used to compute the duration at the IRQ exit handler. See
915 * cond_move_irq_delta_start().
916 */
917void osnoise_trace_irq_entry(int id)
918{
919 struct osnoise_variables *osn_var = this_cpu_osn_var();
920
921 if (!osn_var->sampling)
922 return;
923 /*
924 * This value will be used in the report, but not to compute
925 * the execution time, so it is safe to get it unsafe.
926 */
927 osn_var->irq.arrival_time = time_get();
928 set_int_safe_time(osn_var, &osn_var->irq.delta_start);
929 osn_var->irq.count++;
930
931 local_inc(&osn_var->int_counter);
932}
933
934/*
935 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
936 *
937 * Computes the duration of the IRQ noise, and trace it. Also discounts the
938 * interference from other sources of noise could be currently being accounted.
939 */
940void osnoise_trace_irq_exit(int id, const char *desc)
941{
942 struct osnoise_variables *osn_var = this_cpu_osn_var();
943 s64 duration;
944
945 if (!osn_var->sampling)
946 return;
947
948 duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
949 trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
950 osn_var->irq.arrival_time = 0;
951 cond_move_softirq_delta_start(osn_var, duration);
952 cond_move_thread_delta_start(osn_var, duration);
953}
954
955/*
956 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
957 *
958 * Used to note the starting of an IRQ occurece.
959 */
960static void trace_irqentry_callback(void *data, int irq,
961 struct irqaction *action)
962{
963 osnoise_trace_irq_entry(irq);
964}
965
966/*
967 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
968 *
969 * Used to note the end of an IRQ occurece.
970 */
971static void trace_irqexit_callback(void *data, int irq,
972 struct irqaction *action, int ret)
973{
974 osnoise_trace_irq_exit(irq, action->name);
975}
976
977/*
978 * arch specific register function.
979 */
980int __weak osnoise_arch_register(void)
981{
982 return 0;
983}
984
985/*
986 * arch specific unregister function.
987 */
988void __weak osnoise_arch_unregister(void)
989{
990 return;
991}
992
993/*
994 * hook_irq_events - Hook IRQ handling events
995 *
996 * This function hooks the IRQ related callbacks to the respective trace
997 * events.
998 */
999static int hook_irq_events(void)
1000{
1001 int ret;
1002
1003 ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1004 if (ret)
1005 goto out_err;
1006
1007 ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1008 if (ret)
1009 goto out_unregister_entry;
1010
1011 ret = osnoise_arch_register();
1012 if (ret)
1013 goto out_irq_exit;
1014
1015 return 0;
1016
1017out_irq_exit:
1018 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1019out_unregister_entry:
1020 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1021out_err:
1022 return -EINVAL;
1023}
1024
1025/*
1026 * unhook_irq_events - Unhook IRQ handling events
1027 *
1028 * This function unhooks the IRQ related callbacks to the respective trace
1029 * events.
1030 */
1031static void unhook_irq_events(void)
1032{
1033 osnoise_arch_unregister();
1034 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1035 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1036}
1037
1038#ifndef CONFIG_PREEMPT_RT
1039/*
1040 * trace_softirq_entry_callback - Note the starting of a softirq
1041 *
1042 * Save the starting time of a softirq. As softirqs are non-preemptive to
1043 * other softirqs, it is safe to use a single variable (ons_var->softirq)
1044 * to save the statistics. The arrival_time is used to report... the
1045 * arrival time. The delta_start is used to compute the duration at the
1046 * softirq exit handler. See cond_move_softirq_delta_start().
1047 */
1048static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1049{
1050 struct osnoise_variables *osn_var = this_cpu_osn_var();
1051
1052 if (!osn_var->sampling)
1053 return;
1054 /*
1055 * This value will be used in the report, but not to compute
1056 * the execution time, so it is safe to get it unsafe.
1057 */
1058 osn_var->softirq.arrival_time = time_get();
1059 set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1060 osn_var->softirq.count++;
1061
1062 local_inc(&osn_var->int_counter);
1063}
1064
1065/*
1066 * trace_softirq_exit_callback - Note the end of an softirq
1067 *
1068 * Computes the duration of the softirq noise, and trace it. Also discounts the
1069 * interference from other sources of noise could be currently being accounted.
1070 */
1071static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1072{
1073 struct osnoise_variables *osn_var = this_cpu_osn_var();
1074 s64 duration;
1075
1076 if (!osn_var->sampling)
1077 return;
1078
1079 if (unlikely(timerlat_enabled()))
1080 if (!timerlat_softirq_exit(osn_var))
1081 return;
1082
1083 duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1084 trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1085 cond_move_thread_delta_start(osn_var, duration);
1086 osn_var->softirq.arrival_time = 0;
1087}
1088
1089/*
1090 * hook_softirq_events - Hook softirq handling events
1091 *
1092 * This function hooks the softirq related callbacks to the respective trace
1093 * events.
1094 */
1095static int hook_softirq_events(void)
1096{
1097 int ret;
1098
1099 ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1100 if (ret)
1101 goto out_err;
1102
1103 ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1104 if (ret)
1105 goto out_unreg_entry;
1106
1107 return 0;
1108
1109out_unreg_entry:
1110 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1111out_err:
1112 return -EINVAL;
1113}
1114
1115/*
1116 * unhook_softirq_events - Unhook softirq handling events
1117 *
1118 * This function hooks the softirq related callbacks to the respective trace
1119 * events.
1120 */
1121static void unhook_softirq_events(void)
1122{
1123 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1124 unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1125}
1126#else /* CONFIG_PREEMPT_RT */
1127/*
1128 * softirq are threads on the PREEMPT_RT mode.
1129 */
1130static int hook_softirq_events(void)
1131{
1132 return 0;
1133}
1134static void unhook_softirq_events(void)
1135{
1136}
1137#endif
1138
1139/*
1140 * thread_entry - Record the starting of a thread noise window
1141 *
1142 * It saves the context switch time for a noisy thread, and increments
1143 * the interference counters.
1144 */
1145static void
1146thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
1147{
1148 if (!osn_var->sampling)
1149 return;
1150 /*
1151 * The arrival time will be used in the report, but not to compute
1152 * the execution time, so it is safe to get it unsafe.
1153 */
1154 osn_var->thread.arrival_time = time_get();
1155
1156 set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1157
1158 osn_var->thread.count++;
1159 local_inc(&osn_var->int_counter);
1160}
1161
1162/*
1163 * thread_exit - Report the end of a thread noise window
1164 *
1165 * It computes the total noise from a thread, tracing if needed.
1166 */
1167static void
1168thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
1169{
1170 s64 duration;
1171
1172 if (!osn_var->sampling)
1173 return;
1174
1175 if (unlikely(timerlat_enabled()))
1176 if (!timerlat_thread_exit(osn_var))
1177 return;
1178
1179 duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1180
1181 trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1182
1183 osn_var->thread.arrival_time = 0;
1184}
1185
1186#ifdef CONFIG_TIMERLAT_TRACER
1187/*
1188 * osnoise_stop_exception - Stop tracing and the tracer.
1189 */
1190static __always_inline void osnoise_stop_exception(char *msg, int cpu)
1191{
1192 struct osnoise_instance *inst;
1193 struct trace_array *tr;
1194
1195 rcu_read_lock();
1196 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1197 tr = inst->tr;
1198 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1199 "stop tracing hit on cpu %d due to exception: %s\n",
1200 smp_processor_id(),
1201 msg);
1202
1203 if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1204 panic("tracer hit on cpu %d due to exception: %s\n",
1205 smp_processor_id(),
1206 msg);
1207
1208 tracer_tracing_off(tr);
1209 }
1210 rcu_read_unlock();
1211}
1212
1213/*
1214 * trace_sched_migrate_callback - sched:sched_migrate_task trace event handler
1215 *
1216 * his function is hooked to the sched:sched_migrate_task trace event, and monitors
1217 * timerlat user-space thread migration.
1218 */
1219static void trace_sched_migrate_callback(void *data, struct task_struct *p, int dest_cpu)
1220{
1221 struct osnoise_variables *osn_var;
1222 long cpu = task_cpu(p);
1223
1224 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
1225 if (osn_var->pid == p->pid && dest_cpu != cpu) {
1226 per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
1227 osnoise_taint("timerlat user-thread migrated\n");
1228 osnoise_stop_exception("timerlat user-thread migrated", cpu);
1229 }
1230}
1231
1232static bool monitor_enabled;
1233
1234static int register_migration_monitor(void)
1235{
1236 int ret = 0;
1237
1238 /*
1239 * Timerlat thread migration check is only required when running timerlat in user-space.
1240 * Thus, enable callback only if timerlat is set with no workload.
1241 */
1242 if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options)) {
1243 if (WARN_ON_ONCE(monitor_enabled))
1244 return 0;
1245
1246 ret = register_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1247 if (!ret)
1248 monitor_enabled = true;
1249 }
1250
1251 return ret;
1252}
1253
1254static void unregister_migration_monitor(void)
1255{
1256 if (!monitor_enabled)
1257 return;
1258
1259 unregister_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1260 monitor_enabled = false;
1261}
1262#else
1263static int register_migration_monitor(void)
1264{
1265 return 0;
1266}
1267static void unregister_migration_monitor(void) {}
1268#endif
1269/*
1270 * trace_sched_switch - sched:sched_switch trace event handler
1271 *
1272 * This function is hooked to the sched:sched_switch trace event, and it is
1273 * used to record the beginning and to report the end of a thread noise window.
1274 */
1275static void
1276trace_sched_switch_callback(void *data, bool preempt,
1277 struct task_struct *p,
1278 struct task_struct *n,
1279 unsigned int prev_state)
1280{
1281 struct osnoise_variables *osn_var = this_cpu_osn_var();
1282 int workload = test_bit(OSN_WORKLOAD, &osnoise_options);
1283
1284 if ((p->pid != osn_var->pid) || !workload)
1285 thread_exit(osn_var, p);
1286
1287 if ((n->pid != osn_var->pid) || !workload)
1288 thread_entry(osn_var, n);
1289}
1290
1291/*
1292 * hook_thread_events - Hook the instrumentation for thread noise
1293 *
1294 * Hook the osnoise tracer callbacks to handle the noise from other
1295 * threads on the necessary kernel events.
1296 */
1297static int hook_thread_events(void)
1298{
1299 int ret;
1300
1301 ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1302 if (ret)
1303 return -EINVAL;
1304
1305 ret = register_migration_monitor();
1306 if (ret)
1307 goto out_unreg;
1308
1309 return 0;
1310
1311out_unreg:
1312 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1313 return -EINVAL;
1314}
1315
1316/*
1317 * unhook_thread_events - unhook the instrumentation for thread noise
1318 *
1319 * Unook the osnoise tracer callbacks to handle the noise from other
1320 * threads on the necessary kernel events.
1321 */
1322static void unhook_thread_events(void)
1323{
1324 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1325 unregister_migration_monitor();
1326}
1327
1328/*
1329 * save_osn_sample_stats - Save the osnoise_sample statistics
1330 *
1331 * Save the osnoise_sample statistics before the sampling phase. These
1332 * values will be used later to compute the diff betwneen the statistics
1333 * before and after the osnoise sampling.
1334 */
1335static void
1336save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1337{
1338 s->nmi_count = osn_var->nmi.count;
1339 s->irq_count = osn_var->irq.count;
1340 s->softirq_count = osn_var->softirq.count;
1341 s->thread_count = osn_var->thread.count;
1342}
1343
1344/*
1345 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1346 *
1347 * After a sample period, compute the difference on the osnoise_sample
1348 * statistics. The struct osnoise_sample *s contains the statistics saved via
1349 * save_osn_sample_stats() before the osnoise sampling.
1350 */
1351static void
1352diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1353{
1354 s->nmi_count = osn_var->nmi.count - s->nmi_count;
1355 s->irq_count = osn_var->irq.count - s->irq_count;
1356 s->softirq_count = osn_var->softirq.count - s->softirq_count;
1357 s->thread_count = osn_var->thread.count - s->thread_count;
1358}
1359
1360/*
1361 * osnoise_stop_tracing - Stop tracing and the tracer.
1362 */
1363static __always_inline void osnoise_stop_tracing(void)
1364{
1365 struct osnoise_instance *inst;
1366 struct trace_array *tr;
1367
1368 rcu_read_lock();
1369 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1370 tr = inst->tr;
1371 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1372 "stop tracing hit on cpu %d\n", smp_processor_id());
1373
1374 if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1375 panic("tracer hit stop condition on CPU %d\n", smp_processor_id());
1376
1377 tracer_tracing_off(tr);
1378 }
1379 rcu_read_unlock();
1380}
1381
1382/*
1383 * osnoise_has_tracing_on - Check if there is at least one instance on
1384 */
1385static __always_inline int osnoise_has_tracing_on(void)
1386{
1387 struct osnoise_instance *inst;
1388 int trace_is_on = 0;
1389
1390 rcu_read_lock();
1391 list_for_each_entry_rcu(inst, &osnoise_instances, list)
1392 trace_is_on += tracer_tracing_is_on(inst->tr);
1393 rcu_read_unlock();
1394
1395 return trace_is_on;
1396}
1397
1398/*
1399 * notify_new_max_latency - Notify a new max latency via fsnotify interface.
1400 */
1401static void notify_new_max_latency(u64 latency)
1402{
1403 struct osnoise_instance *inst;
1404 struct trace_array *tr;
1405
1406 rcu_read_lock();
1407 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1408 tr = inst->tr;
1409 if (tracer_tracing_is_on(tr) && tr->max_latency < latency) {
1410 tr->max_latency = latency;
1411 latency_fsnotify(tr);
1412 }
1413 }
1414 rcu_read_unlock();
1415}
1416
1417/*
1418 * run_osnoise - Sample the time and look for osnoise
1419 *
1420 * Used to capture the time, looking for potential osnoise latency repeatedly.
1421 * Different from hwlat_detector, it is called with preemption and interrupts
1422 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1423 * osnoise sampling thread, as they would do with a regular thread.
1424 */
1425static int run_osnoise(void)
1426{
1427 bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options);
1428 struct osnoise_variables *osn_var = this_cpu_osn_var();
1429 u64 start, sample, last_sample;
1430 u64 last_int_count, int_count;
1431 s64 noise = 0, max_noise = 0;
1432 s64 total, last_total = 0;
1433 struct osnoise_sample s;
1434 bool disable_preemption;
1435 unsigned int threshold;
1436 u64 runtime, stop_in;
1437 u64 sum_noise = 0;
1438 int hw_count = 0;
1439 int ret = -1;
1440
1441 /*
1442 * Disabling preemption is only required if IRQs are enabled,
1443 * and the options is set on.
1444 */
1445 disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options);
1446
1447 /*
1448 * Considers the current thread as the workload.
1449 */
1450 osn_var->pid = current->pid;
1451
1452 /*
1453 * Save the current stats for the diff
1454 */
1455 save_osn_sample_stats(osn_var, &s);
1456
1457 /*
1458 * if threshold is 0, use the default value of 1 us.
1459 */
1460 threshold = tracing_thresh ? : 1000;
1461
1462 /*
1463 * Apply PREEMPT and IRQ disabled options.
1464 */
1465 if (disable_irq)
1466 local_irq_disable();
1467
1468 if (disable_preemption)
1469 preempt_disable();
1470
1471 /*
1472 * Make sure NMIs see sampling first
1473 */
1474 osn_var->sampling = true;
1475 barrier();
1476
1477 /*
1478 * Transform the *_us config to nanoseconds to avoid the
1479 * division on the main loop.
1480 */
1481 runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1482 stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1483
1484 /*
1485 * Start timestemp
1486 */
1487 start = time_get();
1488
1489 /*
1490 * "previous" loop.
1491 */
1492 last_int_count = set_int_safe_time(osn_var, &last_sample);
1493
1494 do {
1495 /*
1496 * Get sample!
1497 */
1498 int_count = set_int_safe_time(osn_var, &sample);
1499
1500 noise = time_sub(sample, last_sample);
1501
1502 /*
1503 * This shouldn't happen.
1504 */
1505 if (noise < 0) {
1506 osnoise_taint("negative noise!");
1507 goto out;
1508 }
1509
1510 /*
1511 * Sample runtime.
1512 */
1513 total = time_sub(sample, start);
1514
1515 /*
1516 * Check for possible overflows.
1517 */
1518 if (total < last_total) {
1519 osnoise_taint("total overflow!");
1520 break;
1521 }
1522
1523 last_total = total;
1524
1525 if (noise >= threshold) {
1526 int interference = int_count - last_int_count;
1527
1528 if (noise > max_noise)
1529 max_noise = noise;
1530
1531 if (!interference)
1532 hw_count++;
1533
1534 sum_noise += noise;
1535
1536 trace_sample_threshold(last_sample, noise, interference);
1537
1538 if (osnoise_data.stop_tracing)
1539 if (noise > stop_in)
1540 osnoise_stop_tracing();
1541 }
1542
1543 /*
1544 * In some cases, notably when running on a nohz_full CPU with
1545 * a stopped tick PREEMPT_RCU has no way to account for QSs.
1546 * This will eventually cause unwarranted noise as PREEMPT_RCU
1547 * will force preemption as the means of ending the current
1548 * grace period. We avoid this problem by calling
1549 * rcu_momentary_eqs(), which performs a zero duration
1550 * EQS allowing PREEMPT_RCU to end the current grace period.
1551 * This call shouldn't be wrapped inside an RCU critical
1552 * section.
1553 *
1554 * Note that in non PREEMPT_RCU kernels QSs are handled through
1555 * cond_resched()
1556 */
1557 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
1558 if (!disable_irq)
1559 local_irq_disable();
1560
1561 rcu_momentary_eqs();
1562
1563 if (!disable_irq)
1564 local_irq_enable();
1565 }
1566
1567 /*
1568 * For the non-preemptive kernel config: let threads runs, if
1569 * they so wish, unless set not do to so.
1570 */
1571 if (!disable_irq && !disable_preemption)
1572 cond_resched();
1573
1574 last_sample = sample;
1575 last_int_count = int_count;
1576
1577 } while (total < runtime && !kthread_should_stop());
1578
1579 /*
1580 * Finish the above in the view for interrupts.
1581 */
1582 barrier();
1583
1584 osn_var->sampling = false;
1585
1586 /*
1587 * Make sure sampling data is no longer updated.
1588 */
1589 barrier();
1590
1591 /*
1592 * Return to the preemptive state.
1593 */
1594 if (disable_preemption)
1595 preempt_enable();
1596
1597 if (disable_irq)
1598 local_irq_enable();
1599
1600 /*
1601 * Save noise info.
1602 */
1603 s.noise = time_to_us(sum_noise);
1604 s.runtime = time_to_us(total);
1605 s.max_sample = time_to_us(max_noise);
1606 s.hw_count = hw_count;
1607
1608 /* Save interference stats info */
1609 diff_osn_sample_stats(osn_var, &s);
1610
1611 trace_osnoise_sample(&s);
1612
1613 notify_new_max_latency(max_noise);
1614
1615 if (osnoise_data.stop_tracing_total)
1616 if (s.noise > osnoise_data.stop_tracing_total)
1617 osnoise_stop_tracing();
1618
1619 return 0;
1620out:
1621 return ret;
1622}
1623
1624static struct cpumask osnoise_cpumask;
1625static struct cpumask save_cpumask;
1626static struct cpumask kthread_cpumask;
1627
1628/*
1629 * osnoise_sleep - sleep until the next period
1630 */
1631static void osnoise_sleep(bool skip_period)
1632{
1633 u64 interval;
1634 ktime_t wake_time;
1635
1636 mutex_lock(&interface_lock);
1637 if (skip_period)
1638 interval = osnoise_data.sample_period;
1639 else
1640 interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1641 mutex_unlock(&interface_lock);
1642
1643 /*
1644 * differently from hwlat_detector, the osnoise tracer can run
1645 * without a pause because preemption is on.
1646 */
1647 if (!interval) {
1648 /* Let synchronize_rcu_tasks() make progress */
1649 cond_resched_tasks_rcu_qs();
1650 return;
1651 }
1652
1653 wake_time = ktime_add_us(ktime_get(), interval);
1654 __set_current_state(TASK_INTERRUPTIBLE);
1655
1656 while (schedule_hrtimeout(&wake_time, HRTIMER_MODE_ABS)) {
1657 if (kthread_should_stop())
1658 break;
1659 }
1660}
1661
1662/*
1663 * osnoise_migration_pending - checks if the task needs to migrate
1664 *
1665 * osnoise/timerlat threads are per-cpu. If there is a pending request to
1666 * migrate the thread away from the current CPU, something bad has happened.
1667 * Play the good citizen and leave.
1668 *
1669 * Returns 0 if it is safe to continue, 1 otherwise.
1670 */
1671static inline int osnoise_migration_pending(void)
1672{
1673 if (!current->migration_pending)
1674 return 0;
1675
1676 /*
1677 * If migration is pending, there is a task waiting for the
1678 * tracer to enable migration. The tracer does not allow migration,
1679 * thus: taint and leave to unblock the blocked thread.
1680 */
1681 osnoise_taint("migration requested to osnoise threads, leaving.");
1682
1683 /*
1684 * Unset this thread from the threads managed by the interface.
1685 * The tracers are responsible for cleaning their env before
1686 * exiting.
1687 */
1688 mutex_lock(&interface_lock);
1689 this_cpu_osn_var()->kthread = NULL;
1690 cpumask_clear_cpu(smp_processor_id(), &kthread_cpumask);
1691 mutex_unlock(&interface_lock);
1692
1693 return 1;
1694}
1695
1696/*
1697 * osnoise_main - The osnoise detection kernel thread
1698 *
1699 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1700 * every period.
1701 */
1702static int osnoise_main(void *data)
1703{
1704 unsigned long flags;
1705
1706 /*
1707 * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1708 * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1709 *
1710 * To work around this limitation, disable migration and remove the
1711 * flag.
1712 */
1713 migrate_disable();
1714 raw_spin_lock_irqsave(¤t->pi_lock, flags);
1715 current->flags &= ~(PF_NO_SETAFFINITY);
1716 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
1717
1718 while (!kthread_should_stop()) {
1719 if (osnoise_migration_pending())
1720 break;
1721
1722 /* skip a period if tracing is off on all instances */
1723 if (!osnoise_has_tracing_on()) {
1724 osnoise_sleep(true);
1725 continue;
1726 }
1727
1728 run_osnoise();
1729 osnoise_sleep(false);
1730 }
1731
1732 migrate_enable();
1733 return 0;
1734}
1735
1736#ifdef CONFIG_TIMERLAT_TRACER
1737/*
1738 * timerlat_irq - hrtimer handler for timerlat.
1739 */
1740static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1741{
1742 struct osnoise_variables *osn_var = this_cpu_osn_var();
1743 struct timerlat_variables *tlat;
1744 struct timerlat_sample s;
1745 u64 now;
1746 u64 diff;
1747
1748 /*
1749 * I am not sure if the timer was armed for this CPU. So, get
1750 * the timerlat struct from the timer itself, not from this
1751 * CPU.
1752 */
1753 tlat = container_of(timer, struct timerlat_variables, timer);
1754
1755 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1756
1757 /*
1758 * Enable the osnoise: events for thread an softirq.
1759 */
1760 tlat->tracing_thread = true;
1761
1762 osn_var->thread.arrival_time = time_get();
1763
1764 /*
1765 * A hardirq is running: the timer IRQ. It is for sure preempting
1766 * a thread, and potentially preempting a softirq.
1767 *
1768 * At this point, it is not interesting to know the duration of the
1769 * preempted thread (and maybe softirq), but how much time they will
1770 * delay the beginning of the execution of the timer thread.
1771 *
1772 * To get the correct (net) delay added by the softirq, its delta_start
1773 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1774 * start of the sofitrq will be zeroed, accounting then only the time
1775 * after that.
1776 *
1777 * The thread follows the same principle. However, if a softirq is
1778 * running, the thread needs to receive the softirq delta_start. The
1779 * reason being is that the softirq will be the last to be unfolded,
1780 * resseting the thread delay to zero.
1781 *
1782 * The PREEMPT_RT is a special case, though. As softirqs run as threads
1783 * on RT, moving the thread is enough.
1784 */
1785 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1786 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1787 &osn_var->softirq.delta_start);
1788
1789 copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1790 &osn_var->irq.delta_start);
1791 } else {
1792 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1793 &osn_var->irq.delta_start);
1794 }
1795
1796 /*
1797 * Compute the current time with the expected time.
1798 */
1799 diff = now - tlat->abs_period;
1800
1801 tlat->count++;
1802 s.seqnum = tlat->count;
1803 s.timer_latency = diff;
1804 s.context = IRQ_CONTEXT;
1805
1806 trace_timerlat_sample(&s);
1807
1808 if (osnoise_data.stop_tracing) {
1809 if (time_to_us(diff) >= osnoise_data.stop_tracing) {
1810
1811 /*
1812 * At this point, if stop_tracing is set and <= print_stack,
1813 * print_stack is set and would be printed in the thread handler.
1814 *
1815 * Thus, print the stack trace as it is helpful to define the
1816 * root cause of an IRQ latency.
1817 */
1818 if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
1819 timerlat_save_stack(0);
1820 timerlat_dump_stack(time_to_us(diff));
1821 }
1822
1823 osnoise_stop_tracing();
1824 notify_new_max_latency(diff);
1825
1826 wake_up_process(tlat->kthread);
1827
1828 return HRTIMER_NORESTART;
1829 }
1830 }
1831
1832 wake_up_process(tlat->kthread);
1833
1834 if (osnoise_data.print_stack)
1835 timerlat_save_stack(0);
1836
1837 return HRTIMER_NORESTART;
1838}
1839
1840/*
1841 * wait_next_period - Wait for the next period for timerlat
1842 */
1843static int wait_next_period(struct timerlat_variables *tlat)
1844{
1845 ktime_t next_abs_period, now;
1846 u64 rel_period = osnoise_data.timerlat_period * 1000;
1847
1848 now = hrtimer_cb_get_time(&tlat->timer);
1849 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1850
1851 /*
1852 * Save the next abs_period.
1853 */
1854 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1855
1856 /*
1857 * If the new abs_period is in the past, skip the activation.
1858 */
1859 while (ktime_compare(now, next_abs_period) > 0) {
1860 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1861 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1862 }
1863
1864 set_current_state(TASK_INTERRUPTIBLE);
1865
1866 hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1867 schedule();
1868 return 1;
1869}
1870
1871/*
1872 * timerlat_main- Timerlat main
1873 */
1874static int timerlat_main(void *data)
1875{
1876 struct osnoise_variables *osn_var = this_cpu_osn_var();
1877 struct timerlat_variables *tlat = this_cpu_tmr_var();
1878 struct timerlat_sample s;
1879 struct sched_param sp;
1880 unsigned long flags;
1881 u64 now, diff;
1882
1883 /*
1884 * Make the thread RT, that is how cyclictest is usually used.
1885 */
1886 sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1887 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1888
1889 /*
1890 * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1891 * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1892 *
1893 * To work around this limitation, disable migration and remove the
1894 * flag.
1895 */
1896 migrate_disable();
1897 raw_spin_lock_irqsave(¤t->pi_lock, flags);
1898 current->flags &= ~(PF_NO_SETAFFINITY);
1899 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
1900
1901 tlat->count = 0;
1902 tlat->tracing_thread = false;
1903
1904 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1905 tlat->timer.function = timerlat_irq;
1906 tlat->kthread = current;
1907 osn_var->pid = current->pid;
1908 /*
1909 * Anotate the arrival time.
1910 */
1911 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1912
1913 wait_next_period(tlat);
1914
1915 osn_var->sampling = 1;
1916
1917 while (!kthread_should_stop()) {
1918
1919 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1920 diff = now - tlat->abs_period;
1921
1922 s.seqnum = tlat->count;
1923 s.timer_latency = diff;
1924 s.context = THREAD_CONTEXT;
1925
1926 trace_timerlat_sample(&s);
1927
1928 notify_new_max_latency(diff);
1929
1930 timerlat_dump_stack(time_to_us(diff));
1931
1932 tlat->tracing_thread = false;
1933 if (osnoise_data.stop_tracing_total)
1934 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1935 osnoise_stop_tracing();
1936
1937 if (osnoise_migration_pending())
1938 break;
1939
1940 wait_next_period(tlat);
1941 }
1942
1943 hrtimer_cancel(&tlat->timer);
1944 migrate_enable();
1945 return 0;
1946}
1947#else /* CONFIG_TIMERLAT_TRACER */
1948static int timerlat_main(void *data)
1949{
1950 return 0;
1951}
1952#endif /* CONFIG_TIMERLAT_TRACER */
1953
1954/*
1955 * stop_kthread - stop a workload thread
1956 */
1957static void stop_kthread(unsigned int cpu)
1958{
1959 struct task_struct *kthread;
1960
1961 kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL);
1962 if (kthread) {
1963 if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask) &&
1964 !WARN_ON(!test_bit(OSN_WORKLOAD, &osnoise_options))) {
1965 kthread_stop(kthread);
1966 } else if (!WARN_ON(test_bit(OSN_WORKLOAD, &osnoise_options))) {
1967 /*
1968 * This is a user thread waiting on the timerlat_fd. We need
1969 * to close all users, and the best way to guarantee this is
1970 * by killing the thread. NOTE: this is a purpose specific file.
1971 */
1972 kill_pid(kthread->thread_pid, SIGKILL, 1);
1973 put_task_struct(kthread);
1974 }
1975 } else {
1976 /* if no workload, just return */
1977 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1978 /*
1979 * This is set in the osnoise tracer case.
1980 */
1981 per_cpu(per_cpu_osnoise_var, cpu).sampling = false;
1982 barrier();
1983 }
1984 }
1985}
1986
1987/*
1988 * stop_per_cpu_kthread - Stop per-cpu threads
1989 *
1990 * Stop the osnoise sampling htread. Use this on unload and at system
1991 * shutdown.
1992 */
1993static void stop_per_cpu_kthreads(void)
1994{
1995 int cpu;
1996
1997 cpus_read_lock();
1998
1999 for_each_online_cpu(cpu)
2000 stop_kthread(cpu);
2001
2002 cpus_read_unlock();
2003}
2004
2005/*
2006 * start_kthread - Start a workload tread
2007 */
2008static int start_kthread(unsigned int cpu)
2009{
2010 struct task_struct *kthread;
2011 void *main = osnoise_main;
2012 char comm[24];
2013
2014 /* Do not start a new thread if it is already running */
2015 if (per_cpu(per_cpu_osnoise_var, cpu).kthread)
2016 return 0;
2017
2018 if (timerlat_enabled()) {
2019 snprintf(comm, 24, "timerlat/%d", cpu);
2020 main = timerlat_main;
2021 } else {
2022 /* if no workload, just return */
2023 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2024 per_cpu(per_cpu_osnoise_var, cpu).sampling = true;
2025 barrier();
2026 return 0;
2027 }
2028 snprintf(comm, 24, "osnoise/%d", cpu);
2029 }
2030
2031 kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
2032
2033 if (IS_ERR(kthread)) {
2034 pr_err(BANNER "could not start sampling thread\n");
2035 stop_per_cpu_kthreads();
2036 return -ENOMEM;
2037 }
2038
2039 per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
2040 cpumask_set_cpu(cpu, &kthread_cpumask);
2041
2042 return 0;
2043}
2044
2045/*
2046 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
2047 *
2048 * This starts the kernel thread that will look for osnoise on many
2049 * cpus.
2050 */
2051static int start_per_cpu_kthreads(void)
2052{
2053 struct cpumask *current_mask = &save_cpumask;
2054 int retval = 0;
2055 int cpu;
2056
2057 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2058 if (timerlat_enabled())
2059 return 0;
2060 }
2061
2062 cpus_read_lock();
2063 /*
2064 * Run only on online CPUs in which osnoise is allowed to run.
2065 */
2066 cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
2067
2068 for_each_possible_cpu(cpu) {
2069 if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask)) {
2070 struct task_struct *kthread;
2071
2072 kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL);
2073 if (!WARN_ON(!kthread))
2074 kthread_stop(kthread);
2075 }
2076 }
2077
2078 for_each_cpu(cpu, current_mask) {
2079 retval = start_kthread(cpu);
2080 if (retval) {
2081 cpus_read_unlock();
2082 stop_per_cpu_kthreads();
2083 return retval;
2084 }
2085 }
2086
2087 cpus_read_unlock();
2088
2089 return retval;
2090}
2091
2092#ifdef CONFIG_HOTPLUG_CPU
2093static void osnoise_hotplug_workfn(struct work_struct *dummy)
2094{
2095 unsigned int cpu = smp_processor_id();
2096
2097 mutex_lock(&trace_types_lock);
2098
2099 if (!osnoise_has_registered_instances())
2100 goto out_unlock_trace;
2101
2102 mutex_lock(&interface_lock);
2103 cpus_read_lock();
2104
2105 if (!cpu_online(cpu))
2106 goto out_unlock;
2107 if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
2108 goto out_unlock;
2109
2110 start_kthread(cpu);
2111
2112out_unlock:
2113 cpus_read_unlock();
2114 mutex_unlock(&interface_lock);
2115out_unlock_trace:
2116 mutex_unlock(&trace_types_lock);
2117}
2118
2119static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
2120
2121/*
2122 * osnoise_cpu_init - CPU hotplug online callback function
2123 */
2124static int osnoise_cpu_init(unsigned int cpu)
2125{
2126 schedule_work_on(cpu, &osnoise_hotplug_work);
2127 return 0;
2128}
2129
2130/*
2131 * osnoise_cpu_die - CPU hotplug offline callback function
2132 */
2133static int osnoise_cpu_die(unsigned int cpu)
2134{
2135 stop_kthread(cpu);
2136 return 0;
2137}
2138
2139static void osnoise_init_hotplug_support(void)
2140{
2141 int ret;
2142
2143 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
2144 osnoise_cpu_init, osnoise_cpu_die);
2145 if (ret < 0)
2146 pr_warn(BANNER "Error to init cpu hotplug support\n");
2147
2148 return;
2149}
2150#else /* CONFIG_HOTPLUG_CPU */
2151static void osnoise_init_hotplug_support(void)
2152{
2153 return;
2154}
2155#endif /* CONFIG_HOTPLUG_CPU */
2156
2157/*
2158 * seq file functions for the osnoise/options file.
2159 */
2160static void *s_options_start(struct seq_file *s, loff_t *pos)
2161{
2162 int option = *pos;
2163
2164 mutex_lock(&interface_lock);
2165
2166 if (option >= OSN_MAX)
2167 return NULL;
2168
2169 return pos;
2170}
2171
2172static void *s_options_next(struct seq_file *s, void *v, loff_t *pos)
2173{
2174 int option = ++(*pos);
2175
2176 if (option >= OSN_MAX)
2177 return NULL;
2178
2179 return pos;
2180}
2181
2182static int s_options_show(struct seq_file *s, void *v)
2183{
2184 loff_t *pos = v;
2185 int option = *pos;
2186
2187 if (option == OSN_DEFAULTS) {
2188 if (osnoise_options == OSN_DEFAULT_OPTIONS)
2189 seq_printf(s, "%s", osnoise_options_str[option]);
2190 else
2191 seq_printf(s, "NO_%s", osnoise_options_str[option]);
2192 goto out;
2193 }
2194
2195 if (test_bit(option, &osnoise_options))
2196 seq_printf(s, "%s", osnoise_options_str[option]);
2197 else
2198 seq_printf(s, "NO_%s", osnoise_options_str[option]);
2199
2200out:
2201 if (option != OSN_MAX)
2202 seq_puts(s, " ");
2203
2204 return 0;
2205}
2206
2207static void s_options_stop(struct seq_file *s, void *v)
2208{
2209 seq_puts(s, "\n");
2210 mutex_unlock(&interface_lock);
2211}
2212
2213static const struct seq_operations osnoise_options_seq_ops = {
2214 .start = s_options_start,
2215 .next = s_options_next,
2216 .show = s_options_show,
2217 .stop = s_options_stop
2218};
2219
2220static int osnoise_options_open(struct inode *inode, struct file *file)
2221{
2222 return seq_open(file, &osnoise_options_seq_ops);
2223};
2224
2225/**
2226 * osnoise_options_write - Write function for "options" entry
2227 * @filp: The active open file structure
2228 * @ubuf: The user buffer that contains the value to write
2229 * @cnt: The maximum number of bytes to write to "file"
2230 * @ppos: The current position in @file
2231 *
2232 * Writing the option name sets the option, writing the "NO_"
2233 * prefix in front of the option name disables it.
2234 *
2235 * Writing "DEFAULTS" resets the option values to the default ones.
2236 */
2237static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf,
2238 size_t cnt, loff_t *ppos)
2239{
2240 int running, option, enable, retval;
2241 char buf[256], *option_str;
2242
2243 if (cnt >= 256)
2244 return -EINVAL;
2245
2246 if (copy_from_user(buf, ubuf, cnt))
2247 return -EFAULT;
2248
2249 buf[cnt] = 0;
2250
2251 if (strncmp(buf, "NO_", 3)) {
2252 option_str = strstrip(buf);
2253 enable = true;
2254 } else {
2255 option_str = strstrip(&buf[3]);
2256 enable = false;
2257 }
2258
2259 option = match_string(osnoise_options_str, OSN_MAX, option_str);
2260 if (option < 0)
2261 return -EINVAL;
2262
2263 /*
2264 * trace_types_lock is taken to avoid concurrency on start/stop.
2265 */
2266 mutex_lock(&trace_types_lock);
2267 running = osnoise_has_registered_instances();
2268 if (running)
2269 stop_per_cpu_kthreads();
2270
2271 mutex_lock(&interface_lock);
2272 /*
2273 * avoid CPU hotplug operations that might read options.
2274 */
2275 cpus_read_lock();
2276
2277 retval = cnt;
2278
2279 if (enable) {
2280 if (option == OSN_DEFAULTS)
2281 osnoise_options = OSN_DEFAULT_OPTIONS;
2282 else
2283 set_bit(option, &osnoise_options);
2284 } else {
2285 if (option == OSN_DEFAULTS)
2286 retval = -EINVAL;
2287 else
2288 clear_bit(option, &osnoise_options);
2289 }
2290
2291 cpus_read_unlock();
2292 mutex_unlock(&interface_lock);
2293
2294 if (running)
2295 start_per_cpu_kthreads();
2296 mutex_unlock(&trace_types_lock);
2297
2298 return retval;
2299}
2300
2301/*
2302 * osnoise_cpus_read - Read function for reading the "cpus" file
2303 * @filp: The active open file structure
2304 * @ubuf: The userspace provided buffer to read value into
2305 * @cnt: The maximum number of bytes to read
2306 * @ppos: The current "file" position
2307 *
2308 * Prints the "cpus" output into the user-provided buffer.
2309 */
2310static ssize_t
2311osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
2312 loff_t *ppos)
2313{
2314 char *mask_str;
2315 int len;
2316
2317 mutex_lock(&interface_lock);
2318
2319 len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
2320 mask_str = kmalloc(len, GFP_KERNEL);
2321 if (!mask_str) {
2322 count = -ENOMEM;
2323 goto out_unlock;
2324 }
2325
2326 len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
2327 if (len >= count) {
2328 count = -EINVAL;
2329 goto out_free;
2330 }
2331
2332 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
2333
2334out_free:
2335 kfree(mask_str);
2336out_unlock:
2337 mutex_unlock(&interface_lock);
2338
2339 return count;
2340}
2341
2342/*
2343 * osnoise_cpus_write - Write function for "cpus" entry
2344 * @filp: The active open file structure
2345 * @ubuf: The user buffer that contains the value to write
2346 * @cnt: The maximum number of bytes to write to "file"
2347 * @ppos: The current position in @file
2348 *
2349 * This function provides a write implementation for the "cpus"
2350 * interface to the osnoise trace. By default, it lists all CPUs,
2351 * in this way, allowing osnoise threads to run on any online CPU
2352 * of the system. It serves to restrict the execution of osnoise to the
2353 * set of CPUs writing via this interface. Why not use "tracing_cpumask"?
2354 * Because the user might be interested in tracing what is running on
2355 * other CPUs. For instance, one might run osnoise in one HT CPU
2356 * while observing what is running on the sibling HT CPU.
2357 */
2358static ssize_t
2359osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
2360 loff_t *ppos)
2361{
2362 cpumask_var_t osnoise_cpumask_new;
2363 int running, err;
2364 char buf[256];
2365
2366 if (count >= 256)
2367 return -EINVAL;
2368
2369 if (copy_from_user(buf, ubuf, count))
2370 return -EFAULT;
2371
2372 if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
2373 return -ENOMEM;
2374
2375 err = cpulist_parse(buf, osnoise_cpumask_new);
2376 if (err)
2377 goto err_free;
2378
2379 /*
2380 * trace_types_lock is taken to avoid concurrency on start/stop.
2381 */
2382 mutex_lock(&trace_types_lock);
2383 running = osnoise_has_registered_instances();
2384 if (running)
2385 stop_per_cpu_kthreads();
2386
2387 mutex_lock(&interface_lock);
2388 /*
2389 * osnoise_cpumask is read by CPU hotplug operations.
2390 */
2391 cpus_read_lock();
2392
2393 cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
2394
2395 cpus_read_unlock();
2396 mutex_unlock(&interface_lock);
2397
2398 if (running)
2399 start_per_cpu_kthreads();
2400 mutex_unlock(&trace_types_lock);
2401
2402 free_cpumask_var(osnoise_cpumask_new);
2403 return count;
2404
2405err_free:
2406 free_cpumask_var(osnoise_cpumask_new);
2407
2408 return err;
2409}
2410
2411#ifdef CONFIG_TIMERLAT_TRACER
2412static int timerlat_fd_open(struct inode *inode, struct file *file)
2413{
2414 struct osnoise_variables *osn_var;
2415 struct timerlat_variables *tlat;
2416 long cpu = (long) inode->i_cdev;
2417
2418 mutex_lock(&interface_lock);
2419
2420 /*
2421 * This file is accessible only if timerlat is enabled, and
2422 * NO_OSNOISE_WORKLOAD is set.
2423 */
2424 if (!timerlat_enabled() || test_bit(OSN_WORKLOAD, &osnoise_options)) {
2425 mutex_unlock(&interface_lock);
2426 return -EINVAL;
2427 }
2428
2429 migrate_disable();
2430
2431 osn_var = this_cpu_osn_var();
2432
2433 /*
2434 * The osn_var->pid holds the single access to this file.
2435 */
2436 if (osn_var->pid) {
2437 mutex_unlock(&interface_lock);
2438 migrate_enable();
2439 return -EBUSY;
2440 }
2441
2442 /*
2443 * timerlat tracer is a per-cpu tracer. Check if the user-space too
2444 * is pinned to a single CPU. The tracer laters monitor if the task
2445 * migrates and then disables tracer if it does. However, it is
2446 * worth doing this basic acceptance test to avoid obviusly wrong
2447 * setup.
2448 */
2449 if (current->nr_cpus_allowed > 1 || cpu != smp_processor_id()) {
2450 mutex_unlock(&interface_lock);
2451 migrate_enable();
2452 return -EPERM;
2453 }
2454
2455 /*
2456 * From now on, it is good to go.
2457 */
2458 file->private_data = inode->i_cdev;
2459
2460 get_task_struct(current);
2461
2462 osn_var->kthread = current;
2463 osn_var->pid = current->pid;
2464
2465 /*
2466 * Setup is done.
2467 */
2468 mutex_unlock(&interface_lock);
2469
2470 tlat = this_cpu_tmr_var();
2471 tlat->count = 0;
2472
2473 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
2474 tlat->timer.function = timerlat_irq;
2475
2476 migrate_enable();
2477 return 0;
2478};
2479
2480/*
2481 * timerlat_fd_read - Read function for "timerlat_fd" file
2482 * @file: The active open file structure
2483 * @ubuf: The userspace provided buffer to read value into
2484 * @cnt: The maximum number of bytes to read
2485 * @ppos: The current "file" position
2486 *
2487 * Prints 1 on timerlat, the number of interferences on osnoise, -1 on error.
2488 */
2489static ssize_t
2490timerlat_fd_read(struct file *file, char __user *ubuf, size_t count,
2491 loff_t *ppos)
2492{
2493 long cpu = (long) file->private_data;
2494 struct osnoise_variables *osn_var;
2495 struct timerlat_variables *tlat;
2496 struct timerlat_sample s;
2497 s64 diff;
2498 u64 now;
2499
2500 migrate_disable();
2501
2502 tlat = this_cpu_tmr_var();
2503
2504 /*
2505 * While in user-space, the thread is migratable. There is nothing
2506 * we can do about it.
2507 * So, if the thread is running on another CPU, stop the machinery.
2508 */
2509 if (cpu == smp_processor_id()) {
2510 if (tlat->uthread_migrate) {
2511 migrate_enable();
2512 return -EINVAL;
2513 }
2514 } else {
2515 per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
2516 osnoise_taint("timerlat user thread migrate\n");
2517 osnoise_stop_tracing();
2518 migrate_enable();
2519 return -EINVAL;
2520 }
2521
2522 osn_var = this_cpu_osn_var();
2523
2524 /*
2525 * The timerlat in user-space runs in a different order:
2526 * the read() starts from the execution of the previous occurrence,
2527 * sleeping for the next occurrence.
2528 *
2529 * So, skip if we are entering on read() before the first wakeup
2530 * from timerlat IRQ:
2531 */
2532 if (likely(osn_var->sampling)) {
2533 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2534 diff = now - tlat->abs_period;
2535
2536 /*
2537 * it was not a timer firing, but some other signal?
2538 */
2539 if (diff < 0)
2540 goto out;
2541
2542 s.seqnum = tlat->count;
2543 s.timer_latency = diff;
2544 s.context = THREAD_URET;
2545
2546 trace_timerlat_sample(&s);
2547
2548 notify_new_max_latency(diff);
2549
2550 tlat->tracing_thread = false;
2551 if (osnoise_data.stop_tracing_total)
2552 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
2553 osnoise_stop_tracing();
2554 } else {
2555 tlat->tracing_thread = false;
2556 tlat->kthread = current;
2557
2558 /* Annotate now to drift new period */
2559 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
2560
2561 osn_var->sampling = 1;
2562 }
2563
2564 /* wait for the next period */
2565 wait_next_period(tlat);
2566
2567 /* This is the wakeup from this cycle */
2568 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2569 diff = now - tlat->abs_period;
2570
2571 /*
2572 * it was not a timer firing, but some other signal?
2573 */
2574 if (diff < 0)
2575 goto out;
2576
2577 s.seqnum = tlat->count;
2578 s.timer_latency = diff;
2579 s.context = THREAD_CONTEXT;
2580
2581 trace_timerlat_sample(&s);
2582
2583 if (osnoise_data.stop_tracing_total) {
2584 if (time_to_us(diff) >= osnoise_data.stop_tracing_total) {
2585 timerlat_dump_stack(time_to_us(diff));
2586 notify_new_max_latency(diff);
2587 osnoise_stop_tracing();
2588 }
2589 }
2590
2591out:
2592 migrate_enable();
2593 return 0;
2594}
2595
2596static int timerlat_fd_release(struct inode *inode, struct file *file)
2597{
2598 struct osnoise_variables *osn_var;
2599 struct timerlat_variables *tlat_var;
2600 long cpu = (long) file->private_data;
2601
2602 migrate_disable();
2603 mutex_lock(&interface_lock);
2604
2605 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
2606 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
2607
2608 if (tlat_var->kthread)
2609 hrtimer_cancel(&tlat_var->timer);
2610 memset(tlat_var, 0, sizeof(*tlat_var));
2611
2612 osn_var->sampling = 0;
2613 osn_var->pid = 0;
2614
2615 /*
2616 * We are leaving, not being stopped... see stop_kthread();
2617 */
2618 if (osn_var->kthread) {
2619 put_task_struct(osn_var->kthread);
2620 osn_var->kthread = NULL;
2621 }
2622
2623 mutex_unlock(&interface_lock);
2624 migrate_enable();
2625 return 0;
2626}
2627#endif
2628
2629/*
2630 * osnoise/runtime_us: cannot be greater than the period.
2631 */
2632static struct trace_min_max_param osnoise_runtime = {
2633 .lock = &interface_lock,
2634 .val = &osnoise_data.sample_runtime,
2635 .max = &osnoise_data.sample_period,
2636 .min = NULL,
2637};
2638
2639/*
2640 * osnoise/period_us: cannot be smaller than the runtime.
2641 */
2642static struct trace_min_max_param osnoise_period = {
2643 .lock = &interface_lock,
2644 .val = &osnoise_data.sample_period,
2645 .max = NULL,
2646 .min = &osnoise_data.sample_runtime,
2647};
2648
2649/*
2650 * osnoise/stop_tracing_us: no limit.
2651 */
2652static struct trace_min_max_param osnoise_stop_tracing_in = {
2653 .lock = &interface_lock,
2654 .val = &osnoise_data.stop_tracing,
2655 .max = NULL,
2656 .min = NULL,
2657};
2658
2659/*
2660 * osnoise/stop_tracing_total_us: no limit.
2661 */
2662static struct trace_min_max_param osnoise_stop_tracing_total = {
2663 .lock = &interface_lock,
2664 .val = &osnoise_data.stop_tracing_total,
2665 .max = NULL,
2666 .min = NULL,
2667};
2668
2669#ifdef CONFIG_TIMERLAT_TRACER
2670/*
2671 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2672 * latency is higher than val.
2673 */
2674static struct trace_min_max_param osnoise_print_stack = {
2675 .lock = &interface_lock,
2676 .val = &osnoise_data.print_stack,
2677 .max = NULL,
2678 .min = NULL,
2679};
2680
2681/*
2682 * osnoise/timerlat_period: min 100 us, max 1 s
2683 */
2684static u64 timerlat_min_period = 100;
2685static u64 timerlat_max_period = 1000000;
2686static struct trace_min_max_param timerlat_period = {
2687 .lock = &interface_lock,
2688 .val = &osnoise_data.timerlat_period,
2689 .max = &timerlat_max_period,
2690 .min = &timerlat_min_period,
2691};
2692
2693static const struct file_operations timerlat_fd_fops = {
2694 .open = timerlat_fd_open,
2695 .read = timerlat_fd_read,
2696 .release = timerlat_fd_release,
2697 .llseek = generic_file_llseek,
2698};
2699#endif
2700
2701static const struct file_operations cpus_fops = {
2702 .open = tracing_open_generic,
2703 .read = osnoise_cpus_read,
2704 .write = osnoise_cpus_write,
2705 .llseek = generic_file_llseek,
2706};
2707
2708static const struct file_operations osnoise_options_fops = {
2709 .open = osnoise_options_open,
2710 .read = seq_read,
2711 .llseek = seq_lseek,
2712 .release = seq_release,
2713 .write = osnoise_options_write
2714};
2715
2716#ifdef CONFIG_TIMERLAT_TRACER
2717#ifdef CONFIG_STACKTRACE
2718static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2719{
2720 struct dentry *tmp;
2721
2722 tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2723 &osnoise_print_stack, &trace_min_max_fops);
2724 if (!tmp)
2725 return -ENOMEM;
2726
2727 return 0;
2728}
2729#else /* CONFIG_STACKTRACE */
2730static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2731{
2732 return 0;
2733}
2734#endif /* CONFIG_STACKTRACE */
2735
2736static int osnoise_create_cpu_timerlat_fd(struct dentry *top_dir)
2737{
2738 struct dentry *timerlat_fd;
2739 struct dentry *per_cpu;
2740 struct dentry *cpu_dir;
2741 char cpu_str[30]; /* see trace.c: tracing_init_tracefs_percpu() */
2742 long cpu;
2743
2744 /*
2745 * Why not using tracing instance per_cpu/ dir?
2746 *
2747 * Because osnoise/timerlat have a single workload, having
2748 * multiple files like these are wast of memory.
2749 */
2750 per_cpu = tracefs_create_dir("per_cpu", top_dir);
2751 if (!per_cpu)
2752 return -ENOMEM;
2753
2754 for_each_possible_cpu(cpu) {
2755 snprintf(cpu_str, 30, "cpu%ld", cpu);
2756 cpu_dir = tracefs_create_dir(cpu_str, per_cpu);
2757 if (!cpu_dir)
2758 goto out_clean;
2759
2760 timerlat_fd = trace_create_file("timerlat_fd", TRACE_MODE_READ,
2761 cpu_dir, NULL, &timerlat_fd_fops);
2762 if (!timerlat_fd)
2763 goto out_clean;
2764
2765 /* Record the CPU */
2766 d_inode(timerlat_fd)->i_cdev = (void *)(cpu);
2767 }
2768
2769 return 0;
2770
2771out_clean:
2772 tracefs_remove(per_cpu);
2773 return -ENOMEM;
2774}
2775
2776/*
2777 * init_timerlat_tracefs - A function to initialize the timerlat interface files
2778 */
2779static int init_timerlat_tracefs(struct dentry *top_dir)
2780{
2781 struct dentry *tmp;
2782 int retval;
2783
2784 tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2785 &timerlat_period, &trace_min_max_fops);
2786 if (!tmp)
2787 return -ENOMEM;
2788
2789 retval = osnoise_create_cpu_timerlat_fd(top_dir);
2790 if (retval)
2791 return retval;
2792
2793 return init_timerlat_stack_tracefs(top_dir);
2794}
2795#else /* CONFIG_TIMERLAT_TRACER */
2796static int init_timerlat_tracefs(struct dentry *top_dir)
2797{
2798 return 0;
2799}
2800#endif /* CONFIG_TIMERLAT_TRACER */
2801
2802/*
2803 * init_tracefs - A function to initialize the tracefs interface files
2804 *
2805 * This function creates entries in tracefs for "osnoise" and "timerlat".
2806 * It creates these directories in the tracing directory, and within that
2807 * directory the use can change and view the configs.
2808 */
2809static int init_tracefs(void)
2810{
2811 struct dentry *top_dir;
2812 struct dentry *tmp;
2813 int ret;
2814
2815 ret = tracing_init_dentry();
2816 if (ret)
2817 return -ENOMEM;
2818
2819 top_dir = tracefs_create_dir("osnoise", NULL);
2820 if (!top_dir)
2821 return 0;
2822
2823 tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2824 &osnoise_period, &trace_min_max_fops);
2825 if (!tmp)
2826 goto err;
2827
2828 tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2829 &osnoise_runtime, &trace_min_max_fops);
2830 if (!tmp)
2831 goto err;
2832
2833 tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2834 &osnoise_stop_tracing_in, &trace_min_max_fops);
2835 if (!tmp)
2836 goto err;
2837
2838 tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2839 &osnoise_stop_tracing_total, &trace_min_max_fops);
2840 if (!tmp)
2841 goto err;
2842
2843 tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2844 if (!tmp)
2845 goto err;
2846
2847 tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL,
2848 &osnoise_options_fops);
2849 if (!tmp)
2850 goto err;
2851
2852 ret = init_timerlat_tracefs(top_dir);
2853 if (ret)
2854 goto err;
2855
2856 return 0;
2857
2858err:
2859 tracefs_remove(top_dir);
2860 return -ENOMEM;
2861}
2862
2863static int osnoise_hook_events(void)
2864{
2865 int retval;
2866
2867 /*
2868 * Trace is already hooked, we are re-enabling from
2869 * a stop_tracing_*.
2870 */
2871 if (trace_osnoise_callback_enabled)
2872 return 0;
2873
2874 retval = hook_irq_events();
2875 if (retval)
2876 return -EINVAL;
2877
2878 retval = hook_softirq_events();
2879 if (retval)
2880 goto out_unhook_irq;
2881
2882 retval = hook_thread_events();
2883 /*
2884 * All fine!
2885 */
2886 if (!retval)
2887 return 0;
2888
2889 unhook_softirq_events();
2890out_unhook_irq:
2891 unhook_irq_events();
2892 return -EINVAL;
2893}
2894
2895static void osnoise_unhook_events(void)
2896{
2897 unhook_thread_events();
2898 unhook_softirq_events();
2899 unhook_irq_events();
2900}
2901
2902/*
2903 * osnoise_workload_start - start the workload and hook to events
2904 */
2905static int osnoise_workload_start(void)
2906{
2907 int retval;
2908
2909 /*
2910 * Instances need to be registered after calling workload
2911 * start. Hence, if there is already an instance, the
2912 * workload was already registered. Otherwise, this
2913 * code is on the way to register the first instance,
2914 * and the workload will start.
2915 */
2916 if (osnoise_has_registered_instances())
2917 return 0;
2918
2919 osn_var_reset_all();
2920
2921 retval = osnoise_hook_events();
2922 if (retval)
2923 return retval;
2924
2925 /*
2926 * Make sure that ftrace_nmi_enter/exit() see reset values
2927 * before enabling trace_osnoise_callback_enabled.
2928 */
2929 barrier();
2930 trace_osnoise_callback_enabled = true;
2931
2932 retval = start_per_cpu_kthreads();
2933 if (retval) {
2934 trace_osnoise_callback_enabled = false;
2935 /*
2936 * Make sure that ftrace_nmi_enter/exit() see
2937 * trace_osnoise_callback_enabled as false before continuing.
2938 */
2939 barrier();
2940
2941 osnoise_unhook_events();
2942 return retval;
2943 }
2944
2945 return 0;
2946}
2947
2948/*
2949 * osnoise_workload_stop - stop the workload and unhook the events
2950 */
2951static void osnoise_workload_stop(void)
2952{
2953 /*
2954 * Instances need to be unregistered before calling
2955 * stop. Hence, if there is a registered instance, more
2956 * than one instance is running, and the workload will not
2957 * yet stop. Otherwise, this code is on the way to disable
2958 * the last instance, and the workload can stop.
2959 */
2960 if (osnoise_has_registered_instances())
2961 return;
2962
2963 /*
2964 * If callbacks were already disabled in a previous stop
2965 * call, there is no need to disable then again.
2966 *
2967 * For instance, this happens when tracing is stopped via:
2968 * echo 0 > tracing_on
2969 * echo nop > current_tracer.
2970 */
2971 if (!trace_osnoise_callback_enabled)
2972 return;
2973
2974 trace_osnoise_callback_enabled = false;
2975 /*
2976 * Make sure that ftrace_nmi_enter/exit() see
2977 * trace_osnoise_callback_enabled as false before continuing.
2978 */
2979 barrier();
2980
2981 stop_per_cpu_kthreads();
2982
2983 osnoise_unhook_events();
2984}
2985
2986static void osnoise_tracer_start(struct trace_array *tr)
2987{
2988 int retval;
2989
2990 /*
2991 * If the instance is already registered, there is no need to
2992 * register it again.
2993 */
2994 if (osnoise_instance_registered(tr))
2995 return;
2996
2997 retval = osnoise_workload_start();
2998 if (retval)
2999 pr_err(BANNER "Error starting osnoise tracer\n");
3000
3001 osnoise_register_instance(tr);
3002}
3003
3004static void osnoise_tracer_stop(struct trace_array *tr)
3005{
3006 osnoise_unregister_instance(tr);
3007 osnoise_workload_stop();
3008}
3009
3010static int osnoise_tracer_init(struct trace_array *tr)
3011{
3012 /*
3013 * Only allow osnoise tracer if timerlat tracer is not running
3014 * already.
3015 */
3016 if (timerlat_enabled())
3017 return -EBUSY;
3018
3019 tr->max_latency = 0;
3020
3021 osnoise_tracer_start(tr);
3022 return 0;
3023}
3024
3025static void osnoise_tracer_reset(struct trace_array *tr)
3026{
3027 osnoise_tracer_stop(tr);
3028}
3029
3030static struct tracer osnoise_tracer __read_mostly = {
3031 .name = "osnoise",
3032 .init = osnoise_tracer_init,
3033 .reset = osnoise_tracer_reset,
3034 .start = osnoise_tracer_start,
3035 .stop = osnoise_tracer_stop,
3036 .print_header = print_osnoise_headers,
3037 .allow_instances = true,
3038};
3039
3040#ifdef CONFIG_TIMERLAT_TRACER
3041static void timerlat_tracer_start(struct trace_array *tr)
3042{
3043 int retval;
3044
3045 /*
3046 * If the instance is already registered, there is no need to
3047 * register it again.
3048 */
3049 if (osnoise_instance_registered(tr))
3050 return;
3051
3052 retval = osnoise_workload_start();
3053 if (retval)
3054 pr_err(BANNER "Error starting timerlat tracer\n");
3055
3056 osnoise_register_instance(tr);
3057
3058 return;
3059}
3060
3061static void timerlat_tracer_stop(struct trace_array *tr)
3062{
3063 int cpu;
3064
3065 osnoise_unregister_instance(tr);
3066
3067 /*
3068 * Instruct the threads to stop only if this is the last instance.
3069 */
3070 if (!osnoise_has_registered_instances()) {
3071 for_each_online_cpu(cpu)
3072 per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
3073 }
3074
3075 osnoise_workload_stop();
3076}
3077
3078static int timerlat_tracer_init(struct trace_array *tr)
3079{
3080 /*
3081 * Only allow timerlat tracer if osnoise tracer is not running already.
3082 */
3083 if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
3084 return -EBUSY;
3085
3086 /*
3087 * If this is the first instance, set timerlat_tracer to block
3088 * osnoise tracer start.
3089 */
3090 if (!osnoise_has_registered_instances())
3091 osnoise_data.timerlat_tracer = 1;
3092
3093 tr->max_latency = 0;
3094 timerlat_tracer_start(tr);
3095
3096 return 0;
3097}
3098
3099static void timerlat_tracer_reset(struct trace_array *tr)
3100{
3101 timerlat_tracer_stop(tr);
3102
3103 /*
3104 * If this is the last instance, reset timerlat_tracer allowing
3105 * osnoise to be started.
3106 */
3107 if (!osnoise_has_registered_instances())
3108 osnoise_data.timerlat_tracer = 0;
3109}
3110
3111static struct tracer timerlat_tracer __read_mostly = {
3112 .name = "timerlat",
3113 .init = timerlat_tracer_init,
3114 .reset = timerlat_tracer_reset,
3115 .start = timerlat_tracer_start,
3116 .stop = timerlat_tracer_stop,
3117 .print_header = print_timerlat_headers,
3118 .allow_instances = true,
3119};
3120
3121__init static int init_timerlat_tracer(void)
3122{
3123 return register_tracer(&timerlat_tracer);
3124}
3125#else /* CONFIG_TIMERLAT_TRACER */
3126__init static int init_timerlat_tracer(void)
3127{
3128 return 0;
3129}
3130#endif /* CONFIG_TIMERLAT_TRACER */
3131
3132__init static int init_osnoise_tracer(void)
3133{
3134 int ret;
3135
3136 mutex_init(&interface_lock);
3137
3138 cpumask_copy(&osnoise_cpumask, cpu_all_mask);
3139
3140 ret = register_tracer(&osnoise_tracer);
3141 if (ret) {
3142 pr_err(BANNER "Error registering osnoise!\n");
3143 return ret;
3144 }
3145
3146 ret = init_timerlat_tracer();
3147 if (ret) {
3148 pr_err(BANNER "Error registering timerlat!\n");
3149 return ret;
3150 }
3151
3152 osnoise_init_hotplug_support();
3153
3154 INIT_LIST_HEAD_RCU(&osnoise_instances);
3155
3156 init_tracefs();
3157
3158 return 0;
3159}
3160late_initcall(init_osnoise_tracer);