Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
5 *
6 * Based on "hwlat_detector" tracer by:
7 * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8 * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9 * With feedback from Clark Williams <williams@redhat.com>
10 *
11 * And also based on the rtsl tracer presented on:
12 * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13 * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14 * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
15 *
16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
17 */
18
19#include <linux/kthread.h>
20#include <linux/tracefs.h>
21#include <linux/uaccess.h>
22#include <linux/cpumask.h>
23#include <linux/delay.h>
24#include <linux/sched/clock.h>
25#include <uapi/linux/sched/types.h>
26#include <linux/sched.h>
27#include "trace.h"
28
29#ifdef CONFIG_X86_LOCAL_APIC
30#include <asm/trace/irq_vectors.h>
31#undef TRACE_INCLUDE_PATH
32#undef TRACE_INCLUDE_FILE
33#endif /* CONFIG_X86_LOCAL_APIC */
34
35#include <trace/events/irq.h>
36#include <trace/events/sched.h>
37
38#define CREATE_TRACE_POINTS
39#include <trace/events/osnoise.h>
40
41/*
42 * Default values.
43 */
44#define BANNER "osnoise: "
45#define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
46#define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
47
48#define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
49#define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
50
51/*
52 * osnoise/options entries.
53 */
54enum osnoise_options_index {
55 OSN_DEFAULTS = 0,
56 OSN_WORKLOAD,
57 OSN_PANIC_ON_STOP,
58 OSN_PREEMPT_DISABLE,
59 OSN_IRQ_DISABLE,
60 OSN_MAX
61};
62
63static const char * const osnoise_options_str[OSN_MAX] = {
64 "DEFAULTS",
65 "OSNOISE_WORKLOAD",
66 "PANIC_ON_STOP",
67 "OSNOISE_PREEMPT_DISABLE",
68 "OSNOISE_IRQ_DISABLE" };
69
70#define OSN_DEFAULT_OPTIONS 0x2
71static unsigned long osnoise_options = OSN_DEFAULT_OPTIONS;
72
73/*
74 * trace_array of the enabled osnoise/timerlat instances.
75 */
76struct osnoise_instance {
77 struct list_head list;
78 struct trace_array *tr;
79};
80
81static struct list_head osnoise_instances;
82
83static bool osnoise_has_registered_instances(void)
84{
85 return !!list_first_or_null_rcu(&osnoise_instances,
86 struct osnoise_instance,
87 list);
88}
89
90/*
91 * osnoise_instance_registered - check if a tr is already registered
92 */
93static int osnoise_instance_registered(struct trace_array *tr)
94{
95 struct osnoise_instance *inst;
96 int found = 0;
97
98 rcu_read_lock();
99 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
100 if (inst->tr == tr)
101 found = 1;
102 }
103 rcu_read_unlock();
104
105 return found;
106}
107
108/*
109 * osnoise_register_instance - register a new trace instance
110 *
111 * Register a trace_array *tr in the list of instances running
112 * osnoise/timerlat tracers.
113 */
114static int osnoise_register_instance(struct trace_array *tr)
115{
116 struct osnoise_instance *inst;
117
118 /*
119 * register/unregister serialization is provided by trace's
120 * trace_types_lock.
121 */
122 lockdep_assert_held(&trace_types_lock);
123
124 inst = kmalloc(sizeof(*inst), GFP_KERNEL);
125 if (!inst)
126 return -ENOMEM;
127
128 INIT_LIST_HEAD_RCU(&inst->list);
129 inst->tr = tr;
130 list_add_tail_rcu(&inst->list, &osnoise_instances);
131
132 return 0;
133}
134
135/*
136 * osnoise_unregister_instance - unregister a registered trace instance
137 *
138 * Remove the trace_array *tr from the list of instances running
139 * osnoise/timerlat tracers.
140 */
141static void osnoise_unregister_instance(struct trace_array *tr)
142{
143 struct osnoise_instance *inst;
144 int found = 0;
145
146 /*
147 * register/unregister serialization is provided by trace's
148 * trace_types_lock.
149 */
150 list_for_each_entry_rcu(inst, &osnoise_instances, list,
151 lockdep_is_held(&trace_types_lock)) {
152 if (inst->tr == tr) {
153 list_del_rcu(&inst->list);
154 found = 1;
155 break;
156 }
157 }
158
159 if (!found)
160 return;
161
162 kvfree_rcu(inst);
163}
164
165/*
166 * NMI runtime info.
167 */
168struct osn_nmi {
169 u64 count;
170 u64 delta_start;
171};
172
173/*
174 * IRQ runtime info.
175 */
176struct osn_irq {
177 u64 count;
178 u64 arrival_time;
179 u64 delta_start;
180};
181
182#define IRQ_CONTEXT 0
183#define THREAD_CONTEXT 1
184/*
185 * sofirq runtime info.
186 */
187struct osn_softirq {
188 u64 count;
189 u64 arrival_time;
190 u64 delta_start;
191};
192
193/*
194 * thread runtime info.
195 */
196struct osn_thread {
197 u64 count;
198 u64 arrival_time;
199 u64 delta_start;
200};
201
202/*
203 * Runtime information: this structure saves the runtime information used by
204 * one sampling thread.
205 */
206struct osnoise_variables {
207 struct task_struct *kthread;
208 bool sampling;
209 pid_t pid;
210 struct osn_nmi nmi;
211 struct osn_irq irq;
212 struct osn_softirq softirq;
213 struct osn_thread thread;
214 local_t int_counter;
215};
216
217/*
218 * Per-cpu runtime information.
219 */
220DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
221
222/*
223 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
224 */
225static inline struct osnoise_variables *this_cpu_osn_var(void)
226{
227 return this_cpu_ptr(&per_cpu_osnoise_var);
228}
229
230#ifdef CONFIG_TIMERLAT_TRACER
231/*
232 * Runtime information for the timer mode.
233 */
234struct timerlat_variables {
235 struct task_struct *kthread;
236 struct hrtimer timer;
237 u64 rel_period;
238 u64 abs_period;
239 bool tracing_thread;
240 u64 count;
241};
242
243DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
244
245/*
246 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
247 */
248static inline struct timerlat_variables *this_cpu_tmr_var(void)
249{
250 return this_cpu_ptr(&per_cpu_timerlat_var);
251}
252
253/*
254 * tlat_var_reset - Reset the values of the given timerlat_variables
255 */
256static inline void tlat_var_reset(void)
257{
258 struct timerlat_variables *tlat_var;
259 int cpu;
260 /*
261 * So far, all the values are initialized as 0, so
262 * zeroing the structure is perfect.
263 */
264 for_each_cpu(cpu, cpu_online_mask) {
265 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
266 memset(tlat_var, 0, sizeof(*tlat_var));
267 }
268}
269#else /* CONFIG_TIMERLAT_TRACER */
270#define tlat_var_reset() do {} while (0)
271#endif /* CONFIG_TIMERLAT_TRACER */
272
273/*
274 * osn_var_reset - Reset the values of the given osnoise_variables
275 */
276static inline void osn_var_reset(void)
277{
278 struct osnoise_variables *osn_var;
279 int cpu;
280
281 /*
282 * So far, all the values are initialized as 0, so
283 * zeroing the structure is perfect.
284 */
285 for_each_cpu(cpu, cpu_online_mask) {
286 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
287 memset(osn_var, 0, sizeof(*osn_var));
288 }
289}
290
291/*
292 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
293 */
294static inline void osn_var_reset_all(void)
295{
296 osn_var_reset();
297 tlat_var_reset();
298}
299
300/*
301 * Tells NMIs to call back to the osnoise tracer to record timestamps.
302 */
303bool trace_osnoise_callback_enabled;
304
305/*
306 * osnoise sample structure definition. Used to store the statistics of a
307 * sample run.
308 */
309struct osnoise_sample {
310 u64 runtime; /* runtime */
311 u64 noise; /* noise */
312 u64 max_sample; /* max single noise sample */
313 int hw_count; /* # HW (incl. hypervisor) interference */
314 int nmi_count; /* # NMIs during this sample */
315 int irq_count; /* # IRQs during this sample */
316 int softirq_count; /* # softirqs during this sample */
317 int thread_count; /* # threads during this sample */
318};
319
320#ifdef CONFIG_TIMERLAT_TRACER
321/*
322 * timerlat sample structure definition. Used to store the statistics of
323 * a sample run.
324 */
325struct timerlat_sample {
326 u64 timer_latency; /* timer_latency */
327 unsigned int seqnum; /* unique sequence */
328 int context; /* timer context */
329};
330#endif
331
332/*
333 * Protect the interface.
334 */
335struct mutex interface_lock;
336
337/*
338 * Tracer data.
339 */
340static struct osnoise_data {
341 u64 sample_period; /* total sampling period */
342 u64 sample_runtime; /* active sampling portion of period */
343 u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
344 u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
345#ifdef CONFIG_TIMERLAT_TRACER
346 u64 timerlat_period; /* timerlat period */
347 u64 print_stack; /* print IRQ stack if total > */
348 int timerlat_tracer; /* timerlat tracer */
349#endif
350 bool tainted; /* infor users and developers about a problem */
351} osnoise_data = {
352 .sample_period = DEFAULT_SAMPLE_PERIOD,
353 .sample_runtime = DEFAULT_SAMPLE_RUNTIME,
354 .stop_tracing = 0,
355 .stop_tracing_total = 0,
356#ifdef CONFIG_TIMERLAT_TRACER
357 .print_stack = 0,
358 .timerlat_period = DEFAULT_TIMERLAT_PERIOD,
359 .timerlat_tracer = 0,
360#endif
361};
362
363#ifdef CONFIG_TIMERLAT_TRACER
364static inline bool timerlat_enabled(void)
365{
366 return osnoise_data.timerlat_tracer;
367}
368
369static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
370{
371 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
372 /*
373 * If the timerlat is enabled, but the irq handler did
374 * not run yet enabling timerlat_tracer, do not trace.
375 */
376 if (!tlat_var->tracing_thread) {
377 osn_var->softirq.arrival_time = 0;
378 osn_var->softirq.delta_start = 0;
379 return 0;
380 }
381 return 1;
382}
383
384static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
385{
386 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
387 /*
388 * If the timerlat is enabled, but the irq handler did
389 * not run yet enabling timerlat_tracer, do not trace.
390 */
391 if (!tlat_var->tracing_thread) {
392 osn_var->thread.delta_start = 0;
393 osn_var->thread.arrival_time = 0;
394 return 0;
395 }
396 return 1;
397}
398#else /* CONFIG_TIMERLAT_TRACER */
399static inline bool timerlat_enabled(void)
400{
401 return false;
402}
403
404static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
405{
406 return 1;
407}
408static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
409{
410 return 1;
411}
412#endif
413
414#ifdef CONFIG_PREEMPT_RT
415/*
416 * Print the osnoise header info.
417 */
418static void print_osnoise_headers(struct seq_file *s)
419{
420 if (osnoise_data.tainted)
421 seq_puts(s, "# osnoise is tainted!\n");
422
423 seq_puts(s, "# _-------=> irqs-off\n");
424 seq_puts(s, "# / _------=> need-resched\n");
425 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
426 seq_puts(s, "# || / _----=> hardirq/softirq\n");
427 seq_puts(s, "# ||| / _---=> preempt-depth\n");
428 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
429 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
430
431 seq_puts(s, "# |||||| / ");
432 seq_puts(s, " MAX\n");
433
434 seq_puts(s, "# ||||| / ");
435 seq_puts(s, " SINGLE Interference counters:\n");
436
437 seq_puts(s, "# ||||||| RUNTIME ");
438 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
439
440 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
441 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
442
443 seq_puts(s, "# | | | ||||||| | | ");
444 seq_puts(s, " | | | | | | | |\n");
445}
446#else /* CONFIG_PREEMPT_RT */
447static void print_osnoise_headers(struct seq_file *s)
448{
449 if (osnoise_data.tainted)
450 seq_puts(s, "# osnoise is tainted!\n");
451
452 seq_puts(s, "# _-----=> irqs-off\n");
453 seq_puts(s, "# / _----=> need-resched\n");
454 seq_puts(s, "# | / _---=> hardirq/softirq\n");
455 seq_puts(s, "# || / _--=> preempt-depth\n");
456 seq_puts(s, "# ||| / _-=> migrate-disable ");
457 seq_puts(s, " MAX\n");
458 seq_puts(s, "# |||| / delay ");
459 seq_puts(s, " SINGLE Interference counters:\n");
460
461 seq_puts(s, "# ||||| RUNTIME ");
462 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
463
464 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
465 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
466
467 seq_puts(s, "# | | | ||||| | | ");
468 seq_puts(s, " | | | | | | | |\n");
469}
470#endif /* CONFIG_PREEMPT_RT */
471
472/*
473 * osnoise_taint - report an osnoise error.
474 */
475#define osnoise_taint(msg) ({ \
476 struct osnoise_instance *inst; \
477 struct trace_buffer *buffer; \
478 \
479 rcu_read_lock(); \
480 list_for_each_entry_rcu(inst, &osnoise_instances, list) { \
481 buffer = inst->tr->array_buffer.buffer; \
482 trace_array_printk_buf(buffer, _THIS_IP_, msg); \
483 } \
484 rcu_read_unlock(); \
485 osnoise_data.tainted = true; \
486})
487
488/*
489 * Record an osnoise_sample into the tracer buffer.
490 */
491static void
492__trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer)
493{
494 struct trace_event_call *call = &event_osnoise;
495 struct ring_buffer_event *event;
496 struct osnoise_entry *entry;
497
498 event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
499 tracing_gen_ctx());
500 if (!event)
501 return;
502 entry = ring_buffer_event_data(event);
503 entry->runtime = sample->runtime;
504 entry->noise = sample->noise;
505 entry->max_sample = sample->max_sample;
506 entry->hw_count = sample->hw_count;
507 entry->nmi_count = sample->nmi_count;
508 entry->irq_count = sample->irq_count;
509 entry->softirq_count = sample->softirq_count;
510 entry->thread_count = sample->thread_count;
511
512 if (!call_filter_check_discard(call, entry, buffer, event))
513 trace_buffer_unlock_commit_nostack(buffer, event);
514}
515
516/*
517 * Record an osnoise_sample on all osnoise instances.
518 */
519static void trace_osnoise_sample(struct osnoise_sample *sample)
520{
521 struct osnoise_instance *inst;
522 struct trace_buffer *buffer;
523
524 rcu_read_lock();
525 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
526 buffer = inst->tr->array_buffer.buffer;
527 __trace_osnoise_sample(sample, buffer);
528 }
529 rcu_read_unlock();
530}
531
532#ifdef CONFIG_TIMERLAT_TRACER
533/*
534 * Print the timerlat header info.
535 */
536#ifdef CONFIG_PREEMPT_RT
537static void print_timerlat_headers(struct seq_file *s)
538{
539 seq_puts(s, "# _-------=> irqs-off\n");
540 seq_puts(s, "# / _------=> need-resched\n");
541 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
542 seq_puts(s, "# || / _----=> hardirq/softirq\n");
543 seq_puts(s, "# ||| / _---=> preempt-depth\n");
544 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
545 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
546 seq_puts(s, "# |||||| /\n");
547 seq_puts(s, "# ||||||| ACTIVATION\n");
548 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
549 seq_puts(s, " CONTEXT LATENCY\n");
550 seq_puts(s, "# | | | ||||||| | | ");
551 seq_puts(s, " | |\n");
552}
553#else /* CONFIG_PREEMPT_RT */
554static void print_timerlat_headers(struct seq_file *s)
555{
556 seq_puts(s, "# _-----=> irqs-off\n");
557 seq_puts(s, "# / _----=> need-resched\n");
558 seq_puts(s, "# | / _---=> hardirq/softirq\n");
559 seq_puts(s, "# || / _--=> preempt-depth\n");
560 seq_puts(s, "# ||| / _-=> migrate-disable\n");
561 seq_puts(s, "# |||| / delay\n");
562 seq_puts(s, "# ||||| ACTIVATION\n");
563 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
564 seq_puts(s, " CONTEXT LATENCY\n");
565 seq_puts(s, "# | | | ||||| | | ");
566 seq_puts(s, " | |\n");
567}
568#endif /* CONFIG_PREEMPT_RT */
569
570static void
571__trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer)
572{
573 struct trace_event_call *call = &event_osnoise;
574 struct ring_buffer_event *event;
575 struct timerlat_entry *entry;
576
577 event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
578 tracing_gen_ctx());
579 if (!event)
580 return;
581 entry = ring_buffer_event_data(event);
582 entry->seqnum = sample->seqnum;
583 entry->context = sample->context;
584 entry->timer_latency = sample->timer_latency;
585
586 if (!call_filter_check_discard(call, entry, buffer, event))
587 trace_buffer_unlock_commit_nostack(buffer, event);
588}
589
590/*
591 * Record an timerlat_sample into the tracer buffer.
592 */
593static void trace_timerlat_sample(struct timerlat_sample *sample)
594{
595 struct osnoise_instance *inst;
596 struct trace_buffer *buffer;
597
598 rcu_read_lock();
599 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
600 buffer = inst->tr->array_buffer.buffer;
601 __trace_timerlat_sample(sample, buffer);
602 }
603 rcu_read_unlock();
604}
605
606#ifdef CONFIG_STACKTRACE
607
608#define MAX_CALLS 256
609
610/*
611 * Stack trace will take place only at IRQ level, so, no need
612 * to control nesting here.
613 */
614struct trace_stack {
615 int stack_size;
616 int nr_entries;
617 unsigned long calls[MAX_CALLS];
618};
619
620static DEFINE_PER_CPU(struct trace_stack, trace_stack);
621
622/*
623 * timerlat_save_stack - save a stack trace without printing
624 *
625 * Save the current stack trace without printing. The
626 * stack will be printed later, after the end of the measurement.
627 */
628static void timerlat_save_stack(int skip)
629{
630 unsigned int size, nr_entries;
631 struct trace_stack *fstack;
632
633 fstack = this_cpu_ptr(&trace_stack);
634
635 size = ARRAY_SIZE(fstack->calls);
636
637 nr_entries = stack_trace_save(fstack->calls, size, skip);
638
639 fstack->stack_size = nr_entries * sizeof(unsigned long);
640 fstack->nr_entries = nr_entries;
641
642 return;
643
644}
645
646static void
647__timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size)
648{
649 struct trace_event_call *call = &event_osnoise;
650 struct ring_buffer_event *event;
651 struct stack_entry *entry;
652
653 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
654 tracing_gen_ctx());
655 if (!event)
656 return;
657
658 entry = ring_buffer_event_data(event);
659
660 memcpy(&entry->caller, fstack->calls, size);
661 entry->size = fstack->nr_entries;
662
663 if (!call_filter_check_discard(call, entry, buffer, event))
664 trace_buffer_unlock_commit_nostack(buffer, event);
665}
666
667/*
668 * timerlat_dump_stack - dump a stack trace previously saved
669 */
670static void timerlat_dump_stack(u64 latency)
671{
672 struct osnoise_instance *inst;
673 struct trace_buffer *buffer;
674 struct trace_stack *fstack;
675 unsigned int size;
676
677 /*
678 * trace only if latency > print_stack config, if enabled.
679 */
680 if (!osnoise_data.print_stack || osnoise_data.print_stack > latency)
681 return;
682
683 preempt_disable_notrace();
684 fstack = this_cpu_ptr(&trace_stack);
685 size = fstack->stack_size;
686
687 rcu_read_lock();
688 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
689 buffer = inst->tr->array_buffer.buffer;
690 __timerlat_dump_stack(buffer, fstack, size);
691
692 }
693 rcu_read_unlock();
694 preempt_enable_notrace();
695}
696#else /* CONFIG_STACKTRACE */
697#define timerlat_dump_stack(u64 latency) do {} while (0)
698#define timerlat_save_stack(a) do {} while (0)
699#endif /* CONFIG_STACKTRACE */
700#endif /* CONFIG_TIMERLAT_TRACER */
701
702/*
703 * Macros to encapsulate the time capturing infrastructure.
704 */
705#define time_get() trace_clock_local()
706#define time_to_us(x) div_u64(x, 1000)
707#define time_sub(a, b) ((a) - (b))
708
709/*
710 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
711 *
712 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
713 * to discount the NMI interference.
714 *
715 * See get_int_safe_duration().
716 */
717static inline void
718cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
719{
720 if (osn_var->irq.delta_start)
721 osn_var->irq.delta_start += duration;
722}
723
724#ifndef CONFIG_PREEMPT_RT
725/*
726 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
727 *
728 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
729 * forward to discount the interference.
730 *
731 * See get_int_safe_duration().
732 */
733static inline void
734cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
735{
736 if (osn_var->softirq.delta_start)
737 osn_var->softirq.delta_start += duration;
738}
739#else /* CONFIG_PREEMPT_RT */
740#define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
741#endif
742
743/*
744 * cond_move_thread_delta_start - Forward the delta_start of a running thread
745 *
746 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
747 * is pushed forward to discount the interference.
748 *
749 * See get_int_safe_duration().
750 */
751static inline void
752cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
753{
754 if (osn_var->thread.delta_start)
755 osn_var->thread.delta_start += duration;
756}
757
758/*
759 * get_int_safe_duration - Get the duration of a window
760 *
761 * The irq, softirq and thread varaibles need to have its duration without
762 * the interference from higher priority interrupts. Instead of keeping a
763 * variable to discount the interrupt interference from these variables, the
764 * starting time of these variables are pushed forward with the interrupt's
765 * duration. In this way, a single variable is used to:
766 *
767 * - Know if a given window is being measured.
768 * - Account its duration.
769 * - Discount the interference.
770 *
771 * To avoid getting inconsistent values, e.g.,:
772 *
773 * now = time_get()
774 * ---> interrupt!
775 * delta_start -= int duration;
776 * <---
777 * duration = now - delta_start;
778 *
779 * result: negative duration if the variable duration before the
780 * interrupt was smaller than the interrupt execution.
781 *
782 * A counter of interrupts is used. If the counter increased, try
783 * to capture an interference safe duration.
784 */
785static inline s64
786get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
787{
788 u64 int_counter, now;
789 s64 duration;
790
791 do {
792 int_counter = local_read(&osn_var->int_counter);
793 /* synchronize with interrupts */
794 barrier();
795
796 now = time_get();
797 duration = (now - *delta_start);
798
799 /* synchronize with interrupts */
800 barrier();
801 } while (int_counter != local_read(&osn_var->int_counter));
802
803 /*
804 * This is an evidence of race conditions that cause
805 * a value to be "discounted" too much.
806 */
807 if (duration < 0)
808 osnoise_taint("Negative duration!\n");
809
810 *delta_start = 0;
811
812 return duration;
813}
814
815/*
816 *
817 * set_int_safe_time - Save the current time on *time, aware of interference
818 *
819 * Get the time, taking into consideration a possible interference from
820 * higher priority interrupts.
821 *
822 * See get_int_safe_duration() for an explanation.
823 */
824static u64
825set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
826{
827 u64 int_counter;
828
829 do {
830 int_counter = local_read(&osn_var->int_counter);
831 /* synchronize with interrupts */
832 barrier();
833
834 *time = time_get();
835
836 /* synchronize with interrupts */
837 barrier();
838 } while (int_counter != local_read(&osn_var->int_counter));
839
840 return int_counter;
841}
842
843#ifdef CONFIG_TIMERLAT_TRACER
844/*
845 * copy_int_safe_time - Copy *src into *desc aware of interference
846 */
847static u64
848copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
849{
850 u64 int_counter;
851
852 do {
853 int_counter = local_read(&osn_var->int_counter);
854 /* synchronize with interrupts */
855 barrier();
856
857 *dst = *src;
858
859 /* synchronize with interrupts */
860 barrier();
861 } while (int_counter != local_read(&osn_var->int_counter));
862
863 return int_counter;
864}
865#endif /* CONFIG_TIMERLAT_TRACER */
866
867/*
868 * trace_osnoise_callback - NMI entry/exit callback
869 *
870 * This function is called at the entry and exit NMI code. The bool enter
871 * distinguishes between either case. This function is used to note a NMI
872 * occurrence, compute the noise caused by the NMI, and to remove the noise
873 * it is potentially causing on other interference variables.
874 */
875void trace_osnoise_callback(bool enter)
876{
877 struct osnoise_variables *osn_var = this_cpu_osn_var();
878 u64 duration;
879
880 if (!osn_var->sampling)
881 return;
882
883 /*
884 * Currently trace_clock_local() calls sched_clock() and the
885 * generic version is not NMI safe.
886 */
887 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
888 if (enter) {
889 osn_var->nmi.delta_start = time_get();
890 local_inc(&osn_var->int_counter);
891 } else {
892 duration = time_get() - osn_var->nmi.delta_start;
893
894 trace_nmi_noise(osn_var->nmi.delta_start, duration);
895
896 cond_move_irq_delta_start(osn_var, duration);
897 cond_move_softirq_delta_start(osn_var, duration);
898 cond_move_thread_delta_start(osn_var, duration);
899 }
900 }
901
902 if (enter)
903 osn_var->nmi.count++;
904}
905
906/*
907 * osnoise_trace_irq_entry - Note the starting of an IRQ
908 *
909 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
910 * it is safe to use a single variable (ons_var->irq) to save the statistics.
911 * The arrival_time is used to report... the arrival time. The delta_start
912 * is used to compute the duration at the IRQ exit handler. See
913 * cond_move_irq_delta_start().
914 */
915void osnoise_trace_irq_entry(int id)
916{
917 struct osnoise_variables *osn_var = this_cpu_osn_var();
918
919 if (!osn_var->sampling)
920 return;
921 /*
922 * This value will be used in the report, but not to compute
923 * the execution time, so it is safe to get it unsafe.
924 */
925 osn_var->irq.arrival_time = time_get();
926 set_int_safe_time(osn_var, &osn_var->irq.delta_start);
927 osn_var->irq.count++;
928
929 local_inc(&osn_var->int_counter);
930}
931
932/*
933 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
934 *
935 * Computes the duration of the IRQ noise, and trace it. Also discounts the
936 * interference from other sources of noise could be currently being accounted.
937 */
938void osnoise_trace_irq_exit(int id, const char *desc)
939{
940 struct osnoise_variables *osn_var = this_cpu_osn_var();
941 s64 duration;
942
943 if (!osn_var->sampling)
944 return;
945
946 duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
947 trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
948 osn_var->irq.arrival_time = 0;
949 cond_move_softirq_delta_start(osn_var, duration);
950 cond_move_thread_delta_start(osn_var, duration);
951}
952
953/*
954 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
955 *
956 * Used to note the starting of an IRQ occurece.
957 */
958static void trace_irqentry_callback(void *data, int irq,
959 struct irqaction *action)
960{
961 osnoise_trace_irq_entry(irq);
962}
963
964/*
965 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
966 *
967 * Used to note the end of an IRQ occurece.
968 */
969static void trace_irqexit_callback(void *data, int irq,
970 struct irqaction *action, int ret)
971{
972 osnoise_trace_irq_exit(irq, action->name);
973}
974
975/*
976 * arch specific register function.
977 */
978int __weak osnoise_arch_register(void)
979{
980 return 0;
981}
982
983/*
984 * arch specific unregister function.
985 */
986void __weak osnoise_arch_unregister(void)
987{
988 return;
989}
990
991/*
992 * hook_irq_events - Hook IRQ handling events
993 *
994 * This function hooks the IRQ related callbacks to the respective trace
995 * events.
996 */
997static int hook_irq_events(void)
998{
999 int ret;
1000
1001 ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1002 if (ret)
1003 goto out_err;
1004
1005 ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1006 if (ret)
1007 goto out_unregister_entry;
1008
1009 ret = osnoise_arch_register();
1010 if (ret)
1011 goto out_irq_exit;
1012
1013 return 0;
1014
1015out_irq_exit:
1016 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1017out_unregister_entry:
1018 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1019out_err:
1020 return -EINVAL;
1021}
1022
1023/*
1024 * unhook_irq_events - Unhook IRQ handling events
1025 *
1026 * This function unhooks the IRQ related callbacks to the respective trace
1027 * events.
1028 */
1029static void unhook_irq_events(void)
1030{
1031 osnoise_arch_unregister();
1032 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1033 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1034}
1035
1036#ifndef CONFIG_PREEMPT_RT
1037/*
1038 * trace_softirq_entry_callback - Note the starting of a softirq
1039 *
1040 * Save the starting time of a softirq. As softirqs are non-preemptive to
1041 * other softirqs, it is safe to use a single variable (ons_var->softirq)
1042 * to save the statistics. The arrival_time is used to report... the
1043 * arrival time. The delta_start is used to compute the duration at the
1044 * softirq exit handler. See cond_move_softirq_delta_start().
1045 */
1046static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1047{
1048 struct osnoise_variables *osn_var = this_cpu_osn_var();
1049
1050 if (!osn_var->sampling)
1051 return;
1052 /*
1053 * This value will be used in the report, but not to compute
1054 * the execution time, so it is safe to get it unsafe.
1055 */
1056 osn_var->softirq.arrival_time = time_get();
1057 set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1058 osn_var->softirq.count++;
1059
1060 local_inc(&osn_var->int_counter);
1061}
1062
1063/*
1064 * trace_softirq_exit_callback - Note the end of an softirq
1065 *
1066 * Computes the duration of the softirq noise, and trace it. Also discounts the
1067 * interference from other sources of noise could be currently being accounted.
1068 */
1069static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1070{
1071 struct osnoise_variables *osn_var = this_cpu_osn_var();
1072 s64 duration;
1073
1074 if (!osn_var->sampling)
1075 return;
1076
1077 if (unlikely(timerlat_enabled()))
1078 if (!timerlat_softirq_exit(osn_var))
1079 return;
1080
1081 duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1082 trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1083 cond_move_thread_delta_start(osn_var, duration);
1084 osn_var->softirq.arrival_time = 0;
1085}
1086
1087/*
1088 * hook_softirq_events - Hook softirq handling events
1089 *
1090 * This function hooks the softirq related callbacks to the respective trace
1091 * events.
1092 */
1093static int hook_softirq_events(void)
1094{
1095 int ret;
1096
1097 ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1098 if (ret)
1099 goto out_err;
1100
1101 ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1102 if (ret)
1103 goto out_unreg_entry;
1104
1105 return 0;
1106
1107out_unreg_entry:
1108 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1109out_err:
1110 return -EINVAL;
1111}
1112
1113/*
1114 * unhook_softirq_events - Unhook softirq handling events
1115 *
1116 * This function hooks the softirq related callbacks to the respective trace
1117 * events.
1118 */
1119static void unhook_softirq_events(void)
1120{
1121 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1122 unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1123}
1124#else /* CONFIG_PREEMPT_RT */
1125/*
1126 * softirq are threads on the PREEMPT_RT mode.
1127 */
1128static int hook_softirq_events(void)
1129{
1130 return 0;
1131}
1132static void unhook_softirq_events(void)
1133{
1134}
1135#endif
1136
1137/*
1138 * thread_entry - Record the starting of a thread noise window
1139 *
1140 * It saves the context switch time for a noisy thread, and increments
1141 * the interference counters.
1142 */
1143static void
1144thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
1145{
1146 if (!osn_var->sampling)
1147 return;
1148 /*
1149 * The arrival time will be used in the report, but not to compute
1150 * the execution time, so it is safe to get it unsafe.
1151 */
1152 osn_var->thread.arrival_time = time_get();
1153
1154 set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1155
1156 osn_var->thread.count++;
1157 local_inc(&osn_var->int_counter);
1158}
1159
1160/*
1161 * thread_exit - Report the end of a thread noise window
1162 *
1163 * It computes the total noise from a thread, tracing if needed.
1164 */
1165static void
1166thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
1167{
1168 s64 duration;
1169
1170 if (!osn_var->sampling)
1171 return;
1172
1173 if (unlikely(timerlat_enabled()))
1174 if (!timerlat_thread_exit(osn_var))
1175 return;
1176
1177 duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1178
1179 trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1180
1181 osn_var->thread.arrival_time = 0;
1182}
1183
1184/*
1185 * trace_sched_switch - sched:sched_switch trace event handler
1186 *
1187 * This function is hooked to the sched:sched_switch trace event, and it is
1188 * used to record the beginning and to report the end of a thread noise window.
1189 */
1190static void
1191trace_sched_switch_callback(void *data, bool preempt,
1192 struct task_struct *p,
1193 struct task_struct *n,
1194 unsigned int prev_state)
1195{
1196 struct osnoise_variables *osn_var = this_cpu_osn_var();
1197 int workload = test_bit(OSN_WORKLOAD, &osnoise_options);
1198
1199 if ((p->pid != osn_var->pid) || !workload)
1200 thread_exit(osn_var, p);
1201
1202 if ((n->pid != osn_var->pid) || !workload)
1203 thread_entry(osn_var, n);
1204}
1205
1206/*
1207 * hook_thread_events - Hook the insturmentation for thread noise
1208 *
1209 * Hook the osnoise tracer callbacks to handle the noise from other
1210 * threads on the necessary kernel events.
1211 */
1212static int hook_thread_events(void)
1213{
1214 int ret;
1215
1216 ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1217 if (ret)
1218 return -EINVAL;
1219
1220 return 0;
1221}
1222
1223/*
1224 * unhook_thread_events - *nhook the insturmentation for thread noise
1225 *
1226 * Unook the osnoise tracer callbacks to handle the noise from other
1227 * threads on the necessary kernel events.
1228 */
1229static void unhook_thread_events(void)
1230{
1231 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1232}
1233
1234/*
1235 * save_osn_sample_stats - Save the osnoise_sample statistics
1236 *
1237 * Save the osnoise_sample statistics before the sampling phase. These
1238 * values will be used later to compute the diff betwneen the statistics
1239 * before and after the osnoise sampling.
1240 */
1241static void
1242save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1243{
1244 s->nmi_count = osn_var->nmi.count;
1245 s->irq_count = osn_var->irq.count;
1246 s->softirq_count = osn_var->softirq.count;
1247 s->thread_count = osn_var->thread.count;
1248}
1249
1250/*
1251 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1252 *
1253 * After a sample period, compute the difference on the osnoise_sample
1254 * statistics. The struct osnoise_sample *s contains the statistics saved via
1255 * save_osn_sample_stats() before the osnoise sampling.
1256 */
1257static void
1258diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1259{
1260 s->nmi_count = osn_var->nmi.count - s->nmi_count;
1261 s->irq_count = osn_var->irq.count - s->irq_count;
1262 s->softirq_count = osn_var->softirq.count - s->softirq_count;
1263 s->thread_count = osn_var->thread.count - s->thread_count;
1264}
1265
1266/*
1267 * osnoise_stop_tracing - Stop tracing and the tracer.
1268 */
1269static __always_inline void osnoise_stop_tracing(void)
1270{
1271 struct osnoise_instance *inst;
1272 struct trace_array *tr;
1273
1274 rcu_read_lock();
1275 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1276 tr = inst->tr;
1277 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1278 "stop tracing hit on cpu %d\n", smp_processor_id());
1279
1280 if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1281 panic("tracer hit stop condition on CPU %d\n", smp_processor_id());
1282
1283 tracer_tracing_off(tr);
1284 }
1285 rcu_read_unlock();
1286}
1287
1288/*
1289 * notify_new_max_latency - Notify a new max latency via fsnotify interface.
1290 */
1291static void notify_new_max_latency(u64 latency)
1292{
1293 struct osnoise_instance *inst;
1294 struct trace_array *tr;
1295
1296 rcu_read_lock();
1297 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1298 tr = inst->tr;
1299 if (tr->max_latency < latency) {
1300 tr->max_latency = latency;
1301 latency_fsnotify(tr);
1302 }
1303 }
1304 rcu_read_unlock();
1305}
1306
1307/*
1308 * run_osnoise - Sample the time and look for osnoise
1309 *
1310 * Used to capture the time, looking for potential osnoise latency repeatedly.
1311 * Different from hwlat_detector, it is called with preemption and interrupts
1312 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1313 * osnoise sampling thread, as they would do with a regular thread.
1314 */
1315static int run_osnoise(void)
1316{
1317 bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options);
1318 struct osnoise_variables *osn_var = this_cpu_osn_var();
1319 u64 start, sample, last_sample;
1320 u64 last_int_count, int_count;
1321 s64 noise = 0, max_noise = 0;
1322 s64 total, last_total = 0;
1323 struct osnoise_sample s;
1324 bool disable_preemption;
1325 unsigned int threshold;
1326 u64 runtime, stop_in;
1327 u64 sum_noise = 0;
1328 int hw_count = 0;
1329 int ret = -1;
1330
1331 /*
1332 * Disabling preemption is only required if IRQs are enabled,
1333 * and the options is set on.
1334 */
1335 disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options);
1336
1337 /*
1338 * Considers the current thread as the workload.
1339 */
1340 osn_var->pid = current->pid;
1341
1342 /*
1343 * Save the current stats for the diff
1344 */
1345 save_osn_sample_stats(osn_var, &s);
1346
1347 /*
1348 * if threshold is 0, use the default value of 5 us.
1349 */
1350 threshold = tracing_thresh ? : 5000;
1351
1352 /*
1353 * Apply PREEMPT and IRQ disabled options.
1354 */
1355 if (disable_irq)
1356 local_irq_disable();
1357
1358 if (disable_preemption)
1359 preempt_disable();
1360
1361 /*
1362 * Make sure NMIs see sampling first
1363 */
1364 osn_var->sampling = true;
1365 barrier();
1366
1367 /*
1368 * Transform the *_us config to nanoseconds to avoid the
1369 * division on the main loop.
1370 */
1371 runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1372 stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1373
1374 /*
1375 * Start timestemp
1376 */
1377 start = time_get();
1378
1379 /*
1380 * "previous" loop.
1381 */
1382 last_int_count = set_int_safe_time(osn_var, &last_sample);
1383
1384 do {
1385 /*
1386 * Get sample!
1387 */
1388 int_count = set_int_safe_time(osn_var, &sample);
1389
1390 noise = time_sub(sample, last_sample);
1391
1392 /*
1393 * This shouldn't happen.
1394 */
1395 if (noise < 0) {
1396 osnoise_taint("negative noise!");
1397 goto out;
1398 }
1399
1400 /*
1401 * Sample runtime.
1402 */
1403 total = time_sub(sample, start);
1404
1405 /*
1406 * Check for possible overflows.
1407 */
1408 if (total < last_total) {
1409 osnoise_taint("total overflow!");
1410 break;
1411 }
1412
1413 last_total = total;
1414
1415 if (noise >= threshold) {
1416 int interference = int_count - last_int_count;
1417
1418 if (noise > max_noise)
1419 max_noise = noise;
1420
1421 if (!interference)
1422 hw_count++;
1423
1424 sum_noise += noise;
1425
1426 trace_sample_threshold(last_sample, noise, interference);
1427
1428 if (osnoise_data.stop_tracing)
1429 if (noise > stop_in)
1430 osnoise_stop_tracing();
1431 }
1432
1433 /*
1434 * In some cases, notably when running on a nohz_full CPU with
1435 * a stopped tick PREEMPT_RCU has no way to account for QSs.
1436 * This will eventually cause unwarranted noise as PREEMPT_RCU
1437 * will force preemption as the means of ending the current
1438 * grace period. We avoid this problem by calling
1439 * rcu_momentary_dyntick_idle(), which performs a zero duration
1440 * EQS allowing PREEMPT_RCU to end the current grace period.
1441 * This call shouldn't be wrapped inside an RCU critical
1442 * section.
1443 *
1444 * Note that in non PREEMPT_RCU kernels QSs are handled through
1445 * cond_resched()
1446 */
1447 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
1448 if (!disable_irq)
1449 local_irq_disable();
1450
1451 rcu_momentary_dyntick_idle();
1452
1453 if (!disable_irq)
1454 local_irq_enable();
1455 }
1456
1457 /*
1458 * For the non-preemptive kernel config: let threads runs, if
1459 * they so wish, unless set not do to so.
1460 */
1461 if (!disable_irq && !disable_preemption)
1462 cond_resched();
1463
1464 last_sample = sample;
1465 last_int_count = int_count;
1466
1467 } while (total < runtime && !kthread_should_stop());
1468
1469 /*
1470 * Finish the above in the view for interrupts.
1471 */
1472 barrier();
1473
1474 osn_var->sampling = false;
1475
1476 /*
1477 * Make sure sampling data is no longer updated.
1478 */
1479 barrier();
1480
1481 /*
1482 * Return to the preemptive state.
1483 */
1484 if (disable_preemption)
1485 preempt_enable();
1486
1487 if (disable_irq)
1488 local_irq_enable();
1489
1490 /*
1491 * Save noise info.
1492 */
1493 s.noise = time_to_us(sum_noise);
1494 s.runtime = time_to_us(total);
1495 s.max_sample = time_to_us(max_noise);
1496 s.hw_count = hw_count;
1497
1498 /* Save interference stats info */
1499 diff_osn_sample_stats(osn_var, &s);
1500
1501 trace_osnoise_sample(&s);
1502
1503 notify_new_max_latency(max_noise);
1504
1505 if (osnoise_data.stop_tracing_total)
1506 if (s.noise > osnoise_data.stop_tracing_total)
1507 osnoise_stop_tracing();
1508
1509 return 0;
1510out:
1511 return ret;
1512}
1513
1514static struct cpumask osnoise_cpumask;
1515static struct cpumask save_cpumask;
1516
1517/*
1518 * osnoise_sleep - sleep until the next period
1519 */
1520static void osnoise_sleep(void)
1521{
1522 u64 interval;
1523 ktime_t wake_time;
1524
1525 mutex_lock(&interface_lock);
1526 interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1527 mutex_unlock(&interface_lock);
1528
1529 /*
1530 * differently from hwlat_detector, the osnoise tracer can run
1531 * without a pause because preemption is on.
1532 */
1533 if (!interval) {
1534 /* Let synchronize_rcu_tasks() make progress */
1535 cond_resched_tasks_rcu_qs();
1536 return;
1537 }
1538
1539 wake_time = ktime_add_us(ktime_get(), interval);
1540 __set_current_state(TASK_INTERRUPTIBLE);
1541
1542 while (schedule_hrtimeout_range(&wake_time, 0, HRTIMER_MODE_ABS)) {
1543 if (kthread_should_stop())
1544 break;
1545 }
1546}
1547
1548/*
1549 * osnoise_main - The osnoise detection kernel thread
1550 *
1551 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1552 * every period.
1553 */
1554static int osnoise_main(void *data)
1555{
1556
1557 while (!kthread_should_stop()) {
1558 run_osnoise();
1559 osnoise_sleep();
1560 }
1561
1562 return 0;
1563}
1564
1565#ifdef CONFIG_TIMERLAT_TRACER
1566/*
1567 * timerlat_irq - hrtimer handler for timerlat.
1568 */
1569static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1570{
1571 struct osnoise_variables *osn_var = this_cpu_osn_var();
1572 struct timerlat_variables *tlat;
1573 struct timerlat_sample s;
1574 u64 now;
1575 u64 diff;
1576
1577 /*
1578 * I am not sure if the timer was armed for this CPU. So, get
1579 * the timerlat struct from the timer itself, not from this
1580 * CPU.
1581 */
1582 tlat = container_of(timer, struct timerlat_variables, timer);
1583
1584 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1585
1586 /*
1587 * Enable the osnoise: events for thread an softirq.
1588 */
1589 tlat->tracing_thread = true;
1590
1591 osn_var->thread.arrival_time = time_get();
1592
1593 /*
1594 * A hardirq is running: the timer IRQ. It is for sure preempting
1595 * a thread, and potentially preempting a softirq.
1596 *
1597 * At this point, it is not interesting to know the duration of the
1598 * preempted thread (and maybe softirq), but how much time they will
1599 * delay the beginning of the execution of the timer thread.
1600 *
1601 * To get the correct (net) delay added by the softirq, its delta_start
1602 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1603 * start of the sofitrq will be zeroed, accounting then only the time
1604 * after that.
1605 *
1606 * The thread follows the same principle. However, if a softirq is
1607 * running, the thread needs to receive the softirq delta_start. The
1608 * reason being is that the softirq will be the last to be unfolded,
1609 * resseting the thread delay to zero.
1610 *
1611 * The PREEMPT_RT is a special case, though. As softirqs run as threads
1612 * on RT, moving the thread is enough.
1613 */
1614 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1615 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1616 &osn_var->softirq.delta_start);
1617
1618 copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1619 &osn_var->irq.delta_start);
1620 } else {
1621 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1622 &osn_var->irq.delta_start);
1623 }
1624
1625 /*
1626 * Compute the current time with the expected time.
1627 */
1628 diff = now - tlat->abs_period;
1629
1630 tlat->count++;
1631 s.seqnum = tlat->count;
1632 s.timer_latency = diff;
1633 s.context = IRQ_CONTEXT;
1634
1635 trace_timerlat_sample(&s);
1636
1637 if (osnoise_data.stop_tracing) {
1638 if (time_to_us(diff) >= osnoise_data.stop_tracing) {
1639
1640 /*
1641 * At this point, if stop_tracing is set and <= print_stack,
1642 * print_stack is set and would be printed in the thread handler.
1643 *
1644 * Thus, print the stack trace as it is helpful to define the
1645 * root cause of an IRQ latency.
1646 */
1647 if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
1648 timerlat_save_stack(0);
1649 timerlat_dump_stack(time_to_us(diff));
1650 }
1651
1652 osnoise_stop_tracing();
1653 notify_new_max_latency(diff);
1654
1655 return HRTIMER_NORESTART;
1656 }
1657 }
1658
1659 wake_up_process(tlat->kthread);
1660
1661 if (osnoise_data.print_stack)
1662 timerlat_save_stack(0);
1663
1664 return HRTIMER_NORESTART;
1665}
1666
1667/*
1668 * wait_next_period - Wait for the next period for timerlat
1669 */
1670static int wait_next_period(struct timerlat_variables *tlat)
1671{
1672 ktime_t next_abs_period, now;
1673 u64 rel_period = osnoise_data.timerlat_period * 1000;
1674
1675 now = hrtimer_cb_get_time(&tlat->timer);
1676 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1677
1678 /*
1679 * Save the next abs_period.
1680 */
1681 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1682
1683 /*
1684 * If the new abs_period is in the past, skip the activation.
1685 */
1686 while (ktime_compare(now, next_abs_period) > 0) {
1687 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1688 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1689 }
1690
1691 set_current_state(TASK_INTERRUPTIBLE);
1692
1693 hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1694 schedule();
1695 return 1;
1696}
1697
1698/*
1699 * timerlat_main- Timerlat main
1700 */
1701static int timerlat_main(void *data)
1702{
1703 struct osnoise_variables *osn_var = this_cpu_osn_var();
1704 struct timerlat_variables *tlat = this_cpu_tmr_var();
1705 struct timerlat_sample s;
1706 struct sched_param sp;
1707 u64 now, diff;
1708
1709 /*
1710 * Make the thread RT, that is how cyclictest is usually used.
1711 */
1712 sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1713 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1714
1715 tlat->count = 0;
1716 tlat->tracing_thread = false;
1717
1718 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1719 tlat->timer.function = timerlat_irq;
1720 tlat->kthread = current;
1721 osn_var->pid = current->pid;
1722 /*
1723 * Anotate the arrival time.
1724 */
1725 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1726
1727 wait_next_period(tlat);
1728
1729 osn_var->sampling = 1;
1730
1731 while (!kthread_should_stop()) {
1732 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1733 diff = now - tlat->abs_period;
1734
1735 s.seqnum = tlat->count;
1736 s.timer_latency = diff;
1737 s.context = THREAD_CONTEXT;
1738
1739 trace_timerlat_sample(&s);
1740
1741 timerlat_dump_stack(time_to_us(diff));
1742
1743 tlat->tracing_thread = false;
1744 if (osnoise_data.stop_tracing_total)
1745 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1746 osnoise_stop_tracing();
1747
1748 wait_next_period(tlat);
1749 }
1750
1751 hrtimer_cancel(&tlat->timer);
1752 return 0;
1753}
1754#else /* CONFIG_TIMERLAT_TRACER */
1755static int timerlat_main(void *data)
1756{
1757 return 0;
1758}
1759#endif /* CONFIG_TIMERLAT_TRACER */
1760
1761/*
1762 * stop_kthread - stop a workload thread
1763 */
1764static void stop_kthread(unsigned int cpu)
1765{
1766 struct task_struct *kthread;
1767
1768 kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread;
1769 if (kthread) {
1770 kthread_stop(kthread);
1771 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1772 } else {
1773 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1774 per_cpu(per_cpu_osnoise_var, cpu).sampling = false;
1775 barrier();
1776 return;
1777 }
1778 }
1779}
1780
1781/*
1782 * stop_per_cpu_kthread - Stop per-cpu threads
1783 *
1784 * Stop the osnoise sampling htread. Use this on unload and at system
1785 * shutdown.
1786 */
1787static void stop_per_cpu_kthreads(void)
1788{
1789 int cpu;
1790
1791 cpus_read_lock();
1792
1793 for_each_online_cpu(cpu)
1794 stop_kthread(cpu);
1795
1796 cpus_read_unlock();
1797}
1798
1799/*
1800 * start_kthread - Start a workload tread
1801 */
1802static int start_kthread(unsigned int cpu)
1803{
1804 struct task_struct *kthread;
1805 void *main = osnoise_main;
1806 char comm[24];
1807
1808 if (timerlat_enabled()) {
1809 snprintf(comm, 24, "timerlat/%d", cpu);
1810 main = timerlat_main;
1811 } else {
1812 /* if no workload, just return */
1813 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1814 per_cpu(per_cpu_osnoise_var, cpu).sampling = true;
1815 barrier();
1816 return 0;
1817 }
1818
1819 snprintf(comm, 24, "osnoise/%d", cpu);
1820 }
1821
1822 kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
1823
1824 if (IS_ERR(kthread)) {
1825 pr_err(BANNER "could not start sampling thread\n");
1826 stop_per_cpu_kthreads();
1827 return -ENOMEM;
1828 }
1829
1830 per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
1831
1832 return 0;
1833}
1834
1835/*
1836 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
1837 *
1838 * This starts the kernel thread that will look for osnoise on many
1839 * cpus.
1840 */
1841static int start_per_cpu_kthreads(void)
1842{
1843 struct cpumask *current_mask = &save_cpumask;
1844 int retval = 0;
1845 int cpu;
1846
1847 cpus_read_lock();
1848 /*
1849 * Run only on online CPUs in which osnoise is allowed to run.
1850 */
1851 cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
1852
1853 for_each_possible_cpu(cpu)
1854 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1855
1856 for_each_cpu(cpu, current_mask) {
1857 retval = start_kthread(cpu);
1858 if (retval) {
1859 cpus_read_unlock();
1860 stop_per_cpu_kthreads();
1861 return retval;
1862 }
1863 }
1864
1865 cpus_read_unlock();
1866
1867 return retval;
1868}
1869
1870#ifdef CONFIG_HOTPLUG_CPU
1871static void osnoise_hotplug_workfn(struct work_struct *dummy)
1872{
1873 unsigned int cpu = smp_processor_id();
1874
1875 mutex_lock(&trace_types_lock);
1876
1877 if (!osnoise_has_registered_instances())
1878 goto out_unlock_trace;
1879
1880 mutex_lock(&interface_lock);
1881 cpus_read_lock();
1882
1883 if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
1884 goto out_unlock;
1885
1886 start_kthread(cpu);
1887
1888out_unlock:
1889 cpus_read_unlock();
1890 mutex_unlock(&interface_lock);
1891out_unlock_trace:
1892 mutex_unlock(&trace_types_lock);
1893}
1894
1895static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
1896
1897/*
1898 * osnoise_cpu_init - CPU hotplug online callback function
1899 */
1900static int osnoise_cpu_init(unsigned int cpu)
1901{
1902 schedule_work_on(cpu, &osnoise_hotplug_work);
1903 return 0;
1904}
1905
1906/*
1907 * osnoise_cpu_die - CPU hotplug offline callback function
1908 */
1909static int osnoise_cpu_die(unsigned int cpu)
1910{
1911 stop_kthread(cpu);
1912 return 0;
1913}
1914
1915static void osnoise_init_hotplug_support(void)
1916{
1917 int ret;
1918
1919 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
1920 osnoise_cpu_init, osnoise_cpu_die);
1921 if (ret < 0)
1922 pr_warn(BANNER "Error to init cpu hotplug support\n");
1923
1924 return;
1925}
1926#else /* CONFIG_HOTPLUG_CPU */
1927static void osnoise_init_hotplug_support(void)
1928{
1929 return;
1930}
1931#endif /* CONFIG_HOTPLUG_CPU */
1932
1933/*
1934 * seq file functions for the osnoise/options file.
1935 */
1936static void *s_options_start(struct seq_file *s, loff_t *pos)
1937{
1938 int option = *pos;
1939
1940 mutex_lock(&interface_lock);
1941
1942 if (option >= OSN_MAX)
1943 return NULL;
1944
1945 return pos;
1946}
1947
1948static void *s_options_next(struct seq_file *s, void *v, loff_t *pos)
1949{
1950 int option = ++(*pos);
1951
1952 if (option >= OSN_MAX)
1953 return NULL;
1954
1955 return pos;
1956}
1957
1958static int s_options_show(struct seq_file *s, void *v)
1959{
1960 loff_t *pos = v;
1961 int option = *pos;
1962
1963 if (option == OSN_DEFAULTS) {
1964 if (osnoise_options == OSN_DEFAULT_OPTIONS)
1965 seq_printf(s, "%s", osnoise_options_str[option]);
1966 else
1967 seq_printf(s, "NO_%s", osnoise_options_str[option]);
1968 goto out;
1969 }
1970
1971 if (test_bit(option, &osnoise_options))
1972 seq_printf(s, "%s", osnoise_options_str[option]);
1973 else
1974 seq_printf(s, "NO_%s", osnoise_options_str[option]);
1975
1976out:
1977 if (option != OSN_MAX)
1978 seq_puts(s, " ");
1979
1980 return 0;
1981}
1982
1983static void s_options_stop(struct seq_file *s, void *v)
1984{
1985 seq_puts(s, "\n");
1986 mutex_unlock(&interface_lock);
1987}
1988
1989static const struct seq_operations osnoise_options_seq_ops = {
1990 .start = s_options_start,
1991 .next = s_options_next,
1992 .show = s_options_show,
1993 .stop = s_options_stop
1994};
1995
1996static int osnoise_options_open(struct inode *inode, struct file *file)
1997{
1998 return seq_open(file, &osnoise_options_seq_ops);
1999};
2000
2001/**
2002 * osnoise_options_write - Write function for "options" entry
2003 * @filp: The active open file structure
2004 * @ubuf: The user buffer that contains the value to write
2005 * @cnt: The maximum number of bytes to write to "file"
2006 * @ppos: The current position in @file
2007 *
2008 * Writing the option name sets the option, writing the "NO_"
2009 * prefix in front of the option name disables it.
2010 *
2011 * Writing "DEFAULTS" resets the option values to the default ones.
2012 */
2013static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf,
2014 size_t cnt, loff_t *ppos)
2015{
2016 int running, option, enable, retval;
2017 char buf[256], *option_str;
2018
2019 if (cnt >= 256)
2020 return -EINVAL;
2021
2022 if (copy_from_user(buf, ubuf, cnt))
2023 return -EFAULT;
2024
2025 buf[cnt] = 0;
2026
2027 if (strncmp(buf, "NO_", 3)) {
2028 option_str = strstrip(buf);
2029 enable = true;
2030 } else {
2031 option_str = strstrip(&buf[3]);
2032 enable = false;
2033 }
2034
2035 option = match_string(osnoise_options_str, OSN_MAX, option_str);
2036 if (option < 0)
2037 return -EINVAL;
2038
2039 /*
2040 * trace_types_lock is taken to avoid concurrency on start/stop.
2041 */
2042 mutex_lock(&trace_types_lock);
2043 running = osnoise_has_registered_instances();
2044 if (running)
2045 stop_per_cpu_kthreads();
2046
2047 mutex_lock(&interface_lock);
2048 /*
2049 * avoid CPU hotplug operations that might read options.
2050 */
2051 cpus_read_lock();
2052
2053 retval = cnt;
2054
2055 if (enable) {
2056 if (option == OSN_DEFAULTS)
2057 osnoise_options = OSN_DEFAULT_OPTIONS;
2058 else
2059 set_bit(option, &osnoise_options);
2060 } else {
2061 if (option == OSN_DEFAULTS)
2062 retval = -EINVAL;
2063 else
2064 clear_bit(option, &osnoise_options);
2065 }
2066
2067 cpus_read_unlock();
2068 mutex_unlock(&interface_lock);
2069
2070 if (running)
2071 start_per_cpu_kthreads();
2072 mutex_unlock(&trace_types_lock);
2073
2074 return retval;
2075}
2076
2077/*
2078 * osnoise_cpus_read - Read function for reading the "cpus" file
2079 * @filp: The active open file structure
2080 * @ubuf: The userspace provided buffer to read value into
2081 * @cnt: The maximum number of bytes to read
2082 * @ppos: The current "file" position
2083 *
2084 * Prints the "cpus" output into the user-provided buffer.
2085 */
2086static ssize_t
2087osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
2088 loff_t *ppos)
2089{
2090 char *mask_str;
2091 int len;
2092
2093 mutex_lock(&interface_lock);
2094
2095 len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
2096 mask_str = kmalloc(len, GFP_KERNEL);
2097 if (!mask_str) {
2098 count = -ENOMEM;
2099 goto out_unlock;
2100 }
2101
2102 len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
2103 if (len >= count) {
2104 count = -EINVAL;
2105 goto out_free;
2106 }
2107
2108 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
2109
2110out_free:
2111 kfree(mask_str);
2112out_unlock:
2113 mutex_unlock(&interface_lock);
2114
2115 return count;
2116}
2117
2118/*
2119 * osnoise_cpus_write - Write function for "cpus" entry
2120 * @filp: The active open file structure
2121 * @ubuf: The user buffer that contains the value to write
2122 * @cnt: The maximum number of bytes to write to "file"
2123 * @ppos: The current position in @file
2124 *
2125 * This function provides a write implementation for the "cpus"
2126 * interface to the osnoise trace. By default, it lists all CPUs,
2127 * in this way, allowing osnoise threads to run on any online CPU
2128 * of the system. It serves to restrict the execution of osnoise to the
2129 * set of CPUs writing via this interface. Why not use "tracing_cpumask"?
2130 * Because the user might be interested in tracing what is running on
2131 * other CPUs. For instance, one might run osnoise in one HT CPU
2132 * while observing what is running on the sibling HT CPU.
2133 */
2134static ssize_t
2135osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
2136 loff_t *ppos)
2137{
2138 cpumask_var_t osnoise_cpumask_new;
2139 int running, err;
2140 char buf[256];
2141
2142 if (count >= 256)
2143 return -EINVAL;
2144
2145 if (copy_from_user(buf, ubuf, count))
2146 return -EFAULT;
2147
2148 if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
2149 return -ENOMEM;
2150
2151 err = cpulist_parse(buf, osnoise_cpumask_new);
2152 if (err)
2153 goto err_free;
2154
2155 /*
2156 * trace_types_lock is taken to avoid concurrency on start/stop.
2157 */
2158 mutex_lock(&trace_types_lock);
2159 running = osnoise_has_registered_instances();
2160 if (running)
2161 stop_per_cpu_kthreads();
2162
2163 mutex_lock(&interface_lock);
2164 /*
2165 * osnoise_cpumask is read by CPU hotplug operations.
2166 */
2167 cpus_read_lock();
2168
2169 cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
2170
2171 cpus_read_unlock();
2172 mutex_unlock(&interface_lock);
2173
2174 if (running)
2175 start_per_cpu_kthreads();
2176 mutex_unlock(&trace_types_lock);
2177
2178 free_cpumask_var(osnoise_cpumask_new);
2179 return count;
2180
2181err_free:
2182 free_cpumask_var(osnoise_cpumask_new);
2183
2184 return err;
2185}
2186
2187/*
2188 * osnoise/runtime_us: cannot be greater than the period.
2189 */
2190static struct trace_min_max_param osnoise_runtime = {
2191 .lock = &interface_lock,
2192 .val = &osnoise_data.sample_runtime,
2193 .max = &osnoise_data.sample_period,
2194 .min = NULL,
2195};
2196
2197/*
2198 * osnoise/period_us: cannot be smaller than the runtime.
2199 */
2200static struct trace_min_max_param osnoise_period = {
2201 .lock = &interface_lock,
2202 .val = &osnoise_data.sample_period,
2203 .max = NULL,
2204 .min = &osnoise_data.sample_runtime,
2205};
2206
2207/*
2208 * osnoise/stop_tracing_us: no limit.
2209 */
2210static struct trace_min_max_param osnoise_stop_tracing_in = {
2211 .lock = &interface_lock,
2212 .val = &osnoise_data.stop_tracing,
2213 .max = NULL,
2214 .min = NULL,
2215};
2216
2217/*
2218 * osnoise/stop_tracing_total_us: no limit.
2219 */
2220static struct trace_min_max_param osnoise_stop_tracing_total = {
2221 .lock = &interface_lock,
2222 .val = &osnoise_data.stop_tracing_total,
2223 .max = NULL,
2224 .min = NULL,
2225};
2226
2227#ifdef CONFIG_TIMERLAT_TRACER
2228/*
2229 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2230 * latency is higher than val.
2231 */
2232static struct trace_min_max_param osnoise_print_stack = {
2233 .lock = &interface_lock,
2234 .val = &osnoise_data.print_stack,
2235 .max = NULL,
2236 .min = NULL,
2237};
2238
2239/*
2240 * osnoise/timerlat_period: min 100 us, max 1 s
2241 */
2242u64 timerlat_min_period = 100;
2243u64 timerlat_max_period = 1000000;
2244static struct trace_min_max_param timerlat_period = {
2245 .lock = &interface_lock,
2246 .val = &osnoise_data.timerlat_period,
2247 .max = &timerlat_max_period,
2248 .min = &timerlat_min_period,
2249};
2250#endif
2251
2252static const struct file_operations cpus_fops = {
2253 .open = tracing_open_generic,
2254 .read = osnoise_cpus_read,
2255 .write = osnoise_cpus_write,
2256 .llseek = generic_file_llseek,
2257};
2258
2259static const struct file_operations osnoise_options_fops = {
2260 .open = osnoise_options_open,
2261 .read = seq_read,
2262 .llseek = seq_lseek,
2263 .release = seq_release,
2264 .write = osnoise_options_write
2265};
2266
2267#ifdef CONFIG_TIMERLAT_TRACER
2268#ifdef CONFIG_STACKTRACE
2269static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2270{
2271 struct dentry *tmp;
2272
2273 tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2274 &osnoise_print_stack, &trace_min_max_fops);
2275 if (!tmp)
2276 return -ENOMEM;
2277
2278 return 0;
2279}
2280#else /* CONFIG_STACKTRACE */
2281static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2282{
2283 return 0;
2284}
2285#endif /* CONFIG_STACKTRACE */
2286
2287/*
2288 * init_timerlat_tracefs - A function to initialize the timerlat interface files
2289 */
2290static int init_timerlat_tracefs(struct dentry *top_dir)
2291{
2292 struct dentry *tmp;
2293
2294 tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2295 &timerlat_period, &trace_min_max_fops);
2296 if (!tmp)
2297 return -ENOMEM;
2298
2299 return init_timerlat_stack_tracefs(top_dir);
2300}
2301#else /* CONFIG_TIMERLAT_TRACER */
2302static int init_timerlat_tracefs(struct dentry *top_dir)
2303{
2304 return 0;
2305}
2306#endif /* CONFIG_TIMERLAT_TRACER */
2307
2308/*
2309 * init_tracefs - A function to initialize the tracefs interface files
2310 *
2311 * This function creates entries in tracefs for "osnoise" and "timerlat".
2312 * It creates these directories in the tracing directory, and within that
2313 * directory the use can change and view the configs.
2314 */
2315static int init_tracefs(void)
2316{
2317 struct dentry *top_dir;
2318 struct dentry *tmp;
2319 int ret;
2320
2321 ret = tracing_init_dentry();
2322 if (ret)
2323 return -ENOMEM;
2324
2325 top_dir = tracefs_create_dir("osnoise", NULL);
2326 if (!top_dir)
2327 return 0;
2328
2329 tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2330 &osnoise_period, &trace_min_max_fops);
2331 if (!tmp)
2332 goto err;
2333
2334 tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2335 &osnoise_runtime, &trace_min_max_fops);
2336 if (!tmp)
2337 goto err;
2338
2339 tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2340 &osnoise_stop_tracing_in, &trace_min_max_fops);
2341 if (!tmp)
2342 goto err;
2343
2344 tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2345 &osnoise_stop_tracing_total, &trace_min_max_fops);
2346 if (!tmp)
2347 goto err;
2348
2349 tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2350 if (!tmp)
2351 goto err;
2352
2353 tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL,
2354 &osnoise_options_fops);
2355 if (!tmp)
2356 goto err;
2357
2358 ret = init_timerlat_tracefs(top_dir);
2359 if (ret)
2360 goto err;
2361
2362 return 0;
2363
2364err:
2365 tracefs_remove(top_dir);
2366 return -ENOMEM;
2367}
2368
2369static int osnoise_hook_events(void)
2370{
2371 int retval;
2372
2373 /*
2374 * Trace is already hooked, we are re-enabling from
2375 * a stop_tracing_*.
2376 */
2377 if (trace_osnoise_callback_enabled)
2378 return 0;
2379
2380 retval = hook_irq_events();
2381 if (retval)
2382 return -EINVAL;
2383
2384 retval = hook_softirq_events();
2385 if (retval)
2386 goto out_unhook_irq;
2387
2388 retval = hook_thread_events();
2389 /*
2390 * All fine!
2391 */
2392 if (!retval)
2393 return 0;
2394
2395 unhook_softirq_events();
2396out_unhook_irq:
2397 unhook_irq_events();
2398 return -EINVAL;
2399}
2400
2401static void osnoise_unhook_events(void)
2402{
2403 unhook_thread_events();
2404 unhook_softirq_events();
2405 unhook_irq_events();
2406}
2407
2408/*
2409 * osnoise_workload_start - start the workload and hook to events
2410 */
2411static int osnoise_workload_start(void)
2412{
2413 int retval;
2414
2415 /*
2416 * Instances need to be registered after calling workload
2417 * start. Hence, if there is already an instance, the
2418 * workload was already registered. Otherwise, this
2419 * code is on the way to register the first instance,
2420 * and the workload will start.
2421 */
2422 if (osnoise_has_registered_instances())
2423 return 0;
2424
2425 osn_var_reset_all();
2426
2427 retval = osnoise_hook_events();
2428 if (retval)
2429 return retval;
2430
2431 /*
2432 * Make sure that ftrace_nmi_enter/exit() see reset values
2433 * before enabling trace_osnoise_callback_enabled.
2434 */
2435 barrier();
2436 trace_osnoise_callback_enabled = true;
2437
2438 retval = start_per_cpu_kthreads();
2439 if (retval) {
2440 trace_osnoise_callback_enabled = false;
2441 /*
2442 * Make sure that ftrace_nmi_enter/exit() see
2443 * trace_osnoise_callback_enabled as false before continuing.
2444 */
2445 barrier();
2446
2447 osnoise_unhook_events();
2448 return retval;
2449 }
2450
2451 return 0;
2452}
2453
2454/*
2455 * osnoise_workload_stop - stop the workload and unhook the events
2456 */
2457static void osnoise_workload_stop(void)
2458{
2459 /*
2460 * Instances need to be unregistered before calling
2461 * stop. Hence, if there is a registered instance, more
2462 * than one instance is running, and the workload will not
2463 * yet stop. Otherwise, this code is on the way to disable
2464 * the last instance, and the workload can stop.
2465 */
2466 if (osnoise_has_registered_instances())
2467 return;
2468
2469 /*
2470 * If callbacks were already disabled in a previous stop
2471 * call, there is no need to disable then again.
2472 *
2473 * For instance, this happens when tracing is stopped via:
2474 * echo 0 > tracing_on
2475 * echo nop > current_tracer.
2476 */
2477 if (!trace_osnoise_callback_enabled)
2478 return;
2479
2480 trace_osnoise_callback_enabled = false;
2481 /*
2482 * Make sure that ftrace_nmi_enter/exit() see
2483 * trace_osnoise_callback_enabled as false before continuing.
2484 */
2485 barrier();
2486
2487 stop_per_cpu_kthreads();
2488
2489 osnoise_unhook_events();
2490}
2491
2492static void osnoise_tracer_start(struct trace_array *tr)
2493{
2494 int retval;
2495
2496 /*
2497 * If the instance is already registered, there is no need to
2498 * register it again.
2499 */
2500 if (osnoise_instance_registered(tr))
2501 return;
2502
2503 retval = osnoise_workload_start();
2504 if (retval)
2505 pr_err(BANNER "Error starting osnoise tracer\n");
2506
2507 osnoise_register_instance(tr);
2508}
2509
2510static void osnoise_tracer_stop(struct trace_array *tr)
2511{
2512 osnoise_unregister_instance(tr);
2513 osnoise_workload_stop();
2514}
2515
2516static int osnoise_tracer_init(struct trace_array *tr)
2517{
2518 /*
2519 * Only allow osnoise tracer if timerlat tracer is not running
2520 * already.
2521 */
2522 if (timerlat_enabled())
2523 return -EBUSY;
2524
2525 tr->max_latency = 0;
2526
2527 osnoise_tracer_start(tr);
2528 return 0;
2529}
2530
2531static void osnoise_tracer_reset(struct trace_array *tr)
2532{
2533 osnoise_tracer_stop(tr);
2534}
2535
2536static struct tracer osnoise_tracer __read_mostly = {
2537 .name = "osnoise",
2538 .init = osnoise_tracer_init,
2539 .reset = osnoise_tracer_reset,
2540 .start = osnoise_tracer_start,
2541 .stop = osnoise_tracer_stop,
2542 .print_header = print_osnoise_headers,
2543 .allow_instances = true,
2544};
2545
2546#ifdef CONFIG_TIMERLAT_TRACER
2547static void timerlat_tracer_start(struct trace_array *tr)
2548{
2549 int retval;
2550
2551 /*
2552 * If the instance is already registered, there is no need to
2553 * register it again.
2554 */
2555 if (osnoise_instance_registered(tr))
2556 return;
2557
2558 retval = osnoise_workload_start();
2559 if (retval)
2560 pr_err(BANNER "Error starting timerlat tracer\n");
2561
2562 osnoise_register_instance(tr);
2563
2564 return;
2565}
2566
2567static void timerlat_tracer_stop(struct trace_array *tr)
2568{
2569 int cpu;
2570
2571 osnoise_unregister_instance(tr);
2572
2573 /*
2574 * Instruct the threads to stop only if this is the last instance.
2575 */
2576 if (!osnoise_has_registered_instances()) {
2577 for_each_online_cpu(cpu)
2578 per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
2579 }
2580
2581 osnoise_workload_stop();
2582}
2583
2584static int timerlat_tracer_init(struct trace_array *tr)
2585{
2586 /*
2587 * Only allow timerlat tracer if osnoise tracer is not running already.
2588 */
2589 if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
2590 return -EBUSY;
2591
2592 /*
2593 * If this is the first instance, set timerlat_tracer to block
2594 * osnoise tracer start.
2595 */
2596 if (!osnoise_has_registered_instances())
2597 osnoise_data.timerlat_tracer = 1;
2598
2599 tr->max_latency = 0;
2600 timerlat_tracer_start(tr);
2601
2602 return 0;
2603}
2604
2605static void timerlat_tracer_reset(struct trace_array *tr)
2606{
2607 timerlat_tracer_stop(tr);
2608
2609 /*
2610 * If this is the last instance, reset timerlat_tracer allowing
2611 * osnoise to be started.
2612 */
2613 if (!osnoise_has_registered_instances())
2614 osnoise_data.timerlat_tracer = 0;
2615}
2616
2617static struct tracer timerlat_tracer __read_mostly = {
2618 .name = "timerlat",
2619 .init = timerlat_tracer_init,
2620 .reset = timerlat_tracer_reset,
2621 .start = timerlat_tracer_start,
2622 .stop = timerlat_tracer_stop,
2623 .print_header = print_timerlat_headers,
2624 .allow_instances = true,
2625};
2626
2627__init static int init_timerlat_tracer(void)
2628{
2629 return register_tracer(&timerlat_tracer);
2630}
2631#else /* CONFIG_TIMERLAT_TRACER */
2632__init static int init_timerlat_tracer(void)
2633{
2634 return 0;
2635}
2636#endif /* CONFIG_TIMERLAT_TRACER */
2637
2638__init static int init_osnoise_tracer(void)
2639{
2640 int ret;
2641
2642 mutex_init(&interface_lock);
2643
2644 cpumask_copy(&osnoise_cpumask, cpu_all_mask);
2645
2646 ret = register_tracer(&osnoise_tracer);
2647 if (ret) {
2648 pr_err(BANNER "Error registering osnoise!\n");
2649 return ret;
2650 }
2651
2652 ret = init_timerlat_tracer();
2653 if (ret) {
2654 pr_err(BANNER "Error registering timerlat!\n");
2655 return ret;
2656 }
2657
2658 osnoise_init_hotplug_support();
2659
2660 INIT_LIST_HEAD_RCU(&osnoise_instances);
2661
2662 init_tracefs();
2663
2664 return 0;
2665}
2666late_initcall(init_osnoise_tracer);