Linux Audio

Check our new training course

Loading...
v5.4
  1/* SPDX-License-Identifier: GPL-2.0 */
  2#undef TRACE_SYSTEM
  3#define TRACE_SYSTEM rcu
  4
  5#if !defined(_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ)
  6#define _TRACE_RCU_H
  7
  8#include <linux/tracepoint.h>
  9
 10#ifdef CONFIG_RCU_TRACE
 11#define TRACE_EVENT_RCU TRACE_EVENT
 12#else
 13#define TRACE_EVENT_RCU TRACE_EVENT_NOP
 14#endif
 15
 16/*
 17 * Tracepoint for start/end markers used for utilization calculations.
 18 * By convention, the string is of the following forms:
 19 *
 20 * "Start <activity>" -- Mark the start of the specified activity,
 21 *			 such as "context switch".  Nesting is permitted.
 22 * "End <activity>" -- Mark the end of the specified activity.
 23 *
 24 * An "@" character within "<activity>" is a comment character: Data
 25 * reduction scripts will ignore the "@" and the remainder of the line.
 26 */
 27TRACE_EVENT(rcu_utilization,
 28
 29	TP_PROTO(const char *s),
 30
 31	TP_ARGS(s),
 32
 33	TP_STRUCT__entry(
 34		__field(const char *, s)
 35	),
 36
 37	TP_fast_assign(
 38		__entry->s = s;
 39	),
 40
 41	TP_printk("%s", __entry->s)
 42);
 43
 
 
 44#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU)
 45
 46/*
 47 * Tracepoint for grace-period events.  Takes a string identifying the
 48 * RCU flavor, the grace-period number, and a string identifying the
 49 * grace-period-related event as follows:
 50 *
 51 *	"AccReadyCB": CPU acclerates new callbacks to RCU_NEXT_READY_TAIL.
 52 *	"AccWaitCB": CPU accelerates new callbacks to RCU_WAIT_TAIL.
 53 *	"newreq": Request a new grace period.
 54 *	"start": Start a grace period.
 55 *	"cpustart": CPU first notices a grace-period start.
 56 *	"cpuqs": CPU passes through a quiescent state.
 57 *	"cpuonl": CPU comes online.
 58 *	"cpuofl": CPU goes offline.
 59 *	"cpuofl-bgp": CPU goes offline while blocking a grace period.
 60 *	"reqwait": GP kthread sleeps waiting for grace-period request.
 61 *	"reqwaitsig": GP kthread awakened by signal from reqwait state.
 62 *	"fqswait": GP kthread waiting until time to force quiescent states.
 63 *	"fqsstart": GP kthread starts forcing quiescent states.
 64 *	"fqsend": GP kthread done forcing quiescent states.
 65 *	"fqswaitsig": GP kthread awakened by signal from fqswait state.
 66 *	"end": End a grace period.
 67 *	"cpuend": CPU first notices a grace-period end.
 68 */
 69TRACE_EVENT_RCU(rcu_grace_period,
 70
 71	TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent),
 72
 73	TP_ARGS(rcuname, gp_seq, gpevent),
 74
 75	TP_STRUCT__entry(
 76		__field(const char *, rcuname)
 77		__field(unsigned long, gp_seq)
 78		__field(const char *, gpevent)
 79	),
 80
 81	TP_fast_assign(
 82		__entry->rcuname = rcuname;
 83		__entry->gp_seq = gp_seq;
 84		__entry->gpevent = gpevent;
 85	),
 86
 87	TP_printk("%s %lu %s",
 88		  __entry->rcuname, __entry->gp_seq, __entry->gpevent)
 89);
 90
 91/*
 92 * Tracepoint for future grace-period events.  The caller should pull
 93 * the data from the rcu_node structure, other than rcuname, which comes
 94 * from the rcu_state structure, and event, which is one of the following:
 
 95 *
 96 * "Startleaf": Request a grace period based on leaf-node data.
 97 * "Prestarted": Someone beat us to the request
 98 * "Startedleaf": Leaf node marked for future GP.
 99 * "Startedleafroot": All nodes from leaf to root marked for future GP.
100 * "Startedroot": Requested a nocb grace period based on root-node data.
101 * "NoGPkthread": The RCU grace-period kthread has not yet started.
102 * "StartWait": Start waiting for the requested grace period.
 
103 * "EndWait": Complete wait.
104 * "Cleanup": Clean up rcu_node structure after previous GP.
105 * "CleanupMore": Clean up, and another GP is needed.
106 */
107TRACE_EVENT_RCU(rcu_future_grace_period,
108
109	TP_PROTO(const char *rcuname, unsigned long gp_seq,
110		 unsigned long gp_seq_req, u8 level, int grplo, int grphi,
111		 const char *gpevent),
112
113	TP_ARGS(rcuname, gp_seq, gp_seq_req, level, grplo, grphi, gpevent),
114
115	TP_STRUCT__entry(
116		__field(const char *, rcuname)
117		__field(unsigned long, gp_seq)
118		__field(unsigned long, gp_seq_req)
 
119		__field(u8, level)
120		__field(int, grplo)
121		__field(int, grphi)
122		__field(const char *, gpevent)
123	),
124
125	TP_fast_assign(
126		__entry->rcuname = rcuname;
127		__entry->gp_seq = gp_seq;
128		__entry->gp_seq_req = gp_seq_req;
 
129		__entry->level = level;
130		__entry->grplo = grplo;
131		__entry->grphi = grphi;
132		__entry->gpevent = gpevent;
133	),
134
135	TP_printk("%s %lu %lu %u %d %d %s",
136		  __entry->rcuname, __entry->gp_seq, __entry->gp_seq_req, __entry->level,
137		  __entry->grplo, __entry->grphi, __entry->gpevent)
 
138);
139
140/*
141 * Tracepoint for grace-period-initialization events.  These are
142 * distinguished by the type of RCU, the new grace-period number, the
143 * rcu_node structure level, the starting and ending CPU covered by the
144 * rcu_node structure, and the mask of CPUs that will be waited for.
145 * All but the type of RCU are extracted from the rcu_node structure.
146 */
147TRACE_EVENT_RCU(rcu_grace_period_init,
148
149	TP_PROTO(const char *rcuname, unsigned long gp_seq, u8 level,
150		 int grplo, int grphi, unsigned long qsmask),
151
152	TP_ARGS(rcuname, gp_seq, level, grplo, grphi, qsmask),
153
154	TP_STRUCT__entry(
155		__field(const char *, rcuname)
156		__field(unsigned long, gp_seq)
157		__field(u8, level)
158		__field(int, grplo)
159		__field(int, grphi)
160		__field(unsigned long, qsmask)
161	),
162
163	TP_fast_assign(
164		__entry->rcuname = rcuname;
165		__entry->gp_seq = gp_seq;
166		__entry->level = level;
167		__entry->grplo = grplo;
168		__entry->grphi = grphi;
169		__entry->qsmask = qsmask;
170	),
171
172	TP_printk("%s %lu %u %d %d %lx",
173		  __entry->rcuname, __entry->gp_seq, __entry->level,
174		  __entry->grplo, __entry->grphi, __entry->qsmask)
175);
176
177/*
178 * Tracepoint for expedited grace-period events.  Takes a string identifying
179 * the RCU flavor, the expedited grace-period sequence number, and a string
180 * identifying the grace-period-related event as follows:
181 *
182 *	"snap": Captured snapshot of expedited grace period sequence number.
183 *	"start": Started a real expedited grace period.
184 *	"reset": Started resetting the tree
185 *	"select": Started selecting the CPUs to wait on.
186 *	"selectofl": Selected CPU partially offline.
187 *	"startwait": Started waiting on selected CPUs.
188 *	"end": Ended a real expedited grace period.
189 *	"endwake": Woke piggybackers up.
190 *	"done": Someone else did the expedited grace period for us.
191 */
192TRACE_EVENT_RCU(rcu_exp_grace_period,
193
194	TP_PROTO(const char *rcuname, unsigned long gpseq, const char *gpevent),
195
196	TP_ARGS(rcuname, gpseq, gpevent),
197
198	TP_STRUCT__entry(
199		__field(const char *, rcuname)
200		__field(unsigned long, gpseq)
201		__field(const char *, gpevent)
202	),
203
204	TP_fast_assign(
205		__entry->rcuname = rcuname;
206		__entry->gpseq = gpseq;
207		__entry->gpevent = gpevent;
208	),
209
210	TP_printk("%s %lu %s",
211		  __entry->rcuname, __entry->gpseq, __entry->gpevent)
212);
213
214/*
215 * Tracepoint for expedited grace-period funnel-locking events.  Takes a
216 * string identifying the RCU flavor, an integer identifying the rcu_node
217 * combining-tree level, another pair of integers identifying the lowest-
218 * and highest-numbered CPU associated with the current rcu_node structure,
219 * and a string.  identifying the grace-period-related event as follows:
220 *
221 *	"nxtlvl": Advance to next level of rcu_node funnel
222 *	"wait": Wait for someone else to do expedited GP
223 */
224TRACE_EVENT_RCU(rcu_exp_funnel_lock,
225
226	TP_PROTO(const char *rcuname, u8 level, int grplo, int grphi,
227		 const char *gpevent),
228
229	TP_ARGS(rcuname, level, grplo, grphi, gpevent),
230
231	TP_STRUCT__entry(
232		__field(const char *, rcuname)
233		__field(u8, level)
234		__field(int, grplo)
235		__field(int, grphi)
236		__field(const char *, gpevent)
237	),
238
239	TP_fast_assign(
240		__entry->rcuname = rcuname;
241		__entry->level = level;
242		__entry->grplo = grplo;
243		__entry->grphi = grphi;
244		__entry->gpevent = gpevent;
245	),
246
247	TP_printk("%s %d %d %d %s",
248		  __entry->rcuname, __entry->level, __entry->grplo,
249		  __entry->grphi, __entry->gpevent)
250);
251
252#ifdef CONFIG_RCU_NOCB_CPU
253/*
254 * Tracepoint for RCU no-CBs CPU callback handoffs.  This event is intended
255 * to assist debugging of these handoffs.
256 *
257 * The first argument is the name of the RCU flavor, and the second is
258 * the number of the offloaded CPU are extracted.  The third and final
259 * argument is a string as follows:
260 *
261 *	"WakeEmpty": Wake rcuo kthread, first CB to empty list.
262 *	"WakeEmptyIsDeferred": Wake rcuo kthread later, first CB to empty list.
263 *	"WakeOvf": Wake rcuo kthread, CB list is huge.
264 *	"WakeOvfIsDeferred": Wake rcuo kthread later, CB list is huge.
265 *	"WakeNot": Don't wake rcuo kthread.
266 *	"WakeNotPoll": Don't wake rcuo kthread because it is polling.
267 *	"DeferredWake": Carried out the "IsDeferred" wakeup.
268 *	"Poll": Start of new polling cycle for rcu_nocb_poll.
269 *	"Sleep": Sleep waiting for GP for !rcu_nocb_poll.
270 *	"CBSleep": Sleep waiting for CBs for !rcu_nocb_poll.
271 *	"WokeEmpty": rcuo kthread woke to find empty list.
272 *	"WokeNonEmpty": rcuo kthread woke to find non-empty list.
273 *	"WaitQueue": Enqueue partially done, timed wait for it to complete.
274 *	"WokeQueue": Partial enqueue now complete.
275 */
276TRACE_EVENT_RCU(rcu_nocb_wake,
277
278	TP_PROTO(const char *rcuname, int cpu, const char *reason),
279
280	TP_ARGS(rcuname, cpu, reason),
281
282	TP_STRUCT__entry(
283		__field(const char *, rcuname)
284		__field(int, cpu)
285		__field(const char *, reason)
286	),
287
288	TP_fast_assign(
289		__entry->rcuname = rcuname;
290		__entry->cpu = cpu;
291		__entry->reason = reason;
292	),
293
294	TP_printk("%s %d %s", __entry->rcuname, __entry->cpu, __entry->reason)
295);
296#endif
297
298/*
299 * Tracepoint for tasks blocking within preemptible-RCU read-side
300 * critical sections.  Track the type of RCU (which one day might
301 * include SRCU), the grace-period number that the task is blocking
302 * (the current or the next), and the task's PID.
303 */
304TRACE_EVENT_RCU(rcu_preempt_task,
305
306	TP_PROTO(const char *rcuname, int pid, unsigned long gp_seq),
307
308	TP_ARGS(rcuname, pid, gp_seq),
309
310	TP_STRUCT__entry(
311		__field(const char *, rcuname)
312		__field(unsigned long, gp_seq)
313		__field(int, pid)
314	),
315
316	TP_fast_assign(
317		__entry->rcuname = rcuname;
318		__entry->gp_seq = gp_seq;
319		__entry->pid = pid;
320	),
321
322	TP_printk("%s %lu %d",
323		  __entry->rcuname, __entry->gp_seq, __entry->pid)
324);
325
326/*
327 * Tracepoint for tasks that blocked within a given preemptible-RCU
328 * read-side critical section exiting that critical section.  Track the
329 * type of RCU (which one day might include SRCU) and the task's PID.
330 */
331TRACE_EVENT_RCU(rcu_unlock_preempted_task,
332
333	TP_PROTO(const char *rcuname, unsigned long gp_seq, int pid),
334
335	TP_ARGS(rcuname, gp_seq, pid),
336
337	TP_STRUCT__entry(
338		__field(const char *, rcuname)
339		__field(unsigned long, gp_seq)
340		__field(int, pid)
341	),
342
343	TP_fast_assign(
344		__entry->rcuname = rcuname;
345		__entry->gp_seq = gp_seq;
346		__entry->pid = pid;
347	),
348
349	TP_printk("%s %lu %d", __entry->rcuname, __entry->gp_seq, __entry->pid)
350);
351
352/*
353 * Tracepoint for quiescent-state-reporting events.  These are
354 * distinguished by the type of RCU, the grace-period number, the
355 * mask of quiescent lower-level entities, the rcu_node structure level,
356 * the starting and ending CPU covered by the rcu_node structure, and
357 * whether there are any blocked tasks blocking the current grace period.
358 * All but the type of RCU are extracted from the rcu_node structure.
359 */
360TRACE_EVENT_RCU(rcu_quiescent_state_report,
361
362	TP_PROTO(const char *rcuname, unsigned long gp_seq,
363		 unsigned long mask, unsigned long qsmask,
364		 u8 level, int grplo, int grphi, int gp_tasks),
365
366	TP_ARGS(rcuname, gp_seq, mask, qsmask, level, grplo, grphi, gp_tasks),
367
368	TP_STRUCT__entry(
369		__field(const char *, rcuname)
370		__field(unsigned long, gp_seq)
371		__field(unsigned long, mask)
372		__field(unsigned long, qsmask)
373		__field(u8, level)
374		__field(int, grplo)
375		__field(int, grphi)
376		__field(u8, gp_tasks)
377	),
378
379	TP_fast_assign(
380		__entry->rcuname = rcuname;
381		__entry->gp_seq = gp_seq;
382		__entry->mask = mask;
383		__entry->qsmask = qsmask;
384		__entry->level = level;
385		__entry->grplo = grplo;
386		__entry->grphi = grphi;
387		__entry->gp_tasks = gp_tasks;
388	),
389
390	TP_printk("%s %lu %lx>%lx %u %d %d %u",
391		  __entry->rcuname, __entry->gp_seq,
392		  __entry->mask, __entry->qsmask, __entry->level,
393		  __entry->grplo, __entry->grphi, __entry->gp_tasks)
394);
395
396/*
397 * Tracepoint for quiescent states detected by force_quiescent_state().
398 * These trace events include the type of RCU, the grace-period number
399 * that was blocked by the CPU, the CPU itself, and the type of quiescent
400 * state, which can be "dti" for dyntick-idle mode or "kick" when kicking
401 * a CPU that has been in dyntick-idle mode for too long.
 
402 */
403TRACE_EVENT_RCU(rcu_fqs,
404
405	TP_PROTO(const char *rcuname, unsigned long gp_seq, int cpu, const char *qsevent),
406
407	TP_ARGS(rcuname, gp_seq, cpu, qsevent),
408
409	TP_STRUCT__entry(
410		__field(const char *, rcuname)
411		__field(unsigned long, gp_seq)
412		__field(int, cpu)
413		__field(const char *, qsevent)
414	),
415
416	TP_fast_assign(
417		__entry->rcuname = rcuname;
418		__entry->gp_seq = gp_seq;
419		__entry->cpu = cpu;
420		__entry->qsevent = qsevent;
421	),
422
423	TP_printk("%s %lu %d %s",
424		  __entry->rcuname, __entry->gp_seq,
425		  __entry->cpu, __entry->qsevent)
426);
427
428#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) */
429
430/*
431 * Tracepoint for dyntick-idle entry/exit events.  These take a string
432 * as argument: "Start" for entering dyntick-idle mode, "Startirq" for
433 * entering it from irq/NMI, "End" for leaving it, "Endirq" for leaving it
434 * to irq/NMI, "--=" for events moving towards idle, and "++=" for events
435 * moving away from idle.
 
436 *
437 * These events also take a pair of numbers, which indicate the nesting
438 * depth before and after the event of interest, and a third number that is
439 * the ->dynticks counter.  Note that task-related and interrupt-related
440 * events use two separate counters, and that the "++=" and "--=" events
441 * for irq/NMI will change the counter by two, otherwise by one.
442 */
443TRACE_EVENT_RCU(rcu_dyntick,
444
445	TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks),
446
447	TP_ARGS(polarity, oldnesting, newnesting, dynticks),
448
449	TP_STRUCT__entry(
450		__field(const char *, polarity)
451		__field(long, oldnesting)
452		__field(long, newnesting)
453		__field(int, dynticks)
454	),
455
456	TP_fast_assign(
457		__entry->polarity = polarity;
458		__entry->oldnesting = oldnesting;
459		__entry->newnesting = newnesting;
460		__entry->dynticks = atomic_read(&dynticks);
461	),
462
463	TP_printk("%s %lx %lx %#3x", __entry->polarity,
464		  __entry->oldnesting, __entry->newnesting,
465		  __entry->dynticks & 0xfff)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
466);
467
468/*
469 * Tracepoint for the registration of a single RCU callback function.
470 * The first argument is the type of RCU, the second argument is
471 * a pointer to the RCU callback itself, the third element is the
472 * number of lazy callbacks queued, and the fourth element is the
473 * total number of callbacks queued.
474 */
475TRACE_EVENT_RCU(rcu_callback,
476
477	TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen_lazy,
478		 long qlen),
479
480	TP_ARGS(rcuname, rhp, qlen_lazy, qlen),
481
482	TP_STRUCT__entry(
483		__field(const char *, rcuname)
484		__field(void *, rhp)
485		__field(void *, func)
486		__field(long, qlen_lazy)
487		__field(long, qlen)
488	),
489
490	TP_fast_assign(
491		__entry->rcuname = rcuname;
492		__entry->rhp = rhp;
493		__entry->func = rhp->func;
494		__entry->qlen_lazy = qlen_lazy;
495		__entry->qlen = qlen;
496	),
497
498	TP_printk("%s rhp=%p func=%ps %ld/%ld",
499		  __entry->rcuname, __entry->rhp, __entry->func,
500		  __entry->qlen_lazy, __entry->qlen)
501);
502
503/*
504 * Tracepoint for the registration of a single RCU callback of the special
505 * kfree() form.  The first argument is the RCU type, the second argument
506 * is a pointer to the RCU callback, the third argument is the offset
507 * of the callback within the enclosing RCU-protected data structure,
508 * the fourth argument is the number of lazy callbacks queued, and the
509 * fifth argument is the total number of callbacks queued.
510 */
511TRACE_EVENT_RCU(rcu_kfree_callback,
512
513	TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset,
514		 long qlen_lazy, long qlen),
515
516	TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen),
517
518	TP_STRUCT__entry(
519		__field(const char *, rcuname)
520		__field(void *, rhp)
521		__field(unsigned long, offset)
522		__field(long, qlen_lazy)
523		__field(long, qlen)
524	),
525
526	TP_fast_assign(
527		__entry->rcuname = rcuname;
528		__entry->rhp = rhp;
529		__entry->offset = offset;
530		__entry->qlen_lazy = qlen_lazy;
531		__entry->qlen = qlen;
532	),
533
534	TP_printk("%s rhp=%p func=%ld %ld/%ld",
535		  __entry->rcuname, __entry->rhp, __entry->offset,
536		  __entry->qlen_lazy, __entry->qlen)
537);
538
539/*
540 * Tracepoint for marking the beginning rcu_do_batch, performed to start
541 * RCU callback invocation.  The first argument is the RCU flavor,
542 * the second is the number of lazy callbacks queued, the third is
543 * the total number of callbacks queued, and the fourth argument is
544 * the current RCU-callback batch limit.
545 */
546TRACE_EVENT_RCU(rcu_batch_start,
547
548	TP_PROTO(const char *rcuname, long qlen_lazy, long qlen, long blimit),
549
550	TP_ARGS(rcuname, qlen_lazy, qlen, blimit),
551
552	TP_STRUCT__entry(
553		__field(const char *, rcuname)
554		__field(long, qlen_lazy)
555		__field(long, qlen)
556		__field(long, blimit)
557	),
558
559	TP_fast_assign(
560		__entry->rcuname = rcuname;
561		__entry->qlen_lazy = qlen_lazy;
562		__entry->qlen = qlen;
563		__entry->blimit = blimit;
564	),
565
566	TP_printk("%s CBs=%ld/%ld bl=%ld",
567		  __entry->rcuname, __entry->qlen_lazy, __entry->qlen,
568		  __entry->blimit)
569);
570
571/*
572 * Tracepoint for the invocation of a single RCU callback function.
573 * The first argument is the type of RCU, and the second argument is
574 * a pointer to the RCU callback itself.
575 */
576TRACE_EVENT_RCU(rcu_invoke_callback,
577
578	TP_PROTO(const char *rcuname, struct rcu_head *rhp),
579
580	TP_ARGS(rcuname, rhp),
581
582	TP_STRUCT__entry(
583		__field(const char *, rcuname)
584		__field(void *, rhp)
585		__field(void *, func)
586	),
587
588	TP_fast_assign(
589		__entry->rcuname = rcuname;
590		__entry->rhp = rhp;
591		__entry->func = rhp->func;
592	),
593
594	TP_printk("%s rhp=%p func=%ps",
595		  __entry->rcuname, __entry->rhp, __entry->func)
596);
597
598/*
599 * Tracepoint for the invocation of a single RCU callback of the special
600 * kfree() form.  The first argument is the RCU flavor, the second
601 * argument is a pointer to the RCU callback, and the third argument
602 * is the offset of the callback within the enclosing RCU-protected
603 * data structure.
604 */
605TRACE_EVENT_RCU(rcu_invoke_kfree_callback,
606
607	TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset),
608
609	TP_ARGS(rcuname, rhp, offset),
610
611	TP_STRUCT__entry(
612		__field(const char *, rcuname)
613		__field(void *, rhp)
614		__field(unsigned long, offset)
615	),
616
617	TP_fast_assign(
618		__entry->rcuname = rcuname;
619		__entry->rhp = rhp;
620		__entry->offset	= offset;
621	),
622
623	TP_printk("%s rhp=%p func=%ld",
624		  __entry->rcuname, __entry->rhp, __entry->offset)
625);
626
627/*
628 * Tracepoint for exiting rcu_do_batch after RCU callbacks have been
629 * invoked.  The first argument is the name of the RCU flavor,
630 * the second argument is number of callbacks actually invoked,
631 * the third argument (cb) is whether or not any of the callbacks that
632 * were ready to invoke at the beginning of this batch are still
633 * queued, the fourth argument (nr) is the return value of need_resched(),
634 * the fifth argument (iit) is 1 if the current task is the idle task,
635 * and the sixth argument (risk) is the return value from
636 * rcu_is_callbacks_kthread().
637 */
638TRACE_EVENT_RCU(rcu_batch_end,
639
640	TP_PROTO(const char *rcuname, int callbacks_invoked,
641		 char cb, char nr, char iit, char risk),
642
643	TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
644
645	TP_STRUCT__entry(
646		__field(const char *, rcuname)
647		__field(int, callbacks_invoked)
648		__field(char, cb)
649		__field(char, nr)
650		__field(char, iit)
651		__field(char, risk)
652	),
653
654	TP_fast_assign(
655		__entry->rcuname = rcuname;
656		__entry->callbacks_invoked = callbacks_invoked;
657		__entry->cb = cb;
658		__entry->nr = nr;
659		__entry->iit = iit;
660		__entry->risk = risk;
661	),
662
663	TP_printk("%s CBs-invoked=%d idle=%c%c%c%c",
664		  __entry->rcuname, __entry->callbacks_invoked,
665		  __entry->cb ? 'C' : '.',
666		  __entry->nr ? 'S' : '.',
667		  __entry->iit ? 'I' : '.',
668		  __entry->risk ? 'R' : '.')
669);
670
671/*
672 * Tracepoint for rcutorture readers.  The first argument is the name
673 * of the RCU flavor from rcutorture's viewpoint and the second argument
674 * is the callback address.  The third argument is the start time in
675 * seconds, and the last two arguments are the grace period numbers
676 * at the beginning and end of the read, respectively.  Note that the
677 * callback address can be NULL.
678 */
679#define RCUTORTURENAME_LEN 8
680TRACE_EVENT_RCU(rcu_torture_read,
681
682	TP_PROTO(const char *rcutorturename, struct rcu_head *rhp,
683		 unsigned long secs, unsigned long c_old, unsigned long c),
684
685	TP_ARGS(rcutorturename, rhp, secs, c_old, c),
686
687	TP_STRUCT__entry(
688		__field(char, rcutorturename[RCUTORTURENAME_LEN])
689		__field(struct rcu_head *, rhp)
690		__field(unsigned long, secs)
691		__field(unsigned long, c_old)
692		__field(unsigned long, c)
693	),
694
695	TP_fast_assign(
696		strncpy(__entry->rcutorturename, rcutorturename,
697			RCUTORTURENAME_LEN);
698		__entry->rcutorturename[RCUTORTURENAME_LEN - 1] = 0;
699		__entry->rhp = rhp;
700		__entry->secs = secs;
701		__entry->c_old = c_old;
702		__entry->c = c;
703	),
704
705	TP_printk("%s torture read %p %luus c: %lu %lu",
706		  __entry->rcutorturename, __entry->rhp,
707		  __entry->secs, __entry->c_old, __entry->c)
708);
709
710/*
711 * Tracepoint for rcu_barrier() execution.  The string "s" describes
712 * the rcu_barrier phase:
713 *	"Begin": rcu_barrier() started.
714 *	"EarlyExit": rcu_barrier() piggybacked, thus early exit.
715 *	"Inc1": rcu_barrier() piggyback check counter incremented.
716 *	"OfflineNoCB": rcu_barrier() found callback on never-online CPU
717 *	"OnlineNoCB": rcu_barrier() found online no-CBs CPU.
718 *	"OnlineQ": rcu_barrier() found online CPU with callbacks.
719 *	"OnlineNQ": rcu_barrier() found online CPU, no callbacks.
720 *	"IRQ": An rcu_barrier_callback() callback posted on remote CPU.
721 *	"IRQNQ": An rcu_barrier_callback() callback found no callbacks.
722 *	"CB": An rcu_barrier_callback() invoked a callback, not the last.
723 *	"LastCB": An rcu_barrier_callback() invoked the last callback.
724 *	"Inc2": rcu_barrier() piggyback check counter incremented.
725 * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument
726 * is the count of remaining callbacks, and "done" is the piggybacking count.
727 */
728TRACE_EVENT_RCU(rcu_barrier,
729
730	TP_PROTO(const char *rcuname, const char *s, int cpu, int cnt, unsigned long done),
731
732	TP_ARGS(rcuname, s, cpu, cnt, done),
733
734	TP_STRUCT__entry(
735		__field(const char *, rcuname)
736		__field(const char *, s)
737		__field(int, cpu)
738		__field(int, cnt)
739		__field(unsigned long, done)
740	),
741
742	TP_fast_assign(
743		__entry->rcuname = rcuname;
744		__entry->s = s;
745		__entry->cpu = cpu;
746		__entry->cnt = cnt;
747		__entry->done = done;
748	),
749
750	TP_printk("%s %s cpu %d remaining %d # %lu",
751		  __entry->rcuname, __entry->s, __entry->cpu, __entry->cnt,
752		  __entry->done)
753);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
754
755#endif /* _TRACE_RCU_H */
756
757/* This part must be outside protection */
758#include <trace/define_trace.h>
v4.10.11
 
  1#undef TRACE_SYSTEM
  2#define TRACE_SYSTEM rcu
  3
  4#if !defined(_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ)
  5#define _TRACE_RCU_H
  6
  7#include <linux/tracepoint.h>
  8
 
 
 
 
 
 
  9/*
 10 * Tracepoint for start/end markers used for utilization calculations.
 11 * By convention, the string is of the following forms:
 12 *
 13 * "Start <activity>" -- Mark the start of the specified activity,
 14 *			 such as "context switch".  Nesting is permitted.
 15 * "End <activity>" -- Mark the end of the specified activity.
 16 *
 17 * An "@" character within "<activity>" is a comment character: Data
 18 * reduction scripts will ignore the "@" and the remainder of the line.
 19 */
 20TRACE_EVENT(rcu_utilization,
 21
 22	TP_PROTO(const char *s),
 23
 24	TP_ARGS(s),
 25
 26	TP_STRUCT__entry(
 27		__field(const char *, s)
 28	),
 29
 30	TP_fast_assign(
 31		__entry->s = s;
 32	),
 33
 34	TP_printk("%s", __entry->s)
 35);
 36
 37#ifdef CONFIG_RCU_TRACE
 38
 39#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU)
 40
 41/*
 42 * Tracepoint for grace-period events.  Takes a string identifying the
 43 * RCU flavor, the grace-period number, and a string identifying the
 44 * grace-period-related event as follows:
 45 *
 46 *	"AccReadyCB": CPU acclerates new callbacks to RCU_NEXT_READY_TAIL.
 47 *	"AccWaitCB": CPU accelerates new callbacks to RCU_WAIT_TAIL.
 48 *	"newreq": Request a new grace period.
 49 *	"start": Start a grace period.
 50 *	"cpustart": CPU first notices a grace-period start.
 51 *	"cpuqs": CPU passes through a quiescent state.
 52 *	"cpuonl": CPU comes online.
 53 *	"cpuofl": CPU goes offline.
 
 54 *	"reqwait": GP kthread sleeps waiting for grace-period request.
 55 *	"reqwaitsig": GP kthread awakened by signal from reqwait state.
 56 *	"fqswait": GP kthread waiting until time to force quiescent states.
 57 *	"fqsstart": GP kthread starts forcing quiescent states.
 58 *	"fqsend": GP kthread done forcing quiescent states.
 59 *	"fqswaitsig": GP kthread awakened by signal from fqswait state.
 60 *	"end": End a grace period.
 61 *	"cpuend": CPU first notices a grace-period end.
 62 */
 63TRACE_EVENT(rcu_grace_period,
 64
 65	TP_PROTO(const char *rcuname, unsigned long gpnum, const char *gpevent),
 66
 67	TP_ARGS(rcuname, gpnum, gpevent),
 68
 69	TP_STRUCT__entry(
 70		__field(const char *, rcuname)
 71		__field(unsigned long, gpnum)
 72		__field(const char *, gpevent)
 73	),
 74
 75	TP_fast_assign(
 76		__entry->rcuname = rcuname;
 77		__entry->gpnum = gpnum;
 78		__entry->gpevent = gpevent;
 79	),
 80
 81	TP_printk("%s %lu %s",
 82		  __entry->rcuname, __entry->gpnum, __entry->gpevent)
 83);
 84
 85/*
 86 * Tracepoint for future grace-period events, including those for no-callbacks
 87 * CPUs.  The caller should pull the data from the rcu_node structure,
 88 * other than rcuname, which comes from the rcu_state structure, and event,
 89 * which is one of the following:
 90 *
 91 * "Startleaf": Request a nocb grace period based on leaf-node data.
 92 * "Startedleaf": Leaf-node start proved sufficient.
 93 * "Startedleafroot": Leaf-node start proved sufficient after checking root.
 
 94 * "Startedroot": Requested a nocb grace period based on root-node data.
 
 95 * "StartWait": Start waiting for the requested grace period.
 96 * "ResumeWait": Resume waiting after signal.
 97 * "EndWait": Complete wait.
 98 * "Cleanup": Clean up rcu_node structure after previous GP.
 99 * "CleanupMore": Clean up, and another no-CB GP is needed.
100 */
101TRACE_EVENT(rcu_future_grace_period,
102
103	TP_PROTO(const char *rcuname, unsigned long gpnum, unsigned long completed,
104		 unsigned long c, u8 level, int grplo, int grphi,
105		 const char *gpevent),
106
107	TP_ARGS(rcuname, gpnum, completed, c, level, grplo, grphi, gpevent),
108
109	TP_STRUCT__entry(
110		__field(const char *, rcuname)
111		__field(unsigned long, gpnum)
112		__field(unsigned long, completed)
113		__field(unsigned long, c)
114		__field(u8, level)
115		__field(int, grplo)
116		__field(int, grphi)
117		__field(const char *, gpevent)
118	),
119
120	TP_fast_assign(
121		__entry->rcuname = rcuname;
122		__entry->gpnum = gpnum;
123		__entry->completed = completed;
124		__entry->c = c;
125		__entry->level = level;
126		__entry->grplo = grplo;
127		__entry->grphi = grphi;
128		__entry->gpevent = gpevent;
129	),
130
131	TP_printk("%s %lu %lu %lu %u %d %d %s",
132		  __entry->rcuname, __entry->gpnum, __entry->completed,
133		  __entry->c, __entry->level, __entry->grplo, __entry->grphi,
134		  __entry->gpevent)
135);
136
137/*
138 * Tracepoint for grace-period-initialization events.  These are
139 * distinguished by the type of RCU, the new grace-period number, the
140 * rcu_node structure level, the starting and ending CPU covered by the
141 * rcu_node structure, and the mask of CPUs that will be waited for.
142 * All but the type of RCU are extracted from the rcu_node structure.
143 */
144TRACE_EVENT(rcu_grace_period_init,
145
146	TP_PROTO(const char *rcuname, unsigned long gpnum, u8 level,
147		 int grplo, int grphi, unsigned long qsmask),
148
149	TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
150
151	TP_STRUCT__entry(
152		__field(const char *, rcuname)
153		__field(unsigned long, gpnum)
154		__field(u8, level)
155		__field(int, grplo)
156		__field(int, grphi)
157		__field(unsigned long, qsmask)
158	),
159
160	TP_fast_assign(
161		__entry->rcuname = rcuname;
162		__entry->gpnum = gpnum;
163		__entry->level = level;
164		__entry->grplo = grplo;
165		__entry->grphi = grphi;
166		__entry->qsmask = qsmask;
167	),
168
169	TP_printk("%s %lu %u %d %d %lx",
170		  __entry->rcuname, __entry->gpnum, __entry->level,
171		  __entry->grplo, __entry->grphi, __entry->qsmask)
172);
173
174/*
175 * Tracepoint for expedited grace-period events.  Takes a string identifying
176 * the RCU flavor, the expedited grace-period sequence number, and a string
177 * identifying the grace-period-related event as follows:
178 *
179 *	"snap": Captured snapshot of expedited grace period sequence number.
180 *	"start": Started a real expedited grace period.
 
 
 
 
181 *	"end": Ended a real expedited grace period.
182 *	"endwake": Woke piggybackers up.
183 *	"done": Someone else did the expedited grace period for us.
184 */
185TRACE_EVENT(rcu_exp_grace_period,
186
187	TP_PROTO(const char *rcuname, unsigned long gpseq, const char *gpevent),
188
189	TP_ARGS(rcuname, gpseq, gpevent),
190
191	TP_STRUCT__entry(
192		__field(const char *, rcuname)
193		__field(unsigned long, gpseq)
194		__field(const char *, gpevent)
195	),
196
197	TP_fast_assign(
198		__entry->rcuname = rcuname;
199		__entry->gpseq = gpseq;
200		__entry->gpevent = gpevent;
201	),
202
203	TP_printk("%s %lu %s",
204		  __entry->rcuname, __entry->gpseq, __entry->gpevent)
205);
206
207/*
208 * Tracepoint for expedited grace-period funnel-locking events.  Takes a
209 * string identifying the RCU flavor, an integer identifying the rcu_node
210 * combining-tree level, another pair of integers identifying the lowest-
211 * and highest-numbered CPU associated with the current rcu_node structure,
212 * and a string.  identifying the grace-period-related event as follows:
213 *
214 *	"nxtlvl": Advance to next level of rcu_node funnel
215 *	"wait": Wait for someone else to do expedited GP
216 */
217TRACE_EVENT(rcu_exp_funnel_lock,
218
219	TP_PROTO(const char *rcuname, u8 level, int grplo, int grphi,
220		 const char *gpevent),
221
222	TP_ARGS(rcuname, level, grplo, grphi, gpevent),
223
224	TP_STRUCT__entry(
225		__field(const char *, rcuname)
226		__field(u8, level)
227		__field(int, grplo)
228		__field(int, grphi)
229		__field(const char *, gpevent)
230	),
231
232	TP_fast_assign(
233		__entry->rcuname = rcuname;
234		__entry->level = level;
235		__entry->grplo = grplo;
236		__entry->grphi = grphi;
237		__entry->gpevent = gpevent;
238	),
239
240	TP_printk("%s %d %d %d %s",
241		  __entry->rcuname, __entry->level, __entry->grplo,
242		  __entry->grphi, __entry->gpevent)
243);
244
 
245/*
246 * Tracepoint for RCU no-CBs CPU callback handoffs.  This event is intended
247 * to assist debugging of these handoffs.
248 *
249 * The first argument is the name of the RCU flavor, and the second is
250 * the number of the offloaded CPU are extracted.  The third and final
251 * argument is a string as follows:
252 *
253 *	"WakeEmpty": Wake rcuo kthread, first CB to empty list.
254 *	"WakeEmptyIsDeferred": Wake rcuo kthread later, first CB to empty list.
255 *	"WakeOvf": Wake rcuo kthread, CB list is huge.
256 *	"WakeOvfIsDeferred": Wake rcuo kthread later, CB list is huge.
257 *	"WakeNot": Don't wake rcuo kthread.
258 *	"WakeNotPoll": Don't wake rcuo kthread because it is polling.
259 *	"DeferredWake": Carried out the "IsDeferred" wakeup.
260 *	"Poll": Start of new polling cycle for rcu_nocb_poll.
261 *	"Sleep": Sleep waiting for CBs for !rcu_nocb_poll.
 
262 *	"WokeEmpty": rcuo kthread woke to find empty list.
263 *	"WokeNonEmpty": rcuo kthread woke to find non-empty list.
264 *	"WaitQueue": Enqueue partially done, timed wait for it to complete.
265 *	"WokeQueue": Partial enqueue now complete.
266 */
267TRACE_EVENT(rcu_nocb_wake,
268
269	TP_PROTO(const char *rcuname, int cpu, const char *reason),
270
271	TP_ARGS(rcuname, cpu, reason),
272
273	TP_STRUCT__entry(
274		__field(const char *, rcuname)
275		__field(int, cpu)
276		__field(const char *, reason)
277	),
278
279	TP_fast_assign(
280		__entry->rcuname = rcuname;
281		__entry->cpu = cpu;
282		__entry->reason = reason;
283	),
284
285	TP_printk("%s %d %s", __entry->rcuname, __entry->cpu, __entry->reason)
286);
 
287
288/*
289 * Tracepoint for tasks blocking within preemptible-RCU read-side
290 * critical sections.  Track the type of RCU (which one day might
291 * include SRCU), the grace-period number that the task is blocking
292 * (the current or the next), and the task's PID.
293 */
294TRACE_EVENT(rcu_preempt_task,
295
296	TP_PROTO(const char *rcuname, int pid, unsigned long gpnum),
297
298	TP_ARGS(rcuname, pid, gpnum),
299
300	TP_STRUCT__entry(
301		__field(const char *, rcuname)
302		__field(unsigned long, gpnum)
303		__field(int, pid)
304	),
305
306	TP_fast_assign(
307		__entry->rcuname = rcuname;
308		__entry->gpnum = gpnum;
309		__entry->pid = pid;
310	),
311
312	TP_printk("%s %lu %d",
313		  __entry->rcuname, __entry->gpnum, __entry->pid)
314);
315
316/*
317 * Tracepoint for tasks that blocked within a given preemptible-RCU
318 * read-side critical section exiting that critical section.  Track the
319 * type of RCU (which one day might include SRCU) and the task's PID.
320 */
321TRACE_EVENT(rcu_unlock_preempted_task,
322
323	TP_PROTO(const char *rcuname, unsigned long gpnum, int pid),
324
325	TP_ARGS(rcuname, gpnum, pid),
326
327	TP_STRUCT__entry(
328		__field(const char *, rcuname)
329		__field(unsigned long, gpnum)
330		__field(int, pid)
331	),
332
333	TP_fast_assign(
334		__entry->rcuname = rcuname;
335		__entry->gpnum = gpnum;
336		__entry->pid = pid;
337	),
338
339	TP_printk("%s %lu %d", __entry->rcuname, __entry->gpnum, __entry->pid)
340);
341
342/*
343 * Tracepoint for quiescent-state-reporting events.  These are
344 * distinguished by the type of RCU, the grace-period number, the
345 * mask of quiescent lower-level entities, the rcu_node structure level,
346 * the starting and ending CPU covered by the rcu_node structure, and
347 * whether there are any blocked tasks blocking the current grace period.
348 * All but the type of RCU are extracted from the rcu_node structure.
349 */
350TRACE_EVENT(rcu_quiescent_state_report,
351
352	TP_PROTO(const char *rcuname, unsigned long gpnum,
353		 unsigned long mask, unsigned long qsmask,
354		 u8 level, int grplo, int grphi, int gp_tasks),
355
356	TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
357
358	TP_STRUCT__entry(
359		__field(const char *, rcuname)
360		__field(unsigned long, gpnum)
361		__field(unsigned long, mask)
362		__field(unsigned long, qsmask)
363		__field(u8, level)
364		__field(int, grplo)
365		__field(int, grphi)
366		__field(u8, gp_tasks)
367	),
368
369	TP_fast_assign(
370		__entry->rcuname = rcuname;
371		__entry->gpnum = gpnum;
372		__entry->mask = mask;
373		__entry->qsmask = qsmask;
374		__entry->level = level;
375		__entry->grplo = grplo;
376		__entry->grphi = grphi;
377		__entry->gp_tasks = gp_tasks;
378	),
379
380	TP_printk("%s %lu %lx>%lx %u %d %d %u",
381		  __entry->rcuname, __entry->gpnum,
382		  __entry->mask, __entry->qsmask, __entry->level,
383		  __entry->grplo, __entry->grphi, __entry->gp_tasks)
384);
385
386/*
387 * Tracepoint for quiescent states detected by force_quiescent_state().
388 * These trace events include the type of RCU, the grace-period number
389 * that was blocked by the CPU, the CPU itself, and the type of quiescent
390 * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
391 * or "kick" when kicking a CPU that has been in dyntick-idle mode for
392 * too long.
393 */
394TRACE_EVENT(rcu_fqs,
395
396	TP_PROTO(const char *rcuname, unsigned long gpnum, int cpu, const char *qsevent),
397
398	TP_ARGS(rcuname, gpnum, cpu, qsevent),
399
400	TP_STRUCT__entry(
401		__field(const char *, rcuname)
402		__field(unsigned long, gpnum)
403		__field(int, cpu)
404		__field(const char *, qsevent)
405	),
406
407	TP_fast_assign(
408		__entry->rcuname = rcuname;
409		__entry->gpnum = gpnum;
410		__entry->cpu = cpu;
411		__entry->qsevent = qsevent;
412	),
413
414	TP_printk("%s %lu %d %s",
415		  __entry->rcuname, __entry->gpnum,
416		  __entry->cpu, __entry->qsevent)
417);
418
419#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) */
420
421/*
422 * Tracepoint for dyntick-idle entry/exit events.  These take a string
423 * as argument: "Start" for entering dyntick-idle mode, "End" for
424 * leaving it, "--=" for events moving towards idle, and "++=" for events
425 * moving away from idle.  "Error on entry: not idle task" and "Error on
426 * exit: not idle task" indicate that a non-idle task is erroneously
427 * toying with the idle loop.
428 *
429 * These events also take a pair of numbers, which indicate the nesting
430 * depth before and after the event of interest.  Note that task-related
431 * events use the upper bits of each number, while interrupt-related
432 * events use the lower bits.
 
433 */
434TRACE_EVENT(rcu_dyntick,
435
436	TP_PROTO(const char *polarity, long long oldnesting, long long newnesting),
437
438	TP_ARGS(polarity, oldnesting, newnesting),
439
440	TP_STRUCT__entry(
441		__field(const char *, polarity)
442		__field(long long, oldnesting)
443		__field(long long, newnesting)
 
444	),
445
446	TP_fast_assign(
447		__entry->polarity = polarity;
448		__entry->oldnesting = oldnesting;
449		__entry->newnesting = newnesting;
 
450	),
451
452	TP_printk("%s %llx %llx", __entry->polarity,
453		  __entry->oldnesting, __entry->newnesting)
454);
455
456/*
457 * Tracepoint for RCU preparation for idle, the goal being to get RCU
458 * processing done so that the current CPU can shut off its scheduling
459 * clock and enter dyntick-idle mode.  One way to accomplish this is
460 * to drain all RCU callbacks from this CPU, and the other is to have
461 * done everything RCU requires for the current grace period.  In this
462 * latter case, the CPU will be awakened at the end of the current grace
463 * period in order to process the remainder of its callbacks.
464 *
465 * These tracepoints take a string as argument:
466 *
467 *	"No callbacks": Nothing to do, no callbacks on this CPU.
468 *	"In holdoff": Nothing to do, holding off after unsuccessful attempt.
469 *	"Begin holdoff": Attempt failed, don't retry until next jiffy.
470 *	"Dyntick with callbacks": Entering dyntick-idle despite callbacks.
471 *	"Dyntick with lazy callbacks": Entering dyntick-idle w/lazy callbacks.
472 *	"More callbacks": Still more callbacks, try again to clear them out.
473 *	"Callbacks drained": All callbacks processed, off to dyntick idle!
474 *	"Timer": Timer fired to cause CPU to continue processing callbacks.
475 *	"Demigrate": Timer fired on wrong CPU, woke up correct CPU.
476 *	"Cleanup after idle": Idle exited, timer canceled.
477 */
478TRACE_EVENT(rcu_prep_idle,
479
480	TP_PROTO(const char *reason),
481
482	TP_ARGS(reason),
483
484	TP_STRUCT__entry(
485		__field(const char *, reason)
486	),
487
488	TP_fast_assign(
489		__entry->reason = reason;
490	),
491
492	TP_printk("%s", __entry->reason)
493);
494
495/*
496 * Tracepoint for the registration of a single RCU callback function.
497 * The first argument is the type of RCU, the second argument is
498 * a pointer to the RCU callback itself, the third element is the
499 * number of lazy callbacks queued, and the fourth element is the
500 * total number of callbacks queued.
501 */
502TRACE_EVENT(rcu_callback,
503
504	TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen_lazy,
505		 long qlen),
506
507	TP_ARGS(rcuname, rhp, qlen_lazy, qlen),
508
509	TP_STRUCT__entry(
510		__field(const char *, rcuname)
511		__field(void *, rhp)
512		__field(void *, func)
513		__field(long, qlen_lazy)
514		__field(long, qlen)
515	),
516
517	TP_fast_assign(
518		__entry->rcuname = rcuname;
519		__entry->rhp = rhp;
520		__entry->func = rhp->func;
521		__entry->qlen_lazy = qlen_lazy;
522		__entry->qlen = qlen;
523	),
524
525	TP_printk("%s rhp=%p func=%pf %ld/%ld",
526		  __entry->rcuname, __entry->rhp, __entry->func,
527		  __entry->qlen_lazy, __entry->qlen)
528);
529
530/*
531 * Tracepoint for the registration of a single RCU callback of the special
532 * kfree() form.  The first argument is the RCU type, the second argument
533 * is a pointer to the RCU callback, the third argument is the offset
534 * of the callback within the enclosing RCU-protected data structure,
535 * the fourth argument is the number of lazy callbacks queued, and the
536 * fifth argument is the total number of callbacks queued.
537 */
538TRACE_EVENT(rcu_kfree_callback,
539
540	TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset,
541		 long qlen_lazy, long qlen),
542
543	TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen),
544
545	TP_STRUCT__entry(
546		__field(const char *, rcuname)
547		__field(void *, rhp)
548		__field(unsigned long, offset)
549		__field(long, qlen_lazy)
550		__field(long, qlen)
551	),
552
553	TP_fast_assign(
554		__entry->rcuname = rcuname;
555		__entry->rhp = rhp;
556		__entry->offset = offset;
557		__entry->qlen_lazy = qlen_lazy;
558		__entry->qlen = qlen;
559	),
560
561	TP_printk("%s rhp=%p func=%ld %ld/%ld",
562		  __entry->rcuname, __entry->rhp, __entry->offset,
563		  __entry->qlen_lazy, __entry->qlen)
564);
565
566/*
567 * Tracepoint for marking the beginning rcu_do_batch, performed to start
568 * RCU callback invocation.  The first argument is the RCU flavor,
569 * the second is the number of lazy callbacks queued, the third is
570 * the total number of callbacks queued, and the fourth argument is
571 * the current RCU-callback batch limit.
572 */
573TRACE_EVENT(rcu_batch_start,
574
575	TP_PROTO(const char *rcuname, long qlen_lazy, long qlen, long blimit),
576
577	TP_ARGS(rcuname, qlen_lazy, qlen, blimit),
578
579	TP_STRUCT__entry(
580		__field(const char *, rcuname)
581		__field(long, qlen_lazy)
582		__field(long, qlen)
583		__field(long, blimit)
584	),
585
586	TP_fast_assign(
587		__entry->rcuname = rcuname;
588		__entry->qlen_lazy = qlen_lazy;
589		__entry->qlen = qlen;
590		__entry->blimit = blimit;
591	),
592
593	TP_printk("%s CBs=%ld/%ld bl=%ld",
594		  __entry->rcuname, __entry->qlen_lazy, __entry->qlen,
595		  __entry->blimit)
596);
597
598/*
599 * Tracepoint for the invocation of a single RCU callback function.
600 * The first argument is the type of RCU, and the second argument is
601 * a pointer to the RCU callback itself.
602 */
603TRACE_EVENT(rcu_invoke_callback,
604
605	TP_PROTO(const char *rcuname, struct rcu_head *rhp),
606
607	TP_ARGS(rcuname, rhp),
608
609	TP_STRUCT__entry(
610		__field(const char *, rcuname)
611		__field(void *, rhp)
612		__field(void *, func)
613	),
614
615	TP_fast_assign(
616		__entry->rcuname = rcuname;
617		__entry->rhp = rhp;
618		__entry->func = rhp->func;
619	),
620
621	TP_printk("%s rhp=%p func=%pf",
622		  __entry->rcuname, __entry->rhp, __entry->func)
623);
624
625/*
626 * Tracepoint for the invocation of a single RCU callback of the special
627 * kfree() form.  The first argument is the RCU flavor, the second
628 * argument is a pointer to the RCU callback, and the third argument
629 * is the offset of the callback within the enclosing RCU-protected
630 * data structure.
631 */
632TRACE_EVENT(rcu_invoke_kfree_callback,
633
634	TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset),
635
636	TP_ARGS(rcuname, rhp, offset),
637
638	TP_STRUCT__entry(
639		__field(const char *, rcuname)
640		__field(void *, rhp)
641		__field(unsigned long, offset)
642	),
643
644	TP_fast_assign(
645		__entry->rcuname = rcuname;
646		__entry->rhp = rhp;
647		__entry->offset	= offset;
648	),
649
650	TP_printk("%s rhp=%p func=%ld",
651		  __entry->rcuname, __entry->rhp, __entry->offset)
652);
653
654/*
655 * Tracepoint for exiting rcu_do_batch after RCU callbacks have been
656 * invoked.  The first argument is the name of the RCU flavor,
657 * the second argument is number of callbacks actually invoked,
658 * the third argument (cb) is whether or not any of the callbacks that
659 * were ready to invoke at the beginning of this batch are still
660 * queued, the fourth argument (nr) is the return value of need_resched(),
661 * the fifth argument (iit) is 1 if the current task is the idle task,
662 * and the sixth argument (risk) is the return value from
663 * rcu_is_callbacks_kthread().
664 */
665TRACE_EVENT(rcu_batch_end,
666
667	TP_PROTO(const char *rcuname, int callbacks_invoked,
668		 char cb, char nr, char iit, char risk),
669
670	TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
671
672	TP_STRUCT__entry(
673		__field(const char *, rcuname)
674		__field(int, callbacks_invoked)
675		__field(char, cb)
676		__field(char, nr)
677		__field(char, iit)
678		__field(char, risk)
679	),
680
681	TP_fast_assign(
682		__entry->rcuname = rcuname;
683		__entry->callbacks_invoked = callbacks_invoked;
684		__entry->cb = cb;
685		__entry->nr = nr;
686		__entry->iit = iit;
687		__entry->risk = risk;
688	),
689
690	TP_printk("%s CBs-invoked=%d idle=%c%c%c%c",
691		  __entry->rcuname, __entry->callbacks_invoked,
692		  __entry->cb ? 'C' : '.',
693		  __entry->nr ? 'S' : '.',
694		  __entry->iit ? 'I' : '.',
695		  __entry->risk ? 'R' : '.')
696);
697
698/*
699 * Tracepoint for rcutorture readers.  The first argument is the name
700 * of the RCU flavor from rcutorture's viewpoint and the second argument
701 * is the callback address.  The third argument is the start time in
702 * seconds, and the last two arguments are the grace period numbers
703 * at the beginning and end of the read, respectively.  Note that the
704 * callback address can be NULL.
705 */
706TRACE_EVENT(rcu_torture_read,
 
707
708	TP_PROTO(const char *rcutorturename, struct rcu_head *rhp,
709		 unsigned long secs, unsigned long c_old, unsigned long c),
710
711	TP_ARGS(rcutorturename, rhp, secs, c_old, c),
712
713	TP_STRUCT__entry(
714		__field(const char *, rcutorturename)
715		__field(struct rcu_head *, rhp)
716		__field(unsigned long, secs)
717		__field(unsigned long, c_old)
718		__field(unsigned long, c)
719	),
720
721	TP_fast_assign(
722		__entry->rcutorturename = rcutorturename;
 
 
723		__entry->rhp = rhp;
724		__entry->secs = secs;
725		__entry->c_old = c_old;
726		__entry->c = c;
727	),
728
729	TP_printk("%s torture read %p %luus c: %lu %lu",
730		  __entry->rcutorturename, __entry->rhp,
731		  __entry->secs, __entry->c_old, __entry->c)
732);
733
734/*
735 * Tracepoint for _rcu_barrier() execution.  The string "s" describes
736 * the _rcu_barrier phase:
737 *	"Begin": _rcu_barrier() started.
738 *	"EarlyExit": _rcu_barrier() piggybacked, thus early exit.
739 *	"Inc1": _rcu_barrier() piggyback check counter incremented.
740 *	"OfflineNoCB": _rcu_barrier() found callback on never-online CPU
741 *	"OnlineNoCB": _rcu_barrier() found online no-CBs CPU.
742 *	"OnlineQ": _rcu_barrier() found online CPU with callbacks.
743 *	"OnlineNQ": _rcu_barrier() found online CPU, no callbacks.
744 *	"IRQ": An rcu_barrier_callback() callback posted on remote CPU.
 
745 *	"CB": An rcu_barrier_callback() invoked a callback, not the last.
746 *	"LastCB": An rcu_barrier_callback() invoked the last callback.
747 *	"Inc2": _rcu_barrier() piggyback check counter incremented.
748 * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument
749 * is the count of remaining callbacks, and "done" is the piggybacking count.
750 */
751TRACE_EVENT(rcu_barrier,
752
753	TP_PROTO(const char *rcuname, const char *s, int cpu, int cnt, unsigned long done),
754
755	TP_ARGS(rcuname, s, cpu, cnt, done),
756
757	TP_STRUCT__entry(
758		__field(const char *, rcuname)
759		__field(const char *, s)
760		__field(int, cpu)
761		__field(int, cnt)
762		__field(unsigned long, done)
763	),
764
765	TP_fast_assign(
766		__entry->rcuname = rcuname;
767		__entry->s = s;
768		__entry->cpu = cpu;
769		__entry->cnt = cnt;
770		__entry->done = done;
771	),
772
773	TP_printk("%s %s cpu %d remaining %d # %lu",
774		  __entry->rcuname, __entry->s, __entry->cpu, __entry->cnt,
775		  __entry->done)
776);
777
778#else /* #ifdef CONFIG_RCU_TRACE */
779
780#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
781#define trace_rcu_future_grace_period(rcuname, gpnum, completed, c, \
782				      level, grplo, grphi, event) \
783				      do { } while (0)
784#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, \
785				    qsmask) do { } while (0)
786#define trace_rcu_exp_grace_period(rcuname, gqseq, gpevent) \
787	do { } while (0)
788#define trace_rcu_exp_funnel_lock(rcuname, level, grplo, grphi, gpevent) \
789	do { } while (0)
790#define trace_rcu_nocb_wake(rcuname, cpu, reason) do { } while (0)
791#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)
792#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)
793#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, \
794					 grplo, grphi, gp_tasks) do { } \
795	while (0)
796#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)
797#define trace_rcu_dyntick(polarity, oldnesting, newnesting) do { } while (0)
798#define trace_rcu_prep_idle(reason) do { } while (0)
799#define trace_rcu_callback(rcuname, rhp, qlen_lazy, qlen) do { } while (0)
800#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen_lazy, qlen) \
801	do { } while (0)
802#define trace_rcu_batch_start(rcuname, qlen_lazy, qlen, blimit) \
803	do { } while (0)
804#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
805#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
806#define trace_rcu_batch_end(rcuname, callbacks_invoked, cb, nr, iit, risk) \
807	do { } while (0)
808#define trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
809	do { } while (0)
810#define trace_rcu_barrier(name, s, cpu, cnt, done) do { } while (0)
811
812#endif /* #else #ifdef CONFIG_RCU_TRACE */
813
814#endif /* _TRACE_RCU_H */
815
816/* This part must be outside protection */
817#include <trace/define_trace.h>