1#!/bin/bash
  2# SPDX-License-Identifier: GPL-2.0
  3#
  4# Here's how to use this:
  5#
  6# This script is used to help find functions that are being traced by function
  7# tracer or function graph tracing that causes the machine to reboot, hang, or
  8# crash. Here's the steps to take.
  9#
 10# First, determine if function tracing is working with a single function:
 11#
 12#   (note, if this is a problem with function_graph tracing, then simply
 13#    replace "function" with "function_graph" in the following steps).
 14#
 15#  # cd /sys/kernel/tracing
 16#  # echo schedule > set_ftrace_filter
 17#  # echo function > current_tracer
 18#
 19# If this works, then we know that something is being traced that shouldn't be.
 20#
 21#  # echo nop > current_tracer
 22#
 23# Starting with v5.1 this can be done with numbers, making it much faster:
 24#
 25# The old (slow) way, for kernels before v5.1.
 26#
 27# [old-way] # cat available_filter_functions > ~/full-file
 28#
 29# [old-way] *** Note ***  this process will take several minutes to update the
 30# [old-way] filters. Setting multiple functions is an O(n^2) operation, and we
 31# [old-way] are dealing with thousands of functions. So go have coffee, talk
 32# [old-way] with your coworkers, read facebook. And eventually, this operation
 33# [old-way] will end.
 34#
 35# The new way (using numbers) is an O(n) operation, and usually takes less than a second.
 36#
 37# seq `wc -l available_filter_functions | cut -d' ' -f1` > ~/full-file
 38#
 39# This will create a sequence of numbers that match the functions in
 40# available_filter_functions, and when echoing in a number into the
 41# set_ftrace_filter file, it will enable the corresponding function in
 42# O(1) time. Making enabling all functions O(n) where n is the number of
 43# functions to enable.
 44#
 45# For either the new or old way, the rest of the operations remain the same.
 46#
 47#  # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
 48#  # cat ~/test-file > set_ftrace_filter
 49#
 
 
 
 
 
 50#  # echo function > current_tracer
 51#
 52# If it crashes, we know that ~/test-file has a bad function.
 53#
 54#   Reboot back to test kernel.
 55#
 56#     # cd /sys/kernel/tracing
 57#     # mv ~/test-file ~/full-file
 58#
 59# If it didn't crash.
 60#
 61#     # echo nop > current_tracer
 62#     # mv ~/non-test-file ~/full-file
 63#
 64# Get rid of the other test file from previous run (or save them off somewhere).
 65#  # rm -f ~/test-file ~/non-test-file
 66#
 67# And start again:
 68#
 69#  # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
 70#
 71# The good thing is, because this cuts the number of functions in ~/test-file
 72# by half, the cat of it into set_ftrace_filter takes half as long each
 73# iteration, so don't talk so much at the water cooler the second time.
 74#
 75# Eventually, if you did this correctly, you will get down to the problem
 76# function, and all we need to do is to notrace it.
 77#
 78# The way to figure out if the problem function is bad, just do:
 79#
 80#  # echo <problem-function> > set_ftrace_notrace
 81#  # echo > set_ftrace_filter
 82#  # echo function > current_tracer
 83#
 84# And if it doesn't crash, we are done.
 85#
 86# If it does crash, do this again (there's more than one problem function)
 87# but you need to echo the problem function(s) into set_ftrace_notrace before
 88# enabling function tracing in the above steps. Or if you can compile the
 89# kernel, annotate the problem functions with "notrace" and start again.
 90#
 91
 92
 93if [ $# -ne 3 ]; then
 94  echo 'usage: ftrace-bisect full-file test-file  non-test-file'
 95  exit
 96fi
 97
 98full=$1
 99test=$2
100nontest=$3
101
102x=`cat $full | wc -l`
103if [ $x -eq 1 ]; then
104	echo "There's only one function left, must be the bad one"
105	cat $full
106	exit 0
107fi
108
109let x=$x/2
110let y=$x+1
111
112if [ ! -f $full ]; then
113	echo "$full does not exist"
114	exit 1
115fi
116
117if [ -f $test ]; then
118	echo -n "$test exists, delete it? [y/N]"
119	read a
120	if [ "$a" != "y" -a "$a" != "Y" ]; then
121		exit 1
122	fi
123fi
124
125if [ -f $nontest ]; then
126	echo -n "$nontest exists, delete it? [y/N]"
127	read a
128	if [ "$a" != "y" -a "$a" != "Y" ]; then
129		exit 1
130	fi
131fi
132
133sed -ne "1,${x}p" $full > $test
134sed -ne "$y,\$p" $full > $nontest

  1#!/bin/bash
  2# SPDX-License-Identifier: GPL-2.0
  3#
  4# Here's how to use this:
  5#
  6# This script is used to help find functions that are being traced by function
  7# tracer or function graph tracing that causes the machine to reboot, hang, or
  8# crash. Here's the steps to take.
  9#
 10# First, determine if function tracing is working with a single function:
 11#
 12#   (note, if this is a problem with function_graph tracing, then simply
 13#    replace "function" with "function_graph" in the following steps).
 14#
 15#  # cd /sys/kernel/debug/tracing
 16#  # echo schedule > set_ftrace_filter
 17#  # echo function > current_tracer
 18#
 19# If this works, then we know that something is being traced that shouldn't be.
 20#
 21#  # echo nop > current_tracer
 22#
 23#  # cat available_filter_functions > ~/full-file
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 24#  # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
 25#  # cat ~/test-file > set_ftrace_filter
 26#
 27# *** Note *** this will take several minutes. Setting multiple functions is
 28# an O(n^2) operation, and we are dealing with thousands of functions. So go
 29# have  coffee, talk with your coworkers, read facebook. And eventually, this
 30# operation will end.
 31#
 32#  # echo function > current_tracer
 33#
 34# If it crashes, we know that ~/test-file has a bad function.
 35#
 36#   Reboot back to test kernel.
 37#
 38#     # cd /sys/kernel/debug/tracing
 39#     # mv ~/test-file ~/full-file
 40#
 41# If it didn't crash.
 42#
 43#     # echo nop > current_tracer
 44#     # mv ~/non-test-file ~/full-file
 45#
 46# Get rid of the other test file from previous run (or save them off somewhere).
 47#  # rm -f ~/test-file ~/non-test-file
 48#
 49# And start again:
 50#
 51#  # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
 52#
 53# The good thing is, because this cuts the number of functions in ~/test-file
 54# by half, the cat of it into set_ftrace_filter takes half as long each
 55# iteration, so don't talk so much at the water cooler the second time.
 56#
 57# Eventually, if you did this correctly, you will get down to the problem
 58# function, and all we need to do is to notrace it.
 59#
 60# The way to figure out if the problem function is bad, just do:
 61#
 62#  # echo <problem-function> > set_ftrace_notrace
 63#  # echo > set_ftrace_filter
 64#  # echo function > current_tracer
 65#
 66# And if it doesn't crash, we are done.
 67#
 68# If it does crash, do this again (there's more than one problem function)
 69# but you need to echo the problem function(s) into set_ftrace_notrace before
 70# enabling function tracing in the above steps. Or if you can compile the
 71# kernel, annotate the problem functions with "notrace" and start again.
 72#
 73
 74
 75if [ $# -ne 3 ]; then
 76  echo 'usage: ftrace-bisect full-file test-file  non-test-file'
 77  exit
 78fi
 79
 80full=$1
 81test=$2
 82nontest=$3
 83
 84x=`cat $full | wc -l`
 85if [ $x -eq 1 ]; then
 86	echo "There's only one function left, must be the bad one"
 87	cat $full
 88	exit 0
 89fi
 90
 91let x=$x/2
 92let y=$x+1
 93
 94if [ ! -f $full ]; then
 95	echo "$full does not exist"
 96	exit 1
 97fi
 98
 99if [ -f $test ]; then
100	echo -n "$test exists, delete it? [y/N]"
101	read a
102	if [ "$a" != "y" -a "$a" != "Y" ]; then
103		exit 1
104	fi
105fi
106
107if [ -f $nontest ]; then
108	echo -n "$nontest exists, delete it? [y/N]"
109	read a
110	if [ "$a" != "y" -a "$a" != "Y" ]; then
111		exit 1
112	fi
113fi
114
115sed -ne "1,${x}p" $full > $test
116sed -ne "$y,\$p" $full > $nontest