#!/bin/bash # # tpoint - trace a given tracepoint. Static tracing. # Written using Linux ftrace. # # This will enable a given tracepoint, print events, then disable the tracepoint # when the program ends. This is like a simple version of the "perf" command for # printing live tracepoint events only. Wildcards are currently not supported. # If this is insufficient for any reason, use the perf command instead. # # USAGE: ./tpoint [-hHsv] [-d secs] [-p pid] [-L tid] tracepoint [filter] # ./tpoint -l # # Run "tpoint -h" for full usage. # # I wrote this because I often needed a quick way to dump stack traces for a # given tracepoint. # # OVERHEADS: Relative to the frequency of traced events. You might want to # check their frequency beforehand, using perf_events. Eg: # # perf stat -e block:block_rq_issue -a sleep 5 # # To count occurrences of that tracepoint for 5 seconds. # # REQUIREMENTS: FTRACE and tracepoints, which you may already have on recent # kernel versions. # # From perf-tools: https://github.com/brendangregg/perf-tools # # See the tpoint(8) man page (in perf-tools) for more info. # # COPYRIGHT: Copyright (c) 2014 Brendan Gregg. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # # (http://www.gnu.org/copyleft/gpl.html) # # 22-Jul-2014 Brendan Gregg Created this. ### default variables setglobal tracing = '/sys/kernel/debug/tracing' setglobal flock = '/var/tmp/.ftrace-lock'; setglobal wroteflock = '0' setglobal opt_duration = '0'; setglobal duration = ''; setglobal opt_pid = '0'; setglobal pid = ''; setglobal opt_tid = '0'; setglobal tid = '' setglobal opt_filter = '0'; setglobal filter = ''; setglobal opt_view = '0'; setglobal opt_headers = '0'; setglobal opt_stack = '0'; setglobal dmesg = '2' trap ':' INT QUIT TERM PIPE HUP # sends execution to end tracing section proc usage { cat << """ >&2 USAGE: tpoint [-hHsv] [-d secs] [-p PID] [-L TID] tracepoint [filter] tpoint -l -d seconds # trace duration, and use buffers -p PID # PID to match on event -L TID # thread id to match on event -v # view format file (don't trace) -H # include column headers -l # list all tracepoints -s # show kernel stack traces -h # this usage message eg, tpoint -l | grep open # find tracepoints containing "open" tpoint syscalls:sys_enter_open # trace open() syscall entry tpoint block:block_rq_issue # trace block I/O issue tpoint -s block:block_rq_issue # show kernel stacks See the man page and example file for more info. """ > !2 USAGE: tpoint [-hHsv] [-d secs] [-p PID] [-L TID] tracepoint [filter] tpoint -l -d seconds # trace duration, and use buffers -p PID # PID to match on event -L TID # thread id to match on event -v # view format file (don't trace) -H # include column headers -l # list all tracepoints -s # show kernel stack traces -h # this usage message eg, tpoint -l | grep open # find tracepoints containing "open" tpoint syscalls:sys_enter_open # trace open() syscall entry tpoint block:block_rq_issue # trace block I/O issue tpoint -s block:block_rq_issue # show kernel stacks See the man page and example file for more info. END exit } proc warn { if ! eval @Argv { echo >&2 "WARNING: command failed \"$ifsjoin(Argv)> !2 "WARNING: command failed \"$@\"" } } proc end { # disable tracing echo !2 >/dev/null echo "Ending tracing..." !2 >/dev/null cd $tracing warn "echo 0 > $tdir/enable" if sh-expr ' opt_filter ' { warn "echo 0 > $tdir/filter" } sh-expr ' opt_stack ' && warn "echo 0 > options/stacktrace" warn "echo > trace" sh-expr ' wroteflock ' && warn "rm $flock" } proc die { echo >&2 @Argv> !2 "$@" exit 1 } proc edie { # die with a quiet end() echo >&2 @Argv> !2 "$@" exec >/dev/null !2 > !1 end exit 1 } ### process options while getopts d:hHlp:L:sv opt { match $opt { with d setglobal opt_duration = '1'; setglobal duration = $OPTARG with p setglobal opt_pid = '1'; setglobal pid = $OPTARG with L setglobal opt_tid = '1'; setglobal tid = $OPTARG with H setglobal opt_headers = '1' with l setglobal opt_list = '1' with s setglobal opt_stack = '1' with v setglobal opt_view = '1' with h|? usage } } if sh-expr ' !opt_list ' { shift $shExpr(' $OPTIND - 1 ') sh-expr ' $# ' || usage setglobal tpoint = $1 shift if sh-expr ' $# ' { setglobal opt_filter = '1' setglobal filter = $1 } } ### option logic sh-expr ' opt_pid + opt_filter + opt_tid > 1 ' && \ die "ERROR: use at most one of -p, -L, or filter." sh-expr ' opt_duration && opt_view ' && die "ERROR: use either -d or -v." if sh-expr ' opt_pid ' { # convert to filter setglobal opt_filter = '1' # ftrace common_pid is thread id from user's perspective for tid in [/proc/$pid/task/*] { setglobal filter = ""$filter || common_pid == $(tid##*/)"" } setglobal filter = $(filter:3) # trim leading ' || ' (four characters) } if sh-expr ' opt_tid ' { setglobal opt_filter = '1' setglobal filter = ""common_pid == $tid"" } if sh-expr ' !opt_view && !opt_list ' { if sh-expr ' opt_duration ' { echo "Tracing $tpoint for $duration seconds (buffered)..." } else { echo "Tracing $tpoint. Ctrl-C to end." } } ### check permissions cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE? debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)" ### do list tracepoints if sh-expr ' opt_list ' { cd events for tp in [*/*] { # skip filter/enable files [[ -f $tp ]] && continue echo $(tp/\//:) } exit } ### check tracepoints setglobal tdir = "events/$(tpoint/:/\/)" [[ -e $tdir ]] || die "ERROR: tracepoint $tpoint not found. Exiting" ### view if sh-expr ' opt_view ' { cat $tdir/format exit } ### ftrace lock [[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $[cat $flock] $flock" echo $Pid > $flock || die "ERROR: unable to write $flock." setglobal wroteflock = '1' ### setup and begin tracing echo nop > current_tracer if sh-expr ' opt_filter ' { if ! echo $filter > $tdir/filter { edie "ERROR: setting filter or -p. Exiting." } } if sh-expr ' opt_stack ' { if ! echo 1 > options/stacktrace { edie "ERROR: enabling stack traces (-s). Exiting" } } if ! echo 1 >> $tdir/enable { edie "ERROR: enabling tracepoint $tprobe. Exiting." } ### print trace buffer warn "echo > trace" if sh-expr ' opt_duration ' { sleep $duration if sh-expr ' opt_headers ' { cat trace } else { grep -v '^#' trace } } else { # trace_pipe lack headers, so fetch them from trace sh-expr ' opt_headers ' && cat trace cat trace_pipe } ### end tracing end