Linux性能调试工具之ftrace
1 SA8155 ftrace
1.1 signal知识
shell $?返回值大于128时，表示该脚本收到了signal。
1.2 ftrace-cmd
make LDFLAGS=-static \
CC=aarch64-linux-gnu-gcc trace-cmd
1.3 kill ftrace
1.3.1 enable
echo nop > \
/sys/kernel/debug/tracing/current_tracer
echo syscalls:sys_enter_kill >> \
/sys/kernel/debug/tracing/set_event
echo syscalls:sys_exit_kill >> \
/sys/kernel/debug/tracing/set_event
echo 1 > \
/sys/kernel/debug/tracing/tracing_on
cat /sys/kernel/debug/tracing/trace_pipe
1.3.2 disable
echo 0 > \
/sys/kernel/debug/tracing/tracing_on
echo 0 > \
/sys/kernel/debug/tracing/events/syscalls/sys_exit_kill/enable
echo 0 > \
/sys/kernel/debug/tracing/events/syscalls/sys_enter_kill/enable
echo nop > \
/sys/kernel/debug/tracing/current_tracer
1.4 my_func ftrace
echo 0 > \
/sys/kernel/debug/tracing/tracing_on
echo 0 > \
/sys/kernel/debug/tracing/trace
echo function_graph > \
/sys/kernel/debug/tracing/current_tracer
echo funcgraph-proc > \
/sys/kernel/debug/tracing/trace_options
echo funcgraph-abstime > \
/sys/kernel/debug/tracing/trace_options
echo "my_func1 my_func2" > \
/sys/kernel/debug/tracing/set_ftrace_filter
echo 1 > \
/sys/kernel/debug/tracing/tracing_on
echo 0 > /sys/kernel/debug/tracing/tracing_on
cat /sys/kernel/debug/tracing/trace > \
my_trace_result
1.5 sched_switch
1.5.1 sched_switch
insmod driver, then run ftrace script, this sequence could ensure no latency issue according to my observation.
sched_migrate_task: /proc/sys/kernel/sched_nr_migrate
sched_switch: __schedule()
sched_wakeup: try_to_wake_up()
sched_wakeup_new: for new forked task
# per cpu buffer
echo 10000 > \
/sys/kernel/debug/tracing/buffer_size_kb
# echo "(prev_pid == 1 || next_pid == 1)" > \
# /sys/kernel/debug/tracing/events/sched/sched_switch/filter
# echo 1 > \
# /sys/kernel/debug/tracing/events/irq/irq_handler_entry/enable
# echo 1 > \
# /sys/kernel/debug/tracing/events/irq/irq_handler_exit/enable
echo 1 > \
/sys/kernel/debug/tracing/events/sched/sched_switch/enable
echo 1 > \
/sys/kernel/debug/tracing/events/sched/sched_wakeup_new/enable
cat /sys/kernel/debug/tracing/set_event
echo 1 > \
/sys/kernel/debug/tracing/tracing_on
cat /sys/kernel/debug/tracing/trace
1.5.2 scheduling latency time_diff.py
# cur_line: [CPU number], timestamp
import io
import re
import sys,os
import shutil
arg0_proc_name = ''
def print_usage():
print('\nUsage: python ' + arg0_proc_name +
' <file> <match_string>')
def find_string(src_file,
match_string):
last_line_cnt = 0
cur_line_cnt = 0
last_ts = 0
cur_ts = 0
last_string = ''
num_of_intr = 0
fp = io.open(src_file, 'r',
encoding='utf-8',
errors='ignore')
for cur_string in fp:
cur_line_cnt += 1
if re.findall(match_string, cur_string):
list = cur_string.split()
tmp = float(list[3].rstrip(':'))
cur_ts = int(tmp * 1000)
delta = abs(cur_ts - last_ts)
new_match_string = "next_comm=" +\
match_string
if (delta > 5 and
re.findall(new_match_string,
cur_string)):
print('++++ delta = ' +
str(delta) + 'ms, ' +
'num_of_intr = ' + str(num_of_intr))
print(str(last_line_cnt) + ': ' +
last_string)
print(str(cur_line_cnt) + ': ' +
cur_string)
last_ts = cur_ts
last_line_cnt = cur_line_cnt
last_string = cur_string
num_of_intr = 0
elif re.findall("irq_handler_exit", cur_string):
num_of_intr = num_of_intr + 1
fp.close()
def main():
global arg0_proc_name
arg0_proc_name = sys.argv[0]
if sys.argv[0].rfind(os.path.sep) > 0 :
index = sys.argv[0].rfind(os.path.sep)
arg0_proc_name = sys.argv[0][index+1:]
if len(sys.argv) < 3:
print_usage()
sys.exit(0)
find_string(sys.argv[1], sys.argv[2])
if __name__ == '__main__':
main()
1.5.3 disable cpuidle
/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
i=0
while [ $i -le 7 ]; do
echo 1 > /sys/devices/system/cpu/cpu$i/cpuidle/state0/disable
echo 1 > /sys/devices/system/cpu/cpu$i/cpuidle/state1/disable
i=$((i + 1))
done
1.6 perfetto
perfetto: English Perfect
stress-ng: a next generation tool designed to stress test system hardware
https://ui.perfetto.dev/#!/viewer
tracebox -o /tmp/trace.pf --txt -c /root/xxx.conf
键盘上小写字母A和D用来左右移动，键盘上小写字母W和S用来放大和缩写。
2 htop
ncurses_5.6
./configure \
--host=x86_64-unknown-linux-gnu \
--without-trace \
--without-cxx \
--without-cxx-binding \
--without-ada \
--without-manpages \
--without-progs \
--without-tests \
--with-shared
htop
./configure --disable-unicode \
--host=aarch64-poky-linux
3 ltrace
3.1 Android
https://github.com/LineageOS/android_external_ltrace
3.2 ltrace with Linaro build
3.2.1 zlib
http://www.zlib.net/fossils/
./configure \
--prefix=/path/to/zlib-1.2.11/aarch64
# in Makefile change gcc to
# aarch64-linux-gnu-gcc
make
make install
3.2.2 elfutils
https://sourceware.org/elfutils/ftp/0.178/
LDFLAGS='-L/path/to/zlib-1.2.11/aarch64/lib' \
./configure \
--disable-debuginfod \
--host=aarch64-linux-gnu \
--prefix=/path/to/elfutils-0.178/aarch64
make \
CFLAGS+=\
'-I/path/to/zlib-1.2.11/aarch64/include' \
LDFLAGS+=\
'-Wl,-rpath-link,/path/to/zlib-1.2.11/aarch64/lib' \
LDFLAGS+=\
'-L/path/to/zlib-1.2.11/aarch64/lib'
make install
3.2.3 ltrace
https://github.com/dkogan/ltrace
Do not download ltrace 0.7.3 from http://www.ltrace.org, it is too old, does not support aarch64.
./autogen.sh
CFLAGS=\
'-I/path/to/elfutils-0.178/aarch64/include' \
LDFLAGS=\
'-Wl,-rpath-link,/path/to/zlib-1.2.11/aarch64/lib -L/path/to/elfutils-0.178/aarch64/lib' \
./configure \
--host=aarch64-linux-gnu
[10-Jan-2022] add -ldw, -lelf, -lbz2, -ldl, -llzma, -lpthread for Android 8.0 x86_64.
make \
CFLAGS+=\
'-I/path/to/elfutils-0.178/aarch64/include' \
LDFLAGS+=\
'-L/path/to/elfutils-0.178/aarch64/lib -ldw -lelf' \
CFLAGS+=\
'-I/path/to/zlib-1.2.11/aarch64/include' \
LDFLAGS+=\
'-L/path/to/zlib-1.2.11/aarch64/lib -lz' \
LDFLAGS+='-lbz2' \
LDFLAGS+='-ldl' \
LDFLAGS+='-llzma' \
LDFLAGS+='-lpthread'
3.2.4 static compilation of ltrace
add -all-static to LINK for libtool in Makefile, shall add it before -o $@, as shown below.
LINK=xxx \
-all-static -o $@
4 strace
4.1 aarch64-linux
./configure --enable-mpers=no \
--host=aarch64-linux
4.2 strace
strace -e epoll_wait -x -p $PID
strace -p $PID -tt -T -s 128 -x 2>&1 | \
grep "read(17"
1) sync time to PC
date -s "2020-02-02 16:02:15"
2) only capture SIGALRM
strace -e signal=SIGALRM \
-e trace=none -tt -p <$PID>
3) capture ioctl
strace -e signal=none \
-e trace=ioctl -tt -p <$PID>
4) capture all strace log
strace -tt -p <$PID>
4.3 How To Enable Userspace Dump Stack
prctl(PR_SET_DUMPABLE, 1);
add this code to rild.
需要添加的sepolicy权限如下：
allow xxx self:capability dac_override;
5 systemd
5.1 code
PulseAudio, systemd suite: by Lennart Poettering
https://github.com/systemd/systemd/releases
systemd-244/src/network/networkd.c
5.2 systemd-networkd
systemd-networkd
high priority: /etc/systemd/network
mid priority: /run/systemd/network
low priority: /usr/lib/systemd/network
systemd-networkd-wait-online
/lib/systemd/system
/lib/systemd/system/systemd-networkd.service
[Service]
ExecStartPre=+/bin/sh /usr/bin/xx.sh
Environment=SYSTEMD_LOG_LEVEL=debug
journalctl -b -u systemd-networkd > \
/tmp/networkd.log
5.3 systemd-analyze
opkg install systemd-analyze.ipk
opkg remove systemd-analyze
ipk: Itsy Package
systemd-analyze blame
systemd-analyze critical-chain \
xxx.service
@后面的时刻表示该单元的启动时刻；+后面的时长表示该单元总计花了多长时间才完成启动。
systemd-analyze plot > boot.svg
systemctl list-dependencies xxx.service
6 Abbreviations
OPP: Operating Performance Points
RBCPR: RapidBridge Core Power Reduction