Ftrace - function tracer

ftrace: /sys/kernel/debug/tracing
Ring buffer: /sys/kernel/debug/tracing/trace
debugfs:
$ mount | grep debugfs
debugfs on /sys/kernel/debug type debugfs (rw,relatime)

/sys/kernel/debug/tracing/
├── available_events
├── available_filter_functions
├── available_tracers
├── buffer_size_kb
├── buffer_total_size_kb
├── current_tracer
├── dyn_ftrace_total_info
├── enabled_functions
├── events
├── free_buffer
├── function_profile_enabled
├── hwlat_detector
├── instances
├── kprobe_events
├── kprobe_profile
├── max_graph_depth
├── options
├── per_cpu
├── printk_formats
├── README
├── saved_cmdlines
├── saved_cmdlines_size
├── saved_tgids
├── set_event
├── set_event_pid
├── set_ftrace_filter
├── set_ftrace_notrace
├── set_ftrace_pid
├── set_graph_function
├── set_graph_notrace
├── snapshot
├── stack_max_size
├── stack_trace
├── stack_trace_filter
├── trace
├── trace_clock
├── trace_marker
├── trace_marker_raw
├── trace_options
├── trace_pipe
├── trace_stat
├── tracing_cpumask
├── tracing_max_latency
├── tracing_on
├── tracing_thresh
├── uprobe_events
└── uprobe_profile

root@instance-1:/sys/kernel/debug/tracing# cat current_tracer
nop
root@instance-1:/sys/kernel/debug/tracing# echo function > current_tracer

root@instance-1:/sys/kernel/debug/tracing# cat available_tracers
hwlat blk mmiotrace function_graph wakeup_dl wakeup_rt wakeup function nop
root@instance-1:/sys/kernel/debug/tracing# cat set_graph_function
#### all functions enabled ####
root@instance-1:/sys/kernel/debug/tracing# cat buffer_size_kb
7 (expanded: 1408)

1. event
available_events: A list of events that can be enabled in tracing.
set_event: By echoing in the event into this file, will enable that event.
# cat available_events
net:netif_rx_ni_entry
net:netif_rx_entry
net:netif_receive_skb_entry
net:napi_gro_receive_entry
net:napi_gro_frags_entry
net:netif_rx
net:netif_receive_skb
net:net_dev_queue
net:net_dev_xmit
net:net_dev_start_xmit
skb:skb_copy_datagram_iovec
skb:consume_skb
skb:kfree_skb
syscalls:sys_exit_socket
syscalls:sys_enter_socket

# echo 0 > tracing_on
# echo net:netif_receive_skb > set_event
# echo 1 > tracing_on
# cat trace_pipe
# cat trace
# tracer: nop
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          <idle>-0     [000] ..s.  4175.515810: netif_receive_skb: dev=ens4 skbaddr=ffff91034b184900 len=52
          <idle>-0     [000] ..s.  4175.556855: netif_receive_skb: dev=ens4 skbaddr=ffff91034b184900 len=52

2. function
available_filter_functions / set_ftrace_filter
This lists the functions that ftrace has processed and can trace.
These are the function names that you can pass to
"set_ftrace_filter" or "set_ftrace_notrace".
(See the section "dynamic ftrace" below for more details.)

# cat available_filter_functions
netif_receive_skb
ip_rcv_finish
ip_rcv
ip_local_deliver_finish
ip_local_deliver
ip_forward_finish
ip_forward
ip_output
ip_finish_output2
ip_finish_output
dev_queue_xmit
dev_hard_start_xmit
[...]

3. kprobe: Kernel space;
kprobe_events: Enable dynamic trace points.
kprobe_profile: Dynamic trace points stats.

4. uprobe: User space; See uprobetrace.txt
uprobe_events: Add dynamic tracepoints in programs.
uprobe_profile: Uprobe statistics.

tracer: ftrace, perf, systemtap
debugger: gdb

參考資料:
Debugging the kernel using Ftrace - part 1

https://www.kernel.org/doc/Documentation/trace/ftrace.txt
kernel/linux-4.13/kernel/trace/ftrace.c
http://www.brendangregg.com/blog/2015-07-08/choosing-a-linux-tracer.html

https://www.ibm.com/developerworks/cn/linux/1609_houp_ftrace/index.html