From c4deaf609de2e4382ead028ab78f214e1dde94bb Mon Sep 17 00:00:00 2001 From: Viktor Malik Date: Thu, 10 Sep 2026 10:06:37 +0200 Subject: [PATCH] Update docs for the 0.27 release Signed-off-by: Viktor Malik --- docusaurus.config.js | 12 +- src/pages/docs/pre-release/docs.js | 3 + src/pages/docs/release_023/docs.js | 3 + src/pages/docs/release_024/docs.js | 3 + src/pages/docs/release_025/docs.js | 3 + src/pages/docs/release_026/docs.js | 5 +- src/pages/docs/release_027/cli.js | 737 +++++++++ src/pages/docs/release_027/docs.js | 84 + src/pages/docs/release_027/language.md | 2081 ++++++++++++++++++++++++ src/pages/docs/release_027/stdlib.md | 1657 +++++++++++++++++++ 10 files changed, 4583 insertions(+), 5 deletions(-) create mode 100644 src/pages/docs/release_027/cli.js create mode 100644 src/pages/docs/release_027/docs.js create mode 100644 src/pages/docs/release_027/language.md create mode 100644 src/pages/docs/release_027/stdlib.md diff --git a/docusaurus.config.js b/docusaurus.config.js index 4d0bb8c..7804059 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -147,24 +147,28 @@ const config = { type: 'dropdown', label: 'Docs', position: 'left', - to: '/docs/release_026/docs', + to: '/docs/release_027/docs', items: [ { label: 'The Language', - to: '/docs/release_026/language' + to: '/docs/release_027/language' }, { label: 'The Standard Library', - to: '/docs/release_026/stdlib' + to: '/docs/release_027/stdlib' }, { label: 'The Command Line Tool', - to: '/docs/release_026/cli' + to: '/docs/release_027/cli' }, { label: 'Pre-release docs', to: '/docs/pre-release/docs' }, + { + label: '0.26 docs', + to: '/docs/release_026/docs' + }, { label: '0.25 docs', to: '/docs/release_025/docs' diff --git a/src/pages/docs/pre-release/docs.js b/src/pages/docs/pre-release/docs.js index ecdf180..874a0bf 100644 --- a/src/pages/docs/pre-release/docs.js +++ b/src/pages/docs/pre-release/docs.js @@ -44,6 +44,9 @@ function DocsBody() {
    +
  • + 0.27 +
  • 0.26
    • diff --git a/src/pages/docs/release_023/docs.js b/src/pages/docs/release_023/docs.js index dec702b..132062e 100644 --- a/src/pages/docs/release_023/docs.js +++ b/src/pages/docs/release_023/docs.js @@ -48,6 +48,9 @@ function DocsBody() { pre-release
  • + 0.27 +
    • +
  • 0.26
  • diff --git a/src/pages/docs/release_024/docs.js b/src/pages/docs/release_024/docs.js index 311e00b..77abde8 100644 --- a/src/pages/docs/release_024/docs.js +++ b/src/pages/docs/release_024/docs.js @@ -48,6 +48,9 @@ function DocsBody() { pre-release
  • + 0.27 +
    • +
  • 0.26
  • diff --git a/src/pages/docs/release_025/docs.js b/src/pages/docs/release_025/docs.js index 676180b..f85afa1 100644 --- a/src/pages/docs/release_025/docs.js +++ b/src/pages/docs/release_025/docs.js @@ -48,6 +48,9 @@ function DocsBody() { pre-release
  • + 0.27 +
    • +
  • 0.26
  • diff --git a/src/pages/docs/release_026/docs.js b/src/pages/docs/release_026/docs.js index d440b09..99567a0 100644 --- a/src/pages/docs/release_026/docs.js +++ b/src/pages/docs/release_026/docs.js @@ -18,7 +18,7 @@ function DocsBody() {
    -

    Documentation (Version: 0.26 - latest)

    +

    Documentation (Version: 0.26)

    bpftrace documentation is divided into three sections. However for earlier bpftrace versions these sections are part of a single document, which are available in the dropdown.

    @@ -48,6 +48,9 @@ function DocsBody() { pre-release
  • + 0.27 +
    • +
  • 0.25
  • diff --git a/src/pages/docs/release_027/cli.js b/src/pages/docs/release_027/cli.js new file mode 100644 index 0000000..635ba5e --- /dev/null +++ b/src/pages/docs/release_027/cli.js @@ -0,0 +1,737 @@ +import clsx from 'clsx'; +import Link from '@docusaurus/Link'; +import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; +import Layout from '@theme/Layout'; +import HomepageFeatures from '@site/src/components/HomepageFeatures'; +import useBaseUrl from '@docusaurus/useBaseUrl'; + +import Heading from '@theme/Heading'; +import styles from '../../index.module.css'; + +export default function Home() { + return ( + +
    +
    +
    +

    The Command Line Tool (0.27)

    +
    +
    +

    Synopsis

    +
    +
    +

    bpftrace [OPTIONS] FILENAME
    +bpftrace [OPTIONS] -e 'program code'

    +
    +
    +

    When FILENAME is "-", bpftrace will read program code from stdin.

    +
    +
    +

    A program will continue running until Ctrl-C is hit, or an exit function is called. +When a program exits, all populated maps are printed (more details below).

    +
    +
    +
    +
    +

    Description

    +
    +
    +

    bpftrace is a high-level tracing language for Linux. bpftrace uses LLVM as +a backend to compile scripts to eBPF-bytecode and makes use of libbpf and bcc +for interacting with the Linux BPF subsystem, as well as existing Linux +tracing capabilities.

    +
    +
    +
    +
    +

    Examples

    +
    +
    +
    +
    Trace processes calling sleep
    +
    +
    +
    +
    +
    {`# bpftrace -e 'kprobe:do_nanosleep { printf("%d sleeping\\n", pid); }'`}
    +
    +
    +
    +
    +
    Trace processes calling sleep while spawning sleep 5 as a child process
    +
    +
    +
    +
    +
    {`# bpftrace -e 'kprobe:do_nanosleep { printf("%d sleeping\\n", pid); }' -c 'sleep 5'`}
    +
    +
    +
    +
    +
    List all probes with "sleep" in their name
    +
    +
    +
    +
    +
    {`# bpftrace -l '*sleep*'`}
    +
    +
    +
    +
    +
    List all the probes attached in the program
    +
    +
    +
    +
    +
    {`# bpftrace -l -e 'kprobe:do_nanosleep { printf("%d sleeping\\n", pid); }'`}
    +
    +
    +
    +
    +
    +

    Options

    +
    +
    +

    -B MODE

    +
    +

    Set the buffer mode for stdout.

    +
    +
    +
    +
    Valid values are
    +
    +

    none No buffering. Each I/O is written as soon as possible
    +line Data is written on the first newline or when the buffer is full. This is the default mode.
    +full Data is written once the buffer is full.

    +
    +
    +
    +
    +
    +

    -c COMMAND

    +
    +

    Run COMMAND as a child process. +When the child terminates bpftrace will also terminate, as if 'exit()' had been called. +If bpftrace terminates before the child process does the child process will be terminated with a SIGTERM. +If used, 'USDT' probes will only be attached to the child process. +To avoid a race condition when using 'USDTs', the child is stopped after 'execve' using 'ptrace(2)' and continued when all 'USDT' probes are attached. +The child process runs with the same privileges as bpftrace itself (usually root).

    +
    +
    +

    Unless otherwise specified, bpftrace does not perform any implicit filtering. Therefore, if you are only interested in +events in COMMAND, you may want to filter based on the child PID. The child PID is available to programs as the 'cpid' builtin. +For example, you could add the predicate /pid == cpid/ to probes with userspace context.

    +
    +
    +
    +

    -d STAGE

    +
    +

    Enable debug mode. +For more details see the Debug Output section.

    +
    +
    +
    +

    --dry-run

    +
    +

    Terminate execution right after attaching all the probes. Useful for testing +that the script can be parsed, loaded, and attached, without actually running +it.

    +
    +
    +
    +

    -e PROGRAM

    +
    +

    Execute PROGRAM instead of reading the program from a file or stdin.

    +
    +
    +
    +

    -f FORMAT

    +
    +

    Set the output format.

    +
    +
    +
    +
    Valid values are
    +
    +

    json
    +text

    +
    +
    +
    +
    +

    The JSON output is compatible with NDJSON and JSON Lines, meaning each line of the streamed output is a single blob of valid JSON.

    +
    +
    +
    +

    -h, --help

    +
    +

    Print the help summary.

    +
    +
    +
    +

    -I DIR

    +
    +

    Add the directory DIR to the search path for C headers. +This option can be used multiple times. +For more details see the Preprocessor Options section.

    +
    +
    +
    +

    --include FILENAME

    +
    +

    Add FILENAME as an include for the pre-processor. +This is equal to adding '#include FILENAME' at the top of the program. +This option can be used multiple times. +For more details see the Preprocessor Options section.

    +
    +
    +
    +

    --info

    +
    +

    Print detailed information about features supported by the kernel and the bpftrace build.

    +
    +
    +
    +

    -k

    +
    +

    This flag enables runtime warnings for errors from 'probe_read_*' and map lookup BPF helpers. +When errors occur bpftrace will log an error containing the source location and the error code:

    +
    +
    +
    +
    {`stdin:48-57: WARNING: Failed to probe_read_user_str: Bad address (-14)
+u:lib.so:"fn(char const*)" { printf("arg0:%s\\n", str(arg0));}
+                                                 ~~~~~~~~~`}
    +
    +
    +
    +
    +

    -l [SEARCH|FILENAME]

    +
    +

    List all probes that match the SEARCH pattern. +If the pattern is omitted all probes will be listed. +This pattern supports wildcards in the same way that probes do. +E.g. '-l kprobe:*file*' to list all 'kprobes' with 'file' in the name. +This can be used with a program, which will list all probes in that program. +For more details see the Listing Probes section.

    +
    +
    +
    +

    --no-feature feature,feature,…​

    +
    +
    +
    Disable use of detected features, valid values are
    +
    +

    uprobe_multi to disable uprobe_multi link
    +kprobe_multi to disable kprobe_multi link
    +kprobe_session to disable automatic collapse of kprobe/kretprobe into kprobe session

    +
    +
    +
    +
    +
    +

    --no-warnings

    +
    +

    Suppress all warning messages created by bpftrace.

    +
    +
    +
    +

    -o FILENAME

    +
    +

    Write bpftrace tracing output to FILENAME instead of stdout. +This doesn’t include child process (-c option) output. +Errors are still written to stderr.

    +
    +
    +
    +

    -p PID

    +
    +

    Attach to the process with or filter actions by PID. +If the process terminates, bpftrace will also terminate. +When using USDT, uprobes, uretprobes, hardware, software, profile, interval, watchpoint, or asyncwatchpoint probes they will be attached to only this process. +For all other probes, except begin/end, the pid will act like a predicate to filter out events not from that pid. +For listing uprobes/uretprobes set the target to '*' and the process’s address space will be searched for the symbols.

    +
    +
    +
    +

    -q

    +
    +

    Keep messages quiet.

    +
    +
    +
    +

    --unsafe

    +
    +

    Some calls, like 'system', are marked as unsafe as they can have dangerous side effects ('system("rm -rf")') and are disabled by default. +This flag allows their use.

    +
    +
    +
    +

    --usdt-file-activation

    +
    +

    Activate usdt semaphores based on file path.

    +
    +
    +
    +

    -V, --version

    +
    +

    Print bpftrace version information.

    +
    +
    +
    +

    -v

    +
    +

    Enable verbose messages. +For more details see the Verbose Output section.

    +
    +
    +
    +

    Program Options

    +
    +

    You can also pass custom options to a bpftrace program/script itself via positional or named parameters. +Positional parameters can be placed before or after a double dash but named parameters can ONLY come after a double dash; e.g.

    +
    +
    +
    +
    {`# bpftrace -e 'begin { print(($1, $2, getopt("aa", 1), getopt("bb"))); }' p1 -- --aa=20 --bb p2
+
+// (p1, p2, 20, true) is printed`}
    +
    +
    +
    +

    or

    +
    +
    +
    +
    # bpftrace myscript.bt -- p1 --aa=20 --bb p2`}
    +
    +
    +
    +

    In these examples there are two positional parameters (p1, p2) and two named parameters (aa, which is set to 20, and bb, which is set to true). +Named program parameters require the = to set their value unless they are boolean parameters (like 'bb' above). +Read about how to access positional and named parameters here.

    +
    +
    +
    +
    +
    +

    The Language

    +
    +
    +

    Syntax, types, and concepts for bpftrace are available here.

    +
    +
    +
    +
    +

    Probes

    +
    +
    +

    bpftrace supports various probe types which allow the user to attach BPF programs to different types of events. +Each probe starts with a provider (e.g. kprobe) followed by a colon (:) separated list of options. +The amount of options and their meaning depend on the provider. +Full list of probe types.

    +
    +
    +
    +
    +

    Standard library

    +
    +
    +

    The standard library of all builtins, functions, and map functions is available here.

    +
    +
    +
    +
    +

    Configuration

    +
    +
    +

    Config Variables

    +
    +

    Some behavior can only be controlled through config variables, which are available here. +These can be set via the Config Block directly in a script (before any probes) or via their environment variable equivalent, which is upper case and includes the BPFTRACE_ prefix e.g. stack_mode's environment variable would be BPFTRACE_STACK_MODE.

    +
    +
    +
    +

    Environment Variables

    +
    +

    These are not available as part of the standard set of Config Variables above and can only be set as environment variables.

    +
    +
    +

    BPFTRACE_BTF

    +
    +

    Default: None

    +
    +
    +

    The path to a BTF file. By default, bpftrace searches several locations to find a BTF file. +See src/btf.cpp for the details.

    +
    +
    +
    +

    BPFTRACE_DEBUG_OUTPUT

    +
    +

    Default: 0

    +
    +
    +

    Outputs bpftrace’s runtime debug messages to the trace_pipe. This feature can be turned on by setting +the value of this environment variable to 1.

    +
    +
    +
    +

    BPFTRACE_KERNEL_BUILD

    +
    +

    Default: /lib/modules/$(uname -r)

    +
    +
    +

    Only used with BPFTRACE_KERNEL_SOURCE if it is out-of-tree Linux kernel build.

    +
    +
    +
    +

    BPFTRACE_KERNEL_SOURCE

    +
    +

    Default: /lib/modules/$(uname -r)

    +
    +
    +

    bpftrace requires kernel headers for certain features, which are searched for in this directory.

    +
    +
    +
    +

    BPFTRACE_VMLINUX

    +
    +

    Default: None

    +
    +
    +

    This specifies the vmlinux path used for kernel symbol resolution when attaching kprobe to offset. +If this value is not given, bpftrace searches vmlinux from pre defined locations. +See src/attached_probe.cpp:find_vmlinux() for details.

    +
    +
    +
    +

    BPFTRACE_COLOR

    +
    +

    Default: auto

    +
    +
    +

    Colorize the bpftrace log output message. Valid values are auto, always and never.

    +
    +
    +
    +
    +
    +
    +

    Options Expanded

    +
    +
    +

    Debug Output

    +
    +

    The -d STAGE option produces debug output. It prints the output of the +bpftrace execution stage given by the STAGE argument. The option can be used +multiple times (with different stage names) and the special value all prints +the output of all the supported stages. The option also takes multiple stages +in one invocation as comma separated values.

    +
    +
    +

    Note: This is primarily used for bpftrace developers.

    +
    +
    +

    The supported options are:

    +
    + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    ast

    Prints the Abstract Syntax Tree (AST) after every pass.

    codegen

    Prints the unoptimized LLVM IR as produced by CodegenLLVM.

    codegen-opt

    Prints the optimized LLVM IR, i.e. the code which will be compiled into BPF +bytecode.

    dis

    Disassembles and prints out the generated bytecode that libbpf will see. +Only available in debug builds.

    libbpf

    Captures and prints libbpf log for all libbpf operations that bpftrace uses.

    verifier

    Captures and prints the BPF verifier log.

    all

    Prints the output of all of the above stages.

    +
    +
    +

    Listing Probes

    +
    +

    Probe listing is the method to discover which probes are supported by the current system. +Listing supports the same syntax as normal attachment does and alternatively can be +combined with -e or filename args to see all the probes that a program would attach to.

    +
    +
    +
    +
    {`# bpftrace -l 'kprobe:*'
+# bpftrace -l 't:syscalls:*openat*
+# bpftrace -l 'kprobe:tcp*,trace
+# bpftrace -l 'k:*socket*,tracepoint:syscalls:*tcp*'
+# bpftrace -l -e 'tracepoint:xdp:mem_* { exit(); }'
+# bpftrace -l my_script.bt
+# bpftrace -lv 'enum cpu_usage_stat'`}
    +
    +
    +
    +

    The verbose flag (-v) can be specified to inspect arguments (args) for providers that support it:

    +
    +
    +
    +
    {`# bpftrace -l 'fexit:tcp_reset,tracepoint:syscalls:sys_enter_openat' -v
+fexit:tcp_reset
+    struct sock * sk
+    struct sk_buff * skb
+tracepoint:syscalls:sys_enter_openat
+    int __syscall_nr
+    int dfd
+    const char * filename
+    int flags
+    umode_t mode
+
+# bpftrace -l 'uprobe:/bin/bash:rl_set_prompt' -v    # works only if /bin/bash has DWARF
+uprobe:/bin/bash:rl_set_prompt
+    const char *prompt
+
+# bpftrace -lv 'struct css_task_iter'
+struct css_task_iter {
+        struct cgroup_subsys *ss;
+        unsigned int flags;
+        struct list_head *cset_pos;
+        struct list_head *cset_head;
+        struct list_head *tcset_pos;
+        struct list_head *tcset_head;
+        struct list_head *task_pos;
+        struct list_head *cur_tasks_head;
+        struct css_set *cur_cset;
+        struct css_set *cur_dcset;
+        struct task_struct *cur_task;
+        struct list_head iters_node;
+};`}
    +
    +
    +
    +
    +

    Preprocessor Options

    +
    +

    The -I option can be used to add directories to the list of directories that bpftrace uses to look for headers. +Can be defined multiple times.

    +
    +
    +
    +
    {`# cat program.bt
+#include 
+
+begin { @ = FOO }
+
+# bpftrace program.bt
+
+definitions.h:1:10: fatal error: 'foo.h' file not found
+
+# /tmp/include
+foo.h
+
+# bpftrace -I /tmp/include program.bt
+
+Attached 1 probe`}
    +
    +
    +
    +

    The --include option can be used to include headers by default. +Can be defined multiple times. +Headers are included in the order they are defined, and they are included before any other include in the program being executed.

    +
    +
    +
    +
    {`# bpftrace --include linux/path.h --include linux/dcache.h \
+    -e 'kprobe:vfs_open { printf("open path: %s\\n", str(((struct path *)arg0)->dentry->d_name.name)); }'
+
+Attached 1 probe
+open path: .com.google.Chrome.ASsbu2
+open path: .com.google.Chrome.gimc10
+open path: .com.google.Chrome.R1234s`}
    +
    +
    +
    +
    +

    Verbose Output

    +
    +

    The -v option prints more information about the program as it is run:

    +
    +
    +
    +
    {`# bpftrace -v -e 'tracepoint:syscalls:sys_enter_nanosleep { printf("%s is sleeping.\\n", comm); }'
+AST node count: 7
+Attached 1 probe
+
+load tracepoint:syscalls:sys_enter_nanosleep, with BTF, with func_infos: Success
+
+Program ID: 111
+Attaching tracepoint:syscalls:sys_enter_nanosleep
+iscsid is sleeping.
+iscsid is sleeping.
+[...]`}
    +
    +
    +
    +
    +
    +
    +

    Terminology

    +
    + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    BPF

    Berkeley Packet Filter: a kernel technology originally developed for optimizing the processing of packet filters (eg, tcpdump expressions).

    BPF map

    A BPF memory object, which is used by bpftrace to create many higher-level objects.

    BTF

    BPF Type Format: the metadata format which encodes the debug info related to BPF program/map.

    dynamic tracing

    Also known as dynamic instrumentation, this is a technology that can instrument any software event, such as function calls and returns, by live modification of instruction text. Target software usually does not need special capabilities to support dynamic tracing, other than a symbol table that bpftrace can read. Since this instruments all software text, it is not considered a stable API, and the target functions may not be documented outside of their source code.

    eBPF

    Enhanced BPF: a kernel technology that extends BPF so that it can execute more generic programs on any events, such as the bpftrace programs listed below. It makes use of the BPF sandboxed virtual machine environment. Also note that eBPF is often just referred to as BPF.

    kprobes

    A Linux kernel technology for providing dynamic tracing of kernel functions.

    probe

    An instrumentation point in software or hardware, that generates events that can execute bpftrace programs.

    static tracing

    Hard-coded instrumentation points in code. Since these are fixed, they may be provided as part of a stable API, and documented.

    tracepoints

    A Linux kernel technology for providing static tracing.

    uprobes

    A Linux kernel technology for providing dynamic tracing of user-level functions.

    USDT

    User Statically-Defined Tracing: static tracing points for user-level software. Some applications support USDT.

    +
    +
    +
    +

    Program Files

    +
    +
    +

    Programs saved as files are often called scripts and can be executed by specifying their file name. +It is convention to use the .bt file extension but it is not required.

    +
    +
    +

    For example, listing the sleepers.bt file using cat:

    +
    +
    +
    +
    {`# cat sleepers.bt
+
+tracepoint:syscalls:sys_enter_nanosleep {
+  printf("%s is sleeping.\\n", comm);
+}`}
    +
    +
    +
    +

    And then calling it:

    +
    +
    +
    +
    {`# bpftrace sleepers.bt
+
+Attached 1 probe
+iscsid is sleeping.
+iscsid is sleeping.`}
    +
    +
    +
    +

    It can also be made executable to run stand-alone. +Start by adding an interpreter line at the top (#!) with either the path to your installed bpftrace (/usr/local/bin is the default) or the path to env (usually just /usr/bin/env) followed by bpftrace (so it will find bpftrace in your $PATH):

    +
    +
    +
    +
    {`#!/usr/local/bin/bpftrace
+
+tracepoint:syscalls:sys_enter_nanosleep {
+  printf("%s is sleeping.\\n", comm);
+}`}
    +
    +
    +
    +

    Then make it executable:

    +
    +
    +
    +
    {`# chmod 755 sleepers.bt
+# ./sleepers.bt
+
+Attached 1 probe
+iscsid is sleeping.
+iscsid is sleeping.`}
    +
    +
    +
    +
    +
    +
    + +
    +
    +     + ); +} diff --git a/src/pages/docs/release_027/docs.js b/src/pages/docs/release_027/docs.js new file mode 100644 index 0000000..3f7d904 --- /dev/null +++ b/src/pages/docs/release_027/docs.js @@ -0,0 +1,84 @@ +import clsx from 'clsx'; +import Link from '@docusaurus/Link'; +import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; +import Layout from '@theme/Layout'; +import HomepageFeatures from '@site/src/components/HomepageFeatures'; +import useBaseUrl from '@docusaurus/useBaseUrl'; +import ReactPlayer from 'react-player' +import { useState } from 'react'; + +import Heading from '@theme/Heading'; +import styles from '../../index.module.css'; + + +function DocsBody() { + const {siteConfig} = useDocusaurusContext(); + const [url, setUrl] = useState('https://www.youtube.com/watch?v=18xPsqYjUhE'); + return ( +
    +
    +
    +

    Documentation (Version: 0.27 - latest)

    +

    bpftrace documentation is divided into three sections. However for earlier bpftrace versions these sections are part of a single document, which are available in the dropdown.

    +
    +
    +
    +

    The Language

    +

    Everything you wanted to know about bpftrace's syntax, types, concepts, and structure.

    +

    Read More

    +
    +
    +

    The Standard Library

    +

    This includes builtins, functions, map functions, and macros.

    +

    Read More

    +
    +
    +

    The Command Line Tool

    +

    The man page. This include information on how to invoke bpftrace including all CLI options, ENV variables, and run modes.

    +

    Read More

    +
    +
    +
    +
    +
    +
    + + +
    +
    +
    +
    + ); +} + +export default function Home() { + const {siteConfig} = useDocusaurusContext(); + return ( + + + + ); +} diff --git a/src/pages/docs/release_027/language.md b/src/pages/docs/release_027/language.md new file mode 100644 index 0000000..614e480 --- /dev/null +++ b/src/pages/docs/release_027/language.md @@ -0,0 +1,2081 @@ +# The bpftrace Language (0.27) + +The `bpftrace` (`bt`) language is inspired by the D language used by `dtrace` and uses the same program structure. +Each script consists of a [Preamble](#preamble) and one or more [Action Blocks](#action-blocks). + +``` +preamble + +actionblock1 +actionblock2 +``` + +## Action Blocks + +Each action block consists of three parts: + +``` +probe[,probe] +/predicate/ { + action +} +``` + +* **Probes**\ + A probe specifies the event and event type to attach to. [Probes list](#probes). +* **Predicate**\ + The predicate is an optional condition that must be met for the action to be executed. +* **Action**\ + Actions are the programs that run when an event fires (and the predicate is met). +An action is a semicolon (`;`) separated list of statements and always enclosed by brackets `{}`. + +A basic script that traces the `open(2)` and `openat(2)` system calls can be written as follows: + +``` +begin { + printf("Tracing open syscalls... Hit Ctrl-C to end.\n"); +} + +tracepoint:syscalls:sys_enter_open, +tracepoint:syscalls:sys_enter_openat { + printf("%-6d %-16s %s\n", pid, comm, str(args.filename)); +} +``` + +The above script has two action blocks and a total of 3 probes. + +The first action block uses the special `begin` probe, which fires once during `bpftrace` startup. +This probe is used to print a header, indicating that the tracing has started. + +The second action block uses two probes, one for `open` and one for `openat`, and defines an action that prints the file being `open` ed as well as the `pid` and `comm` of the process that execute the syscall. +See the [Probes](#probes) section for details on the available probe types. + +## Arguments + +This refers to traced function and tracepoint arguments. + +There are two ways to access these arguments and the way you choose depends on the [probe type](#probes). + +- For `uprobe`, `kprobe`, and `usdt` use the `argN` format. +- For `rawtracepoint`, `tracepoint`, `fentry`, `fexit`, and `uprobe` (with DWARF) use the `args` format. + +### argN + +These keywords allow access to the nth argument passed to the function being traced. +For the first argument use `arg0`, for the second `arg1`, and so forth. +The type of each arg is an `int64` and will often require casting to non scalar types, e.g., `$x = (struct qstr *)arg1`. +These are extracted from the CPU registers. +The amount of args passed in registers depends on the CPU architecture. + +### args + +This keyword represents the struct of all arguments of the traced function. +You can print the entire structure via `print(args)` or access particular fields using the dot syntax, e.g., `$x = str(args.filename);`. +To see the args for a particular function, you can use [verbose listing mode](cli#listing-probes). +Example: +``` +# bpftrace -lv 'fentry:tcp_reset' + +fentry:tcp_reset + struct sock * sk + struct sk_buff * skb +``` + +## Arrays + +bpftrace supports accessing one-dimensional arrays like those found in `C`. + +Constructing arrays from scratch, like `int a[] = {1,2,3}` in `C`, is not supported. +They can only be read into a variable from a pointer. + +The `[]` operator is used to access elements. + +``` +struct MyStruct { + int y[4]; +} + +kprobe:dummy { + $s = (struct MyStruct *) arg0; + print($s->y[0]); +} +``` + +## Command Line Parameters + +Custom options can be passed to a bpftrace program itself via positional or named parameters. It is recommended to use **named parameters** because they are less ambiguous in terms of their types and meaning (they have actual names). + +### Named Parameters + +Named parameters can be accessed in a bpftrace program via the `getopt` function call, e.g., `getopt("my_named_param", 5)`, `getopt("my_bool_param")`. The first argument is the parameter name and the second is the default value when that argument is not passed on the command line. If the second argument is not provided it indicates that the parameter is a `boolean` type. + +Named parameters must come AFTER a double dash (`--`) when being passed on the command line, e.g., +``` +# bpftrace -e 'begin { print((getopt("aa", 1), getopt("bb"))); }' -- --aa=20 --bb +``` +Here `getopt("aa", 1)` would evaluate to `20` and `getopt("bb")` would evaluate to `true`. + +Named parameters require the `=` to set their value unless they are boolean parameters (like 'bb' above). The supported types are string, number, and boolean. + +### Positional Parameters + +Positional parameters can be accessed in a bpftrace program via, what looks like, a numbered scratch variable, e.g. `$1`, `$2`, ..., `$N`. So `$1` would be the first positional parameter and so on. + +Positional parameters can be placed before or after a double dash, e.g., +``` +# bpftrace -e 'begin { print(($1, $2)); }' p1 -- 20 +``` + +Here `$1` would evaluate to a string `p1` and `$2` would evaluate to a number `20`. + +If a parameter is used that was not provided, it will default to zero for a numeric context, and "" for a string context. +Positional parameters may also be used in probe arguments and will be treated as a string parameter, e.g., `tracepoint:block:block_rq_issue +/args.bytes > $1/`. + +If a positional parameter is used in `str()`, it is interpreted as a pointer to the actual given string literal, which allows to do pointer arithmetic on it. +Only addition of a single constant, less or equal to the length of the supplied string, is allowed. Example: + +``` +# bpftrace -e 'begin { printf("I got %d, %s (%d args)\n", $1, str($2), $#); }' 42 "hello" + +I got 42, hello (2 args) + +# bpftrace -e 'begin { printf("%s\n", str($1 + 1)) }' "hello" + +ello +``` + +`$#` is a special builtin that returns the number of positional arguments supplied. + +## Comments + +Both single line and multi line comments are supported. + +``` +// A single line comment +interval:s:1 { // can also be used to comment inline +/* + a multi line comment + +*/ + print(/* inline comment block */ 1); +} +``` + +## Conditionals + +Conditional expressions are supported in the form of if/else statements and the ternary operator. + +The ternary operator consists of three operands: a condition followed by a `?`, the expression to execute when the condition is true followed by a `:` and the expression to execute if the condition is false. + +``` +condition ? ifTrue : ifFalse +``` + +Both the `ifTrue` and `ifFalse` expressions must be of the same type, mixing types is not allowed. + +The ternary operator can be used as part of an assignment. + +``` +$a == 1 ? print("true") : print("false"); +$b = $a > 0 ? $a : -1; +``` + +If/else statements are supported. + +``` +if (condition) { + ifblock +} else if (condition) { + if2block +} else { + elseblock +} +``` + +## Config Block + +To improve script portability, you can set bpftrace [Config Variables](#config-variables) via the config block, +which can only be placed at the top of the script (in the [preamble](#preamble)) before any action blocks. + +``` +config = { + stack_mode=perf; + max_map_keys=2 +} + +begin { ... } + +uprobe:./testprogs/uprobe_test:uprobeFunction1 { ... } +``` + +The names of the config variables can be in the format of environment variables +or their lowercase equivalent without the `BPFTRACE_` prefix. For example, +`BPFTRACE_STACK_MODE`, `STACK_MODE`, and `stack_mode` are equivalent. + +***Note***: Environment variables for the same config take precedence over those set +inside a script config block. + +[List of All Config Variables](#config-variables) + +## Config Variables + +Some behavior can only be controlled through config variables, which are listed here. +These can be set via the [Config Block](#config-block) directly in a script (before any probes) or via their environment variable equivalent, which is upper case and includes the `BPFTRACE_` prefix e.g. ``stack_mode`’s environment variable would be `BPFTRACE_STACK_MODE`. + +### cache_user_symbols + +Default: PER_PROGRAM if ASLR disabled or `-c` option given, PER_PID otherwise. + +* PER_PROGRAM - each program has its own cache. If there are more processes with enabled ASLR for a single program, this might produce incorrect results. +* PER_PID - each process has its own cache. This is accurate for processes with ASLR enabled, and enables bpftrace to preload caches for processes running at probe attachment time. +If there are many processes running, it will consume a lot of a memory. +* NONE - caching disabled. This saves the most memory, but at the cost of speed. + +### cpp_demangle + +Default: true + +C++ symbol demangling in userspace stack traces is enabled by default. + +This feature can be turned off by setting the value of this variable to `false`. + +### lazy_symbolication + +Default: false + +For user space symbols, symbolicate lazily/on-demand (`true`) or symbolicate everything ahead of time (`false`). + +### license + +Default: "GPL" + +The license bpftrace will use to load BPF programs into the linux kernel. + +### log_size + +Default: 1000000 + +Log size in bytes. + +### max_bpf_progs + +Default: 1024 + +This is the maximum number of BPF programs (functions) that bpftrace can generate. +The main purpose of this limit is to prevent bpftrace from hanging since generating a lot of probes +takes a lot of resources (and it should not happen often). + +### max_cat_bytes + +Default: 10240 + +Maximum bytes read by cat builtin. + +### max_map_keys + +Default: 4096 + +This is the maximum number of keys that can be stored in a map. +Increasing the value will consume more memory and increase startup times. +There are some cases where you will want to, for example: sampling stack traces, recording timestamps for each page, etc. + +### max_probes + +Default: 1024 + +This is the maximum number of probes that bpftrace can attach to. +Increasing the value will consume more memory, increase startup times, and can incur high performance overhead or even freeze/crash the +system. + +### max_strlen + +Default: 1024 + +The maximum length (in bytes) for values created by `str()`, `buf()` and `path()`. + +This limit is necessary because BPF requires the size of all dynamically-read strings (and similar) to be declared up front. This is the size for all strings (and similar) in bpftrace unless specified at the call site. +There is no artificial limit on what you can tune this to. But you may be wasting resources (memory and cpu) if you make this too high. + +### missing_probes + +Default: `error` + +Controls handling of probes which cannot be attached because they do not exist (in the kernel or in the traced binary) or there was an issue during attachment. + +The possible options are: +- `error` - always fail on missing probes +- `warn` - print a warning but continue execution +- `ignore` - silently ignore missing probes + +### on_stack_limit + +Default: 32 + +The maximum size (in bytes) of individual objects that will be stored on the BPF stack. If they are larger than this limit they will be stored in pre-allocated memory. + +This exists because the BPF stack is limited to 512 bytes and large objects make it more likely that we’ll run out of space. bpftrace can store objects that are larger than the `on_stack_limit` in pre-allocated memory to prevent this stack error. However, storing in pre-allocated memory may be less memory efficient. Lower this default number if you are still seeing a stack memory error or increase it if you’re worried about memory consumption. + +### perf_rb_pages + +Default: 64 + +Number of pages to allocate per CPU perf ring buffer. +The value must be a power of 2. +If you’re getting a lot of dropped events bpftrace may not be processing events in the ring buffer fast enough. +It may be useful to bump the value higher so more events can be queued up. +The tradeoff is that bpftrace will use more memory. + +### show_debug_info + +This is only available if the [Blazesym](https://github.com/libbpf/blazesym) library is available at build time. If it is available this defaults to `true`, meaning that when printing ustack and kstack symbols bpftrace will also show (if debug info is available) symbol file and line ('bpftrace' stack mode) and a label if the function was inlined ('bpftrace' and 'perf' stack modes). +There might be a performance difference when symbolicating, which is the only reason to disable this. + +### stack_mode + +Default: bpftrace + +Output format for ustack and kstack builtins. +Available modes/formats: + +* bpftrace +* perf +* raw: no symbolication + +This can be overwritten at the call site. + +### str_trunc_trailer + +Default: `..` + +Trailer to add to strings that were truncated. +Set to empty string to disable truncation trailers. + +### print_maps_on_exit + +Default: true + +Controls whether maps are printed on exit. Set to `false` in order to change the default behavior and not automatically print maps at program exit. + +### unstable features + +These are the list of unstable features: +- `unstable_macro` - feature flag for bpftrace macros +- `unstable_map_decl` - feature flag for map declarations +- `unstable_tseries` - feature flag for time series map type +- `unstable_addr` - feature flag for address of operator (&) + +All of these accept the following options: + +Default: warn + +- `error` - fail if this feature is used +- `warn` - enable feature but print a warning +- `enable` - enable feature + +## Data Types + +The following fundamental types are provided by the language. +Note: Integers are by default represented as 64 bit signed but that can be +changed by either casting them or, for scratch variables, explicitly specifying +the type upon declaration. + +| | | +| --- | --- | +| **Type** | **Description** | +| bool | `true` or `false` | +| uint8 | Unsigned 8 bit integer | +| int8 | Signed 8 bit integer | +| uint16 | Unsigned 16 bit integer | +| int16 | Signed 16 bit integer | +| uint32 | Unsigned 32 bit integer | +| int32 | Signed 32 bit integer | +| uint64 | Unsigned 64 bit integer | +| int64 | Signed 64 bit integer | + +``` +begin { $x = 1<<16; printf("%d %d\n", (uint16)$x, $x); } + +/* + * Output: + * 0 65536 + */ +``` + +## Filters/Predicates + +Filters (also known as predicates) can be added after probe names. +The probe still fires, but it will skip the action unless the filter is true. + +``` +kprobe:vfs_read /arg2 < 16/ { + printf("small read: %d byte buffer\n", arg2); +} + +kprobe:vfs_read /comm == "bash"/ { + printf("read by %s\n", comm); +} +``` + +## Floating-point + +Floating-point numbers are not supported by BPF and therefore not by bpftrace. + +## Identifiers + +Identifiers must match the following regular expression: `[_a-zA-Z][_a-zA-Z0-9]*` + +## Keywords + +`break`, `config`, `continue`, `else`, `for`, `if`, `import`, `let`, `macro`, `offsetof`, `return`, `sizeof`, `unroll`, `while`. + +* `return` - The return keyword is used to exit the current probe. This differs from `exit()` in that it doesn’t exit bpftrace. + +## Literals + +Integer and string literals are supported. + +Integer literals can be defined in the following formats: + +* decimal (base 10) +* octal (base 8) +* hexadecimal (base 16) +* scientific (base 10) + +Octal literals have to be prefixed with a `0` e.g. `0123`. +Hexadecimal literals start with either `0x` or `0X` e.g. `0x10`. +Scientific literals are written in the `e` format which is a shorthand for `m*10^n` e.g. `$i = 2e3;`. +Note that scientific literals are integer only due to the lack of floating point support e.g. `1e-3` is not valid. + +To improve the readability of big literals an underscore `_` can be used as field separator e.g. 1_000_123_000. + +Integer suffixes as found in the C language are parsed by bpftrace to ensure compatibility with C headers/definitions but they’re not used as size specifiers. +`123UL`, `123U` and `123LL` all result in the same integer type with a value of `123`. + +These duration suffixes are also supported: `ns`, `us`, `ms`, `s`, `m`, `h`, and `d`. All get turned into integer values in nanoseconds, e.g. +``` +$a = 1m; +print($a); // prints 60000000000 +``` + +Character literals are not supported at this time, and the corresponding ASCII code must be used instead: + +``` +begin { + printf("Echo A: %c\n", 65); +} +``` + +String literals can be defined by enclosing the character string in double quotes e.g. `$str = "Hello world";`. + +Strings support the following escape sequences: + +| | | +| --- | --- | +| \n | Newline | +| \t | Tab | +| \0nn | Octal value nn | +| \xnn | Hexadecimal value nn | + +## Loops + +### For + +`for` loops can be used to iterate over elements in a map, or over a range of integers, provided as two unary expressions separated by `..`. + +``` +for ($kv : @map) { + block; +} +``` + +``` +for ($i : start..end) { + block; +} +``` + +The variable declared in the `for` loop will be initialised on each iteration. + +If the iteration is over a map, the value will be a tuple containing a key and a value from the map, i.e. `$kv = (key, value)`: + +``` +@map[10] = 20; +for ($kv : @map) { + print($kv.0); // key + print($kv.1); // value +} +``` + +If a map has multiple keys, the loop variable will be initialised with nested tuple of the form: `((key1, key2, ...), value)`: + +``` +@map[10,11] = 20; +for ($kv : @map) { + print($kv.0.0); // key 1 + print($kv.0.1); // key 2 + print($kv.1); // value +} +``` + +If an integer range is provided, the value will be an integer value for each element in the range, inclusive of the start value and exclusive of the end value: + +``` +for ($cpu : 0..ncpus) { + print($cpu); // current value in range +} +``` + +Note that you cannot adjust the range itself after the loop has started. +The `for` start and end values are evaluated once, not on each loop iteration. +For example, the following will print `0` through `9`: + +``` +$a = 10; +for ($i : 0..$a) { + print($i); + $a--; +} +``` + +Both `for` loops support the following control flow statements: + +| | | +| --- | --- | +| continue | skip processing of the rest of the block and proceed to the next iteration | +| break | terminate the loop | + +### While + +BPF supports `while` loops as long as the verifier can prove they’re bounded and fit within the instruction limit. + +``` +while (condition) { + block; +} +``` + +``` +interval:s:1 { + $i = 0; + while ($i <= 100) { + printf("%d ", $i); + if ($i > 5) { + break; + } + $i++ + } + printf("\n"); +} +``` + +The `while` loop supports the following control flow statements: + +| | | +| --- | --- | +| continue | skip processing of the rest of the block and return to the conditional | +| break | terminate the loop | + +### Unroll + +Loop unrolling is also supported with the `unroll` statement. + +``` +unroll(n) { + block; +} +``` + +The compiler will evaluate the block `n` times and generate the BPF code for the block `n` times. +As this happens at compile time `n` must be a constant greater than 0 (`n > 0`). + +The following two probes compile into the same code: + +``` +interval:s:1 { + unroll(3) { + print("Unrolled") + } +} + +interval:s:1 { + print("Unrolled") + print("Unrolled") + print("Unrolled") +} +``` + +## Macros + +***Warning*** this feature is experimental and may be subject to changes. +Stabilization is tracked in [#4079](https://github.com/bpftrace/bpftrace/issues/4079). + +bpftrace macros (as opposed to C macros) provide a way for you to structure your script. +They can be useful when you want to factor out code into smaller, more understandable parts. +Or if you want to share code between probes. + +At a high level, macros can be thought of as semantic aware text replacement. +They accept (optional) variable, map, and expression arguments. +The body of the macro may only access maps and external variables passed in through the arguments, which is why these are often referred to as "hygienic macros", but the macro body can create new variables which only exist inside the body. +A macro's parameter signature specifies how an argument will be used. +For example `macro test($a, b, @c)` indicates that `$a` needs to be a scratch variable (which might be mutated), that `b` needs to be an expression that will be inserted where ever `b` is used in the macro body, and that `@c` needs to be a map (which might be mutated). +A valid use of this macro could be `test($x, 1 + 2, @y)`. +Variables and maps can also be used for ident parameters that expect expressions and would be the same as writing `{ @y }` (Block Expression). + +Here are some valid usages of macros: + +``` +macro one() { + 1 +} + +macro add_one(x) { + x + 1 +} + +macro add_one_to_each($a, @b) { + $a += 1; + @b += 1; +} + +macro side_effects(x) { + x; + x; + x; +} + +macro add_two(x) { + add_one(x) + 1 +} + +begin { + print(one()); // prints 1 + print(one); // prints 1 (bare identifier works if the macro accepts 0 args) + + $a = 10; + print(add_one($a)); // prints 11 + print(add_one(1 + 1)); // prints 3 + + @b = 5; + add_one_to_each($a, @b); + print($a + @b) // prints 17 + + side_effects({ printf("hi") }) // prints hihihi + + print(add_two(1)); // prints 3 +} +``` + +Some examples of invalid macro usage: + +``` +macro unhygienic_access() { + @x++ // BAD: @x not passed in +} + +macro wrong_parameter_type($x) { + $x++ +} + +begin { + @x = 1; + unhygienic_access(); + + wrong_parameter_type(@x); // BAD: macro expects a scratch variable + wrong_parameter_type(1 + 1); // BAD: macro expects a scratch variable +} +``` + +Note: If you want the passed in expression to only be executed once, simply bind it to a variable, e.g., +``` +macro add_one(x) { + let $x = x; + $x + 1 +} +``` + +## Operators and Expressions + +### Arithmetic Operators + +The following operators are available for integer arithmetic: + +| | | +| --- | --- | +| + | integer addition | +| - | integer subtraction | +| * | integer multiplication | +| / | integer division | +| % | integer modulo | + +Operations between a signed and an unsigned integer are allowed providing +bpftrace can statically prove a safe conversion is possible. If safe conversion +is not guaranteed, the operation is undefined behavior and a corresponding +warning will be emitted. + +If the two operands are different size, the smaller integer is implicitly +promoted to the size of the larger one. Sign is preserved in the promotion. +For example, `(uint32)5 + (uint8)3` is converted to `(uint32)5 + (uint32)3` +which results in `(uint32)8`. + +Pointers may be used with arithmetic operators but only for addition and +subtraction. For subtraction, the pointer must appear on the left side of the +operator. Pointers may also be used with logical operators; they are considered +true when non-null. + +### Logical Operators + +| | | +| --- | --- | +| && | Logical AND | +| \|\| | Logical OR | +| ! | Logical NOT | + +### Bitwise Operators + +| | | +| --- | --- | +| & | AND | +| \| | OR | +| ^ | XOR | +| << | Left shift the left-hand operand by the number of bits specified by the right-hand expression value | +| >> | Right shift the left-hand operand by the number of bits specified by the right-hand expression value | + +### Relational Operators + +The following relational operators are defined for integers and pointers. + +| | | +| --- | --- | +| < | left-hand expression is less than right-hand | +| <= | left-hand expression is less than or equal to right-hand | +| > | left-hand expression is bigger than right-hand | +| >= | left-hand expression is bigger or equal to than right-hand | +| == | left-hand expression equal to right-hand | +| != | left-hand expression not equal to right-hand | + +The following relation operators are available for comparing strings and integer arrays. + +| | | +| --- | --- | +| == | left-hand string equal to right-hand | +| != | left-hand string not equal to right-hand | + +### Assignment Operators + +The following assignment operators can be used on both `map` and `scratch` variables: + +| | | +| --- | --- | +| = | Assignment, assign the right-hand expression to the left-hand variable | +| <<= | Update the variable with its value left shifted by the number of bits specified by the right-hand expression value | +| >>= | Update the variable with its value right shifted by the number of bits specified by the right-hand expression value | +| += | Increment the variable by the right-hand expression value | +| -= | Decrement the variable by the right-hand expression value | +| *= | Multiple the variable by the right-hand expression value | +| /= | Divide the variable by the right-hand expression value | +| %= | Modulo the variable by the right-hand expression value | +| &= | Bitwise AND the variable by the right-hand expression value | +| \|= | Bitwise OR the variable by the right-hand expression value | +| ^= | Bitwise XOR the variable by the right-hand expression value | + +All these operators are syntactic sugar for combining assignment with the specified operator. +`@ -= 5` is equal to `@ = @ - 5`. + +### Increment and Decrement Operators + +The increment (`++`) and decrement (`--`) operators can be used on integer and pointer variables to increment their value by one. +They can only be used on variables and can either be applied as prefix or suffix. +The difference is that the expression `x++` returns the original value of `x`, before it got incremented while `++x` returns the value of `x` post increment. + +``` +$x = 10; +$y = $x--; // y = 10; x = 9 +$a = 10; +$b = --$a; // a = 9; b = 9 +``` + +Note that maps will be implicitly declared and initialized to 0 if not already declared or defined. +Scratch variables must be initialized before using these operators. + +Note `++`/`--` on a shared global variable can lose updates. See [`count()`](stdlib#count) for more details. + +### Block Expressions + +A block can be used as expression, as long as the last statement of the block +is an expression with no trailing semi-colon. + +``` +let $a = { + let $b = 1; + $b +}; +``` + +This can be used anywhere an expression can be used. + +## Preamble + +The preamble consists of multiple optional pieces: +- preprocessor definitions +- type definitions +- a [config block](#config-block) +- [map declarations](#map-declarations) + +For example: + +``` +#include +#define RED "\033[31m" + +struct S { + int x; +} + +config = { + stack_mode=perf +} + +let @a = lruhash(100); + +``` + +## Probes + +bpftrace supports various probe types which allow the user to attach BPF programs to different types of events. +Each probe starts with a provider (e.g. `kprobe`) followed by a colon (`:`) separated list of options. +The amount of options and their meaning depend on the provider and are detailed below. +The valid values for options can depend on the system or binary being traced, e.g. for uprobes it depends on the binary. +Also see [Listing Probes](cli#listing-probes). + +It is possible to associate multiple probes with a single action as long as the action is valid for all specified probes. +Multiple probes can be specified as a comma (`,`) separated list: + +``` +kprobe:tcp_reset,kprobe:tcp_v4_rcv { + printf("Entered: %s\n", probe); +} +``` + +Wildcards are supported too: + +``` +kprobe:tcp_* { + printf("Entered: %s\n", probe); +} +``` + +Both can be combined: + +``` +kprobe:tcp_reset,kprobe:*socket* { + printf("Entered: %s\n", probe); +} +``` + +By default, bpftrace requires all probes to attach successfully or else an error is returned. However this can be changed using the `missing_probes` config variable. + +Most providers also support a short name which can be used instead of the full name, e.g. `kprobe:f` and `k:f` are identical. + +| | | | +| --- | --- | --- | +| **Probe Name** | **Short Name** | **Description** | +| [`begin/end`](#beginend) | - | Built-in events | +| [`bench`](#bench) | - | Micro benchmarks | +| [`self`](#self) | - | Built-in events | +| [`hardware`](#hardware) | `h` | Processor-level events | +| [`interval`](#interval) | `i` | Timed output | +| [`iter`](#iterator) | `it` | Iterators tracing | +| [`fentry/fexit`](#fentry-and-fexit) | `f`/`fr` | Kernel functions tracing with BTF support | +| [`kprobe/kretprobe`](#kprobe-and-kretprobe) | `k`/`kr` | Kernel function start/return | +| [`profile`](#profile) | `p` | Timed sampling | +| [`rawtracepoint`](#rawtracepoint) | `rt` | Kernel static tracepoints with raw arguments | +| [`software`](#software) | `s` | Kernel software events | +| [`tracepoint`](#tracepoint) | `t` | Kernel static tracepoints | +| [`uprobe/uretprobe`](#uprobe-uretprobe) | `u`/`ur` | User-level function start/return | +| [`usdt`](#usdt) | `U` | User-level static tracepoints | +| [`watchpoint/asyncwatchpoint`](#watchpoint-and-asyncwatchpoint) | `w`/`aw` | Memory watchpoints | + +### begin/end + +These are special built-in events provided by the bpftrace runtime. +`begin` is triggered before all other probes are attached. +`end` is triggered after all other probes are detached. + +Note that specifying an `end` probe doesn’t override the printing of 'non-empty' maps at exit. +To prevent printing all used maps need be cleared in the `end` probe: + +``` +end { + clear(@map1); + clear(@map2); +} +``` + +### bench + +`bench` is a special built-in probe type for creating micro benchmarks. +bpftrace executes each `bench` probe repeatedly to measure the average +execution time of the contained code. If multiple `bench` probes exist +in a script, bpftrace executes them sequentially in the order they are +specified. To run `bench` probes, you must run bpftrace in bench mode: +`bpftrace --test-mode bench ...`; otherwise, `bench` probes will be +ignored. + +``` +bench:lhist { + @a = lhist(rand % 10, 1, 10, 1); +} + +bench:count { + @b = count(); +} + +bench:my_loop { + $a = 0; + for ($i : 0..10) { + $a++; + } +} +``` + +``` +Attached 3 probes + + ++-----------+-------------+ +| BENCHMARK | NANOSECONDS | ++-----------+-------------+ +| count | 40 | +| lhist | 88 | +| my_loop | 124 | ++-----------+-------------+ +``` + +### self + +**variants** + +* `self:signal:SIGUSR1` + +These are special built-in events provided by the bpftrace runtime. +The trigger function is called by the bpftrace runtime when the bpftrace process receives specific events, such as a `SIGUSR1` signal. + +``` +self:signal:SIGUSR1 { + print("abc"); +} +``` + +### hardware + +**variants** + +* `hardware:event_name:` +* `hardware:event_name:count` + +**short name** + +* `h` + +These are the pre-defined hardware events provided by the Linux kernel, as commonly traced by the perf utility. +They are implemented using performance monitoring counters (PMCs): hardware resources on the processor. +There are about ten of these, and they are documented in the perf_event_open(2) man page. +The event names are: + +* `cpu-cycles` or `cycles` +* `instructions` +* `cache-references` +* `cache-misses` +* `branch-instructions` or `branches` +* `branch-misses` +* `bus-cycles` +* `frontend-stalls` +* `backend-stalls` +* `ref-cycles` + +The `count` option specifies how many events must happen before the probe fires (sampling interval). +If `count` is left unspecified a default value is used. + +This will fire once for every 1,000,000 cache misses. + +``` +hardware:cache-misses:1e6 { @[pid] = count(); } +``` + +### interval + +**variants** + +* `interval:count` +* `interval:us:count` +* `interval:ms:count` +* `interval:s:count` +* `interval:hz:rate` + +**short name** + +* `i` + +The interval probe fires at a fixed interval as specified by its time spec. +Interval fires on one CPU at a time, unlike [profile](#profile) probes. +If a unit of time is not specified in the second position, the number is interpreted as nanoseconds; e.g., `interval:1s`, `interval:1000000000`, and `interval:s:1` are all equivalent. + +This prints the rate of syscalls per second. + +``` +tracepoint:raw_syscalls:sys_enter { @syscalls = count(); } +interval:1s { print(@syscalls); clear(@syscalls); } +``` + +### iterator + +**variants** + +* `iter:task` +* `iter:task:pin` +* `iter:task_file` +* `iter:task_file:pin` +* `iter:task_vma` +* `iter:task_vma:pin` + +**short name** + +* `it` + +***Warning*** this feature is experimental and may be subject to interface changes. + +These are eBPF iterator probes that allow iteration over kernel objects. +Iterator probe can’t be mixed with any other probe, not even another iterator. +Each iterator probe provides a set of fields that could be accessed with the +ctx pointer. Users can display the set of available fields for each iterator via +-lv options as described below. + +``` +iter:task { printf("%s:%d\n", ctx->task->comm, ctx->task->pid); } + +/* + * Sample output: + * systemd:1 + * kthreadd:2 + * rcu_gp:3 + * rcu_par_gp:4 + * kworker/0:0H:6 + * mm_percpu_wq:8 + */ +``` + +``` +iter:task_file { + printf("%s:%d %d:%s\n", ctx->task->comm, ctx->task->pid, ctx->fd, path(ctx->file->f_path)); +} + +/* + * Sample output: + * systemd:1 1:/dev/null + * systemd:1 3:/dev/kmsg + * ... + * su:1622 2:/dev/pts/1 + * ... + * bpftrace:1892 2:/dev/pts/1 + * bpftrace:1892 6:anon_inode:bpf-prog + */ +``` + +``` +iter:task_vma { + printf("%s %d %lx-%lx\n", comm, pid, ctx->vma->vm_start, ctx->vma->vm_end); +} + +/* + * Sample output: + * bpftrace 119480 55b92c380000-55b92c386000 + * ... + * bpftrace 119480 7ffd55dde000-7ffd55de2000 + */ +``` + +It’s possible to pin an iterator by specifying the optional probe ':pin' part, that defines the pin file. +It can be specified as an absolute or relative path to /sys/fs/bpf. + +**relative pin** + +``` +iter:task:list { printf("%s:%d\n", ctx->task->comm, ctx->task->pid); } + +/* + * Sample output: + * Program pinned to /sys/fs/bpf/list + */ +``` + +**absolute pin** + +``` +iter:task_file:/sys/fs/bpf/files { + printf("%s:%d %s\n", ctx->task->comm, ctx->task->pid, path(ctx->file->f_path)); +} + +/* + * Sample output: + * Program pinned to /sys/fs/bpf/files + */ +``` + +### fentry and fexit + +**variants** + +* `fentry[:module]:fn` +* `fexit[:module]:fn` +* `fentry:bpf[:prog_id]:prog_name` +* `fexit:bpf[:prog_id]:prog_name` + +**short names** + +* `f` (`fentry`) +* `fr` (`fexit`) + +**requires (`--info`)** + +* Kernel features:BTF +* Probe types:fentry + +``fentry``/``fexit`` probes attach to kernel functions similar to [kprobe and kretprobe](#kprobe-and-kretprobe). +They make use of eBPF trampolines which allow kernel code to call into BPF programs with near zero overhead. +Originally, these were called `kfunc` and `kretfunc` but were later renamed to `fentry` and `fexit` to match +how these are referenced in the kernel and to prevent confusion with [BPF Kernel Functions](https://docs.kernel.org/bpf/kfuncs.html). +The original names are still supported for backwards compatibility. + +``fentry``/``fexit`` probes make use of BTF type information to derive the type of function arguments at compile time. +This removes the need for manual type casting and makes the code more resilient against small signature changes in the kernel. +The function arguments are available in the `args` struct which can be inspected by doing verbose listing (see [Listing Probes](cli#listing-probes)). +These arguments are also available in the return probe (`fexit`), unlike `kretprobe`. + +The bpf variants (e.g. `fentry:bpf[:prog_id]:prog_name`) allow attaching to running BPF programs and sub-programs. +For example, if bpftrace was already running with a script like `uprobe:./testprogs/uprobe_test:uprobeFunction1 { print("hello"); }` then you could attach to this program with `fexit:bpf:uprobe___testprogs_uprobe_test_uprobeFunction1_1 { print("bye"); }` and this probe would execute after (because it's `fexit`) the `print("hello")` probe executes. +You can specify just the program name, and in this case bpftrace will attach to all running programs and sub-programs with that name. +You can differentiate between them using the `probe` builtin. +You can also specify the program id (e.g. `fentry:bpf:123:*`) to attach to a specific running BPF program or sub-programs called in that running BPF program. +To see a list of running, valid BPF programs and sub-programs use `bpftrace -l 'fentry:bpf:*'`. +Note: only BPF programs with a BTF Id can be attached to. +Also, the `args` builtin is not yet available for this variant. + +``` +# bpftrace -lv 'fentry:tcp_reset' + +fentry:tcp_reset + struct sock * sk + struct sk_buff * skb +``` + +``` +fentry:x86_pmu_stop { + printf("pmu %s stop\n", str(args.event->pmu->name)); +} +``` + +The fget function takes one argument as file descriptor and you can access it via args.fd and the return value is accessible via retval: + +``` +fexit:fget { + printf("fd %d name %s\n", args.fd, str(retval->f_path.dentry->d_name.name)); +} + +/* + * Sample output: + * fd 3 name ld.so.cache + * fd 3 name libselinux.so.1 + */ +``` + +### kprobe and kretprobe + +**variants** + +* `kprobe[:module]:fn` +* `kprobe[:module]:fn+offset` +* `kretprobe[:module]:fn` + +**short names** + +* `k` +* `kr` + +``kprobe``s allow for dynamic instrumentation of kernel functions. +Each time the specified kernel function is executed the attached BPF programs are ran. + +``` +kprobe:tcp_reset { + @tcp_resets = count() +} +``` + +Function arguments are available through the `argN` for register args. Arguments passed on stack are available using the stack pointer, e.g. `$stack_arg0 = **(int64**)reg("sp") + 16`. +Whether arguments passed on stack or in a register depends on the architecture and the number or arguments used, e.g. on x86_64 the first 6 non-floating point arguments are passed in registers and all following arguments are passed on the stack. +Note that floating point arguments are typically passed in special registers which don’t count as `argN` arguments which can cause confusion. +Consider a function with the following signature: + +``` +void func(int a, double d, int x) +``` + +Due to `d` being a floating point, `x` is accessed through `arg1` where one might expect `arg2`. + +bpftrace does not detect the function signature so it is not aware of the argument count or their type. +It is up to the user to perform [Type conversion](#type-conversion) when needed, e.g. + +``` +#include +#include + +kprobe:vfs_open +{ + printf("open path: %s\n", str(((struct path *)arg0)->dentry->d_name.name)); +} +``` + +Here arg0 was cast as a (struct path *), since that is the first argument to vfs_open. +The struct support is the same as bcc and based on available kernel headers. +This means that many, but not all, structs will be available, and you may need to manually define structs. + +If the kernel has BTF (BPF Type Format) data, all kernel structs are always available without defining them. For example: + +``` +kprobe:vfs_open { + printf("open path: %s\n", str(((struct path *)arg0)->dentry->d_name.name)); +} +``` + +You can optionally specify a kernel module, either to include BTF data from that module, or to specify that the traced function should come from that module. + +``` +kprobe:kvm:x86_emulate_insn +{ + $ctxt = (struct x86_emulate_ctxt *) arg0; + printf("eip = 0x%lx\n", $ctxt->eip); +} +``` + +See [BTF Support](#btf-support) for more details. + +`kprobe` s are not limited to function entry, they can be attached to any instruction in a function by specifying an offset from the start of the function. + +`kretprobe` s trigger on the return from a kernel function. +Return probes do not have access to the function (input) arguments, only to the return value (through `retval`). +A common pattern to work around this is by storing the arguments in a map on function entry and retrieving in the return probe: + +``` +kprobe:d_lookup +{ + $name = (struct qstr *)arg1; + @fname[tid] = $name->name; +} + +kretprobe:d_lookup +/@fname[tid]/ +{ + printf("%-8d %-6d %-16s M %s\n", elapsed / 1e6, pid, comm, + str(@fname[tid])); +} +``` + +### profile + +**variants** + +* `profile:count` +* `profile:us:count` +* `profile:ms:count` +* `profile:s:count` +* `profile:hz:rate` + +**short name** + +* `p` + +Profile probes fire on each CPU on the specified interval. +These operate using perf_events (a Linux kernel facility, which is also used by the perf command). +If a unit of time is not specified in the second position, the number is interpreted as nanoseconds; e.g., `interval:1s`, `interval:1000000000`, and `interval:s:1` are all equivalent. + +``` +profile:hz:99 { @[tid] = count(); } +``` + +### rawtracepoint + +**variants** + +* `rawtracepoint[:module]:event` + +**short name** + +* `rt` + +Raw tracepoints are attached to the same tracepoints as normal tracepoint programs. +The reason why you might want to use raw tracepoints over normal tracepoints is due to the performance improvement - [Read More](https://docs.ebpf.io/linux/program-type/BPF_PROG_TYPE_RAW_TRACEPOINT/). + +`rawtracepoint` arguments can be accessed via the `argN` builtins AND via the `args` builtin. + +``` +rawtracepoint:vmlinux:kfree_skb { + printf("%llx %llx\n", arg0, args.skb); +} +``` +`arg0` and `args.skb` will print the same address. + +`rawtracepoint` probes make use of BTF type information to derive the type of function arguments at compile time. +This removes the need for manual type casting and makes the code more resilient against small signature changes in the kernel. +The arguments accessible by a `rawtracepoint` are different from the arguments you can access from the `tracepoint` of the same name. +The function arguments are available in the `args` struct which can be inspected by doing verbose listing (see [Listing Probes](cli#listing-probes)). + +### software + +**variants** + +* `software:event:` +* `software:event:count` + +**short name** + +* `s` + +These are the pre-defined software events provided by the Linux kernel, as commonly traced via the perf utility. +They are similar to tracepoints, but there is only about a dozen of these, and they are documented in the perf_event_open(2) man page. +If the count is not provided, a default is used. + +The event names are: + +* `cpu-clock` or `cpu` +* `task-clock` +* `page-faults` or `faults` +* `context-switches` or `cs` +* `cpu-migrations` +* `minor-faults` +* `major-faults` +* `alignment-faults` +* `emulation-faults` +* `dummy` +* `bpf-output` + +``` +software:faults:100 { @[comm] = count(); } +``` + +This roughly counts who is causing page faults, by sampling the process name for every one in one hundred faults. + +### tracepoint + +**variants** + +* `tracepoint:subsys:event` + +**short name** + +* `t` + +Tracepoints are hooks into events in the kernel. +Tracepoints are defined in the kernel source and compiled into the kernel binary which makes them a form of static tracing. +Unlike `kprobe` s, new tracepoints cannot be added without modifying the kernel. + +The advantage of tracepoints is that they generally provide a more stable interface than `kprobe` s do, they do not depend on the existence of a kernel function. + +``` +tracepoint:syscalls:sys_enter_openat { + printf("%s %s\n", comm, str(args.filename)); +} +``` + +Tracepoint arguments are available in the `args` struct which can be inspected with verbose listing, see the [Listing Probes](cli#listing-probes) section for more details. + +``` +# bpftrace -lv "tracepoint:*" + +tracepoint:xhci-hcd:xhci_setup_device_slot + u32 info + u32 info2 + u32 tt_info + u32 state +... +``` + +Alternatively members for each tracepoint can be listed from their /format file in /sys. + +Apart from the filename member, we can also print flags, mode, and more. +After the "common" members listed first, the members are specific to the tracepoint. + +**Additional information** + +* https://www.kernel.org/doc/html/latest/trace/tracepoints.html + +### uprobe, uretprobe + +**variants** + +* `uprobe:binary:func` +* `uprobe:binary:func+offset` +* `uprobe:binary:offset` +* `uretprobe:binary:func` + +**short names** + +* `u` +* `ur` + +`uprobe` s or user-space probes are the user-space equivalent of `kprobe` s. +The same limitations that apply [kprobe and kretprobe](#kprobe-and-kretprobe) also apply to `uprobe` s and `uretprobe` s, namely: arguments are available via the `argN` builtins and can only be accessed with a uprobe. +retval is the return value for the instrumented function and can only be accessed with a uretprobe. +**Note**: When tracing some languages, like C++, `arg0` and even `arg1` may refer to runtime internals such as the current object instance (`this`) and/or the eventual return value for large returned objects where copy elision is used. +This will push the actual function arguments to possibly start at `arg1` or `arg2` - the only way to know is to experiment. + +``` +uprobe:/bin/bash:readline { printf("arg0: %d\n", arg0); } +``` + +What does arg0 of readline() in /bin/bash contain? +I don’t know, so I’ll need to look at the bash source code to find out what its arguments are. + +When tracing libraries, it is sufficient to specify the library name instead of +a full path. The path will be then automatically resolved using `/etc/ld.so.cache`: + +``` +uprobe:libc:malloc { printf("Allocated %d bytes\n", arg0); } +``` + +If multiple versions of the same shared library exist (e.g. `libssl.so.3` and +`libssl.so.59`), bpftrace may resolve the wrong one. To fix this, you can specify +a versioned SONAME to ensure the correct library is traced: + +``` +uprobe:libssl.so.3:SSL_write { ... } +``` + +If the traced binary has DWARF included, function arguments are available in the `args` struct which can be inspected with verbose listing, see the [Listing Probes](cli#listing-probes) section for more details. + +``` +# bpftrace -lv 'uprobe:/bin/bash:rl_set_prompt' + +uprobe:/bin/bash:rl_set_prompt + const char* prompt +``` + +When tracing C++ programs, it’s possible to turn on automatic symbol demangling by using the `:cpp` prefix: + +``` +# bpftrace:cpp:"bpftrace::BPFtrace::add_probe" { ... } +``` + +It is important to note that for `uretprobe` s to work the kernel runs a special helper on user-space function entry which overrides the return address on the stack. +This can cause issues with languages that have their own runtime like Golang: + +**example.go** + +``` +func myprint(s string) { + fmt.Printf("Input: %s\n", s) +} + +func main() { + ss := []string{"a", "b", "c"} + for _, s := range ss { + go myprint(s) + } + time.Sleep(1*time.Second) +} +``` + +**bpftrace** + +``` +# bpftrace -e 'uretprobe:./test:main.myprint { @=count(); }' -c ./test +runtime: unexpected return pc for main.myprint called from 0x7fffffffe000 +stack: frame={sp:0xc00008cf60, fp:0xc00008cfd0} stack=[0xc00008c000,0xc00008d000) +fatal error: unknown caller pc +``` + +### usdt + +**variants** + +* `usdt:binary_path:probe_name` +* `usdt:binary_path:[probe_namespace]:probe_name` +* `usdt:library_path:probe_name` +* `usdt:library_path:[probe_namespace]:probe_name` + +**short name** + +* `U` + +Where probe_namespace is optional if probe_name is unique within the binary. + +You can target the entire host (or an entire process’s address space by using the `-p` arg) by using a single wildcard in place of the binary_path/library_path: + +``` +usdt:*:loop { printf("hi\n"); } +``` + +Please note that if you use wildcards for the probe_name or probe_namespace and end up targeting multiple USDTs for the same probe you might get errors if you also utilize the USDT argument builtin (e.g. arg0) as they could be of different types. + +Arguments are available via the `argN` builtins: + +``` +usdt:/root/tick:loop { printf("%s: %d\n", str(arg0), arg1); } +``` + +bpftrace also supports USDT semaphores. +If both your environment and bpftrace support uprobe refcounts, then USDT semaphores are automatically activated for all processes upon probe attachment (and --usdt-file-activation becomes a noop). +You can check if your system supports uprobe refcounts by running: + +``` +# bpftrace --info 2>&1 | grep "uprobe refcount" +bcc bpf_attach_uprobe refcount: yes + uprobe refcount (depends on Build:bcc bpf_attach_uprobe refcount): yes +``` + +If your system does not support uprobe refcounts, you may activate semaphores by passing in -p $PID or --usdt-file-activation. +--usdt-file-activation looks through /proc to find processes that have your probe’s binary mapped with executable permissions into their address space and then tries to attach your probe. +Note that file activation occurs only once (during attach time). +In other words, if later during your tracing session a new process with your executable is spawned, your current tracing session will not activate the new process. +Also note that --usdt-file-activation matches based on file path. +This means that if bpftrace runs from the root host, things may not work as expected if there are processes execved from private mount namespaces or bind mounted directories. +One workaround is to run bpftrace inside the appropriate namespaces (i.e. the container). + +### watchpoint and asyncwatchpoint + +**variants** + +* `watchpoint:absolute_address:length:mode` +* `watchpoint:function+argN:length:mode` + +**short names** + +* `w` +* `aw` + +This feature is experimental and may be subject to interface changes. +Memory watchpoints are also architecture dependent. + +These are memory watchpoints provided by the kernel. +Whenever a memory address is written to (`w`), read +from (`r`), or executed (`x`), the kernel can generate an event. + +In the first form, an absolute address is monitored. +If a pid (`-p`) or a command (`-c`) is provided, bpftrace takes the address as a userspace address and monitors the appropriate process. +If not, bpftrace takes the address as a kernel space address. + +In the second form, the address present in `argN` when `function` is entered is +monitored. +A pid or command must be provided for this form. +If synchronous (`watchpoint`), a `SIGSTOP` is sent to the tracee upon function entry. +The tracee will be ``SIGCONT``ed after the watchpoint is attached. +This is to ensure events are not missed. +If you want to avoid the `SIGSTOP` + `SIGCONT` use `asyncwatchpoint`. + +Note that on most architectures you may not monitor for execution while monitoring read or write. + +``` +# bpftrace -e 'watchpoint:0x10000000:8:rw { printf("hit!\n"); }' -c ./testprogs/watchpoint +``` + +Print the call stack every time the `jiffies` variable is updated: + +``` +watchpoint:0x$(awk '$3 == "jiffies" {print $1}' /proc/kallsyms):8:w { + @[kstack] = count(); +} +``` + +"hit" and exit when the memory pointed to by `arg1` of `increment` is written to: + +```C +# cat wpfunc.c +#include +#include +#include + +__attribute__((noinline)) +void increment(__attribute__((unused)) int _, int *i) +{ + (*i)++; +} + +int main() +{ + int *i = malloc(sizeof(int)); + while (1) + { + increment(0, i); + (*i)++; + usleep(1000); + } +} +``` + +``` +# bpftrace -e 'watchpoint:increment+arg1:4:w { printf("hit!\n"); exit() }' -c ./wpfunc +``` + +Note that threads are monitored, but only for threads created after watchpoint attachment. +The is a limitation from the kernel. +Additionally, because of how watchpoints are implemented in bpftrace the specified function must be called at least once in the main thread in order to observe future calls to this function in child threads. + +## Pointers + +Pointers in bpftrace are similar to those found in `C`. + +## Structs + +`C` like structs are supported by bpftrace. +Fields are accessed with the `.` operator. +Fields of a pointer to a struct can be accessed with the `\->` operator. + +Custom structs can be defined in the preamble. + +Constructing structs from scratch, like `struct X var = {.f1 = 1}` in `C`, is not supported. +They can only be read into a variable from a pointer. + +``` +struct MyStruct { + int a; +} + +kprobe:dummy { + $ptr = (struct MyStruct *) arg0; + $st = *$ptr; + print($st.a); + print($ptr->a); +} +``` + +## Tuples + +bpftrace has support for immutable N-tuples (`n > 1`). +A tuple is a sequence type (like an array) where, unlike an array, every element can have a different type. + +Tuples are a comma separated list of expressions, enclosed in brackets, `(1,2)` +Individual fields can be accessed with the `.` operator. +Tuples are zero indexed like arrays are. + +``` +interval:s:1 { + $a = (1,2); + $b = (3,4, $a); + print($a); + print($b); + print($b.0); +} + +/* + * Sample output: + * (1, 2) + * (3, 4, (1, 2)) + * 3 + */ +``` + +## Type conversion + +Integer and pointer types can be converted using explicit type conversion with an expression like: + +``` +$y = (uint32) $z; +$py = (int16 *) $pz; +``` + +Integer casts to a higher rank are sign extended. +Conversion to a lower rank is done by zeroing leading bits. + +It is also possible to cast between integers and integer arrays using the same syntax: + +``` +$a = (uint8[8]) 12345; +$x = (uint64) $a; +``` + +Both the cast and the destination type must have the same size. +When casting to an array, it is possible to omit the size which will be determined automatically from the size of the cast value. + +Integers are internally represented as 64 bit signed. If you need another representation, you may cast to the supported [Data Types](#data-types). + +### Array casts + +It is possible to cast between integer arrays and integers. +Both the source and the destination type must have the same size. +The main purpose of this is to allow casts from/to byte arrays. + +``` +begin { + $a = (int8[8])12345; + printf("%x %x\n", $a[0], $a[1]); + printf("%d\n", (uint64)$a); +} + +/* + * Output: + * 39 30 + * 12345 + */ +``` + +When casting to an array, it is possible to omit the size which will be determined automatically from the size of the cast value. + +This feature is especially useful when working with IP addresses since various libraries, builtins, and parts of the kernel use different approaches to represent addresses (usually byte arrays vs. integers). +Array casting allows seamless comparison of such representations: + +``` +fentry:tcp_connect { + if (args->sk->__sk_common.skc_daddr == (uint32)pton("127.0.0.1")) + ... +} +``` + +## Variables and Maps + +bpftrace knows two types of variables, 'scratch' and 'map'. + +'scratch' variables are kept on the BPF stack and their names always start +with a `$`, e.g. `$myvar`. +'scratch' variables cannot be accessed outside of their lexical block e.g. +``` +$a = 1; +if ($a == 1) { + $b = "hello" + $a = 2; +} +``` + +'scratch' variables can also declared before or during initialization with `let` e.g. +``` +let $a = 1; +let $b; +if ($a == 1) { + $b = "hello" + $a = 2; +} +``` + +If no assignment is specified variables will initialize to 0. + +'map' variables use BPF 'maps'. +These exist for the lifetime of `bpftrace` itself and can be accessed from all action blocks and user-space. +Map names always start with a `@`, e.g. `@mymap`. + +All valid identifiers can be used as `name`. + +The data type of a variable is automatically determined during first assignment and cannot be changed afterwards. + +### Map Declarations + +***Warning*** this feature is experimental and may be subject to changes. +Stabilization is tracked in [#4077](https://github.com/bpftrace/bpftrace/issues/4077). + +Maps can also be declared in the global scope, before probes and after the config e.g. +``` +let @a = hash(100); +let @b = percpulruhash(20); + +begin { ... } +``` + +The utility of this is that you can specify different underlying BPF map types. +Currently these are available in bpftrace: +- hash (BPF_MAP_TYPE_HASH) +- lruhash (BPF_MAP_TYPE_LRU_HASH) +- percpuhash (BPF_MAP_TYPE_PERCPU_HASH) +- percpulruhash (BPF_MAP_TYPE_LRU_PERCPU_HASH) +- percpuarray (BPF_MAP_TYPE_PERCPU_ARRAY) + +Additionally, map declarations must supply a single argument: ***max entries*** e.g. `let @a = lruhash(100);` +All maps that are not declared in the global scope utilize the default set in the config variable "max_map_keys". +However, it’s best practice to declare maps up front as using the default can lead to lost map update events (if the map is full) or over allocation of memory if the map is intended to only store a few entries. + +***Warning*** The "lru" variants of hash and percpuhash evict the approximately least recently used elements. In other words, users should not rely on the accuracy on the part of the eviction algorithm. Adding a single new element may cause one or multiple elements to be deleted if the map is at capacity. [Read more about LRU internals](https://docs.ebpf.io/linux/map-type/BPF_MAP_TYPE_LRU_HASH/). + +### Maps without Explicit Keys + +Values can be assigned directly to maps without a key (sometimes refered to as scalar maps). +Note: you can’t iterate over these maps as they don’t have an accessible key. + +``` +@name = expression +``` + +### Map Keys + +Setting single value map keys. + +``` +@name[key] = expression +``` + +Map keys that are composed of multiple values are represented as tuples e.g. + +``` +@name[(key1,key2)] = expression +``` + +However, this, more concise, syntax is supported and the same as the explicit +tuple above: + +``` +@name[key1,key2] = expression +``` + +Just like with any variable the type is determined on first use and cannot be modified afterwards. +This applies to both the key(s) and the value type. + +The following snippets create a map with key signature `(int64, string)` and a value type of `int64`: + +``` +@[pid, comm]++ +@[(pid, comm)]++ +``` + +### Per-Thread Variables + +These can be implemented as a map keyed on the thread ID. For example, `@start[tid]`: + +``` +kprobe:do_nanosleep { + @start[tid] = nsecs; +} + +kretprobe:do_nanosleep /has_key(@start, tid)/ { + printf("slept for %d ms\n", (nsecs - @start[tid]) / 1000000); + delete(@start, tid); +} + +/* + * Sample output: + * slept for 1000 ms + * slept for 1009 ms + * slept for 2002 ms + * ... + */ +``` + +This style of map may also be useful for capturing output parameters, or other context, between two different probes. For example: + +``` +tracepoint:syscalls:sys_enter_wait4 +{ + @out[tid] = args.ru; +} + +tracepoint:syscalls:sys_exit_wait4 +{ + $ru = @out[tid]; + delete(@out, tid); + if ($ru != 0) { + printf("got usage ...", ...); + } +} +``` + +--- + +## Advanced Topics + +### Address Spaces + +Kernel and user pointers live in different address spaces which, depending on the CPU architecture, might overlap. +Trying to read a pointer that is in the wrong address space results in a runtime error. +This error is hidden by default but can be enabled with the `-k` flag: + +``` +stdin:1:9-12: WARNING: Failed to probe_read_user: Bad address (-14) +begin { @=*uptr(kaddr("do_poweroff")) } + ~~~ +``` + +bpftrace tries to automatically set the correct address space for a pointer based on the probe type, but might fail in cases where it is unclear. +The address space can be changed with the [kptrs](stdlib#kptr) and [uptr](stdlib#uptr) functions. + +### BPF License + +By default bpftrace uses "GPL", which is actually "GPL version 2", as the license it uses to load BPF programs into the kernel. +Some other examples of compatible licenses are: "GPL v2" and "Dual MPL/GPL". +You can specify a different license using the "license" config variable. +[Read more about BPF programs and licensing](https://docs.kernel.org/bpf/bpf_licensing.html#using-bpf-programs-in-the-linux-kernel). + +### BTF Support + +If the kernel version has BTF support, kernel types are automatically available and there is no need to include additional headers to use them. +It is not recommended to mix definitions from multiple sources (ie. BTF and header files). +If your program mixes definitions, bpftrace will do its best but can easily get confused due to redefinition conflicts. +Prefer to exclusively use BTF as it can never get out of sync on a running system. BTF is also less susceptible to parsing failures (C is constantly evolving). +Almost all current linux deployments will support BTF. + +To allow users to detect this situation in scripts, the preprocessor macro `BPFTRACE_HAVE_BTF` is defined if BTF is detected. +See `tools/` for examples of its usage. + +Requirements for using BTF for vmlinux: + +* Linux 4.18+ with CONFIG_DEBUG_INFO_BTF=y + * Building requires dwarves with pahole v1.13+ +* bpftrace v0.9.3+ with BTF support (built with libbpf v0.0.4+) + +Additional requirements for using BTF for kernel modules: + +* Linux 5.11+ with CONFIG_DEBUG_INFO_BTF_MODULES=y + * Building requires dwarves with pahole v1.19+ + +See kernel documentation for more information on BTF. + +### Clang Environment Variables + +bpftrace parses header files using libclang, the C interface to Clang. +Thus environment variables affecting the clang toolchain can be used. +For example, if header files are included from a non-default directory, the `CPATH` or `C_INCLUDE_PATH` environment variables can be set to allow clang to locate the files. +See clang documentation for more information on these environment variables and their usage. + +### Complex Tools + +bpftrace can be used to create some powerful one-liners and some simple tools. +For complex tools, which may involve command line options, positional parameters, argument processing, and customized output, consider switching to bcc. +bcc provides Python (and other) front-ends, enabling usage of all the other Python libraries (including argparse), as well as a direct control of the kernel BPF program. +The down side is that bcc is much more verbose and laborious to program. +Together, bpftrace and bcc are complimentary. + +An expected development path would be exploration with bpftrace one-liners, then and ad hoc scripting with bpftrace, then finally, when needed, advanced tooling with bcc. + +As an example of bpftrace vs bcc differences, the bpftrace xfsdist.bt tool also exists in bcc as xfsdist.py. Both measure the same functions and produce the same summary of information. +However, the bcc version supports various arguments: + +``` +# ./xfsdist.py -h +usage: xfsdist.py [-h] [-T] [-m] [-p PID] [interval] [count] + +Summarize XFS operation latency + +positional arguments: + interval output interval, in seconds + count number of outputs + +optional arguments: + -h, --help show this help message and exit + -T, --notimestamp don't include timestamp on interval output + -m, --milliseconds output in milliseconds + -p PID, --pid PID trace this PID only + +examples: + ./xfsdist # show operation latency as a histogram + ./xfsdist -p 181 # trace PID 181 only + ./xfsdist 1 10 # print 1 second summaries, 10 times + ./xfsdist -m 5 # 5s summaries, milliseconds +``` + +The bcc version is 131 lines of code. The bpftrace version is 22. + +### Errors + +1. Looks like the BPF stack limit of 512 bytes is exceeded BPF programs that operate on many data items may hit this limit. +There are a number of things you can try to stay within the limit: + 1. Find ways to reduce the size of the data used in the program. Eg, avoid strings if they are unnecessary: use pid instead of comm. Use fewer map keys. + 2. Split your program over multiple probes. + 3. Check the status of the BPF stack limit in Linux (it may be increased in the future, maybe as a tuneable). + 4. (advanced): Run -d and examine the LLVM IR, and look for ways to optimize src/ast/codegen_llvm.cpp. +2. Kernel headers not found +bpftrace requires kernel headers for certain features, which are searched for by default in: `/lib/modules/$(uname -r)`. +The default search directory can be overridden using the environment variable BPFTRACE_KERNEL_SOURCE and also BPFTRACE_KERNEL_BUILD if it is out-of-tree Linux kernel build. + +### Map Printing + +By default when a bpftrace program exits it will print all maps to stdout. +If you don’t want this, you can either override the `print_maps_on_exit` configuration option or you can specify an `end` probe and `clear` the maps you don’t want printed. + +For example, these two scripts are equivalent and will print nothing on exit: +``` +config = { + print_maps_on_exit=0 +} + +begin { + @a = 1; + @b[1] = 1; +} +``` + +``` +begin { + @a = 1; + @b[1] = 1; +} + +end { + clear(@a); + clear(@b); +} +``` + +### PERCPU types + +For bpftrace PERCPU map types (e.g., those created by using [`count()`](stdlib#count) or [`sum()`](stdlib#sum)) you may coerce +(and thus force a more expensive synchronous read) the type to an integer using +a cast or by doing a comparison. This is useful for when you need an integer +during comparisons, `printf()`, or other. + +For example: + +``` +begin { + @c = count(); + @s = sum(3); + @s = sum(9); + + if (@s == 12) { // Coerces @s + printf("%d %d\n", (int64)@c, (int64)@s); // Coerces @c and @s and prints "1 12" + } +} +``` + +### Supported architectures + +x86_64, arm64, s390x, arm32, loongarch64, mips64, ppc64, riscv64 + +### Systemd support + +If bpftrace has been built with `-DENABLE_SYSTEMD=1`, one can run bpftrace in +the background using systemd:: +``` +# systemd-run --unit=bpftrace --service-type=notify bpftrace -e 'kprobe:do_nanosleep { printf("%d sleeping\n", pid); }' +``` + +In the above example, systemd-run will not finish until bpftrace has attached +its probes, so you can be sure that all following commands will be traced. To +stop tracing, run `systemctl stop bpftrace`. + +To debug early boot issues, bpftrace can be invoked via a systemd service +ordered before the service that needs to be traced. A basic unit file to run +bpftrace before another service looks as follows:: +``` +[Unit] +Before=service-i-want-to-trace.service + +[Service] +Type=notify +ExecStart=bpftrace -e 'kprobe:do_nanosleep { printf("%d sleeping\n", pid); }' +``` + +Similarly to the systemd-run example, the service to be traced will not start +until bpftrace started by the systemd unit has attached its probes. + +### Unstable Features + +Some features added to bpftrace are not yet stable. +They are enabled by default but come with a warning if used. +If you explicitly add the config variable to your script the warning will not be shown e.g. +``` +config = { + unstable_map_decl=enable; +} +``` + +To opt-out of these unstable features (and ensure they are not used) add the config variable and set it to `error` e.g. +``` +config = { + unstable_map_decl=error; +} +``` + +Note: all unstable features are subject to change and/or removal. \ No newline at end of file diff --git a/src/pages/docs/release_027/stdlib.md b/src/pages/docs/release_027/stdlib.md new file mode 100644 index 0000000..55070ea --- /dev/null +++ b/src/pages/docs/release_027/stdlib.md @@ -0,0 +1,1657 @@ +# bpftrace Standard Library (0.27) + +This includes builtins, functions, macros, and [map value functions](#map-value-functions). +The boundaries for the first three are blurred, by design, to allow for more flexible usage and are grouped below as "Helpers". +For example `pid` and `pid()` are equivalent; both yielding the process id. +Basically all functions or macros that don't have arguments or have default arguments can be invoked with or without the call syntax. + +**async** helpers are asynchronous, which can lead to unexpected behaviour. See the [Invocation Mode](#invocation-mode) section for more information. + +**compile time** helpers are evaluated at compile time, a static value will be compiled into the program. + +**unsafe** helpers can have dangerous side effects and should be used with care, the `--unsafe` flag is required for use. + +## Helpers + +### assert +- `void assert(bool condition, string message)` + +Simple assertion macro that will exit the entire script with an error code if the condition is not met. + + +### bswap +- `uint8 bswap(uint8 n)` +- `uint16 bswap(uint16 n)` +- `uint32 bswap(uint32 n)` +- `uint64 bswap(uint64 n)` + +`bswap` reverses the order of the bytes in integer `n`. In case of 8 bit integers, `n` is returned without being modified. +The return type is an unsigned integer of the same width as `n`. + + +### buf +- `buffer buf(void * data, [int64 length])` + +`buf` reads `length` amount of bytes from address `data`. +The maximum value of `length` is limited to the `BPFTRACE_MAX_STRLEN` variable. +For arrays the `length` is optional, it is automatically inferred from the signature. + +`buf` is address space aware and will call the correct helper based on the address space associated with `data`. + +The `buffer` object returned by `buf` can safely be printed as a hex encoded string with the `%r` format specifier. + +Bytes with values >=32 and \<=126 are printed using their ASCII character, other bytes are printed in hex form (e.g. `\x00`). The `%rx` format specifier can be used to print everything in hex form, including ASCII characters. The similar `%rh` format specifier prints everything in hex form without `\x` and with spaces between bytes (e.g. `0a fe`). + +``` +interval:s:1 { + printf("%r\n", buf(kaddr("avenrun"), 8)); +} +``` + +``` +\x00\x03\x00\x00\x00\x00\x00\x00 +\xc2\x02\x00\x00\x00\x00\x00\x00 +``` + + +### cat +- `void cat(string namefmt, [...args])` + +**async** + +Dump the contents of the named file to stdout. +`cat` supports the same format string and arguments that `printf` does. +If the file cannot be opened or read an error is printed to stderr. + +``` +tracepoint:syscalls:sys_enter_execve { + cat("/proc/%d/maps", pid); +} +``` + +``` +55f683ebd000-55f683ec1000 r--p 00000000 08:01 1843399 /usr/bin/ls +55f683ec1000-55f683ed6000 r-xp 00004000 08:01 1843399 /usr/bin/ls +55f683ed6000-55f683edf000 r--p 00019000 08:01 1843399 /usr/bin/ls +55f683edf000-55f683ee2000 rw-p 00021000 08:01 1843399 /usr/bin/ls +55f683ee2000-55f683ee3000 rw-p 00000000 00:00 0 +``` + + +### cgroup +- `uint64 cgroup()` +- `uint64 cgroup` + +ID of the cgroup the current process belongs to + +Only works with cgroupv2 + +This utilizes the BPF helper `get_current_cgroup_id` + + +### cgroup_path +- `cgroup_path_t cgroup_path(int cgroupid, string filter)` + +Convert cgroup id to cgroup path. +This is done asynchronously in userspace when the cgroup_path value is printed, +therefore it can resolve to a different value if the cgroup id gets reassigned. +This also means that the returned value can only be used for printing. + +A string literal may be passed as an optional second argument to filter cgroup +hierarchies in which the cgroup id is looked up by a wildcard expression (cgroup2 +is always represented by "unified", regardless of where it is mounted). + +The currently mounted hierarchy at /sys/fs/cgroup is used to do the lookup. If +the cgroup with the given id isn’t present here (e.g. when running in a Docker +container), the cgroup path won’t be found (unlike when looking up the cgroup +path of a process via /proc/.../cgroup). + +``` +BEGIN { + $cgroup_path = cgroup_path(3436); + print($cgroup_path); + print($cgroup_path); /* This may print a different path */ + printf("%s %s", $cgroup_path, $cgroup_path); /* This may print two different paths */ +} +``` + + +### cgroupid +- `uint64 cgroupid(const string path)` + +**compile time** + +`cgroupid` retrieves the cgroupv2 ID of the cgroup available at `path`. + +``` +BEGIN { + print(cgroupid("/sys/fs/cgroup/system.slice")); +} +``` + + +### clear +- `void clear(map m)` + +**async** + +Clear all keys/values from map `m`. + +``` +interval:ms:100 { + @[rand % 10] = count(); +} + +interval:s:10 { + print(@); + clear(@); +} +``` + + +### comm +- `string comm()` +- `string comm` + +Name of the current thread + +This utilizes the BPF helper `get_current_comm` + + +### cpid +- `uint32 cpid()` +- `uint32 cpid` + +Child process ID, if bpftrace is invoked with `-c` + + +### cpu +- `uint32 cpu()` +- `uint32 cpu` + +ID of the processor executing the BPF program + +BPF program, in this case, is the probe body + +This utilizes the BPF helper `raw_smp_processor_id` + + +### curtask +- `uint64 curtask()` +- `uint64 curtask` + +Pointer to `struct task_struct` of the current task + +This utilizes the BPF helper `get_current_task` + + +### delete +- `bool delete(map m, mapkey k)` +- deprecated `bool delete(mapkey k)` + +Delete a single key from a map. +For scalar maps (e.g. no explicit keys), the key is omitted and is equivalent to calling `clear`. +For map keys that are composed of multiple values (e.g. `@mymap[3, "hello"] = 1` - remember these values are represented as a tuple) the syntax would be: `delete(@mymap, (3, "hello"));` + +If deletion fails (e.g. the key doesn’t exist) the function returns false (0). +Additionally, if the return value for `delete` is discarded, and deletion fails, you will get a warning. + +``` +@a[1] = 1; + +delete(@a, 1); // no warning (the key exists) + +if (delete(@a, 2)) { // no warning (return value is used) + ... +} + +$did_delete = delete(@a, 2); // no warning (return value is used) + +delete(@a, 2); // warning (return value is discarded and the key doesn’t exist) +``` + +The, now deprecated, API (supported in version <= 0.21.x) of passing map arguments with the key is still supported: +e.g. `delete(@mymap[3, "hello"]);`. + +``` +kprobe:dummy { + @scalar = 1; + delete(@scalar); // ok + @single["hello"] = 1; + delete(@single, "hello"); // ok + @associative[1,2] = 1; + delete(@associative, (1,2)); // ok + delete(@associative); // error + delete(@associative, 1); // error + + // deprecated but ok + delete(@single["hello"]); + delete(@associative[1, 2]); +} +``` + + +### elapsed +- `uint64 elapsed()` +- `uint64 elapsed` + +ktime_get_ns - ktime_get_boot_ns + + +### errorf +- `void errorf(const string fmt, args...)` + +**async** + +`errorf()` formats and prints data (similar to [`printf`](#printf)) as an error message with the source location. + +``` +BEGIN { errorf("Something bad with args: %d, %s", 10, "arg2"); } +``` + +Prints: + +``` +EXPECT stdin:1:9-62: ERROR: Something bad with args: 10, arg2 +``` + + +### exit +- `void exit([int code])` + +**async** + +Terminate bpftrace, as if a `SIGTERM` was received. +The `END` probe will still trigger (if specified) and maps will be printed. +An optional exit code can be provided. + +``` +BEGIN { + exit(); +} +``` + +Or + +``` +BEGIN { + exit(1); +} +``` + + +### func +- `string func()` +- `string func` + +Name of the current function being traced (kprobes,uprobes,fentry) + + +### getopt +- `bool getopt(string arg_name)` +- `string getopt(string arg_name, string default_value)` +- `int getopt(string arg_name, int default_value)` +- `bool getopt(string arg_name, bool default_value)` + +Get the named command line argument/option e.g. +``` +# bpftrace -e 'BEGIN { print(getopt("hello", 1)); }' -- --hello=5 + +``` + +`getopt` defines the type of the argument by the default value’s type. +If no default type is provided, the option is treated like a boolean arg e.g. `getopt("hello")` would evaluate to `false` if `--hello` is not specified on the command line or `true` if `--hello` is passed or set to one of the following values: `true`, `1`. +Additionally, boolean args accept the following false values: `0`, `false` e.g. `--hello=false`. +If the arg is not set on the command line, the default value is used. + +``` +# bpftrace -e 'BEGIN { print((getopt("aa", 10), getopt("bb", "hello"), getopt("cc"), getopt("dd", false))); }' -- --cc --bb=bye + +``` + + +### gid +- `uint64 gid()` +- `uint64 gid` + +Group ID of the current thread, as seen from the init namespace + +This utilizes the BPF helper `get_current_uid_gid` + + +### has_key +- `boolean has_key(map m, mapkey k)` + +Return true (1) if the key exists in this map. +Otherwise return false (0). +Error if called with a map that has no keys (aka scalar map). +Return value can also be used for scratch variables and map keys/values. + +``` +kprobe:dummy { + @associative[1,2] = 1; + if (!has_key(@associative, (1,3))) { // ok + print(("bye")); + } + + @scalar = 1; + if (has_key(@scalar)) { // error + print(("hello")); + } + + $a = has_key(@associative, (1,2)); // ok + @b[has_key(@associative, (1,2))] = has_key(@associative, (1,2)); // ok +} +``` + + +### jiffies +- `uint64 jiffies()` +- `uint64 jiffies` + +Jiffies of the kernel + +On 32-bit systems, using this builtin might be slower + +This utilizes the BPF helper `get_jiffies_64` + + +### join +- `void join(char *arr[], [char * sep = ' '])` + +**async** + +`join` joins a char * `arr` with `sep` as separator into one string. +This string will be printed to stdout directly, it cannot be used as string value. + +The concatenation of the array members is done in BPF and the printing happens in userspace. + +``` +tracepoint:syscalls:sys_enter_execve { + join(args.argv); +} +``` + + +### kaddr +- `uint64 kaddr(const string name)` + +**compile time** + +Get the address of the kernel symbol `name`. + +``` +interval:s:1 { + $avenrun = kaddr("avenrun"); + $load1 = *$avenrun; +} +``` + +You can find all kernel symbols at `/proc/kallsyms`. + + +### kptr +- `T * kptr(T * ptr)` + +Marks `ptr` as a kernel address space pointer. +See the address-spaces section for more information on address-spaces. +The pointer type is left unchanged. + + +### kstack +- `kstack_t kstack([StackMode mode, ][int limit])` + +These are implemented using BPF stack maps. + +``` +kprobe:ip_output { @[kstack()] = count(); } + +/* + * Sample output: + * @[ + * ip_output+1 + * tcp_transmit_skb+1308 + * tcp_write_xmit+482 + * tcp_release_cb+225 + * release_sock+64 + * tcp_sendmsg+49 + * sock_sendmsg+48 + * sock_write_iter+135 + * __vfs_write+247 + * vfs_write+179 + * sys_write+82 + * entry_SYSCALL_64_fastpath+30 + * ]: 1708 + */ +``` + +Sampling only three frames from the stack (limit = 3): + +``` +kprobe:ip_output { @[kstack(3)] = count(); } + +/* + * Sample output: + * @[ + * ip_output+1 + * tcp_transmit_skb+1308 + * tcp_write_xmit+482 + * ]: 1708 + */ +``` + +You can also choose a different output format. +Available formats are `bpftrace`, `perf`, and `raw` (no symbolication): + +``` +kprobe:ip_output { @[kstack(perf, 3)] = count(); } + +/* + * Sample output: + * @[ + * ffffffffb4019501 do_mmap+1 + * ffffffffb401700a sys_mmap_pgoff+266 + * ffffffffb3e334eb sys_mmap+27 + * ]: 1708 + */ +``` + + +### ksym +- `ksym_t ksym(uint64 addr)` + +**async** + +Retrieve the name of the function that contains address `addr`. +The address to name mapping happens in user-space. + +The `ksym_t` type can be printed with the `%s` format specifier. + +``` +kprobe:do_nanosleep +{ + printf("%s\n", ksym(reg("ip"))); +} + +/* + * Sample output: + * do_nanosleep + */ +``` + + +### len +- `int64 len(map m)` +- `int64 len(ustack stack)` +- `int64 len(kstack stack)` + +For maps, return the number of elements in the map. + +For kstack/ustack, return the depth (measured in # of frames) of the call stack. + + +### macaddr +- `macaddr_t macaddr(char [6] mac)` + +Create a buffer that holds a macaddress as read from `mac` +This buffer can be printed in the canonical string format using the `%s` format specifier. + +``` +kprobe:arp_create { + $stack_arg0 = *(uint8*)(reg("sp") + 8); + $stack_arg1 = *(uint8*)(reg("sp") + 16); + printf("SRC %s, DST %s\n", macaddr($stack_arg0), macaddr($stack_arg1)); +} + +/* + * Sample output: + * SRC 18:C0:4D:08:2E:BB, DST 74:83:C2:7F:8C:FF + */ +``` + + +### ncpus +- `uint64 ncpus()` +- `uint64 ncpus` + +Number of CPUs + + +### nsecs +- `timestamp nsecs([TimestampMode mode])` +- `nsecs(monotonic) - nanosecond timestamp since boot, exclusive of time the system spent suspended (CLOCK_MONOTONIC)` +- `nsecs(boot) - nanoseconds since boot, inclusive of time the system spent suspended (CLOCK_BOOTTIME)` +- `nsecs(tai) - TAI timestamp in nanoseconds (CLOCK_TAI)` +- `nsecs(sw_tai) - approximation of TAI timestamp in nanoseconds, is obtained through the "triple vdso sandwich" method. For older kernels without direct TAI timestamp access in BPF.` + +Returns a timestamp in nanoseconds, as given by the requested kernel clock. +Defaults to `boot` if no clock is explicitly requested. + + +``` +interval:s:1 { + $sw_tai1 = nsecs(sw_tai); + $tai = nsecs(tai); + $sw_tai2 = nsecs(sw_tai); + printf("sw_tai precision: %lldns\n", ($sw_tai1 + $sw_tai2)/2 - $tai); +} + +/* + * Sample output: + * sw_tai precision: -98ns + * sw_tai precision: -99ns + * ... + */ +``` + + +### ntop +- `inet ntop([int64 af, ] int addr)` +- `inet ntop([int64 af, ] char addr[4])` +- `inet ntop([int64 af, ] char addr[16])` + +`ntop` returns the string representation of an IPv4 or IPv6 address. +`ntop` will infer the address type (IPv4 or IPv6) based on the `addr` type and size. +If an integer or `char[4]` is given, ntop assumes IPv4, if a `char[16]` is given, ntop assumes IPv6. +You can also pass the address type (e.g. AF_INET) explicitly as the first parameter. + + +### numaid +- `uint32 numaid()` +- `uint32 numaid` + +ID of the NUMA node executing the BPF program + +BPF program, in this case, is the probe body + +This utilizes the BPF helper `numa_node_id` + + +### offsetof +- `uint64 offsetof(STRUCT, FIELD[.SUBFIELD])` +- `uint64 offsetof(EXPRESSION, FIELD[.SUBFIELD])` + +**compile time** + +Returns offset of the field offset bytes in struct. +Similar to kernel `offsetof` operator. + +Support any number of sub field levels, for example: + +``` +struct Foo { + struct { + struct { + struct { + int d; + } c; + } b; + } a; +} +BEGIN { + @x = offsetof(struct Foo, a.b.c.d); + exit(); +} +``` + + +### override +- `void override(uint64 rc)` + +**unsafe** + +**Kernel** 4.16 + +This utilizes the BPF helper `bpf_override` + +**Supported probes** + +* kprobe + +When using `override` the probed function will not be executed and instead `rc` will be returned. + +``` +kprobe:__x64_sys_getuid +/comm == "id"/ { + override(2<<21); +} +``` + +``` +uid=4194304 gid=0(root) euid=0(root) groups=0(root) +``` + +This feature only works on kernels compiled with `CONFIG_BPF_KPROBE_OVERRIDE` and only works on functions tagged `ALLOW_ERROR_INJECTION`. + +bpftrace does not test whether error injection is allowed for the probed function, instead if will fail to load the program into the kernel: + +``` +ioctl(PERF_EVENT_IOC_SET_BPF): Invalid argument +Error attaching probe: 'kprobe:vfs_read' +``` + + +### path +- `char * path(struct path * path [, int32 size])` + +**Kernel** 5.10 + +This utilizes the BPF helper `bpf_d_path` + +Return full path referenced by struct path pointer in argument. If `size` is set, +the path will be clamped by `size` otherwise `BPFTRACE_MAX_STRLEN` is used. + +If `size` is smaller than the resolved path, the resulting string will be truncated at the front rather than at the end. + +This function can only be used by functions that are allowed to, these functions are contained in the `btf_allowlist_d_path` set in the kernel. + + +### percpu_kaddr +- `uint64 *percpu_kaddr(const string name)` +- `uint64 *percpu_kaddr(const string name, int cpu)` + +**sync** + +Get the address of the percpu kernel symbol `name` for CPU `cpu`. When `cpu` is +omitted, the current CPU is used. + +``` +interval:s:1 { + $proc_cnt = percpu_kaddr("process_counts"); + printf("% processes are running on CPU %d\n", *$proc_cnt, cpu); +} +``` + +The second variant may return NULL if `cpu` is higher than the number of +available CPUs. Therefore, it is necessary to perform a NULL-check on the result +when accessing fields of the pointed structure, otherwise the BPF program will +be rejected. + +``` +interval:s:1 { + $runqueues = (struct rq *)percpu_kaddr("runqueues", 0); + if ($runqueues != 0) { // The check is mandatory here + print($runqueues->nr_running); + } +} +``` + + +### pid +- `uint32 pid([curr_ns|init])` +- `uint32 pid` + +Returns the process ID of the current thread. +Defaults to `curr_ns`. + +* `pid(curr_ns)` - The process ID as seen from the PID namespace of bpftrace. +* `pid(init)` - The process ID as seen from the initial PID namespace. + + +### ppid +- `uint32 ppid(struct task_struct * task)` + +Get the pid of the parent process + + +### print +- `void print(T val)` + +**async** + + +### printf +- `void printf(const string fmt, args...)` + +**async** + +`printf()` formats and prints data. +It behaves similar to `printf()` found in `C` and many other languages. + +The format string has to be a constant, it cannot be modified at runtime. +The formatting of the string happens in user space. +Values are copied and passed by value. + +bpftrace supports all the typical format specifiers like `%llx` and `%hhu`. +The non-standard ones can be found in the table below: + +| Specifier | Type | Description | +| --- | --- | --- | +| r | buffer | Hex-formatted string to print arbitrary binary content returned by the [buf](#buf) function. | +| rh | buffer | Prints in hex-formatted string without `\x` and with spaces between bytes (e.g. `0a fe`) | + +`printf()` can also symbolize enums as strings. User defined enums as well as enums +defined in the kernel are supported. For example: + +``` +enum custom { + CUSTOM_ENUM = 3, +}; + +BEGIN { + $r = SKB_DROP_REASON_SOCKET_FILTER; + printf("%d, %s, %s\n", $r, $r, CUSTOM_ENUM); + exit(); +} +``` + +yields: + +``` +6, SKB_DROP_REASON_SOCKET_FILTER, CUSTOM_ENUM +``` + +Colors are supported too, using standard terminal escape sequences: + +``` +print("\033[31mRed\t\033[33mYellow\033[0m\n") +``` + + +### probe +- `string probe()` +- `string probe` + +Name of the fully expanded probe + +For example: `kprobe:do_nanosleep` + + +### pton +- `char addr[4] pton(const string *addr_v4)` +- `char addr[16] pton(const string *addr_v6)` + +**compile time** + +`pton` converts a text representation of an IPv4 or IPv6 address to byte array. +`pton` infers the address family based on `.` or `:` in the given argument. +`pton` comes in handy when we need to select packets with certain IP addresses. + + +### rand +- `uint32 rand()` +- `uint32 rand` + +Get a pseudo random number + +This utilizes the BPF helper `get_prandom_u32` + + +### reg +- `uint64 reg(const string name)` + +**Supported probes** + +* kprobe +* uprobe + +Get the contents of the register identified by `name`. +Valid names depend on the CPU architecture. + + +### retval +- `uint64 retval()` +- `uint64 retval` + +Value returned by the function being traced + +(kretprobe, uretprobe, fexit) +For kretprobe and uretprobe, its type is uint64, but for fexit it depends. You can look up the type using `bpftrace -lv` + + +### signal +- `void signal(const string sig)` +- `void signal(uint32 signum)` + +**unsafe** + +**Kernel** 5.3 + +This utilizes the BPF helper `bpf_send_signal` + +Probe types: k(ret)probe, u(ret)probe, USDT, profile + +Send a signal to the process being traced. +The signal can either be identified by name, e.g. `SIGSTOP` or by ID, e.g. `19` as found in `kill -l`. + +``` +kprobe:__x64_sys_execve +/comm == "bash"/ { + signal(5); +} +``` +``` +$ ls +Trace/breakpoint trap (core dumped) +``` + + +### sizeof +- `uint64 sizeof(TYPE)` +- `uint64 sizeof(EXPRESSION)` + +**compile time** + +Returns size of the argument in bytes. +Similar to C/C++ `sizeof` operator. +Note that the expression does not get evaluated. + + +### skboutput +- `uint32 skboutput(const string path, struct sk_buff *skb, uint64 length, const uint64 offset)` + +**Kernel** 5.5 + +This utilizes the BPF helper `bpf_skb_output` + +Write sk_buff `skb` 's data section to a PCAP file in the `path`, starting from `offset` to `offset` + `length`. + +The PCAP file is encapsulated in RAW IP, so no ethernet header is included. +The `data` section in the struct `skb` may contain ethernet header in some kernel contexts, you may set `offset` to 14 bytes to exclude ethernet header. + +Each packet’s timestamp is determined by adding `nsecs` and boot time, the accuracy varies on different kernels, see `nsecs`. + +This function returns 0 on success, or a negative error in case of failure. + +Environment variable `BPFTRACE_PERF_RB_PAGES` should be increased in order to capture large packets, or else these packets will be dropped. + +Usage + +``` +# cat dump.bt +fentry:napi_gro_receive { + $ret = skboutput("receive.pcap", args.skb, args.skb->len, 0); +} + +fentry:dev_queue_xmit { + // setting offset to 14, to exclude ethernet header + $ret = skboutput("output.pcap", args.skb, args.skb->len, 14); + printf("skboutput returns %d\n", $ret); +} + +# export BPFTRACE_PERF_RB_PAGES=1024 +# bpftrace dump.bt +... + +# tcpdump -n -r ./receive.pcap | head -3 +reading from file ./receive.pcap, link-type RAW (Raw IP) +dropped privs to tcpdump +10:23:44.674087 IP 22.128.74.231.63175 > 192.168.0.23.22: Flags [.], ack 3513221061, win 14009, options [nop,nop,TS val 721277750 ecr 3115333619], length 0 +10:23:45.823194 IP 100.101.2.146.53 > 192.168.0.23.46619: 17273 0/1/0 (130) +10:23:45.823229 IP 100.101.2.146.53 > 192.168.0.23.46158: 45799 1/0/0 A 100.100.45.106 (60) +``` + + +### socket_cookie +- `uint64 socket_cookie(struct sock *sk)` + +This utilizes the BPF helper `bpf_get_socket_cookie` + +Retrieve the cookie (generated by the kernel) of the socket. +If no cookie has been set yet, generate a new cookie. Once generated, the socket cookie remains stable for the life of the socket. + +This function returns a `uint64` unique number on success, or 0 if **sk** is NULL. + +``` +fentry:tcp_rcv_established +{ + $cookie = socket_cookie(args->sk); + @psize[$cookie] = hist(args->skb->len); +} +``` + +Prints: + +``` +@psize[65551]: +[32, 64) 4 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| + +@psize[504]: +[32, 64) 4 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| +[64, 128) 1 |@@@@@@@@@@@@@ | +[128, 256) 0 | | +[256, 512) 1 |@@@@@@@@@@@@@ | +[512, 1K) 0 | | +[1K, 2K) 0 | | +[2K, 4K) 1 |@@@@@@@@@@@@@ | +``` + + +### str +- `string str(char * data [, uint32 length)` + +This utilizes the BPF helpers `probe_read_str, probe_read_{kernel,user}_str` + +`str` reads a NULL terminated (`\0`) string from `data`. +The maximum string length is limited by the `BPFTRACE_MAX_STRLEN` env variable, unless `length` is specified and shorter than the maximum. +In case the string is longer than the specified length only `length - 1` bytes are copied and a NULL byte is appended at the end. + +When available (starting from kernel 5.5, see the `--info` flag) bpftrace will automatically use the `kernel` or `user` variant of `probe_read_{kernel,user}_str` based on the address space of `data`, see [Address-spaces](./language#address-spaces) for more information. + + +### strcontains +- `int64 strcontains(const char *haystack, const char *needle)` + +`strcontains` compares whether the string haystack contains the string needle. +If needle is contained `1` is returned, else zero is returned. + +bpftrace doesn’t read past the length of the shortest string. + + +### strerror +- `strerror_t strerror(int error)` + +Convert errno code to string. +This is done asynchronously in userspace when the strerror value is printed, hence the returned value can only be used for printing. + +``` +#include +BEGIN { + print(strerror(EPERM)); +} +``` + + +### strftime +- `timestamp strftime(const string fmt, int64 timestamp_ns)` + +**async** + +Format the nanoseconds since boot timestamp `timestamp_ns` according to the format specified by `fmt`. +The time conversion and formatting happens in user space, therefore the `timestamp` value returned can only be used for printing using the `%s` format specifier. + +bpftrace uses the `strftime(3)` function for formatting time and supports the same format specifiers. + +``` +interval:s:1 { + printf("%s\n", strftime("%H:%M:%S", nsecs)); +} +``` + +bpftrace also supports the following format string extensions: + +| Specifier | Description | +| --- | --- | +| `%f` | Microsecond as a decimal number, zero-padded on the left | + + +### strncmp +- `int64 strncmp(char * s1, char * s2, int64 n)` + +`strncmp` compares up to `n` characters string `s1` and string `s2`. +If they’re equal `0` is returned, else a non-zero value is returned. + +bpftrace doesn’t read past the length of the shortest string. + +The use of the `==` and `!=` operators is recommended over calling `strncmp` directly. + + +### system +- `void system(string namefmt [, ...args])` + +**unsafe** +**async** + +`system` lets bpftrace run the specified command (`fork` and `exec`) until it completes and print its stdout. +The `command` is run with the same privileges as bpftrace and it blocks execution of the processing threads which can lead to missed events and delays processing of async events. + +``` +interval:s:1 { + time("%H:%M:%S: "); + printf("%d\n", @++); +} +interval:s:10 { + system("/bin/sleep 10"); +} +interval:s:30 { + exit(); +} +``` + +Note how the async `time` and `printf` first print every second until the `interval:s:10` probe hits, then they print every 10 seconds due to bpftrace blocking on `sleep`. + +``` +Attached 3 probes +08:50:37: 0 +08:50:38: 1 +08:50:39: 2 +08:50:40: 3 +08:50:41: 4 +08:50:42: 5 +08:50:43: 6 +08:50:44: 7 +08:50:45: 8 +08:50:46: 9 +08:50:56: 10 +08:50:56: 11 +08:50:56: 12 +08:50:56: 13 +08:50:56: 14 +08:50:56: 15 +08:50:56: 16 +08:50:56: 17 +08:50:56: 18 +08:50:56: 19 +``` + +`system` supports the same format string and arguments that `printf` does. + +``` +tracepoint:syscalls:sys_enter_execve { + system("/bin/grep %s /proc/%d/status", "vmswap", pid); +} +``` + + +### tid +- `uint32 tid([curr_ns|init])` +- `uint32 tid` + +Returns the thread ID of the current thread. +Defaults to `curr_ns`. + +* `tid(curr_ns)` - The thread ID as seen from the PID namespace of bpftrace. +* `tid(init)` - The thread ID as seen from the initial PID namespace. + + +### time +- `void time(const string fmt)` + +**async** + +Format the current wall time according to the format specifier `fmt` and print it to stdout. +Unlike `strftime()` `time()` doesn’t send a timestamp from the probe, instead it is the time at which user-space processes the event. + +bpftrace uses the `strftime(3)` function for formatting time and supports the same format specifiers. + + +### uaddr +- `T * uaddr(const string sym)` + +**Supported probes** + +* uprobes +* uretprobes +* USDT + +***Does not work with ASLR, see issue [#75](https://github.com/bpftrace/bpftrace/issues/75)*** + +The `uaddr` function returns the address of the specified symbol. +This lookup happens during program compilation and cannot be used dynamically. + +The default return type is `uint64*`. +If the ELF object size matches a known integer size (1, 2, 4 or 8 bytes) the return type is modified to match the width (`uint8*`, `uint16*`, `uint32*` or `uint64*` resp.). +As ELF does not contain type info the type is always assumed to be unsigned. + +``` +uprobe:/bin/bash:readline { + printf("PS1: %s\n", str(*uaddr("ps1_prompt"))); +} +``` + + +### uid +- `uint64 uid()` +- `uint64 uid` + +User ID of the current thread, as seen from the init namespace + +This utilizes the BPF helper `get_current_uid_gid` + + +### unwatch +- `void unwatch(void * addr)` + +**async** + +Removes a watchpoint + + +### uptr +- `T * uptr(T * ptr)` + +Marks `ptr` as a user address space pointer. +See the address-spaces section for more information on address-spaces. +The pointer type is left unchanged. + + +### usermode +- `uint8 usermode()` +- `uint8 usermode` + +Returns 1 if the current process is in user mode, 0 otherwise + +Currently only available on x86_64. + + +### username +- `string username()` +- `string username` + +Get the current username + +Often this is just "root" + + +### ustack +- `ustack_t ustack([StackMode mode, ][int limit])` + +These are implemented using BPF stack maps. + +``` +kprobe:do_sys_open /comm == "bash"/ { @[ustack()] = count(); } + +/* + * Sample output: + * @[ + * __open_nocancel+65 + * command_word_completion_function+3604 + * rl_completion_matches+370 + * bash_default_completion+540 + * attempt_shell_completion+2092 + * gen_completion_matches+82 + * rl_complete_internal+288 + * rl_complete+145 + * _rl_dispatch_subseq+647 + * _rl_dispatch+44 + * readline_internal_char+479 + * readline_internal_charloop+22 + * readline_internal+23 + * readline+91 + * yy_readline_get+152 + * yy_readline_get+429 + * yy_getc+13 + * shell_getc+469 + * read_token+251 + * yylex+192 + * yyparse+777 + * parse_command+126 + * read_command+207 + * reader_loop+391 + * main+2409 + * __libc_start_main+231 + * 0x61ce258d4c544155 + * ]: 9 + */ +``` + +Sampling only three frames from the stack (limit = 3): + +``` +kprobe:ip_output { @[ustack(3)] = count(); } + +/* + * Sample output: + * @[ + * __open_nocancel+65 + * command_word_completion_function+3604 + * rl_completion_matches+370 + * ]: 20 + */ +``` + +You can also choose a different output format. +Available formats are `bpftrace`, `perf`, and `raw` (no symbolication): + +``` +kprobe:ip_output { @[ustack(perf, 3)] = count(); } + +/* + * Sample output: + * @[ + * 5649feec4090 readline+0 (/home/mmarchini/bash/bash/bash) + * 5649fee2bfa6 yy_readline_get+451 (/home/mmarchini/bash/bash/bash) + * 5649fee2bdc6 yy_getc+13 (/home/mmarchini/bash/bash/bash) + * ]: 20 + */ +``` + +Note that for these examples to work, bash had to be recompiled with frame pointers. + + +### usym +- `usym_t usym(uint64 * addr)` + +**async** + +**Supported probes** + +* uprobes +* uretprobes + +Equal to [ksym](#ksym) but resolves user space symbols. + +If ASLR is enabled, user space symbolication only works when the process is running at either the time of the symbol resolution or the time of the probe attachment. The latter requires `BPFTRACE_CACHE_USER_SYMBOLS` to be set to `PER_PID`, and might not work with older versions of BCC. A similar limitation also applies to dynamically loaded symbols. + +``` +uprobe:/bin/bash:readline +{ + printf("%s\n", usym(reg("ip"))); +} + +/* + * Sample output: + * readline + */ +``` + + +### zero +- `void zero(map m)` + +**async** + +Set all values (for all keys) in the map to zero. + + +## Map Value Functions + +Map value functions can only be assigned to maps (when scalar) or map keys. +The data types associated with these functions are only for internal use but many can be cast to integers (e.g. `count_t` and `sum_t`). + +### avg + +* `avg_t avg(int64 n)` + +Calculate the running average of `n` between consecutive calls. + +``` +interval:s:1 { + @x++; + @y = avg(@x); + print(@x); + print(@y); +} +``` + +Internally this keeps two values in the map: value count and running total. +The average is computed in user-space when printing by dividing the total by the +count. However, you can get the average in kernel space in expressions like +`if (@y == 5)` but this is expensive as bpftrace needs to iterate over all the +cpus to collect and sum BOTH count and total. + +### count + +* `count_t count()` + +Count how often this function is called. + +Using `@=count()` is conceptually similar to `@++`. +The difference is that the `count()` function uses a map type optimized for +performance and correctness using cheap, thread-safe writes ([PERCPU](./language#percpu-types)). However, sync reads +can be expensive as bpftrace needs to iterate over all the cpus to collect and +sum these values. + +Note: This differs from "raw" writes (e.g. `@++`) where multiple writers to a +shared location might lose updates, as bpftrace does not generate any atomic instructions +for `++`. + +Example one: +``` +BEGIN { + @ = count(); + @ = count(); + printf("%d\n", (int64)@); // prints 2 + exit(); +} +``` + +Example two: +``` +interval:ms:100 { + @ = count(); +} + +interval:s:10 { + // async read + print(@); + // sync read + if (@ > 10) { + print(("hello")); + } + clear(@); +} +``` + +### hist + +* `hist_t hist(int64 n[, int k])` + +Create a log2 histogram of `n` using $2^k$ buckets per power of 2, +0 <= k <= 5, defaults to 0. + +``` +kretprobe:vfs_read { + @bytes = hist(retval); +} +``` + +Prints: + +``` +@: +[1M, 2M) 3 | | +[2M, 4M) 2 | | +[4M, 8M) 2 | | +[8M, 16M) 6 | | +[16M, 32M) 16 | | +[32M, 64M) 27 | | +[64M, 128M) 48 |@ | +[128M, 256M) 98 |@@@ | +[256M, 512M) 191 |@@@@@@ | +[512M, 1G) 394 |@@@@@@@@@@@@@ | +[1G, 2G) 820 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +``` + +### lhist + +* `lhist_t lhist(int64 n, int64 min, int64 max, int64 step)` + +Create a linear histogram of `n`. +`lhist` creates `M` (`(max - min) / step`) buckets in the range `[min,max)` where each bucket is `step` in size. +Values in the range `(-inf, min)` and `(max, inf)` get their get their own bucket too, bringing the total amount of buckets created to `M+2`. + +``` +interval:ms:1 { + @ = lhist(rand %10, 0, 10, 1); +} + +interval:s:5 { + exit(); +} +``` + +Prints: + +``` +@: +[0, 1) 306 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[1, 2) 284 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[2, 3) 294 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[3, 4) 318 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[4, 5) 311 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[5, 6) 362 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| +[6, 7) 336 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[7, 8) 326 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[8, 9) 328 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[9, 10) 318 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +``` + +### max + +* `max_t max(int64 n)` + +Update the map with `n` if `n` is bigger than the current value held. +Similar to `count` this uses a [PERCPU](./language#percpu-types) map (thread-safe, fast writes, slow reads). + +Note: this is different than the typical userspace `max()` in that bpftrace’s `max()` +only takes a single argument. The logical "other" argument to compare to is the value +in the map the "result" is being assigned to. + +For example, compare the two logically equivalent samples (C++ vs bpftrace): + +In C++: +``` +int x = std::max(3, 33); // x contains 33 +``` + +In bpftrace: +``` +@x = max(3); +@x = max(33); // @x contains 33 +``` + +Also note that bpftrace takes care to handle the unset case. In other words, +there is no default value. The first value you pass to `max()` will always +be returned. + +### min + +* `min_t min(int64 n)` + +Update the map with `n` if `n` is smaller than the current value held. +Similar to `count` this uses a [PERCPU](./language#percpu-types) map (thread-safe, fast writes, slow reads). + +See `max()` above for how this differs from the typical userspace `min()`. + +### stats + +* `stats_t stats(int64 n)` + +`stats` combines the `count`, `avg` and `sum` calls into one. + +``` +kprobe:vfs_read { + @bytes[comm] = stats(arg2); +} +``` + +``` +@bytes[bash]: count 7, average 1, total 7 +@bytes[sleep]: count 5, average 832, total 4160 +@bytes[ls]: count 7, average 886, total 6208 +@ +``` + +### sum + +* `sum_t sum(int64 n)` + +Calculate the sum of all `n` passed. + +Using `@=sum(5)` is conceptually similar to `@+=5`. +The difference is that the `sum()` function uses a map type optimized for +performance and correctness using cheap, thread-safe writes ([PERCPU](./language#percpu-types)). However, sync reads +can be expensive as bpftrace needs to iterate over all the cpus to collect and +sum these values. + +Note: This differs from "raw" writes (e.g. `@+=5`) where multiple writers to a +shared location might lose updates, as bpftrace does not generate any implicit +atomic operations. + +Example one: +``` +BEGIN { + @ = sum(5); + @ = sum(6); + printf("%d\n", (int64)@); // prints 11 + clear(@); + exit(); +} +``` + +Example two: +``` +interval:ms:100 { + @ = sum(5); +} + +interval:s:10 { + // async read + print(@); + // sync read + if (@ > 10) { + print(("hello")); + } + clear(@); +} +``` + +### tseries + +* `tseries_t tseries(int64 n, int64 interval_ns, int64 num_intervals)` +* `tseries_t tseries(int64 n, int64 interval_ns, int64 num_intervals, const string agg)` + +Create a time series that tracks an integer value. `tseries` records up to +`num_intervals` intervals representing `interval_ns` nanoseconds. + +#### Durations + +`interval_ns` is an unsigned integer that specifies the interval duration. You +may use numbers with duration suffixes to improve readability: + +``` +@a = tseries(1, 100ns, 5); // 100 nanoseconds +@b = tseries(1, 100us, 5); // 100 microseconds +@c = tseries(1, 100ms, 5); // 100 milliseconds +@d = tseries(1, 1s, 5); // 1 second +``` + +#### Aggregation Functions + +By default, each interval in `tseries` contains the last value recorded in that +interval. The optional `agg` parameter specifies how values in the same interval +are aggregated. + +| Aggregation Function | Example | Description | +| --- | --- | --- | +| `avg` | `@ = tseries(@v, 1s, 5, "avg")` | Calculate the running average of all the values in each interval. | +| `max` | `@ = tseries(@v, 1s, 5, "max")` | Calculate the maximum of all values in each interval. | +| `min` | `@ = tseries(@v, 1s, 5, "min")` | Calculate the minimum of all values in each interval. | +| `sum` | `@ = tseries(@v, 1s, 5, "sum")` | Calculate the sum of all values in each interval. | + +#### Examples + +Example one: + +``` +// Record the minimum of ten random values generated during each 100ms interval. +i:ms:10 { + @ = tseries(rand % 10, 100ms, 20, "min"); +} +``` + +``` +Attached 2 probes + + +@: + 0 2 +hh:mm:ss.ms |___________________________________________________| +10:41:46.700 * | 0 +10:41:46.800 * | 0 +10:41:46.900 | * | 1 +10:41:47.000 | * 2 +10:41:47.100 | * | 1 +10:41:47.200 * | 0 +10:41:47.300 | * | 1 +10:41:47.400 * | 0 +10:41:47.500 | * | 1 +10:41:47.600 * | 0 +10:41:47.700 | * 2 +10:41:47.800 * | 0 +10:41:47.900 * | 0 +10:41:48.000 | * | 1 +10:41:48.100 | * | 1 +10:41:48.200 | * | 1 +10:41:48.300 * | 0 +10:41:48.400 | * 2 +10:41:48.500 | * | 1 +10:41:48.600 | * | 1 + v___________________________________________________v + 0 2 +``` + +Example two: + +``` +// Create a zigzag pattern +BEGIN { + @dir = 1; + @c = -5; +} + +i:ms:100 { + @ = tseries(@c, 100ms, 20); + @c += @dir; + + if (@c > 5) { + @dir = -1; + @c = 4 + } else if (@c < -5) { + @dir = 1; + @c = -4; + } +} +``` + +``` +Attached 2 probes + + +@: + -5 5 +hh:mm:ss.ms |___________________________________________________| +10:39:49.300 * . | -5 +10:39:49.400 | * . | -4 +10:39:49.500 | * . | -3 +10:39:49.600 | * . | -2 +10:39:49.700 | * . | -1 +10:39:49.800 | * | 0 +10:39:49.900 | . * | 1 +10:39:50.000 | . * | 2 +10:39:50.100 | . * | 3 +10:39:50.200 | . * | 4 +10:39:50.300 | . * 5 +10:39:50.400 | . * | 4 +10:39:50.500 | . * | 3 +10:39:50.600 | . * | 2 +10:39:50.700 | . * | 1 +10:39:50.800 | * | 0 +10:39:50.900 | * . | -1 +10:39:51.000 | * . | -2 +10:39:51.100 | * . | -3 +10:39:51.200 | * . | -4 + v___________________________________________________v + -5 5 +``` + +## Invocation Mode + +There are three invocation modes for bpftrace built-in functions. + +| | | | +| --- | --- | --- | +| Mode | Description | Example functions | +| Synchronous | The value/effect of the built-in function is determined/handled right away by the bpf program in the kernel space. | `reg(), str(), ntop()` | +| Asynchronous | The value/effect of the built-in function is determined/handled later by the bpftrace process in the user space. | `printf(), clear(), exit()` | +| Compile-time | The value of the built-in function is determined before bpf programs are running. | `kaddr(), cgroupid(), offsetof()` | + +While BPF in the kernel can do a lot there are still things that can only be done from user space, like the outputting (printing) of data. +The way bpftrace handles this is by sending events from the BPF program which user-space will pick up some time in the future (usually in milliseconds). +Operations that happen in the kernel are 'synchronous' ('sync') and those that are handled in user space are 'asynchronous' ('async') + +The asynchronous behaviour can lead to some unexpected behavior as updates can happen before user space had time to process the event. The following situations may occur: + +* event loss: when using printf(), the amount of data printed may be less than the actual number of events generated by the kernel during BPF program’s execution. +* delayed exit: when using the exit() to terminate the program, bpftrace needs to handle the exit signal asynchronously causing the BPF program may continue to run for some additional time. + +One example is updating a map value in a tight loop: + +``` +BEGIN { + @=0; + unroll(10) { + print(@); + @++; + } + exit() +} +``` + +Maps are printed by reference not by value and as the value gets updated right after the print user-space will likely only see the final value once it processes the event: + +``` +@: 10 +@: 10 +@: 10 +@: 10 +@: 10 +@: 10 +@: 10 +@: 10 +@: 10 +@: 10 +``` + +Therefore, when you need precise event statistics, it is recommended to use synchronous functions (e.g. count() and hist()) to ensure more reliable and accurate results. \ No newline at end of file