debugging/strace

Strace

Basic Usage

Set up strace environment

module purge
module add compiler/intel/2022
module add devel/strace

Trace all linux syscalls of benchmark stream
```
strace ./stream
```

Filter for openat syscalls

Discard standard output
Redirect standard error output to standard output to to allow forwarding to grep
Filter non-successful openat by grep

strace -e openat ./stream 2>&1 1>/dev/null |
    grep -v "No such file or directory"
# or by matching a regular expression
strace --trace='/.*open.*' ./stream 2>&1 1>/dev/null |
    grep -v "No such file or directory"

openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib64/libgomp.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
+++ exited with 0 +++

Usage scenarios with OpenMPI

Set up strace environment

module purge
module add \
    compiler/intel/2022 \
    mpi/openmpi/4.1
module add devel/strace

Strace on all MPI ranks to different files (e.g. for comparison)

mpirun -np 4 bash -c \
    'strace -o strace.out.${OMPI_COMM_WORLD_RANK} ./osu_alltoall'
ll -h strace.out.*

-rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:27 strace.out.0
-rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:27 strace.out.1
-rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:27 strace.out.2
-rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:27 strace.out.3

Strace on first rank to file (e.g. for strace data reduction)

mpirun -np 4 bash -c \
    'if [[ ${OMPI_COMM_WORLD_RANK} -eq 0 ]]; then
        strace -o strace.out \
            ./osu_alltoall
     else
            ./osu_alltoall
     fi'
ll -h strace.out

-rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:28 strace.ou

Strace on first rank filter for openat syscalls, grep for loaded dynamic libraries

mpirun -np 4 bash -c \
    'if [[ ${OMPI_COMM_WORLD_RANK} -eq 0 ]]; then
        strace -e openat -o"| grep [.]so | grep -v ENOENT" \
            ./osu_alltoall
     else
            ./osu_alltoall
     fi'

openat(AT_FDCWD, "/software/all/mpi/openmpi/4.1_intel_2022.0.2/lib/libmpi.so.40", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/liblustreapi.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libgpfs.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/software/all/mpi/openmpi/4.1.2_intel_2022.0.2/lib/libopen-rte.so.40", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/software/all/mpi/openmpi/4.1.2_intel_2022.0.2/lib/libopen-pal.so.40", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libucp.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libuct.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libucs.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libnuma.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libucm.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libfabric.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libutil.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/opt/hwloc/2.7/lib/libhwloc.so.15", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libudev.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libpciaccess.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libxml2.so.2", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/software/all/toolkit/Intel_OneAPI/compiler/2022.0.2/linux/compiler/lib/intel64_lin/libimf.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/software/all/toolkit/Intel_OneAPI/compiler/2022.0.2/linux/compiler/lib/intel64_lin/libsvml.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/software/all/toolkit/Intel_OneAPI/compiler/2022.0.2/linux/compiler/lib/intel64_lin/libirng.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/software/all/toolkit/Intel_OneAPI/compiler/2022.0.2/linux/compiler/lib/intel64_lin/libintlc.so.5", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libreadline.so.7", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/librdmacm.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libibverbs.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libnl-3.so.200", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libnl-route-3.so.200", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libpsm2.so.2", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libmount.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/liblzma.so.5", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libtinfo.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib64/libucs.so.0/../etc/ucx.conf", O_RDONLY) = -1 ENOTDIR (Not a directory)
openat(AT_FDCWD, "/usr/lib64/libibverbs/libmlx5-rdmav34.so", O_RDONLY|O_CLOEXEC) = 15
openat(AT_FDCWD, "/usr/lib64/libcuda.so.1", O_RDONLY|O_CLOEXEC) = 21
openat(AT_FDCWD, "/usr/lib64/ucx/libuct_cuda.so.0", O_RDONLY|O_CLOEXEC) = 22
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 22
openat(AT_FDCWD, "/software/all/devel/cuda/11.2/targets/x86_64-linux/lib/libcudart.so.11.0", O_RDONLY|O_CLOEXEC) = 22
openat(AT_FDCWD, "/lib64/libnvidia-ml.so.1", O_RDONLY|O_CLOEXEC) = 22
openat(AT_FDCWD, "/usr/lib64/ucx/libuct_cuda_gdrcopy.so.0", O_RDONLY|O_CLOEXEC) = 22
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 22
openat(AT_FDCWD, "/lib64/libgdrapi.so.2", O_RDONLY|O_CLOEXEC) = 22
openat(AT_FDCWD, "/usr/lib64/ucx/libuct_ib.so.0", O_RDONLY|O_CLOEXEC) = 22
openat(AT_FDCWD, "/usr/lib64/ucx/libuct_rdmacm.so.0", O_RDONLY|O_CLOEXEC) = 22