debugging/strace
Strace
Basic Usage
Set up strace environment
module purge module add compiler/intel/2022 module add devel/strace
Trace all linux syscalls of benchmark stream
strace ./stream
Filter for
openat
syscalls- Discard standard output
- Redirect standard error output to standard output to to allow
forwarding to
grep
- Filter non-successful
openat
bygrep
strace -e openat ./stream 2>&1 1>/dev/null | grep -v "No such file or directory" # or by matching a regular expression strace --trace='/.*open.*' ./stream 2>&1 1>/dev/null | grep -v "No such file or directory"
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/lib64/libgomp.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3 +++ exited with 0 +++
Usage scenarios with OpenMPI
Set up strace environment
module purge module add \ \ compiler/intel/2022 mpi/openmpi/4.1module add devel/strace
Strace on all MPI ranks to different files (e.g. for comparison)
mpirun -np 4 bash -c \ 'strace -o strace.out.${OMPI_COMM_WORLD_RANK} ./osu_alltoall' ll -h strace.out.*
-rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:27 strace.out.0 -rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:27 strace.out.1 -rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:27 strace.out.2 -rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:27 strace.out.3
Strace on first rank to file (e.g. for strace data reduction)
mpirun -np 4 bash -c \ 'if [[ ${OMPI_COMM_WORLD_RANK} -eq 0 ]]; then strace -o strace.out \ ./osu_alltoall else ./osu_alltoall fi' ll -h strace.out
-rw-r--r-- 1 bq0742 hk-project-scs 3.8M Jun 15 14:28 strace.ou
Strace on first rank filter for openat syscalls, grep for loaded dynamic libraries
mpirun -np 4 bash -c \ 'if [[ ${OMPI_COMM_WORLD_RANK} -eq 0 ]]; then strace -e openat -o"| grep [.]so | grep -v ENOENT" \ ./osu_alltoall else ./osu_alltoall fi'
openat(AT_FDCWD, "/software/all/mpi/openmpi/4.1_intel_2022.0.2/lib/libmpi.so.40", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/liblustreapi.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libgpfs.so", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/software/all/mpi/openmpi/4.1.2_intel_2022.0.2/lib/libopen-rte.so.40", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/software/all/mpi/openmpi/4.1.2_intel_2022.0.2/lib/libopen-pal.so.40", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libucp.so.0", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libuct.so.0", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libucs.so.0", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libnuma.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libucm.so.0", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libfabric.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/librt.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libutil.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/opt/hwloc/2.7/lib/libhwloc.so.15", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libudev.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libpciaccess.so.0", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libxml2.so.2", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/software/all/toolkit/Intel_OneAPI/compiler/2022.0.2/linux/compiler/lib/intel64_lin/libimf.so", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/software/all/toolkit/Intel_OneAPI/compiler/2022.0.2/linux/compiler/lib/intel64_lin/libsvml.so", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/software/all/toolkit/Intel_OneAPI/compiler/2022.0.2/linux/compiler/lib/intel64_lin/libirng.so", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/software/all/toolkit/Intel_OneAPI/compiler/2022.0.2/linux/compiler/lib/intel64_lin/libintlc.so.5", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libreadline.so.7", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/librdmacm.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libibverbs.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libnl-3.so.200", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libnl-route-3.so.200", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libpsm2.so.2", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libmount.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/liblzma.so.5", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libtinfo.so.6", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3 openat(AT_FDCWD, "/usr/lib64/libucs.so.0/../etc/ucx.conf", O_RDONLY) = -1 ENOTDIR (Not a directory) openat(AT_FDCWD, "/usr/lib64/libibverbs/libmlx5-rdmav34.so", O_RDONLY|O_CLOEXEC) = 15 openat(AT_FDCWD, "/usr/lib64/libcuda.so.1", O_RDONLY|O_CLOEXEC) = 21 openat(AT_FDCWD, "/usr/lib64/ucx/libuct_cuda.so.0", O_RDONLY|O_CLOEXEC) = 22 openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 22 openat(AT_FDCWD, "/software/all/devel/cuda/11.2/targets/x86_64-linux/lib/libcudart.so.11.0", O_RDONLY|O_CLOEXEC) = 22 openat(AT_FDCWD, "/lib64/libnvidia-ml.so.1", O_RDONLY|O_CLOEXEC) = 22 openat(AT_FDCWD, "/usr/lib64/ucx/libuct_cuda_gdrcopy.so.0", O_RDONLY|O_CLOEXEC) = 22 openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 22 openat(AT_FDCWD, "/lib64/libgdrapi.so.2", O_RDONLY|O_CLOEXEC) = 22 openat(AT_FDCWD, "/usr/lib64/ucx/libuct_ib.so.0", O_RDONLY|O_CLOEXEC) = 22 openat(AT_FDCWD, "/usr/lib64/ucx/libuct_rdmacm.so.0", O_RDONLY|O_CLOEXEC) = 22