From 92e22b350fe6320e6f298b9f7429b6bf1085b542 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Sat, 20 Jul 2024 15:50:38 -0700 Subject: [PATCH] resolve a few hdf5 asan issues related to cleanup during exceptions (#1304) * resolve a few hdf5 asan issues related to cleanup during exception * add alloc cleanup logic for hdf5 variable string case * add lsan supressions file * add gh actions asan build recipe for build w/ mpi * gactemu --- .../workflows/build_conduit_clang_asan.yml | 116 +++++++ .github/workflows/windows.yml | 2 +- scripts/build_conduit/build_conduit.sh | 289 ++++++++++++++++++ scripts/gactemu/gactemu-config.yaml | 7 + scripts/gactemu/gactemu.py | 252 +++++++++++++++ src/cmake/lsan.supp | 9 + src/libs/relay/conduit_relay_io_hdf5.cpp | 46 +-- 7 files changed, 702 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/build_conduit_clang_asan.yml create mode 100755 scripts/build_conduit/build_conduit.sh create mode 100644 scripts/gactemu/gactemu-config.yaml create mode 100644 scripts/gactemu/gactemu.py create mode 100644 src/cmake/lsan.supp diff --git a/.github/workflows/build_conduit_clang_asan.yml b/.github/workflows/build_conduit_clang_asan.yml new file mode 100644 index 000000000..1800a15af --- /dev/null +++ b/.github/workflows/build_conduit_clang_asan.yml @@ -0,0 +1,116 @@ +name: build_conduit_clang_asan + +on: + pull_request: + branches: [ develop ] + +jobs: + build_basic: + name: Ubuntu Build Conduit Clang Asan + runs-on: ubuntu-latest + env: + CC: clang + CXX: clang++ + MPICH_CC: clang + MPICH_CXX: clang++ + CFLAGS: "-fsanitize=address -DNDEBUG" + CXXFLAGS: "-fsanitize=address -DNDEBUG" + steps: + - name: Install System Deps + run: | + sudo apt-get update -y + sudo apt-get install -y binutils \ + clang \ + llvm \ + python3 \ + perl \ + git \ + git-lfs \ + curl \ + wget \ + tar \ + unzip \ + build-essential \ + libncurses-dev \ + libssl-dev \ + libblas-dev \ + liblapack-dev \ + zlib1g-dev \ + libgdbm-dev \ + libreadline-dev \ + libsqlite3-dev \ + libbz2-dev \ + mpich \ + libmpich-dev \ + cmake + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Build TPLs + run: | + env enable_mpi=OFF \ + build_config=Debug \ + enable_fortran=OFF \ + enable_tests=OFF \ + enable_verbose=OFF \ + build_conduit=false \ + build_jobs=4 \ + prefix=tpls \ + ./scripts/build_conduit/build_conduit.sh + # - name: Configure Conduit (MPI) + # run: | + # cmake --version + # echo "**** Configuring Conduit" + # # Note: for system clang + mpich, we need to pass + # # mpi compiler wrappers are the c & cxx compilers + # cmake -S src \ + # -B build \ + # -C tpls/conduit-config.cmake \ + # -DCMAKE_C_COMPILER=/usr/bin/mpicc \ + # -DCMAKE_CXX_COMPILER=/usr/bin/mpicxx \ + # -DENABLE_TESTS=ON \ + # -DCMAKE_INSTALL_PREFIX=install + - name: Configure Conduit + run: | + cmake --version + echo "**** Configuring Conduit" + cmake -S src \ + -B build \ + -C tpls/conduit-config.cmake \ + -DENABLE_TESTS=ON \ + -DCMAKE_INSTALL_PREFIX=install + - name: Build Conduit + run: | + echo "**** Building Conduit" + cmake --build build -j4 --config Debug + - name: Run Conduit Unit Tests + run: | + echo "**** Conduit Unit Tests (asan)" + export ASAN_SYMBOLIZER_PATH=`which llvm-symbolizer` + export CWD=`pwd` + export LSAN_OPTIONS="suppressions=${CWD}/src/cmake/lsan.supp" + echo "ASAN_SYMBOLIZER_PATH=${ASAN_SYMBOLIZER_PATH}" + echo "LSAN_OPTIONS=${LSAN_OPTIONS}" + export CTEST_OUTPUT_ON_FAILURE=1 + ctest --test-dir build + - name: Install Conduit + run: | + echo "**** Installing Conduit" + cmake --install build --config Debug + - name: Check Install + run: | + echo "**** Checking Conduit using-with-cmake example" + cd install/examples/conduit/using-with-cmake + cmake -S . -B build + cmake --build build --verbose -j4 + ./build/conduit_example + # - name: Check Install (MPI) + # run: | + # echo "**** Checking Conduit using-with-cmake-mpi example" + # export CC=/usr/bin/mpicc + # export CXX=/usr/bin/mpicxx + # cd install/examples/conduit/using-with-cmake-mpi + # cmake -S . -B build + # cmake --build build --verbose -j4 + # mpiexec -n 2 ./build/conduit_mpi_example diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index f76e19ae4..12fd6aa38 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -73,9 +73,9 @@ jobs: strategy: matrix: python-version: - - '3.7' - '3.8' - '3.9' + - '3.10' steps: - uses: actions/setup-python@v2 with: diff --git a/scripts/build_conduit/build_conduit.sh b/scripts/build_conduit/build_conduit.sh new file mode 100755 index 000000000..006cfd49e --- /dev/null +++ b/scripts/build_conduit/build_conduit.sh @@ -0,0 +1,289 @@ +#!/bin/bash + +############################################################################## +# Demonstrates how to manually build Conduit and its dependencies, including: +# +# zlib, hdf5 +# +# usage example: +# env enable_mpi=ON enable_openmp=ON ./build_conduit.sh +# +# +# Assumes: +# - cmake is in your path +# - selected compilers are in your path or set via env vars +# - [when enabled] MPI and Python (+numpy and mpi4py), are in your path +# +############################################################################## +set -eu -o pipefail + +############################################################################## +# Build Options +############################################################################## + +# shared options +enable_fortran="${enable_fortran:=OFF}" +enable_python="${enable_python:=OFF}" +enable_openmp="${enable_openmp:=OFF}" +enable_mpi="${enable_mpi:=OFF}" +enable_find_mpi="${enable_find_mpi:=ON}" +enable_tests="${enable_tests:=OFF}" +enable_verbose="${enable_verbose:=ON}" +build_jobs="${build_jobs:=6}" +build_config="${build_config:=Release}" +build_shared_libs="${build_shared_libs:=ON}" + +# tpl controls +build_zlib="${build_zlib:=true}" +build_hdf5="${build_hdf5:=true}" + +# conduit options +build_conduit="${build_conduit:=true}" + +# see if we are building on windows +build_windows="${build_windows:=OFF}" + +# see if we are building on macOS +build_macos="${build_macos:=OFF}" + +case "$OSTYPE" in + win*) build_windows="ON";; + msys*) build_windows="ON";; + darwin*) build_macos="ON";; + *) ;; +esac + +if [[ "$build_windows" == "ON" ]]; then + echo "*** configuring for windows" +fi + +if [[ "$build_macos" == "ON" ]]; then + echo "*** configuring for macos" +fi + +################ +# path helpers +################ +function ospath() +{ + if [[ "$build_windows" == "ON" ]]; then + echo `cygpath -m $1` + else + echo $1 + fi +} + +function abs_path() +{ + if [[ "$build_macos" == "ON" ]]; then + echo "$(cd $(dirname "$1");pwd)/$(basename "$1")" + else + echo `realpath $1` + fi +} + +root_dir=$(pwd) +root_dir="${prefix:=${root_dir}}" +root_dir=$(ospath ${root_dir}) +root_dir=$(abs_path ${root_dir}) +script_dir=$(abs_path "$(dirname "${BASH_SOURCE[0]}")") + +# root_dir is where we will build and install +# override with `prefix` env var +if [ ! -d ${root_dir} ]; then + mkdir -p ${root_dir} +fi + +cd ${root_dir} + +# install_dir is where we will install +# override with `prefix` env var +install_dir="${install_dir:=$root_dir/install}" + +echo "*** prefix: ${root_dir}" +echo "*** build root: ${root_dir}/build" +echo "*** install root: ${install_dir}" +echo "*** script dir: ${script_dir}" + +################ +# CMake Compiler Settings +################ +cmake_compiler_settings="" + +# capture compilers if they are provided via env vars +if [ ! -z ${CC+x} ]; then + cmake_compiler_settings="-DCMAKE_C_COMPILER:PATH=${CC}" +fi + +if [ ! -z ${CXX+x} ]; then + cmake_compiler_settings="${cmake_compiler_settings} -DCMAKE_CXX_COMPILER:PATH=${CXX}" +fi + +if [ ! -z ${FTN+x} ]; then + cmake_compiler_settings="${cmake_compiler_settings} -DCMAKE_Fortran_COMPILER:PATH=${FTN}" +fi + +################ +# print all build_ZZZ and enable_ZZZ options +################ +echo "*** cmake_compiler_settings: ${cmake_compiler_settings}" +echo "*** build_conduit `enable` settings:" +set | grep enable_ +echo "*** build_conduit `build` settings:" +set | grep build_ + +################ +# Zlib +################ +zlib_version=1.3.1 +zlib_src_dir=$(ospath ${root_dir}/zlib-${zlib_version}) +zlib_build_dir=$(ospath ${root_dir}/build/zlib-${zlib_version}/) +zlib_install_dir=$(ospath ${install_dir}/zlib-${zlib_version}/) +zlib_tarball=zlib-${zlib_version}.tar.gz + +# build only if install doesn't exist +if [ ! -d ${zlib_install_dir} ]; then +if ${build_zlib}; then +if [ ! -d ${zlib_src_dir} ]; then + echo "**** Downloading ${zlib_tarball}" + curl -L https://github.com/madler/zlib/releases/download/v${zlib_version}/zlib-${zlib_version}.tar.gz -o ${zlib_tarball} + tar -xzf ${zlib_tarball} +fi + +echo "**** Configuring Zlib ${zlib_version}" +cmake -S ${zlib_src_dir} -B ${zlib_build_dir} ${cmake_compiler_settings} \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose} \ + -DCMAKE_BUILD_TYPE=${build_config} \ + -DCMAKE_INSTALL_PREFIX=${zlib_install_dir} + +echo "**** Building Zlib ${zlib_version}" +cmake --build ${zlib_build_dir} --config ${build_config} -j${build_jobs} +echo "**** Installing Zlib ${zlib_version}" +cmake --install ${zlib_build_dir} --config ${build_config} + +fi +else + echo "**** Skipping Zlib build, install found at: ${zlib_install_dir}" +fi # build_zlib + + +################ +# HDF5 +################ +# release 1-2 GAH! +hdf5_version=1.14.1-2 +hdf5_middle_version=1.14.1 +hdf5_short_version=1.14 +hdf5_src_dir=$(ospath ${root_dir}/hdf5-${hdf5_version}) +hdf5_build_dir=$(ospath ${root_dir}/build/hdf5-${hdf5_version}/) +hdf5_install_dir=$(ospath ${install_dir}/hdf5-${hdf5_version}/) +hdf5_tarball=hdf5-${hdf5_version}.tar.gz + +# build only if install doesn't exist +if [ ! -d ${hdf5_install_dir} ]; then +if ${build_hdf5}; then +if [ ! -d ${hdf5_src_dir} ]; then + echo "**** Downloading ${hdf5_tarball}" + curl -L https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${hdf5_short_version}/hdf5-${hdf5_middle_version}/src/hdf5-${hdf5_version}.tar.gz -o ${hdf5_tarball} + tar -xzf ${hdf5_tarball} +fi + +################# +# +# hdf5 1.14.x CMake recipe for using zlib +# +# -DHDF5_ENABLE_Z_LIB_SUPPORT=ON +# Add zlib install dir to CMAKE_PREFIX_PATH +# +################# + +echo "**** Configuring HDF5 ${hdf5_version}" +cmake -S ${hdf5_src_dir} -B ${hdf5_build_dir} ${cmake_compiler_settings} \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose} \ + -DCMAKE_BUILD_TYPE=${build_config} \ + -DHDF5_ENABLE_Z_LIB_SUPPORT=ON \ + -DCMAKE_PREFIX_PATH=${zlib_install_dir} \ + -DCMAKE_INSTALL_PREFIX=${hdf5_install_dir} + +echo "**** Building HDF5 ${hdf5_version}" +cmake --build ${hdf5_build_dir} --config ${build_config} -j${build_jobs} +echo "**** Installing HDF5 ${hdf5_version}" +cmake --install ${hdf5_build_dir} --config ${build_config} + +fi +else + echo "**** Skipping HDF5 build, install found at: ${hdf5_install_dir}" +fi # build_hdf5 + +################ +# Conduit +################ +conduit_version=develop +conduit_src_dir=$(ospath ${root_dir}/conduit/src) +conduit_build_dir=$(ospath ${root_dir}/build/conduit-${conduit_version}/) +conduit_install_dir=$(ospath ${install_dir}/conduit-${conduit_version}/) + +echo "**** Creating Conduit host-config (conduit-config.cmake)" +# +echo '# host-config file generated by build_conduit.sh' > ${root_dir}/conduit-config.cmake + +# capture compilers if they are provided via env vars +if [ ! -z ${CC+x} ]; then + echo 'set(CMAKE_C_COMPILER ' ${CC} ' CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +fi + +if [ ! -z ${CXX+x} ]; then + echo 'set(CMAKE_CXX_COMPILER ' ${CXX} ' CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +fi + +if [ ! -z ${FTN+x} ]; then + echo 'set(CMAKE_Fortran_COMPILER ' ${FTN} ' CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +fi + +# capture compiler flags if they are provided via env vars +if [ ! -z ${CFLAGS+x} ]; then + echo 'set(CMAKE_C_FLAGS "' ${CFLAGS} '" CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +fi + +if [ ! -z ${CXXFLAGS+x} ]; then + echo 'set(CMAKE_CXX_FLAGS "' ${CXXFLAGS} '" CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +fi + +if [ ! -z ${FFLAGS+x} ]; then + echo 'set(CMAKE_F_FLAGS "' ${FFLAGS} '" CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +fi + +echo 'set(CMAKE_VERBOSE_MAKEFILE ' ${enable_verbose} ' CACHE BOOL "")' >> ${root_dir}/conduit-config.cmake +echo 'set(CMAKE_BUILD_TYPE ' ${build_config} ' CACHE STRING "")' >> ${root_dir}/conduit-config.cmake +echo 'set(BUILD_SHARED_LIBS ' ${build_shared_libs} ' CACHE STRING "")' >> ${root_dir}/conduit-config.cmake +echo 'set(CMAKE_INSTALL_PREFIX ' ${conduit_install_dir} ' CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +echo 'set(ENABLE_TESTS ' ${enable_tests} ' CACHE BOOL "")' >> ${root_dir}/conduit-config.cmake +echo 'set(ENABLE_MPI ' ${enable_mpi} ' CACHE BOOL "")' >> ${root_dir}/conduit-config.cmake +echo 'set(ENABLE_FIND_MPI ' ${enable_find_mpi} ' CACHE BOOL "")' >> ${root_dir}/conduit-config.cmake +echo 'set(ENABLE_FORTRAN ' ${enable_fortran} ' CACHE BOOL "")' >> ${root_dir}/conduit-config.cmake +echo 'set(ENABLE_PYTHON ' ${enable_python} ' CACHE BOOL "")' >> ${root_dir}/conduit-config.cmake +echo 'set(CONDUIT_DIR ' ${conduit_install_dir} ' CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +echo 'set(HDF5_DIR ' ${hdf5_install_dir} ' CACHE PATH "")' >> ${root_dir}/conduit-config.cmake +echo 'set(ZLIB_DIR ' ${zlib_install_dir} ' CACHE PATH "")' >> ${root_dir}/conduit-config.cmake + +# build only if install doesn't exist +if [ ! -d ${conduit_install_dir} ]; then +if ${build_conduit}; then +if [ ! -d ${conduit_src_dir} ]; then + echo "**** Cloning Conduit" + git clone --recursive https://github.com/LLNL/conduit.git +fi + +echo "**** Configuring Conduit" +cmake -S ${conduit_src_dir} -B ${conduit_build_dir} -C ${root_dir}/conduit-config.cmake + +echo "**** Building Conduit" +cmake --build ${conduit_build_dir} --config ${build_config} -j${build_jobs} + +echo "**** Installing Conduit" +cmake --install ${conduit_build_dir} --config ${build_config} + +fi +else + echo "**** Skipping Conduit build, install found at: ${conduit_install_dir}" +fi # build_conduit diff --git a/scripts/gactemu/gactemu-config.yaml b/scripts/gactemu/gactemu-config.yaml new file mode 100644 index 000000000..7a5130a6e --- /dev/null +++ b/scripts/gactemu/gactemu-config.yaml @@ -0,0 +1,7 @@ +name: conduit +repo_url: https://github.com/LLNL/conduit.git +# This tells azemu to find current branch via a git cmd +repo_branch: +runners: + ubuntu-latest: "ubuntu:22.04" + ubuntu-20.04: "ubuntu:20.04" diff --git a/scripts/gactemu/gactemu.py b/scripts/gactemu/gactemu.py new file mode 100644 index 000000000..1c8c0f16d --- /dev/null +++ b/scripts/gactemu/gactemu.py @@ -0,0 +1,252 @@ +############################################################################### +# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent +# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and +# other details. No copyright assignment is required to contribute to Ascent. +############################################################################### +""" +gactemu.py (Generally Awesome Emu Wrangler) + +Digests .github/workflows/ specs and generates scripts to run steps +locally using docker. + + +""" + +import yaml +import os +import stat +import glob +import subprocess +import sys + + +class CTX: + def __init__(self,ctx=None): + self.name = "" + self.txt = "" + self.cwd = "" + self.container = "" + if not ctx is None: + self.name = ctx.name + self.txt = ctx.txt + self.container = ctx.container + self.cwd = ctx.cwd + + def set_name(self,name): + self.name = name + + def set_container(self,container): + self.container = container + + def set_cwd(self,cwd): + self.cwd = cwd + + def print(self,txt): + self.txt += txt + "\n" + + def print_esc(self,txt, tag = None): + res = "" + if not tag is None: + res = "# [{0}: {1}]".format(tag,txt) + else: + res = "# [{0}]".format(txt) + self.txt += res + "\n" + + def finish(self): + print("[creating: {0}".format(self.script_file())) + f = open(self.script_file(),"w",newline='\n') + f.write(self.gen_script()) + os.chmod(self.script_file(), stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO ) + + print("[creating: {0}".format(self.launch_file())) + f= open(self.launch_file(),"w",newline='\n') + f.write(self.gen_launch()) + os.chmod(self.launch_file(), stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO ) + + def script_file(self): + return "GACTEMU-SCRIPT-" + self.name + ".sh" + + def launch_file(self): + return "GACTEMU-LAUNCH-" + self.name + ".sh" + + def gen_script(self): + res = "#!/bin/bash\n" + res += "# CONTAINER: {0}\n".format(self.container) + res += "set -e\n" + res += "set -x\n" + res += str(self) + return res + + def gen_launch(self): + res = "#!/bin/bash\n" + res += "set -x\n" + res += "docker stop gactemu_exec\n" + res += "docker rm gactemu_exec\n" + res += "docker run --name gactemu_exec -t -d {0}\n".format(self.container) + res += 'docker exec gactemu_exec sh -c "apt-get update && env apt-get install -y sudo"\n' + res += 'docker exec gactemu_exec sh -c "useradd -ms /bin/bash -G sudo user && echo \\"user:docker\\\" | chpasswd"\n' + res += 'docker exec gactemu_exec sh -c "echo \\"user ALL=(root) NOPASSWD:ALL\\\" > /etc/sudoers.d/user && chmod 0440 /etc/sudoers.d/user"\n' + res += 'docker cp {0} gactemu_exec://home/user/\n'.format(self.script_file()) + res += 'docker exec -u user gactemu_exec sh -c "echo [GACT EXEC SCRIPT]!"\n' + res += 'docker exec -u user --workdir=//home/user/ -i gactemu_exec ./{0}\n'.format(self.script_file()) + return res + + def __str__(self): + res ="" + if self.name != "": + res += "# [[{0}]]\n".format(self.name) + res += self.txt + return res + + +def shexe(cmd,ret_output=False,echo = True): + """ Helper for executing shell commands. """ + if echo: + print("[exe: {}]".format(cmd)) + if ret_output: + p = subprocess.Popen(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + res = p.communicate()[0] + res = res.decode('utf8') + return p.returncode,res + else: + return subprocess.call(cmd,shell=True) + +def sanitize_var(v): + if type(v)==bool: + if v: + return "ON" + else: + return "OFF" + return v + +def proc_root(tree, config): + for k,v in tree.items(): + if k == "name": + config["root_name"] = v + if k == "jobs": + proc_jobs(v, config) + +def map_gact_runners(runner,config): + if runner in config["runners"]: + return config["runners"][runner] + else: + print("# unsupported runner:" + runner) + return "UNSUPPORTED" + +def proc_jobs(tree, config): + for job_name, job in tree.items(): + job_ctx = CTX() + job_full_name = config["root_name"] + "-" + job_name + job_ctx.print_esc(tag = "job", txt =job_full_name ) + if "runs-on" in job.keys(): + job_ctx.set_container(map_gact_runners(job["runs-on"],config)) + else: + job_ctx.set_container(config["default_container"]) + job_ctx.set_name(job_full_name) + if "env" in job.keys(): + job_ctx.print_esc("job env vars") + for k,v in job["env"].items(): + job_ctx.print('export {0}="{1}"'.format(k,sanitize_var(v))) + steps = job["steps"] + proc_steps(steps,config, job_ctx) + job_ctx.finish() + ## fancier cases (matrix specs) not yet supported + +def proc_matrix_entry(steps, + config, + matrix_entry_name, + env_vars, + ctx): + ctx.print("#-------------------------------------") + ctx.print_esc(tag = "matrix entry", txt = matrix_entry_name) + ctx.print("#-------------------------------------") + ctx.print_esc(tag = "azure global scope vars", txt = config["azure_vars"]) + ctx.print_esc("matrix env vars") + for k,v in env_vars.items(): + ctx.print("export {0}={1}".format(k,sanitize_var(v))) + ctx.print("") + proc_steps(steps, config, ctx) + +def proc_action_checkout(step, config, ctx): + ctx.print("") + ctx.print("#++++++++++++++++++++++++++++++++") + ctx.print_esc("checkout") + ctx.print_esc(step) + ctx.print('echo ">start checkout"') + ctx.print("date") + ctx.print("git clone --recursive --depth=1 -b {0} {1} ".format( + config["repo_branch"], + config["repo_url"])) + ctx.set_cwd(config["name"]) + ctx.print('echo ">end checkout"') + ctx.print("date") + ctx.print("#++++++++++++++++++++++++++++++++") + +def proc_steps(steps, config, ctx): + ctx.print("#-------------------------------------") + ctx.print_esc("STEPS") + ctx.print("#-------------------------------------") + for s in steps: + # we only process "uses:actions/checkout and run + if "uses" in s.keys(): + # support checkout ... + if s["uses"].count("actions/checkout") > 0: + proc_action_checkout(s,config,ctx) + elif "run" in s.keys(): + ctx.print("") + ctx.print("#++++++++++++++++++++++++++++++++") + ctx.print_esc(tag = "name", txt = s["name"]) + ctx.print_esc("script") + ctx.print('echo ">start {0}"'.format(s["name"])) + ctx.print("date") + if not ctx.cwd is None: + ctx.print("cd ~/{0}".format(ctx.cwd)) + lines = s["run"].strip().split("\n") + ctx.print_esc("turn ON halt on error") + ctx.print("set -e") + for l in lines: + ctx.print(l) + ctx.print('echo ">end {0}"'.format(s["name"])) + ctx.print("date") + ctx.print("#++++++++++++++++++++++++++++++++") + else: + if "name" in s.keys(): + ctx.print_esc("STEP with name:{0} not SUPPORTED".format(s["name"])) + else: + ctx.print_esc("STEP not SUPPORTED") + +def proc_config(config): + if config["repo_branch"] == "": + rcode,rout = shexe("git rev-parse --abbrev-ref HEAD",ret_output=True,echo=True) + if rcode == 0: + config["repo_branch"] = rout.strip() + else: + print("[error finding current git branch]") + sys.exit(-1) + return config + + + +def main(): + gactrep_yaml_files = glob.glob("../../.github/workflows/*yml") + print(gactrep_yaml_files) + config_yaml_file = "gactemu-config.yaml" + config = yaml.load(open(config_yaml_file), Loader=yaml.Loader) + config = proc_config(config) + for gactrep_yaml_file in gactrep_yaml_files: + root = yaml.load(open(gactrep_yaml_file), Loader=yaml.Loader) + try: + if os.path.isfile(root): + root = yaml.load(open(root), Loader=yaml.Loader) + except: + pass + print(gactrep_yaml_file) + #print(root) + proc_root(root, config) + +if __name__ == "__main__": + main() + diff --git a/src/cmake/lsan.supp b/src/cmake/lsan.supp new file mode 100644 index 000000000..5b6fc12d1 --- /dev/null +++ b/src/cmake/lsan.supp @@ -0,0 +1,9 @@ +# h5literate related +leak:H5G__compact_build_table +# h5literate related +leak:H5MM_xstrdup +# mpi related +leak:PMPI_Init +leak:libmpi.so +leak:libfabric.so + diff --git a/src/libs/relay/conduit_relay_io_hdf5.cpp b/src/libs/relay/conduit_relay_io_hdf5.cpp index af6a7966e..40c0a8e24 100644 --- a/src/libs/relay/conduit_relay_io_hdf5.cpp +++ b/src/libs/relay/conduit_relay_io_hdf5.cpp @@ -2789,8 +2789,8 @@ fill_dataset_opts(const std::string & ref_path, const Node & inopts, Node& nsizes = filled_opts["slabparams/dataset_sizes"]; nsizes.set(DataType::index_t(rank)); index_t_array nsizes_array = nsizes.value(); - hsize_t* psizes = new hsize_t[rank]; - H5Sget_simple_extent_dims(dataspace_id, psizes, nullptr); + std::vector psizes(rank); + H5Sget_simple_extent_dims(dataspace_id, psizes.data(), nullptr); for (int d = 0; d < rank; ++d) { nsizes_array[d] = psizes[d]; @@ -2813,22 +2813,21 @@ fill_dataset_opts(const std::string & ref_path, const Node & inopts, index_t readcount = calculate_readsize(readsz, rank, nsizes_array, offset, stride); filled_opts["slabparams/readcount"] = readcount; - - delete[] psizes; } //---------------------------------------------------------------------------// -hsize_t* -make_dataset_opt_copy(const Node& opts, const std::string opt_name) +void +make_dataset_opt_copy(const Node& opts, + const std::string opt_name, + std::vector &values) { const index_t_accessor hdf5array = opts["slabparams"].fetch_existing(opt_name).as_index_t_accessor(); int rank = hdf5array.number_of_elements(); - hsize_t* retval = new hsize_t[rank]; + values.resize(rank); for (int d = 0; d < rank; ++d) { - retval[d] = (hsize_t)hdf5array[d]; + values[d] = (hsize_t)hdf5array[d]; } - return retval; } //---------------------------------------------------------------------------// @@ -2871,9 +2870,18 @@ read_hdf5_dataset_into_conduit_node(hid_t hdf5_dset_id, index_t rank = slab_params["rank"].to_long_long(); hsize_t readtotal = slab_params["readcount"].to_unsigned_long_long(); index_t nelems = (index_t)readtotal; - hsize_t* readsize = make_dataset_opt_copy(filled_opts, "sizes"); - hsize_t* offset = make_dataset_opt_copy(filled_opts, "offsets"); - hsize_t* stride = make_dataset_opt_copy(filled_opts, "strides"); + // use std::vector vs raw allocd pointer here b/c + // the vector will be cleaned up in exception cases + std::vector readsize_vec; + std::vector offset_vec; + std::vector stride_vec; + + make_dataset_opt_copy(filled_opts, "sizes",readsize_vec); + make_dataset_opt_copy(filled_opts, "offsets", offset_vec); + make_dataset_opt_copy(filled_opts, "strides", stride_vec); + hsize_t* readsize = readsize_vec.data(); + hsize_t* offset = offset_vec.data(); + hsize_t* stride = stride_vec.data(); // copy metadata to the node under hard-coded keys if (only_get_metadata) @@ -2952,8 +2960,14 @@ read_hdf5_dataset_into_conduit_node(hid_t hdf5_dset_id, H5P_DEFAULT, read_ptr); - // copy the data out to the conduit node - dest.set_string(read_ptr[0]); + if(read_ptr[0] != NULL) + { + // copy the data out to the conduit node + dest.set_string(read_ptr[0]); + // hdf5 allocates this for us, so we need + // to clean up + free(read_ptr[0]); + } } // check for bad # of elements else if( dt.number_of_elements() < 0 ) @@ -3000,10 +3014,6 @@ read_hdf5_dataset_into_conduit_node(hid_t hdf5_dset_id, H5Sclose(dataspace); } - delete[] stride; - delete[] offset; - delete[] readsize; - if(opts.dtype().is_empty()) { CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status,