From 9a5495ad740a85c492ab5869f12c9c1baf0d0939 Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser Date: Tue, 30 Jun 2026 09:51:49 -0500 Subject: [PATCH 1/5] [NFCI] Make check_smoke.sh shellcheck-compliant Addressed ~130 ShellCheck findings Actual behavior change might stem from `|| exit / return` Co-authored-by: Cursor --- test/smoke/check_smoke.sh | 366 +++++++++++++++++++------------------- 1 file changed, 179 insertions(+), 187 deletions(-) diff --git a/test/smoke/check_smoke.sh b/test/smoke/check_smoke.sh index e08b9b3a1..ab7160427 100755 --- a/test/smoke/check_smoke.sh +++ b/test/smoke/check_smoke.sh @@ -17,57 +17,57 @@ ulimit -t 600 function printfails(){ # Print run logs for runtime fails, EPSDB only - if [ "$EPSDB" == 1 ] ; then + if [ "${EPSDB}" == 1 ] ; then files='make-fail.txt failing-tests.txt' flags_test_done=0 - for file in $files; do - if [ -e "$file" ]; then - echo ---------- Printing $file Logs --------- + for file in ${files}; do + if [ -e "${file}" ]; then + echo "---------- Printing ${file} Logs ---------" + # shellcheck disable=SC2094 # ${file} is only read here, never written while read -r line; do - trimline=$(echo $line | sed 's/\: Make Failed//') - line=$trimline + line=${line/: Make Failed/} echo - if [ "$file" == "failing-tests.txt" ]; then - echo "--> Test: $line - Runtime Error! Print Log:" + if [ "${file}" == "failing-tests.txt" ]; then + echo "--> Test: ${line} - Runtime Error! Print Log:" else - echo "--> Test: $line - Compile Error! Print Log:" + echo "--> Test: ${line} - Compile Error! Print Log:" fi # The flags test has multiple numbered runs. We cannot pushd flags 1 because only the flags dir exists. # We must re-run the entire flags test to get run.log. If more than one flags subtest fails only run once. - if [[ "$line" =~ "flags" ]]; then - if [[ "$flags_test_done" == 0 ]]; then + if [[ "${line}" =~ "flags" ]]; then + if [[ "${flags_test_done}" == 0 ]]; then echo - pushd flags > /dev/null - echo The flags test must run all iterations if one subtest fails. - if [ "$file" == "failing-tests.txt" ]; then - cat run.log | sed -e '1,/lockfile/d' - else - if [ -e "run.log" ]; then + pushd flags > /dev/null || return + echo "The flags test must run all iterations if one subtest fails." + if [ "${file}" == "failing-tests.txt" ]; then + sed -e '1,/lockfile/d' run.log + else + if [ -e "run.log" ]; then cat run.log - else - cat make-log.txt - fi - fi + else + cat make-log.txt + fi + fi flags_test_done=1 - popd > /dev/null + popd > /dev/null || return fi else - pushd $line > /dev/null + pushd "${line}" > /dev/null || return echo - if [ "$file" == "failing-tests.txt" ]; then - cat run.log | sed -e '1,/lockfile/d' - else - if [ -e "run.log" ]; then + if [ "${file}" == "failing-tests.txt" ]; then + sed -e '1,/lockfile/d' run.log + else + if [ -e "run.log" ]; then cat run.log - else - cat make-log.txt - fi - fi - popd > /dev/null + else + cat make-log.txt + fi + fi + popd > /dev/null || return fi - done < "$file" + done < "${file}" echo - echo "**** End of $file ****" + echo "**** End of ${file} ****" echo fi done @@ -90,7 +90,7 @@ function gatherdata(){ passing_tests=0 if [ -e passing-tests.txt ]; then ((passing_tests=$(wc -l < passing-tests.txt))) - total_tests=$passing_tests + total_tests=${passing_tests} fi if [ -e make-fail.txt ]; then ((total_tests+=$(wc -l < make-fail.txt))) @@ -100,16 +100,16 @@ function gatherdata(){ fi # Print Results - echo -e "$BLU"-------------------- Results --------------------"$BLK" - echo -e "$BLU"Number of tests: $total_tests"$BLK" + echo -e "${BLU}-------------------- Results --------------------${BLK}" + echo -e "${BLU}Number of tests: ${total_tests}${BLK}" echo "" - echo -e "$GRN"Passing tests: $passing_tests/$total_tests"$BLK" + echo -e "${GRN}Passing tests: ${passing_tests}/${total_tests}${BLK}" echo "" # Print failed tests - echo -e "$RED" - if [ "$SKIP_FAILS" != 1 ] && [ "$known_fails" != "" ] ; then - echo "Known failures: $known_fails" + echo -e "${RED}" + if [ "${SKIP_FAILS}" != 1 ] && [ "${known_fails}" != "" ] ; then + echo "Known failures: ${known_fails}" fi echo "" if [ -e failing-tests.txt ]; then @@ -124,14 +124,14 @@ function gatherdata(){ echo "--------------------" cat make-fail.txt fi - echo -e "$BLK" + echo -e "${BLK}" # Tests that need visual inspection echo "" - echo -e "$ORG" + echo -e "${ORG}" echo "---------- Please inspect the output above to verify the following tests ----------" echo "devices, stream" - echo -e "$BLK" + echo -e "${BLK}" } if [ "$1" == "gatherdata" ]; then @@ -147,124 +147,121 @@ cleanup(){ } script_dir=$(dirname "$0") -pushd $script_dir +pushd "${script_dir}" || exit path=$(pwd) -script_dir_name=$(basename "$path") +script_dir_name=$(basename "${path}") SMOKE_DIRS=${SMOKE_DIRS:-./*/} # test directories to run SMOKE_LRUN=${SMOKE_LRUN:-1} # number of times to run test list SMOKE_NRUN=${SMOKE_NRUN:-1} # number of times to run one test set_my_nrun() { - my_nrun=$SMOKE_NRUN + my_nrun=${SMOKE_NRUN} # If doing reruns, also check for NRUN file in test directory - if [ $my_nrun -gt 1 ] && [ -r NRUN ]; then - base_nrun=`cat NRUN`; + if [ "${my_nrun}" -gt 1 ] && [ -r NRUN ]; then + base_nrun=$(cat NRUN) # Use value from NRUN if larger than default - if [ $base_nrun -gt $my_nrun ]; then my_nrun=$base_nrun; fi + if [ "${base_nrun}" -gt "${my_nrun}" ]; then my_nrun=${base_nrun}; fi fi } cleanup if [ "$1" == "-clean" ]; then - for directory in $SMOKE_DIRS; do - if [ ! -r $directory/Makefile ]; then continue; fi - pushd $directory > /dev/null - if [ $? -ne 0 ]; then continue; fi + for directory in ${SMOKE_DIRS}; do + if [ ! -r "${directory}/Makefile" ]; then continue; fi + if ! pushd "${directory}" > /dev/null; then continue; fi make clean - popd > /dev/null + popd > /dev/null || exit done exit 0 fi export OMP_TARGET_OFFLOAD=${OMP_TARGET_OFFLOAD:-MANDATORY} -echo OMP_TARGET_OFFLOAD=$OMP_TARGET_OFFLOAD +echo "OMP_TARGET_OFFLOAD=${OMP_TARGET_OFFLOAD}" echo "" -echo -e "$ORG"RUNNING ALL TESTS IN: $path"$BLK" +echo -e "${ORG}RUNNING ALL TESTS IN: ${path}${BLK}" echo "" -echo "************************************************************************************" > check-smoke.txt -echo " A non-zero exit code means a failure occured." >> check-smoke.txt -echo "Tests that need to be visually inspected: devices, stream" >> check-smoke.txt -echo "***********************************************************************************" >> check-smoke.txt +{ + echo "************************************************************************************" + echo " A non-zero exit code means a failure occured." + echo "Tests that need to be visually inspected: devices, stream" + echo "***********************************************************************************" +} > check-smoke.txt known_fails="" skip_tests="" +# shellcheck disable=SC2010 # version-sorted listing of /opt; glob would not preserve ls ordering newest_rocm=$(ls /opt | grep -e "rocm-[0-9].[0-9].[0-9].*" | tail -1) -AOMPROCM=${AOMPROCM:-/opt/"$newest_rocm"} +AOMPROCM=${AOMPROCM:-/opt/${newest_rocm}} -if [ ${KNOWN_FAIL_FILE+x} ] ; then - known_fails+="`cat $KNOWN_FAIL_FILE` " +if [ "${KNOWN_FAIL_FILE+x}" ] ; then + known_fails+="$(cat "${KNOWN_FAIL_FILE}") " export SKIP_FAILURES=${SKIP_FAILURES:-1} export SKIP_FAILS=${SKIP_FAILS:-1} fi -if [ ${KNOWN_FAIL_TESTS+x} ] ; then - known_fails+="$KNOWN_FAIL_TESTS " +if [ "${KNOWN_FAIL_TESTS+x}" ] ; then + known_fails+="${KNOWN_FAIL_TESTS} " export SKIP_FAILURES=${SKIP_FAILURES:-1} export SKIP_FAILS=${SKIP_FAILS:-1} fi -if [ "$SKIP_FAILURES" == 1 ] ; then - skip_tests=$known_fails +if [ "${SKIP_FAILURES}" == 1 ] ; then + skip_tests=${known_fails} else skip_tests="" fi -if [ "$SKIP_FORTRAN" == 1 ] ; then - skip_tests+="`find . -iname '*.f9[50]' | sed s^./^^ | awk -F/ '{print $1}'` " +if [ "${SKIP_FORTRAN}" == 1 ] ; then + skip_tests+="$(find . -iname '*.f9[50]' | sed s^./^^ | awk -F/ '{print $1}') " fi -if [ "$skip_tests" != "" ]; then - echo "skip_tests: $skip_tests" +if [ "${skip_tests}" != "" ]; then + echo "skip_tests: ${skip_tests}" echo "" fi # ---------- Begin parallel logic ---------- -if [ "$AOMP_PARALLEL_SMOKE" == 1 ]; then - if [ -z ${AOMP_NO_PREREQ+x} ]; then +if [ "${AOMP_PARALLEL_SMOKE}" == 1 ]; then + if [ -z "${AOMP_NO_PREREQ+x}" ]; then export AOMP_NO_PREREQ=1 # disable prereq target so builds can be reused fi - sem --help > /dev/null - if [ $? -eq 0 ]; then + if sem --help > /dev/null; then COMP_THREADS=1 - MAX_THREADS=16 - if [ ! -z `which "getconf"` ]; then - COMP_THREADS=$(`which "getconf"` _NPROCESSORS_ONLN) - if [ "$AOMP_PROC" == "ppc64le" ] ; then + if [ -n "$(which getconf)" ]; then + COMP_THREADS=$(getconf _NPROCESSORS_ONLN) + if [ "${AOMP_PROC}" == "ppc64le" ] ; then COMP_THREADS=$(( COMP_THREADS / 6)) fi - if [ "$AOMP_PROC" == "aarch64" ] ; then + if [ "${AOMP_PROC}" == "aarch64" ] ; then COMP_THREADS=$(( COMP_THREADS / 4)) fi fi AOMP_JOB_THREADS=${AOMP_JOB_THREADS:-$COMP_THREADS} - if [ $AOMP_JOB_THREADS -gt 16 ]; then + if [ "${AOMP_JOB_THREADS}" -gt 16 ]; then AOMP_JOB_THREADS=16 - echo "Limiting job threads to $AOMP_JOB_THREADS." + echo "Limiting job threads to ${AOMP_JOB_THREADS}." fi - echo THREADS: $AOMP_JOB_THREADS + echo "THREADS: ${AOMP_JOB_THREADS}" # Parallel Make - for directory in $SMOKE_DIRS; do - if [ ! -r $directory/Makefile ]; then continue; fi - pushd $directory > /dev/null - if [ $? -ne 0 ]; then continue; fi - base=$(basename `pwd`) - echo Make: $base - if [ $base == 'hip_rocblas' ] ; then - ls $AOMPROCM/rocblas > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo -e "$RED"$base - needs rocblas installed at $AOMPROCM/rocblas:"$BLK" - echo -e "$RED"$base - ROCBLAS NOT FOUND!!! SKIPPING TEST!"$BLK" - popd > /dev/null + for directory in ${SMOKE_DIRS}; do + if [ ! -r "${directory}/Makefile" ]; then continue; fi + if ! pushd "${directory}" > /dev/null; then continue; fi + base=$(basename "$(pwd)") + echo "Make: ${base}" + if [ "${base}" == 'hip_rocblas' ] ; then + if ! ls "${AOMPROCM}/rocblas" > /dev/null 2>&1; then + echo -e "${RED}${base} - needs rocblas installed at ${AOMPROCM}/rocblas:${BLK}" + echo -e "${RED}${base} - ROCBLAS NOT FOUND!!! SKIPPING TEST!${BLK}" + popd > /dev/null || exit continue fi fi - if [ $base == "gpus" ]; then # Compile and link only test + if [ "${base}" == "gpus" ]; then # Compile and link only test make clean > /dev/null - make &> make-log.txt - if [ $? -ne 0 ]; then + if ! make &> make-log.txt; then flock -e lockfile -c "echo $base: Make Failed >> ../make-fail.txt" else flock -e lockfile -c "echo $base >> ../passing-tests.txt" @@ -272,7 +269,7 @@ if [ "$AOMP_PARALLEL_SMOKE" == 1 ]; then else sem --jobs $AOMP_JOB_THREADS --id def_sem -u 'base=$(basename $(pwd)); make clean > /dev/null; make &> make-log.txt; if [ $? -ne 0 ]; then flock -e lockfile -c "echo $base: Make Failed >> ../make-fail.txt"; fi;' fi - popd > /dev/null + popd > /dev/null || exit done # Wait for jobs to finish before execution @@ -281,59 +278,57 @@ if [ "$AOMP_PARALLEL_SMOKE" == 1 ]; then # Parallel execution, currently limited to 4 jobs #--- Begin list iteration lrun=0 - while [ $lrun -lt $SMOKE_LRUN ]; do + while [ "${lrun}" -lt "${SMOKE_LRUN}" ]; do #--- - for directory in $SMOKE_DIRS; do - if [ ! -r $directory/Makefile ]; then continue; fi - pushd $directory > /dev/null - if [ $? -ne 0 ]; then continue; fi - base=$(basename `pwd`) - echo RUN $base - if [ $base == 'hip_rocblas' ] ; then - ls $AOMPROCM/rocblas > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo -e "$RED"$base - needs rocblas installed at $AOMPROCM/rocblas:"$BLK" - echo -e "$RED"$base - ROCBLAS NOT FOUND!!! SKIPPING TEST!"$BLK" - popd > /dev/null + for directory in ${SMOKE_DIRS}; do + if [ ! -r "${directory}/Makefile" ]; then continue; fi + if ! pushd "${directory}" > /dev/null; then continue; fi + base=$(basename "$(pwd)") + echo "RUN ${base}" + if [ "${base}" == 'hip_rocblas' ] ; then + if ! ls "${AOMPROCM}/rocblas" > /dev/null 2>&1; then + echo -e "${RED}${base} - needs rocblas installed at ${AOMPROCM}/rocblas:${BLK}" + echo -e "${RED}${base} - ROCBLAS NOT FOUND!!! SKIPPING TEST!${BLK}" + popd > /dev/null || exit continue fi fi #--- Begin test iteration run=0 set_my_nrun - while [ $run -lt $my_nrun ]; do + while [ "${run}" -lt "${my_nrun}" ]; do #--- - if [ $base == 'devices' ] || [ $base == 'stream' ]; then + if [ "${base}" == 'devices' ] || [ "${base}" == 'stream' ]; then sem --jobs 4 --id def_sem -u 'make run > /dev/null 2>&1' sem --jobs 4 --id def_sem -u 'make check > /dev/null 2>&1' - elif [ $base == 'printf_parallel_for_target' ] || [ $base == 'omp_places' ] || [ $base == 'pfspecifier' ] || [ $base == 'pfspecifier_str' ] ; then + elif [ "${base}" == 'printf_parallel_for_target' ] || [ "${base}" == 'omp_places' ] || [ "${base}" == 'pfspecifier' ] || [ "${base}" == 'pfspecifier_str' ] ; then sem --jobs 4 --id def_sem -u 'make verify-log > /dev/null' - elif [ $base == 'flags' ] ; then + elif [ "${base}" == 'flags' ] ; then make run - elif [ $base == 'liba_bundled' ] || [ $base == 'liba_bundled_cmdline' ]; then + elif [ "${base}" == 'liba_bundled' ] || [ "${base}" == 'liba_bundled_cmdline' ]; then sem --jobs 4 --id def_sem -u 'base=$(basename $(pwd)); make check > /dev/null; if [ $? -ne 0 ]; then flock -e lockfile -c "echo $base: Make Failed >> ../make-fail.txt"; fi;' - elif [ $base == "gpus" ]; then # Compile and link only test - echo gpus is compile only! - elif [ "$AOMP_SANITIZER" == 1 ] && [ "$script_dir_name" == "smoke-asan" ]; then + elif [ "${base}" == "gpus" ]; then # Compile and link only test + echo "gpus is compile only!" + elif [ "${AOMP_SANITIZER}" == 1 ] && [ "${script_dir_name}" == "smoke-asan" ]; then sem --jobs 4 --id def_sem -u 'make check-asan > /dev/null 2>&1' else sem --jobs 4 --id def_sem -u 'base=$(basename $(pwd)); make check &> run.log; rc=$?; if [ $rc -ne 0 ]; then [ -e ../make-fail.txt ] && mytest=$(grep -E "$base:" ../make-fail.txt); flock -e lockfile -c "if [[ ! -e ../make-fail.txt || -z \"$mytest\" ]]; then echo $base make check failed: $rc >> ../make-fail.txt; fi"; fi' fi #--- if [ -r "TEST_STATUS" ]; then - test_status=`cat TEST_STATUS` - if [ "$AOMP_SANITIZER" == 1 ] && [ "$test_status" -ne 0 ]; then - test_status=`cat FILECHECK_STATUS` + test_status=$(cat TEST_STATUS) + if [ "${AOMP_SANITIZER}" == 1 ] && [ "${test_status}" -ne 0 ]; then + test_status=$(cat FILECHECK_STATUS) fi - if [ "$test_status" == "124" ]; then break; fi # don't rerun timeouts + if [ "${test_status}" == "124" ]; then break; fi # don't rerun timeouts fi - run=$(($run+1)) + run=$((run+1)) done #--- End test iteration - popd > /dev/null + popd > /dev/null || exit done #--- - lrun=$(($lrun+1)) + lrun=$((lrun+1)) done #--- End list iteration @@ -353,115 +348,112 @@ fi # Loop over all directories and make run / make check depending on directory name #--- Begin list iteration lrun=0 -while [ $lrun -lt $SMOKE_LRUN ]; do +while [ "${lrun}" -lt "${SMOKE_LRUN}" ]; do #--- -for directory in $SMOKE_DIRS; do - if [ ! -r $directory/Makefile ]; then continue; fi - if [ "$SKIP_OMPT" == "1" ]; then - if [[ $directory =~ "ompt" ]]; then continue; fi - if [[ $directory =~ "lib-debug" ]]; then continue; fi +for directory in ${SMOKE_DIRS}; do + if [ ! -r "${directory}/Makefile" ]; then continue; fi + if [ "${SKIP_OMPT}" == "1" ]; then + if [[ ${directory} =~ "ompt" ]]; then continue; fi + if [[ ${directory} =~ "lib-debug" ]]; then continue; fi fi - pushd $directory > /dev/null - if [ $? -ne 0 ]; then continue; fi - if [ $lrun -eq 0 ]; then + if ! pushd "${directory}" > /dev/null; then continue; fi + if [ "${lrun}" -eq 0 ]; then make clean fi path=$(pwd) - base=$(basename $path) + base=$(basename "${path}") # Skip tests that are known failures skip=0 - for test in $skip_tests ; do - if [ $test == $base ] ; then + for test in ${skip_tests} ; do + if [ "${test}" == "${base}" ] ; then skip=1 break fi done - if [ $skip -ne 0 ] ; then - echo "Skip $base!" + if [ "${skip}" -ne 0 ] ; then + echo "Skip ${base}!" echo "" - popd > /dev/null + popd > /dev/null || exit continue fi - if [ $base == 'hip_rocblas' ] ; then - ls $AOMPROCM/rocblas > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo -e "$RED"$base - needs rocblas installed at $AOMPROCM/rocblas:"$BLK" - echo -e "$RED"$base - ROCBLAS NOT FOUND!!! SKIPPING TEST!"$BLK" - popd > /dev/null + if [ "${base}" == 'hip_rocblas' ] ; then + if ! ls "${AOMPROCM}/rocblas" > /dev/null 2>&1; then + echo -e "${RED}${base} - needs rocblas installed at ${AOMPROCM}/rocblas:${BLK}" + echo -e "${RED}${base} - ROCBLAS NOT FOUND!!! SKIPPING TEST!${BLK}" + popd > /dev/null || exit continue fi fi #--- Begin test iteration run=0 set_my_nrun - while [ $run -lt $my_nrun ]; do + while [ "${run}" -lt "${my_nrun}" ]; do #--- make 2>&1 | tee make-log.txt - if [ ${PIPESTATUS[0]} -ne 0 ]; then - echo "$base: Make Failed" >> ../make-fail.txt + if [ "${PIPESTATUS[0]}" -ne 0 ]; then + echo "${base}: Make Failed" >> ../make-fail.txt if [ -r "TEST_STATUS" ]; then - test_status=`cat TEST_STATUS` - if [ "$test_status" == "124" ]; then break; fi # don't rerun timeouts + test_status=$(cat TEST_STATUS) + if [ "${test_status}" == "124" ]; then break; fi # don't rerun timeouts fi - run=$(($run+1)) + run=$((run+1)) continue fi - if [ $base == 'devices' ] || [ $base == 'stream' ]; then + if [ "${base}" == 'devices' ] || [ "${base}" == 'stream' ]; then make run > /dev/null 2>&1 make check > /dev/null 2>&1 - elif [ $base == 'flags' ] ; then # Flags has multiple runs + elif [ "${base}" == 'flags' ] ; then # Flags has multiple runs make run > /dev/null 2>&1 - elif [ $base == "gpus" ]; then # Compile and link only test - echo "$base" >> ../passing-tests.txt - elif [ $base == 'printf_parallel_for_target' ] || [ $base == 'omp_places' ] || [ $base == 'pfspecifier' ] || [ $base == 'pfspecifier_str' ] ; then + elif [ "${base}" == "gpus" ]; then # Compile and link only test + echo "${base}" >> ../passing-tests.txt + elif [ "${base}" == 'printf_parallel_for_target' ] || [ "${base}" == 'omp_places' ] || [ "${base}" == 'pfspecifier' ] || [ "${base}" == 'pfspecifier_str' ] ; then make verify-log - elif [ "$AOMP_SANITIZER" == 1 ] && [ "$script_dir_name" == "smoke-asan" ]; then + elif [ "${AOMP_SANITIZER}" == 1 ] && [ "${script_dir_name}" == "smoke-asan" ]; then make check-asan > /dev/null 2>&1 else make check &> run.log rc=$? # liba_bundled has an additional Makefile, that may fail on the make check - if [ $rc -ne 0 ]; then - if [ $base == 'liba_bundled' ] || [ $base == 'liba_bundled_cmdline' ]; then - echo "$base: Make Failed" >> ../make-fail.txt + if [ "${rc}" -ne 0 ]; then + if [ "${base}" == 'liba_bundled' ] || [ "${base}" == 'liba_bundled_cmdline' ]; then + echo "${base}: Make Failed" >> ../make-fail.txt else - echo "$base 'make check' failed: $rc" >> ../make-fail.txt + echo "${base} 'make check' failed: ${rc}" >> ../make-fail.txt fi if [ -r "TEST_STATUS" ]; then - test_status=`cat TEST_STATUS` - if [ "$test_status" == "124" ]; then break; fi # don't rerun timeouts + test_status=$(cat TEST_STATUS) + if [ "${test_status}" == "124" ]; then break; fi # don't rerun timeouts fi fi fi echo "" #--- if [ -r "TEST_STATUS" ]; then - test_status=`cat TEST_STATUS` - if [ "$AOMP_SANITIZER" == 1 ] && [ "$test_status" -ne 0 ]; then - test_status=`cat FILECHECK_STATUS` + test_status=$(cat TEST_STATUS) + if [ "${AOMP_SANITIZER}" == 1 ] && [ "${test_status}" -ne 0 ]; then + test_status=$(cat FILECHECK_STATUS) fi - if [ "$test_status" == "124" ]; then break; fi # don't rerun timeouts + if [ "${test_status}" == "124" ]; then break; fi # don't rerun timeouts fi - run=$(($run+1)) + run=$((run+1)) done #--- End test iteration - popd > /dev/null + popd > /dev/null || exit done #--- -lrun=$(($lrun+1)) +lrun=$((lrun+1)) done #--- End list iteration # Print run.log for all tests that need visual inspection -for directory in $SMOKE_DIRS; do - if [ ! -r $directory/Makefile ]; then continue; fi - pushd $directory > /dev/null - if [ $? -ne 0 ]; then continue; fi +for directory in ${SMOKE_DIRS}; do + if [ ! -r "${directory}/Makefile" ]; then continue; fi + if ! pushd "${directory}" > /dev/null; then continue; fi path=$(pwd) - base=$(basename $path) - if [ $base == 'devices' ] || [ $base == 'stream' ] ; then + base=$(basename "${path}") + if [ "${base}" == 'devices' ] || [ "${base}" == 'stream' ] ; then echo "" - echo -e "$ORG"$base - Run Log:"$BLK" + echo -e "${ORG}${base} - Run Log:${BLK}" echo "--------------------------" if [ -e run.log ]; then cat run.log @@ -469,19 +461,19 @@ for directory in $SMOKE_DIRS; do echo "" echo "" fi - popd > /dev/null + popd > /dev/null || exit done gatherdata printfails # Clean up, hide output -if [ "$EPSDB" != 1 ] && [ "$CLEANUP" != 0 ]; then +if [ "${EPSDB}" != 1 ] && [ "${CLEANUP}" != 0 ]; then cleanup fi -popd +popd || exit -realpath=`realpath $0` -thisdir=`dirname $realpath` -$thisdir/../../bin/check_amdgpu_modversion.sh +realpath=$(realpath "$0") +thisdir=$(dirname "${realpath}") +"${thisdir}/../../bin/check_amdgpu_modversion.sh" From 66f94515239b628120da763ec8a17cda0619260b Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser Date: Tue, 30 Jun 2026 10:18:31 -0500 Subject: [PATCH 2/5] fixup! [NFCI] Make check_smoke.sh shellcheck-compliant --- test/smoke/check_smoke.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/smoke/check_smoke.sh b/test/smoke/check_smoke.sh index ab7160427..280c21bd0 100755 --- a/test/smoke/check_smoke.sh +++ b/test/smoke/check_smoke.sh @@ -267,7 +267,7 @@ if [ "${AOMP_PARALLEL_SMOKE}" == 1 ]; then flock -e lockfile -c "echo $base >> ../passing-tests.txt" fi else - sem --jobs $AOMP_JOB_THREADS --id def_sem -u 'base=$(basename $(pwd)); make clean > /dev/null; make &> make-log.txt; if [ $? -ne 0 ]; then flock -e lockfile -c "echo $base: Make Failed >> ../make-fail.txt"; fi;' + sem --jobs "${AOMP_JOB_THREADS}" --id def_sem -u 'base=$(basename $(pwd)); make clean > /dev/null; make &> make-log.txt; if [ $? -ne 0 ]; then flock -e lockfile -c "echo $base: Make Failed >> ../make-fail.txt"; fi;' fi popd > /dev/null || exit done From 80cccc1cc3c6b0ae4431132b4f51c23a34897c32 Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser Date: Tue, 30 Jun 2026 11:12:07 -0500 Subject: [PATCH 3/5] [CK] Remove DPP kernel workaround Currently, this is disabled by CK itself, see: https://github.com/ROCm/rocm-libraries/pull/8501 --- bin/run_composable-kernels.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bin/run_composable-kernels.sh b/bin/run_composable-kernels.sh index e2d6feaff..0184330f2 100755 --- a/bin/run_composable-kernels.sh +++ b/bin/run_composable-kernels.sh @@ -318,13 +318,7 @@ else fi # For some reason, CK on gfx12 wants this set. -CKCmakeCmd+="-DBUILD_DEV=On " - -# TODO: Remove this workaround when possible. -# As of 2026-JUN-12 this is needed to bypass DPP-related build errors: -# error: cannot compile inline asm -# :2:33: error: not a valid operand. -CKCmakeCmd+="-DDISABLE_DPP_KERNELS=ON" +CKCmakeCmd+="-DBUILD_DEV=On" # Ensure CK build directory is cleaned. if [ "${ShouldRebuildCK}" == 'yes' ]; then From 3aab18f1c4e103f52a026c865e0f4d10b187ef04 Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser Date: Mon, 29 Jun 2026 12:34:40 -0500 Subject: [PATCH 4/5] [CI] Add semaphore isolation helper to avoid hangs Fix possible stale semaphores during parallel runs - isolate semaphores per test run - steal semaphore after timeout Issue: Semaphores including hostnames, which in turn include uppercase letters, are not reaped correctly. This lead to hangs of CI runs. Very recent parallel versions 20260522 and newer handle this case by converting hostname to lowercase: https://cgit.git.savannah.gnu.org/cgit/parallel.git/tree/src/parallel?h=20260522#n7565 --- bin/semaphore_isolation.src | 33 +++++++++++++++++++++++++++++++++ test/smoke/check_smoke.sh | 8 ++++++++ 2 files changed, 41 insertions(+) create mode 100644 bin/semaphore_isolation.src diff --git a/bin/semaphore_isolation.src b/bin/semaphore_isolation.src new file mode 100644 index 000000000..d93e6a401 --- /dev/null +++ b/bin/semaphore_isolation.src @@ -0,0 +1,33 @@ +# shellcheck shell=bash +# +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT +# +# semaphore_isolation.src - shared helper for GNU parallel's "sem" semaphore. +# +# Source this file and call isolate_semaphore before any "sem" use. It gives +# "sem" a private, self-cleaning semaphore namespace (its own PARALLEL_HOME) so +# that a leaked/stale token can neither deadlock a run nor leak across runs, and +# optionally bounds a stuck semaphore with a timeout. +# +# Background: on hosts whose name contains uppercase letters, GNU parallel's +# dead-lock reaper (remove_dead_locks) never matches its own token files, so a +# token left behind by an unclean death is never reclaimed and every later +# "sem --wait" deadlocks. A per-run private namespace sidesteps this entirely. +# +# Design: +# - No-op when PARALLEL_HOME is already set, so an outer caller never collides. +# - AOMP_SEMAPHORE_TIMEOUT (seconds), if set, is passed to every "sem" call as +# --semaphoretimeout: a wedged sem/sem --wait then steals and proceeds +# instead of hanging. Left unset => pure isolation, no timeout. + +isolate_semaphore() { + [ -n "${PARALLEL_HOME:-}" ] && return 0 + _semaphore_home=$(mktemp -d "${TMPDIR:-/tmp}/aomp-sem.XXXXXX") || return 0 + export PARALLEL_HOME="$_semaphore_home" + if [ -n "${AOMP_SEMAPHORE_TIMEOUT:-}" ]; then + export PARALLEL="--semaphoretimeout ${AOMP_SEMAPHORE_TIMEOUT}${PARALLEL:+ $PARALLEL}" + fi + trap 'rm -rf "$_semaphore_home"' EXIT +} diff --git a/test/smoke/check_smoke.sh b/test/smoke/check_smoke.sh index 280c21bd0..604af70e3 100755 --- a/test/smoke/check_smoke.sh +++ b/test/smoke/check_smoke.sh @@ -1,5 +1,9 @@ #!/bin/bash # +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT +# # Checks all tests in smoke directory using make check. Programs return 0 for success or a number > 0 for failure. # Tests that need to be visually inspected: devices, stream # @@ -228,6 +232,10 @@ if [ "${AOMP_PARALLEL_SMOKE}" == 1 ]; then export AOMP_NO_PREREQ=1 # disable prereq target so builds can be reused fi if sem --help > /dev/null; then + # Give "sem" a private, self-cleaning semaphore namespace (no-op if the + # caller already set PARALLEL_HOME). Prevents stale-token deadlocks/leaks. + # shellcheck source=/dev/null + source "${path}/../../bin/semaphore_isolation.src" 2>/dev/null && isolate_semaphore COMP_THREADS=1 if [ -n "$(which getconf)" ]; then COMP_THREADS=$(getconf _NPROCESSORS_ONLN) From 2d6d2c2dd9e69dd9fdcf447c62c9b12891a44b1a Mon Sep 17 00:00:00 2001 From: theRonShark Date: Wed, 1 Jul 2026 17:53:47 -0400 Subject: [PATCH 5/5] version of HipDevCov showing annotated source (#2316) * version of HipDevCov showing annotated source * filechecked --- test/smoke-dev/OutputHipDevCov/Makefile | 30 +++++++ test/smoke-dev/OutputHipDevCov/main.hip | 62 ++++++++++++++ test/smoke-dev/OutputHipDevCov/mod.hip | 42 ++++++++++ .../OutputHipDevCov/run_and_check.sh | 84 +++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 test/smoke-dev/OutputHipDevCov/Makefile create mode 100644 test/smoke-dev/OutputHipDevCov/main.hip create mode 100644 test/smoke-dev/OutputHipDevCov/mod.hip create mode 100755 test/smoke-dev/OutputHipDevCov/run_and_check.sh diff --git a/test/smoke-dev/OutputHipDevCov/Makefile b/test/smoke-dev/OutputHipDevCov/Makefile new file mode 100644 index 000000000..749ada2af --- /dev/null +++ b/test/smoke-dev/OutputHipDevCov/Makefile @@ -0,0 +1,30 @@ +include ../../Makefile.defs + +# HipDevCov: device-side coverage drained via the in-tree HSA introspection +# pass. main (TESTNAME) is built by the harness with source-based coverage; it +# hipModuleLoad()s mod.co (built+extracted by run_and_check.sh) whose kernel has +# no host shadow and is therefore only reachable by the HSA drain. + +TESTNAME = main +TESTSRC_MAIN = main.hip +TESTSRC_AUX = +TESTSRC_ALL = $(TESTSRC_MAIN) $(TESTSRC_AUX) + +AOMPHIP ?= $(AOMP) +ROCM_PATH ?= $(AOMPHIP) + +# Extra budget: run_and_check.sh also compiles mod.hip and extracts mod.co. +TIMEOUT = 300s + +CFLAGS = -x hip $(TARGET) -fno-gpu-rdc -fprofile-instr-generate -fcoverage-mapping --rocm-path=$(ROCM_PATH) +LINK_FLAGS = -L$(ROCM_PATH)/lib -lamdhip64 -Wl,-rpath,$(ROCM_PATH)/lib + +CC = $(AOMP)/bin/clang $(VERBOSE) + +RUNCMD = ./run_and_check.sh "$(AOMP)" "$(GPU_W_FEATURES)" "$(FILECHECK)" "$(ROCM_PATH)" + +include ../Makefile.rules + +clean:: + rm -f mod.co builder builder.*.host-* builder.*.hip-amdgcn-amd-amdhsa--* \ + *.profraw merged.profdata diff --git a/test/smoke-dev/OutputHipDevCov/main.hip b/test/smoke-dev/OutputHipDevCov/main.hip new file mode 100644 index 000000000..9712b02ec --- /dev/null +++ b/test/smoke-dev/OutputHipDevCov/main.hip @@ -0,0 +1,62 @@ +// HipDevCov: prove the HSA-introspection device-coverage drain captures device +// code that the HIP host-shadow drain cannot see. +// +// This program contains exactly one statically-registered kernel (host_kernel) +// -- it has a host-side shadow, so the host-shadow drain covers it. It then +// hipModuleLoad()s mod.co (built+extracted by run_and_check.sh) and launches +// mod_kernel, which is NOT compiled into this program and therefore has no host +// shadow here. mod_kernel's device counters can only reach the profile via the +// HSA drain. run_and_check.sh asserts BOTH kernels appear in the merged +// profile, so a passing run requires the HSA drain to be working. + +#include +#include + +__global__ void host_kernel(int *p, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + p[i] = p[i] * 3; // taken region + else + p[0] = p[0] - 1; // untaken region +} + +int main() { + const int n = 64; + int *d = nullptr; + if (hipMalloc(&d, n * sizeof(int)) != hipSuccess) + return 1; + if (hipMemset(d, 0, n * sizeof(int)) != hipSuccess) + return 2; + + // (1) host-shadow path: a normally-registered kernel. + host_kernel<<>>(d, n); + if (hipDeviceSynchronize() != hipSuccess) + return 3; + + // (2) HSA-only path: a kernel from a separately loaded code object, with no + // host shadow in this process. + hipModule_t mod; + if (hipModuleLoad(&mod, "mod.co") != hipSuccess) { + fprintf(stderr, "hipModuleLoad(mod.co) failed\n"); + return 4; + } + hipFunction_t fn; + if (hipModuleGetFunction(&fn, mod, "mod_kernel") != hipSuccess) { + fprintf(stderr, "hipModuleGetFunction(mod_kernel) failed\n"); + return 5; + } + int nn = n; + void *args[] = {&d, &nn}; + if (hipModuleLaunchKernel(fn, 1, 1, 1, 64, 1, 1, 0, nullptr, args, nullptr) != + hipSuccess) + return 6; + if (hipDeviceSynchronize() != hipSuccess) + return 7; + + // Intentionally do NOT hipModuleUnload(mod): the HSA-introspection drain runs + // at process exit and walks the code objects still loaded on the agent. + // Unloading mod.co here would remove its counters before the drain sees them. + (void)hipFree(d); + printf("HipDevCov: ran host_kernel + module mod_kernel\n"); + return 0; +} diff --git a/test/smoke-dev/OutputHipDevCov/mod.hip b/test/smoke-dev/OutputHipDevCov/mod.hip new file mode 100644 index 000000000..eb1590321 --- /dev/null +++ b/test/smoke-dev/OutputHipDevCov/mod.hip @@ -0,0 +1,42 @@ +// Device-only kernel used to produce a standalone, loadable code object +// (mod.co) that carries its own instrumentation (__llvm_profile_sections + +// the device profile runtime). main.hip hipModuleLoad()s this code object, so +// in that process mod_kernel has NO host-side shadow registration. The HIP +// host-shadow drain therefore cannot see it; only the HSA-introspection drain +// (which walks every code object loaded on the agent) can collect its +// counters. That is exactly what this test asserts. +// +// When built with -DBUILD_MODULE_EXE this file is a normal HIP executable; the +// full executable link makes clang's driver add the amdgcn device profile +// runtime to the device link (addProfileRTLibs), so the embedded device code +// object is fully instrumented. run_and_check.sh extracts that object as +// mod.co. (The standalone --offload-device-only / hipcc --genco paths do NOT +// add the device profile RT, leaving __llvm_profile_instrument_gpu / +// __llvm_profile_raw_version unresolved -- hence the extract-from-executable +// approach.) + +#include + +extern "C" __global__ void mod_kernel(int *p, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + p[i] = p[i] + 7; // taken region + else + p[0] = p[0] - 1; // untaken region (non-trivial coverage) +} + +#ifdef BUILD_MODULE_EXE +int main() { + const int n = 64; + int *d = nullptr; + if (hipMalloc(&d, n * sizeof(int)) != hipSuccess) + return 1; + if (hipMemset(d, 0, n * sizeof(int)) != hipSuccess) + return 2; + mod_kernel<<>>(d, n); + if (hipDeviceSynchronize() != hipSuccess) + return 3; + (void)hipFree(d); + return 0; +} +#endif diff --git a/test/smoke-dev/OutputHipDevCov/run_and_check.sh b/test/smoke-dev/OutputHipDevCov/run_and_check.sh new file mode 100755 index 000000000..46b1abdf2 --- /dev/null +++ b/test/smoke-dev/OutputHipDevCov/run_and_check.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# HipDevCov check driver. +# +# Args (passed from the Makefile): +# $1 = AOMP (LLVM dir containing bin/clang, llvm-profdata, llvm-objdump) +# $2 = GPU arch (e.g. gfx90a, may include :features) +# $3 = FILECHECK (path to FileCheck) +# $4 = ROCM_PATH (HIP/ROCm install for --rocm-path and libamdhip64) +# +# Returns 0 only if BOTH the host-shadow kernel and the module-only kernel are +# present in the merged profile (the latter requires the HSA drain). + +set -u +AOMP="$1"; ARCH="$2"; ROCM="$4" +CLANG="$AOMP/bin/clang" +PROFDATA="$AOMP/bin/llvm-profdata" +OBJDUMP="$AOMP/bin/llvm-objdump" +COV="$AOMP/bin/llvm-cov" + +here="$(cd "$(dirname "$0")" && pwd)" +cd "$here" || exit 1 + +set -x + +# Capability gate: skip cleanly on toolchains that do not ship the device +# profile runtime / HSA drain yet (so this does not red-fail CI before the +# feature lands). A real, drain-capable toolchain has the amdgcn device profile +# RT and the host drain symbol. +resdir="$("$CLANG" -print-resource-dir 2>/dev/null)" +devrt="$resdir/lib/amdgcn-amd-amdhsa/libclang_rt.profile.a" +if [ ! -f "$devrt" ]; then + echo "SKIP HipDevCov: no device profile runtime at $devrt" + exit 0 +fi +if ! ls "$resdir"/lib/*/libclang_rt.profile_rocm*.a >/dev/null 2>&1; then + echo "SKIP HipDevCov: no host profile_rocm runtime (HSA drain) in toolchain" + exit 0 +fi + +set -e +rm -f ./*.profraw mod.co merged.profdata builder builder.*.host-* \ + builder.*.hip-amdgcn-amd-amdhsa--* 2>/dev/null || true + +# 1. Build mod.hip as a full executable so the device link gets the device +# profile RT, then extract its device code object as the loadable mod.co. +"$CLANG" -x hip --offload-arch="$ARCH" -fno-gpu-rdc -DBUILD_MODULE_EXE \ + -fprofile-instr-generate -fcoverage-mapping --rocm-path="$ROCM" \ + mod.hip -o builder -L"$ROCM/lib" -lamdhip64 -Wl,-rpath,"$ROCM/lib" +"$OBJDUMP" --offloading builder >/dev/null 2>&1 || true +shopt -s nullglob +extracted=(builder*.hip-amdgcn-amd-amdhsa--*gfx*) +if [ ${#extracted[@]} -eq 0 ]; then + echo "FAIL HipDevCov: could not extract device code object from builder" + exit 1 +fi +cp "${extracted[0]}" mod.co + +# 2. main is built by the smoke harness (TESTNAME=main). Build it here too if it +# is missing, so the script also works when run standalone. +if [ ! -x ./main ]; then + "$CLANG" -x hip --offload-arch="$ARCH" -fno-gpu-rdc \ + -fprofile-instr-generate -fcoverage-mapping --rocm-path="$ROCM" \ + main.hip -o main -L"$ROCM/lib" -lamdhip64 -Wl,-rpath,"$ROCM/lib" +fi + +# 3. Run. Device .profraw files are written to CWD (arch-prefixed for the +# host-shadow drain, arch.hsa-prefixed for the HSA drain); the host one +# goes to LLVM_PROFILE_FILE. +rm -f ./*.profraw +LLVM_PROFILE_FILE="$here/host.profraw" \ + LD_LIBRARY_PATH="$ROCM/lib:${LD_LIBRARY_PATH:-}" \ + ./main + +# 4. Merge host + all device profraws and assert both kernels are present. +"$PROFDATA" merge -sparse -o merged.profdata ./*.profraw +"$PROFDATA" show --all-functions merged.profdata + +# 5. Sanity (non-fatal): llvm-cov can consume the merged device+host profile. +# main's covmap only describes host_kernel/main, so llvm-cov warns about the +# module-only function; that is expected, hence non-fatal. +"$COV" report ./main -instr-profile=merged.profdata +"$COV" show ./main -instr-profile=merged.profdata + +echo "HipDevCov PASSED"