diff --git a/tests/integrate/Autotest.sh b/tests/integrate/Autotest.sh index 4f49b3ade9..1adf2f924c 100755 --- a/tests/integrate/Autotest.sh +++ b/tests/integrate/Autotest.sh @@ -387,7 +387,11 @@ run_case() check_out result.out $my_threshold $my_force_threshold $my_stress_threshold $my_fatal_threshold $my_descriptor_threshold fi else - bash -e ../../integrate/validation_tools/catch_properties.sh result.ref + if ! bash -e ../../integrate/validation_tools/catch_properties.sh result.ref; then + echo -e "\e[0;31m [ERROR ] Failed to generate result.ref \e[0m" + let fatal++ + fatal_case_list+=$dir'\n' + fi fi fi @@ -530,5 +534,11 @@ else exit 1 fi else -echo "Generate test cases complete." +if [ "$fatal" -eq 0 ] && [ "$failed" -eq 0 ]; then + echo "Generate test cases complete." +else + echo -e "\e[0;31m[ERROR ]\e[0m Failed to generate references for $[ $fatal + $failed ] case(s)." + printf "%b" "$fatal_case_list$failed_case_list" + exit 1 +fi fi diff --git a/tests/integrate/CMakeLists.txt b/tests/integrate/CMakeLists.txt index af107559be..4ec3c7b26e 100644 --- a/tests/integrate/CMakeLists.txt +++ b/tests/integrate/CMakeLists.txt @@ -1,4 +1,22 @@ find_program(BASH bash) +# Validation-script regressions do not impose a Python configure dependency +# on builds that only use a dedicated output validator. +find_package(Python3 QUIET COMPONENTS Interpreter) +if(Python3_Interpreter_FOUND) + add_test( + NAME compare_file_test + COMMAND ${Python3_EXECUTABLE} -m unittest validation_tools/test_compare_file.py + WORKING_DIRECTORY ${ABACUS_TEST_DIR}/integrate + ) + + add_test( + NAME catch_properties_test + COMMAND ${BASH} validation_tools/test_catch_properties.sh + WORKING_DIRECTORY ${ABACUS_TEST_DIR}/integrate + ) +else() + message(STATUS "Python3 interpreter not found; skipping validation-script regression tests") +endif() if(ENABLE_ASAN) add_test( diff --git a/tests/integrate/validation_tools/CompareFile.py b/tests/integrate/validation_tools/CompareFile.py index 4aa6cb1354..17a5728a09 100755 --- a/tests/integrate/validation_tools/CompareFile.py +++ b/tests/integrate/validation_tools/CompareFile.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import math import sys usage = ''' @@ -46,6 +47,8 @@ def ReadParam(): return file1,file2,epsilon,useAbsoluteVal,compareAbsoluteDiff def is_two_data_diff(data1,data2,epsilon,compareAbsoluteDiff): + if not math.isfinite(data1) or not math.isfinite(data2): + return True if compareAbsoluteDiff: if abs(data1-data2) > epsilon: return True else: @@ -60,6 +63,11 @@ def ReadFile(file1,lines): try: with open(file1) as f1: for line in f1.readlines(): lines.append(line) + # A final blank line carries no numerical information and varies among + # otherwise equivalent output writers. Keep interior blank lines so a + # missing or shifted data row is still detected below. + while lines and not lines[-1].strip(): + lines.pop() except OSError: print("Error: can not find file %s" % file1) sys.exit(1) @@ -86,6 +94,16 @@ def IsComplex(x): return False return [realp,imagp] +def HasNonFiniteNumber(line): + for token in line.split(): + complex_value = IsComplex(token) + if complex_value: + if not math.isfinite(complex_value[0]) or not math.isfinite(complex_value[1]): + return True + elif IsFloat(token) and not math.isfinite(float(token)): + return True + return False + def ExitError(iline,line1,line2,jnumber=-1): if jnumber < 0: print('Compare Error: line %d\n %s:%s %s:%s'%(iline+1,file1,line1,file2,line2)) @@ -102,7 +120,9 @@ def ExitError(iline,line1,line2,jnumber=-1): ReadFile(file2,lines2) for i in range(min(len(lines1),len(lines2))): - if lines1[i].strip() == lines2[i].strip():continue + if lines1[i].strip() == lines2[i].strip(): + if HasNonFiniteNumber(lines1[i]): ExitError(i,lines1[i],lines2[i]) + continue elif '' in [lines1[i].strip(),lines2[i].strip()]: ExitError(i,lines1[i],lines2[i]) elif len(lines1[i].split()) != len(lines2[i].split()): ExitError(i,lines1[i],lines2[i]) else: @@ -128,3 +148,9 @@ def ExitError(iline,line1,line2,jnumber=-1): if is_two_data_diff(float(sline1[j]), float(sline2[j]), epsilon, compareAbsoluteDiff): ExitError(i,sline1[j],sline2[j],j) elif sline1[j] != sline2[j]: ExitError(i,sline1[j],sline2[j],j) + + if len(lines1) != len(lines2): + i = min(len(lines1),len(lines2)) + line1 = lines1[i] if i < len(lines1) else '' + line2 = lines2[i] if i < len(lines2) else '' + ExitError(i,line1,line2) diff --git a/tests/integrate/validation_tools/catch_properties.sh b/tests/integrate/validation_tools/catch_properties.sh index 22ef71b9e3..3912333d88 100755 --- a/tests/integrate/validation_tools/catch_properties.sh +++ b/tests/integrate/validation_tools/catch_properties.sh @@ -7,7 +7,24 @@ # one-time INPUT switch parsing) and then runs each property block below. PROPS_SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) source "$PROPS_SCRIPT_DIR/props_common.sh" -props_init "$1" +props_result_file=$1 +props_reference_mode=0 +case "$props_result_file" in + result.ref|*/result.ref) + props_reference_mode=1 + cleanup_failed_reference(){ + reference_status=$? + trap - EXIT + if [ "$reference_status" -ne 0 ]; then + rm -f "$props_result_file" + fi + exit "$reference_status" + } + trap cleanup_failed_reference EXIT + set -e + ;; +esac +props_init "$props_result_file" source "$PROPS_SCRIPT_DIR/props_basic.sh" source "$PROPS_SCRIPT_DIR/props_mat.sh" source "$PROPS_SCRIPT_DIR/props_cube.sh" @@ -72,8 +89,14 @@ run_ml_rpa_props # Before the split this block ran as a separate `bash` process without -e, # so a failing helper (get_sum_*.py, a missing deepks_desc.dat) left an # empty value for the threshold check instead of aborting the collection. -# Keep that behavior with a subshell that disables errexit. -( set +e; run_deepks_props ) +# Keep that behavior for result.out. Reference generation instead inherits the +# entry point's errexit mode, so an early failure cannot be hidden by a later +# successful command in the collector. +if [ "$props_reference_mode" -eq 1 ]; then + run_deepks_props +else + ( set +e; run_deepks_props ) || true +fi #-------------------------------------------- # basic collectors that run after deepks: symmetry diff --git a/tests/integrate/validation_tools/props_basic.sh b/tests/integrate/validation_tools/props_basic.sh index a73902d097..e984400c5a 100644 --- a/tests/integrate/validation_tools/props_basic.sh +++ b/tests/integrate/validation_tools/props_basic.sh @@ -82,8 +82,7 @@ fi if ! test -z "$has_cond" && [ $has_cond == 1 ]; then onref=refOnsager.txt oncal=OUT.autotest/Onsager.txt - python3 $COMPARE_SCRIPT $onref $oncal 3 -com_type 0 - echo "CompareH_Failed $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareH_Failed" "$onref" "$oncal" 3 0 0 rm -f je-je.txt Chebycoef fi diff --git a/tests/integrate/validation_tools/props_common.sh b/tests/integrate/validation_tools/props_common.sh index 590f157b0a..d314de9174 100644 --- a/tests/integrate/validation_tools/props_common.sh +++ b/tests/integrate/validation_tools/props_common.sh @@ -59,6 +59,40 @@ sanitize_result_key(){ } +record_compare_status(){ + result_file=$1 + result_key=$2 + compare_status=$3 + + if [ "$compare_status" -ne 0 ]; then + case "$result_file" in + result.ref|*/result.ref) + echo "Error: refusing to record failed comparison $result_key in $result_file" >&2 + rm -f "$result_file" + exit "$compare_status" + ;; + esac + fi + echo "$result_key $compare_status" >> "$result_file" +} + + +run_and_record_status(){ + result_file=$1 + result_key=$2 + shift 2 + + # A failed check remains useful data in result.out, but it must never be + # accepted as the expected value while generating result.ref. + if "$@"; then + command_status=0 + else + command_status=$? + fi + record_compare_status "$result_file" "$result_key" "$command_status" +} + + record_compare_result(){ result_file=$1 result_key=$2 @@ -66,14 +100,28 @@ record_compare_result(){ cal_file=$4 accuracy=${5:-8} use_abs=${6:-0} + compare_type=${7:-} if [ ! -f "$ref_file" ] || [ ! -f "$cal_file" ]; then - echo "$result_key 1" >> "$result_file" - return + record_compare_status "$result_file" "$result_key" 1 + return $? fi - python3 $COMPARE_SCRIPT "$ref_file" "$cal_file" "$accuracy" -abs "$use_abs" - echo "$result_key $?" >> "$result_file" + if [ -n "$compare_type" ]; then + if python3 "$COMPARE_SCRIPT" "$ref_file" "$cal_file" "$accuracy" \ + -abs "$use_abs" -com_type "$compare_type"; then + compare_status=0 + else + compare_status=$? + fi + else + if python3 "$COMPARE_SCRIPT" "$ref_file" "$cal_file" "$accuracy" -abs "$use_abs"; then + compare_status=0 + else + compare_status=$? + fi + fi + record_compare_status "$result_file" "$result_key" "$compare_status" } diff --git a/tests/integrate/validation_tools/props_cube.sh b/tests/integrate/validation_tools/props_cube.sh index 1dfd83270f..a73fad2897 100644 --- a/tests/integrate/validation_tools/props_cube.sh +++ b/tests/integrate/validation_tools/props_cube.sh @@ -22,8 +22,7 @@ run_cube_pot_props(){ if ! test -z "$out_pot" && [ $out_pot == 1 ]; then pot1ref=pot.cube.ref pot1cal=OUT.autotest/pot.cube - python3 $COMPARE_SCRIPT $pot1ref $pot1cal 3 - echo "ComparePot1_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "ComparePot1_pass" "$pot1ref" "$pot1cal" 3 fi #------------------------------- @@ -32,8 +31,7 @@ fi if ! test -z "$out_pot" && [ $out_pot == 2 ]; then pot1ref=potes.cube.ref pot1cal=OUT.autotest/potes.cube - python3 $COMPARE_SCRIPT $pot1ref $pot1cal 8 - echo "ComparePot_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "ComparePot_pass" "$pot1ref" "$pot1cal" 8 fi #------------------------------- @@ -43,8 +41,7 @@ fi if ! test -z "$out_elf" && [ $out_elf == 1 ]; then elf1ref=elftot.cube.ref elf1cal=OUT.autotest/elftot.cube - python3 $COMPARE_SCRIPT $elf1ref $elf1cal 3 - echo "ComparePot1_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "ComparePot1_pass" "$elf1ref" "$elf1cal" 3 fi } @@ -66,8 +63,8 @@ fi #--------------------------------------- if ! test -z "$has_scan" && [ $has_scan == "scan" ] && \ ! test -z "$out_chg" && [ $out_chg == 1 ]; then - python3 $COMPARE_SCRIPT tau.cube.ref OUT.autotest/tau.cube 8 - echo "tau.cube_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "tau.cube_pass" \ + "tau.cube.ref" "OUT.autotest/tau.cube" 8 fi #--------------------------------------- @@ -76,8 +73,8 @@ fi #--------------------------------------- if ! test -z "$has_ldos" && [ $has_ldos == 1 ]; then stm_bias=$(get_input_key_value "stm_bias" "OUT.autotest/INPUT.info") - python3 $COMPARE_SCRIPT ldos.cube.ref OUT.autotest/ldos_"$stm_bias"ev.cube 8 - echo "ldos.cube_pass $?" >> $props_result_file + record_compare_result "$props_result_file" "ldos.cube_pass" \ + "ldos.cube.ref" "OUT.autotest/ldos_${stm_bias}ev.cube" 8 fi #--------------------------------------- @@ -159,7 +156,7 @@ if ! test -z "$has_lowf" && [ "$has_lowf" == 1 ]; then wfc_name=wfk2_nao input_file=wfk2_nao fi - awk 'BEGIN {flag=999} + awk 'BEGIN {flag=999} { if($2 == "(band)") {flag=2;print $0} else if(flag>0) {flag-=1;print $0} @@ -175,8 +172,8 @@ if ! test -z "$has_lowf" && [ "$has_lowf" == 1 ]; then wfc_ref="$wfc_name"_mod.txt.ref fi - python3 $COMPARE_SCRIPT $wfc_cal $wfc_ref 8 -abs 1 - echo "Compare_wfc_lcao_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "Compare_wfc_lcao_pass" \ + "$wfc_cal" "$wfc_ref" 8 1 fi } @@ -187,8 +184,8 @@ run_cube_tail_props(){ # mulliken charge #-------------------------------------------- if ! test -z "$out_mul" && [ $out_mul == 1 ]; then - python3 $COMPARE_SCRIPT mulliken.txt.ref OUT.autotest/mulliken.txt 3 - echo "Compare_mulliken_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "Compare_mulliken_pass" \ + "mulliken.txt.ref" "OUT.autotest/mulliken.txt" 3 fi #-------------------------------------------- @@ -279,11 +276,8 @@ nspin=$(get_input_key_value "nspin" "INPUT") if_separate_k=$(get_input_key_value "if_separate_k" "INPUT") if [ "$nspin" = "4" ] && { [ "$if_separate_k" = "1" ] || [ "$if_separate_k" = "true" ]; } \ && [ -n "$out_wfc_norm" ] && [ -n "$out_wfc_re_im" ] && [ -n "$out_pchg" ]; then - if python3 "$CUBE_TOOL" check-spinor OUT.autotest; then - echo "pw_spinor_cube_identity 0" >> "$props_result_file" - else - echo "pw_spinor_cube_identity 1" >> "$props_result_file" - fi + run_and_record_status "$props_result_file" "pw_spinor_cube_identity" \ + python3 "$CUBE_TOOL" check-spinor OUT.autotest fi } diff --git a/tests/integrate/validation_tools/props_deepks.sh b/tests/integrate/validation_tools/props_deepks.sh index 918eee161a..2f8b062d53 100644 --- a/tests/integrate/validation_tools/props_deepks.sh +++ b/tests/integrate/validation_tools/props_deepks.sh @@ -177,15 +177,11 @@ if ! test -z "$deepks_out_labels" && [ $deepks_out_labels == 1 ]; then fi # For deepks_v_delta < 0 - # CompareFile.py exits 1 when the files differ; capture the status in a - # named variable so it is recorded even when sourced under `bash -e`. if ! test -z "$deepks_v_delta" && [ $deepks_v_delta -lt 0 ]; then - hr_pass=0 - python3 $COMPARE_SCRIPT "deepks_hrtot.csr.ref" "OUT.autotest/deepks_hrtot.csr" 8 || hr_pass=$? - echo "deepks_hr_label_pass $hr_pass" >>$props_result_file - hrdelta_pass=0 - python3 $COMPARE_SCRIPT "deepks_hrdelta.csr.ref" "OUT.autotest/deepks_hrdelta.csr" 8 || hrdelta_pass=$? - echo "deepks_vdelta_r_pass $hrdelta_pass" >>$props_result_file + record_compare_result "$props_result_file" "deepks_hr_label_pass" \ + "deepks_hrtot.csr.ref" "OUT.autotest/deepks_hrtot.csr" 8 + record_compare_result "$props_result_file" "deepks_vdelta_r_pass" \ + "deepks_hrdelta.csr.ref" "OUT.autotest/deepks_hrdelta.csr" 8 # For deepks_v_delta = -1 if [ $deepks_v_delta -eq -1 ]; then process_npy "single" "abs" "vdrpre" "" "deepks_vdrp" "$props_result_file" diff --git a/tests/integrate/validation_tools/props_mat.sh b/tests/integrate/validation_tools/props_mat.sh index 3123a8d4fc..20aa1a48db 100644 --- a/tests/integrate/validation_tools/props_mat.sh +++ b/tests/integrate/validation_tools/props_mat.sh @@ -29,12 +29,12 @@ compare_hr_csr(){ echo "Error: can not find file $ref or $cal" return 1 fi - python3 $COMPARE_SCRIPT <(sed "$strip" "$ref") <(sed "$strip" "$cal") 8 || return 1 + python3 "$COMPARE_SCRIPT" <(sed "$strip" "$ref") <(sed "$strip" "$cal") 8 || return 1 if [ "$(sed -n "$pick" "$ref" | wc -l)" -ne "$(sed -n "$pick" "$cal" | wc -l)" ]; then echo "Error: E_Fermi annotation count differs between $ref and $cal" return 1 fi - python3 $COMPARE_SCRIPT <(sed -n "$pick" "$ref") <(sed -n "$pick" "$cal") 3 + python3 "$COMPARE_SCRIPT" <(sed -n "$pick" "$ref") <(sed -n "$pick" "$cal") 3 } run_mat_dm1_props(){ @@ -45,8 +45,7 @@ run_mat_dm1_props(){ if ! test -z "$out_dm1" && [ $out_dm1 == 1 ]; then dm1ref=dmr_nao.csr.ref dm1cal=OUT.autotest/dmr_nao.csr - python3 $COMPARE_SCRIPT $dm1ref $dm1cal 8 - echo "CompareDM1_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareDM1_pass" "$dm1ref" "$dm1cal" 8 fi } @@ -60,8 +59,7 @@ run_mat_props(){ if [ "$calculation" == "get_s" ]; then sref=sr_nao.csr.ref scal=OUT.autotest/sr_nao.csr - python3 $COMPARE_SCRIPT $sref $scal 8 - echo "CompareS_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareS_pass" "$sref" "$scal" 8 fi #------------------------------- @@ -71,8 +69,7 @@ fi if ! test -z "$out_pband" && [ $out_pband == 1 ]; then orbref=refOrbital orbcal=OUT.autotest/Orbital - python3 $COMPARE_SCRIPT $orbref $orbcal 8 - echo "CompareOrb_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareOrb_pass" "$orbref" "$orbcal" 8 fi #------------------------------- @@ -86,14 +83,11 @@ if ! test -z "$toW90" && [ $toW90 == 1 ]; then mmncal=OUT.autotest/diamond.mmn eigref=diamond.eig eigcal=OUT.autotest/diamond.eig - sed -i '1d' $amncal - sed -i '1d' $mmncal - python3 $COMPARE_SCRIPT $amnref $amncal 1 -abs 8 - echo "CompareAMN_pass $?" >>$props_result_file - python3 $COMPARE_SCRIPT $mmnref $mmncal 1 -abs 8 - echo "CompareMMN_pass $?" >>$props_result_file - python3 $COMPARE_SCRIPT $eigref $eigcal 8 - echo "CompareEIG_pass $?" >>$props_result_file + sed -i '1d' "$amncal" + sed -i '1d' "$mmncal" + record_compare_result "$props_result_file" "CompareAMN_pass" "$amnref" "$amncal" 1 8 + record_compare_result "$props_result_file" "CompareMMN_pass" "$mmnref" "$mmncal" 1 8 + record_compare_result "$props_result_file" "CompareEIG_pass" "$eigref" "$eigcal" 8 fi #------------------------------- @@ -104,8 +98,7 @@ fi if ! test -z "$has_band" && [ $has_band == 1 ]; then bandref=band.txt.ref bandcal=OUT.autotest/band.txt - python3 $COMPARE_SCRIPT $bandref $bandcal 8 - echo "CompareBand_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareBand_pass" "$bandref" "$bandcal" 8 fi #-------------------------------- @@ -124,14 +117,11 @@ if ! test -z "$has_hs" && [ $has_hs == 1 ]; then sref=sk_nao.txt.ref scal=OUT.autotest/sk_nao.txt # Compare Hamiltonian matrix for spin 1 - python3 $COMPARE_SCRIPT $h1ref $h1cal 6 - echo "CompareH1_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareH1_pass" "$h1ref" "$h1cal" 6 # Compare Hamiltonian matrix for spin 2 - python3 $COMPARE_SCRIPT $h2ref $h2cal 6 - echo "CompareH2_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareH2_pass" "$h2ref" "$h2cal" 6 # Compare overlap matrix - python3 $COMPARE_SCRIPT $sref $scal 8 - echo "CompareS_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareS_pass" "$sref" "$scal" 8 elif ! test -z "$nspin" && [ $nspin == 4 ]; then # nspin=4 : do nothing, only matching condition without any operation true @@ -142,11 +132,9 @@ if ! test -z "$has_hs" && [ $has_hs == 1 ]; then sref=sk_nao.txt.ref scal=OUT.autotest/sk_nao.txt # Compare Hamiltonian matrix - python3 $COMPARE_SCRIPT $href $hcal 6 - echo "CompareH_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareH_pass" "$href" "$hcal" 6 # Compare overlap matrix - python3 $COMPARE_SCRIPT $sref $scal 8 - echo "CompareS_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareS_pass" "$sref" "$scal" 8 fi else # ========== Multiple k-points calculation ========== @@ -159,14 +147,11 @@ if ! test -z "$has_hs" && [ $has_hs == 1 ]; then sref=sk2_nao.txt.ref scal=OUT.autotest/sk2_nao.txt # Compare Hamiltonian matrix for spin 1 - python3 $COMPARE_SCRIPT $h1ref $h1cal 6 - echo "CompareH1_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareH1_pass" "$h1ref" "$h1cal" 6 # Compare Hamiltonian matrix for spin 2 - python3 $COMPARE_SCRIPT $h2ref $h2cal 6 - echo "CompareH2_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareH2_pass" "$h2ref" "$h2cal" 6 # Compare overlap matrix - python3 $COMPARE_SCRIPT $sref $scal 8 - echo "CompareS_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareS_pass" "$sref" "$scal" 8 elif ! test -z "$nspin" && [ $nspin == 4 ]; then # nspin=4 : do nothing, only matching condition without any operation true @@ -177,27 +162,25 @@ if ! test -z "$has_hs" && [ $has_hs == 1 ]; then sref=sk2_nao.txt.ref scal=OUT.autotest/sk2_nao.txt # Compare Hamiltonian matrix - python3 $COMPARE_SCRIPT $href $hcal 6 - echo "CompareH_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareH_pass" "$href" "$hcal" 6 # Compare overlap matrix - python3 $COMPARE_SCRIPT $sref $scal 8 - echo "CompareS_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareS_pass" "$sref" "$scal" 8 fi fi elif ! test -z "$has_hs" && [ $has_hs == 2 ]; then HSK_BINARY_COMPARE="$PROPS_TOOLS_DIR/compare_hsk_binary.py" if ! test -z "$gamma_only" && [ $gamma_only == 1 ]; then HSK_TEXT_REFERENCE_DIR="../scf_out_hk" - python3 $HSK_BINARY_COMPARE OUT.autotest/hk_nao.dat "$HSK_TEXT_REFERENCE_DIR/hk_nao.txt.ref" real 3 - echo "CompareH_pass $?" >>$props_result_file - python3 $HSK_BINARY_COMPARE OUT.autotest/sk_nao.dat "$HSK_TEXT_REFERENCE_DIR/sk_nao.txt.ref" real 3 - echo "CompareS_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "CompareH_pass" python3 "$HSK_BINARY_COMPARE" \ + OUT.autotest/hk_nao.dat "$HSK_TEXT_REFERENCE_DIR/hk_nao.txt.ref" real 3 + run_and_record_status "$props_result_file" "CompareS_pass" python3 "$HSK_BINARY_COMPARE" \ + OUT.autotest/sk_nao.dat "$HSK_TEXT_REFERENCE_DIR/sk_nao.txt.ref" real 3 else HSK_TEXT_REFERENCE_DIR="../scf_out_hsk" - python3 $HSK_BINARY_COMPARE OUT.autotest/hk2_nao.dat "$HSK_TEXT_REFERENCE_DIR/hk2_nao.txt.ref" complex 3 - echo "CompareH_pass $?" >>$props_result_file - python3 $HSK_BINARY_COMPARE OUT.autotest/sk2_nao.dat "$HSK_TEXT_REFERENCE_DIR/sk2_nao.txt.ref" complex 3 - echo "CompareS_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "CompareH_pass" python3 "$HSK_BINARY_COMPARE" \ + OUT.autotest/hk2_nao.dat "$HSK_TEXT_REFERENCE_DIR/hk2_nao.txt.ref" complex 3 + run_and_record_status "$props_result_file" "CompareS_pass" python3 "$HSK_BINARY_COMPARE" \ + OUT.autotest/sk2_nao.dat "$HSK_TEXT_REFERENCE_DIR/sk2_nao.txt.ref" complex 3 fi fi @@ -214,10 +197,8 @@ if ! test -z "$has_xc" && [ "$has_xc" == 1 ]; then fi oeref=vxc.ref oecal=OUT.autotest/vxc.txt - python3 $COMPARE_SCRIPT $xcref $xccal 4 - echo "CompareVXC_pass $?" >>$props_result_file - python3 $COMPARE_SCRIPT $oeref $oecal 5 - echo "CompareOrbXC_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareVXC_pass" "$xcref" "$xccal" 4 + record_compare_result "$props_result_file" "CompareOrbXC_pass" "$oeref" "$oecal" 5 fi #-------------------------------- @@ -226,8 +207,7 @@ fi if ! test -z "$has_xc2" && [ $has_xc2 == 1 ]; then xc2ref=Vxc_R_spin0.ref xc2cal=OUT.autotest/Vxc_R_spin0.csr - python3 $COMPARE_SCRIPT $xc2ref $xc2cal 8 - echo "CompareVXC_R_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareVXC_R_pass" "$xc2ref" "$xc2cal" 8 fi #-------------------------------- @@ -236,20 +216,16 @@ fi if ! test -z "$has_eband_separate" && [ $has_eband_separate == 1 ]; then ekref=kinetic_out.ref ekcal=OUT.autotest/kinetic_out.dat - python3 $COMPARE_SCRIPT $ekref $ekcal 4 - echo "CompareOrbKinetic_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareOrbKinetic_pass" "$ekref" "$ekcal" 4 vlref=vpp_local_out.ref vlcal=OUT.autotest/vpp_local_out.dat - python3 $COMPARE_SCRIPT $vlref $vlcal 4 - echo "CompareOrbVL_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareOrbVL_pass" "$vlref" "$vlcal" 4 vnlref=vpp_nonlocal_out.ref vnlcal=OUT.autotest/vpp_nonlocal_out.dat - python3 $COMPARE_SCRIPT $vnlref $vnlcal 4 - echo "CompareOrbVNL_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareOrbVNL_pass" "$vnlref" "$vnlcal" 4 vhref=vhartree_out.ref vhcal=OUT.autotest/vhartree_out.dat - python3 $COMPARE_SCRIPT $vhref $vhcal 4 - echo "CompareOrbVHartree_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareOrbVHartree_pass" "$vhref" "$vhcal" 4 fi #----------------------------------- @@ -257,44 +233,41 @@ fi #----------------------------------- #echo $has_hs2 if ! test -z "$has_hs2" && [ $has_hs2 == 1 ]; then - compare_hr_csr hrs1_nao.csr.ref OUT.autotest/hrs1_nao.csr - echo "CompareHR_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "CompareHR_pass" \ + compare_hr_csr hrs1_nao.csr.ref OUT.autotest/hrs1_nao.csr if ! test -z "$nspin" && [ "$nspin" -eq 2 ]; then - compare_hr_csr hrs2_nao.csr.ref OUT.autotest/hrs2_nao.csr - echo "CompareHR2_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "CompareHR2_pass" \ + compare_hr_csr hrs2_nao.csr.ref OUT.autotest/hrs2_nao.csr fi - python3 $COMPARE_SCRIPT sr_nao.csr.ref OUT.autotest/sr_nao.csr 8 - echo "CompareSR_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareSR_pass" \ + "sr_nao.csr.ref" "OUT.autotest/sr_nao.csr" 8 elif ! test -z "$has_hs2" && [ "$has_hs2" == 2 ]; then HSR_BINARY_COMPARE="$PROPS_TOOLS_DIR/compare_hsr_binary.py" - python3 $HSR_BINARY_COMPARE OUT.autotest/hrs1_nao.dat hrs1_nao.csr.ref real 4 - echo "CompareHR_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "CompareHR_pass" python3 "$HSR_BINARY_COMPARE" \ + OUT.autotest/hrs1_nao.dat hrs1_nao.csr.ref real 4 if ! test -z "$nspin" && [ "$nspin" -eq 2 ]; then - python3 $HSR_BINARY_COMPARE OUT.autotest/hrs2_nao.dat hrs2_nao.csr.ref real 4 - echo "CompareHR2_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "CompareHR2_pass" python3 "$HSR_BINARY_COMPARE" \ + OUT.autotest/hrs2_nao.dat hrs2_nao.csr.ref real 4 fi - python3 $HSR_BINARY_COMPARE OUT.autotest/sr_nao.dat sr_nao.csr.ref real 4 - echo "CompareSR_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "CompareSR_pass" python3 "$HSR_BINARY_COMPARE" \ + OUT.autotest/sr_nao.dat sr_nao.csr.ref real 4 fi #----------------------------------- # H(R), S(R), and DM(R) matrices in NPZ format #----------------------------------- if { ! test -z "$out_hsr" && [ "$out_hsr" == 3 ]; } || { ! test -z "$out_hsr_npz" && [ "$out_hsr_npz" == 1 ]; }; then - test -f OUT.autotest/sr_nao.npz - echo "OutputSRNPZ_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "OutputSRNPZ_pass" test -f OUT.autotest/sr_nao.npz fi if { ! test -z "$out_hr_npz" && [ "$out_hr_npz" == 1 ]; } \ || { ! test -z "$out_hsr" && [ "$out_hsr" == 3 ]; } \ || { ! test -z "$out_hsr_npz" && [ "$out_hsr_npz" == 1 ]; }; then - test -f OUT.autotest/hrs1_nao.npz - echo "OutputHRNPZ_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "OutputHRNPZ_pass" test -f OUT.autotest/hrs1_nao.npz fi if ! test -z "$out_dm_npz" && [ "$out_dm_npz" == 1 ]; then - test -f OUT.autotest/output_DM0.npz - echo "OutputDMNPZ_pass $?" >>$props_result_file + run_and_record_status "$props_result_file" "OutputDMNPZ_pass" test -f OUT.autotest/output_DM0.npz fi #----------------------------------- @@ -302,8 +275,8 @@ fi #----------------------------------- #echo $has_mat_r if ! test -z "$has_mat_r" && [ $has_mat_r == 1 ]; then - python3 $COMPARE_SCRIPT rr_nao.txt.ref OUT.autotest/rr_nao.txt 8 - echo "ComparerR_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "ComparerR_pass" \ + "rr_nao.txt.ref" "OUT.autotest/rr_nao.txt" 8 fi #----------------------------------- @@ -311,8 +284,8 @@ fi #----------------------------------- #echo $has_mat_t if ! test -z "$has_mat_t" && [ $has_mat_t == 1 ]; then - python3 $COMPARE_SCRIPT tr_nao.csr.ref OUT.autotest/tr_nao.csr 8 - echo "ComparerTR_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "ComparerTR_pass" \ + "tr_nao.csr.ref" "OUT.autotest/tr_nao.csr" 8 fi #----------------------------------- @@ -320,8 +293,8 @@ fi #----------------------------------- #echo $has_mat_syns if ! test -z "$has_mat_syns" && [ $has_mat_syns == 1 ]; then - python3 $COMPARE_SCRIPT syns_nao.csr.ref OUT.autotest/syns_nao.csr 8 - echo "CompareSYNS_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareSYNS_pass" \ + "syns_nao.csr.ref" "OUT.autotest/syns_nao.csr" 8 fi #----------------------------------- @@ -329,12 +302,12 @@ fi #----------------------------------- #echo $has_mat_dh if ! test -z "$has_mat_dh" && [ $has_mat_dh == 1 ] && [ $gamma_only != 1 ]; then - python3 $COMPARE_SCRIPT dhrxs1_nao.csr.ref OUT.autotest/dhrxs1_nao.csr 8 - echo "ComparerdHRx_pass $?" >>$props_result_file - python3 $COMPARE_SCRIPT dhrys1_nao.csr.ref OUT.autotest/dhrys1_nao.csr 8 - echo "ComparerdHRy_pass $?" >>$props_result_file - python3 $COMPARE_SCRIPT dhrzs1_nao.csr.ref OUT.autotest/dhrzs1_nao.csr 8 - echo "ComparerdHRz_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "ComparerdHRx_pass" \ + "dhrxs1_nao.csr.ref" "OUT.autotest/dhrxs1_nao.csr" 8 + record_compare_result "$props_result_file" "ComparerdHRy_pass" \ + "dhrys1_nao.csr.ref" "OUT.autotest/dhrys1_nao.csr" 8 + record_compare_result "$props_result_file" "ComparerdHRz_pass" \ + "dhrzs1_nao.csr.ref" "OUT.autotest/dhrzs1_nao.csr" 8 fi #----------------------------------- @@ -361,8 +334,7 @@ run_mat_dm_props(){ if ! test -z "$out_dm" && [ $out_dm == 1 ]; then dmfile=OUT.autotest/dm_nao.txt dmref=dm_nao.txt.ref - python3 $COMPARE_SCRIPT $dmref $dmfile 5 - echo "DM_different $?" >>$props_result_file + record_compare_result "$props_result_file" "DM_different" "$dmref" "$dmfile" 5 fi } diff --git a/tests/integrate/validation_tools/props_tddft.sh b/tests/integrate/validation_tools/props_tddft.sh index 5d3f7d1eac..6070f02919 100644 --- a/tests/integrate/validation_tools/props_tddft.sh +++ b/tests/integrate/validation_tools/props_tddft.sh @@ -14,8 +14,8 @@ run_tddft_props(){ if ! test -z "$out_current" && [ $out_current ]; then current1ref=current_tot.txt.ref current1cal=OUT.autotest/current_tot.txt - python3 $COMPARE_SCRIPT $current1ref $current1cal 10 - echo "CompareCurrent_pass $?" >>$props_result_file + record_compare_result "$props_result_file" "CompareCurrent_pass" \ + "$current1ref" "$current1cal" 10 fi #-------------------------------------------- @@ -24,7 +24,7 @@ fi if ! test -z "$out_efield" && [ "$out_efield" == 1 ]; then efield_refs=(efield_*.txt.ref) if [ ! -e "${efield_refs[0]}" ]; then - echo "CompareEfieldReference_pass 1" >>$props_result_file + record_compare_status "$props_result_file" "CompareEfieldReference_pass" 1 else for efield_ref in "${efield_refs[@]}"; do efield_name=${efield_ref%.ref} diff --git a/tests/integrate/validation_tools/run_check.sh b/tests/integrate/validation_tools/run_check.sh index a1d390ab2c..ed2c960612 100755 --- a/tests/integrate/validation_tools/run_check.sh +++ b/tests/integrate/validation_tools/run_check.sh @@ -73,16 +73,19 @@ test -d OUT.autotest || exit 0 #---------------------------------------------------------------------- if test -z $1 then -$CATCH_SCRIPT result.out +bash -e "$CATCH_SCRIPT" result.out check_out result.out elif [ $1 == "debug" ] then # Single_job.sh debug copies validation_tools/ into the case directory; # use that copy so local edits to the collectors take effect. -./validation_tools/catch_properties.sh result.out +bash -e ./validation_tools/catch_properties.sh result.out check_out result.out else -$CATCH_SCRIPT result.ref +if ! bash -e "$CATCH_SCRIPT" result.ref; then + echo "Failed to generate result.ref." >&2 + exit 1 +fi rm -r OUT.autotest rm log.txt fi diff --git a/tests/integrate/validation_tools/test_catch_properties.sh b/tests/integrate/validation_tools/test_catch_properties.sh new file mode 100644 index 0000000000..7a5516dfdc --- /dev/null +++ b/tests/integrate/validation_tools/test_catch_properties.sh @@ -0,0 +1,380 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +integrate_dir=$(realpath "${script_dir}/..") +test_root=$(mktemp -d "${TMPDIR:-/tmp}/catch-properties-test.XXXXXX") +trap 'rm -rf "${test_root}"' EXIT + +case_dir="${test_root}/tests/01_PW/test_case" +mkdir -p "${case_dir}/OUT.autotest" +ln -s "${integrate_dir}" "${test_root}/tests/integrate" + +cat > "${case_dir}/INPUT" <<'EOF' +INPUT_PARAMETERS +calculation get_s +out_band 1 +EOF + +cat > "${case_dir}/OUT.autotest/running_get_s.log" <<'EOF' +Driver atomic_world 0.01 +EOF + +touch "${case_dir}/sr_nao.csr.ref" +touch "${case_dir}/OUT.autotest/sr_nao.csr" + +( + cd "${case_dir}" + bash -e ../../integrate/validation_tools/catch_properties.sh result.out +) + +grep -qx 'CompareBand_pass 1' "${case_dir}/result.out" + +if ( + cd "${case_dir}" + bash ../../integrate/validation_tools/catch_properties.sh result.ref +); then + echo "catch_properties.sh accepted a failed comparison while generating result.ref" >&2 + exit 1 +fi + +if [ -e "${case_dir}/result.ref" ]; then + echo "catch_properties.sh left a partial result.ref after comparison failure" >&2 + exit 1 +fi + +npz_case_dir="${test_root}/tests/01_PW/npz_case" +mkdir -p "${npz_case_dir}/OUT.autotest" + +cat > "${npz_case_dir}/INPUT" <<'EOF' +INPUT_PARAMETERS +calculation get_s +out_hsr_npz 1 +EOF + +cat > "${npz_case_dir}/OUT.autotest/running_get_s.log" <<'EOF' +Driver atomic_world 0.01 +EOF + +touch "${npz_case_dir}/sr_nao.csr.ref" +touch "${npz_case_dir}/OUT.autotest/sr_nao.csr" + +( + cd "${npz_case_dir}" + bash -e ../../integrate/validation_tools/catch_properties.sh result.out +) + +grep -qx 'OutputSRNPZ_pass 1' "${npz_case_dir}/result.out" +grep -qx 'OutputHRNPZ_pass 1' "${npz_case_dir}/result.out" + +if ( + cd "${npz_case_dir}" + bash ../../integrate/validation_tools/catch_properties.sh result.ref +); then + echo "catch_properties.sh accepted a missing NPZ file while generating result.ref" >&2 + exit 1 +fi + +if [ -e "${npz_case_dir}/result.ref" ]; then + echo "catch_properties.sh left a partial NPZ result.ref after generation failure" >&2 + exit 1 +fi + +hsk_case_dir="${test_root}/tests/01_PW/hsk_case" +mkdir -p "${hsk_case_dir}/OUT.autotest" + +cat > "${hsk_case_dir}/INPUT" <<'EOF' +INPUT_PARAMETERS +calculation get_s +out_hsk 2 +gamma_only 1 +EOF + +cat > "${hsk_case_dir}/OUT.autotest/running_get_s.log" <<'EOF' +Driver atomic_world 0.01 +EOF + +touch "${hsk_case_dir}/sr_nao.csr.ref" +touch "${hsk_case_dir}/OUT.autotest/sr_nao.csr" + +( + cd "${hsk_case_dir}" + bash -e ../../integrate/validation_tools/catch_properties.sh result.out +) + +grep -qx 'CompareH_pass 1' "${hsk_case_dir}/result.out" +grep -qx 'CompareS_pass 1' "${hsk_case_dir}/result.out" + +if ( + cd "${hsk_case_dir}" + bash ../../integrate/validation_tools/catch_properties.sh result.ref +); then + echo "catch_properties.sh accepted a failed HSK comparison while generating result.ref" >&2 + exit 1 +fi + +if [ -e "${hsk_case_dir}/result.ref" ]; then + echo "catch_properties.sh left a partial HSK result.ref after generation failure" >&2 + exit 1 +fi + +# Keep upstream's renamed VXC files while retaining failure propagation in +# both Gamma-only and multi-k collectors. No legacy vxc_out.* files exist. +for gamma in 0 1; do + vxc_case_dir="${test_root}/tests/01_PW/vxc_case_${gamma}" + mkdir -p "${vxc_case_dir}/OUT.autotest" + printf 'INPUT_PARAMETERS\ncalculation get_s\nout_mat_xc 1\ngamma_only %s\n' \ + "${gamma}" > "${vxc_case_dir}/INPUT" + printf 'Driver atomic_world 0.01\n' > "${vxc_case_dir}/OUT.autotest/running_get_s.log" + touch "${vxc_case_dir}/sr_nao.csr.ref" "${vxc_case_dir}/OUT.autotest/sr_nao.csr" + if [ "${gamma}" -eq 1 ]; then + vxc_name=vxc_nao.txt + else + vxc_name=vxck2_nao.txt + fi + printf 'value 1\n' > "${vxc_case_dir}/${vxc_name}.ref" + printf 'value 1\n' > "${vxc_case_dir}/OUT.autotest/${vxc_name}" + printf 'value 1\n' > "${vxc_case_dir}/vxc.ref" + printf 'value 1\n' > "${vxc_case_dir}/OUT.autotest/vxc.txt" + ( + cd "${vxc_case_dir}" + bash -e ../../integrate/validation_tools/catch_properties.sh result.ref + ) + grep -qx 'CompareVXC_pass 0' "${vxc_case_dir}/result.ref" + grep -qx 'CompareOrbXC_pass 0' "${vxc_case_dir}/result.ref" + + for failure in different missing; do + if [ "${failure}" = different ]; then + printf 'value 2\n' > "${vxc_case_dir}/OUT.autotest/vxc.txt" + else + rm "${vxc_case_dir}/OUT.autotest/vxc.txt" + fi + ( + cd "${vxc_case_dir}" + bash -e ../../integrate/validation_tools/catch_properties.sh result.out + ) + grep -qx 'CompareVXC_pass 0' "${vxc_case_dir}/result.out" + grep -qx 'CompareOrbXC_pass 1' "${vxc_case_dir}/result.out" + if ( + cd "${vxc_case_dir}" + bash -e ../../integrate/validation_tools/catch_properties.sh result.ref + ); then + echo "catch_properties.sh accepted ${failure} VXC output in result.ref" >&2 + exit 1 + fi + if [ -e "${vxc_case_dir}/result.ref" ]; then + echo "catch_properties.sh left a partial VXC result.ref" >&2 + exit 1 + fi + done +done + +deepks_case_dir="${test_root}/tests/01_PW/deepks_case" +mkdir -p "${deepks_case_dir}/OUT.autotest" + +cat > "${deepks_case_dir}/INPUT" <<'EOF' +INPUT_PARAMETERS +calculation get_s +deepks_out_labels 1 +deepks_v_delta -1 +EOF + +cat > "${deepks_case_dir}/OUT.autotest/running_get_s.log" <<'EOF' +Driver atomic_world 0.01 +EOF + +touch "${deepks_case_dir}/sr_nao.csr.ref" +touch "${deepks_case_dir}/OUT.autotest/sr_nao.csr" +printf 'value 1\n' > "${deepks_case_dir}/deepks_hrtot.csr.ref" +printf 'value 2\n' > "${deepks_case_dir}/OUT.autotest/deepks_hrtot.csr" +printf 'value 1\n' > "${deepks_case_dir}/deepks_hrdelta.csr.ref" +printf 'value 1\n' > "${deepks_case_dir}/OUT.autotest/deepks_hrdelta.csr" + +( + cd "${deepks_case_dir}" + bash -e ../../integrate/validation_tools/catch_properties.sh result.out +) + +grep -qx 'deepks_hr_label_pass 1' "${deepks_case_dir}/result.out" +grep -qx 'deepks_vdelta_r_pass 0' "${deepks_case_dir}/result.out" + +if ( + cd "${deepks_case_dir}" + bash ../../integrate/validation_tools/catch_properties.sh result.ref +); then + echo "catch_properties.sh accepted a failed DeePKS comparison while generating result.ref" >&2 + exit 1 +fi + +if [ -e "${deepks_case_dir}/result.ref" ]; then + echo "catch_properties.sh left a partial DeePKS result.ref after generation failure" >&2 + exit 1 +fi + +deepks_missing_case_dir="${test_root}/tests/01_PW/deepks_missing_case" +mkdir -p "${deepks_missing_case_dir}/OUT.autotest" + +cat > "${deepks_missing_case_dir}/INPUT" <<'EOF' +INPUT_PARAMETERS +calculation get_s +deepks_scf 1 +EOF + +cat > "${deepks_missing_case_dir}/OUT.autotest/running_get_s.log" <<'EOF' +Driver atomic_world 0.01 +EOF + +touch "${deepks_missing_case_dir}/sr_nao.csr.ref" +touch "${deepks_missing_case_dir}/OUT.autotest/sr_nao.csr" + +( + cd "${deepks_missing_case_dir}" + bash -e ../../integrate/validation_tools/catch_properties.sh result.out +) + +grep -qx 'deepks_desc 0.0' "${deepks_missing_case_dir}/result.out" + +if ( + cd "${deepks_missing_case_dir}" + bash ../../integrate/validation_tools/catch_properties.sh result.ref +); then + echo "catch_properties.sh accepted a missing DeePKS descriptor while generating result.ref" >&2 + exit 1 +fi + +if [ -e "${deepks_missing_case_dir}/result.ref" ]; then + echo "catch_properties.sh left a partial result.ref after a missing DeePKS descriptor" >&2 + exit 1 +fi + +ml_case_dir="${test_root}/tests/01_PW/ml_case" +mkdir -p "${ml_case_dir}/OUT.autotest/MLKEDF_Descriptors" + +cat > "${ml_case_dir}/INPUT" <<'EOF' +INPUT_PARAMETERS +calculation get_s +EOF + +cat > "${ml_case_dir}/OUT.autotest/running_get_s.log" <<'EOF' +Driver atomic_world 0.01 +EOF + +touch "${ml_case_dir}/sr_nao.csr.ref" +touch "${ml_case_dir}/OUT.autotest/sr_nao.csr" +printf 'not a NumPy file\n' > "${ml_case_dir}/OUT.autotest/MLKEDF_Descriptors/broken.npy" + +if ( + cd "${ml_case_dir}" + bash ../../integrate/validation_tools/catch_properties.sh result.ref +); then + echo "catch_properties.sh accepted a failed ML descriptor collector" >&2 + exit 1 +fi + +if [ -e "${ml_case_dir}/result.ref" ]; then + echo "catch_properties.sh left a partial result.ref after an ML collector failure" >&2 + exit 1 +fi + +cat > "${test_root}/fake_abacus" <<'EOF' +#!/usr/bin/env bash +mkdir -p OUT.autotest +cat > OUT.autotest/running_get_s.log <<'INNER_EOF' +Driver atomic_world 0.01 +INNER_EOF +cat > OUT.autotest/running_scf.log <<'INNER_EOF' +Driver atomic_world 0.01 +INNER_EOF +cat > OUT.autotest/running_get_wf.log <<'INNER_EOF' +Driver atomic_world 0.01 +INNER_EOF +touch OUT.autotest/sr_nao.csr +EOF +chmod +x "${test_root}/fake_abacus" + +mkdir -p "${test_root}/fake_bin" +cat > "${test_root}/fake_bin/mpirun" <<'EOF' +#!/usr/bin/env bash +if [ "$1" = "-np" ]; then + shift 2 +fi +exec "$@" +EOF +chmod +x "${test_root}/fake_bin/mpirun" + +run_check_log="${test_root}/run-check.log" +if ( + cd "${deepks_missing_case_dir}" + PATH="${test_root}/fake_bin:${PATH}" \ + ABACUS_EXE="${test_root}/fake_abacus" \ + bash ../../integrate/validation_tools/run_check.sh generate +) >"${run_check_log}" 2>&1; then + cat "${run_check_log}" >&2 + echo "run_check.sh reported success after reference generation failed" >&2 + exit 1 +fi + +if [ -e "${deepks_missing_case_dir}/result.ref" ]; then + echo "run_check.sh left a partial result.ref after generation failure" >&2 + exit 1 +fi + +echo test_case > "${test_root}/tests/01_PW/CASES.txt" +autotest_log="${test_root}/autotest.log" +if ( + cd "${test_root}/tests/01_PW" + OMP_NUM_THREADS=1 bash ../integrate/Autotest.sh \ + -a "${test_root}/fake_abacus" -n 0 -o 1 -j 1 -f CASES.txt -g +) >"${autotest_log}" 2>&1; then + cat "${autotest_log}" >&2 + echo "Autotest.sh reported success after reference generation failed" >&2 + exit 1 +fi + +if [ -e "${case_dir}/result.ref" ]; then + echo "Autotest.sh left a partial result.ref after generation failure" >&2 + exit 1 +fi + +# Upstream's dedicated-validator path must still honor a custom MPI launcher, +# its flags, the OFML single-rank exception, and commented case-list entries. +launcher_case_dir="${test_root}/tests/01_PW/launcher_case" +mkdir -p "${launcher_case_dir}" +printf 'INPUT_PARAMETERS\ncalculation get_s\nout_band 1\n' > "${launcher_case_dir}/INPUT" +printf '\n # absent_case\nlauncher_case\n' > "${test_root}/tests/01_PW/LAUNCHER_CASES.txt" +cat > "${test_root}/fake_bin/custom_mpi" <<'EOF' +#!/usr/bin/env bash +set -eu +[ "$#" -eq 6 ] +[ "$1" = "--ranks" ] +[ "$2" = "$EXPECTED_RANKS" ] +[ "$3" = "--test-pre-one" ] +[ "$4" = "--test-pre-two" ] +[ "$6" = "--test-post" ] +touch "$LAUNCHER_MARKER" +exec "$5" +EOF +chmod +x "${test_root}/fake_bin/custom_mpi" +for expected_ranks in 4 1; do + if [ "${expected_ranks}" -eq 1 ]; then + printf 'of_ml_gene_data 1\n' >> "${launcher_case_dir}/INPUT" + fi + launcher_log="${test_root}/launcher-${expected_ranks}.log" + ( + cd "${test_root}/tests/01_PW" + EXPECTED_RANKS="${expected_ranks}" \ + LAUNCHER_MARKER="${test_root}/launcher-${expected_ranks}.used" \ + ABACUS_MPIEXEC="${test_root}/fake_bin/custom_mpi" \ + ABACUS_MPIEXEC_NUMPROC_FLAG=--ranks \ + ABACUS_MPIEXEC_PREFLAGS='--test-pre-one --test-pre-two' \ + ABACUS_MPIEXEC_POSTFLAGS=--test-post \ + OMP_NUM_THREADS=1 bash ../integrate/Autotest.sh \ + -a "${test_root}/fake_abacus" -n 4 -o 1 -j 1 -f LAUNCHER_CASES.txt -p + ) > "${launcher_log}" 2>&1 + test -f "${test_root}/launcher-${expected_ranks}.used" + grep -q 'Property collectors skipped' "${launcher_log}" + if [ -e "${launcher_case_dir}/result.out" ] || [ -e "${launcher_case_dir}/result.ref" ]; then + echo "Autotest.sh ran property collectors despite -p" >&2 + exit 1 + fi +done diff --git a/tests/integrate/validation_tools/test_compare_file.py b/tests/integrate/validation_tools/test_compare_file.py new file mode 100644 index 0000000000..8b2d33969c --- /dev/null +++ b/tests/integrate/validation_tools/test_compare_file.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +import pathlib +import subprocess +import sys +import tempfile +import unittest + + +COMPARE_FILE = pathlib.Path(__file__).with_name("CompareFile.py") + + +class CompareFileTest(unittest.TestCase): + def run_compare(self, first_content, second_content, extra_arguments=None): + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_path = pathlib.Path(temporary_directory) + first_path = temporary_path / "first.txt" + second_path = temporary_path / "second.txt" + first_path.write_text(first_content, encoding="utf-8") + second_path.write_text(second_content, encoding="utf-8") + command = [sys.executable, str(COMPARE_FILE), str(first_path), str(second_path)] + if extra_arguments: + command.extend(extra_arguments) + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + + def test_equal_files_with_comments_and_blank_lines_pass(self): + result = self.run_compare("# heading\n\nvalue 1.0\n", " # heading \n \nvalue 1.0\n") + + self.assertEqual(result.returncode, 0, result.stdout) + + def test_missing_data_line_fails_in_either_file(self): + for first_content, second_content in ( + ("value 1.0\nextra 2.0\n", "value 1.0\n"), + ("value 1.0\n", "value 1.0\nextra 2.0\n"), + ): + with self.subTest(first_content=first_content, second_content=second_content): + result = self.run_compare(first_content, second_content) + + self.assertEqual(result.returncode, 1) + self.assertIn("", result.stdout) + + def test_extra_comment_line_fails(self): + result = self.run_compare("value 1.0\n# extra\n", "value 1.0\n") + + self.assertEqual(result.returncode, 1) + + def test_trailing_blank_lines_are_ignored(self): + result = self.run_compare("value 1.0\n\n", "value 1.0\n") + + self.assertEqual(result.returncode, 0, result.stdout) + + def test_extra_interior_blank_line_fails(self): + result = self.run_compare("first 1.0\n\nsecond 2.0\n", "first 1.0\nsecond 2.0\n") + + self.assertEqual(result.returncode, 1) + + def test_final_newline_is_not_significant(self): + result = self.run_compare("value 1.0\n", "value 1.0") + + self.assertEqual(result.returncode, 0, result.stdout) + + def test_non_finite_numbers_fail(self): + comparisons = ( + ("value nan\n", "value nan\n", []), + ("value nan\n", "value 1.0\n", []), + ("value inf\n", "value inf\n", ["-com_type", "0"]), + ("value inf\n", "value -inf\n", ["-com_type", "0"]), + ("value (1.0,nan)\n", "value (1.0,nan)\n", []), + ) + for first_content, second_content, extra_arguments in comparisons: + with self.subTest(first_content=first_content, second_content=second_content): + result = self.run_compare(first_content, second_content, extra_arguments) + + self.assertEqual(result.returncode, 1, result.stdout) + + +if __name__ == "__main__": + unittest.main()