Fix invalid integration comparisons and reference-generation failures

tests/integrate/Autotest.sh

@@ -387,7 +387,11 @@ run_case()
                         check_out result.out $my_threshold $my_force_threshold $my_stress_threshold $my_fatal_threshold $my_descriptor_threshold
                     fi
                 else
-                    bash -e ../../integrate/validation_tools/catch_properties.sh result.ref
+                    if ! bash -e ../../integrate/validation_tools/catch_properties.sh result.ref; then
+                        echo -e "\e[0;31m [ERROR     ]  Failed to generate result.ref \e[0m"
+                        let fatal++
+                        fatal_case_list+=$dir'\n'
+                    fi
                 fi
             fi
 
@@ -530,5 +534,11 @@ else
     exit 1
 fi
 else
-echo "Generate test cases complete."
+if [ "$fatal" -eq 0 ] && [ "$failed" -eq 0 ]; then
+    echo "Generate test cases complete."
+else
+    echo -e "\e[0;31m[ERROR     ]\e[0m Failed to generate references for $[ $fatal + $failed ] case(s)."
+    printf "%b" "$fatal_case_list$failed_case_list"
+    exit 1
+fi
 fi

tests/integrate/CMakeLists.txt

@@ -1,4 +1,22 @@
 find_program(BASH bash)
+# Validation-script regressions do not impose a Python configure dependency
+# on builds that only use a dedicated output validator.
+find_package(Python3 QUIET COMPONENTS Interpreter)
+if(Python3_Interpreter_FOUND)
+    add_test(
+        NAME compare_file_test
+        COMMAND ${Python3_EXECUTABLE} -m unittest validation_tools/test_compare_file.py
+        WORKING_DIRECTORY ${ABACUS_TEST_DIR}/integrate
+    )
+
+    add_test(
+        NAME catch_properties_test
+        COMMAND ${BASH} validation_tools/test_catch_properties.sh
+        WORKING_DIRECTORY ${ABACUS_TEST_DIR}/integrate
+    )
+else()
+    message(STATUS "Python3 interpreter not found; skipping validation-script regression tests")
+endif()
 
 if(ENABLE_ASAN)
     add_test(

tests/integrate/validation_tools/CompareFile.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import math
 import sys
 
 usage = '''
@@ -46,6 +47,8 @@ def ReadParam():
     return file1,file2,epsilon,useAbsoluteVal,compareAbsoluteDiff
 
 def is_two_data_diff(data1,data2,epsilon,compareAbsoluteDiff):
+    if not math.isfinite(data1) or not math.isfinite(data2):
+        return True
     if compareAbsoluteDiff:
         if abs(data1-data2) > epsilon: return True
     else:
@@ -60,6 +63,11 @@ def ReadFile(file1,lines):
     try:
         with open(file1) as f1:
             for line in f1.readlines(): lines.append(line)
+        # A final blank line carries no numerical information and varies among
+        # otherwise equivalent output writers. Keep interior blank lines so a
+        # missing or shifted data row is still detected below.
+        while lines and not lines[-1].strip():
+            lines.pop()
     except OSError:
         print("Error: can not find file %s" % file1)
         sys.exit(1)
@@ -86,6 +94,16 @@ def IsComplex(x):
         return False
     return [realp,imagp]
 
+def HasNonFiniteNumber(line):
+    for token in line.split():
+        complex_value = IsComplex(token)
+        if complex_value:
+            if not math.isfinite(complex_value[0]) or not math.isfinite(complex_value[1]):
+                return True
+        elif IsFloat(token) and not math.isfinite(float(token)):
+            return True
+    return False
+
 def ExitError(iline,line1,line2,jnumber=-1):
     if jnumber < 0:
         print('Compare Error: line %d\n %s:%s %s:%s'%(iline+1,file1,line1,file2,line2))
@@ -102,7 +120,9 @@ def ExitError(iline,line1,line2,jnumber=-1):
     ReadFile(file2,lines2)
 
     for i in range(min(len(lines1),len(lines2))):
-        if lines1[i].strip() == lines2[i].strip():continue
+        if lines1[i].strip() == lines2[i].strip():
+            if HasNonFiniteNumber(lines1[i]): ExitError(i,lines1[i],lines2[i])
+            continue
         elif '' in [lines1[i].strip(),lines2[i].strip()]: ExitError(i,lines1[i],lines2[i])
         elif len(lines1[i].split()) != len(lines2[i].split()): ExitError(i,lines1[i],lines2[i])
         else:
@@ -128,3 +148,9 @@ def ExitError(iline,line1,line2,jnumber=-1):
                         if is_two_data_diff(float(sline1[j]), float(sline2[j]), epsilon, compareAbsoluteDiff):
                             ExitError(i,sline1[j],sline2[j],j)
                 elif sline1[j] != sline2[j]: ExitError(i,sline1[j],sline2[j],j)
+
+    if len(lines1) != len(lines2):
+        i = min(len(lines1),len(lines2))
+        line1 = lines1[i] if i < len(lines1) else '<missing>'
+        line2 = lines2[i] if i < len(lines2) else '<missing>'
+        ExitError(i,line1,line2)

tests/integrate/validation_tools/catch_properties.sh

@@ -7,7 +7,24 @@
 # one-time INPUT switch parsing) and then runs each property block below.
 PROPS_SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 source "$PROPS_SCRIPT_DIR/props_common.sh"
-props_init "$1"
+props_result_file=$1
+props_reference_mode=0
+case "$props_result_file" in
+    result.ref|*/result.ref)
+        props_reference_mode=1
+        cleanup_failed_reference(){
+            reference_status=$?
+            trap - EXIT
+            if [ "$reference_status" -ne 0 ]; then
+                rm -f "$props_result_file"
+            fi
+            exit "$reference_status"
+        }
+        trap cleanup_failed_reference EXIT
+        set -e
+        ;;
+esac
+props_init "$props_result_file"
 source "$PROPS_SCRIPT_DIR/props_basic.sh"
 source "$PROPS_SCRIPT_DIR/props_mat.sh"
 source "$PROPS_SCRIPT_DIR/props_cube.sh"
@@ -72,8 +89,14 @@ run_ml_rpa_props
 # Before the split this block ran as a separate `bash` process without -e,
 # so a failing helper (get_sum_*.py, a missing deepks_desc.dat) left an
 # empty value for the threshold check instead of aborting the collection.
-# Keep that behavior with a subshell that disables errexit.
-( set +e; run_deepks_props )
+# Keep that behavior for result.out. Reference generation instead inherits the
+# entry point's errexit mode, so an early failure cannot be hidden by a later
+# successful command in the collector.
+if [ "$props_reference_mode" -eq 1 ]; then
+    run_deepks_props
+else
+    ( set +e; run_deepks_props ) || true
+fi
 
 #--------------------------------------------
 # basic collectors that run after deepks: symmetry

tests/integrate/validation_tools/props_basic.sh

@@ -82,8 +82,7 @@ fi
 if ! test -z "$has_cond"  && [  $has_cond == 1 ]; then
 	onref=refOnsager.txt
 	oncal=OUT.autotest/Onsager.txt
-	python3 $COMPARE_SCRIPT $onref $oncal 3 -com_type 0
-    echo "CompareH_Failed $?" >>$props_result_file
+	record_compare_result "$props_result_file" "CompareH_Failed" "$onref" "$oncal" 3 0 0
 	rm -f je-je.txt Chebycoef
 fi
 

tests/integrate/validation_tools/props_common.sh

@@ -59,21 +59,69 @@ sanitize_result_key(){
 }
 
 
+record_compare_status(){
+	result_file=$1
+	result_key=$2
+	compare_status=$3
+
+	if [ "$compare_status" -ne 0 ]; then
+		case "$result_file" in
+			result.ref|*/result.ref)
+				echo "Error: refusing to record failed comparison $result_key in $result_file" >&2
+				rm -f "$result_file"
+				exit "$compare_status"
+				;;
+		esac
+	fi
+	echo "$result_key $compare_status" >> "$result_file"
+}
+
+
+run_and_record_status(){
+	result_file=$1
+	result_key=$2
+	shift 2
+
+	# A failed check remains useful data in result.out, but it must never be
+	# accepted as the expected value while generating result.ref.
+	if "$@"; then
+		command_status=0
+	else
+		command_status=$?
+	fi
+	record_compare_status "$result_file" "$result_key" "$command_status"
+}
+
+
 record_compare_result(){
 	result_file=$1
 	result_key=$2
 	ref_file=$3
 	cal_file=$4
 	accuracy=${5:-8}
 	use_abs=${6:-0}
+	compare_type=${7:-}
 
 	if [ ! -f "$ref_file" ] || [ ! -f "$cal_file" ]; then
-		echo "$result_key 1" >> "$result_file"
-		return
+		record_compare_status "$result_file" "$result_key" 1
+		return $?
 	fi
 
-	python3 $COMPARE_SCRIPT "$ref_file" "$cal_file" "$accuracy" -abs "$use_abs"
-	echo "$result_key $?" >> "$result_file"
+	if [ -n "$compare_type" ]; then
+		if python3 "$COMPARE_SCRIPT" "$ref_file" "$cal_file" "$accuracy" \
+			-abs "$use_abs" -com_type "$compare_type"; then
+			compare_status=0
+		else
+			compare_status=$?
+		fi
+	else
+		if python3 "$COMPARE_SCRIPT" "$ref_file" "$cal_file" "$accuracy" -abs "$use_abs"; then
+			compare_status=0
+		else
+			compare_status=$?
+		fi
+	fi
+	record_compare_status "$result_file" "$result_key" "$compare_status"
 }
 
 

tests/integrate/validation_tools/props_cube.sh

@@ -22,8 +22,7 @@ run_cube_pot_props(){
 if ! test -z "$out_pot"  && [  $out_pot == 1 ]; then
 	pot1ref=pot.cube.ref
 	pot1cal=OUT.autotest/pot.cube
-	python3 $COMPARE_SCRIPT $pot1ref $pot1cal 3
-	echo "ComparePot1_pass $?" >>$props_result_file
+	record_compare_result "$props_result_file" "ComparePot1_pass" "$pot1ref" "$pot1cal" 3
 fi
 
 #-------------------------------
@@ -32,8 +31,7 @@ fi
 if ! test -z "$out_pot"  && [  $out_pot == 2 ]; then
 	pot1ref=potes.cube.ref
 	pot1cal=OUT.autotest/potes.cube
-	python3 $COMPARE_SCRIPT $pot1ref $pot1cal 8
-	echo "ComparePot_pass $?" >>$props_result_file
+	record_compare_result "$props_result_file" "ComparePot_pass" "$pot1ref" "$pot1cal" 8
 fi
 
 #-------------------------------
@@ -43,8 +41,7 @@ fi
 if ! test -z "$out_elf"  && [  $out_elf == 1 ]; then
 	elf1ref=elftot.cube.ref
 	elf1cal=OUT.autotest/elftot.cube
-	python3 $COMPARE_SCRIPT $elf1ref $elf1cal 3
-	echo "ComparePot1_pass $?" >>$props_result_file
+	record_compare_result "$props_result_file" "ComparePot1_pass" "$elf1ref" "$elf1cal" 3
 fi
 
 }
@@ -66,8 +63,8 @@ fi
 #---------------------------------------
 if ! test -z "$has_scan"  && [  $has_scan == "scan" ] && \
        ! test -z "$out_chg" && [ $out_chg == 1 ]; then
-    python3 $COMPARE_SCRIPT tau.cube.ref OUT.autotest/tau.cube 8
-    echo "tau.cube_pass $?" >>$props_result_file
+    record_compare_result "$props_result_file" "tau.cube_pass" \
+        "tau.cube.ref" "OUT.autotest/tau.cube" 8
 fi
 
 #---------------------------------------
@@ -76,8 +73,8 @@ fi
 #---------------------------------------
 if ! test -z "$has_ldos"  && [  $has_ldos == 1 ]; then
     stm_bias=$(get_input_key_value "stm_bias" "OUT.autotest/INPUT.info")
-    python3 $COMPARE_SCRIPT ldos.cube.ref OUT.autotest/ldos_"$stm_bias"ev.cube 8
-    echo "ldos.cube_pass $?" >> $props_result_file
+    record_compare_result "$props_result_file" "ldos.cube_pass" \
+        "ldos.cube.ref" "OUT.autotest/ldos_${stm_bias}ev.cube" 8
 fi
 
 #---------------------------------------
@@ -159,7 +156,7 @@ if ! test -z "$has_lowf" && [ "$has_lowf" == 1 ]; then
 			wfc_name=wfk2_nao
 			input_file=wfk2_nao
 		fi
-	awk 'BEGIN {flag=999}
+		awk 'BEGIN {flag=999}
 	{
 		if($2 == "(band)") {flag=2;print $0}
 		else if(flag>0) {flag-=1;print $0}
@@ -175,8 +172,8 @@ if ! test -z "$has_lowf" && [ "$has_lowf" == 1 ]; then
 		wfc_ref="$wfc_name"_mod.txt.ref
 	fi
 
-	python3 $COMPARE_SCRIPT $wfc_cal $wfc_ref 8 -abs 1
-	echo "Compare_wfc_lcao_pass $?" >>$props_result_file
+	record_compare_result "$props_result_file" "Compare_wfc_lcao_pass" \
+		"$wfc_cal" "$wfc_ref" 8 1
 fi
 
 }
@@ -187,8 +184,8 @@ run_cube_tail_props(){
 # mulliken charge
 #--------------------------------------------
 if ! test -z "$out_mul"  && [ $out_mul == 1 ]; then
-    python3 $COMPARE_SCRIPT mulliken.txt.ref OUT.autotest/mulliken.txt 3
-	echo "Compare_mulliken_pass $?" >>$props_result_file
+    record_compare_result "$props_result_file" "Compare_mulliken_pass" \
+        "mulliken.txt.ref" "OUT.autotest/mulliken.txt" 3
 fi
 
 #--------------------------------------------
@@ -279,11 +276,8 @@ nspin=$(get_input_key_value "nspin" "INPUT")
 if_separate_k=$(get_input_key_value "if_separate_k" "INPUT")
 if [ "$nspin" = "4" ] && { [ "$if_separate_k" = "1" ] || [ "$if_separate_k" = "true" ]; } \
     && [ -n "$out_wfc_norm" ] && [ -n "$out_wfc_re_im" ] && [ -n "$out_pchg" ]; then
-    if python3 "$CUBE_TOOL" check-spinor OUT.autotest; then
-        echo "pw_spinor_cube_identity 0" >> "$props_result_file"
-    else
-        echo "pw_spinor_cube_identity 1" >> "$props_result_file"
-    fi
+    run_and_record_status "$props_result_file" "pw_spinor_cube_identity" \
+        python3 "$CUBE_TOOL" check-spinor OUT.autotest
 fi
 
 }

tests/integrate/validation_tools/props_deepks.sh

@@ -177,15 +177,11 @@ if ! test -z "$deepks_out_labels" && [ $deepks_out_labels == 1 ]; then
     fi
 
     # For deepks_v_delta < 0
-    # CompareFile.py exits 1 when the files differ; capture the status in a
-    # named variable so it is recorded even when sourced under `bash -e`.
     if ! test -z "$deepks_v_delta" && [ $deepks_v_delta -lt 0 ]; then
-        hr_pass=0
-        python3 $COMPARE_SCRIPT "deepks_hrtot.csr.ref" "OUT.autotest/deepks_hrtot.csr" 8 || hr_pass=$?
-        echo "deepks_hr_label_pass $hr_pass" >>$props_result_file
-        hrdelta_pass=0
-        python3 $COMPARE_SCRIPT "deepks_hrdelta.csr.ref" "OUT.autotest/deepks_hrdelta.csr" 8 || hrdelta_pass=$?
-        echo "deepks_vdelta_r_pass $hrdelta_pass" >>$props_result_file
+        record_compare_result "$props_result_file" "deepks_hr_label_pass" \
+            "deepks_hrtot.csr.ref" "OUT.autotest/deepks_hrtot.csr" 8
+        record_compare_result "$props_result_file" "deepks_vdelta_r_pass" \
+            "deepks_hrdelta.csr.ref" "OUT.autotest/deepks_hrdelta.csr" 8
         # For deepks_v_delta = -1
         if [ $deepks_v_delta -eq -1 ]; then
             process_npy "single" "abs" "vdrpre" "" "deepks_vdrp" "$props_result_file"