Enhanced a lot of testing routines

First release can be prepared.

report/get_files.py

@@ -11,7 +11,7 @@ for path, subdirs, files in os.walk(root):
     for name in files:
         if name == "intro.md":
             continue
-        if ".ipynb" in name or ".md" in name or ".rst" in name:
+        if ".md" in name or ".rst" in name:
             print(name)
             currentFile = str(pathlib.PurePath(path, name))
             toc.append(currentFile)

test.sh

@@ -33,6 +33,16 @@ if [ -z "$src" ] || [ -z "${main_dir}" ] || [ -z "${destination}" ] || [ "${#set
     exit
 fi
 
+if [ -z "${from_scratch}" ]; then
+    from_scratch=false
+fi
+
+if ${from_scratch}; then
+    if [ -d "${main_dir}" ]; then
+        echo "Remove existing ${main_dir}"
+        rm -r ${main_dir}
+    fi
+fi
 mkdir -p ${main_dir}
 
 if [ -d $src ]; then
@@ -59,11 +69,53 @@ fi
 echo "Register the configured destination"
 echo "${destination}" > DESTINATIONS
 
-for keyword in `awk '{if(NR==1){print $1}}' ./DESTINATIONS` ; do
+run_targets=()
+sync_targets=()
+
+for keyword in  `awk '{print $1}' ./DESTINATIONS` ; do
+    if [ "${keyword##*_}" == "sync" ]; then
+        sync_targets=(${sync_targets[@]} $keyword)
+        continue
+    fi
+
+    run_targets=(${run_targets[@]} $keyword)
+
+done
+
+echo "$setups" > SETUPS
+
+echo ""
+echo "##   Testing with following targets/setups   ##"
+echo "###############################################"
+echo ""
+
+for run_target in "${run_targets[@]}"; do
+    for setup in `awk '{print $1}' SETUPS`; do
+        if [ "${setup%%_*}" != "${run_target%%_*}" ]; then
+            continue
+        fi
+        sync_to=""
+        for sync_target in "${sync_targets[@]}"; do
+            if [[ "${sync_target}" =~ "${setup}" ]]; then
+                sync_to="| synchronize to: "${sync_target}
+                break
+            fi
+        done
+        echo "Run on:" $run_target $sync_to "| with setup: "$setup
+    done
+done
+
+
 
-    source ./local_scripts/identify_target.sh ${keyword}
+total_runs=0
+for run_target in "${run_targets[@]}"; do
 
-    echo ${target} ${dest} ${dest_folder} ${user_at_dest}
+    source ./local_scripts/identify_target.sh ${run_target}
+
+    if ${from_scratch}; then
+        echo "Remove ${dest} if existing..."
+        ssh -t ${user_at_dest} "if [ -d ${dest_folder} ]; then rm -r ${dest_folder}; fi"
+    fi
 
     echo "Register base directory for all setups."
     echo "${target}_base ${dest}/base" >> DESTINATIONS
@@ -77,6 +129,10 @@ for keyword in `awk '{if(NR==1){print $1}}' ./DESTINATIONS` ; do
     echo "Run all the given test setups..."
     for setup in `awk '{print $1}' SETUPS`; do
 
+        if [ "${setup%%_*}" != "${run_target%%_*}" ]; then
+            continue
+        fi
+
         echo "  Prepare work directory for setup $setup."
         ssh -t ${user_at_dest} "if [ -d ${dest_folder}/${setup} ]; then rm -r ${dest_folder}/${setup}; fi"
         ssh -t ${user_at_dest} "cp -as ${dest_folder}/base ${dest_folder}/${setup}"
@@ -96,24 +152,71 @@ for keyword in `awk '{if(NR==1){print $1}}' ./DESTINATIONS` ; do
         for d in "${dirs[@]}"; do 
             echo "   with input folder $d"
             ./run.sh ${target}_${setup} prepare-before-run $d
+            let total_runs=total_runs+1
         done
+
+        for sync_target in "${sync_targets[@]}"; do
+            if [[ ! "${sync_target}" =~ "${setup}" ]]; then
+                continue
+            fi
+
+            if ${from_scratch}; then
+                sync_dest="`awk -v dest="${sync_target}" '{if($1==dest){print $2}}' ./DESTINATIONS`"
+                echo "Remove ${sync_dest} if existing..."
+                user_at_sync_dest="${sync_dest%:*}"
+                sync_dest_folder="${sync_dest#*:}"
+                ssh -t ${user_at_sync_dest} "if [ -d ${sync_dest_folder} ]; then rm -r ${sync_dest_folder}; fi"
+            fi
+
+            ./sync.sh "${target}_${setup}" "${sync_target}"
+            break
+        done
+
+         
     done
 
 done
 
+echo "Waiting for test jobs to finish..."
+
 let finished=0
-while  [ ${finished} -lt `cat SETUPS | wc -l` ]; do
-    let finished=0
-    for setup in `awk '{print $1}' SETUPS`; do
-        marker=`ssh -t ${user_at_dest} "if [ -f ${dest_folder}/${setup}/*_finished.txt ]; then echo -n ${setup}; fi"`
-        if [ "$marker" == "$setup" ]; then
-            let finished++
-        fi
+while [ ${finished} -lt ${total_runs} ]; do
+
+    for run_target in "${run_targets[@]}"; do
+
+        source ./local_scripts/identify_target.sh ${run_target} > /dev/null 2>&1
+
+        for setup in `awk '{print $1}' SETUPS`; do
+
+            if [ "${setup%%_*}" != "${run_target%%_*}" ]; then
+                continue
+            fi
+
+            if [ ! -z ${setup_done} ]; then
+                continue
+            fi
+
+            dirs=(`ssh ${user_at_dest} "if [ -d ${dest_folder}/${setup}/input ]; then ls ${dest_folder}/${setup}/input; fi"`)
+            if [[ "${dirs[*]}" =~ "global_settings.py" ]]; then 
+                dirs=("")
+            fi
+            success=`ssh -t ${user_at_dest} "ls ${dest_folder}/${setup}/*_finished.txt 2> /dev/null | wc -l" | sed s.'\r'.''.g`
+            failed=`ssh -t ${user_at_dest} "ls ${dest_folder}/${setup}/*_failed.txt 2> /dev/null | wc -l" | sed s.'\r'.''.g`
+            let completed=0
+            let completed=completed+success+failed
+            if [[ "$completed" == "${#dirs[@]}" ]]; then 
+                let finished=finished+completed
+                eval ${setup}_done=true
+                echo "   Setup $setup done."
+                continue 3 # skip waiting in while loop since this might be the last setup
+            fi
+        done
     done
-    sleep 10
+    sleep 30
 done
+echo "... done."
 
-exit
-
+echo "Create test report..."
 cd -
 ./test_report.sh ${test_config}
+echo "... done."

test_config.example

@@ -1,3 +1,5 @@
+from_scratch=true
+
 # directory where the code will be stored (can be relative to the directory where the test.sh script is)
 main_dir="./IOW_ESM"
 
@@ -10,12 +12,15 @@ src=".."
 
 # user-specific settings
 
-# destination
-destination="hlrnb mvkkarst@blogin:/scratch/usr/mvkkarst/test_area"
-
 # setups that will be tested in format of the usual SETUPS file
-setups="parallel_example mvkkarst@blogin:/scratch/usr/mviowmod/IOW_ESM/setups/example/1.00.00
-single_example mvkkarst@blogin:/scratch/usr/mviowmod/IOW_ESM/setups/MOM5_Baltic-CCLM_Eurocordex/example_3nm_0.22deg/1.00.00"
+setups="hlrng_example mvkkarst@glogin:/scratch/usr/mviowmod/IOW_ESM/setups/example/1.00.00
+hlrnb_example mvkkarst@blogin:/scratch/usr/mviowmod/IOW_ESM/setups/example/1.00.00"
+
+# destination
+destination="hlrng mvkkarst@glogin:/scratch/usr/mvkkarst/test_area
+hlrnb mvkkarst@blogin:/scratch/usr/mvkkarst/test_area
+phy-10_hlrng_example_sync karsten@phy-10:/silod8/karsten/test_area/hlrng_sync
+phy-10_hlrnb_example_sync karsten@phy-10:/silod8/karsten/test_area/hlrnb_sync"
 
 
 module load miniconda3

test_report.py

@@ -1,7 +1,9 @@
 import sys
 
-test_dir = str(sys.argv[1])
-setups_str = str(sys.argv[2])
+target = str(sys.argv[1])
+dest = str(sys.argv[2])
+setups_str = str(sys.argv[3])
+
 
 setups=[]
 for setup in setups_str.split("\n"):
@@ -9,7 +11,7 @@ for setup in setups_str.split("\n"):
 
 from test_reporter import TestReporter
 
-test_reporter = TestReporter(test_dir, setups)
+test_reporter = TestReporter(target, dest, setups)
 
 test_reporter.check_build()
 test_reporter.check_output()

test_report.sh

@@ -9,12 +9,59 @@ fi
 source ${test_config}
 
 echo "Check for mandatory settings and apply defaults."
-if [ -z $src ] || [ -z ${main_dir} ] || [ -z $machine ] || [ -z $user_at_host ] || [ -z $test_dir ] || [ "${#setups[@]}" -eq 0 ]; then
+if [ -z "$src" ] || [ -z "${main_dir}" ] || [ -z "${destination}" ] || [ "${#setups[@]}" -eq 0 ]; then
     echo "Some mandatory variable is not correctly set in $test_config"
     exit
 fi
 
-python3 test_report.py ${test_dir} "$setups"
+for keyword in  `awk '{print $1}' ${main_dir}/DESTINATIONS` ; do
+    if [ "${keyword##*_}" == "sync" ]; then
+        sync_targets=(${sync_targets[@]} $keyword)
+        continue
+    fi
+
+    # skip automatically added destinations
+
+    # for different setups
+    for setup in `awk '{print $1}' ${main_dir}/SETUPS`; do
+        if [[ "${keyword}" =~ "${setup}" ]]; then
+            continue 2
+        fi
+    done
+
+    # for the base
+    if [[ "${keyword}" =~ "_base" ]]; then
+        continue 
+    fi
+
+    run_targets=(${run_targets[@]} $keyword)
+
+
+done
+
+cat <<EOF > report/intro.md
+# IOW ESM test report
+
+This is an IOW ESM test report.
+The test thas been performed in `${run_targets[@]}`.
+
+EOF
+
+for run_target in "${run_targets[@]}"; do
+
+    source ${main_dir}/local_scripts/identify_target.sh ${run_target}
+
+    # test if target is associated with any setup
+
+    #echo ${target} ${dest} ${dest_folder} ${user_at_dest}
+
+    python3 test_report.py ${run_target} ${dest} "$setups" 
+
+done
 
 cd report
 source ./build_report.sh
+
+for d in `ls -d  output_*/*/*/figures`; do
+    cp -r $d _build/html/$d
+done

test_reporter.py

@@ -1,23 +1,40 @@
 import glob
 import os
+import subprocess
+
+class Logger:
+    def __init__(self, directory, file_name, parent = None, overwrite = True):
+        self.directory = directory
+        self.file_name = file_name
+        self.parent = parent
+        if glob.glob(self.directory) and overwrite:
+            os.system("rm -r "+self.directory)
+        os.system("mkdir -p "+self.directory)
+        self.file_path = self.directory+'/'+self.file_name
+
+        if overwrite:
+            self.file = open(self.file_path, "w")
+        else:
+            self.file = open(self.file_path, "a")
+
+        if self.parent is not None:
+            self.parent.report_files.append(self.file_path)
+    
+    def log(self, text):
+        self.file.write(text)
+
+    def __del__(self):
+        self.file.close()
 
 class TestReporter:
 
-    def __init__(self, test_dir, setups, report_dir = "./report"):
+    def __init__(self, target, dest, setups, report_dir = "./report"):
 
-        self.test_dir = test_dir
+        self.target = target
+        self.dest = dest
         self.report_dir = report_dir
         self.setups = setups
 
-        self.intro_file_name = self.report_dir + "/intro.md"
-        with open(self.intro_file_name, "w") as file:
-            file.write("# IOW ESM test report\n\n")
-
-            file.write("This is an IOW ESM test report.\n")
-            file.write("The tes thas been performed in `"+test_dir+"` with the setups `"+str(setups)+"`.\n")
-            file.write("\n\n")
-
-
         self.binaries = {
             "components/CCLM" : ["cclm/bin_PRODUCTION/lmparbin"],
             "components/MOM5" : ["exec/IOW_ESM_PRODUCTION/MOM_SIS/fms_MOM_SIS.x"],
@@ -27,117 +44,139 @@ class TestReporter:
         }
 
         self.summary = {}
+        self.report_files = []
+
+        self.user_at_host, self.host_path = self.dest.split(":")
+
+        self.summary_logger = Logger(self.report_dir, "intro.md", self, overwrite=False)
+        self.summary_logger.log("\n\n## Test summary for "+self.target+"\n\n")
+
+        print("# Reporter for "+self.target+" started.")
+
+    def _glob_remote(self, pattern):
+        cmd = "ssh "+self.user_at_host+" \"ls "+pattern+" 2> /dev/null\""
+        sp = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+        output = sp.stdout.read().decode("utf-8").split("\n")
+        return [i for i in output if i != ""]      
 
     def check_build(self):
 
-        build_report_dir = self.report_dir+"/build"
-        if glob.glob(build_report_dir):
-            os.system("rm -r "+build_report_dir)
-        os.system("mkdir "+build_report_dir)
-
-        check_build_file_name = build_report_dir+"/build.md"
-        with open(check_build_file_name, "w") as file:
-            file.write("# IOW ESM build report\n\n\n")
-
-            for component in self.binaries.keys():
-                file.write("## Build report for "+component+"\n\n")
-
-                found_binaries = []
-                for binary in self.binaries[component]:
-                    found_binaries += glob.glob(self.test_dir+"/base/"+component+"/"+binary)
-                
-                if not found_binaries:
-                    file.write("**Could not find binaries "+str(self.binaries[component])+" for "+component+"**\n")
-                    self.summary["Build:"+component] = False
-                else:
-                    file.write("Binaries `"+str(found_binaries)+"` has been built for "+component+"\n")
-                    self.summary["Build:"+component] = True
+        print("Check build...")
+
+        logger = Logger(self.report_dir+"/build_"+self.target, "build.md", self)
+        logger.log("# IOW ESM build report\n\n\n")
+
+        for component in self.binaries.keys():
+            logger.log("## Build report for "+component+"\n\n")
+
+            found_binaries = []
 
-                file.write("\n\n")
+            for binary in self.binaries[component]:
+                found_binaries = self._glob_remote(self.host_path+"/base/"+component+"/"+binary)
+            
+            if not found_binaries:
+                logger.log("**Could not find binaries "+str(self.binaries[component])+" for "+component+"**\n")
+                self.summary["Build:"+component] = False
+            else:
+                logger.log("Binaries `"+str(found_binaries)+"` has been built for "+component+"\n")
+                self.summary["Build:"+component] = True
+
+            logger.log("\n\n")
+
+        print("...done.")
 
     def check_output(self):
-        output_report_dir = self.report_dir+"/output"
-        if glob.glob(output_report_dir):
-            os.system("rm -r "+output_report_dir)
-        os.system("mkdir "+output_report_dir)
 
+        print("Check output...")
+
+        logger = Logger(self.report_dir+"/output_"+self.target, "output.md", self)
+        logger.log("# IOW ESM output report\n\n\n")
 
-        check_output_file_name = output_report_dir+"/output.md"
-        with open(check_output_file_name, "w") as file:
-            file.write("# IOW ESM output report\n\n\n")
-            for setup in self.setups:
-                file.write("## Output report for "+setup+"\n\n")
+        for setup in self.setups:
 
-                found_output = glob.glob(self.test_dir+"/"+setup+"/output/*")
+            if self.target not in setup:
+                continue
+
+            print(" Check setup "+setup+"...")
+
+            input_folders = self._glob_remote("-d "+self.host_path+"/"+setup+"/input/*/ | grep -v ^_")
+
+            if input_folders == []:
+                input_folders = self._glob_remote(self.host_path+"/"+setup+"/input/global_settings.py")
+                if input_folders == []:
+                    continue
+
+                input_folders = [""]
+
+            for input_folder in input_folders:
+                print("  Check output for input folder "+input_folder+"...")
+                if input_folder != "" and input_folder[-1] == "/":
+                    input_folder = input_folder[:-1] 
+                input_folder = input_folder.split("/")[-1]
+
+                logger.log("## Output report for `"+setup+"` and input folder `"+input_folder+"`\n\n")
+
+                found_output = self._glob_remote(self.host_path+"/"+setup+"/output/"+input_folder)
                 if not found_output:
-                    file.write("**Could not find output for "+setup+"**\n")
-                    self.summary["Output:"+setup] = False
+                    logger.log("**Could not find output for "+setup+" and input folder `"+input_folder+"`**\n")
+                    self.summary["Output:"+setup+":"+input_folder] = False
                 else:
-                    file.write("Output `"+str(found_output)+"` has been generated for "+setup+".\n")
-                    self.summary["Output:"+setup] = True
+                    logger.log("Output `"+str(found_output)+"` has been generated for "+setup+" and input folder `"+input_folder+"`.\n")
+                    self.summary["Output:"+setup+":"+input_folder] = True
 
-                file.write("\n")
+                logger.log("\n")
+
+                model_outputs = self._glob_remote("-d "+self.host_path+"/"+setup+"/output/"+input_folder+"/*")
 
-                model_outputs = glob.glob(self.test_dir+"/"+setup+"/output/*/*")
                 for model_output in model_outputs:
+                    
+                    if model_output[-1] == "/":
+                        model_output = model_output[:-1]
+                                                    
                     model = model_output.split("/")[-1].split("_")[0]
+
+                    print("   Check model "+model+"...")
                     results_dir = model_output.split("/output/")[-1].replace("/", "_")
-                    fig_dir = self.report_dir+"/output/figures_"+setup+"/"+model
-                    found_plots = glob.glob(self.test_dir+"/"+setup+"/postprocess/"+model+"/plot*/results/"+results_dir+"*/*.pdf")
+                    found_report = self._glob_remote(self.host_path+"/"+setup+"/postprocess/"+model+"/create_validation_report/results/"+results_dir+"*/validation_report.md")                      
                     
-                    if not found_plots:
+                    print("   ...done.")
+                    if not found_report:
                         continue
 
-                    file.write("### "+model+"\n\n")
-                    file.write(str(len(found_plots))+" figures have been generated for "+model+".\n\n")
-
-                    file.write("<details>\n\n")
-
-                    os.system("mkdir -p "+fig_dir)
-
-                    found_plots = sorted(found_plots)
+                    validation_report_dir = self.report_dir+"/output_"+self.target+"/"+input_folder+"/"+model
 
-                    for plot in found_plots:
-                        figure = plot.split("/")[-1]
-                        try:
-                            from pdf2image import convert_from_path
-                            print("Convert "+plot+" to png file")
-                            figure = figure.replace("pdf","png")
-                            pages = convert_from_path(plot, 300)
-                            for page in pages:
-                                page.save(fig_dir+"/"+figure,'PNG')
-                        except: 
-                            os.system("cp "+plot+" "+fig_dir+"/")
-                        file.write("```{figure} ./figures_"+setup+"/"+model+"/"+figure+"\n")
-                        file.write("---\n")
-                        file.write("height: 500px\n")
-                        file.write("name: fig-"+setup+"-"+model+"-"+figure+"\n")
-                        file.write("---\n")
-                        file.write(figure+"\n")
-                        file.write("```\n\n")
+                    os.system("mkdir -p "+validation_report_dir)
+                    cmd = "scp -r "+self.dest+"/"+setup+"/postprocess/"+model+"/create_validation_report/results/"+results_dir+"*/* "+validation_report_dir
+                    sp = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
 
-                        self.summary["Output:"+setup+":"+model+":"+figure] = True
+                    logger.log("### "+model+"\n\n")
+                    logger.log("Validation report has been generated for "+model+".\n\n")
 
-                    file.write("</details>\n\n")
-
-                file.write("\n\n")
+                logger.log("\n\n")
+                print("  ...done.")
+            print(" ...done.")
+        print("...done.")
     
     def write_summary(self):
+        print("Write summary...")
+
         failed = 0
-        with open(self.intro_file_name, "a") as file:
-            file.write("## Summary\n\n")
+        self.summary_logger.log("| checkpoint       | success  |\n")
+        self.summary_logger.log("|---               |---       |\n")
+        for checkpoint in self.summary.keys():
+            self.summary_logger.log("|"+checkpoint+"|`"+str(self.summary[checkpoint])+"`|\n")
 
-            file.write("| checkpoint       | success  |\n")
-            file.write("|---               |---       |\n")
-            for checkpoint in self.summary.keys():
-                file.write("|"+checkpoint+"|`"+str(self.summary[checkpoint])+"`|\n")
+            if not self.summary[checkpoint]:
+                failed += 1
 
-                if not self.summary[checkpoint]:
-                    failed += 1
+        if failed > 0:
+            self.summary_logger.log("### Failure\n\n")
+            self.summary_logger.log("\n"+str(failed)+" checkpoints failed!\n")
+        else:
+            self.summary_logger.log("### Success\n\n")
+            self.summary_logger.log("\n<span style=\"color:green\">All checkpoints succeeded!</span>\n")
 
-            if failed > 0:
-                file.write("## Failure\n\n")
-                file.write("\n"+str(failed)+" checkpoints failed!\n")
-            else:
-                file.write("## Success\n\n")
-                file.write("\n<span style=\"color:green\">All checkpoints succeeded!</span>\n")
+        print("...done.")
+
+    def __del__(self):
+        print("# Reporter for "+self.target+" finished.")