clm5/test/tools/test_driver.sh
2024-05-09 15:14:01 +08:00

723 lines
18 KiB
Bash

#!/bin/sh
#
# test_driver.sh: driver script for the offline testing of CLM of tools
#
# interactive usage on all machines:
#
# env ./test_driver.sh -i
#
# valid arguments:
# -i interactive usage
# -d debug usage -- display tests that will run -- but do NOT actually execute them
# -f force batch submission (avoids user prompt)
# -h displays this help message
#
#
# **pass environment variables by preceding above commands
# with 'env var1=setting var2=setting '
# **more details in the CLM testing user's guide, accessible
# from the CLM developers web page
#will attach timestamp onto end of script name to prevent overwriting
cur_time=`date '+%H:%M:%S'`
hostname=`hostname`
echo $hostname
case $hostname in
##Derecho
derecho* | dec*)
submit_script="test_driver_derecho${cur_time}.sh"
##vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv
cat > ./${submit_script} << EOF
#!/bin/sh
#
interactive="YES"
input_file="tests_pretag_derecho_nompi"
c_threads=128
export INITMODULES="/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/sh"
. \$INITMODULES
module --force purge
module load ncarenv
module load craype
module load intel
module load mkl
module load ncarcompilers
module load netcdf
module load nco
module load ncl
#omp threads
if [ -z "\$CLM_THREADS" ]; then #threads NOT set on command line
export CLM_THREADS=\$c_threads
fi
# Stop on first failed test
if [ -z "\$CLM_SOFF" ]; then #CLM_SOFF NOT set
export CLM_SOFF=FALSE
fi
export CESM_MACH="derecho"
export CESM_COMP="intel"
export NETCDF_DIR=\$NETCDF
export INC_NETCDF=\$NETCDF/include
export LIB_NETCDF=\$NETCDF/lib
export MAKE_CMD="gmake -j "
export CFG_STRING=""
export TOOLS_MAKE_STRING="USER_FC=ifort USER_LINKER=ifort USER_CPPDEFS=-DLINUX"
export MACH_WORKSPACE=\$SCRATCH
export CPRNC_EXE="$CESMDATAROOT/cprnc/cprnc"
dataroot="$CESMDATAROOT/inputdata"
export TOOLSLIBS=""
export REGRID_PROC=1
export TOOLS_CONF_STRING="--mpilib mpi-serial"
echo_arg=""
EOF
#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^
;;
##cheyenne
cheyenne* | r*i*n*)
submit_script="test_driver_cheyenne${cur_time}.sh"
#vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv
at > ./${submit_script} << EOF
!/bin/sh
interactive="YES"
input_file="tests_pretag_cheyenne_nompi"
c_threads=36
export INITMODULES="/glade/u/apps/ch/opt/lmod/8.1.7/lmod/lmod/init/sh"
. \$INITMODULES
module purge
module load ncarenv
module load intel
module load mkl
module load ncarcompilers
module load netcdf
module load nco
module load ncl
module load conda
##omp threads
if [ -z "\$CLM_THREADS" ]; then #threads NOT set on command line
export CLM_THREADS=\$c_threads
fi
# Stop on first failed test
if [ -z "\$CLM_SOFF" ]; then #CLM_SOFF NOT set
export CLM_SOFF=FALSE
fi
export CESM_MACH="cheyenne"
export CESM_COMP="intel"
export NETCDF_DIR=\$NETCDF
export INC_NETCDF=\$NETCDF/include
export LIB_NETCDF=\$NETCDF/lib
export MAKE_CMD="gmake -j "
export CFG_STRING=""
export TOOLS_MAKE_STRING="USER_FC=ifort USER_LINKER=ifort USER_CPPDEFS=-DLINUX"
export MACH_WORKSPACE="/glade/scratch"
export CPRNC_EXE="$CESMDATAROOT/tools/cime/tools/cprnc/cprnc.cheyenne"
dataroot="$CESMDATAROOT"
export TOOLSLIBS=""
export REGRID_PROC=1
export TOOLS_CONF_STRING="--mpilib mpi-serial"
echo_arg=""
EOF
##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^
;;
## DAV cluster
casper* | pronghorn*)
submit_script="test_driver_dav${cur_time}.sh"
##vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv
cat > ./${submit_script} << EOF
#!/bin/sh
#
interactive="YES"
input_file="tests_posttag_dav_mpi"
c_threads=36
export INITMODULES="/glade/u/apps/ch/opt/lmod/8.1.7/lmod/lmod/init/sh"
. \$INITMODULES
module purge
module load ncarenv
module load intel
module load mkl
module load ncarcompilers
module load netcdf
module load openmpi
module load nco
module load conda
module load ncl
##omp threads
if [ -z "\$CLM_THREADS" ]; then #threads NOT set on command line
export CLM_THREADS=\$c_threads
fi
# Stop on first failed test
if [ -z "\$CLM_SOFF" ]; then #CLM_SOFF NOT set
export CLM_SOFF=FALSE
fi
export CESM_MACH="cheyenne"
export CESM_COMP="intel"
export NETCDF_DIR=\$NETCDF
export INC_NETCDF=\$NETCDF/include
export LIB_NETCDF=\$NETCDF/lib
export MAKE_CMD="gmake -j "
export CFG_STRING=""
export TOOLS_MAKE_STRING="USER_FC=ifort USER_LINKER=ifort USER_CPPDEFS=-DLINUX"
export MACH_WORKSPACE="/glade/scratch"
export CPRNC_EXE="$CESMDATAROOT/tools/cime/tools/cprnc/cprnc.cheyenne"
dataroot="$CESMDATAROOT"
export TOOLSLIBS=""
export TOOLS_CONF_STRING="--mpilib mpich"
echo_arg=""
EOF
##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^
;;
## hobart
hobart* | h*.cgd.ucar.edu)
submit_script="test_driver_hobart_${cur_time}.sh"
export PATH=/cluster/torque/bin:${PATH}
##vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv
cat > ./${submit_script} << EOF
#!/bin/sh
#
# Name of the queue (CHANGE THIS if needed)
#PBS -q long
# Number of nodes (CHANGE THIS if needed)
#PBS -l nodes=1:ppn=24
# output file base name
#PBS -N test_dr
# Put standard error and standard out in same file
#PBS -j oe
# Export all Environment variables
#PBS -V
# End of options
if [ -n "\$PBS_JOBID" ]; then #batch job
export JOBID=\`echo \${PBS_JOBID} | cut -f1 -d'.'\`
initdir=\${PBS_O_WORKDIR}
fi
if [ "\$PBS_ENVIRONMENT" = "PBS_BATCH" ]; then
interactive="NO"
input_file="tests_posttag_hobart"
else
interactive="YES"
input_file="tests_posttag_hobart_nompi"
fi
##omp threads
if [ -z "\$CLM_THREADS" ]; then #threads NOT set on command line
export CLM_THREADS=2
fi
export CLM_RESTART_THREADS=1
##mpi tasks
export CLM_TASKS=24
export CLM_RESTART_TASKS=20
export P4_GLOBMEMSIZE=500000000
export CESM_MACH="hobart"
ulimit -s unlimited
ulimit -c unlimited
export CESM_COMP="intel"
export TOOLS_MAKE_STRING="USER_FC=ifort USER_CC=icc "
export TOOLS_CONF_STRING=" -mpilib mpi-serial"
export CFG_STRING=""
export INITMODULES="/usr/share/Modules/init/sh"
. \$INITMODULES
module purge
module load compiler/intel
module load tool/nco
module load tool/netcdf
module load lang/python
export NETCDF_DIR=\$NETCDF_PATH
export INC_NETCDF=\${NETCDF_PATH}/include
export LIB_NETCDF=\${NETCDF_PATH}/lib
export MAKE_CMD="gmake -j 5" ##using hyper-threading on hobart
export MACH_WORKSPACE="/scratch/cluster"
export CPRNC_EXE=/fs/cgd/csm/tools/cprnc/cprnc
export DATM_QIAN_DATA_DIR="/project/tss/atm_forcing.datm7.Qian.T62.c080727"
dataroot="/fs/cgd/csm"
export TOOLSSLIBS=""
echo_arg="-e"
EOF
##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^
;;
## izumi
izumi* | i*.unified.ucar.edu)
submit_script="test_driver_izumi_${cur_time}.sh"
export PATH=/cluster/torque/bin:${PATH}
##vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv
cat > ./${submit_script} << EOF
#!/bin/sh
#
# Name of the queue (CHANGE THIS if needed)
#PBS -q long
# Number of nodes (CHANGE THIS if needed)
#PBS -l nodes=1:ppn=24
# output file base name
#PBS -N test_dr
# Put standard error and standard out in same file
#PBS -j oe
# Export all Environment variables
#PBS -V
# End of options
if [ -n "\$PBS_JOBID" ]; then #batch job
export JOBID=\`echo \${PBS_JOBID} | cut -f1 -d'.'\`
initdir=\${PBS_O_WORKDIR}
fi
if [ "\$PBS_ENVIRONMENT" = "PBS_BATCH" ]; then
interactive="NO"
input_file="tests_posttag_izumi"
else
interactive="YES"
input_file="tests_posttag_izumi_nompi"
fi
##omp threads
if [ -z "\$CLM_THREADS" ]; then #threads NOT set on command line
export CLM_THREADS=2
fi
export CLM_RESTART_THREADS=1
##mpi tasks
export CLM_TASKS=24
export CLM_RESTART_TASKS=20
export P4_GLOBMEMSIZE=500000000
export CESM_MACH="izumi"
ulimit -s unlimited
ulimit -c unlimited
export CESM_COMP="intel"
export TOOLS_MAKE_STRING="USER_FC=ifort USER_CC=icc "
export TOOLS_CONF_STRING=" -mpilib mpi-serial"
export CFG_STRING=""
export INITMODULES="/usr/share/Modules/init/sh"
. \$INITMODULES
module purge
module load compiler/intel
module load tool/nco
module load tool/netcdf
module load lang/python
export NETCDF_DIR=\$NETCDF_PATH
export INC_NETCDF=\${NETCDF_PATH}/include
export LIB_NETCDF=\${NETCDF_PATH}/lib
export MAKE_CMD="gmake -j 5" ##using hyper-threading on izumi
export MACH_WORKSPACE="/scratch/cluster"
export CPRNC_EXE=/fs/cgd/csm/tools/cprnc/cprnc.izumi
export DATM_QIAN_DATA_DIR="/project/tss/atm_forcing.datm7.Qian.T62.c080727"
dataroot="/fs/cgd/csm"
export TOOLSSLIBS=""
echo_arg="-e"
EOF
##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^
;;
* )
echo "Only setup to work on: derecho, cheyenne, hobart and izumi"
exit
esac
##vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv
cat >> ./${submit_script} << EOF
export CPRNC_OPT=""
if [ -n "\${CLM_JOBID}" ]; then
export JOBID=\${CLM_JOBID}
fi
##check if interactive job
if [ "\$interactive" = "YES" ]; then
if [ -z "\${JOBID}" ]; then
export JOBID=\$\$
fi
echo "test_driver.sh: interactive run - setting JOBID to \$JOBID"
if [ \$0 = "test_driver.sh" ]; then
initdir="."
else
initdir=\${0%/*}
fi
else
echo "ERROR: you *always* need to use the interactive option (-i)"
echo " currently doesn't work without it"
exit 3
fi
##establish script dir and clm_root
if [ -f \${initdir}/test_driver.sh ]; then
export CLM_SCRIPTDIR=\`cd \${initdir}; pwd \`
export CLM_ROOT=\`cd \${CLM_SCRIPTDIR}/../..; pwd \`
export CTSM_ROOT=\${CLM_ROOT}
if [ -d \${CLM_ROOT}/cime ]; then
export CIME_ROOT=\${CLM_ROOT}/cime
else
export CIME_ROOT=\${CLM_ROOT}/../../cime
fi
if [ ! -d \${CIME_ROOT} ]; then
echo "ERROR: trouble finding the CIME_ROOT directory: \$CIME_ROOT"
exit 3
fi
else
if [ -n "\${CLM_ROOT}" ] && [ -f \${CLM_ROOT}/test/tools/test_driver.sh ]; then
export CLM_SCRIPTDIR=\`cd \${CLM_ROOT}/test/tools; pwd \`
else
echo "ERROR: unable to determine script directory "
echo " if initiating batch job from directory other than the one containing test_driver.sh, "
echo " you must set the environment variable CLM_ROOT to the full path of directory containing "
echo " <cime/scripts>. "
exit 3
fi
fi
# Setup conda environment
conda activate ctsm_pylib
if [ \$? -ne 0 ]; then
echo "ERROR: Trouble activating the ctsm_pylib conda environment, be sure it's setup with \$CLM_ROOT/py_env_create, then rerun"
exit 4
fi
##output files
clm_log=\${initdir}/td.\${JOBID}.log
if [ -f \$clm_log ]; then
rm \$clm_log
fi
clm_status=\${initdir}/td.\${JOBID}.status
if [ -f \$clm_status ]; then
rm \$clm_status
fi
##setup test work directory
if [ -z "\$CLM_TESTDIR" ]; then
export CLM_TESTDIR=\${MACH_WORKSPACE}/\$LOGNAME/clmTests/test-driver.\${JOBID}
if [ -d \$CLM_TESTDIR ] && [ \$CLM_RETAIN_FILES != "TRUE" ]; then
rm -r \$CLM_TESTDIR
fi
fi
if [ ! -d \$CLM_TESTDIR ]; then
mkdir -p \$CLM_TESTDIR
if [ \$? -ne 0 ]; then
echo "ERROR: unable to create work directory \$CLM_TESTDIR"
exit 4
fi
fi
## MCT and PIO build directorys
export MCT_LIBDIR=\$CLM_TESTDIR/mct
export PIO_LIBDIR=\$CLM_TESTDIR/pio
##set our own environment vars
export CSMDATA=\${dataroot}/inputdata
export DIN_LOC_ROOT=\${CSMDATA}
export MPI_TYPE_MAX=100000
##process other env vars possibly coming in
if [ -z "\$CLM_RETAIN_FILES" ]; then
export CLM_RETAIN_FILES=FALSE
fi
if [ -n "\${CLM_INPUT_TESTS}" ]; then
input_file=\$CLM_INPUT_TESTS
else
input_file=\${CLM_SCRIPTDIR}/\${input_file}
fi
if [ ! -f \${input_file} ]; then
echo "ERROR: unable to locate input file \${input_file}"
exit 5
fi
if [ \$interactive = "YES" ]; then
echo "reading tests from \${input_file}"
else
echo "reading tests from \${input_file}" >> \${clm_log}
fi
num_tests=\`wc -w < \${input_file}\`
echo "STATUS OF CLM TESTING UNDER JOB \${JOBID}; scheduled to run \$num_tests tests from:" >> \${clm_status}
echo "\$input_file" >> \${clm_status}
echo "" >> \${clm_status}
echo " on machine: $hostname" >> \${clm_status}
if [ -n "${BL_ROOT}" ]; then
echo "tests of baseline will use source code from:" >> \${clm_status}
echo "\$BL_ROOT" >> \${clm_status}
fi
if [ \$interactive = "NO" ]; then
echo "see \${clm_log} for more detailed output" >> \${clm_status}
fi
echo "" >> \${clm_status}
test_list=""
while read input_line; do
test_list="\${test_list}\${input_line} "
done < \${input_file}
##initialize flags, counter
skipped_tests="NO"
pending_tests="NO"
count=0
##loop through the tests of input file
for test_id in \${test_list}; do
count=\`expr \$count + 1\`
while [ \${#count} -lt 3 ]; do
count="0\${count}"
done
master_line=\`grep \$test_id \${CLM_SCRIPTDIR}/input_tests_master\`
status_out=""
for arg in \${master_line}; do
status_out="\${status_out}\${arg} "
done
if [ -z "\$status_out" ]; then
echo "No test matches \$test_id in \${CLM_SCRIPTDIR}/input_tests_master"
exit 3
fi
test_cmd=\${status_out#* }
status_out="\${count} \${status_out}"
if [ \$interactive = "YES" ]; then
echo ""
echo "***********************************************************************************"
echo "\${status_out}"
echo "***********************************************************************************"
else
echo "" >> \${clm_log}
echo "***********************************************************************************"\
>> \${clm_log}
echo "\$status_out" >> \${clm_log}
echo "***********************************************************************************"\
>> \${clm_log}
fi
if [ \${#status_out} -gt 94 ]; then
status_out=\`echo "\${status_out}" | cut -c1-100\`
fi
while [ \${#status_out} -lt 97 ]; do
status_out="\${status_out}."
done
echo \$echo_arg "\$status_out\c" >> \${clm_status}
if [ \$interactive = "YES" ]; then
\${CLM_SCRIPTDIR}/\${test_cmd}
rc=\$?
else
\${CLM_SCRIPTDIR}/\${test_cmd} >> \${clm_log} 2>&1
rc=\$?
fi
if [ \$rc -eq 0 ]; then
echo "PASS" >> \${clm_status}
elif [ \$rc -eq 255 ]; then
echo "SKIPPED*" >> \${clm_status}
skipped_tests="YES"
elif [ \$rc -eq 254 ]; then
echo "PENDING**" >> \${clm_status}
pending_tests="YES"
else
echo " rc=\$rc FAIL" >> \${clm_status}
if [ "\$CLM_SOFF" = "TRUE" ]; then
echo "stopping on first failure" >> \${clm_status}
echo "stopping on first failure" >> \${clm_log}
exit 6
fi
fi
done
echo "end of input" >> \${clm_status}
if [ \$interactive = "YES" ]; then
echo "end of input"
else
echo "end of input" >> \${clm_log}
fi
if [ \$skipped_tests = "YES" ]; then
echo "* please verify that any skipped tests are not required of your clm commit" >> \${clm_status}
fi
if [ \$pending_tests = "YES" ]; then
echo "** tests that are pending must be checked manually for a successful completion" >> \${clm_status}
if [ \$interactive = "NO" ]; then
echo " see the test's output in \${clm_log} " >> \${clm_status}
echo " for the location of test results" >> \${clm_status}
fi
fi
if [ "\$interactive" = "YES" ]; then
passInt="test_driver.sh-i"
else
passInt="test_driver.sh"
fi
../../bld/unit_testers/xFail/wrapClmTests.pl -statusFile "\${clm_status}" -numberOfTests "\${num_tests}" -callingScript "\${passInt}"
exit 0
EOF
##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^
chmod a+x $submit_script
if [ ! -z "$CLM_RETAIN_FILES" ]; then
export CLM_RETAIN_FILES="FALSE"
fi
arg1=${1##*-}
case $arg1 in
[iI]* )
debug="NO"
interactive="YES"
compile_only="NO"
export debug
export interactive
export compile_only
./${submit_script}
exit 0
;;
[cC]* )
debug="NO"
interactive="YES"
compile_only="YES"
export debug
export CLM_RETAIN_FILES="TRUE"
export interactive
export compile_only
export CLM_RETAIN_FILES="TRUE"
./${submit_script}
exit 0
;;
[dD]* )
debug="YES"
interactive="YES"
compile_only="NO"
export debug
export interactive
export compile_only
./${submit_script}
exit 0
;;
[fF]* )
debug="NO"
interactive="NO"
compile_only="NO"
export debug
export interactive
export compile_only
;;
"" )
echo ""
echo "**********************"
echo "$submit_script has been created and will be submitted to the batch queue..."
echo "(ret) to continue, (a) to abort"
read ans
case $ans in
[aA]* )
echo "aborting...type ./test_driver.sh -h for help message"
exit 0
;;
esac
debug="NO"
interactive="NO"
compile_only="NO"
export debug
export interactive
export compile_only
;;
* )
echo ""
echo "**********************"
echo "usage on derecho, cheyenne, hobart, and izumi: "
echo "./test_driver.sh -i"
echo ""
echo "valid arguments: "
echo "-i interactive usage"
echo "-c compile-only usage (run configure and compile do not run clm)"
echo "-d debug-only usage (run configure and build-namelist do NOT compile or run clm)"
echo "-f force batch submission (avoids user prompt)"
echo "-h displays this help message"
echo ""
echo "**pass environment variables by preceding above commands "
echo " with 'env var1=setting var2=setting '"
echo ""
echo "**********************"
exit 0
;;
esac
echo "submitting..."
case $hostname in
#default
* )
echo "no submission capability on this machine use the interactive option: -i"
exit 0
;;
esac
exit 0