#!/bin/csh -f
#
# run_clmtowers.csh
# Purpose: This script will run any number of flux tower sites. You will need to
#          run the script for each spinup and post spinup set of simulations (i.e., 
#          for BGC on, run it separately for AD, PostAD, and post spinup simulations;
#          for BGC off, run it separately for spinup and post spinup simulations)
#          You will need to do two things:
#          1. Copy this script into $Clm_Tag_Dir/tools/shared/PTCLM
#             where $Clm_Tag_Dir is the location of your clm tag
#          2. Set up a directory structure where you can put any sourcemods you might want.
#             These sourcemods will be copied into the appropriate case directory.
#             The structure is up to you but here is an example:
#             cd $Clm_Tag_Dir/tools/shared/PTCLM
#             mkdir SourceMods
#             mkdir SourceMods/clm4_5 
#             mkdir SourceMods/clm5_0
#             mkdir SourceMods/clm4_5/BASE  ; This might contain any sourcemods that you want
#                                           ;  to use in your clm4_5 control experiment
#             mkdir SourceMods/clm5_0/BASE  ; This might contain any sourcemods that you want
#                                           ;  to use in your clm5_0 control experiment
#             mkdir SourceMods/clm4_5/EXP1  ; This might contain any sourcemods that you want
#                                           ;  to use in your first clm4_5 experiment
#             mkdir SourceMods/clm5_0/EXP1  ; This might contain any sourcemods that you want
#                                           ;  to use in your first clm5_0 experiment
# Author: Keith Oleson
# Last Revised: Jan 24 2019
# Last CLM Tag that this worked on: release-clm5.0.12
# Warning: This script is complicated and does not have good (any) error checking currently.
#          You might want to ask me for a quick tutorial before using this.
#
# ASSUMES that PTCLMmkdata has already been run for the tower sites chosen below
# (Surface datasets and shell commands have already been created)
# To run this script on cheyenne: qcmd -- ./run_clmtowers.csh >&! run_clmtowers.out &
#

set pwd=`pwd`

# =================================Start User Mods================================
# Pick a compset (these are the only two compsets supported, they are both SP compsets
# but BGC will be added if requested below)
#set compset = I1PtClm45SpGs
set compset = I1PtClm50SpGs
if ($compset == I1PtClm45SpGs) then
  set model = clm4_5
else
  set model = clm5_0
endif

# Set location of your run directories
set rundata = /glade/derecho/scratch/oleson
# Set the location of your CLM tag
set Clm_Tag_Dir   = /glade/work/oleson/release-clm5.0.12
# Set the location of your surface datasets and shell commands that were generated by PTCLM. 
# This will not necessarily be in the same location as the CLM tag that you are running above
#set User_Mods_Dir = /glade/derecho/scratch/oleson/release-clm5.0.12  # This is my version for SP simulations
set User_Mods_Dir = /glade/derecho/scratch/oleson/release-clm5.0.12.BGC  # This is my version for BGC simulations

# What sites to run?
# These are the sites that can be evaluated with some combination of level 2 data and synthesis (gap-filled) data
#set sites     = ( US-Var US-Bo1 US-UMB US-Brw US-ARM US-Ho1 US-Me4 US-Me2 US-Dk3 US-NR1 DE-Tha ES-ES1 FL-Hyy CA-Man BR-Sa3 BR-Sa1 IT-Cpz US-Dk2 US-MOz US-WCr US-MMS US-Ha1 BE-Vie IT-Col CA-Let US-FPe FL-Kaa US-IB1 US-Ne3 CA-Qfo BR-Sa1LBA BR-Sa3LBA BR-Ma1LBA BR-RJA BR-Ji1 CA-Obs CA-Ojp CA-Ca1 CA-Oas US-Dk1)
#set startyear = ( 2000   1996   1999   1998   2000   1996   1996   2002   1998   1998   1998   1999   1997   1994   2001   2002   2001   2003   2004   1998   1999   1991   1997   1996   1998   2000   2000   2005   2001   2004   2002      2001      2000      2000   1999   2000   2000   1998   1997   2001)
#set endyear   = ( 2007   2008   2006   2006   2007   2004   2000   2010   2005   2007   2003   2005   2005   2003   2003   2004   2005   2005   2007   2006   2007   2006   2005   2001   2007   2007   2005   2007   2006   2006   2004      2003      2005      2002   2001   2006   2006   2006   2006   2005)
# Or you could just do one site
set sites     = ( US-Var )
set startyear = ( 2000 )
set endyear   = ( 2007 ) 

set BGC   = "ON"  # ON or OFF

# USER MODS FOR BGC ON
# For BGC on, the sequence of simulations is AD spinup (300 years),
#                                            PostAD spinup (100 years),
#                                            post spinup (the number of tower years with atmospheric forcing)
# For BGC on, AD spinup is            SPINUP_P1=TRUE,  SPINUP_P2=FALSE
#             PostAD spinup is        SPINUP_P1=TRUE,  SPINUP_P2=TRUE
#             post spinup is          SPINUP_P1=FALSE, SPINUP_P2=FALSE

if ($BGC == ON) then
  setenv SPINUP_P1 "TRUE"
  setenv SPINUP_P2 "TRUE"
endif

# For BGC on, you could use these for either type of spinup (AD or PostAD)
if ($BGC == ON) then
  if ($SPINUP_P1 == TRUE) then
    if ($model == clm5_0) then
      set newcase   = spinclm50conr12AD
      set clonecase = spinclm50conr12pAD
    else 
      if ($model == clm4_5) then
        set newcase   = spinclm45conr12AD
        set clonecase = spinclm45conr12pAD
     endif
    endif
  endif
endif

# For BGC on, you could use these for post spinup (the number of tower years with atmospheric forcing)
# You should change the "r12" for the clonecase to whatever tag you are using (e.g., r12 is used here to 
#   denote release-clm5.0.12) and/or add some designation for your particular experiment with that tag (e.g.,
#   conclm50r12wspinbgc)
if ($BGC == ON) then
  if ($SPINUP_P1 == FALSE) then
    if ($model == clm5_0) then
      set newcase   = spinclm50conr12pAD
      set clonecase = conclm50r12wspinbgc
    else
      if ($model == clm4_5) then
        set newcase   = spinclm45conr12pAD
        set clonecase = conclm45r12wspinbgc
      endif
    endif
  endif
endif

# USER MODS FOR BGC OFF
# For BGC off, the sequence of simulations is normal spinup (32 years)
#                                             post spinup (the number of tower years with atmospheric forcing)
# For BGC off, normal spinup is SPINUP_P1=TRUE,  SPINUP_P2=FALSE
#              post spinup is   SPINUP_P1=FALSE, SPINUP_P2=FALSE
if ($BGC == OFF) then
  setenv SPINUP_P1 "FALSE"
  setenv SPINUP_P2 "FALSE"
endif

# For BGC off, use these for either normal spinup or post spinup
# You should change the "r12" for the clonecase to whatever tag you are using (e.g., r12 is used here to 
#   denote release-clm5.0.12) and/or add some designation for your particular experiment with that tag (e.g., 
#   conclm50r12wspinsp)
if ($BGC == OFF) then
  if ($model == clm5_0) then
    set newcase   = spinclm50conr12sp
    set clonecase = conclm50r12wspinsp
  else
    if ($model == clm4_5) then
      set newcase   = spinclm45conr12sp
      set clonecase = conclm45r12wspinsp
    endif
  endif
endif

# These sourcemods will be copied into every case directory (you will need to setup a 
# directory structure for your sourcemods, see instructions at top of script)
set sourcemods_dir = {$Clm_Tag_Dir}/tools/PTCLM/SourceMods/
echo $sourcemods_dir
set sourcemods = {$sourcemods_dir}{$model}/BASE/*.F90 
echo $sourcemods

# Set some namelist options if required
# If you set any of these you will need to also set them below (search on namelist_opts)
#set namelist_opts1 = "paramfile='/glade/campaign/cgd/tss/people/oleson/modify_param/CLM5_SP_ens_dec_5D_mcalib_psi50BET3_BETKr9_Cropkrmax5e-10_calmbboptleafcn.nc'"
#set namelist_opts2 = "baseflow_scalar= 0.001d00"
# BGC
#set namelist_opts3 = "pot_hmn_ign_counts_alpha= 0.012d00"
#set namelist_opts4 = "cli_scale= 0.022d00"
#set namelist_opts5 = "boreal_peatfire_c= 0.2d-4"

# =================================End User Mods================================

@ cnt = 1
foreach mysite ( $sites )
  @ numyears = $endyear[$cnt] - $startyear[$cnt] + 1
  if ($SPINUP_P1 == TRUE) then
    @ numfour = $numyears / 4
    # If have three years or less (numfour = 0) just repeat first year  
    # unless first year is leap year then use next year.
    # Since just using one year that is not a leap year we can use 
    # an alignyear of 1 and endyear is the startyear
    if ( $numfour == 0 ) then
      if ( $startyear[$cnt] % 4 == 0 ) then
        @ startyears = $startyear[$cnt] + 1
        @ endyears  = $startyears
      else
        @ endyears = $startyear[$cnt]
        @ startyears = $endyears
      endif
      @ alignyear = 1
    endif
    if ( $numfour != 0 ) then
      @ startyears = $startyear[$cnt]
      @ endyears  = $startyear[$cnt] + $numfour * 4 - 1
      @ alignyear = $startyear[$cnt]
    endif
    echo $endyear[$cnt]
    echo $endyears
    echo $startyears
    echo $alignyear
  endif
  cd {$Clm_Tag_Dir}/cime/scripts
  if ($SPINUP_P1 == FALSE) then
    set casename = ${clonecase}_${mysite}_$compset
    ./create_clone --case $casename --clone ${newcase}_${mysite}_${compset}
  else
    if ($BGC == ON && $SPINUP_P2 == TRUE) then
      set casename = ${clonecase}_${mysite}_$compset
      echo $casename
      ./create_clone --case $casename --clone ${newcase}_${mysite}_${compset}
    else
      set casename = ${newcase}_${mysite}_$compset
      ./create_newcase --user-mods-dir {$User_Mods_Dir}/tools/PTCLM/mydatafiles/1x1pt_${mysite} --case $casename --mach cheyenne --compset $compset --res CLM_USRDAT --project P93300041 --run-unsupported
    endif
  endif
  cd {$Clm_Tag_Dir}/cime/scripts/${casename}
  ./xmlchange --id PIO_TYPENAME --val netcdf
  ./xmlchange --id PIO_REARRANGER --val 1
  if ($SPINUP_P1 == FALSE) then
    rm -f cesm.stderr*
    rm -f cesm.stdout*
    rm -f STATUS.out
    ./xmlchange --id STOP_OPTION --val nyears
    ./xmlchange --id STOP_N --val $numyears
    ./xmlchange --id RUN_STARTDATE --val $startyear[$cnt]-01-01
    ./xmlchange --id DATM_CLMNCEP_YR_ALIGN --val $startyear[$cnt]
    ./xmlchange --id DATM_CLMNCEP_YR_START --val $startyear[$cnt]
    ./xmlchange --id DATM_CLMNCEP_YR_END --val $endyear[$cnt]
    ./xmlchange --id CALENDAR --val GREGORIAN
    if ($BGC == ON) then
      ./xmlchange --id CLM_BLDNML_OPTS --val "-mask navy -bgc bgc -crop"
      ./xmlchange --id CLM_ACCELERATED_SPINUP --val "off"
    endif
  else
    if ($BGC == ON && $SPINUP_P2 == TRUE) then
      rm -f cesm.stderr*
      rm -f cesm.stdout*
      rm -f STATUS.out
      ./xmlchange --id STOP_OPTION --val nyears
      ./xmlchange --id STOP_N --val 100
      ./xmlchange --id CLM_BLDNML_OPTS --val "-mask navy -bgc bgc -crop"
      ./xmlchange --id CLM_ACCELERATED_SPINUP --val "off"
    else
      ./xmlchange --id STOP_OPTION --val nyears
      if ($BGC == ON) then
        ./xmlchange --id STOP_N --val 300
      else
        ./xmlchange --id STOP_N --val 32
      endif
      if ($alignyear == 1) then
        ./xmlchange --id RUN_STARTDATE --val 000{$alignyear}-01-01
      else
        ./xmlchange --id RUN_STARTDATE --val $startyear[$cnt]-01-01
      endif
      ./xmlchange --id DATM_CLMNCEP_YR_ALIGN --val $alignyear
      ./xmlchange --id DATM_CLMNCEP_YR_START --val $startyears
      ./xmlchange --id DATM_CLMNCEP_YR_END --val $endyears
      if ($alignyear == 1) then
        ./xmlchange --id CALENDAR --val NO_LEAP
      endif
      if ($BGC == ON) then
        ./xmlchange --id CLM_BLDNML_OPTS --val "-mask navy -bgc bgc -crop"
        ./xmlchange --id CLM_ACCELERATED_SPINUP --val "on"
      endif
    endif
  endif
  if ($mysite == BR-Sa1LBA || $mysite == BR-Sa3LBA || $mysite == BR-Ma1LBA || $mysite == BR-RJA || $mysite == BR-Ji1) then
    if ($SPINUP_P1 == FALSE) then
      rm -f user_datm.streams.txt.CLM1PT.CLM_USRDAT
    endif
  endif
  ./xmlchange --id MAX_TASKS_PER_NODE --val 1
  ./xmlchange --id MAX_MPITASKS_PER_NODE --val 1
  ./case.setup
  ./preview_namelists
  # Have to force this for some reason
  if ($SPINUP_P1 == FALSE) then
    ./xmlchange --id DATM_CLMNCEP_YR_END --val $endyear[$cnt]
    ./preview_namelists
  endif
  if ( $status != 0 )then
     echo "CESM_SETUP FAIL $status" >> ./STATUS.out
     exit -1
  else
     echo "CESM_SETUP PASS" >> ./STATUS.out
  endif
  if ($SPINUP_P1 == TRUE && $BGC == ON) then
    sed "/BSUB  -R/d" ./.case.run > tmp.run
    ./xmlchange --subgroup case.run --id JOB_WALLCLOCK_TIME --val 5:59 
#   ./xmlchange --subgroup case.run --id JOB_QUEUE --val regular
    ./xmlchange --subgroup case.run --id JOB_QUEUE --val share
    ./xmlchange --subgroup case.run --id PROJECT --val P93300041
  else
    sed "/BSUB  -R/d" ./.case.run > tmp.run
#   ./xmlchange --subgroup case.run --id JOB_QUEUE --val regular
    ./xmlchange --subgroup case.run --id JOB_WALLCLOCK_TIME --val 5:59
    ./xmlchange --subgroup case.run --id JOB_QUEUE --val share
    ./xmlchange --subgroup case.run --id PROJECT --val P93300041
  endif
  mv tmp.run ./.case.run
  chmod u+x ./.case.run
  cp $sourcemods SourceMods/src.clm
  echo $mysite
  if ($mysite == BR-Sa1LBA || $mysite == BR-Sa3LBA || $mysite == BR-Ma1LBA || $mysite == BR-RJA || $mysite == BR-Ji1) then
    cp CaseDocs/datm.streams.txt.CLM1PT.CLM_USRDAT ./user_datm.streams.txt.CLM1PT.CLM_USRDAT
    chmod u+wx ./user_datm.streams.txt.CLM1PT.CLM_USRDAT
    sed "s/RH   /QBOT /g" user_datm.streams.txt.CLM1PT.CLM_USRDAT > tmp.user_datm.streams.txt.CLM1PT.CLM_USRDAT
    sed "s/  rh/  shum/g" tmp.user_datm.streams.txt.CLM1PT.CLM_USRDAT > tmp2.user_datm.streams.txt.CLM1PT.CLM_USRDAT
    rm -f tmp.user_datm.streams.txt.CLM1PT.CLM_USRDAT
    mv tmp2.user_datm.streams.txt.CLM1PT.CLM_USRDAT ./user_datm.streams.txt.CLM1PT.CLM_USRDAT
  endif
  if ($SPINUP_P1 == FALSE) then
    sed "/taxmode = 'cycle','cycle'/d" user_nl_datm > tmp.user_nl_datm
    mv tmp.user_nl_datm ./user_nl_datm
    sed "s/hist_nhtfrq = 0/hist_nhtfrq = 0,1/g" ./user_nl_clm > tmp.user_nl_clm
    sed "s/hist_mfilt  = 1200/hist_mfilt  = 1,350400/g" ./tmp.user_nl_clm > tmp2.user_nl_clm
    rm -f tmp.user_nl_clm
    sed "/finidat/d" ./tmp2.user_nl_clm > tmp3.user_nl_clm
    rm -f tmp2.user_nl_clm
    if ($BGC == ON) then
      echo " hist_fincl2  = 'FSDS','FLDS','FSR','FSA','FIRE','FIRA','FSH','FCTR','FCEV','FGEV','FGR','FGR12','FSM','TSOI','COSZEN','RAIN','SNOW','H2OSOI','WA','ZWT','GPP','NEE','ELAI','BTRAN','TV','RSSUN','RSSHA','FSH_G','RHAF','RH_LEAF','RH','T10','TG','SABG','SABV'" >> tmp3.user_nl_clm
    else
      if ($model == clm5_0) then
        echo " hist_fincl2  = 'FSDS','FLDS','FSR','FSA','FIRE','FIRA','FSH','FCTR','FCEV','FGEV','FGR','FGR12','FSM','TSOI','COSZEN','RAIN','SNOW','H2OSOI','WA','ZWT','ELAI','BTRAN','FPSN','TV','RSSUN','RSSHA','FSH_G','RHAF','RH_LEAF','RH','T10','TG','SABG','SABV','VEGWP'" >> tmp3.user_nl_clm
      else
        echo " hist_fincl2  = 'FSDS','FLDS','FSR','FSA','FIRE','FIRA','FSH','FCTR','FCEV','FGEV','FGR','FGR12','FSM','TSOI','COSZEN','RAIN','SNOW','H2OSOI','WA','ZWT','ELAI','BTRAN','FPSN','TV','RSSUN','RSSHA','FSH_G','RHAF','RH_LEAF','RH','T10','TG','SABG','SABV'" >> tmp3.user_nl_clm
      endif
    endif
    set finidat=`ls -1 $rundata/${newcase}_${mysite}_${compset}/run/${newcase}_${mysite}_${compset}.clm?.r.*.nc | tail -1`
    echo $finidat
    echo " finidat = '$finidat'" >> tmp3.user_nl_clm
    mv tmp3.user_nl_clm ./user_nl_clm
  else
    if ($BGC == ON && $SPINUP_P2 == TRUE) then 
      set finidat=`ls -1 $rundata/${newcase}_${mysite}_${compset}/run/${newcase}_${mysite}_${compset}.clm?.r.*.nc | tail -1`
      echo $finidat
      echo " finidat = '$finidat'" >> user_nl_clm
    else
      echo "taxmode = 'cycle','cycle'" >> user_nl_datm
#      echo $namelist_opts1 >> user_nl_clm
#      echo $namelist_opts2 >> user_nl_clm
#      echo $namelist_opts3 >> user_nl_clm
#      echo $namelist_opts4 >> user_nl_clm
#      echo $namelist_opts5 >> user_nl_clm
      if ($BGC == ON) then
        sed "s/hist_mfilt  = 1200/hist_mfilt  = 12000/g" ./user_nl_clm > tmp.user_nl_clm
        echo " hist_fincl1  = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'FSH', 'FCTR', 'FCEV', 'FGEV', 'QRUNOFF', 'ZWT', 'NEE', 'NEP'" >> tmp.user_nl_clm
        mv tmp.user_nl_clm ./user_nl_clm
      endif
    endif
  endif
  ./case.build
  if ( $status != 0 )then
     echo "BUILD FAIL $status" >> ./STATUS.out
     exit -1
  else
     echo "BUILD PASS" >> ./STATUS.out
  endif
  ./case.submit
  if ( $status != 0 )then
     echo "SUBMIT FAIL $status" >> ./STATUS.out
     exit -1
  else
     echo "SUBMIT PASS" >> ./STATUS.out
  endif
  cd $pwd
  @ cnt++
end