diff --git a/tools/spinup_stability/README.md b/tools/spinup_stability/README.md new file mode 100644 index 0000000000..432defa22f --- /dev/null +++ b/tools/spinup_stability/README.md @@ -0,0 +1,56 @@ +# Spinup stability +## a python script to assess CLM spinup stability + +## Requirements: + +1. ctsm_pylib + - see zqz for details + +2. a completed CLM simulation + - needs area and landfrac variables + - should have completed at least two forcing cycles + +3. spinup stability configuration file + - more info below + +4. missing bits + - only implemented for annualized output presently + - only implemented for regular grids + - no sparsegrid + - no spectral element + +## Files + +1. spinup_stability.py + - a python script to calculate and evaluate model drift + - will create a set of diagnostic plots + - exits with 0 if the simulation meets all criteria + - exits with 11 if the simulation is drifting + +2. default.yml + - a configuration file for spinup_stability.py + - includes our standard stability criteria + - points to the case that will be evaulated + - thresholds with _gridded, are interpreted as pct landarea gridded disequilibrium + - threshoulds without _gridded, are interpreted as global drifts + - cf = conversion factor + - land area (la) multiplication is also already included + - i.e. should be interpreted as GPP=cf*(la*ds.GPP).sum(dim=['lat','lon']) + - lists of conversion factors will be multiplied together (see for example GPP) + +3. batch.job + - an example job submission script in lieu of running from the command line + +## Usage + +1. load a suitable python environment. +2. python spinup_stability.py my_config.yml + +May be preferred as a batch job, e.g. +qsub batch.job + +Output: + - The script will write some text to stdout detailing spinup status. + - The script will exit 0 for spun up, 11 for drifting, 1 for other problems. + - The script will also create a set of plots $CASE.png + \ No newline at end of file diff --git a/tools/spinup_stability/batch.job b/tools/spinup_stability/batch.job new file mode 100644 index 0000000000..78e489a48e --- /dev/null +++ b/tools/spinup_stability/batch.job @@ -0,0 +1,13 @@ +#!/bin/bash +#PBS -N spinup_stability +#PBS -q develop +#PBS -l walltime=1:00:00 +#PBS -A P93300041 +#PBS -k eod +#PBS -l select=1:ncpus=5 +# --------------------------------------------------------- + +module load conda +conda activate npl-2025b + +python spinup_stability.py default.yml diff --git a/tools/spinup_stability/default.yml b/tools/spinup_stability/default.yml new file mode 100644 index 0000000000..ebf8c68ba3 --- /dev/null +++ b/tools/spinup_stability/default.yml @@ -0,0 +1,45 @@ +# detailing the spinup case and stability thresholds +case: I1850Clm50Bgc.CPLHIST.default.pAD +hist_dir: /glade/derecho/scratch/djk2120/archive/I1850Clm50Bgc.CPLHIST.default.pAD/lnd/hist +cycle_years: 20 +pct_landarea: 3 +thresholds: + TOTECOSYSC: 0.02 + TOTVEGC: 0.02 + GPP: 0.02 + TLAI: 0.02 + TEC_gridded: 1 + TVC_gridded: 1 + TSC_gridded: 1 +cfs: + TOTECOSYSC: 1e-9 + TOTSOMC: 1e-9 + TOTVEGC: 1e-9 + GPP: + - 1e-9 + - 86400 + - 365 + TLAI: 1/lasum + TWS: 1/lasum + TEC_gridded: 1 + TSC_gridded: 1 + TVC_gridded: 1 +units: + TOTECOSYSC: PgC + TOTSOMC: PgC + TOTVEGC: PgC + GPP: PgC + TLAI: m2/m2 + TEC_gridded: gC/m2 + TSC_gridded: gC/m2 + TVC_gridded: gC/m2 + TWS: mm +data_vars: + - TLAI + - GPP + - TOTECOSYSC + - TOTVEGC + - TOTSOMC + - TWS + + diff --git a/tools/spinup_stability/spinup_stability.py b/tools/spinup_stability/spinup_stability.py new file mode 100644 index 0000000000..a952286023 --- /dev/null +++ b/tools/spinup_stability/spinup_stability.py @@ -0,0 +1,216 @@ +import xarray as xr +import numpy as np +import matplotlib.pyplot as plt +import glob +import sys +import yaml + +def parse_cf(cf,lasum): + ''' logic for parsing each conversion factor entry in the yaml file ''' + if type(cf)==str: + cf=parse_cfstr(cf,lasum) + elif type(cf)==list: + cftot=1 + for f in cf: + cftot*=parse_cf(f,lasum) + cf=cftot + return cf + +def parse_cfstr(cf,lasum): + ''' + replaces the str 1/lasum with the actual number 1/lasum, where lasum is the sum of the landarea, typically (ds.area*ds.landfrac).sum(dim=['lat','lon']) + ''' + if cf=='1/lasum': + return 1/lasum + else: + return float(cf) + +def parse_cfs(cfs,lasum): + ''' + translate the conversion factors inherited from the yaml to a dictionary of conversion factors, with two logic rules, lists are returned as the group product, and strings are converted to float unless they are special reserved phrases (e.g. '1/lasum') + ''' + for v in cfs: + cfs[v]=parse_cf(cfs[v],lasum) + return cfs + +def check_freq(tmp): + '''infer if the dataset time frequency is monthly or annual''' + # there has to be a better way to do this + # is there alreay a function for this? + # tricky to handle old vs. new clm history files + nsecs_per_day=24*60*60*1e9 + if 'nbnd' in tmp.time_bounds.dims: + dt=(tmp.time_bounds.isel(time=-1,nbnd=1)- + tmp.time_bounds.isel(time=-1,nbnd=0))/nsecs_per_day + else: + dt=(tmp.time_bounds.isel(time=-1,hist_interval=1)- + tmp.time_bounds.isel(time=-1,hist_interval=0))/nsecs_per_day + if dt<40: + freq='monthly' + else: + freq='annual' + return freq + +def get_ds(files,freq,dvs): + #open dataset and process to an orderly annual dataset + def pp(ds): + return ds[dvs] + if freq=='monthly': + # this is really quite slow without dask + # I have ideas for this, but will have to come back later + raise NotImplementedError + else: + ds=xr.open_mfdataset(files,combine='by_coords',preprocess=pp, + decode_timedelta=False, parallel = False) + ds=ds.isel(time=slice(1,len(ds.time))) + return ds + +def plot_drifts(xs,thiscase,ncycles,nyears,units,thresholds,drifts,equils,tpct,la,lasum): + plt.figure(figsize=[16,12]) + for j,v in enumerate(xs): + plt.subplot(3,3,j+1) #hard-coded, should probably revise + if 'gridded' not in v: + # this is one type of plot, for global variables + for i in range(ncycles): + plt.plot(range(nyears),xs[v].isel(time=np.arange(nyears)+i*nyears),label='cycle_'+str(i).zfill(3)) + plt.ylabel(v+' ['+units[v]+']') + if j==5: + plt.legend() + dstr=str(np.round(drifts[v],3)) + tstr='drift='+dstr+units[v]+'/yr' + gl='><' + if v in thresholds: + tstr+=gl[int(equils[v])] + tstr+=str(thresholds[v]) + else: + tstr+=' [not evaluated]' + else: + # this is an alternative plot for gridded landarea disequilibrium + x=xs[v] + if v in thresholds: + thresh=thresholds[v] + else: + thresh=1 + diseq=abs(x-x.shift(time=nyears))/nyears>thresh + pct=100*(la*diseq).sum(dim=['lat','lon'])/lasum + ix=np.arange(len(x.time))>=nyears + pct.where(ix).plot() + ystr=(r'abs($\Delta$'+v.split('_')[0]+')>'+ + str(thresh)+units[v]+'/yr'+'\n[% landarea]') + plt.ylabel(ystr) + plt.ylim([0,100]) + plt.xlabel('') + dstr=str(np.round(drifts[v],1)) + tstr=dstr+'%' + if v in equils: + tstr+=gl[int(equils[v])] + tstr+=str(tpct) + else: + tstr+=' [not evaluated]' + if v in equils: + if not equils[v]: + tstr='FAILED: '+tstr + + plt.title(tstr) + plt.subplots_adjust(hspace=0.2,wspace=0.3) + plt.savefig(thiscase+'.png',dpi=300,bbox_inches='tight') + + + +def main(): + cfile = sys.argv[1] + config = yaml.safe_load(open(cfile)) + thiscase = config['case'] + d = config['hist_dir'] + files = sorted(glob.glob(d+'/*.h0*')) + print(d) + print(len(files)) + if len(files) < 1: + print('no files found') + print('matchstr: '+d+'/*.h0*') + sys.exit(1) + else: + #import config and parse conversion factors + tmp = xr.open_dataset(files[0],decode_timedelta=True) + la = tmp.area*tmp.landfrac + lasum = la.sum().values + thresholds=config['thresholds'] + units=config['units'] + cfs=parse_cfs(config['cfs'],lasum) + + freq=check_freq(tmp) + dvs=config['data_vars'] + ds=get_ds(files,freq,dvs) + + #set up year variables + nyears=config['cycle_years'] #years per met forcing cycle + ncycles=int(len(ds.time)/nyears) + y2=nyears*ncycles + y1=y2-nyears + y0=y1-nyears + + #abbrev dict + stocks={'TEC':'TOTECOSYSC', + 'TSC':'TOTSOMC', + 'TVC':'TOTVEGC'} + + #evaluate global drifts + drifts={} + xs={} + for v in cfs: + cf=cfs[v] + if 'gridded' in v: + vlong=stocks[v.split('_')[0]] + x=cf*ds[vlong] + else: + x=cf*(la*ds[v]).sum(dim=['lat','lon']) + xs[v]=x + drift=abs(x.isel(time=slice(y1,y2)).mean(dim='time')- + x.isel(time=slice(y0,y1)).mean(dim='time'))/nyears + if 'gridded' in v: + if v in thresholds: + thresh=thresholds[v] + else: + thresh=1 + pct=100*(la*(drift>thresh)).sum(dim=['lat','lon'])/lasum + drifts[v]=pct.values + else: + drifts[v]=drift.values + + equils={} + for v in thresholds: + if 'gridded' in v: + equils[v]=drifts[v]SASU->ND + - checks for spinup stability periodically, resubmitting until stability, then proceeding to the next segment + - I tried to prototype some config files to make this a bit more user friendly and easy to update + - It's currently configured to run 2deg CLM60Bgc global, with loose spinup criteria + +3. old_spinup + - safe to ignore for now + - I needed this for stuff I'm doing presently, and can eventually make this more like sasu_spinup if desired + + +## Generic tether information + +### Basic tether call sequence: +1. run the commands specified in commands.txt +2. submit the case segment specified in case.txt +3. call tether.sh to presubmit your next case segment + +### tether.sh syntax + - ```./tether.sh $WDIR commands.txt $template``` + +### tether.sh tips + - I prefer to write a stub shell script with a PBS header to run tether, in lieu of calling it from the command line, because: + 1. I am often running scripts that are not well-suited to the login nodes + 2. I can automatically pipe stdout and stderr to an appropriately named log file + - As such, I will create a file: **segment001.job**, which can be a one-liner: + - ```$TDIR/tether.sh $WDIR commands.txt $template``` + - where ```$TDIR``` provides the full path to tether.sh + - with the addition of some standard PBS magic in the header + - see tether/example1/segment001.job + - then to run tether, I will opt for: + - ```qsub segment001.job``` + +### $WDIR + - the working directory within which all your cases and log files will exist + +### commands.txt + - a text file that will be run as bash commands line by line + - example: +``` +./part1.sh +``` + +### commands.txt tips + - commands.txt should exist within ```$WDIR``` + - ```./``` refers to ```$WDIR```, any other directories should utilize full path + - my intuition generally is to call a single script here, but I could see how eventually we may have smaller reusable scripts that are called in sequence, so commands.txt can have as many lines as needed + - within the commands, you should create, setup, build, and customize whatever case you are planning to submit + - somewhere within the commands, you need to write case.txt to indicate which case will be submitted in the current call to tether + - all cases should be created in ```$WDIR``` + - you should **NOT** call ```case.submit``` within your commands, tether.sh is responsible for case submission + - within the final command, you need to rewrite commands.txt for your next segment + - e.g. the last line of part1.sh is: ```echo "./part2.sh" > commands.txt``` + - if there are no future segments, ```rm commands.txt``` will signal tether to exit after the current job submission + - this is pretty important, because as currently written it's pretty easy to accidentally submit an infinite job sequence to the queue (**not advised**) + +### $template + - a stub shell script with PBS magic that is used to presubmit the next tethered segment + - see ```tether/derecho.template``` for an example + - you should only need to edit the project code within this template + - this will end up looking very similar to segment001.job, apart from an extra PBS command with the ```afterok``` flag + +### what it might look like to configure a tethered sequence: + 1. mkdir $WDIR + 2. write the necessary case configuration scripts + - taking care to include the tethering step of editing commands.txt + - and writing the casename to case.txt + 3. write the initial commands.txt + - typically just a one-liner, e.g.: ./part1.sh + 4. prepare segment001.job + 5. qsub segment001.job + +### what it looks like when tether is running: + - if you run ```qstat -u $USER```, you'll probably see segment001 queued or running + - as segment001 is running it will write log files with stdout and stderr in ```$WDIR``` + - it will also prepare segment002.job, which you can inspect but won't be very interesting + - more interesting would be ```cat commands.txt``` + - this should eventually update to reflect the commands required for your second segment + - then after segment001 is finished, if you rerun ```qstat -u $USER``` you may see something like: + + +| Job id | Name | User | Time Use | S | Queue +| :------- | :------: | :------: | :------: |:------: |-------: | +| 2542575.desched1 | run.mycase | djk2120 | | Q | cpu +| 2542576.desched1 | st_archive.m | djk2120 | | H | cpu +| 2542577.desched1 | segment002 | djk2120 | | H | cpudev + + - The Q means queued, and the H's mean on hold. in this case job 2542576 is waiting for 2542575 to finish, and 2542577 is waiting for 2542576 to finish + - In this situation once mycase finishes running, and the short-term archiver completes, segment002.job will submit the next case in the tethered sequence + +### when things go wrong + - you may end up needing to frantically qdel + - e.g. ```qdel 2542577``` + - useful but very committing is: + - ```qselect -u $USER | xargs qdel``` + - which will kill all your derecho jobs + - alternatively you can pipe to tail and head as appropriate + - ```qselect -u $USER | head -n 5 | tail -n 3``` etc. + - until you isolate the set of jobs that you want to delete + - and then call: + - ```qselect -u $USER | head -n 5 | tail -n 3 | xargs qdel``` + + +### restarting tether, after a debugging fix + - clean house as needed, e.g. you may need to rm -r mycase and rm -r /glade/derecho/scratch/USER/mycase + - edit commands.txt to reference the script you want to start with + - edit case.txt to point to the case that will be submitted + - remove the afterok line from the last segment0XX.job + - qsub segment0XX.job + diff --git a/tools/tether/derecho.template b/tools/tether/derecho.template new file mode 100644 index 0000000000..7bab2fc64b --- /dev/null +++ b/tools/tether/derecho.template @@ -0,0 +1,13 @@ +#!/bin/bash +#PBS -N jobname +#PBS -q develop +#PBS -l walltime=1:00:00 +#PBS -A P93300041 +#PBS -j oe +#PBS -k eod +#PBS -l select=1:ncpus=1 +#PBS -W depend=afterok:jobid +# --------------------------------------------------------- + +tdir="/glade/u/home/djk2120/vp/scripts/" +$tdir"tether.sh" wdir commands template diff --git a/tools/tether/example1/.pre_config b/tools/tether/example1/.pre_config new file mode 100644 index 0000000000..91e366cd38 --- /dev/null +++ b/tools/tether/example1/.pre_config @@ -0,0 +1,20 @@ +[user] +USER=ustr +PROJECT=UNSET + +[ctsm tag] +TAG=tagstr + +[key directories] +SDIR=pwd +TDIR=UNSET +WDIR=UNSET +CDIR=UNSET + +[case specifications] +PI_COMPSET=UNSET +GRID=UNSET + +[case names] +CASE1=UNSET +CASE2=UNSET diff --git a/tools/tether/example1/README.txt b/tools/tether/example1/README.txt new file mode 100644 index 0000000000..881a7356e4 --- /dev/null +++ b/tools/tether/example1/README.txt @@ -0,0 +1,9 @@ +This is a simple demonstration example of a two-segment tethered simulation. + +This will run a 1-month 4x5deg CLM-SP simulation, then spawn a hybrid case that picks up and continues that for another month. + +Steps to make this work for you: +1) ./create_config.sh +2) ./setup_config.sh +3) cd to tethered_sims/c2025xxxx directory +4) qsub segment001.job diff --git a/tools/tether/example1/commands.txt b/tools/tether/example1/commands.txt new file mode 100644 index 0000000000..bad3a4a007 --- /dev/null +++ b/tools/tether/example1/commands.txt @@ -0,0 +1 @@ +bash part1.sh diff --git a/tools/tether/example1/create_config.sh b/tools/tether/example1/create_config.sh new file mode 100755 index 0000000000..768869e4a0 --- /dev/null +++ b/tools/tether/example1/create_config.sh @@ -0,0 +1,9 @@ + +TAG=$(git describe --tags | cut -d- -f1) +NLDIR=$(pwd)"/namelists" + + +sed "s/ustr/"$USER"/g" .pre_config > example1.config +sed -i "s:pwd:"$(pwd)":g" example1.config +sed -i "s/tagstr/"$TAG"/g" example1.config +sed -i "s:nldir:"$NLDIR":g" example1.config diff --git a/tools/tether/example1/derecho.template b/tools/tether/example1/derecho.template new file mode 100644 index 0000000000..7700f79d72 --- /dev/null +++ b/tools/tether/example1/derecho.template @@ -0,0 +1,12 @@ +#!/bin/bash +#PBS -N jobname +#PBS -q develop +#PBS -l walltime=1:00:00 +#PBS -A P93300041 +#PBS -k eod +#PBS -l select=1:ncpus=1 +#PBS -W depend=afterok:jobid +# --------------------------------------------------------- + +tdir="/glade/work/djk2120/ctsm_tether/tools/tether/" +$tdir"tether.sh" wdir commands template diff --git a/tools/tether/example1/part1.sh b/tools/tether/example1/part1.sh new file mode 100755 index 0000000000..4438b85a3f --- /dev/null +++ b/tools/tether/example1/part1.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# function to abort tether when handling errors +kill_tether () { + cd $WDIR + rm commands.txt + exit 1 +} + + +# import a bunch of variables from config file +# e.g. $WDIR $PI_COMPSET +config=$1 +source <(grep = $1) + + +# create case +caseroot=$WDIR"/"$CASE1 +cd $CDIR +./create_newcase --case $caseroot --compset $PI_COMPSET --res $GRID --project $PROJECT --mach derecho --run-unsupported + + +# setup case +cd $caseroot +./case.setup + + +# some xml changes +./xmlchange STOP_N=1 +./xmlchange STOP_OPTION=nmonths +./xmlchange JOB_PRIORITY=premium +./xmlchange JOB_WALLCLOCK_TIME="01:00:00" + +#build the case (DO NOT SUBMIT!!!) +./case.build + + +# tether commands +cd $WDIR +echo "./part2.sh example1.config" > commands.txt +echo $CASE1 > case.txt + diff --git a/tools/tether/example1/part2.sh b/tools/tether/example1/part2.sh new file mode 100755 index 0000000000..8c0cf349a5 --- /dev/null +++ b/tools/tether/example1/part2.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# function to abort tether when handling errors +kill_tether () { + cd $WDIR + rm commands.txt + exit 1 +} + + +# import a bunch of variables from config file +# e.g. $WDIR $PI_COMPSET +config=$1 +source <(grep = $1) + + +# create case +caseroot=$WDIR"/"$CASE2 +cd $CDIR +./create_newcase --case $caseroot --compset $PI_COMPSET --res $GRID --project $PROJECT --mach derecho --run-unsupported + + +# setup case +cd $caseroot +./case.setup + + +# finding the latest restart from previous case +# this code is likely very brittle +ARCHIVE="/glade/derecho/scratch/"$USER"/archive" +REFCASE=$CASE1 +REFREST=$ARCHIVE"/"$REFCASE"/rest" +last_date=$(ls $REFREST | tail -n1) +REFDIR=$REFREST"/"$last_date +REFDATE=${last_date%-*} + + +# some xml changes +./xmlchange RUN_TYPE=hybrid +./xmlchange STOP_N=1 +./xmlchange STOP_OPTION=nmonths +./xmlchange JOB_PRIORITY=premium +./xmlchange JOB_WALLCLOCK_TIME="01:00:00" +./xmlchange RUN_REFCASE=$REFCASE +./xmlchange GET_REFCASE="True" +./xmlchange RUN_REFDIR=$REFDIR +./xmlchange RUN_REFDATE=$REFDATE + + +#build the case (DO NOT SUBMIT!!!) +./case.build + + +# tether commands +cd $WDIR +rm commands.txt +echo $CASE2 > case.txt + diff --git a/tools/tether/example1/setup_config.py b/tools/tether/example1/setup_config.py new file mode 100644 index 0000000000..4d640e2a20 --- /dev/null +++ b/tools/tether/example1/setup_config.py @@ -0,0 +1,105 @@ +import os +from datetime import date +import sys + + +def get_x(x, lines): + strips=["'", '"'] + for line in lines: + if x in line: + xout = line.split('=')[1] + for s in strips: + if s in xout: + xout = xout.replace(s, '') + return xout + + +def get_tdir(pwd): + tools = '/tools/' + if tools in pwd: + base = pwd.split(tools)[0] + tdir = base+tools+'tether' + else: + tdir = 'TETHER_DIR_NOT_FOUND' + return tdir + + +def find_cime(pwd,user): + ''' search upwards from pwd to find ./cime/scripts ''' + go = True + thisdir = pwd + while go: + ld = os.listdir(thisdir) + if 'cime' in ld: + cime_dir = thisdir + '/cime/scripts' + break + else: + last = os.path.basename(thisdir) + if last == user: + print('cannot find CIME') + cime_dir = 'CIME/SCRIPTS_DIR_NOT_FOUND' + break + else: + nx = len(last) + 1 + thisdir = thisdir[:-nx] + if not os.path.isdir(thisdir): + print('cannot find CIME') + cime_dir = 'CIME/SCRIPTS_DIR_NOT_FOUND' + break + return cime_dir + + +def get_defaults(lines): + user = get_x('USER', lines) + pwd = get_x('SDIR', lines) + tag = get_x('TAG',lines).replace('.','') + tdir = get_tdir(pwd) + dstr = 'c'+date.today().strftime('%Y%m%d') + wdir = '/glade/u/home/'+user+'/tethered_sims/'+dstr + cdir = find_cime(pwd,user) + defaults = {'PI_COMPSET': 'I1850Clm60Sp', + 'GRID': 'f45_g37', + 'PROJECT': 'P93300041'} + + for v in ['PI_COMPSET','GRID']: + x = get_x('PI_COMPSET',lines) + if x!='UNSET': + defaults[v]=x + + dirs = {'TDIR': tdir, 'WDIR': wdir, 'CDIR': cdir} + for k in dirs: + defaults[k] = dirs[k] + + compset = defaults['PI_COMPSET'] + grid = defaults['GRID'] + for suff in ['1', '2']: + defaults['CASE' + suff] = '{}.{}.{}.{}'.format(tag, compset, grid, suff) + + return defaults + + +def write_new_file(lines, defaults): + with open('config.tmp', 'w') as f: + for line in lines: + if '=' in line: + k, v = line.split('=') + if v == 'UNSET': + if k in defaults: + f.write(line.replace(v,defaults[k]) + '\n') + else: + f.write(line + '\n') + else: + f.write(line + '\n') + else: + f.write(line + '\n') + + +def main(): + with open('example1.config', 'r') as f: + lines = [line.rstrip() for line in f] + defaults = get_defaults(lines) + write_new_file(lines, defaults) + + +if __name__ == '__main__': + main() diff --git a/tools/tether/example1/setup_config.sh b/tools/tether/example1/setup_config.sh new file mode 100755 index 0000000000..f46f41a7aa --- /dev/null +++ b/tools/tether/example1/setup_config.sh @@ -0,0 +1,44 @@ + + +# apply defaults to any unset config fields +python setup_config.py +rm example1.config # old unset file + +# source variables from config +source <(grep = config.tmp) + +# mk WDIR and mv config to WDIR +mkdir -p $WDIR +mv config.tmp $WDIR"/example1.config" +echo "config file now located at: "$WDIR"/example1.config" + + +# make sure scratch directory exists +SD='/glade/derecho/scratch/'$USER +if [ ! -d $SD ]; then + echo $SD" is not a directory, $USER may be set badly" + echo "examine "$WDIR"/spinup.config" + exit 1 +fi + + +# cp scripts to WDIR +cp part*.sh $WDIR + + +# configure the PBS template +sed "s:TDIR:"$TDIR":g" derecho.template > $WDIR"/derecho.template" +sed -i "s:project:"$PROJECT":g" $WDIR"/derecho.template" + + +# create the initial qsub job +sed '/afterok/d' $WDIR"/derecho.template" > $WDIR"/segment001.job" +sed -i 's/jobname/segment001/g' $WDIR"/segment001.job" +sed -i 's:wdir:'$WDIR':g' $WDIR"/segment001.job" +sed -i 's/commands/commands.txt/g' $WDIR"/segment001.job" +sed -i 's/template/derecho.template/g' $WDIR"/segment001.job" + + +# create the initial commands.txt +echo "./part1.sh example1.config" > $WDIR"/commands.txt" + diff --git a/tools/tether/example1/spinup.config b/tools/tether/example1/spinup.config new file mode 100644 index 0000000000..80ad52791e --- /dev/null +++ b/tools/tether/example1/spinup.config @@ -0,0 +1,20 @@ +[user] +USER=djk2120 +PROJECT=UNSET + +[ctsm tag] +TAG=ctsm5.3.072 + +[key directories] +SDIR=/glade/work/djk2120/ctsm_tether/tools/tether/example1 +TDIR=UNSET +WDIR=UNSET +CDIR=UNSET + +[case specifications] +PI_COMPSET=UNSET +GRID=UNSET + +[case names] +CASE1=UNSET +CASE2=UNSET diff --git a/tools/tether/old_spinup/AD.yml b/tools/tether/old_spinup/AD.yml new file mode 100644 index 0000000000..ddb3d7d690 --- /dev/null +++ b/tools/tether/old_spinup/AD.yml @@ -0,0 +1,45 @@ +# detailing the spinup case and stability thresholds +case: I1850Clm50Bgc.CPLHIST.default.AD +hist_dir: /glade/derecho/scratch/djk2120/archive/I1850Clm50Bgc.CPLHIST.default.AD/lnd/hist +cycle_years: 20 +pct_landarea: 3 +thresholds: + TOTECOSYSC: 0.02 + TOTVEGC: 0.02 + GPP: 0.02 + TLAI: 0.02 + TEC_gridded: 1 + TVC_gridded: 1 + TSC_gridded: 1 +cfs: + TOTECOSYSC: 1e-9 + TOTSOMC: 1e-9 + TOTVEGC: 1e-9 + GPP: + - 1e-9 + - 86400 + - 365 + TLAI: 1/lasum + TWS: 1/lasum + TEC_gridded: 1 + TSC_gridded: 1 + TVC_gridded: 1 +units: + TOTECOSYSC: PgC + TOTSOMC: PgC + TOTVEGC: PgC + GPP: PgC + TLAI: m2/m2 + TEC_gridded: gC/m2 + TSC_gridded: gC/m2 + TVC_gridded: gC/m2 + TWS: mm +data_vars: + - TLAI + - GPP + - TOTECOSYSC + - TOTVEGC + - TOTSOMC + - TWS + + diff --git a/tools/tether/old_spinup/PAD.yml b/tools/tether/old_spinup/PAD.yml new file mode 100644 index 0000000000..ebf8c68ba3 --- /dev/null +++ b/tools/tether/old_spinup/PAD.yml @@ -0,0 +1,45 @@ +# detailing the spinup case and stability thresholds +case: I1850Clm50Bgc.CPLHIST.default.pAD +hist_dir: /glade/derecho/scratch/djk2120/archive/I1850Clm50Bgc.CPLHIST.default.pAD/lnd/hist +cycle_years: 20 +pct_landarea: 3 +thresholds: + TOTECOSYSC: 0.02 + TOTVEGC: 0.02 + GPP: 0.02 + TLAI: 0.02 + TEC_gridded: 1 + TVC_gridded: 1 + TSC_gridded: 1 +cfs: + TOTECOSYSC: 1e-9 + TOTSOMC: 1e-9 + TOTVEGC: 1e-9 + GPP: + - 1e-9 + - 86400 + - 365 + TLAI: 1/lasum + TWS: 1/lasum + TEC_gridded: 1 + TSC_gridded: 1 + TVC_gridded: 1 +units: + TOTECOSYSC: PgC + TOTSOMC: PgC + TOTVEGC: PgC + GPP: PgC + TLAI: m2/m2 + TEC_gridded: gC/m2 + TSC_gridded: gC/m2 + TVC_gridded: gC/m2 + TWS: mm +data_vars: + - TLAI + - GPP + - TOTECOSYSC + - TOTVEGC + - TOTSOMC + - TWS + + diff --git a/tools/tether/old_spinup/README.txt b/tools/tether/old_spinup/README.txt new file mode 100644 index 0000000000..04446bb79f --- /dev/null +++ b/tools/tether/old_spinup/README.txt @@ -0,0 +1,10 @@ +This is a more complicated sequence. And I didn't make this as friendly as example 1. It would be interesting to see how we could reconfigure this use scenario into an actual set of tools. My plan is to write this a bit more generically with a newer tag, using the AD/SASU/pSASU/transient sequence. This current example is for actual work that I'm doing right now. + +It feels like we could pull out things like CIMEROOT COMPSET GRID etc into a config file, and then have a utility that would cp the shell scripts and yamls into a user-specified WDIR, and give some instructions on how to proceed and the various options for customization. We could also add a line in the script for user_mods so user hopefully wouldn't need to edit the main script very often. + +The general sequence is: + - Run AD for 100 years, from an existing restart + - Test for spinup stability and run 20 more years, as needed + - Run postAD for 100 years, picking up from the AD case + - Test for spinup stability and run 20 more years, as needed + - Run a transient case from 1880-1949, picking up from the postAD case diff --git a/tools/tether/old_spinup/commands.txt b/tools/tether/old_spinup/commands.txt new file mode 100644 index 0000000000..db0c0296b1 --- /dev/null +++ b/tools/tether/old_spinup/commands.txt @@ -0,0 +1 @@ +./setupAD.sh diff --git a/tools/tether/old_spinup/derecho.template b/tools/tether/old_spinup/derecho.template new file mode 100644 index 0000000000..a1a3565ebd --- /dev/null +++ b/tools/tether/old_spinup/derecho.template @@ -0,0 +1,14 @@ +#!/bin/bash +#PBS -N jobname +#PBS -q develop +#PBS -l walltime=2:00:00 +#PBS -A P93300041 +#PBS -k eod +#PBS -l select=1:ncpus=5 +#PBS -W depend=afterok:jobid +# --------------------------------------------------------- + +source ~/.bashrc +conda activate ppe-py +tdir="/glade/work/djk2120/ctsm_tether/tools/tether/" +$tdir"tether.sh" wdir commands template diff --git a/tools/tether/old_spinup/mirror_wdir.sh b/tools/tether/old_spinup/mirror_wdir.sh new file mode 100644 index 0000000000..9dea84d70a --- /dev/null +++ b/tools/tether/old_spinup/mirror_wdir.sh @@ -0,0 +1,5 @@ +cp /glade/u/home/djk2120/vp/sims/ICLM50Bgc.CPLHIST.default/*.sh ./ +cp /glade/u/home/djk2120/vp/sims/ICLM50Bgc.CPLHIST.default/*.yml ./ +cp /glade/u/home/djk2120/vp/sims/ICLM50Bgc.CPLHIST.default/segment001.job ./ +cp /glade/u/home/djk2120/vp/sims/ICLM50Bgc.CPLHIST.default/derecho.template ./ +echo "./setupAD.sh" >commands.txt diff --git a/tools/tether/old_spinup/segment001.job b/tools/tether/old_spinup/segment001.job new file mode 100644 index 0000000000..e6324a769d --- /dev/null +++ b/tools/tether/old_spinup/segment001.job @@ -0,0 +1,19 @@ +#!/bin/bash +#PBS -N segment001 +#PBS -q develop +#PBS -l walltime=1:00:00 +#PBS -A P93300041 +#PBS -k eod +#PBS -l select=1:ncpus=5 + +# --------------------------------------------------------- + +source ~/.bashrc +conda activate ppe-py + + +tdir="/glade/work/djk2120/ctsm_tether/tools/tether/" +wdir="/glade/u/home/djk2120/vp/sims/ICLM50Bgc.CPLHIST.default/" +$tdir"tether.sh" $wdir commands.txt derecho.template + + diff --git a/tools/tether/old_spinup/setupAD.sh b/tools/tether/old_spinup/setupAD.sh new file mode 100755 index 0000000000..2fcc07438f --- /dev/null +++ b/tools/tether/old_spinup/setupAD.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +WDIR="/glade/u/home/djk2120/vp/sims/ICLM50Bgc.CPLHIST.default/" +NAMELISTS="/glade/u/home/djk2120/vp/scripts/namelists/" +CASENAME="I1850Clm50Bgc.CPLHIST.default.AD" +PROJECT=P93300041 + +COMPSET=1850_DATM%CPLHIST_CLM50%BGC-CROP_SICE_SOCN_MOSART_CISM2%NOEVOLVE_SWAV +GRID=f09_f09_mg17 +CESMROOT="/glade/work/djk2120/cesm2.1.5/" + +curdir=$(pwd) +CASEROOTBASE=$WDIR +caseroot=$CASEROOTBASE$CASENAME +echo $caseroot + +cd $CESMROOT/cime/scripts +./create_newcase --case $caseroot --compset $COMPSET --res $GRID --project $PROJECT --mach derecho --run-unsupported +cd $caseroot +./case.setup + +./xmlchange RUN_TYPE=hybrid +./xmlchange PROJECT=$PROJECT +./xmlchange JOB_PRIORITY="premium" +./xmlchange RUN_STARTDATE="0001-01-01" +./xmlchange RUN_REFCASE="b.e21.BHISTcmip6.f09_g17.LE2-1001.001" +./xmlchange RUN_REFDIR="/glade/campaign/cesm/collections/CESM2-LE/restarts/b.e21.BHISTcmip6.f09_g17.LE2-1001.001/rest/1880-01-01-00000" +./xmlchange RUN_REFDATE="1880-01-01" +./xmlchange GET_REFCASE="True" +./xmlchange DATM_MODE="CPLHIST" +./xmlchange DATM_PRESAERO="cplhist" +./xmlchange DATM_TOPO="cplhist" +./xmlchange DATM_CPLHIST_CASE="f.e21.FHIST_BGC.f09_f09.ersstv5.cplhist" +./xmlchange DATM_CPLHIST_DIR="/glade/derecho/scratch/djk2120/archive/f.e21.FHIST_BGC.f09_f09.ersstv5.cplhist/cpl/proc/" +./xmlchange DATM_CPLHIST_YR_ALIGN="1880" +./xmlchange DATM_CPLHIST_YR_START="1880" +./xmlchange DATM_CPLHIST_YR_END="1899" +./xmlchange STOP_N=100,STOP_OPTION=nyears +./xmlchange CLM_ACCELERATED_SPINUP=on + + +cp $NAMELISTS"AD/"* ./ +./case.build + +cd $WDIR +echo $CASENAME>case.txt +echo "./stabAD.sh">commands.txt diff --git a/tools/tether/old_spinup/setupHIST.sh b/tools/tether/old_spinup/setupHIST.sh new file mode 100755 index 0000000000..89ad459cf4 --- /dev/null +++ b/tools/tether/old_spinup/setupHIST.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +WDIR="/glade/u/home/djk2120/vp/sims/ICLM50Bgc.CPLHIST.default/" +NAMELISTS="/glade/u/home/djk2120/vp/scripts/namelists/" +CASENAME="IHistClm50Bgc.CPLHIST.default" +REFCASE="I1850Clm50Bgc.CPLHIST.default.pAD" +ARCHIVE="/glade/derecho/scratch/djk2120/archive/" +PROJECT=P93300041 + +COMPSET=HIST_DATM%CPLHIST_CLM50%BGC-CROP_SICE_SOCN_MOSART_CISM2%NOEVOLVE_SWAV +GRID=f09_f09_mg17 +CESMROOT="/glade/work/djk2120/cesm2.1.5/" + +curdir=$(pwd) +CASEROOTBASE=$WDIR +caseroot=$CASEROOTBASE$CASENAME +echo $caseroot + +cd $CESMROOT/cime/scripts +./create_newcase --case $caseroot --compset $COMPSET --res $GRID --project $PROJECT --mach derecho --run-unsupported +cd $caseroot +./case.setup + +./xmlchange RUN_TYPE=hybrid +./xmlchange PROJECT=$PROJECT +./xmlchange RUN_STARTDATE="1880-01-01" +./xmlchange STOP_OPTION="nyears" +./xmlchange STOP_N=70 +./xmlchange JOB_WALLCLOCK_TIME="12:00:00" + + +# finding the latest restart from AD +# this code is likely very brittle +./xmlchange RUN_REFCASE=$REFCASE +./xmlchange GET_REFCASE="True" +REFREST=$ARCHIVE$REFCASE"/rest" +last_date=$(ls $REFREST | tail -n1) +REFDIR=$REFREST"/"$last_date +REFDATE=${last_date%-*} +./xmlchange RUN_REFDIR=$REFDIR +./xmlchange RUN_REFDATE=$REFDATE + + +#./xmlchange RUN_REFCASE="b.e21.BHISTcmip6.f09_g17.LE2-1001.001" +#./xmlchange RUN_REFDIR="/glade/campaign/cesm/collections/CESM2-LE/restarts/b.e21.BHISTcmip6.f09_g17.LE2-1001.001/rest/1880-01-01-00000" +#./xmlchange RUN_REFDATE="1880-01-01" +#./xmlchange GET_REFCASE="True" + + + +./xmlchange DATM_MODE="CPLHIST" +./xmlchange DATM_PRESAERO="cplhist" +./xmlchange DATM_TOPO="cplhist" +./xmlchange DATM_CPLHIST_CASE="f.e21.FHIST_BGC.f09_f09.ersstv5.cplhist" +./xmlchange DATM_CPLHIST_DIR="/glade/derecho/scratch/djk2120/archive/f.e21.FHIST_BGC.f09_f09.ersstv5.cplhist/cpl/proc/" +./xmlchange DATM_CPLHIST_YR_ALIGN="1880" +./xmlchange DATM_CPLHIST_YR_START="1880" +./xmlchange DATM_CPLHIST_YR_END="1949" + + + + +cp $NAMELISTS"IHIST/*" . +./case.build + +cd $WDIR +echo $CASENAME>case.txt +rm commands.txt + + + + diff --git a/tools/tether/old_spinup/setupPAD.sh b/tools/tether/old_spinup/setupPAD.sh new file mode 100755 index 0000000000..b6a280e433 --- /dev/null +++ b/tools/tether/old_spinup/setupPAD.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +WDIR="/glade/u/home/djk2120/vp/sims/ICLM50Bgc.CPLHIST.default/" +NAMELISTS="/glade/u/home/djk2120/vp/scripts/namelists/" +CLONENAME="I1850Clm50Bgc.CPLHIST.default.AD" +CASENAME="I1850Clm50Bgc.CPLHIST.default.pAD" +PROJECT=P93300041 +ARCHIVE="/glade/derecho/scratch/djk2120/archive/" +CESMROOT="/glade/work/djk2120/cesm2.1.5/" + +curdir=$(pwd) +CASEROOTBASE=$WDIR +cloneroot=$CASEROOTBASE$CLONENAME +caseroot=$CASEROOTBASE$CASENAME +echo $caseroot + + +cd $CESMROOT/cime/scripts +./create_clone --case $caseroot --clone $cloneroot +cd $caseroot +./case.setup + + +./xmlchange RUN_TYPE=hybrid +./xmlchange CONTINUE_RUN=False +./xmlchange PROJECT=$PROJECT +./xmlchange JOB_PRIORITY="regular" +./xmlchange JOB_WALLCLOCK_TIME="12:00:00" +./xmlchange RUN_STARTDATE="0001-01-01" +./xmlchange STOP_N=100,STOP_OPTION=nyears +./xmlchange CLM_ACCELERATED_SPINUP=off + +# finding the latest restart from AD +# this code is likely very brittle +./xmlchange RUN_REFCASE=$CLONENAME +./xmlchange GET_REFCASE="True" +REFREST=$ARCHIVE$CLONENAME"/rest" +last_date=$(ls $REFREST | tail -n1) +REFDIR=$REFREST"/"$last_date +REFDATE=${last_date%-*} +./xmlchange RUN_REFDIR=$REFDIR +./xmlchange RUN_REFDATE=$REFDATE + + +cp $NAMELISTS"pAD/"* ./ +./case.build + +cd $WDIR +echo $CASENAME>case.txt +echo "./stabPAD.sh">commands.txt + + + + + diff --git a/tools/tether/old_spinup/spinup_stability.py b/tools/tether/old_spinup/spinup_stability.py new file mode 100644 index 0000000000..92c6f5bca2 --- /dev/null +++ b/tools/tether/old_spinup/spinup_stability.py @@ -0,0 +1,203 @@ +import xarray as xr +import numpy as np +import matplotlib.pyplot as plt +import glob +import sys +import yaml + +def parse_cf(cf,lasum): + if type(cf)==str: + cf=parse_cfstr(cf,lasum) + elif type(cf)==list: + cftot=1 + for f in cf: + cftot*=parse_cf(f,lasum) + cf=cftot + return cf + +def parse_cfstr(cf,lasum): + if cf=='1/lasum': + return 1/lasum + else: + return float(cf) + +def parse_cfs(cfs,lasum): + for v in cfs: + cfs[v]=parse_cf(cfs[v],lasum) + return cfs + +def check_freq(tmp): + #infer if history is monthly or annual + nsecs_per_day=24*60*60*1e9 + dt=(tmp.time_bounds.isel(time=-1,hist_interval=1)- + tmp.time_bounds.isel(time=-1,hist_interval=0))/nsecs_per_day + if dt<40: + freq='monthly' + else: + freq='annual' + return freq + +def get_ds(files,freq,dvs): + #open dataset and process to an orderly annual dataset + def pp(ds): + return ds[dvs] + if freq=='monthly': + t=xr.DataArray(xr.date_range('1999',freq='MS',periods=12),dims='time') + dpm=xr.DataArray(t['time.daysinmonth'].values,dims='time') + #splitting files up by year + #too slow to have one big open_mfdataset + fsets=[files[i:i + 12] for i in range(0, len(files), 12)] + dsets=[] + for i,fset in enumerate(fsets): + ds=xr.open_mfdataset(fset,combine='by_coords',preprocess=pp, + decode_timedelta=False) + dsets.append((dpm*ds).sum(dim='time')/365) + ds=xr.concat(dsets,dim='time') + else: + ds=xr.open_mfdataset(files,combine='by_coords',preprocess=pp, + decode_timedelta=False) + ds=ds.isel(time=slice(1,len(ds.time))) + return ds + +def plot_drifts(xs,thiscase,ncycles,nyears,units,thresholds,drifts,equils,tpct,la,lasum): + plt.figure(figsize=[16,12]) + for j,v in enumerate(xs): + plt.subplot(3,3,j+1) + if 'gridded' not in v: + for i in range(ncycles): + plt.plot(range(nyears),xs[v].isel(time=np.arange(nyears)+i*nyears),label='cycle_'+str(i).zfill(3)) + plt.ylabel(v+' ['+units[v]+']') + if j==5: + plt.legend() + dstr=str(np.round(drifts[v],3)) + tstr='drift='+dstr+units[v]+'/yr' + gl='><' + if v in thresholds: + tstr+=gl[int(equils[v])] + tstr+=str(thresholds[v]) + else: + tstr+=' [not evaluated]' + else: + x=xs[v] + if v in thresholds: + thresh=thresholds[v] + else: + thresh=1 + diseq=abs(x-x.shift(time=nyears))/nyears>thresh + pct=100*(la*diseq).sum(dim=['lat','lon'])/lasum + ix=np.arange(len(x.time))>=nyears + pct.where(ix).plot() + ystr=('abs($\Delta$'+v.split('_')[0]+')>'+ + str(thresh)+units[v]+'/yr'+'\n[% landarea]') + plt.ylabel(ystr) + plt.ylim([0,100]) + plt.xlabel('') + dstr=str(np.round(drifts[v],1)) + tstr=dstr+'%' + if v in equils: + tstr+=gl[int(equils[v])] + tstr+=str(tpct) + else: + tstr+=' [not evaluated]' + if v in equils: + if not equils[v]: + tstr='FAILED: '+tstr + + plt.title(tstr) + plt.subplots_adjust(hspace=0.2,wspace=0.3) + plt.savefig(thiscase+'.png',dpi=300,bbox_inches='tight') + + + +def main(): + cfile = sys.argv[1] + config = yaml.safe_load(open(cfile)) + thiscase = config['case'] + d = config['hist_dir'] + files = sorted(glob.glob(d+'/*.h0.*')) + if len(files) < 1: + failed = True + else: + #import config and parse conversion factors + tmp = xr.open_dataset(files[0],decode_timedelta=True) + la = tmp.area*tmp.landfrac + lasum = la.sum().values + thresholds=config['thresholds'] + units=config['units'] + cfs=parse_cfs(config['cfs'],lasum) + + freq=check_freq(tmp) + dvs=config['data_vars'] + ds=get_ds(files,freq,dvs) + + #set up year variables + nyears=config['cycle_years'] #years per met forcing cycle + ncycles=int(len(ds.time)/nyears) + y2=nyears*ncycles + y1=y2-nyears + y0=y1-nyears + + #abbrev dict + stocks={'TEC':'TOTECOSYSC', + 'TSC':'TOTSOMC', + 'TVC':'TOTVEGC'} + + #evaluate global drifts + drifts={} + xs={} + for v in cfs: + cf=cfs[v] + if 'gridded' in v: + vlong=stocks[v.split('_')[0]] + x=cf*ds[vlong] + else: + x=cf*(la*ds[v]).sum(dim=['lat','lon']) + xs[v]=x + drift=abs(x.isel(time=slice(y1,y2)).mean(dim='time')- + x.isel(time=slice(y0,y1)).mean(dim='time'))/nyears + if 'gridded' in v: + if v in thresholds: + thresh=thresholds[v] + else: + thresh=1 + pct=100*(la*(drift>thresh)).sum(dim=['lat','lon'])/lasum + drifts[v]=pct.values + else: + drifts[v]=drift.values + + equils={} + for v in thresholds: + if 'gridded' in v: + equils[v]=drifts[v]case.txt + echo "stabAD.sh">commands.txt +elif [[ "$status" == "0" ]]; then + echo "spinup appears sufficient" + ./setupPAD.sh + echo "stabPAD.sh">commands.txt +else + echo "something looks wrong, halting tether" + rm commands.txt +fi diff --git a/tools/tether/old_spinup/stabPAD.sh b/tools/tether/old_spinup/stabPAD.sh new file mode 100755 index 0000000000..e1de1d5a60 --- /dev/null +++ b/tools/tether/old_spinup/stabPAD.sh @@ -0,0 +1,27 @@ + + +SDIR="/glade/work/djk2120/ctsm_tether/tools/tether/old_spinup/" +WDIR="/glade/u/home/djk2120/vp/sims/" +case=$(case.txt + echo "stabPAD.sh">commands.txt +elif [[ "$status" == "0" ]]; then + echo "PAD spinup appears sufficient" + ./setupHIST.sh + rm commands.txt +else + echo "something looks wrong, halting tether" + rm commands.txt +fi diff --git a/tools/tether/sasu_spinup/.pre_config b/tools/tether/sasu_spinup/.pre_config new file mode 100644 index 0000000000..580c29b506 --- /dev/null +++ b/tools/tether/sasu_spinup/.pre_config @@ -0,0 +1,23 @@ +[user] +USER=ustr +PROJECT=UNSET +PWD=pwd + +[ctsm tag] +TAG=tagstr + +[key directories] +SDIR=UNSET +TDIR=UNSET +WDIR=UNSET +CDIR=UNSET + +[case specifications] +PI_COMPSET=UNSET +GRID=UNSET +NLDIR=nldir # specify top dir; needs subdirs AD SASU ND + +[case names] +CASE_AD=UNSET +CASE_SASU=UNSET +CASE_ND=UNSET diff --git a/tools/tether/sasu_spinup/AD.yml b/tools/tether/sasu_spinup/AD.yml new file mode 100644 index 0000000000..6c037afdf8 --- /dev/null +++ b/tools/tether/sasu_spinup/AD.yml @@ -0,0 +1,45 @@ +# detailing the spinup case and stability thresholds +case: CASE_AD +hist_dir: HIST_AD +cycle_years: 20 +pct_landarea: 3 +thresholds: +# TOTECOSYSC: 0.02 +# TOTVEGC: 0.02 + GPP: 0.1 +# TLAI: 0.02 +# TEC_gridded: 1 +# TVC_gridded: 1 +# TSC_gridded: 1 +cfs: + TOTECOSYSC: 1e-9 + TOTSOMC: 1e-9 + TOTVEGC: 1e-9 + GPP: + - 1e-9 + - 86400 + - 365 + TLAI: 1/lasum + TWS: 1/lasum + TEC_gridded: 1 + TSC_gridded: 1 + TVC_gridded: 1 +units: + TOTECOSYSC: PgC + TOTSOMC: PgC + TOTVEGC: PgC + GPP: PgC + TLAI: m2/m2 + TEC_gridded: gC/m2 + TSC_gridded: gC/m2 + TVC_gridded: gC/m2 + TWS: mm +data_vars: + - TLAI + - GPP + - TOTECOSYSC + - TOTVEGC + - TOTSOMC + - TWS + + diff --git a/tools/tether/sasu_spinup/ND.yml b/tools/tether/sasu_spinup/ND.yml new file mode 100644 index 0000000000..a88bb4c261 --- /dev/null +++ b/tools/tether/sasu_spinup/ND.yml @@ -0,0 +1,45 @@ +# detailing the spinup case and stability thresholds +case: CASE_ND +hist_dir: HIST_ND +cycle_years: 20 +pct_landarea: 3 +thresholds: +# TOTECOSYSC: 0.02 +# TOTVEGC: 0.02 + GPP: 0.1 +# TLAI: 0.02 +# TEC_gridded: 1 +# TVC_gridded: 1 +# TSC_gridded: 1 +cfs: + TOTECOSYSC: 1e-9 + TOTSOMC: 1e-9 + TOTVEGC: 1e-9 + GPP: + - 1e-9 + - 86400 + - 365 + TLAI: 1/lasum + TWS: 1/lasum + TEC_gridded: 1 + TSC_gridded: 1 + TVC_gridded: 1 +units: + TOTECOSYSC: PgC + TOTSOMC: PgC + TOTVEGC: PgC + GPP: PgC + TLAI: m2/m2 + TEC_gridded: gC/m2 + TSC_gridded: gC/m2 + TVC_gridded: gC/m2 + TWS: mm +data_vars: + - TLAI + - GPP + - TOTECOSYSC + - TOTVEGC + - TOTSOMC + - TWS + + diff --git a/tools/tether/sasu_spinup/README.txt b/tools/tether/sasu_spinup/README.txt new file mode 100644 index 0000000000..aa91cd67d5 --- /dev/null +++ b/tools/tether/sasu_spinup/README.txt @@ -0,0 +1,12 @@ +steps: + +1) ./create_config.sh +2) edit spinup.config a bit if you want, + - especially project, grid, and compset information + - setup_config will apply nice defaults for any unset variables +3) ./setup_config.sh +4) examine the resulting config file + - it will be located in a separate directory + - if you don't like the default names or directories, return to step 1 + - tinker and repeat as needed +5) proceed to tether parent directory and qsub segment001.job diff --git a/tools/tether/sasu_spinup/SASU.yml b/tools/tether/sasu_spinup/SASU.yml new file mode 100644 index 0000000000..ac1de0572c --- /dev/null +++ b/tools/tether/sasu_spinup/SASU.yml @@ -0,0 +1,45 @@ +# detailing the spinup case and stability thresholds +case: CASE_SASU +hist_dir: HIST_SASU +cycle_years: 20 +pct_landarea: 10 +thresholds: +# TOTECOSYSC: 0.02 +# TOTVEGC: 0.02 + GPP: 0.1 +# TLAI: 0.02 +# TEC_gridded: 1 +# TVC_gridded: 1 +# TSC_gridded: 1 +cfs: + TOTECOSYSC: 1e-9 + TOTSOMC: 1e-9 + TOTVEGC: 1e-9 + GPP: + - 1e-9 + - 86400 + - 365 + TLAI: 1/lasum + TWS: 1/lasum + TEC_gridded: 1 + TSC_gridded: 1 + TVC_gridded: 1 +units: + TOTECOSYSC: PgC + TOTSOMC: PgC + TOTVEGC: PgC + GPP: PgC + TLAI: m2/m2 + TEC_gridded: gC/m2 + TSC_gridded: gC/m2 + TVC_gridded: gC/m2 + TWS: mm +data_vars: + - TLAI + - GPP + - TOTECOSYSC + - TOTVEGC + - TOTSOMC + - TWS + + diff --git a/tools/tether/sasu_spinup/create_config.sh b/tools/tether/sasu_spinup/create_config.sh new file mode 100755 index 0000000000..761d962f49 --- /dev/null +++ b/tools/tether/sasu_spinup/create_config.sh @@ -0,0 +1,9 @@ + +TAG=$(git describe --tags | cut -d- -f1) +NLDIR=$(pwd)"/namelists" + + +sed "s/ustr/"$USER"/g" .pre_config > spinup.config +sed -i "s:pwd:"$(pwd)":g" spinup.config +sed -i "s/tagstr/"$TAG"/g" spinup.config +sed -i "s:nldir:"$NLDIR":g" spinup.config diff --git a/tools/tether/sasu_spinup/derecho.template b/tools/tether/sasu_spinup/derecho.template new file mode 100644 index 0000000000..3dd34936f6 --- /dev/null +++ b/tools/tether/sasu_spinup/derecho.template @@ -0,0 +1,18 @@ +#!/bin/bash +#PBS -N jobname +#PBS -q develop +#PBS -l walltime=2:00:00 +#PBS -A project +#PBS -k eod +#PBS -l select=1:ncpus=5 +#PBS -W depend=afterok:jobid +# --------------------------------------------------------- + +#source .bashrc +#conda activate ctsm_pylib + +module load conda +conda activate npl-2025b + +tdir=TDIR +$tdir"/tether.sh" wdir commands template diff --git a/tools/tether/sasu_spinup/namelists/AD/unset_user_nl_clm b/tools/tether/sasu_spinup/namelists/AD/unset_user_nl_clm new file mode 100644 index 0000000000..90e7baa682 --- /dev/null +++ b/tools/tether/sasu_spinup/namelists/AD/unset_user_nl_clm @@ -0,0 +1,6 @@ +! ---------------------------------------------------------------------------------- +! AD spinup namelist +! copied from nlpath/user_nl_clm +! ---------------------------------------------------------------------------------- + +reseed_dead_plants=.true. diff --git a/tools/tether/sasu_spinup/namelists/ND/unset_user_nl_clm b/tools/tether/sasu_spinup/namelists/ND/unset_user_nl_clm new file mode 100644 index 0000000000..0b2052201f --- /dev/null +++ b/tools/tether/sasu_spinup/namelists/ND/unset_user_nl_clm @@ -0,0 +1,12 @@ +! ---------------------------------------------------------------------------------- +! ND spinup namelist +! copied from nlpath/user_nl_clm +! ---------------------------------------------------------------------------------- + + hist_empty_htapes = .true. + hist_nhtfrq = -8760 + hist_mfilt = 10 + + hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS','H2OSNO','NFIX' + reseed_dead_plants=.false. + diff --git a/tools/tether/sasu_spinup/namelists/SASU/unset_user_nl_clm b/tools/tether/sasu_spinup/namelists/SASU/unset_user_nl_clm new file mode 100644 index 0000000000..77f96566f8 --- /dev/null +++ b/tools/tether/sasu_spinup/namelists/SASU/unset_user_nl_clm @@ -0,0 +1,12 @@ +! ---------------------------------------------------------------------------------- +! SASU spinup namelist +! copied from nlpath/user_nl_clm +! ---------------------------------------------------------------------------------- + + hist_empty_htapes = .true. + hist_nhtfrq = -8760 + hist_mfilt = 10 + + hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS','H2OSNO','NFIX' + reseed_dead_plants=.false. + diff --git a/tools/tether/sasu_spinup/setupAD.sh b/tools/tether/sasu_spinup/setupAD.sh new file mode 100755 index 0000000000..eec34ddf3c --- /dev/null +++ b/tools/tether/sasu_spinup/setupAD.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# function to abort tether when handling errors +kill_tether () { + cd $WDIR + rm commands.txt + exit 1 +} + + +# import a bunch of variables from config file +# e.g. $WDIR $PI_COMPSET +config=$1 +source <(grep = $1) + + +# create case +caseroot=$WDIR"/"$CASE_AD +cd $CDIR +./create_newcase --case $caseroot --compset $PI_COMPSET --res $GRID --project $PROJECT --mach derecho --run-unsupported + + +# setup case +cd $caseroot +./case.setup + + +# some xml changes +./xmlchange JOB_PRIORITY=premium +./xmlchange STOP_OPTION=nyears +./xmlchange STOP_N=80 +./xmlchange CLM_ACCELERATED_SPINUP=on +./xmlchange MOSART_MODE=NULL + +# cp in namelist +ad_namelists=$NLDIR"/AD" +if [ -d $ad_namelists ]; then + cd $ad_namelists + if [ -f user_nl_clm ]; then + cp user_nl_* $caseroot + else + echo "ERROR: no user_nl_clm in "$ad_namelists + kill_tether + fi +else + echo "ERROR: "$ad_namelists" is not a directory" + kill_tether +fi + + +# build case +# do not submit! tether.sh will submit +cd $caseroot +./case.build + + +# tether commands +cd $WDIR +echo "./stabAD.sh spinup.config" > commands.txt +echo $CASE_AD > case.txt diff --git a/tools/tether/sasu_spinup/setupND.sh b/tools/tether/sasu_spinup/setupND.sh new file mode 100755 index 0000000000..9cdb4f32ab --- /dev/null +++ b/tools/tether/sasu_spinup/setupND.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# function to abort tether when handling errors +kill_tether () { + cd $WDIR + rm commands.txt + exit 1 +} + + +# import a bunch of variables from config file +# e.g. $WDIR $PI_COMPSET +config=$1 +source <(grep = $config) + + +# create case +caseroot=$WDIR"/"$CASE_ND +cd $CDIR +./create_newcase --case $caseroot --compset $PI_COMPSET --res $GRID --project $PROJECT --mach derecho --run-unsupported + + +# setup case +cd $caseroot +./case.setup + + +# finding the latest restart from SASU +# this code is likely very brittle +ARCHIVE="/glade/derecho/scratch/"$USER"/archive" +REFCASE=$CASE_SASU +REFREST=$ARCHIVE"/"$REFCASE"/rest" +last_date=$(ls $REFREST | tail -n1) +REFDIR=$REFREST"/"$last_date +REFDATE=${last_date%-*} + + +# xml changes +./xmlchange JOB_PRIORITY=premium +./xmlchange STOP_OPTION=nyears +./xmlchange STOP_N=80 +./xmlchange RUN_TYPE=hybrid +./xmlchange RUN_REFCASE=$REFCASE +./xmlchange GET_REFCASE="True" +./xmlchange RUN_REFDIR=$REFDIR +./xmlchange RUN_REFDATE=$REFDATE +./xmlchange MOSART_MODE=NULL + +# cp in namelist +namelists=$NLDIR"/ND" +if [ -d $namelists ]; then + cd $namelists + if [ -f user_nl_clm ]; then + cp user_nl_* $caseroot + else + echo "ERROR: no user_nl_clm in "$namelists + kill_tether + fi +else + echo "ERROR: "$namelists" is not a directory" + kill_tether +fi + + +# build case +# do not submit! tether.sh will submit +cd $caseroot +./case.build + + +# tether commands +cd $WDIR +echo "./stabND.sh spinup.config" > commands.txt +echo $CASE_ND > case.txt diff --git a/tools/tether/sasu_spinup/setupSASU.sh b/tools/tether/sasu_spinup/setupSASU.sh new file mode 100755 index 0000000000..24f1eb60e2 --- /dev/null +++ b/tools/tether/sasu_spinup/setupSASU.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# function to abort tether when handling errors +kill_tether () { + cd $WDIR + rm commands.txt + exit 1 +} + + +# import a bunch of variables from config file +# e.g. $WDIR $PI_COMPSET +config=$1 +source <(grep = $config) + + +# create case +caseroot=$WDIR"/"$CASE_SASU +cd $CDIR +./create_newcase --case $caseroot --compset $PI_COMPSET --res $GRID --project $PROJECT --mach derecho --run-unsupported + + +# setup case +cd $caseroot +./case.setup + + +# finding the latest restart from AD +# this code is likely very brittle +ARCHIVE="/glade/derecho/scratch/"$USER"/archive" +REFCASE=$CASE_AD +REFREST=$ARCHIVE"/"$REFCASE"/rest" +last_date=$(ls $REFREST | tail -n1) +REFDIR=$REFREST"/"$last_date +REFDATE=${last_date%-*} + + +# xml changes +./xmlchange JOB_PRIORITY=premium +./xmlchange STOP_OPTION=nyears +./xmlchange STOP_N=80 +./xmlchange MOSART_MODE=NULL +./xmlchange RUN_TYPE=hybrid +./xmlchange CLM_ACCELERATED_SPINUP=sasu +./xmlchange RUN_REFCASE=$REFCASE +./xmlchange GET_REFCASE="True" +./xmlchange RUN_REFDIR=$REFDIR +./xmlchange RUN_REFDATE=$REFDATE + + +# cp in namelist +namelists=$NLDIR"/SASU" +if [ -d $namelists ]; then + cd $namelists + if [ -f user_nl_clm ]; then + cp user_nl_* $caseroot + else + echo "ERROR: no user_nl_clm in "$namelists + kill_tether + fi +else + echo "ERROR: "$namelists" is not a directory" + kill_tether +fi + + +# build case +# do not submit! tether.sh will submit +cd $caseroot +./case.build + + +# tether commands +cd $WDIR +echo "./stabSASU.sh spinup.config" > commands.txt +echo $CASE_SASU > case.txt diff --git a/tools/tether/sasu_spinup/setup_config.py b/tools/tether/sasu_spinup/setup_config.py new file mode 100644 index 0000000000..5cdfdcf524 --- /dev/null +++ b/tools/tether/sasu_spinup/setup_config.py @@ -0,0 +1,107 @@ +import os +from datetime import date +import sys + + +def get_x(x, lines): + strips=["'", '"'] + for line in lines: + if x in line: + xout = line.split('=')[1] + for s in strips: + if s in xout: + xout = xout.replace(s, '') + return xout + + +def get_tdir(pwd,tool): + tools = '/tools/' + if tools in pwd: + base = pwd.split(tools)[0] + tdir = base+tools+tool + else: + tdir = tool.upper()+'_DIR_NOT_FOUND' + return tdir + + +def find_cime(pwd,user): + ''' search upwards from pwd to find ./cime/scripts ''' + # this is probably way overkill + go = True + thisdir = pwd + while go: + ld = os.listdir(thisdir) + if 'cime' in ld: + cime_dir = thisdir + '/cime/scripts' + break + else: + last = os.path.basename(thisdir) + if last == user: + print('cannot find CIME') + cime_dir = 'CIME/SCRIPTS_DIR_NOT_FOUND' + break + else: + nx = len(last) + 1 + thisdir = thisdir[:-nx] + if not os.path.isdir(thisdir): + print('cannot find CIME') + cime_dir = 'CIME/SCRIPTS_DIR_NOT_FOUND' + break + return cime_dir + + +def get_defaults(lines): + user = get_x('USER', lines) + pwd = get_x('PWD', lines) + tag = get_x('TAG',lines).replace('.','') + tdir = get_tdir(pwd,'tether') + sdir = get_tdir(pwd,'spinup_stability') + dstr = 'c'+date.today().strftime('%Y%m%d') + wdir = '/glade/u/home/'+user+'/tethered_sims/'+dstr + cdir = find_cime(pwd,user) + defaults = {'PI_COMPSET': 'I1850Clm60BgcCrujra', + 'GRID': 'f19_g17', + 'PROJECT': 'P93300041'} + + for v in ['PI_COMPSET','GRID']: + x = get_x('PI_COMPSET',lines) + if x!='UNSET': + defaults[v]=x + + dirs = {'TDIR': tdir, 'SDIR': sdir, 'WDIR': wdir, 'CDIR': cdir} + for k in dirs: + defaults[k] = dirs[k] + + compset = defaults['PI_COMPSET'] + grid = defaults['GRID'] + for suff in ['AD', 'SASU', 'ND']: + defaults['CASE_' + suff] = '{}.{}.{}.{}'.format(tag, compset, grid, suff) + + return defaults + + +def write_new_file(lines, defaults): + with open('config.tmp', 'w') as f: + for line in lines: + if '=' in line: + k, v = line.split('=') + if v == 'UNSET': + if k in defaults: + f.write(line.replace(v,defaults[k]) + '\n') + else: + f.write(line + '\n') + else: + f.write(line + '\n') + else: + f.write(line + '\n') + + +def main(): + with open('spinup.config', 'r') as f: + lines = [line.rstrip() for line in f] + defaults = get_defaults(lines) + write_new_file(lines, defaults) + + +if __name__ == '__main__': + main() diff --git a/tools/tether/sasu_spinup/setup_config.sh b/tools/tether/sasu_spinup/setup_config.sh new file mode 100755 index 0000000000..ff741f119e --- /dev/null +++ b/tools/tether/sasu_spinup/setup_config.sh @@ -0,0 +1,69 @@ + + +# apply defaults to any unset config fields +python setup_config.py +rm spinup.config # old unset file + +# source variables from config +source <(grep = config.tmp) + +# mk WDIR and mv config to WDIR +mkdir -p $WDIR +mv config.tmp $WDIR"/spinup.config" +echo "config file now located at: "$WDIR"/spinup.config" + + +# make sure scratch directory exists +SD='/glade/derecho/scratch/'$USER +if [ ! -d $SD ]; then + echo $SD" is not a directory, $USER may be set badly" + echo "examine "$WDIR"/spinup.config" + exit 1 +fi + +# cp scripts to WDIR +cp *AD.sh $WDIR +cp *SASU.sh $WDIR +cp *ND.sh $WDIR + + +# add a path comment to the namelist files +# it probably makes sense to move these to WDIR +segments=('AD' 'SASU' 'ND') +cd namelists +for segment in ${segments[@]}; do + cd $segment + sed 's:nlpath:'$(pwd)':g' unset_user_nl_clm > user_nl_clm + cd .. +done +cd .. + + +# configure the PBS template +sed "s:TDIR:"$TDIR":g" derecho.template > $WDIR"/derecho.template" +sed -i "s:project:"$PROJECT":g" $WDIR"/derecho.template" + + +# configure yamls +HD=$SD"/archive/"$CASE_AD"/lnd/hist" +sed 's/CASE_AD/'$CASE_AD'/g' AD.yml > $WDIR"/AD.yml" +sed -i 's:HIST_AD:'$HD':g' $WDIR"/AD.yml" +HD=$SD"/archive/"$CASE_SASU"/lnd/hist" +sed 's/CASE_SASU/'$CASE_SASU'/g' SASU.yml > $WDIR"/SASU.yml" +sed -i 's:HIST_SASU:'$HD':g' $WDIR"/SASU.yml" +HD=$SD"/archive/"$CASE_ND"/lnd/hist" +sed 's/CASE_ND/'$CASE_ND'/g' ND.yml > $WDIR"/ND.yml" +sed -i 's:HIST_ND:'$HD':g' $WDIR"/ND.yml" + + +# create the initial qsub job +sed '/afterok/d' $WDIR"/derecho.template" > $WDIR"/segment001.job" +sed -i 's/jobname/segment001/g' $WDIR"/segment001.job" +sed -i 's:wdir:'$WDIR':g' $WDIR"/segment001.job" +sed -i 's/commands/commands.txt/g' $WDIR"/segment001.job" +sed -i 's/template/derecho.template/g' $WDIR"/segment001.job" + + +# create the initial commands.txt +echo "./setupAD.sh spinup.config" > $WDIR"/commands.txt" + diff --git a/tools/tether/sasu_spinup/stabAD.sh b/tools/tether/sasu_spinup/stabAD.sh new file mode 100755 index 0000000000..88b5e8e30a --- /dev/null +++ b/tools/tether/sasu_spinup/stabAD.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# function to abort tether when handling errors +kill_tether () { + cd $WDIR + rm commands.txt + exit 1 +} + + +# import a bunch of variables from config file +# e.g. $WDIR $PI_COMPSET +config=$1 +source <(grep = $config) + + +# run the spinup stability script +python $SDIR"/spinup_stability.py" AD.yml +status=$? +echo "status: "$status + + +# proceed accordingly +if [[ "$status" == "11" ]]; then + echo "needs more spinup" + cd $CASE_AD + ./xmlchange CONTINUE_RUN=True + ./xmlchange STOP_N=20 + ./xmlchange JOB_WALLCLOCK_TIME=4:00:00 --subgroup case.run + cd $WDIR + echo $CASE_AD>case.txt + echo "./stabAD.sh spinup.config">commands.txt +elif [[ "$status" == "0" ]]; then + echo "spinup appears sufficient" + ./setupSASU.sh spinup.config +else + echo "something looks wrong, halting tether" + echo $TDIR"spinup_stability.py AD.yml experienced an error" + kill_tether +fi diff --git a/tools/tether/sasu_spinup/stabND.sh b/tools/tether/sasu_spinup/stabND.sh new file mode 100755 index 0000000000..2c2e70eeaa --- /dev/null +++ b/tools/tether/sasu_spinup/stabND.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# function to abort tether when handling errors +kill_tether () { + cd $WDIR + rm commands.txt + exit 1 +} + + +# import a bunch of variables from config file +# e.g. $WDIR $PI_COMPSET +config=$1 +source <(grep = $config) + + +# run the spinup stability script +python $SDIR"/spinup_stability.py" ND.yml +status=$? +echo "status: "$status + + +# proceed accordingly +if [[ "$status" == "11" ]]; then + echo "needs more spinup" + cd $CASE_ND + ./xmlchange CONTINUE_RUN=True + ./xmlchange STOP_N=20 + ./xmlchange JOB_WALLCLOCK_TIME=4:00:00 --subgroup case.run + cd $WDIR + echo $CASE_ND>case.txt + echo "./stabND.sh spinup.config">commands.txt +elif [[ "$status" == "0" ]]; then + echo "spinup appears sufficient" + rm commands.txt # sequence complete, exit tether +else + echo "something looks wrong, halting tether" + echo $TDIR"spinup_stability.py ND.yml experienced an error" + kill_tether +fi diff --git a/tools/tether/sasu_spinup/stabSASU.sh b/tools/tether/sasu_spinup/stabSASU.sh new file mode 100755 index 0000000000..9369693f56 --- /dev/null +++ b/tools/tether/sasu_spinup/stabSASU.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# function to abort tether when handling errors +kill_tether () { + cd $WDIR + rm commands.txt + exit 1 +} + + +# import a bunch of variables from config file +# e.g. $WDIR $PI_COMPSET +config=$1 +source <(grep = $config) + + +# run the spinup stability script +python $SDIR"/spinup_stability.py" SASU.yml +status=$? +echo "status: "$status + + +# proceed accordingly +if [[ "$status" == "11" ]]; then + echo "needs more spinup" + cd $CASE_SASU + ./xmlchange CONTINUE_RUN=True + ./xmlchange STOP_N=20 + ./xmlchange JOB_WALLCLOCK_TIME=4:00:00 --subgroup case.run + cd $WDIR + echo $CASE_SASU>case.txt + echo "./stabSASU.sh spinup.config">commands.txt +elif [[ "$status" == "0" ]]; then + echo "spinup appears sufficient" + ./setupND.sh spinup.config +else + echo "something looks wrong, halting tether" + echo $TDIR"spinup_stability.py SASU.yml experienced an error" + kill_tether +fi diff --git a/tools/tether/sasu_spinup/todo.txt b/tools/tether/sasu_spinup/todo.txt new file mode 100644 index 0000000000..e06199c88c --- /dev/null +++ b/tools/tether/sasu_spinup/todo.txt @@ -0,0 +1,3 @@ +- make a readme that will appear in the tether WDIR +- check that you can use sasu in the specified codebase, before anything +- come up with a better default WDIR diff --git a/tools/tether/tether.sh b/tools/tether/tether.sh new file mode 100755 index 0000000000..e1d9f8b1df --- /dev/null +++ b/tools/tether/tether.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# creates a jobname +get_jobname () { + if [ -f segment001.job ]; then + segment=$(ls segment*.job | wc -l) + ((segment++)) + else + segment="1" + fi + jobname="segment"$(printf %03d $segment) + echo $jobname +} + +# adds newline if commands file does not end in newline +prep_commands () { + if ! [[ $(tail -c1 $commands | wc -l) -gt 0 ]]; then + echo "" >> $commands + fi +} + +# logfile header +header () { + echo "-----------------------------" + echo "TETHER COMMANDS:" + echo "-----------------------------" +} + +main () { + wdir=$1 + commands=$2 + template=$3 + cd $wdir + if [ -f $commands ]; then + prep_commands + header + while read -r line; do + echo $line #log command + echo "-----------------------------" + command ${line} #run command + echo "-----------------------------" + done< $commands + + #read case + case=$( $qj + sed -i 's:jobid:'$jobid':g' $qj + sed -i 's:wdir:'$wdir':g' $qj + sed -i 's:commands:'$commands':g' $qj + sed -i 's:template:'$template':g' $qj + qsub $qj + else + echo "No commands file found, exiting tether" + fi +} + +main $1 $2 $3