cron_FLEXPART.sh

# @Author: Christopher Symonds
# @Date:   2020-11-18T10:07:25+00:00
# @Email:  C.C.Symonds@leeds.ac.uk
# @Project: FAZE-In
# @Filename: cron_FLEXPART.sh
# @Last modified by:   chris
# @Last modified time: 2021-01-20T19:49:21+00:00
# @License: MIT
# @Copyright: University of Leeds

#! /bin/bash -l

#Replace with own conda path
. /etc/profile.d/modules.sh
. /scratch/cemac/cemac.sh
. /nfs/earcemac/chmcsy/anaconda3/etc/profile.d/conda.sh

module purge
module load user
module load gnu/4.8.1
module load flexpart

# Dir paths
rundir=$PWD
scratchdir="/scratch/chmcsy/data"
flexdir="/scratch/chmcsy/FlexPart"
testdir="/scratch/chmcsy/fwd_gfs_test"
out_dir_base="${flexdir}/cronflex"
flextractdir="/scratch/chmcsy/flex_extract/"
spritedir="/nfs/see-fs-02_users/chmcsy/Git_Repos/Faze-In_App/Applications/static/sprites"

#Looks at day before yesterday to yesterday. UTC to account for any daylight savings effects.
day=$(TZ=":UTC" date -d '-4 days' +"%Y%m%d" )

#make directory on a68 for this to go in, and set as output directory in pathnames

out_dir=${out_dir_base}/daily

mkdir -p ${out_dir}

if [ -f "${out_dir}/partposit_$( date -d $day +'%Y%m%d%H%M%S' )" ]; then
  warm_strt=TRUE
else
  warm_strt=FALSE
fi

echo "Warm Start = $warm_strt"

# Make pathnames file
cat > ${testdir}/pathnames << -EOF
$testdir/options
$out_dir
$scratchdir
$scratchdir/AVAILABLE

-EOF

if [ "$warm_strt" = TRUE ]; then
  datechunk=3 # 1 day, plus one on either side
  strtday=$day
  endday=$day
else
  datechunk=27 # 20 days, plus five days of output data and one day either side
  endday=$day
  strtday=$( date -d "$day -$datechunk days" +"%Y%m%d" )
fi

echo "Data to be retrieved between $strtday and $endday"

#Retrieve ERA5 data using flex_extract tool and make AVAILABLE file

conda activate flex_extract
#${flextractdir}/Source/Python/submit.py --controlfile CONTROL_EA5.0.5.6h --date_chunk $datechunk --start_date $strtday --end_date $endday --outputdir ${scratchdir}

echo "Conda env activated for flex_extract"

python $rundir/download_gfs.py ${strtday} ${endday} ${scratchdir}

cp make_available ${scratchdir}
cd ${scratchdir}
./make_available
cd ${testdir}

echo "AVAILABLE file made"

#Retrieve GFAS data and make RELEASES file

conda activate fazein

echo "Fazein conda activated"

rm ${testdir}/options/RELEASES*
cp -f $rundir/get_fire_RELEASES_GFAS_daily.py ${testdir}
cd ${testdir}
python get_fire_RELEASES_GFAS_daily.py $( date -d $strtday +'%F' ) --enddate $( date -d $endday +'%F' )

echo "GFAS files retrieved"

# Add combo for releases

#update lines in COMMAND file to have start date of this day and end date of this day+run length
sed -i "9s/.*/ IBDATE=         ${strtday}, ! Start date of the simulation   ; YYYYMMDD: YYYY=year, MM=month, DD=day /" ${testdir}/options/COMMAND
sed -i "11s/.*/ IEDATE=         $( date -d "${endday} +1 days" +"%Y%m%d" ),! End date of the simulation     ; same format as IBDATE  /" ${testdir}/options/COMMAND
sed -i '25s/.*/ IPIN=                  0, ! Warm start from particle dump (needs previous partposit_end file); [0]no 1]yes  /' ${testdir}/options/COMMAND

#if its not the first day then want to use a warm start
#remove partposit_end file and replace it with partposit_file from run before
#update line in COMMAND file to have a warm start
if [ "$warm_strt" == TRUE ]; then
  #if [ -d ${out_dir_base}/backup_data ]; then rm -rf ${out_dir_base}/backup_data; fi
  #cp -r ${out_dir} ${out_dir_base}/backup_data
  echo 'warm start'
  if [ -f ${out_dir}/partposit_end ]; then rm ${out_dir}/partposit_end; fi
  cp -f "${out_dir}/partposit_$( date -d $day +'%Y%m%d%H%M%S' )" "${out_dir}/partposit_end"
  sed -i '25s/.*/ IPIN=                  1, ! Warm start from particle dump (needs previous partposit_end file); [0]no 1]yes  /' ${testdir}/options/COMMAND
  start_year=$( date -d $day +'%Y' )
  rm ${out_dir}/partposit_$( date -d $day +'%Y%m%d' )*
fi

#run flexpart for this day
FLEXPART

#plot the output files
conda activate flex_extract

python ${rundir}/plot_flexpart.py ${strtday} ${out_dir} -v BC

echo "Plots created"

cd ${out_dir}
spritedays=5
mkdir -p ${out_dir}/spritebuild
plotday=$( date -d "${day} +1 days" +"%Y%m%d" )
cp 20m-${plotday}* spritebuild
for i in `seq 0 $spritedays`; do
  plotday=$( date -d "${day} -${i} days" +"%Y%m%d" )
  cp 20m-${plotday}* spritebuild
done

cd ${spritedir}
git pull

cd ${out_dir}/spritebuild
rm 20m-${plotday}0000.png

montage -background transparent -tile 8x -geometry 675x600+0+0 *.png ${spritedir}/spritesheet.png

cd ${spritedir}
git add -u
git commit -m "Spritesheet updated for ${day}"
git push # credentials stored
cd ${out_dir}
rm -rf spritebuild

#put partposit files and header in folder
mkdir -p "${out_dir}/${day}"
for file in ${out_dir}/partposit_*000000; do
  cp $file "${out_dir}/${day}/"
done
cp "${out_dir}/header" "${out_dir}/${day}/"

# Remove old files (AVAILABLE files older than 5 days, all others older than 30 days)
cleanupdate=$( date -d "${day} -30 days" +"%Y%m%d" )
cd ${scratchdir}
for f in gfs_4_*; do
  filedate=${f:6:8}
  if [ $filedate -lt $cleanupdate ]; then
    rm $f
  fi
done
cleanupdate2=$( date -d "${day} -5 days" +"%s" )
for f in AVAILABLE.*; do
  filedate=${f:10:10}
  if [ $filedate -lt $cleanupdate2 ]; then
    rm $f
  fi
done
cd ${out_dir}
for f in 20m-*.png; do
  filedate=${f:4:8}
  if [ $filedate -lt $cleanupdate ]; then
    rm $f
  fi
done
for f in partposit_20*; do
  filedate=${f:10:8}
  if [ $filedate -lt $cleanupdate ]; then
    rm $f
  fi
done
for f in shortposit_20*; do
  filedate=${f:11:8}
  if [ $filedate -lt $cleanupdate ]; then
    rm $f
  fi
done
for f in grid_conc_*.nc; do
  filedate=${f:10:8}
  if [ $filedate -lt $cleanupdate ]; then
    rm $f
  fi
done