#!/bin/tcsh
#
# RELION 4.0 job submission template for SLURM at MRC-LMB cluster
# This script is maintained by Takanori Nakane.
#
#SBATCH --job-name=RELION4.0
#SBATCH --error=Class2D/job027/run.err
#SBATCH --output=Class2D/job027/run.out
#SBATCH --open-mode=append
#SBATCH --time=2-00:00:00 # TODO: should be enough for most jobs but consider
#SBATCH --mail-type=FAIL
##SBATCH --partition=cpu
##SBATCH --gres=gpu:0
##SBATCH --ntasks=9
##SBATCH --cpus-per-task=4
##SBATCH --mem=60G #TODO: --mem-per-cpu=2500M might be better but user have to worry...
#SBATCH --partition=gpu
#SBATCH --gres=gpu:4
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=32
#SBATCH --mem=128G

source /public/EM/RELION/relion-4.0-dev.csh

# This is created by the job prolog script.
# For some reason, the environmental variable is not properly set,
# so I do here.
set  RELION_SCRATCH_DIR=/ssd/${SLURM_JOB_USER}-${SLURM_JOBID}

echo "* RELION 4.0 JOB ENVIRONMENT *" # csh expands * ...
echo
echo PATH TO mpiexec: `which mpiexec`
echo PATH TO relion_refine_mpi: `which relion_refine_mpi`
echo LIBRARIES FOR relion_refine_mpi:
ldd  `which relion_refine_mpi`
echo RELION_EXTERNAL_RECONSTRUCT_EXECUTABLE: $RELION_EXTERNAL_RECONSTRUCT_EXECUTABLE
echo SIDESPLITTER: $SIDESPLITTER
echo

set  STARTTIME = `date +%s`
echo RUN STARTED AT `date -d@${STARTTIME}`
echo TO KILL THIS JOB, RUN: scancel $SLURM_JOB_ID
echo ------------------------------------------------------------------------------------

# --oversubscribe is intentional to allow almost idle rank 0
mpiexec --oversubscribe -n 9  `which relion_refine_mpi` --continue Class2D/job027/run_it020_optimiser.star --o Class2D/job027/run --iter 25 --dont_combine_weights_via_disc --pool 30 --pad 2  --tau2_fudge 2 --particle_diameter 340 --center_classes  --oversampling 1 --psi_step 2 --offset_range 5 --offset_step 2 --helical_outer_diameter 200 --bimodal_psi --sigma_psi 2 --helix --helical_rise_initial 4.75 --j 4 --gpu ""  --pipeline_control Class2D/job027/

echo ------------------------------------------------------------------------------------
set  ENDTIME = `date +%s`
echo RUN FINISHED AT `date -d@${ENDTIME}`
echo ELAPSED WALL CLOCK TIME IN SECONDS: `echo $ENDTIME - $STARTTIME | bc`

exit 0