#!/bin/bash --login
#SBATCH --job-name=phi090_theta047
#SBATCH --nodes=10
#SBATCH --tasks-per-node=36
#SBATCH --cpus-per-task=1
#SBATCH --time=16:00:00
#SBATCH --exclusive
# Replace [budget code] below with your project code (e.g. t01)
#SBATCH --account=e720
#SBATCH --partition=standard
#SBATCH --qos=lowpriority
############################################################
module load intel-20.4/compilers
module load intel-20.4/mpi
module load intel-20.4/cmkl
module load mpt
module list
############################################################
RUNDIR=$SLURM_SUBMIT_DIR
ulimit -s unlimited
EXEC=/work/e720/e720/wen94/wt_mpi.x
echo =========================================================   
echo SLURM job: submitted  date = `date`      
date_start=`date +%s`
echo Executable file:                              
echo MPI parallel job.                                  
echo -------------  
echo Job output begins                                           
echo -----------------                                           
hostname 
echo "Print the following environmetal variables:"
echo "Job name                     : $SLURM_JOB_NAME"
echo "Job ID                       : $SLURM_JOB_ID"
echo "Job user                     : $SLURM_JOB_USER"
echo "Job array index              : $SLURM_ARRAY_TASK_ID"
echo "Submit directory             : $SLURM_SUBMIT_DIR"
echo "Temporary directory          : $TMPDIR"
echo "Submit host                  : $SLURM_SUBMIT_HOST"
echo "Queue/Partition name         : $SLURM_JOB_PARTITION"
echo "Node list                    : $SLURM_JOB_NODELIST"
echo "Hostname of 1st node         : $HOSTNAME"
echo "Number of nodes allocated    : $SLURM_JOB_NUM_NODES or $SLURM_NNODES"
echo "Number of processes          : $SLURM_NTASKS"
echo "Number of processes per node : $SLURM_TASKS_PER_NODE"
echo "Requested tasks per node     : $SLURM_NTASKS_PER_NODE"
echo "Requested CPUs per task      : $SLURM_CPUS_PER_TASK"
echo "Scheduling priority          : $SLURM_PRIO_PROCESS"
echo "Running parallel job"
srun --distribution=block:block --hint=nomultithread $EXEC
ret=$? 
echo ---------------                                           
echo Job output ends                                           
date_end=`date +%s`
seconds=$((date_end-date_start)) 
minutes=$((seconds/60)) 
seconds=$((seconds-60*minutes)) 
hours=$((minutes/60)) 
minutes=$((minutes-60*hours)) 
echo =========================================================   
echo SLURM job: finished   date = `date`   
echo Total run time : $hours Hours $minutes Minutes $seconds Seconds
echo =========================================================   
exit $ret
