RUNopenMP.bsub

#BSUB -J MPIbind
#BSUB -q hpcintro
#BSUB -n 120
#BSUB -R "span[ptile=24]"
#BSUB -R "rusage[mem=10GB]"
#BSUB -o MPIbind_%J.out
#BSUB -e MPIbind_%J.err
#BSUB -W 40

# clear out the LSB affinity file (it disturbs in this setup)
/bin/rm -f $LSB_AFFINITY_HOSTFILE
touch $LSB_AFFINITY_HOSTFILE

# load MPI version
module load mpi/3.1.3-gcc-8.2.0

# mpirun options
BIND="--bind-to socket"
MAP="--map-by node"
RANK="--rank-by socket"
TAG="--tag-output"
REPORT="--report-bindings"

# do a clean build to ensure everything is updated
make clean
make CFLAGS="-O3 -march=native"

tol=0

# Loop matrix size
for N_n in 100 500 1000 1500 2000 2500 3000 3500 4000
do

if [[ $N_n -lt 200 ]]
then
    iter=100
elif [[ $N_n -lt 1000 ]]
then
    iter=20
else
    iter=5
fi

# Loop px
all_p="1 4 9 16 25 36 49 64 81 100" # start at 1? end at 100?
if [[ $N_n -lt 2750 ]]
then
    all_p_sqr="1 2 3 4 5 6 7 8 9 10"    # squared number of processes
elif [[ $N_n -lt 3250 ]]
then
    all_p_sqr="7 8 9 10"
elif [[ $N_n -lt 3750 ]]
then
    all_p_sqr="9 10"
else
    all_p_sqr="10"
fi

for p_n in $all_p_sqr ; do

N=$(($N_n-($N_n%$p_n)))
p=$(($p_n*$p_n))

# skip calculations with more processors than available
[ $p -gt $LSB_DJOB_NUMPROC ] && continue

# adjust here for the different algorithms
all_algo="6"
for algo in $all_algo ; do

if [[ $algo -eq 8 ]]
then
    export OMP_NUM_THREADS=24
fi

out="data/${N}_p${p}_${algo}"
[ -e $out ] && continue

# OpenMP threads
echo OMP_NUM_THREADS $OMP_NUM_THREADS

#mpirun -np $p $MAP $BIND $RANK $TAG $REPORT /bin/hostname  
mpirun -np $p $MAP $BIND $RANK $TAG $REPORT \
	./main $N $iter $tol $algo > $out 2>${out}_binding
if [ $? -ne 0 ]; then
    echo "Exitting before done!!!"
    exit 1
fi

done #algo

done #p_n

done #N_n