Added OMP_Threadpool struct

Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:18.04
+
+# DYNAMIC BRANCH CAN BE OVERWRITTEN DURING RUNTIME
+ENV BRANCH=""
+
+# NEEDS TO BE UPDATED FOR SPECIFIC VERSIONS
+RUN apt update && \
+    apt install -y g++ libomp-dev libgsl-dev libhdf5-serial-dev git cmake
+
+WORKDIR /home/ubuntu/
+
+# COPY PROJECT DIRECTORY
+# RUN git clone https://github.com/pelahi/VELOCIraptor-STF.git 
+COPY . /home/ubuntu/VELOCIraptor-STF
+
+WORKDIR /home/ubuntu/VELOCIraptor-STF
+
+#RUN git checkout ${BRANCH} 
+#&& git submodule update --init --recursive
+
+# BUILD BINARY
+RUN mkdir build && cd build && cmake .. && make all

examples/profiling/flamegraph.Makefile

@@ -0,0 +1,48 @@
+# This Makefile profiles stf using perf to produce flame graphs
+.PHONEY: info record_prereq fold flamegraph run display
+
+#script that produces lots of qsub scripts to run velociraptor on simulation output
+info:
+    @echo "This script profiles a VR run."
+
+# PERF
+_PERF_OPT_RECORD=--call-graph lbr -g -s -o
+_PERF_PATH_ROOT=`pwd`/fg
+_PERF_FILE_RAW=$(_PERF_PATH_ROOT)/perf.raw.`date +%Y%m%d`.`hostname`.data
+
+# FLAMEGRAPH
+_FLAMEGRAPH_GIT_URL=https://github.com/brendangregg/FlameGraph.git
+_FLAMEGRAPH_PATH=$(_PERF_PATH_ROOT)/FlameGraph
+_FLAMEGRAPH_FILE_FOLDED=$(_PERF_PATH_FOLDED)/perf.processed.`date +%Y%m%d`.`hostname`.folded
+_FLAMEGRAPH_SCRIPT_STACKCOLLAPSE=$(_FLAMEGRAPH_PATH)/stackcollapse-perf.pl
+_FLAMEGRAPH_SCRIPT_MAIN=$(_FLAMEGRAPH_PATH)/flamegraph.pl
+_FLAMEGRAPH_FILE_SVG=$(_PERF_PATH_ROOT)
+
+# STF
+# Dynamic stf input parameters through environment variables to be set in an input payload "./payloadname.env "
+_STF_PARAMETERS="DEFAULT"
+include ./payloadname.env 
+
+record_prereq:
+    @mkdir -p $(_PERF_PATH_ROOT)
+    @cd $(_PERF_PATH_ROOT) && git clone $(_FLAMEGRAPH_GIT_URL) && cd..
+
+record: record_prereq
+    perf record $(_PERF_OPT_RECORD) -o $(_PERF_FILE_RAW) ./stf $(_STF_PARAMETERS)
+    @echo "To manually test recorded data, execute: "
+    @echo "perf report -i $(_PERF_FILE_RAW)"
+
+fold:
+    perf script -i $(_PERF_FILE_RAW) | $(_FLAMEGRAPH_SCRIPT_STACKCOLLAPSE) > $(_FLAMEGRAPH_FILE_FOLDED)
+
+flamegraph:
+    cat $(_FLAMEGRAPH_FILE_FOLDED) | $(_FLAMEGRAPH_SCRIPT_MAIN) > $(_FLAMEGRAPH_FILE_SVG)
+
+run: record fold flamegraph
+
+# Display output for proofreading
+# It can be appended to job scripts or test run in CLI
+display:
+    @echo "perf record $(_PERF_OPT_RECORD) -o $(_PERF_FILE_RAW) ./stf $(_STF_PARAMETERS)"
+    @echo "perf script -i $(_PERF_FILE_RAW) | $(_FLAMEGRAPH_SCRIPT_STACKCOLLAPSE) > $(_FLAMEGRAPH_FILE_FOLDED)"
+    @echo "cat $(_FLAMEGRAPH_FILE_FOLDED) | $(_FLAMEGRAPH_SCRIPT_MAIN) > $(_FLAMEGRAPH_FILE_SVG)"

src/ompvar.h

@@ -50,4 +50,75 @@ struct OMP_ImportInfo {
     int task;
 };
 #endif
-#endif
+
+/*
+ * OMP_Threadpool_datastructure
+ * - coupled to OMP_Threadpool
+ * - structure to store threadpool information
+ * - captures the following:
+ *      thread state active(True)/inactive(false)
+ *      total number of particles
+ */
+struct OMP_Threadpool_datastructure {
+    bool active;
+    Int_t particle_total;
+};
+
+/*
+ * OMP_Threadpool
+ * - contains routines used for optimised parallel computing/processing
+ *      throttle_up - increase number of active threads
+ *      throttle_down - decrease number of active threads
+ * - captures the following:
+ *      specific threadpool information stored to OMP_Threadpool_datastructure
+ *      total thread size
+ *      number of active threads
+ */
+struct OMP_Threadpool {
+    // Total thread available
+    // Active threads
+    // Unsigned long long for number of particles
+    // omp thread class - 
+    // omp retrieve active threads
+    // specify offset of thread
+
+    /* Private block */
+    private:
+    map<int, OMP_Threadpool_datastructure> details;
+    int thread_count, active_threads;
+
+    /* Public block */
+    public:
+    void throttle_up(){
+        // TODO: add the following logic:
+        //      OMP_Threadpool_datastructure update state of thread from inactive to active
+        active_threads += 1;
+    }
+
+    void throttle_down(){
+        // TODO: add the following logic:
+        //      OMP_Threadpool_datastructure update state of thread from active to inactive
+        active_threads -= 1;
+    }
+
+    void set_total_threads(int thread_count){
+        thread_count = thread_count;
+    }
+
+    int get_total_threads(){
+        return thread_count;
+    }
+
+    int get_active_threads(){
+        return active_threads;
+    }
+
+    /* Confirmation Functions */
+    void print_total_threads(){
+        printf("Total Threads: %d\n", thread_count);
+    }
+
+    void print_active_threads(){
+        printf("Active Threads: %d\n", active_threads);
+    }
+};

src/search.cxx

@@ -416,7 +416,7 @@ Int_t* SearchFullSet(Options &opt, const Int_t nbodies, vector<Particle> &Part,
             numingroup=BuildNumInGroupTyped(Nlocal,numgroups,pfof,Part.data(),DARKTYPE);
             for (i=0;i<Nlocal;i++) {
                 if (Part[i].GetType()==DARKTYPE) Part[i].SetType(numingroup[pfof[Part[i].GetID()]]>=MINSUBSIZE);
-                else Part[i].SetType(-1);
+                else Part[i].SetType(0);
                 numlocalden += (Part[i].GetType()>0);
             }
             delete[] numingroup;
@@ -2634,10 +2634,6 @@ void RemoveSpuriousDynamicalSubstructures(Options &opt, const Int_t nsubset, Int
     }
 }
 
-int setNthreads(){
-    return 0;
-}
-
 ///adjust to phase centre
 inline void AdjustSubPartToPhaseCM(Int_t num, Particle *subPart, GMatrix &cmphase)
 {
@@ -2746,11 +2742,12 @@ inline void CleanAndUpdateGroupsFromSubSearch(Options &opt,
             }
         }
     }
-
+    
     for (auto j=0;j<subnumingroup;j++)
     {
         if (subpfof[j]>0) pfof[subpglist[j]]=ngroup+ngroupidoffset+subpfof[j];
     }
+
     //ngroupidoffset+=subngroup;
     //now alter subsubpglist so that index pointed is global subset index as global subset is used to get the particles to be searched for subsubstructure
     for (auto j=1;j<=subngroup;j++)
@@ -2920,12 +2917,19 @@ void SearchSubSub(Options &opt, const Int_t nsubset, vector<Particle> &Partsubse
                 //this routine is within this file, also has internal parallelisation
                 AdjustSubPartToPhaseCM(subnumingroup[i], subPart, cmphase);
             }
+            // TODO: Optimise bottleneck
+            double time_temp;
+            time_temp = MyGetTime();
             PreCalcSearchSubSet(opt, subnumingroup[i], subPart, sublevel);
+            cout<<"DURATION[PreCalcSearchSubSet]: "<<MyGetTime()-time_temp<<endl;
             subpfof = SearchSubset(opt, subnumingroup[i], subnumingroup[i], subPart,
                 subngroup[i], sublevel, &numcores[i]);
+            cout<<"DURATION[SearchSubset]: "<<MyGetTime()-time_temp<<endl;
+            time_temp = MyGetTime();
             CleanAndUpdateGroupsFromSubSearch(opt, subnumingroup[i], subPart, subpfof,
                     subngroup[i], subsubnumingroup[i], subsubpglist[i], numcores[i],
                     subpglist[i], pfof, ngroup, ngroupidoffset_old[i]);
+            cout<<"DURATION[CleanAndUpdateGroupsFromSubSearch]: "<<MyGetTime()-time_temp<<endl;
             delete[] subpfof;
             delete[] subPart;
             ns+=subngroup[i];
@@ -2952,6 +2956,9 @@ void SearchSubSub(Options &opt, const Int_t nsubset, vector<Particle> &Partsubse
                     //this routine is within this file, also has internal parallelisation
                     AdjustSubPartToPhaseCM(subnumingroup[i], subPart, cmphase);
                 }
+                // TODO: Optimise bottleneck
+                double time_temp;
+                time_temp = MyGetTime();
                 PreCalcSearchSubSet(opt2, subnumingroup[i], subPart, sublevel);
                 subpfof = SearchSubset(opt2, subnumingroup[i], subnumingroup[i], subPart,
                     subngroup[i], sublevel, &numcores[i]);
@@ -3843,7 +3850,7 @@ private(i,tid,p1,pindex,x1,D2,dval,rval,icheck,nnID,dist2,baryonfofold)
             map<Int_t, Int_t> remap;
             Int_t newng=0, oldpid, newpid;
             remap[0]=0;
-            for (i=1;i<=ng;i++) {
+            for (auto i=1;i<=ng;i++) {
                 if (ningall[i]>0) {
                     newng++;
                     remap[i]=newng;