diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e4522b12 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:18.04 + +# DYNAMIC BRANCH CAN BE OVERWRITTEN DURING RUNTIME +ENV BRANCH="" + +# NEEDS TO BE UPDATED FOR SPECIFIC VERSIONS +RUN apt update && \ + apt install -y g++ libomp-dev libgsl-dev libhdf5-serial-dev git cmake + +WORKDIR /home/ubuntu/ + +# COPY PROJECT DIRECTORY +# RUN git clone https://github.com/pelahi/VELOCIraptor-STF.git +COPY . /home/ubuntu/VELOCIraptor-STF + +WORKDIR /home/ubuntu/VELOCIraptor-STF + +#RUN git checkout ${BRANCH} +#&& git submodule update --init --recursive + +# BUILD BINARY +RUN mkdir build && cd build && cmake .. && make all diff --git a/examples/profile_build.sh b/examples/profile_build.sh deleted file mode 100755 index bd6111b0..00000000 --- a/examples/profile_build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -l -# This script profiles stf using perf and hotspot to produce flame graphs -# It produces a build directory with a certian labels -# runs cmake with the desired options, runs the code on the desired input -# using the desired config and running the hotspot stuff -# it assumes the existence of perf that is in linux-tools-common -# also makes use of wget - -#script that produces lots of qsub scripts to run velociraptor on simulation output -if [ $# -eq 0 ] || [ "$1" == "--help" ] -then - echo "This script profiles a VR run." - echo "The interface is as follows:" - echo "buildlabel buildoptions VRargs VRconfig" - echo "buildlabel: string, label for the build" - echo "buildoptions: string, cmake options for build. Ex: \" -DVR_USE_GAS=ON \"" - echo "VRargs: string, options for VR. Ex: \"-i inputfile -I 2 -s 1 -o outputfile \"" - echo "VRconfig: string, path and file name of the config file" - exit -fi - - -#initial and final snapshot numbers -buildlabel=$1 -buildoptions=$2 -VRargs=$3 -VRconfig=$4 - -workingdir=`pwd` -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -#build exe -cd ${scriptdir}/../ -mkdir build-${buildlabel} -cd build-${buildlabel} -rm -rf * -cmake ${buildoptions} -DCMAKE_BUILD_TYPE=RelWithDebugInfo ../ -make -j - -wget https://github.com/KDAB/hotspot/releases/download/v1.1.0/hotspot-v1.1.0-x86_64.AppImage -chmod +x hotspot-v1.1.0-x86_64.AppImage - -#run code -#this will have produced a perf.data -perf record ./stf ${VRargs} -C ${VRconfig} - -#run hotspot -./hotspot-v1.1.0-x86_64.AppImage perf.data diff --git a/examples/profiling/flamegraph.Makefile b/examples/profiling/flamegraph.Makefile new file mode 100644 index 00000000..e300654e --- /dev/null +++ b/examples/profiling/flamegraph.Makefile @@ -0,0 +1,48 @@ +# This Makefile profiles stf using perf to produce flame graphs +.PHONEY: info record_prereq fold flamegraph run display + +#script that produces lots of qsub scripts to run velociraptor on simulation output +info: + @echo "This script profiles a VR run." + +# PERF +_PERF_OPT_RECORD=--call-graph lbr -g -s -o +_PERF_PATH_ROOT=`pwd`/fg +_PERF_FILE_RAW=$(_PERF_PATH_ROOT)/perf.raw.`date +%Y%m%d`.`hostname`.data + +# FLAMEGRAPH +_FLAMEGRAPH_GIT_URL=https://github.com/brendangregg/FlameGraph.git +_FLAMEGRAPH_PATH=$(_PERF_PATH_ROOT)/FlameGraph +_FLAMEGRAPH_FILE_FOLDED=$(_PERF_PATH_FOLDED)/perf.processed.`date +%Y%m%d`.`hostname`.folded +_FLAMEGRAPH_SCRIPT_STACKCOLLAPSE=$(_FLAMEGRAPH_PATH)/stackcollapse-perf.pl +_FLAMEGRAPH_SCRIPT_MAIN=$(_FLAMEGRAPH_PATH)/flamegraph.pl +_FLAMEGRAPH_FILE_SVG=$(_PERF_PATH_ROOT) + +# STF +# Dynamic stf input parameters through environment variables to be set in an input payload "./payloadname.env " +_STF_PARAMETERS="DEFAULT" +include ./payloadname.env + +record_prereq: + @mkdir -p $(_PERF_PATH_ROOT) + @cd $(_PERF_PATH_ROOT) && git clone $(_FLAMEGRAPH_GIT_URL) && cd.. + +record: record_prereq + perf record $(_PERF_OPT_RECORD) -o $(_PERF_FILE_RAW) ./stf $(_STF_PARAMETERS) + @echo "To manually test recorded data, execute: " + @echo "perf report -i $(_PERF_FILE_RAW)" + +fold: + perf script -i $(_PERF_FILE_RAW) | $(_FLAMEGRAPH_SCRIPT_STACKCOLLAPSE) > $(_FLAMEGRAPH_FILE_FOLDED) + +flamegraph: + cat $(_FLAMEGRAPH_FILE_FOLDED) | $(_FLAMEGRAPH_SCRIPT_MAIN) > $(_FLAMEGRAPH_FILE_SVG) + +run: record fold flamegraph + +# Display output for proofreading +# It can be appended to job scripts or test run in CLI +display: + @echo "perf record $(_PERF_OPT_RECORD) -o $(_PERF_FILE_RAW) ./stf $(_STF_PARAMETERS)" + @echo "perf script -i $(_PERF_FILE_RAW) | $(_FLAMEGRAPH_SCRIPT_STACKCOLLAPSE) > $(_FLAMEGRAPH_FILE_FOLDED)" + @echo "cat $(_FLAMEGRAPH_FILE_FOLDED) | $(_FLAMEGRAPH_SCRIPT_MAIN) > $(_FLAMEGRAPH_FILE_SVG)" \ No newline at end of file diff --git a/examples/baryonsample.cfg b/examples/stfconfig/baryonsample.cfg similarity index 100% rename from examples/baryonsample.cfg rename to examples/stfconfig/baryonsample.cfg diff --git a/examples/genesis2019_configuration.cfg b/examples/stfconfig/genesis2019_configuration.cfg similarity index 100% rename from examples/genesis2019_configuration.cfg rename to examples/stfconfig/genesis2019_configuration.cfg diff --git a/examples/sample.cfg b/examples/stfconfig/sample.cfg similarity index 100% rename from examples/sample.cfg rename to examples/stfconfig/sample.cfg diff --git a/examples/sample_dmcosmological_run.cfg b/examples/stfconfig/sample_dmcosmological_run.cfg similarity index 100% rename from examples/sample_dmcosmological_run.cfg rename to examples/stfconfig/sample_dmcosmological_run.cfg diff --git a/examples/sample_eaglehydro_3dfof_subhalo.cfg b/examples/stfconfig/sample_eaglehydro_3dfof_subhalo.cfg similarity index 100% rename from examples/sample_eaglehydro_3dfof_subhalo.cfg rename to examples/stfconfig/sample_eaglehydro_3dfof_subhalo.cfg diff --git a/examples/sample_eaglehydro_6dfof_subhalo.cfg b/examples/stfconfig/sample_eaglehydro_6dfof_subhalo.cfg similarity index 100% rename from examples/sample_eaglehydro_6dfof_subhalo.cfg rename to examples/stfconfig/sample_eaglehydro_6dfof_subhalo.cfg diff --git a/examples/sample_galaxycatalog_run.cfg b/examples/stfconfig/sample_galaxycatalog_run.cfg similarity index 100% rename from examples/sample_galaxycatalog_run.cfg rename to examples/stfconfig/sample_galaxycatalog_run.cfg diff --git a/examples/sample_hydrocosmological_run.cfg b/examples/stfconfig/sample_hydrocosmological_run.cfg similarity index 100% rename from examples/sample_hydrocosmological_run.cfg rename to examples/stfconfig/sample_hydrocosmological_run.cfg diff --git a/examples/sample_swiftdm_3dfof_subhalo.cfg b/examples/stfconfig/sample_swiftdm_3dfof_subhalo.cfg similarity index 100% rename from examples/sample_swiftdm_3dfof_subhalo.cfg rename to examples/stfconfig/sample_swiftdm_3dfof_subhalo.cfg diff --git a/examples/sample_swifthydro_3dfof_subhalo.cfg b/examples/stfconfig/sample_swifthydro_3dfof_subhalo.cfg similarity index 100% rename from examples/sample_swifthydro_3dfof_subhalo.cfg rename to examples/stfconfig/sample_swifthydro_3dfof_subhalo.cfg diff --git a/examples/sample_swifthydro_3dfof_subhalo_extra_properties.cfg b/examples/stfconfig/sample_swifthydro_3dfof_subhalo_extra_properties.cfg similarity index 100% rename from examples/sample_swifthydro_3dfof_subhalo_extra_properties.cfg rename to examples/stfconfig/sample_swifthydro_3dfof_subhalo_extra_properties.cfg diff --git a/examples/sample_swifthydro_6dfof_subhalo.cfg b/examples/stfconfig/sample_swifthydro_6dfof_subhalo.cfg similarity index 100% rename from examples/sample_swifthydro_6dfof_subhalo.cfg rename to examples/stfconfig/sample_swifthydro_6dfof_subhalo.cfg diff --git a/examples/sample_zoomdmcosmological_run.cfg b/examples/stfconfig/sample_zoomdmcosmological_run.cfg similarity index 100% rename from examples/sample_zoomdmcosmological_run.cfg rename to examples/stfconfig/sample_zoomdmcosmological_run.cfg diff --git a/examples/sample_zoomhydrocosmological_run.cfg b/examples/stfconfig/sample_zoomhydrocosmological_run.cfg similarity index 100% rename from examples/sample_zoomhydrocosmological_run.cfg rename to examples/stfconfig/sample_zoomhydrocosmological_run.cfg diff --git a/examples/surfs2018_configuration.cfg b/examples/stfconfig/surfs2018_configuration.cfg similarity index 100% rename from examples/surfs2018_configuration.cfg rename to examples/stfconfig/surfs2018_configuration.cfg diff --git a/src/ompvar.h b/src/ompvar.h index b1699809..b9b84e7d 100644 --- a/src/ompvar.h +++ b/src/ompvar.h @@ -50,4 +50,75 @@ struct OMP_ImportInfo { int task; }; #endif -#endif + +/* + * OMP_Threadpool_datastructure + * - coupled to OMP_Threadpool + * - structure to store threadpool information + * - captures the following: + * thread state active(True)/inactive(false) + * total number of particles + */ +struct OMP_Threadpool_datastructure { + bool active; + Int_t particle_total; +}; + +/* + * OMP_Threadpool + * - contains routines used for optimised parallel computing/processing + * throttle_up - increase number of active threads + * throttle_down - decrease number of active threads + * - captures the following: + * specific threadpool information stored to OMP_Threadpool_datastructure + * total thread size + * number of active threads + */ +struct OMP_Threadpool { + // Total thread available + // Active threads + // Unsigned long long for number of particles + // omp thread class - + // omp retrieve active threads + // specify offset of thread + + /* Private block */ + private: + map details; + int thread_count, active_threads; + + /* Public block */ + public: + void throttle_up(){ + // TODO: add the following logic: + // OMP_Threadpool_datastructure update state of thread from inactive to active + active_threads += 1; + } + + void throttle_down(){ + // TODO: add the following logic: + // OMP_Threadpool_datastructure update state of thread from active to inactive + active_threads -= 1; + } + + void set_total_threads(int thread_count){ + thread_count = thread_count; + } + + int get_total_threads(){ + return thread_count; + } + + int get_active_threads(){ + return active_threads; + } + + /* Confirmation Functions */ + void print_total_threads(){ + printf("Total Threads: %d\n", thread_count); + } + + void print_active_threads(){ + printf("Active Threads: %d\n", active_threads); + } +}; diff --git a/src/search.cxx b/src/search.cxx index 75a2013a..5704c6a5 100644 --- a/src/search.cxx +++ b/src/search.cxx @@ -416,7 +416,7 @@ Int_t* SearchFullSet(Options &opt, const Int_t nbodies, vector &Part, numingroup=BuildNumInGroupTyped(Nlocal,numgroups,pfof,Part.data(),DARKTYPE); for (i=0;i=MINSUBSIZE); - else Part[i].SetType(-1); + else Part[i].SetType(0); numlocalden += (Part[i].GetType()>0); } delete[] numingroup; @@ -2634,10 +2634,6 @@ void RemoveSpuriousDynamicalSubstructures(Options &opt, const Int_t nsubset, Int } } -int setNthreads(){ - return 0; -} - ///adjust to phase centre inline void AdjustSubPartToPhaseCM(Int_t num, Particle *subPart, GMatrix &cmphase) { @@ -2746,11 +2742,12 @@ inline void CleanAndUpdateGroupsFromSubSearch(Options &opt, } } } - + for (auto j=0;j0) pfof[subpglist[j]]=ngroup+ngroupidoffset+subpfof[j]; } + //ngroupidoffset+=subngroup; //now alter subsubpglist so that index pointed is global subset index as global subset is used to get the particles to be searched for subsubstructure for (auto j=1;j<=subngroup;j++) @@ -2920,12 +2917,19 @@ void SearchSubSub(Options &opt, const Int_t nsubset, vector &Partsubse //this routine is within this file, also has internal parallelisation AdjustSubPartToPhaseCM(subnumingroup[i], subPart, cmphase); } + // TODO: Optimise bottleneck + double time_temp; + time_temp = MyGetTime(); PreCalcSearchSubSet(opt, subnumingroup[i], subPart, sublevel); + cout<<"DURATION[PreCalcSearchSubSet]: "< &Partsubse //this routine is within this file, also has internal parallelisation AdjustSubPartToPhaseCM(subnumingroup[i], subPart, cmphase); } + // TODO: Optimise bottleneck + double time_temp; + time_temp = MyGetTime(); PreCalcSearchSubSet(opt2, subnumingroup[i], subPart, sublevel); subpfof = SearchSubset(opt2, subnumingroup[i], subnumingroup[i], subPart, subngroup[i], sublevel, &numcores[i]); @@ -3843,7 +3850,7 @@ private(i,tid,p1,pindex,x1,D2,dval,rval,icheck,nnID,dist2,baryonfofold) map remap; Int_t newng=0, oldpid, newpid; remap[0]=0; - for (i=1;i<=ng;i++) { + for (auto i=1;i<=ng;i++) { if (ningall[i]>0) { newng++; remap[i]=newng;