From dcb52afbd287b30584d0119bffc9b78fa14dea34 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Thu, 14 Sep 2023 22:43:15 -0700 Subject: [PATCH 01/18] added support to query BP5 files --- examples/query/test.cpp | 65 +++++++++++++----------- source/adios2/toolkit/query/BlockIndex.h | 55 +++++++++++++++++--- source/adios2/toolkit/query/Query.cpp | 2 +- 3 files changed, 85 insertions(+), 37 deletions(-) diff --git a/examples/query/test.cpp b/examples/query/test.cpp index 94da39a0dc..589aef577b 100644 --- a/examples/query/test.cpp +++ b/examples/query/test.cpp @@ -12,6 +12,38 @@ // #include "adios2/toolkit/query/Worker.h" +void queryWithStreaming(adios2::IO &queryIO, std::string &dataFileName, std::string &queryFile) +{ + adios2::Engine reader = queryIO.Open(dataFileName, adios2::Mode::Read, MPI_COMM_WORLD); + // adios2::QueryWorker* worker = NULL; + queryIO.SetParameter("StreamReader", "true"); + std::vector> touched_blocks; + + while (reader.BeginStep() == adios2::StepStatus::OK) + { + adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader); + adios2::Box empty; + w.GetResultCoverage(empty, touched_blocks); + // adios2::Box tt({10,10}, {12,12}); + // w.GetResultCoverage(tt, touched_blocks); + std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size() + << std::endl; + for (auto n : touched_blocks) + { + std::ostringstream startStr; + std::ostringstream countStr; + for (size_t k = 0; k < n.first.size(); k++) + { + startStr << n.first[k] << " "; + countStr << n.second[k] << " "; + } + std::cout << "\t[" << startStr.str() << "] [" << countStr.str() << "]" << std::endl; + } + reader.EndStep(); + } + reader.Close(); +} + int main(int argc, char *argv[]) { int provided; @@ -47,43 +79,16 @@ int main(int argc, char *argv[]) adios2::ADIOS ad = adios2::ADIOS(configFileName, MPI_COMM_WORLD); adios2::IO queryIO = ad.DeclareIO("query"); - adios2::Engine reader = queryIO.Open(dataFileName, adios2::Mode::Read, MPI_COMM_WORLD); -#ifdef NEVER - adios2::QueryWorker w = adios2::QueryWorker(configFileName, reader); -#else + std::string queryFile = configFileName; if (argc > 3) { queryFile = argv[3]; } std::cout << "Testing query file ..." << queryFile << std::endl; - adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader); -#endif - std::vector> touched_blocks; - while (reader.BeginStep() == adios2::StepStatus::OK) - { - adios2::Box empty; - w.GetResultCoverage(empty, touched_blocks); - // adios2::Box tt({10,10}, {12,12}); - // w.GetResultCoverage(tt, touched_blocks); - std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size() - << std::endl; - for (auto n : touched_blocks) - { - std::ostringstream startStr; - std::ostringstream countStr; - for (size_t k = 0; k < n.first.size(); k++) - { - startStr << n.first[k] << " "; - countStr << n.second[k] << " "; - } - std::cout << "\t[" << startStr.str() << "] [" << countStr.str() << "]" - << std::endl; - } - reader.EndStep(); - } - reader.Close(); + queryWithStreaming(queryIO, dataFileName, queryFile); + return 0; } catch (std::exception &e) diff --git a/source/adios2/toolkit/query/BlockIndex.h b/source/adios2/toolkit/query/BlockIndex.h index dc6bb8ee7a..e532f7d27f 100644 --- a/source/adios2/toolkit/query/BlockIndex.h +++ b/source/adios2/toolkit/query/BlockIndex.h @@ -22,9 +22,9 @@ class BlockIndex }; public: - BlockIndex(adios2::core::Variable &var, adios2::core::IO &io, + BlockIndex(adios2::core::Variable *var, adios2::core::IO &io, adios2::core::Engine &reader) - : m_Var(var), m_IdxIO(io), m_IdxReader(reader) + : m_VarPtr(var), m_IdxIO(io), m_IdxReader(reader) { } @@ -32,18 +32,57 @@ class BlockIndex void Evaluate(const QueryVar &query, std::vector> &resultSubBlocks) { - RunBP4Stat(query, resultSubBlocks); + if (m_IdxReader.m_EngineType.find("5") != std::string::npos) // a bp5 reader + RunBP5Stat(query, resultSubBlocks); + else + RunBP4Stat(query, resultSubBlocks); + } + + void RunBP5Stat(const QueryVar &query, std::vector> &hitBlocks) + { + size_t currStep = m_IdxReader.CurrentStep(); + adios2::Dims currShape = m_VarPtr->Shape(); + if (!query.IsSelectionValid(currShape)) + return; + + auto MinBlocksInfo = m_IdxReader.MinBlocksInfo(*m_VarPtr, currStep); + if (!MinBlocksInfo) + { // no info, can't do anything + return; + } + for (auto &blockInfo : MinBlocksInfo->BlocksInfo) + { + Dims ss(MinBlocksInfo->Dims); + Dims cc(MinBlocksInfo->Dims); + for (auto i = 0; i < ss.size(); i++) + { + ss[i] = blockInfo.Start[i]; + cc[i] = blockInfo.Count[i]; + } + if (!query.TouchSelection(ss, cc)) + continue; + + T bmin = *(T *)&blockInfo.MinMax.MinUnion; + T bmax = *(T *)&blockInfo.MinMax.MaxUnion; + bool isHit = query.m_RangeTree.CheckInterval(bmin, bmax); + if (isHit) + { + adios2::Box box = {ss, cc}; + hitBlocks.push_back(box); + } + } + delete MinBlocksInfo; } void RunBP4Stat(const QueryVar &query, std::vector> &hitBlocks) { size_t currStep = m_IdxReader.CurrentStep(); - adios2::Dims currShape = m_Var.Shape(); + adios2::Dims currShape = m_VarPtr->Shape(); if (!query.IsSelectionValid(currShape)) return; std::vector::BPInfo> varBlocksInfo = - m_IdxReader.BlocksInfo(m_Var, currStep); + m_IdxReader.BlocksInfo(*m_VarPtr, currStep); for (auto &blockInfo : varBlocksInfo) { @@ -113,7 +152,11 @@ class BlockIndex */ Tree m_Content; - adios2::core::Variable m_Var; + + // can not be unique_ptr as it changes with bp5 through steps + // as BP5Deserializer::SetupForStep calls io.RemoveVariables() + // must use ptr as bp5 associates ptrs with blockinfo, see MinBlocksInfo() in bp5 + adios2::core::Variable *m_VarPtr; private: // diff --git a/source/adios2/toolkit/query/Query.cpp b/source/adios2/toolkit/query/Query.cpp index 97dc3517e6..6dc85b94c5 100644 --- a/source/adios2/toolkit/query/Query.cpp +++ b/source/adios2/toolkit/query/Query.cpp @@ -310,7 +310,7 @@ void QueryVar::BlockIndexEvaluate(adios2::core::IO &io, adios2::core::Engine &re if (varType == adios2::helper::GetDataType()) \ { \ core::Variable *var = io.InquireVariable(m_VarName); \ - BlockIndex idx(*var, io, reader); \ + BlockIndex idx(var, io, reader); \ idx.Evaluate(*this, touchedBlocks); \ } // ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_type) //skip complex types From 94e3d2ab1fdbf35475d3dafc5138b189eb850538 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Thu, 14 Sep 2023 23:25:31 -0700 Subject: [PATCH 02/18] size_type fix --- source/adios2/toolkit/query/BlockIndex.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/toolkit/query/BlockIndex.h b/source/adios2/toolkit/query/BlockIndex.h index e532f7d27f..36dd370e72 100644 --- a/source/adios2/toolkit/query/BlockIndex.h +++ b/source/adios2/toolkit/query/BlockIndex.h @@ -54,7 +54,7 @@ class BlockIndex { Dims ss(MinBlocksInfo->Dims); Dims cc(MinBlocksInfo->Dims); - for (auto i = 0; i < ss.size(); i++) + for (std::vector::size_type i = 0; i < ss.size(); i++) { ss[i] = blockInfo.Start[i]; cc[i] = blockInfo.Count[i]; From b177b4f94edfa388a2ecf121293b5b79563d2929 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Sep 2023 13:17:52 -0700 Subject: [PATCH 03/18] updated python test and added to ctest --- testing/adios2/bindings/python/CMakeLists.txt | 1 + testing/adios2/bindings/python/TestQuery.py | 156 +++++++++++------- 2 files changed, 96 insertions(+), 61 deletions(-) diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt index b90d79a346..dd3c292a75 100644 --- a/testing/adios2/bindings/python/CMakeLists.txt +++ b/testing/adios2/bindings/python/CMakeLists.txt @@ -25,6 +25,7 @@ if(ADIOS2_HAVE_MPI) add_python_mpi_test(BPBlocksInfo) add_python_mpi_test(BPChangingShapeHighLevelAPI) add_python_mpi_test(NullEngine) + add_python_mpi_test(Query) # Currently hangs in H5Fclose for unknown reasons #if(ADIOS2_HAVE_HDF5) # add_python_mpi_test(BPWriteTypesHighLevelAPI_HDF5) diff --git a/testing/adios2/bindings/python/TestQuery.py b/testing/adios2/bindings/python/TestQuery.py index f53df651f4..009fe05f12 100644 --- a/testing/adios2/bindings/python/TestQuery.py +++ b/testing/adios2/bindings/python/TestQuery.py @@ -1,21 +1,86 @@ +# from mpi4py import MPI import numpy as np import adios2 -import os +import sys # MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() -configFile = './defaultConfig.xml' -queryFile = './sampleQuery.xml' -dataPath = './heat.bp' +######################################## +## usage: [bp4 | bp5=default] ## +######################################## +numSteps = 5 +queryFile='query.xml' +targetVarName='var0' +# User data +myArray = np.array([0, 1., 2., 3., 4., 5., 6., 7., 8., 9.]) +Nx = myArray.size -def doAnalysis(reader, touched_blocks, varList): +# ADIOS MPI Communicator +adios = adios2.ADIOS(comm) + +supportedEngines=['bp5', 'bp4'] +engineType='bp5' +if (len(sys.argv) > 1): + engineType = sys.argv[1].lower() + +if (engineType in supportedEngines): + if (rank == 0): + print ('Using engine type:', engineType) +else: + sys.exit('specified engine does not exist') + + +dataFileName= 'test_'+engineType+'.bp' +def writeDataFile(): + # ADIOS IO + bpIO = adios.DeclareIO("Writer") + bpIO.SetEngine(engineType) + + ioArray = bpIO.DefineVariable( + targetVarName, myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims) + + # ADIOS Engine + bpFileWriter = bpIO.Open(dataFileName, adios2.Mode.Write) + + for i in range(numSteps): + bpFileWriter.BeginStep() + bpFileWriter.Put(ioArray, i*10.0 + myArray/(rank+1), adios2.Mode.Sync) + bpFileWriter.EndStep() + + bpFileWriter.Close() + + +def createQueryFile(): + print(".. Writing query file to: ", queryFile) + + file1 = open(queryFile, 'w') + queryContent = [ + "\n", "\n", + " \n" + " \n", + " \n", + " \n", + " \n", " \n", + " \n", " \n", "\n" + ] + file1.writelines(queryContent) + file1.close() + +def doAnalysis(reader, touched_blocks, varList): print(" Step: ", reader.CurrentStep(), " num touched blocks: ", len(touched_blocks)) + if ( 0 == reader.CurrentStep() ): + assert(len(touched_blocks) == min(size, 2)) + if ( 1 == reader.CurrentStep() ): + assert(len(touched_blocks) == size) + if ( 1 < reader.CurrentStep() ): + assert(len(touched_blocks) == 0) + values = [] data = {} @@ -32,76 +97,45 @@ def doAnalysis(reader, touched_blocks, varList): # do analysis with data here -def runQuery(): - adios = adios2.ADIOS(configFile, comm, True) - queryIO = adios.DeclareIO("query") - reader = queryIO.Open(dataPath, adios2.Mode.Read, comm) - w = adios2.Query(queryFile, reader) - +def queryDataFile(): + adios_nompi = adios2.ADIOS() + queryIO = adios_nompi.DeclareIO("query") + reader = queryIO.Open(dataFileName, adios2.Mode.Read) + print("dataFile=", dataFileName, "queryFile=", queryFile) + touched_blocks = [] print("Num steps: ", reader.Steps()) while (reader.BeginStep() == adios2.StepStatus.OK): + # bp5 loads metadata after beginstep(). so query has to be called per step + w = adios2.Query(queryFile, reader) # say only rank 0 wants to process result - var = [queryIO.InquireVariable("T")] + var = [queryIO.InquireVariable(targetVarName)] if (rank == 0): touched_blocks = w.GetResult() doAnalysis(reader, touched_blocks, var) - reader.EndStep() + reader.EndStep() reader.Close() -def createConfigFile(): - print(".. Writing config file to: ", configFile) - file1 = open(configFile, 'w') - - xmlContent = ["\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", "\n"] - - file1.writelines(xmlContent) - file1.close() - - -def createQueryFile(): - print(".. Writing query file to: ", queryFile) - - file1 = open(queryFile, 'w') - queryContent = [ - "\n", "\n", - " \n" - " \n", - " \n", - " \n", - " \n", " \n", - " \n", " \n", "\n" - ] - file1.writelines(queryContent) - file1.close() - - -if (os.path.exists(dataPath) is False): - print("Please generate data file:", dataPath, - " from heat transfer example first.") -else: - # configFile created - createConfigFile() +def cleanUp(): + import os + import shutil + os.remove(queryFile) + shutil.rmtree(dataFileName) + print (" Cleanup generated files: ", queryFile, dataFileName) - # queryFile Generated - createQueryFile() - print(".. Running query against: ", dataPath) - runQuery() + +# +# actual setup: +# +writeDataFile() - print("Now clean up.") - os.remove(queryFile) - os.remove(configFile) +if ( 0 == rank): + createQueryFile(); + queryDataFile() + cleanUp() From dc4ec1bba1210b7755de5fc7eac540d9cdeb3c2b Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Sep 2023 13:19:41 -0700 Subject: [PATCH 04/18] moved QueryWorker to the beginstep/endstep loop because bp5 metadata loading pattern deprecate bp3 engine test and replaced with bp5 engine test --- testing/adios2/performance/query/TestBPQuery.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/testing/adios2/performance/query/TestBPQuery.cpp b/testing/adios2/performance/query/TestBPQuery.cpp index 1ae23c8e87..420660f205 100644 --- a/testing/adios2/performance/query/TestBPQuery.cpp +++ b/testing/adios2/performance/query/TestBPQuery.cpp @@ -14,7 +14,6 @@ #include -// #include "../engine/SmallTestData.h" // std::string engineName; // comes from command line @@ -98,7 +97,6 @@ void BPQueryTest::QueryIntVar(const std::string &fname, adios2::ADIOS &adios, std::string queryFile = "./" + ioName + "test.xml"; //"./test.xml"; std::cout << ioName << std::endl; WriteXmlQuery1D(queryFile, ioName, "intV"); - adios2::QueryWorker w = adios2::QueryWorker(queryFile, bpReader); std::vector rr; if (engineName.compare("BP4") == 0) @@ -108,6 +106,7 @@ void BPQueryTest::QueryIntVar(const std::string &fname, adios2::ADIOS &adios, while (bpReader.BeginStep() == adios2::StepStatus::OK) { + adios2::QueryWorker w = adios2::QueryWorker(queryFile, bpReader); std::vector> touched_blocks; adios2::Box empty; w.GetResultCoverage(empty, touched_blocks); @@ -135,7 +134,6 @@ void BPQueryTest::QueryDoubleVar(const std::string &fname, adios2::ADIOS &adios, // std::string queryFile = "./.test.xml"; std::string queryFile = "./" + ioName + "test.xml"; WriteXmlQuery1D(queryFile, ioName, "doubleV"); - adios2::QueryWorker w = adios2::QueryWorker(queryFile, bpReader); std::vector rr; //= {0,9,9}; if (engineName.compare("BP4") == 0) @@ -144,6 +142,7 @@ void BPQueryTest::QueryDoubleVar(const std::string &fname, adios2::ADIOS &adios, rr = {0, 1, 1}; while (bpReader.BeginStep() == adios2::StepStatus::OK) { + adios2::QueryWorker w = adios2::QueryWorker(queryFile, bpReader); std::vector> touched_blocks; adios2::Box empty; w.GetResultCoverage(empty, touched_blocks); @@ -234,9 +233,9 @@ void BPQueryTest::WriteFile(const std::string &fname, adios2::ADIOS &adios, // 1D test data //****************************************************************************** -TEST_F(BPQueryTest, BP3) +TEST_F(BPQueryTest, BP5) { - std::string engineName = "BP3"; + std::string engineName = "BP5"; // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array const std::string fname(engineName + "Query1D.bp"); From eb35c5b6814e720df8cf7a53c66762f01eec542e Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Sep 2023 13:23:10 -0700 Subject: [PATCH 05/18] clang-format --- testing/adios2/performance/query/TestBPQuery.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/adios2/performance/query/TestBPQuery.cpp b/testing/adios2/performance/query/TestBPQuery.cpp index 420660f205..d84784db59 100644 --- a/testing/adios2/performance/query/TestBPQuery.cpp +++ b/testing/adios2/performance/query/TestBPQuery.cpp @@ -14,7 +14,6 @@ #include - // std::string engineName; // comes from command line struct QueryTestData From 0a63b60db65efd2d79d25be52da1e20e9e4db4a0 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Sep 2023 13:25:57 -0700 Subject: [PATCH 06/18] clang-format --- testing/adios2/bindings/python/TestQuery.py | 25 +++++++++------------ 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/testing/adios2/bindings/python/TestQuery.py b/testing/adios2/bindings/python/TestQuery.py index 009fe05f12..766d986234 100644 --- a/testing/adios2/bindings/python/TestQuery.py +++ b/testing/adios2/bindings/python/TestQuery.py @@ -4,7 +4,7 @@ import adios2 import sys -# MPI +#MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() @@ -16,11 +16,11 @@ queryFile='query.xml' targetVarName='var0' -# User data +#User data myArray = np.array([0, 1., 2., 3., 4., 5., 6., 7., 8., 9.]) Nx = myArray.size -# ADIOS MPI Communicator +#ADIOS MPI Communicator adios = adios2.ADIOS(comm) supportedEngines=['bp5', 'bp4'] @@ -37,14 +37,14 @@ dataFileName= 'test_'+engineType+'.bp' def writeDataFile(): - # ADIOS IO +#ADIOS IO bpIO = adios.DeclareIO("Writer") bpIO.SetEngine(engineType) ioArray = bpIO.DefineVariable( targetVarName, myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims) - # ADIOS Engine +#ADIOS Engine bpFileWriter = bpIO.Open(dataFileName, adios2.Mode.Write) for i in range(numSteps): @@ -94,7 +94,7 @@ def doAnalysis(reader, touched_blocks, varList): var.SetSelection(n) reader.Get(var, values, adios2.Mode.Sync) data[var].extend(values) - # do analysis with data here +#do analysis with data here def queryDataFile(): @@ -108,9 +108,9 @@ def queryDataFile(): print("Num steps: ", reader.Steps()) while (reader.BeginStep() == adios2.StepStatus.OK): - # bp5 loads metadata after beginstep(). so query has to be called per step +#bp5 loads metadata after beginstep().so query has to be called per step w = adios2.Query(queryFile, reader) - # say only rank 0 wants to process result +#say only rank 0 wants to process result var = [queryIO.InquireVariable(targetVarName)] if (rank == 0): @@ -128,14 +128,11 @@ def cleanUp(): shutil.rmtree(dataFileName) print (" Cleanup generated files: ", queryFile, dataFileName) - - # -# actual setup: +#actual setup: # writeDataFile() if ( 0 == rank): - createQueryFile(); - queryDataFile() - cleanUp() + createQueryFile(); +queryDataFile() cleanUp() From 4de4b38e9003589f30e19954e425829a30faeeb0 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Sep 2023 13:29:40 -0700 Subject: [PATCH 07/18] removed some comments --- testing/adios2/bindings/python/TestQuery.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/testing/adios2/bindings/python/TestQuery.py b/testing/adios2/bindings/python/TestQuery.py index 766d986234..33a39f9a9c 100644 --- a/testing/adios2/bindings/python/TestQuery.py +++ b/testing/adios2/bindings/python/TestQuery.py @@ -37,14 +37,12 @@ dataFileName= 'test_'+engineType+'.bp' def writeDataFile(): -#ADIOS IO bpIO = adios.DeclareIO("Writer") bpIO.SetEngine(engineType) ioArray = bpIO.DefineVariable( targetVarName, myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims) -#ADIOS Engine bpFileWriter = bpIO.Open(dataFileName, adios2.Mode.Write) for i in range(numSteps): @@ -98,8 +96,10 @@ def doAnalysis(reader, touched_blocks, varList): def queryDataFile(): - adios_nompi = adios2.ADIOS() + ## use no mpi + adios_nompi = adios2.ADIOS() queryIO = adios_nompi.DeclareIO("query") + reader = queryIO.Open(dataFileName, adios2.Mode.Read) print("dataFile=", dataFileName, "queryFile=", queryFile) @@ -108,9 +108,9 @@ def queryDataFile(): print("Num steps: ", reader.Steps()) while (reader.BeginStep() == adios2.StepStatus.OK): -#bp5 loads metadata after beginstep().so query has to be called per step + # bp5 loads metadata after beginstep().so query has to be called per step w = adios2.Query(queryFile, reader) -#say only rank 0 wants to process result + # assume only rank 0 wants to process result var = [queryIO.InquireVariable(targetVarName)] if (rank == 0): @@ -135,4 +135,6 @@ def cleanUp(): if ( 0 == rank): createQueryFile(); -queryDataFile() cleanUp() + + queryDataFile() + cleanUp() From f5339ecc3992bba8f5817c912b6f48c9ced4b518 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Sep 2023 13:55:05 -0700 Subject: [PATCH 08/18] flake8 --- .../adios2/bindings/python/TestNullEngine.py | 163 +++++++++++++----- 1 file changed, 116 insertions(+), 47 deletions(-) diff --git a/testing/adios2/bindings/python/TestNullEngine.py b/testing/adios2/bindings/python/TestNullEngine.py index dfe1c0069c..928b3ca104 100644 --- a/testing/adios2/bindings/python/TestNullEngine.py +++ b/testing/adios2/bindings/python/TestNullEngine.py @@ -1,75 +1,144 @@ # -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -# -# TestNullEngine.py -# -# -# Created on : Apr 11th, 2019 -# Author : William F Godoy godoywf @ornl.gov -# - from mpi4py import MPI import numpy as np import adios2 +import sys # MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() +# ####################################### +# # usage: [bp4 | bp5=default] ## +# ####################################### +numSteps = 5 +queryFile = 'query.xml' +targetVarName = 'var0' + # User data -Nx = 10 -Ny = 10 +myArray = np.array([0, 1., 2., 3., 4., 5., 6., 7., 8., 9.]) +Nx = myArray.size -count = [Nx, Ny] -start = [rank * Nx, 0] -shape = [size * Nx, Ny] +# ADIOS MPI Communicator +adios = adios2.ADIOS(comm) -temperatures = np.zeros(count, dtype=np.int32) +supportedEngines = ['bp5', 'bp4'] +engineType = 'bp5' +if (len(sys.argv) > 1): + engineType = sys.argv[1].lower() -for i in range(0, Nx): - for j in range(0, Ny): - temperatures[i, j] = (start[0] + i) * shape[1] + (j + start[1]) +if (engineType in supportedEngines): + if (rank == 0): + print('Using engine type:', engineType) +else: + sys.exit('specified engine does not exist') -# ADIOS write -adios = adios2.ADIOS(comm) -ioWrite = adios.DeclareIO("ioWriter") -varTemperature = ioWrite.DefineVariable("temperature2D", temperatures, shape, - start, count, adios2.ConstantDims) -ioWrite.SetEngine("NULL") +dataFileName = 'test_'+engineType+'.bp' + + +def writeDataFile(): + bpIO = adios.DeclareIO("Writer") + bpIO.SetEngine(engineType) + + ioArray = bpIO.DefineVariable( + targetVarName, myArray, [size * Nx], [rank * Nx], + [Nx], adios2.ConstantDims) + + bpFileWriter = bpIO.Open(dataFileName, adios2.Mode.Write) + + for i in range(numSteps): + bpFileWriter.BeginStep() + bpFileWriter.Put(ioArray, i*10.0 + myArray/(rank+1), adios2.Mode.Sync) + bpFileWriter.EndStep() -nullWriter = ioWrite.Open('NULL_py.bp', adios2.Mode.Write) + bpFileWriter.Close() -assert (nullWriter.Type() == "NullWriter") -status = nullWriter.BeginStep() -assert (status == adios2.StepStatus.OK) +def createQueryFile(): + print(".. Writing query file to: ", queryFile) -nullWriter.Put(varTemperature, temperatures) -nullWriter.EndStep() -nullWriter.Close() + file1 = open(queryFile, 'w') + queryContent = [ + "\n", "\n", + " \n" + " \n", + " \n", + " \n", + " \n", " \n", + " \n", " \n", "\n" + ] + file1.writelines(queryContent) + file1.close() -# ADIOS2 read -ioRead = adios.DeclareIO("ioReader") -ioRead.SetEngine("null") -nullReader = ioRead.Open('NULL_py.bp', adios2.Mode.Read, MPI.COMM_SELF) -assert (nullReader.Type() == "NullReader") +def doAnalysis(reader, touched_blocks, varList): + print(" Step: ", reader.CurrentStep(), + " num touched blocks: ", len(touched_blocks)) + if (0 == reader.CurrentStep()): + assert (len(touched_blocks) == min(size, 2)) + if (1 == reader.CurrentStep()): + assert (len(touched_blocks) == size) + if (1 < reader.CurrentStep()): + assert (len(touched_blocks) == 0) -inTemperatures = np.zeros(1, dtype=np.int32) + values = [] + data = {} -status = nullReader.BeginStep() -assert (status == adios2.StepStatus.EndOfStream) + for var in varList: + data[var] = [] -var_inTemperature = ioRead.InquireVariable("temperature2D") + if (len(touched_blocks) > 0): + for n in touched_blocks: + for var in varList: + values = np.zeros(n[1], dtype=np.double) + var.SetSelection(n) + reader.Get(var, values, adios2.Mode.Sync) + data[var].extend(values) + + +def queryDataFile(): + # # use no mpi + adios_nompi = adios2.ADIOS() + queryIO = adios_nompi.DeclareIO("query") + + reader = queryIO.Open(dataFileName, adios2.Mode.Read) + print("dataFile=", dataFileName, "queryFile=", queryFile) + touched_blocks = [] + + print("Num steps: ", reader.Steps()) + + while (reader.BeginStep() == adios2.StepStatus.OK): + # bp5 loads metadata after beginstep(), + # therefore query has to be called per step + w = adios2.Query(queryFile, reader) + # assume only rank 0 wants to process result + var = [queryIO.InquireVariable(targetVarName)] + + if (rank == 0): + touched_blocks = w.GetResult() + doAnalysis(reader, touched_blocks, var) + + reader.EndStep() + reader.Close() + + +def cleanUp(): + import os + import shutil + os.remove(queryFile) + shutil.rmtree(dataFileName) + print(" Cleanup generated files: ", queryFile, dataFileName) + +# +# actual setup: +# -if (var_inTemperature is True): - raise ValueError('var_inTemperature is not False') -# nullReader.Get(var_inTemperature, inTemperatures) +writeDataFile() -nullReader.PerformGets() -nullReader.EndStep() -nullReader.Close() +if (0 == rank): + createQueryFile() + queryDataFile() + cleanUp() From e8628d311b79d2d886fe1c423c9ea3058b6866e5 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Sep 2023 14:01:09 -0700 Subject: [PATCH 09/18] correct previous commit. Wrong files changed --- .../adios2/bindings/python/TestNullEngine.py | 163 +++++------------- testing/adios2/bindings/python/TestQuery.py | 76 ++++---- 2 files changed, 87 insertions(+), 152 deletions(-) diff --git a/testing/adios2/bindings/python/TestNullEngine.py b/testing/adios2/bindings/python/TestNullEngine.py index 928b3ca104..dfe1c0069c 100644 --- a/testing/adios2/bindings/python/TestNullEngine.py +++ b/testing/adios2/bindings/python/TestNullEngine.py @@ -1,144 +1,75 @@ # +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# TestNullEngine.py +# +# +# Created on : Apr 11th, 2019 +# Author : William F Godoy godoywf @ornl.gov +# + from mpi4py import MPI import numpy as np import adios2 -import sys # MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() -# ####################################### -# # usage: [bp4 | bp5=default] ## -# ####################################### -numSteps = 5 -queryFile = 'query.xml' -targetVarName = 'var0' - # User data -myArray = np.array([0, 1., 2., 3., 4., 5., 6., 7., 8., 9.]) -Nx = myArray.size - -# ADIOS MPI Communicator -adios = adios2.ADIOS(comm) - -supportedEngines = ['bp5', 'bp4'] -engineType = 'bp5' -if (len(sys.argv) > 1): - engineType = sys.argv[1].lower() - -if (engineType in supportedEngines): - if (rank == 0): - print('Using engine type:', engineType) -else: - sys.exit('specified engine does not exist') - - -dataFileName = 'test_'+engineType+'.bp' - +Nx = 10 +Ny = 10 -def writeDataFile(): - bpIO = adios.DeclareIO("Writer") - bpIO.SetEngine(engineType) +count = [Nx, Ny] +start = [rank * Nx, 0] +shape = [size * Nx, Ny] - ioArray = bpIO.DefineVariable( - targetVarName, myArray, [size * Nx], [rank * Nx], - [Nx], adios2.ConstantDims) +temperatures = np.zeros(count, dtype=np.int32) - bpFileWriter = bpIO.Open(dataFileName, adios2.Mode.Write) +for i in range(0, Nx): + for j in range(0, Ny): + temperatures[i, j] = (start[0] + i) * shape[1] + (j + start[1]) - for i in range(numSteps): - bpFileWriter.BeginStep() - bpFileWriter.Put(ioArray, i*10.0 + myArray/(rank+1), adios2.Mode.Sync) - bpFileWriter.EndStep() - - bpFileWriter.Close() - - -def createQueryFile(): - print(".. Writing query file to: ", queryFile) - - file1 = open(queryFile, 'w') - queryContent = [ - "\n", "\n", - " \n" - " \n", - " \n", - " \n", - " \n", " \n", - " \n", " \n", "\n" - ] - file1.writelines(queryContent) - file1.close() - - -def doAnalysis(reader, touched_blocks, varList): - print(" Step: ", reader.CurrentStep(), - " num touched blocks: ", len(touched_blocks)) - if (0 == reader.CurrentStep()): - assert (len(touched_blocks) == min(size, 2)) - if (1 == reader.CurrentStep()): - assert (len(touched_blocks) == size) - if (1 < reader.CurrentStep()): - assert (len(touched_blocks) == 0) - - values = [] - data = {} - - for var in varList: - data[var] = [] - - if (len(touched_blocks) > 0): - for n in touched_blocks: - for var in varList: - values = np.zeros(n[1], dtype=np.double) - var.SetSelection(n) - reader.Get(var, values, adios2.Mode.Sync) - data[var].extend(values) +# ADIOS write +adios = adios2.ADIOS(comm) +ioWrite = adios.DeclareIO("ioWriter") +varTemperature = ioWrite.DefineVariable("temperature2D", temperatures, shape, + start, count, adios2.ConstantDims) +ioWrite.SetEngine("NULL") -def queryDataFile(): - # # use no mpi - adios_nompi = adios2.ADIOS() - queryIO = adios_nompi.DeclareIO("query") +nullWriter = ioWrite.Open('NULL_py.bp', adios2.Mode.Write) - reader = queryIO.Open(dataFileName, adios2.Mode.Read) - print("dataFile=", dataFileName, "queryFile=", queryFile) - touched_blocks = [] +assert (nullWriter.Type() == "NullWriter") - print("Num steps: ", reader.Steps()) +status = nullWriter.BeginStep() +assert (status == adios2.StepStatus.OK) - while (reader.BeginStep() == adios2.StepStatus.OK): - # bp5 loads metadata after beginstep(), - # therefore query has to be called per step - w = adios2.Query(queryFile, reader) - # assume only rank 0 wants to process result - var = [queryIO.InquireVariable(targetVarName)] +nullWriter.Put(varTemperature, temperatures) +nullWriter.EndStep() +nullWriter.Close() - if (rank == 0): - touched_blocks = w.GetResult() - doAnalysis(reader, touched_blocks, var) +# ADIOS2 read +ioRead = adios.DeclareIO("ioReader") +ioRead.SetEngine("null") +nullReader = ioRead.Open('NULL_py.bp', adios2.Mode.Read, MPI.COMM_SELF) - reader.EndStep() - reader.Close() +assert (nullReader.Type() == "NullReader") +inTemperatures = np.zeros(1, dtype=np.int32) -def cleanUp(): - import os - import shutil - os.remove(queryFile) - shutil.rmtree(dataFileName) - print(" Cleanup generated files: ", queryFile, dataFileName) +status = nullReader.BeginStep() +assert (status == adios2.StepStatus.EndOfStream) -# -# actual setup: -# +var_inTemperature = ioRead.InquireVariable("temperature2D") +if (var_inTemperature is True): + raise ValueError('var_inTemperature is not False') -writeDataFile() +# nullReader.Get(var_inTemperature, inTemperatures) -if (0 == rank): - createQueryFile() - queryDataFile() - cleanUp() +nullReader.PerformGets() +nullReader.EndStep() +nullReader.Close() diff --git a/testing/adios2/bindings/python/TestQuery.py b/testing/adios2/bindings/python/TestQuery.py index 33a39f9a9c..928b3ca104 100644 --- a/testing/adios2/bindings/python/TestQuery.py +++ b/testing/adios2/bindings/python/TestQuery.py @@ -4,44 +4,47 @@ import adios2 import sys -#MPI +# MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() -######################################## -## usage: [bp4 | bp5=default] ## -######################################## +# ####################################### +# # usage: [bp4 | bp5=default] ## +# ####################################### numSteps = 5 -queryFile='query.xml' -targetVarName='var0' +queryFile = 'query.xml' +targetVarName = 'var0' -#User data +# User data myArray = np.array([0, 1., 2., 3., 4., 5., 6., 7., 8., 9.]) Nx = myArray.size -#ADIOS MPI Communicator +# ADIOS MPI Communicator adios = adios2.ADIOS(comm) -supportedEngines=['bp5', 'bp4'] -engineType='bp5' +supportedEngines = ['bp5', 'bp4'] +engineType = 'bp5' if (len(sys.argv) > 1): engineType = sys.argv[1].lower() if (engineType in supportedEngines): - if (rank == 0): - print ('Using engine type:', engineType) + if (rank == 0): + print('Using engine type:', engineType) else: sys.exit('specified engine does not exist') -dataFileName= 'test_'+engineType+'.bp' +dataFileName = 'test_'+engineType+'.bp' + + def writeDataFile(): bpIO = adios.DeclareIO("Writer") bpIO.SetEngine(engineType) ioArray = bpIO.DefineVariable( - targetVarName, myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims) + targetVarName, myArray, [size * Nx], [rank * Nx], + [Nx], adios2.ConstantDims) bpFileWriter = bpIO.Open(dataFileName, adios2.Mode.Write) @@ -49,13 +52,13 @@ def writeDataFile(): bpFileWriter.BeginStep() bpFileWriter.Put(ioArray, i*10.0 + myArray/(rank+1), adios2.Mode.Sync) bpFileWriter.EndStep() - + bpFileWriter.Close() def createQueryFile(): print(".. Writing query file to: ", queryFile) - + file1 = open(queryFile, 'w') queryContent = [ "\n", "\n", @@ -69,16 +72,17 @@ def createQueryFile(): file1.writelines(queryContent) file1.close() -def doAnalysis(reader, touched_blocks, varList): + +def doAnalysis(reader, touched_blocks, varList): print(" Step: ", reader.CurrentStep(), " num touched blocks: ", len(touched_blocks)) - if ( 0 == reader.CurrentStep() ): - assert(len(touched_blocks) == min(size, 2)) - if ( 1 == reader.CurrentStep() ): - assert(len(touched_blocks) == size) - if ( 1 < reader.CurrentStep() ): - assert(len(touched_blocks) == 0) - + if (0 == reader.CurrentStep()): + assert (len(touched_blocks) == min(size, 2)) + if (1 == reader.CurrentStep()): + assert (len(touched_blocks) == size) + if (1 < reader.CurrentStep()): + assert (len(touched_blocks) == 0) + values = [] data = {} @@ -92,23 +96,22 @@ def doAnalysis(reader, touched_blocks, varList): var.SetSelection(n) reader.Get(var, values, adios2.Mode.Sync) data[var].extend(values) -#do analysis with data here def queryDataFile(): - ## use no mpi - adios_nompi = adios2.ADIOS() + # # use no mpi + adios_nompi = adios2.ADIOS() queryIO = adios_nompi.DeclareIO("query") - + reader = queryIO.Open(dataFileName, adios2.Mode.Read) - print("dataFile=", dataFileName, "queryFile=", queryFile) - + print("dataFile=", dataFileName, "queryFile=", queryFile) touched_blocks = [] print("Num steps: ", reader.Steps()) while (reader.BeginStep() == adios2.StepStatus.OK): - # bp5 loads metadata after beginstep().so query has to be called per step + # bp5 loads metadata after beginstep(), + # therefore query has to be called per step w = adios2.Query(queryFile, reader) # assume only rank 0 wants to process result var = [queryIO.InquireVariable(targetVarName)] @@ -126,15 +129,16 @@ def cleanUp(): import shutil os.remove(queryFile) shutil.rmtree(dataFileName) - print (" Cleanup generated files: ", queryFile, dataFileName) + print(" Cleanup generated files: ", queryFile, dataFileName) # -#actual setup: +# actual setup: # + + writeDataFile() -if ( 0 == rank): - createQueryFile(); - +if (0 == rank): + createQueryFile() queryDataFile() cleanUp() From 7bcdfb02f684add85afcf0befad4c9d00406ad83 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Sep 2023 14:04:25 -0700 Subject: [PATCH 10/18] furthur formatting --- testing/adios2/bindings/python/TestQuery.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/adios2/bindings/python/TestQuery.py b/testing/adios2/bindings/python/TestQuery.py index 928b3ca104..e406ce3cf5 100644 --- a/testing/adios2/bindings/python/TestQuery.py +++ b/testing/adios2/bindings/python/TestQuery.py @@ -35,7 +35,7 @@ sys.exit('specified engine does not exist') -dataFileName = 'test_'+engineType+'.bp' +dataFileName = 'test_' + engineType + '.bp' def writeDataFile(): @@ -50,7 +50,7 @@ def writeDataFile(): for i in range(numSteps): bpFileWriter.BeginStep() - bpFileWriter.Put(ioArray, i*10.0 + myArray/(rank+1), adios2.Mode.Sync) + bpFileWriter.Put(ioArray, i * 10.0 + myArray / (rank + 1), adios2.Mode.Sync) bpFileWriter.EndStep() bpFileWriter.Close() @@ -63,7 +63,7 @@ def createQueryFile(): queryContent = [ "\n", "\n", " \n" - " \n", + " \n", " \n", " \n", " \n", " \n", From e4c749d94e40d6a4b8b982770a145b02cecf2651 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Mon, 18 Sep 2023 17:48:53 -0700 Subject: [PATCH 11/18] removed comments --- examples/query/test.cpp | 3 +-- source/adios2/toolkit/query/BlockIndex.h | 32 ------------------------ 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/examples/query/test.cpp b/examples/query/test.cpp index 589aef577b..8d866a3085 100644 --- a/examples/query/test.cpp +++ b/examples/query/test.cpp @@ -24,8 +24,7 @@ void queryWithStreaming(adios2::IO &queryIO, std::string &dataFileName, std::str adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader); adios2::Box empty; w.GetResultCoverage(empty, touched_blocks); - // adios2::Box tt({10,10}, {12,12}); - // w.GetResultCoverage(tt, touched_blocks); + std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size() << std::endl; for (auto n : touched_blocks) diff --git a/source/adios2/toolkit/query/BlockIndex.h b/source/adios2/toolkit/query/BlockIndex.h index 36dd370e72..38babc492d 100644 --- a/source/adios2/toolkit/query/BlockIndex.h +++ b/source/adios2/toolkit/query/BlockIndex.h @@ -119,38 +119,6 @@ class BlockIndex } } - /* - void RunDefaultBPStat(const QueryVar &query, - std::vector> &hitBlocks) - { - size_t currStep = m_IdxReader.CurrentStep(); - adios2::Dims currShape = m_Var.Shape(); - if (!query.IsSelectionValid(currShape)) - return; - - std::vector::BPInfo> varBlocksInfo = - m_IdxReader.BlocksInfo(m_Var, currStep); - - for (auto &blockInfo : varBlocksInfo) - { - if (!query.TouchSelection(blockInfo.Start, blockInfo.Count)) - continue; - - T min = blockInfo.Min; - T max = blockInfo.Max; - - // std::cout<<" min: "< box = {blockInfo.Start, - blockInfo.Count}; - hitBlocks.push_back(box); - } - } - } - */ - Tree m_Content; // can not be unique_ptr as it changes with bp5 through steps From de8ad7b692790fa945465c19af9af3f6e59cdd8e Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Mon, 18 Sep 2023 17:51:27 -0700 Subject: [PATCH 12/18] added new doc file --- docs/user_guide/source/advanced/query.rst | 86 +++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 docs/user_guide/source/advanced/query.rst diff --git a/docs/user_guide/source/advanced/query.rst b/docs/user_guide/source/advanced/query.rst new file mode 100644 index 0000000000..dc43045034 --- /dev/null +++ b/docs/user_guide/source/advanced/query.rst @@ -0,0 +1,86 @@ +################# +ADIOS2 query API +################# + +The query API in ADIOS2 allows a client to pass query in XML or json format, +and get back a list of blocks or subblocks that contains hits. + + +The interface +============= +User is expected to pass a query file (configFile), and init a read engine (engine) +to construct a query and evaluate using the engine. +(note that the engine and query should be using the same ADIOS IO) + +.. code-block:: c++ + + class QueryWorker + { + public: + // configFile has query, can be either xml or json + QueryWorker(const std::string &configFile, adios2::Engine &engine); + // touched_blocks is a list of regions specified by (start, count), + // that contains data that satisfies the query file + void GetResultCoverage(adios2::Box &, + std::vector> &touched_blocks); + } + + +Sample Query: +------------- +.. code-block:: xml + + + + + + + + + + + + + + + + + +EXAMPLES: +========= +C++: +---- +.. code-block:: c++ + + while (reader.BeginStep() == adios2::StepStatus::OK) + { + adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader); + adios2::Box empty; // no output restriction + w.GetResultCoverage(empty, touched_blocks); + + std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size() + << std::endl; + } + +Full C++ example is here: + https://github.com/ornladios/ADIOS2/blob/master/examples/query/test.cpp + + +Python: +------- + +.. code-block:: python + + while (reader.BeginStep() == adios2.StepStatus.OK): + # say only rank 0 wants to process result + var = [queryIO.InquireVariable("T")] + + if (rank == 0): + touched_blocks = w.GetResult() + doAnalysis(reader, touched_blocks, var) + +Full python example is here: + https://github.com/ornladios/ADIOS2/blob/master/testing/adios2/bindings/python/TestQuery.py + + This example generates data, the query file (in xml) and runs the query, all in python. + From 935ce0a22276f7902a7f69b232181fbd7611ec3f Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Mon, 18 Sep 2023 18:10:30 -0700 Subject: [PATCH 13/18] Polished query.rst --- docs/user_guide/source/advanced/query.rst | 43 ++++++++++++----------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/docs/user_guide/source/advanced/query.rst b/docs/user_guide/source/advanced/query.rst index dc43045034..bd092bd9b3 100644 --- a/docs/user_guide/source/advanced/query.rst +++ b/docs/user_guide/source/advanced/query.rst @@ -24,30 +24,31 @@ to construct a query and evaluate using the engine. void GetResultCoverage(adios2::Box &, std::vector> &touched_blocks); } - -Sample Query: -------------- +A Sample Compound Query: +---------------------- + .. code-block:: xml - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + -EXAMPLES: -========= +Code EXAMPLES: +============== C++: ---- .. code-block:: c++ @@ -62,7 +63,7 @@ C++: << std::endl; } -Full C++ example is here: +The Full C++ example is here: https://github.com/ornladios/ADIOS2/blob/master/examples/query/test.cpp From 796f5db856868969f315e2f3203506f8ff583c77 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 19 Sep 2023 11:18:59 -0700 Subject: [PATCH 14/18] more touch ups --- docs/user_guide/source/advanced/query.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/user_guide/source/advanced/query.rst b/docs/user_guide/source/advanced/query.rst index bd092bd9b3..852e0e52dc 100644 --- a/docs/user_guide/source/advanced/query.rst +++ b/docs/user_guide/source/advanced/query.rst @@ -2,8 +2,9 @@ ADIOS2 query API ################# -The query API in ADIOS2 allows a client to pass query in XML or json format, +The query API in ADIOS2 allows a client to pass a query in XML or json format, and get back a list of blocks or subblocks that contains hits. +Both BP4 and BP5 engines are supported. The interface @@ -19,20 +20,25 @@ to construct a query and evaluate using the engine. public: // configFile has query, can be either xml or json QueryWorker(const std::string &configFile, adios2::Engine &engine); + // touched_blocks is a list of regions specified by (start, count), // that contains data that satisfies the query file - void GetResultCoverage(adios2::Box &, - std::vector> &touched_blocks); + void GetResultCoverage(std::vector> &touched_blocks); + ... } -A Sample Compound Query: +A Sample Compound Query ---------------------- +This query targets a 1D variable "doubleV", data of interest is (x > 6.6) or (x < -0.17) or (2.8 < x < 2.9) +In addition, this query also specied an output region [start=5,count=80]. + + .. code-block:: xml - + @@ -56,13 +62,13 @@ C++: while (reader.BeginStep() == adios2::StepStatus::OK) { adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader); - adios2::Box empty; // no output restriction - w.GetResultCoverage(empty, touched_blocks); + w.GetResultCoverage(touched_blocks); std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size() << std::endl; } + The Full C++ example is here: https://github.com/ornladios/ADIOS2/blob/master/examples/query/test.cpp From 97ac3a9aeb2fafc9f23670fb5e336b334c0b185a Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 19 Sep 2023 12:47:25 -0700 Subject: [PATCH 15/18] more clean ups --- bindings/CXX11/adios2/cxx11/Query.cpp | 8 +++++++- bindings/CXX11/adios2/cxx11/Query.h | 8 +++++++- bindings/Python/py11Query.cpp | 3 +-- bindings/Python/py11Query.h | 2 -- examples/query/test.cpp | 3 +-- testing/adios2/performance/query/TestBPQuery.cpp | 8 +++----- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/bindings/CXX11/adios2/cxx11/Query.cpp b/bindings/CXX11/adios2/cxx11/Query.cpp index 23bbd8c6c6..ae5b3f3eb0 100644 --- a/bindings/CXX11/adios2/cxx11/Query.cpp +++ b/bindings/CXX11/adios2/cxx11/Query.cpp @@ -14,7 +14,13 @@ QueryWorker::QueryWorker(const std::string &configFile, adios2::Engine &reader) delete m; } -void QueryWorker::GetResultCoverage(adios2::Box &outputSelection, +void QueryWorker::GetResultCoverage(std::vector> &touched_blocks) +{ + adios2::Box empty; + GetResultCoverage(empty, touched_blocks); +} + +void QueryWorker::GetResultCoverage(const adios2::Box &outputSelection, std::vector> &touched_blocks) { if (m_Worker) diff --git a/bindings/CXX11/adios2/cxx11/Query.h b/bindings/CXX11/adios2/cxx11/Query.h index a4dd046d38..1b26ab62d6 100644 --- a/bindings/CXX11/adios2/cxx11/Query.h +++ b/bindings/CXX11/adios2/cxx11/Query.h @@ -26,9 +26,15 @@ class Worker; class QueryWorker { public: + // configFile has query, can be either xml or json QueryWorker(const std::string &configFile, adios2::Engine &engine); - void GetResultCoverage(adios2::Box &, + // touched_blocks is a list of regions specified by (start, count), + // that contains data that satisfies the query file + void GetResultCoverage(std::vector> &touched_blocks); + + // supply output bound for the results + void GetResultCoverage(const adios2::Box &, std::vector> &touched_blocks); private: diff --git a/bindings/Python/py11Query.cpp b/bindings/Python/py11Query.cpp index 87c2014967..0e82cae6bb 100644 --- a/bindings/Python/py11Query.cpp +++ b/bindings/Python/py11Query.cpp @@ -30,9 +30,8 @@ Query::operator bool() const noexcept { return (m_QueryWorker == nullptr) ? fals std::vector> Query::GetResult() { - // std::cout<<"Do something"< empty; // look into all data std::vector> touched_blocks; + adios2::Box empty; m_QueryWorker->GetResultCoverage(empty, touched_blocks); return touched_blocks; } diff --git a/bindings/Python/py11Query.h b/bindings/Python/py11Query.h index df04fd3e59..b8cd29018b 100644 --- a/bindings/Python/py11Query.h +++ b/bindings/Python/py11Query.h @@ -34,8 +34,6 @@ class Query explicit operator bool() const noexcept; std::vector> GetResult(); - // const Box< Dims > & refinedSelectionIfAny, - // std::vector< Box< Dims > > &touched_blocks private: Query(adios2::query::Worker *qw); diff --git a/examples/query/test.cpp b/examples/query/test.cpp index 8d866a3085..c825fbfea6 100644 --- a/examples/query/test.cpp +++ b/examples/query/test.cpp @@ -22,8 +22,7 @@ void queryWithStreaming(adios2::IO &queryIO, std::string &dataFileName, std::str while (reader.BeginStep() == adios2::StepStatus::OK) { adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader); - adios2::Box empty; - w.GetResultCoverage(empty, touched_blocks); + w.GetResultCoverage(touched_blocks); std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size() << std::endl; diff --git a/testing/adios2/performance/query/TestBPQuery.cpp b/testing/adios2/performance/query/TestBPQuery.cpp index d84784db59..a3ff5ae4a2 100644 --- a/testing/adios2/performance/query/TestBPQuery.cpp +++ b/testing/adios2/performance/query/TestBPQuery.cpp @@ -107,8 +107,7 @@ void BPQueryTest::QueryIntVar(const std::string &fname, adios2::ADIOS &adios, { adios2::QueryWorker w = adios2::QueryWorker(queryFile, bpReader); std::vector> touched_blocks; - adios2::Box empty; - w.GetResultCoverage(empty, touched_blocks); + w.GetResultCoverage(touched_blocks); ASSERT_EQ(touched_blocks.size(), rr[bpReader.CurrentStep()]); bpReader.EndStep(); } @@ -143,8 +142,7 @@ void BPQueryTest::QueryDoubleVar(const std::string &fname, adios2::ADIOS &adios, { adios2::QueryWorker w = adios2::QueryWorker(queryFile, bpReader); std::vector> touched_blocks; - adios2::Box empty; - w.GetResultCoverage(empty, touched_blocks); + w.GetResultCoverage(touched_blocks); ASSERT_EQ(touched_blocks.size(), rr[bpReader.CurrentStep()]); bpReader.EndStep(); } @@ -263,7 +261,7 @@ TEST_F(BPQueryTest, BP4) std::string engineName = "BP4"; // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname(engineName + "4Query1D.bp"); + const std::string fname(engineName + "Query1D.bp"); #if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); From c515be3f2b932a5bb8666be8ca36cd92d3188870 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 19 Sep 2023 13:19:22 -0700 Subject: [PATCH 16/18] subblock calculated by the helper function is relative to block so added block start before return to user --- source/adios2/toolkit/query/BlockIndex.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/adios2/toolkit/query/BlockIndex.h b/source/adios2/toolkit/query/BlockIndex.h index 38babc492d..94890de269 100644 --- a/source/adios2/toolkit/query/BlockIndex.h +++ b/source/adios2/toolkit/query/BlockIndex.h @@ -101,6 +101,10 @@ class BlockIndex { adios2::Box currSubBlock = adios2::helper::GetSubBlock(blockInfo.Count, blockInfo.SubBlockInfo, i); + for (size_t d = 0; d < blockInfo.Count.size(); ++d) + { + currSubBlock.first[d] += blockInfo.Start[d]; + } if (!query.TouchSelection(currSubBlock.first, currSubBlock.second)) continue; hitBlocks.push_back(currSubBlock); From 01d8f292ea7d99952f46867622096ef1c5fe8bec Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 19 Sep 2023 13:37:42 -0700 Subject: [PATCH 17/18] clang-format fix --- bindings/CXX11/adios2/cxx11/Query.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/CXX11/adios2/cxx11/Query.cpp b/bindings/CXX11/adios2/cxx11/Query.cpp index ae5b3f3eb0..140d1485c6 100644 --- a/bindings/CXX11/adios2/cxx11/Query.cpp +++ b/bindings/CXX11/adios2/cxx11/Query.cpp @@ -16,8 +16,8 @@ QueryWorker::QueryWorker(const std::string &configFile, adios2::Engine &reader) void QueryWorker::GetResultCoverage(std::vector> &touched_blocks) { - adios2::Box empty; - GetResultCoverage(empty, touched_blocks); + adios2::Box empty; + GetResultCoverage(empty, touched_blocks); } void QueryWorker::GetResultCoverage(const adios2::Box &outputSelection, From 1efdff058c03a26b3338910cf4df9154bb004498 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 19 Sep 2023 13:44:17 -0700 Subject: [PATCH 18/18] clang-format fix --- examples/query/test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/query/test.cpp b/examples/query/test.cpp index c825fbfea6..b97a2d7714 100644 --- a/examples/query/test.cpp +++ b/examples/query/test.cpp @@ -22,7 +22,7 @@ void queryWithStreaming(adios2::IO &queryIO, std::string &dataFileName, std::str while (reader.BeginStep() == adios2::StepStatus::OK) { adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader); - w.GetResultCoverage(touched_blocks); + w.GetResultCoverage(touched_blocks); std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size() << std::endl;