diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 7e3dccf1acb4..400225d8443c 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -13,6 +13,10 @@ add_library( gdalalg_raster.cpp gdalalg_raster_info.cpp gdalalg_raster_convert.cpp + gdalalg_raster_pipeline.cpp + gdalalg_raster_read.cpp + gdalalg_raster_reproject.cpp + gdalalg_raster_write.cpp gdalalg_vector.cpp gdalalg_vector_info.cpp gdalalg_vector_convert.cpp diff --git a/apps/gdalalg_pipeline.cpp b/apps/gdalalg_pipeline.cpp index 528592d5ea3d..b060e854573a 100644 --- a/apps/gdalalg_pipeline.cpp +++ b/apps/gdalalg_pipeline.cpp @@ -12,67 +12,24 @@ #include "cpl_error.h" #include "gdalalgorithm.h" -//#include "gdalalg_raster_pipeline.h" +#include "gdalalg_raster_pipeline.h" #include "gdalalg_vector_pipeline.h" #include "gdalalg_dispatcher.h" #include "gdal_priv.h" -/************************************************************************/ -/* GDALDummyRasterPipelineAlgorithm */ -/************************************************************************/ - -class GDALDummyRasterPipelineAlgorithm final : public GDALAlgorithm -{ - public: - static constexpr const char *NAME = "pipeline"; - static constexpr const char *DESCRIPTION = "Dummy raster pipeline."; - static constexpr const char *HELP_URL = ""; - - static std::vector GetAliases() - { - return {}; - } - - explicit GDALDummyRasterPipelineAlgorithm(bool = false) - : GDALAlgorithm(NAME, DESCRIPTION, HELP_URL) - { - } - - bool ParseCommandLineArguments(const std::vector &) override - { - return false; - } - - /* cppcheck-suppress functionStatic */ - GDALDataset *GetDatasetRef() - { - return nullptr; - } - - /* cppcheck-suppress functionStatic */ - void SetDataset(GDALDataset *) - { - } - - private: - bool RunImpl(GDALProgressFunc, void *) override - { - return false; - } -}; - /************************************************************************/ /* GDALPipelineAlgorithm */ /************************************************************************/ class GDALPipelineAlgorithm final - : public GDALDispatcherAlgorithm { public: static constexpr const char *NAME = "pipeline"; static constexpr const char *DESCRIPTION = - "Execute a pipeline (shortcut for 'gdal vector pipeline')."; + "Execute a pipeline (shortcut for 'gdal raster pipeline' or 'gdal " + "vector pipeline')."; static constexpr const char *HELP_URL = ""; // TODO static std::vector GetAliases() @@ -94,7 +51,7 @@ class GDALPipelineAlgorithm final } private: - std::unique_ptr m_rasterPipeline{}; + std::unique_ptr m_rasterPipeline{}; std::unique_ptr m_vectorPipeline{}; std::string m_format{}; diff --git a/apps/gdalalg_raster.cpp b/apps/gdalalg_raster.cpp index 368748f42eb4..0523730aea1c 100644 --- a/apps/gdalalg_raster.cpp +++ b/apps/gdalalg_raster.cpp @@ -14,6 +14,8 @@ #include "gdalalg_raster_info.h" #include "gdalalg_raster_convert.h" +#include "gdalalg_raster_pipeline.h" +#include "gdalalg_raster_reproject.h" /************************************************************************/ /* GDALRasterAlgorithm */ @@ -35,6 +37,8 @@ class GDALRasterAlgorithm final : public GDALAlgorithm { RegisterSubAlgorithm(); RegisterSubAlgorithm(); + RegisterSubAlgorithm(); + RegisterSubAlgorithm(); } private: diff --git a/apps/gdalalg_raster_pipeline.cpp b/apps/gdalalg_raster_pipeline.cpp new file mode 100644 index 000000000000..4376a1333841 --- /dev/null +++ b/apps/gdalalg_raster_pipeline.cpp @@ -0,0 +1,553 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "raster pipeline" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "gdalalg_raster_pipeline.h" +#include "gdalalg_raster_read.h" +#include "gdalalg_raster_reproject.h" +#include "gdalalg_raster_write.h" + +#include "cpl_conv.h" +#include "cpl_json.h" +#include "cpl_string.h" + +#include + +//! @cond Doxygen_Suppress + +#ifndef _ +#define _(x) (x) +#endif + +/************************************************************************/ +/* GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm() */ +/************************************************************************/ + +GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm( + const std::string &name, const std::string &description, + const std::string &helpURL, bool standaloneStep) + : GDALAlgorithm(name, description, helpURL), + m_standaloneStep(standaloneStep) +{ + if (m_standaloneStep) + { + AddInputArgs(false, false); + AddProgressArg(); + AddOutputArgs(false); + } +} + +/************************************************************************/ +/* GDALRasterPipelineStepAlgorithm::AddInputArgs() */ +/************************************************************************/ + +void GDALRasterPipelineStepAlgorithm::AddInputArgs( + bool openForMixedRasterVector, bool hiddenForCLI) +{ + AddInputFormatsArg(&m_inputFormats) + .AddMetadataItem( + GAAMDI_REQUIRED_CAPABILITIES, + openForMixedRasterVector + ? std::vector{GDAL_DCAP_RASTER, GDAL_DCAP_VECTOR} + : std::vector{GDAL_DCAP_RASTER}) + .SetHiddenForCLI(hiddenForCLI); + AddOpenOptionsArg(&m_openOptions).SetHiddenForCLI(hiddenForCLI); + AddInputDatasetArg(&m_inputDataset, + openForMixedRasterVector + ? (GDAL_OF_RASTER | GDAL_OF_VECTOR) + : GDAL_OF_RASTER, + /* positionalAndRequired = */ !hiddenForCLI); +} + +/************************************************************************/ +/* GDALRasterPipelineStepAlgorithm::AddOutputArgs() */ +/************************************************************************/ + +void GDALRasterPipelineStepAlgorithm::AddOutputArgs(bool hiddenForCLI) +{ + AddOutputFormatArg(&m_format) + .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES, + {GDAL_DCAP_RASTER, GDAL_DCAP_CREATECOPY}) + .SetHiddenForCLI(hiddenForCLI); + AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER, + /* positionalAndRequired = */ !hiddenForCLI) + .SetHiddenForCLI(hiddenForCLI); + m_outputDataset.SetInputFlags(GADV_NAME | GADV_OBJECT); + AddCreationOptionsArg(&m_creationOptions).SetHiddenForCLI(hiddenForCLI); + AddOverwriteArg(&m_overwrite).SetHiddenForCLI(hiddenForCLI); +} + +/************************************************************************/ +/* GDALRasterPipelineStepAlgorithm::RunImpl() */ +/************************************************************************/ + +bool GDALRasterPipelineStepAlgorithm::RunImpl(GDALProgressFunc pfnProgress, + void *pProgressData) +{ + if (m_standaloneStep) + { + GDALRasterReadAlgorithm readAlg; + for (auto &arg : readAlg.GetArgs()) + { + auto stepArg = GetArg(arg->GetName()); + if (stepArg && stepArg->IsExplicitlySet()) + { + arg->SetSkipIfAlreadySet(true); + arg->SetFrom(*stepArg); + } + } + + GDALRasterWriteAlgorithm writeAlg; + for (auto &arg : writeAlg.GetArgs()) + { + auto stepArg = GetArg(arg->GetName()); + if (stepArg && stepArg->IsExplicitlySet()) + { + arg->SetSkipIfAlreadySet(true); + arg->SetFrom(*stepArg); + } + } + + bool ret = false; + if (readAlg.Run()) + { + m_inputDataset.Set(readAlg.m_outputDataset.GetDatasetRef()); + m_outputDataset.Set(nullptr); + if (RunStep(nullptr, nullptr)) + { + writeAlg.m_inputDataset.Set(m_outputDataset.GetDatasetRef()); + if (writeAlg.Run(pfnProgress, pProgressData)) + { + m_outputDataset.Set( + writeAlg.m_outputDataset.GetDatasetRef()); + ret = true; + } + } + } + + return ret; + } + else + { + return RunStep(pfnProgress, pProgressData); + } +} + +/************************************************************************/ +/* GDALRasterPipelineAlgorithm::GDALRasterPipelineAlgorithm() */ +/************************************************************************/ + +GDALRasterPipelineAlgorithm::GDALRasterPipelineAlgorithm( + bool openForMixedRasterVector) + : GDALRasterPipelineStepAlgorithm(NAME, DESCRIPTION, HELP_URL, + /*standaloneStep=*/false) +{ + AddInputArgs(openForMixedRasterVector, /* hiddenForCLI = */ true); + AddProgressArg(); + AddArg("pipeline", 0, _("Pipeline string"), &m_pipeline) + .SetHiddenForCLI() + .SetPositional(); + AddOutputArgs(/* hiddenForCLI = */ true); + + m_stepRegistry.Register(); + m_stepRegistry.Register(); + m_stepRegistry.Register(); +} + +/************************************************************************/ +/* GDALRasterPipelineAlgorithm::GetStepAlg() */ +/************************************************************************/ + +std::unique_ptr +GDALRasterPipelineAlgorithm::GetStepAlg(const std::string &name) const +{ + auto alg = m_stepRegistry.Instantiate(name); + return std::unique_ptr( + cpl::down_cast(alg.release())); +} + +/************************************************************************/ +/* GDALRasterPipelineAlgorithm::ParseCommandLineArguments() */ +/************************************************************************/ + +bool GDALRasterPipelineAlgorithm::ParseCommandLineArguments( + const std::vector &args) +{ + if (args.size() == 1 && (args[0] == "-h" || args[0] == "--help" || + args[0] == "help" || args[0] == "--json-usage")) + return GDALAlgorithm::ParseCommandLineArguments(args); + + for (const auto &arg : args) + { + if (arg.find("--pipeline") == 0) + return GDALAlgorithm::ParseCommandLineArguments(args); + + // gdal raster pipeline [--progress] "read in.tif ..." + if (arg.find("read ") == 0) + return GDALAlgorithm::ParseCommandLineArguments(args); + } + + if (!m_steps.empty()) + { + ReportError(CE_Failure, CPLE_AppDefined, + "ParseCommandLineArguments() can only be called once per " + "instance."); + return false; + } + + struct Step + { + std::unique_ptr alg{}; + std::vector args{}; + }; + + std::vector steps; + steps.resize(1); + + for (const auto &arg : args) + { + if (arg == "--progress") + { + m_progressBarRequested = true; + continue; + } + + auto &curStep = steps.back(); + + if (arg == "!" || arg == "|") + { + if (curStep.alg) + { + steps.resize(steps.size() + 1); + } + } +#ifdef GDAL_PIPELINE_PROJ_NOSTALGIA + else if (arg == "+step") + { + if (curStep.alg) + { + steps.resize(steps.size() + 1); + } + } + else if (arg.find("+gdal=") == 0) + { + const std::string stepName = arg.substr(strlen("+gdal=")); + curStep.alg = GetStepAlg(stepName); + if (!curStep.alg) + { + ReportError(CE_Failure, CPLE_AppDefined, + "unknown step name: %s", stepName.c_str()); + return false; + } + } +#endif + else if (!curStep.alg) + { + std::string algName = arg; +#ifdef GDAL_PIPELINE_PROJ_NOSTALGIA + if (!algName.empty() && algName[0] == '+') + algName = algName.substr(1); +#endif + curStep.alg = GetStepAlg(algName); + if (!curStep.alg) + { + ReportError(CE_Failure, CPLE_AppDefined, + "unknown step name: %s", algName.c_str()); + return false; + } + } + else + { +#ifdef GDAL_PIPELINE_PROJ_NOSTALGIA + if (!arg.empty() && arg[0] == '+') + { + curStep.args.push_back("--" + arg.substr(1)); + continue; + } +#endif + curStep.args.push_back(arg); + } + } + + // As we initially added a step without alg to bootstrap things, make + // sure to remove it if it hasn't been filled, or the user has terminated + // the pipeline with a '!' separator. + if (!steps.back().alg) + steps.pop_back(); + + if (steps.size() < 2) + { + ReportError(CE_Failure, CPLE_AppDefined, + "At least 2 steps must be provided"); + return false; + } + + if (steps.front().alg->GetName() != GDALRasterReadAlgorithm::NAME) + { + ReportError(CE_Failure, CPLE_AppDefined, "First step should be '%s'", + GDALRasterReadAlgorithm::NAME); + return false; + } + for (size_t i = 1; i < steps.size() - 1; ++i) + { + if (steps[i].alg->GetName() == GDALRasterReadAlgorithm::NAME) + { + ReportError(CE_Failure, CPLE_AppDefined, + "Only first step can be '%s'", + GDALRasterReadAlgorithm::NAME); + return false; + } + } + if (steps.back().alg->GetName() != GDALRasterWriteAlgorithm::NAME) + { + ReportError(CE_Failure, CPLE_AppDefined, "Last step should be '%s'", + GDALRasterWriteAlgorithm::NAME); + return false; + } + for (size_t i = 0; i < steps.size() - 1; ++i) + { + if (steps[i].alg->GetName() == GDALRasterWriteAlgorithm::NAME) + { + ReportError(CE_Failure, CPLE_AppDefined, + "Only last step can be '%s'", + GDALRasterWriteAlgorithm::NAME); + return false; + } + } + + if (!m_pipeline.empty()) + { + // Propagate input parameters set at the pipeline level to the + // "read" step + { + auto &step = steps.front(); + for (auto &arg : step.alg->GetArgs()) + { + auto pipelineArg = GetArg(arg->GetName()); + if (pipelineArg && pipelineArg->IsExplicitlySet()) + { + arg->SetSkipIfAlreadySet(true); + arg->SetFrom(*pipelineArg); + } + } + } + + // Same with "write" step + { + auto &step = steps.back(); + for (auto &arg : step.alg->GetArgs()) + { + auto pipelineArg = GetArg(arg->GetName()); + if (pipelineArg && pipelineArg->IsExplicitlySet()) + { + arg->SetSkipIfAlreadySet(true); + arg->SetFrom(*pipelineArg); + } + } + } + } + + // Parse each step, but without running the validation + for (const auto &step : steps) + { + step.alg->m_skipValidationInParseCommandLine = true; + if (!step.alg->ParseCommandLineArguments(step.args)) + return false; + } + + // Evaluate "input" argument of "read" step, together with the "output" + // argument of the "write" step, in case they point to the same dataset. + auto inputArg = steps.front().alg->GetArg(GDAL_ARG_NAME_INPUT); + if (inputArg && inputArg->IsExplicitlySet() && + inputArg->GetType() == GAAT_DATASET) + { + steps.front().alg->ProcessDatasetArg(inputArg, steps.back().alg.get()); + } + + for (const auto &step : steps) + { + if (!step.alg->ValidateArguments()) + return false; + } + + for (auto &step : steps) + m_steps.push_back(std::move(step.alg)); + + return true; +} + +/************************************************************************/ +/* GDALRasterPipelineAlgorithm::RunStep() */ +/************************************************************************/ + +bool GDALRasterPipelineAlgorithm::RunStep(GDALProgressFunc pfnProgress, + void *pProgressData) +{ + if (m_steps.empty()) + { + // If invoked programmatically, not from the command line. + + if (m_pipeline.empty()) + { + ReportError(CE_Failure, CPLE_AppDefined, + "'pipeline' argument not set"); + return false; + } + + const CPLStringList aosTokens(CSLTokenizeString(m_pipeline.c_str())); + if (!ParseCommandLineArguments(aosTokens)) + return false; + } + + GDALDataset *poCurDS = nullptr; + for (size_t i = 0; i < m_steps.size(); ++i) + { + auto &step = m_steps[i]; + if (i > 0) + { + if (step->m_inputDataset.GetDatasetRef()) + { + // Shouldn't happen + ReportError(CE_Failure, CPLE_AppDefined, + "Step nr %d (%s) has already an input dataset", + static_cast(i), step->GetName().c_str()); + return false; + } + step->m_inputDataset.Set(poCurDS); + } + if (i + 1 < m_steps.size() && step->m_outputDataset.GetDatasetRef()) + { + // Shouldn't happen + ReportError(CE_Failure, CPLE_AppDefined, + "Step nr %d (%s) has already an output dataset", + static_cast(i), step->GetName().c_str()); + return false; + } + if (!step->Run(i < m_steps.size() - 1 ? nullptr : pfnProgress, + i < m_steps.size() - 1 ? nullptr : pProgressData)) + { + return false; + } + poCurDS = step->m_outputDataset.GetDatasetRef(); + if (!poCurDS) + { + ReportError(CE_Failure, CPLE_AppDefined, + "Step nr %d (%s) failed to produce an output dataset", + static_cast(i), step->GetName().c_str()); + return false; + } + } + + if (!m_outputDataset.GetDatasetRef()) + { + m_outputDataset.Set(poCurDS); + } + + return true; +} + +/************************************************************************/ +/* GDALAlgorithm::Finalize() */ +/************************************************************************/ + +bool GDALRasterPipelineAlgorithm::Finalize() +{ + bool ret = GDALAlgorithm::Finalize(); + for (auto &step : m_steps) + { + ret = step->Finalize() && ret; + } + return ret; +} + +/************************************************************************/ +/* GDALRasterPipelineAlgorithm::GetUsageForCLI() */ +/************************************************************************/ + +std::string GDALRasterPipelineAlgorithm::GetUsageForCLI( + bool shortUsage, const UsageOptions &usageOptions) const +{ + std::string ret = GDALAlgorithm::GetUsageForCLI(shortUsage, usageOptions); + if (shortUsage) + return ret; + + ret += "\n is of the form: read [READ-OPTIONS] " + "( ! [STEP-OPTIONS] )* ! write [WRITE-OPTIONS]\n"; + ret += '\n'; + ret += "Example: 'gdal raster pipeline --progress ! read in.tif ! \\\n"; + ret += " reproject --dst-crs=EPSG:32632 ! "; + ret += "write out.tif --overwrite'\n"; + ret += '\n'; + ret += "Potential steps are:\n"; + + UsageOptions stepUsageOptions; + stepUsageOptions.isPipelineStep = true; + + for (const std::string &name : m_stepRegistry.GetNames()) + { + auto alg = GetStepAlg(name); + auto [options, maxOptLen] = alg->GetArgNamesForCLI(); + stepUsageOptions.maxOptLen = + std::max(stepUsageOptions.maxOptLen, maxOptLen); + } + + { + const auto name = GDALRasterReadAlgorithm::NAME; + ret += '\n'; + auto alg = GetStepAlg(name); + alg->SetCallPath({name}); + ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions); + } + for (const std::string &name : m_stepRegistry.GetNames()) + { + if (name != GDALRasterReadAlgorithm::NAME && + name != GDALRasterWriteAlgorithm::NAME) + { + ret += '\n'; + auto alg = GetStepAlg(name); + alg->SetCallPath({name}); + ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions); + } + } + { + const auto name = GDALRasterWriteAlgorithm::NAME; + ret += '\n'; + auto alg = GetStepAlg(name); + alg->SetCallPath({name}); + ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions); + } + + return ret; +} + +/************************************************************************/ +/* GDALRasterPipelineAlgorithm::GetUsageAsJSON() */ +/************************************************************************/ + +std::string GDALRasterPipelineAlgorithm::GetUsageAsJSON() const +{ + CPLJSONDocument oDoc; + oDoc.LoadMemory(GDALAlgorithm::GetUsageAsJSON()); + + CPLJSONArray jPipelineSteps; + for (const std::string &name : m_stepRegistry.GetNames()) + { + auto alg = GetStepAlg(name); + CPLJSONDocument oStepDoc; + oStepDoc.LoadMemory(alg->GetUsageAsJSON()); + jPipelineSteps.Add(oStepDoc.GetRoot()); + } + oDoc.GetRoot().Add("pipeline_algorithms", jPipelineSteps); + + return oDoc.SaveAsString(); +} + +//! @endcond diff --git a/apps/gdalalg_raster_pipeline.h b/apps/gdalalg_raster_pipeline.h new file mode 100644 index 000000000000..5004603eb217 --- /dev/null +++ b/apps/gdalalg_raster_pipeline.h @@ -0,0 +1,123 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "raster pipeline" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef GDALALG_RASTER_PIPELINE_INCLUDED +#define GDALALG_RASTER_PIPELINE_INCLUDED + +#include "gdalalgorithm.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALRasterPipelineStepAlgorithm */ +/************************************************************************/ + +class GDALRasterPipelineStepAlgorithm /* non final */ : public GDALAlgorithm +{ + protected: + GDALRasterPipelineStepAlgorithm(const std::string &name, + const std::string &description, + const std::string &helpURL, + bool standaloneStep); + + friend class GDALRasterPipelineAlgorithm; + + virtual bool RunStep(GDALProgressFunc pfnProgress, void *pProgressData) = 0; + + void AddInputArgs(bool openForMixedRasterVector, bool hiddenForCLI); + void AddOutputArgs(bool hiddenForCLI); + + bool m_standaloneStep = false; + + // Input arguments + GDALArgDatasetValue m_inputDataset{}; + std::vector m_openOptions{}; + std::vector m_inputFormats{}; + std::vector m_inputLayerNames{}; + + // Output arguments + GDALArgDatasetValue m_outputDataset{}; + std::string m_format{}; + std::vector m_creationOptions{}; + bool m_overwrite = false; + std::string m_outputLayerName{}; + + private: + bool RunImpl(GDALProgressFunc pfnProgress, void *pProgressData) override; +}; + +/************************************************************************/ +/* GDALRasterPipelineAlgorithm */ +/************************************************************************/ + +// This is an easter egg to pay tribute to PROJ pipeline syntax +// We accept "gdal vector +gdal=pipeline +step +gdal=read +input=in.tif +step +gdal=reproject +dst-crs=EPSG:32632 +step +gdal=write +output=out.tif +overwrite" +// as an alternative to (recommended): +// "gdal vector pipeline ! read in.tif ! reproject--dst-crs=EPSG:32632 ! write out.tif --overwrite" +#define GDAL_PIPELINE_PROJ_NOSTALGIA + +class GDALRasterPipelineAlgorithm final : public GDALRasterPipelineStepAlgorithm +{ + public: + static constexpr const char *NAME = "pipeline"; + static constexpr const char *DESCRIPTION = "Process a raster dataset."; + static constexpr const char *HELP_URL = + "https://gdal.org/en/stable/programs/gdal_raster_pipeline.html"; + + static std::vector GetAliases() + { + return { +#ifdef GDAL_PIPELINE_PROJ_NOSTALGIA + GDALAlgorithmRegistry::HIDDEN_ALIAS_SEPARATOR, + "+pipeline", + "+gdal=pipeline", +#endif + }; + } + + GDALRasterPipelineAlgorithm(bool openForMixedRasterVector = false); + + bool + ParseCommandLineArguments(const std::vector &args) override; + + bool Finalize() override; + + std::string GetUsageForCLI(bool shortUsage, + const UsageOptions &usageOptions) const override; + + std::string GetUsageAsJSON() const override; + + GDALDataset *GetDatasetRef() + { + return m_inputDataset.GetDatasetRef(); + } + + /* cppcheck-suppress functionStatic */ + void SetDataset(GDALDataset *) + { + } + + private: + std::string m_pipeline{}; + + bool RunStep(GDALProgressFunc pfnProgress, void *pProgressData) override; + + std::unique_ptr + GetStepAlg(const std::string &name) const; + + GDALAlgorithmRegistry m_stepRegistry{}; + std::vector> m_steps{}; +}; + +//! @endcond + +#endif diff --git a/apps/gdalalg_raster_read.cpp b/apps/gdalalg_raster_read.cpp new file mode 100644 index 000000000000..f727a519c665 --- /dev/null +++ b/apps/gdalalg_raster_read.cpp @@ -0,0 +1,47 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: "read" step of "raster pipeline" + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "gdalalg_raster_read.h" + +#include "gdal_priv.h" +#include "ogrsf_frmts.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALRasterReadAlgorithm::GDALRasterReadAlgorithm() */ +/************************************************************************/ + +GDALRasterReadAlgorithm::GDALRasterReadAlgorithm() + : GDALRasterPipelineStepAlgorithm(NAME, DESCRIPTION, HELP_URL, + /* standaloneStep =*/false) +{ + AddInputArgs(/* openForMixedRasterVector = */ false, + /* hiddenForCLI = */ false); +} + +/************************************************************************/ +/* GDALRasterReadAlgorithm::RunStep() */ +/************************************************************************/ + +bool GDALRasterReadAlgorithm::RunStep(GDALProgressFunc, void *) +{ + CPLAssert(m_inputDataset.GetDatasetRef()); + CPLAssert(m_outputDataset.GetName().empty()); + CPLAssert(!m_outputDataset.GetDatasetRef()); + + m_outputDataset.Set(m_inputDataset.GetDatasetRef()); + + return true; +} + +//! @endcond diff --git a/apps/gdalalg_raster_read.h b/apps/gdalalg_raster_read.h new file mode 100644 index 000000000000..39b1e55f436a --- /dev/null +++ b/apps/gdalalg_raster_read.h @@ -0,0 +1,45 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: "read" step of "raster pipeline" + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef GDALALG_RASTER_READ_INCLUDED +#define GDALALG_RASTER_READ_INCLUDED + +#include "gdalalg_raster_pipeline.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALRasterReadAlgorithm */ +/************************************************************************/ + +class GDALRasterReadAlgorithm final : public GDALRasterPipelineStepAlgorithm +{ + public: + static constexpr const char *NAME = "read"; + static constexpr const char *DESCRIPTION = "Read a raster dataset."; + static constexpr const char *HELP_URL = + "https://gdal.org/en/stable/programs/gdal_raster_pipeline.html"; + + static std::vector GetAliases() + { + return {}; + } + + GDALRasterReadAlgorithm(); + + private: + bool RunStep(GDALProgressFunc, void *) override; +}; + +//! @endcond + +#endif diff --git a/apps/gdalalg_raster_reproject.cpp b/apps/gdalalg_raster_reproject.cpp new file mode 100644 index 000000000000..c251f25bbcd3 --- /dev/null +++ b/apps/gdalalg_raster_reproject.cpp @@ -0,0 +1,174 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: "reproject" step of "raster pipeline" + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "gdalalg_raster_reproject.h" + +#include "gdal_priv.h" +#include "gdal_utils.h" + +//! @cond Doxygen_Suppress + +#ifndef _ +#define _(x) (x) +#endif + +/************************************************************************/ +/* GDALRasterReprojectAlgorithm::GDALRasterReprojectAlgorithm() */ +/************************************************************************/ + +GDALRasterReprojectAlgorithm::GDALRasterReprojectAlgorithm(bool standaloneStep) + : GDALRasterPipelineStepAlgorithm(NAME, DESCRIPTION, HELP_URL, + standaloneStep) +{ + AddArg("src-crs", 's', _("Source CRS"), &m_srsCrs).AddHiddenAlias("s_srs"); + AddArg("dst-crs", 'd', _("Destination CRS"), &m_dstCrs) + .AddHiddenAlias("t_srs"); + AddArg("resampling", 'r', _("Resampling method"), &m_resampling) + .SetChoices("near", "bilinear", "cubic", "cubicspline", "lanczos", + "average", "rms", "mode", "min", "max", "med", "q1", "q3", + "sum"); + + auto &resArg = + AddArg("resolution", 0, + _("Target resolution (in destination CRS units)"), &m_resolution) + .SetMinCount(2) + .SetMaxCount(2) + .SetRepeatedArgAllowed(false) + .SetDisplayHintAboutRepetition(false) + .SetMetaVar(","); + resArg.AddValidationAction( + [&resArg]() + { + const auto &val = resArg.Get>(); + CPLAssert(val.size() == 2); + if (!(val[0] >= 0 && val[1] >= 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Target resolution should be strictly positive."); + return false; + } + return true; + }); + + auto &extentArg = + AddArg("extent", 0, _("Target extent (in destination CRS units)"), + &m_extent) + .SetMinCount(4) + .SetMaxCount(4) + .SetRepeatedArgAllowed(false) + .SetDisplayHintAboutRepetition(false) + .SetMetaVar(",,,"); + extentArg.AddValidationAction( + [&extentArg]() + { + const auto &val = extentArg.Get>(); + CPLAssert(val.size() == 4); + if (!(val[0] <= val[2]) || !(val[1] <= val[3])) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Value of 'extent' should be xmin,ymin,xmax,ymax with " + "xmin <= xmax and ymin <= ymax"); + return false; + } + return true; + }); + + AddArg("target-aligned-pixels", 0, + _("Round target extent to target resolution"), + &m_targetAlignedPixels) + .AddHiddenAlias("tap"); +} + +/************************************************************************/ +/* GDALRasterReprojectAlgorithm::RunStep() */ +/************************************************************************/ + +bool GDALRasterReprojectAlgorithm::RunStep(GDALProgressFunc, void *) +{ + CPLAssert(m_inputDataset.GetDatasetRef()); + CPLAssert(m_outputDataset.GetName().empty()); + CPLAssert(!m_outputDataset.GetDatasetRef()); + + if (!m_srsCrs.empty()) + { + OGRSpatialReference oSRS; + if (oSRS.SetFromUserInput(m_srsCrs.c_str()) != OGRERR_NONE) + { + ReportError(CE_Failure, CPLE_AppDefined, + "Invalid value for '--src-crs'"); + return false; + } + } + + if (!m_dstCrs.empty()) + { + OGRSpatialReference oSRS; + if (oSRS.SetFromUserInput(m_dstCrs.c_str()) != OGRERR_NONE) + { + ReportError(CE_Failure, CPLE_AppDefined, + "Invalid value for '--dst-crs'"); + return false; + } + } + + CPLStringList aosOptions; + aosOptions.AddString("-of"); + aosOptions.AddString("VRT"); + if (!m_srsCrs.empty()) + { + aosOptions.AddString("-s_srs"); + aosOptions.AddString(m_srsCrs.c_str()); + } + if (!m_dstCrs.empty()) + { + aosOptions.AddString("-t_srs"); + aosOptions.AddString(m_dstCrs.c_str()); + } + if (!m_resampling.empty()) + { + aosOptions.AddString("-r"); + aosOptions.AddString(m_resampling.c_str()); + } + if (!m_resolution.empty()) + { + aosOptions.AddString("-tr"); + aosOptions.AddString(CPLSPrintf("%.17g", m_resolution[0])); + aosOptions.AddString(CPLSPrintf("%.17g", m_resolution[1])); + } + if (!m_extent.empty()) + { + aosOptions.AddString("-te"); + aosOptions.AddString(CPLSPrintf("%.17g", m_extent[0])); + aosOptions.AddString(CPLSPrintf("%.17g", m_extent[1])); + aosOptions.AddString(CPLSPrintf("%.17g", m_extent[2])); + aosOptions.AddString(CPLSPrintf("%.17g", m_extent[3])); + } + if (m_targetAlignedPixels) + { + aosOptions.AddString("-tap"); + } + GDALWarpAppOptions *psOptions = + GDALWarpAppOptionsNew(aosOptions.List(), nullptr); + + GDALDatasetH hSrcDS = GDALDataset::ToHandle(m_inputDataset.GetDatasetRef()); + auto poRetDS = GDALDataset::FromHandle( + GDALWarp("", nullptr, 1, &hSrcDS, psOptions, nullptr)); + GDALWarpAppOptionsFree(psOptions); + if (!poRetDS) + return false; + + m_outputDataset.Set(std::unique_ptr(poRetDS)); + + return true; +} + +//! @endcond diff --git a/apps/gdalalg_raster_reproject.h b/apps/gdalalg_raster_reproject.h new file mode 100644 index 000000000000..7bc1d7a91a87 --- /dev/null +++ b/apps/gdalalg_raster_reproject.h @@ -0,0 +1,67 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: "reproject" step of "raster pipeline" + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef GDALALG_RASTER_REPROJECT_INCLUDED +#define GDALALG_RASTER_REPROJECT_INCLUDED + +#include "gdalalg_raster_pipeline.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALRasterReprojectAlgorithm */ +/************************************************************************/ + +class GDALRasterReprojectAlgorithm /* non final */ + : public GDALRasterPipelineStepAlgorithm +{ + public: + static constexpr const char *NAME = "reproject"; + static constexpr const char *DESCRIPTION = "Reproject a raster dataset."; + static constexpr const char *HELP_URL = + "https://gdal.org/en/stable/programs/gdal_raster_reproject.html"; + + static std::vector GetAliases() + { + return {}; + } + + explicit GDALRasterReprojectAlgorithm(bool standaloneStep = false); + + private: + bool RunStep(GDALProgressFunc pfnProgress, void *pProgressData) override; + + std::string m_srsCrs{}; + std::string m_dstCrs{}; + std::string m_resampling{}; + std::vector m_resolution{}; + std::vector m_extent{}; + bool m_targetAlignedPixels = false; +}; + +/************************************************************************/ +/* GDALRasterReprojectAlgorithmStandalone */ +/************************************************************************/ + +class GDALRasterReprojectAlgorithmStandalone final + : public GDALRasterReprojectAlgorithm +{ + public: + GDALRasterReprojectAlgorithmStandalone() + : GDALRasterReprojectAlgorithm(/* standaloneStep = */ true) + { + } +}; + +//! @endcond + +#endif /* GDALALG_RASTER_REPROJECT_INCLUDED */ diff --git a/apps/gdalalg_raster_write.cpp b/apps/gdalalg_raster_write.cpp new file mode 100644 index 000000000000..79dd634e095c --- /dev/null +++ b/apps/gdalalg_raster_write.cpp @@ -0,0 +1,74 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: "write" step of "raster pipeline" + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "gdalalg_raster_write.h" + +#include "cpl_string.h" +#include "gdal_utils.h" +#include "gdal_priv.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALRasterWriteAlgorithm::GDALRasterWriteAlgorithm() */ +/************************************************************************/ + +GDALRasterWriteAlgorithm::GDALRasterWriteAlgorithm() + : GDALRasterPipelineStepAlgorithm(NAME, DESCRIPTION, HELP_URL, + /* standaloneStep =*/false) +{ + AddOutputArgs(/* hiddenForCLI = */ false); +} + +/************************************************************************/ +/* GDALRasterWriteAlgorithm::RunStep() */ +/************************************************************************/ + +bool GDALRasterWriteAlgorithm::RunStep(GDALProgressFunc pfnProgress, + void *pProgressData) +{ + CPLAssert(m_inputDataset.GetDatasetRef()); + CPLAssert(!m_outputDataset.GetDatasetRef()); + + CPLStringList aosOptions; + if (!m_overwrite) + { + aosOptions.AddString("--no-overwrite"); + } + if (!m_format.empty()) + { + aosOptions.AddString("-of"); + aosOptions.AddString(m_format.c_str()); + } + for (const auto &co : m_creationOptions) + { + aosOptions.AddString("-co"); + aosOptions.AddString(co.c_str()); + } + + GDALTranslateOptions *psOptions = + GDALTranslateOptionsNew(aosOptions.List(), nullptr); + GDALTranslateOptionsSetProgress(psOptions, pfnProgress, pProgressData); + + GDALDatasetH hSrcDS = GDALDataset::ToHandle(m_inputDataset.GetDatasetRef()); + auto poRetDS = GDALDataset::FromHandle(GDALTranslate( + m_outputDataset.GetName().c_str(), hSrcDS, psOptions, nullptr)); + GDALTranslateOptionsFree(psOptions); + if (!poRetDS) + return false; + + m_outputDataset.Set(std::unique_ptr(poRetDS)); + + return true; +} + +//! @endcond diff --git a/apps/gdalalg_raster_write.h b/apps/gdalalg_raster_write.h new file mode 100644 index 000000000000..88197ae9ed00 --- /dev/null +++ b/apps/gdalalg_raster_write.h @@ -0,0 +1,45 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: "write" step of "raster pipeline" + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef GDALALG_RASTER_WRITE_INCLUDED +#define GDALALG_RASTER_WRITE_INCLUDED + +#include "gdalalg_raster_pipeline.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALRasterWriteAlgorithm */ +/************************************************************************/ + +class GDALRasterWriteAlgorithm final : public GDALRasterPipelineStepAlgorithm +{ + public: + static constexpr const char *NAME = "write"; + static constexpr const char *DESCRIPTION = "Write a raster dataset."; + static constexpr const char *HELP_URL = + "https://gdal.org/en/stable/programs/gdal_raster_pipeline.html"; + + static std::vector GetAliases() + { + return {}; + } + + GDALRasterWriteAlgorithm(); + + private: + bool RunStep(GDALProgressFunc pfnProgress, void *pProgressData) override; +}; + +//! @endcond + +#endif /* GDALALG_RASTER_WRITE_INCLUDED */ diff --git a/autotest/utilities/test_gdalalg_raster_pipeline.py b/autotest/utilities/test_gdalalg_raster_pipeline.py new file mode 100755 index 000000000000..5af4be70fb2a --- /dev/null +++ b/autotest/utilities/test_gdalalg_raster_pipeline.py @@ -0,0 +1,541 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: 'gdal raster pipeline' testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +import json + +import pytest + +from osgeo import gdal + + +def get_pipeline_alg(): + reg = gdal.GetGlobalAlgorithmRegistry() + raster = reg.InstantiateAlg("raster") + return raster.InstantiateSubAlgorithm("pipeline") + + +def test_gdalalg_raster_pipeline_read_and_write(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + last_pct = [0] + + def my_progress(pct, msg, user_data): + last_pct[0] = pct + return True + + pipeline = get_pipeline_alg() + assert pipeline.ParseRunAndFinalize( + ["read", "../gcore/data/byte.tif", "!", "write", out_filename], my_progress + ) + assert last_pct[0] == 1.0 + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + with pytest.raises(Exception, match="can only be called once per instance"): + pipeline.ParseRunAndFinalize( + ["read", "../gcore/data/byte.tif", "!", "write", out_filename], my_progress + ) + + +def test_gdalalg_raster_pipeline_pipeline_arg(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + # Also test extra pipes / exclamation mark + assert pipeline.ParseRunAndFinalize( + ["--pipeline", f"! read ../gcore/data/byte.tif | | write {out_filename} !"] + ) + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +def test_gdalalg_raster_pipeline_as_api(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + pipeline.GetArg("pipeline").Set( + f"read ../gcore/data/byte.tif ! write {out_filename}" + ) + assert pipeline.Run() + ds = pipeline.GetArg("output").Get().GetDataset() + assert ds.GetRasterBand(1).Checksum() == 4672 + assert pipeline.Finalize() + ds = None + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +def test_gdalalg_raster_pipeline_input_through_api(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + pipeline.GetArg("input").Get().SetDataset(gdal.OpenEx("../gcore/data/byte.tif")) + pipeline.GetArg("pipeline").Set(f"read ! write {out_filename}") + assert pipeline.Run() + assert pipeline.Finalize() + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +def test_gdalalg_raster_pipeline_input_through_api_run_twice(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + pipeline.GetArg("input").Get().SetDataset(gdal.OpenEx("../gcore/data/byte.tif")) + pipeline.GetArg("pipeline").Set(f"read ! write {out_filename}") + assert pipeline.Run() + with pytest.raises( + Exception, match=r"pipeline: Step nr 0 \(read\) has already an output dataset" + ): + pipeline.Run() + + +def test_gdalalg_raster_pipeline_output_through_api(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + pipeline.GetArg("output").Get().SetName(out_filename) + pipeline.GetArg("pipeline").Set("read ../gcore/data/byte.tif ! write") + assert pipeline.Run() + assert pipeline.Finalize() + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +def test_gdalalg_raster_pipeline_as_api_error(): + + pipeline = get_pipeline_alg() + pipeline.GetArg("pipeline").Set("read") + with pytest.raises(Exception, match="pipeline: At least 2 steps must be provided"): + pipeline.Run() + + +def test_gdalalg_raster_pipeline_usage_as_json(): + + pipeline = get_pipeline_alg() + j = json.loads(pipeline.GetUsageAsJSON()) + assert "pipeline_algorithms" in j + + +def test_gdalalg_raster_pipeline_quoted(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + assert pipeline.ParseRunAndFinalize( + [f"read ../gcore/data/byte.tif ! write {out_filename}"] + ) + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +def test_gdalalg_raster_pipeline_progress(tmp_path): + + out_filename = str(tmp_path / "out.tif") + import gdaltest + import test_cli_utilities + + gdal_path = test_cli_utilities.get_gdal_path() + if gdal_path is None: + pytest.skip("gdal binary missing") + out, err = gdaltest.runexternal_out_and_err( + f"{gdal_path} raster pipeline --progress read ../gcore/data/byte.tif ! write {out_filename}" + ) + assert out.startswith( + "0...10...20...30...40...50...60...70...80...90...100 - done." + ) + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +def test_gdalalg_raster_easter_egg(tmp_path): + + out_filename = str(tmp_path / "out.tif") + import gdaltest + import test_cli_utilities + + gdal_path = test_cli_utilities.get_gdal_path() + if gdal_path is None: + pytest.skip("gdal binary missing") + gdaltest.runexternal( + f"{gdal_path} raster +gdal=pipeline +step +gdal=read +input=../gcore/data/byte.tif +step +write +output={out_filename}" + ) + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +def test_gdalalg_raster_pipeline_usage_as_json_bis(): + import gdaltest + import test_cli_utilities + + gdal_path = test_cli_utilities.get_gdal_path() + if gdal_path is None: + pytest.skip("gdal binary missing") + out, err = gdaltest.runexternal_out_and_err( + f"{gdal_path} raster pipeline --json-usage" + ) + j = json.loads(out) + assert "pipeline_algorithms" in j + + +def test_gdalalg_raster_pipeline_missing_at_run(): + + pipeline = get_pipeline_alg() + with pytest.raises(Exception, match="pipeline: 'pipeline' argument not set"): + pipeline.Run() + + +def test_gdalalg_raster_pipeline_empty_args(): + + pipeline = get_pipeline_alg() + with pytest.raises(Exception, match="pipeline: At least 2 steps must be provided"): + pipeline.ParseRunAndFinalize([]) + + +def test_gdalalg_raster_pipeline_unknow_step(): + + pipeline = get_pipeline_alg() + with pytest.raises(Exception, match="pipeline: unknown step name: unknown_step"): + pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "unknown_step", + "!", + "write", + "/vsimem/foo.tif", + ] + ) + + +def test_gdalalg_raster_pipeline_read_read(): + + pipeline = get_pipeline_alg() + with pytest.raises(Exception, match="pipeline: Last step should be 'write'"): + pipeline.ParseRunAndFinalize( + ["read", "../gcore/data/byte.tif", "!", "read", "../gcore/data/byte.tif"] + ) + + +def test_gdalalg_raster_pipeline_write_write(): + + pipeline = get_pipeline_alg() + with pytest.raises(Exception, match="pipeline: First step should be 'read'"): + pipeline.ParseRunAndFinalize( + ["write", "/vsimem/out.tif", "!", "write", "/vsimem/out.tif"] + ) + + +def test_gdalalg_raster_pipeline_read_write_write(): + + pipeline = get_pipeline_alg() + with pytest.raises(Exception, match="pipeline: Only last step can be 'write'"): + pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "write", + "/vsimem/out.tif", + "!", + "write", + "/vsimem/out.tif", + ] + ) + + +def test_gdalalg_raster_pipeline_read_read_write(): + + pipeline = get_pipeline_alg() + with pytest.raises(Exception, match="pipeline: Only first step can be 'read'"): + pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "read", + "../gcore/data/byte.tif", + "!", + "write", + "/vsimem/out.tif", + ] + ) + + +def test_gdalalg_raster_pipeline_invalid_step_during_parsing(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + with pytest.raises( + Exception, match="write: Long name option '--invalid' is unknown" + ): + pipeline.ParseRunAndFinalize( + ["read", "../gcore/data/byte.tif", "!", "write", "--invalid", out_filename] + ) + + +def test_gdalalg_raster_pipeline_invalid_step_during_validation(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + with pytest.raises( + Exception, + match="read: Positional arguments starting at 'INPUT' have not been specified", + ): + pipeline.ParseRunAndFinalize(["read", "!", "write", "--invalid", out_filename]) + + +@pytest.mark.require_driver("GPKG") +def test_gdalalg_raster_pipeline_write_options(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.gpkg") + + pipeline = get_pipeline_alg() + assert pipeline.ParseRunAndFinalize( + ["read", "../gcore/data/byte.tif", "!", "write", "--of=GPKG", out_filename] + ) + + pipeline = get_pipeline_alg() + with pytest.raises( + Exception, + match="already exists. Specify the --overwrite option to overwrite it", + ): + assert pipeline.ParseRunAndFinalize( + ["read", "../gcore/data/byte.tif", "!", "write", out_filename] + ) + + pipeline = get_pipeline_alg() + assert pipeline.ParseRunAndFinalize( + ["read", "../gcore/data/byte.tif", "!", "write", "--overwrite", out_filename] + ) + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +@pytest.mark.require_driver("GPKG") +def test_gdalalg_raster_pipeline_write_co(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.gpkg") + + pipeline = get_pipeline_alg() + assert pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "write", + out_filename, + "--co", + "ADD_GPKG_OGR_CONTENTS=NO", + ] + ) + + with gdal.OpenEx(out_filename) as ds: + with ds.ExecuteSQL( + "SELECT * FROM sqlite_master WHERE name = 'gpkg_ogr_contents'" + ) as lyr: + assert lyr.GetFeatureCount() == 0 + + +def test_gdalalg_raster_pipeline_reproject_invalid_src_crs(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + with pytest.raises( + Exception, + match="reproject: Invalid value for '--src-crs'", + ): + pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "reproject", + "--src-crs=invalid", + "--dst-crs=EPSG:4326", + "!", + "write", + out_filename, + ] + ) + + +def test_gdalalg_raster_pipeline_reproject_invalid_dst_crs(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + with pytest.raises( + Exception, + match="reproject: Invalid value for '--dst-crs'", + ): + pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "reproject", + "--dst-crs=invalid", + "!", + "write", + out_filename, + ] + ) + + +def test_gdalalg_raster_pipeline_reproject_invalid_resolution(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + with pytest.raises( + Exception, + match="Target resolution should be strictly positive", + ): + pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "reproject", + "--resolution=1,-1", + "!", + "write", + out_filename, + ] + ) + + +def test_gdalalg_raster_pipeline_reproject_no_args(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + assert pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "reproject", + "!", + "write", + "--overwrite", + out_filename, + ] + ) + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetSpatialRef().GetAuthorityCode(None) == "26711" + assert ds.GetRasterBand(1).Checksum() == 4672 + + +def test_gdalalg_raster_pipeline_reproject_invalid_extent(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + with pytest.raises( + Exception, + match="Value of 'extent' should be xmin,ymin,xmax,ymax with xmin <= xmax and ymin <= ymax", + ): + pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "reproject", + "--extent=3,4,2,1", + "!", + "write", + out_filename, + ] + ) + + +def test_gdalalg_raster_pipeline_reproject_extent_arg(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + assert pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "reproject", + "--src-crs=EPSG:32611", + "--dst-crs=EPSG:4326", + "--extent=-117.641,33.89,-117.628,33.9005", + "!", + "write", + "--overwrite", + out_filename, + ] + ) + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetSpatialRef().GetAuthorityCode(None) == "4326" + assert ds.GetGeoTransform() == pytest.approx( + (-117.641, 0.0005909090909093286, 0.0, 33.9005, 0.0, -0.0005833333333333554) + ) + assert ds.GetRasterBand(1).Checksum() == 4585 + + +def test_gdalalg_raster_pipeline_reproject_almost_all_args(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + pipeline = get_pipeline_alg() + assert pipeline.ParseRunAndFinalize( + [ + "read", + "../gcore/data/byte.tif", + "!", + "reproject", + "--src-crs=EPSG:32611", + "--dst-crs=EPSG:4326", + "--resampling=bilinear", + "--resolution=0.0005,0.0004", + "--target-aligned-pixels", + "!", + "write", + "--overwrite", + out_filename, + ] + ) + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetSpatialRef().GetAuthorityCode(None) == "4326" + assert ds.GetGeoTransform() == pytest.approx( + (-117.641, 0.0005, 0.0, 33.9008, 0.0, -0.0004), rel=1e-8 + ) + assert ds.GetRasterBand(1).Checksum() == 8515 diff --git a/autotest/utilities/test_gdalalg_raster_reproject.py b/autotest/utilities/test_gdalalg_raster_reproject.py new file mode 100755 index 000000000000..dfdcdc0e7364 --- /dev/null +++ b/autotest/utilities/test_gdalalg_raster_reproject.py @@ -0,0 +1,46 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: 'gdal raster reproject' testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +from osgeo import gdal + + +def get_reproject_alg(): + reg = gdal.GetGlobalAlgorithmRegistry() + raster = reg.InstantiateAlg("raster") + return raster.InstantiateSubAlgorithm("reproject") + + +def test_gdalalg_raster_reproject(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.tif") + + last_pct = [0] + + def my_progress(pct, msg, user_data): + last_pct[0] = pct + return True + + pipeline = get_reproject_alg() + assert pipeline.ParseRunAndFinalize( + [ + "--src-crs=EPSG:32611", + "--dst-crs=EPSG:4326", + "../gcore/data/byte.tif", + out_filename, + ], + my_progress, + ) + assert last_pct[0] == 1.0 + + with gdal.OpenEx(out_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4727 diff --git a/doc/source/conf.py b/doc/source/conf.py index b7e00fe6df67..437d700e546a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -229,6 +229,20 @@ [author_evenr], 1, ), + ( + "programs/gdal_raster_pipeline", + "gdal-raster-pipeline", + "Process a raster dataset", + [author_evenr], + 1, + ), + ( + "programs/gdal_raster_reproject", + "gdal-raster-reproject", + "Reproect a raster dataset", + [author_evenr], + 1, + ), ( "programs/gdal_vector", "gdal-vector", diff --git a/doc/source/programs/gdal_raster.rst b/doc/source/programs/gdal_raster.rst index 5b7eea40e074..118bf2074caf 100644 --- a/doc/source/programs/gdal_raster.rst +++ b/doc/source/programs/gdal_raster.rst @@ -21,12 +21,16 @@ Synopsis where is one of: - convert: Convert a raster dataset. - info: Return information on a raster dataset. + - pipeline: Process a raster dataset. + - reproject: Reproject a raster dataset. Available sub-commands ---------------------- - :ref:`gdal_raster_info_subcommand` - :ref:`gdal_raster_convert_subcommand` +- :ref:`gdal_raster_pipeline_subcommand` +- :ref:`gdal_raster_reproject_subcommand` Examples -------- diff --git a/doc/source/programs/gdal_raster_pipeline.rst b/doc/source/programs/gdal_raster_pipeline.rst new file mode 100644 index 000000000000..6786d7699255 --- /dev/null +++ b/doc/source/programs/gdal_raster_pipeline.rst @@ -0,0 +1,98 @@ +.. _gdal_raster_pipeline_subcommand: + +================================================================================ +"gdal raster pipeline" sub-command +================================================================================ + +.. versionadded:: 3.11 + +.. only:: html + + Process a raster dataset. + +.. Index:: gdal raster pipeline + +Synopsis +-------- + +.. code-block:: + + Usage: gdal raster pipeline [OPTIONS] + + Process a raster dataset. + + Positional arguments: + + Common Options: + -h, --help Display help message and exit + --json-usage Display usage as JSON document and exit + --progress Display progress bar + + is of the form: read [READ-OPTIONS] ( ! [STEP-OPTIONS] )* ! write [WRITE-OPTIONS] + + +A pipeline chains several steps, separated with the `!` (quotation mark) character. +The first step must be ``read``, and the last one ``write``. + +Potential steps are: + +* read [OPTIONS] + +.. code-block:: + + Read a raster dataset. + + Positional arguments: + -i, --input Input raster dataset [required] + + Advanced Options: + --if, --input-format Input formats [may be repeated] + --oo, --open-option Open options [may be repeated] + + +* reproject [OPTIONS] + +.. code-block:: + + Reproject a raster dataset. + + Options: + -s, --src-crs Source CRS + -d, --dst-crs Destination CRS + -r, --resampling Resampling method. RESAMPLING=near|bilinear|cubic|cubicspline|lanczos|average|rms|mode|min|max|med|q1|q3|sum + --resolution , Target resolution (in destination CRS units) + --extent ,,, Target extent (in destination CRS units) + --target-aligned-pixels Round target extent to target resolution + + +* write [OPTIONS] + +.. code-block:: + + Write a raster dataset. + + Positional arguments: + -o, --output Output raster dataset [required] + + Options: + -f, --of, --format, --output-format Output format + --co, --creation-option = Creation option [may be repeated] + --overwrite Whether overwriting existing output is allowed + + + +Description +----------- + +:program:`gdal raster pipeline` can be used to process a raster dataset and +perform various on-the-fly processing steps. + +Examples +-------- + +.. example:: + :title: Reproject a GeoTIFF file to CRS EPSG:32632 ("WGS 84 / UTM zone 32N") + + .. code-block:: bash + + $ gdal raster pipeline --progress ! read in.tif ! reproject --dst-crs=EPSG:32632 ! write out.tif --overwrite diff --git a/doc/source/programs/gdal_raster_reproject.rst b/doc/source/programs/gdal_raster_reproject.rst new file mode 100644 index 000000000000..5ac8f506b0d1 --- /dev/null +++ b/doc/source/programs/gdal_raster_reproject.rst @@ -0,0 +1,64 @@ +.. _gdal_raster_reproject_subcommand: + +================================================================================ +"gdal raster reproject" sub-command +================================================================================ + +.. versionadded:: 3.11 + +.. only:: html + + Reproject a raster dataset. + +.. Index:: gdal raster reproject + +Synopsis +-------- + +.. code-block:: + + Usage: gdal raster reproject [OPTIONS] + + Reproject a raster dataset. + + Positional arguments: + -i, --input Input raster dataset [required] + -o, --output Output raster dataset [required] + + Common Options: + -h, --help Display help message and exit + --version Display GDAL version and exit + --json-usage Display usage as JSON document and exit + --drivers Display driver list as JSON document and exit + --progress Display progress bar + + Options: + -f, --of, --format, --output-format Output format + --co, --creation-option = Creation option [may be repeated] + --overwrite Whether overwriting existing output is allowed + -s, --src-crs Source CRS + -d, --dst-crs Destination CRS + -r, --resampling Resampling method. RESAMPLING=near|bilinear|cubic|cubicspline|lanczos|average|rms|mode|min|max|med|q1|q3|sum + --resolution , Target resolution (in destination CRS units) + --extent ,,, Target extent (in destination CRS units) + --target-aligned-pixels Round target extent to target resolution + + Advanced Options: + --if, --input-format Input formats [may be repeated] + --oo, --open-option Open options [may be repeated] + + +Description +----------- + +:program:`gdal raster reproject` can be used to reproject a raster dataset. + +Examples +-------- + +.. example:: + :title: Reproject a GeoTIFF file to CRS EPSG:32632 ("WGS 84 / UTM zone 32N") + + .. code-block:: bash + + $ gdal raster reproject --dst-crs=EPSG:32632 in.tif out.tif --overwrite diff --git a/doc/source/programs/index.rst b/doc/source/programs/index.rst index c5ded0399aca..62cd29932139 100644 --- a/doc/source/programs/index.rst +++ b/doc/source/programs/index.rst @@ -31,6 +31,8 @@ single :program:`gdal` program that accepts commands and subcommands. gdal_raster gdal_raster_info gdal_raster_convert + gdal_raster_pipeline + gdal_raster_reproject gdal_vector gdal_vector_info gdal_vector_convert @@ -44,6 +46,8 @@ single :program:`gdal` program that accepts commands and subcommands. - :ref:`gdal_raster_command`: Entry point for raster commands - :ref:`gdal_raster_info_subcommand`: Get information on a raster dataset - :ref:`gdal_raster_convert_subcommand`: Convert a raster dataset + - :ref:`gdal_raster_pipeline_subcommand`: Process a raster dataset + - :ref:`gdal_raster_reproject_subcommand`: Reproject a raster dataset - :ref:`gdal_vector_command`: Entry point for vector commands - :ref:`gdal_vector_info_subcommand`: Get information on a vector dataset - :ref:`gdal_vector_convert_subcommand`: Convert a vector dataset