From 6d0e59ee4daecadfa6956809fbb9d8f621871c7f Mon Sep 17 00:00:00 2001 From: Anthony Papetti Date: Mon, 16 Sep 2024 18:35:02 -0400 Subject: [PATCH 1/6] Added driver --- examples/CMakeLists.txt | 1 + examples/driver.cpp | 43 +++++++++++++ examples/extended_fuzzer.cpp | 118 +++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 examples/driver.cpp create mode 100644 examples/extended_fuzzer.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2eb1667740..2f2c057c1e 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -20,6 +20,7 @@ set(examples bag_read bag_vr_create bag_vr_read + driver ) foreach(example ${examples}) diff --git a/examples/driver.cpp b/examples/driver.cpp new file mode 100644 index 0000000000..0160a8ed31 --- /dev/null +++ b/examples/driver.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "extended_fuzzer.cpp" + +int main(int argc, char** argv)//argv, which is an array of pointers to strings representing command-line arguments + // argc, which is an integer representing the number of command-line arguments +{ + assert(argc == 2); + std::string filename = argv[1]; //initialize a string variable named filename with the value of the first command-line argument + + // https://stackoverflow.com/questions/7880/how-do-you-open-a-file-in-c + std::ifstream input(filename, std::ios::binary); + //create an input file stream object named input and associates it with the file whose name is stored in the filename variable + + std::vector bytes;// vector used to store the bytes read from the input file + + uint8_t byte; + while (input >> byte) //reads bytes from the input file stream input + // The loop continues until there are no more bytes to read from the file + { + bytes.push_back(byte);//each byte read from the file is appended to the bytes vector using the push_back method + } + + uint8_t *array = &bytes[0];//array that points to the memory address of the first element of the bytes vector + //BAG uses this + size_t len = bytes.size(); + + std::cout << "got " << len << " bytes" << std::endl; + + //char* filename_cstr = new char[filename.length() + 1]; + + //strcpy(filename_cstr, filename.c_str()); + + LLVMFuzzerTestOneInputByFile(filename.c_str()); +} \ No newline at end of file diff --git a/examples/extended_fuzzer.cpp b/examples/extended_fuzzer.cpp new file mode 100644 index 0000000000..4bb4a8f026 --- /dev/null +++ b/examples/extended_fuzzer.cpp @@ -0,0 +1,118 @@ +#include +#include + +#include +#include +#include + +#include "bag_dataset.h" + +using BAG::Dataset; + + +void printLayerDescriptor( + const BAG::LayerDescriptor& descriptor) +{ + std::cout << "\t\tchunkSize == " << descriptor.getChunkSize() << '\n'; + std::cout << "\t\tcompression level == " << descriptor.getCompressionLevel() << '\n'; + std::cout << "\t\tdata type == " << descriptor.getDataType() << '\n'; + std::cout << "\t\telement size == " << +descriptor.getElementSize() << '\n'; + std::cout << "\t\tinternalPath == " << descriptor.getInternalPath() << '\n'; + std::cout << "\t\tlayer type == " << descriptor.getLayerType() << '\n'; + + const auto minMax = descriptor.getMinMax(); + std::cout << "\t\tmin max == (" << std::get<0>(minMax) << ", " << + std::get<1>(minMax) << ")\n"; +} + +extern "C" int LLVMFuzzerTestOneInputByFile(const char* filename) { + + + auto pDataset = Dataset::open(filename, BAG_OPEN_READONLY); + if (pDataset == NULL) { + return EXIT_FAILURE; + } + + const auto& descriptor = pDataset->getDescriptor(); + uint64_t numRows = 0, numCols = 0; + std::tie(numRows, numCols) = descriptor.getDims(); + std::cout << "\trows, columns == " << numRows << ", " << numCols << '\n'; + + double minX = 0., minY = 0., maxX = 0., maxY = 0.; + std::tie(minX, minY) = pDataset->gridToGeo(0, 0); + std::tie(maxX, maxY) = pDataset->gridToGeo(numRows - 1, numCols - 1); + + std::cout << "\tgrid cover (llx, lly), (urx, ury) == (" << + std::setprecision(10) << minX << ", " << minY << "), (" << maxX << + ", " << maxY << ")\n"; + + const auto& dims = descriptor.getDims(); + std::cout << "\tdims == (" << std::get<0>(dims) << ", " << + std::get<1>(dims) << ")\n"; + + const auto& gridSpacing = descriptor.getGridSpacing(); + std::cout << "\tgrid spacing == (" << std::get<0>(gridSpacing) << ", " << + std::get<1>(gridSpacing) << ")\n"; + + const auto& origin = descriptor.getOrigin(); + std::cout << "\torigin == (" << std::get<0>(origin) << ", " + << std::get<1>(origin) << ")\n"; + + const auto& projCover = descriptor.getProjectedCover(); + std::cout << "\tprojected cover (llx, lly), (urx, ury) == (" << std::get<0>(projCover) << ", " << + std::get<1>(projCover) << "), (" << std::get<2>(projCover) << ", " << + std::get<3>(projCover) << ")\n"; + + std::cout << "\tversion == " << descriptor.getVersion() << '\n'; + + std::cout << "\thorizontal reference system ==\n" << + descriptor.getHorizontalReferenceSystem() << "\n\n"; + + std::cout << "\tvertical reference system ==\n" << descriptor.getVerticalReferenceSystem() << '\n'; + + std::cout << "\nLayers:\n"; + + for (const auto& layer : pDataset->getLayers()) + { + auto pDescriptor = layer->getDescriptor(); + + std::cout << "\t" << pDescriptor->getName() << " Layer .. id(" << + pDescriptor->getId() << ")\n"; + + printLayerDescriptor(*pDescriptor); + } + + const auto& trackingList = pDataset->getTrackingList(); + std::cout << "\nTracking List: (" << trackingList.size() << " items)\n"; + + size_t itemNum = 0; + for (const auto& item : trackingList) + { + std::cout << "\tTracking list item #" << itemNum++ << '\n'; + std::cout << "\t\trow == " << item.row << '\n'; + std::cout << "\t\tcol == " << item.col << '\n'; + std::cout << "\t\tdepth == " << item.depth << '\n'; + std::cout << "\t\tuncertainty == " << item.uncertainty << '\n'; + std::cout << "\t\ttrack_code == " << item.track_code << '\n'; + std::cout << "\t\tlist_series == " << item.list_series << '\n'; + } + + pDataset->close(); + + return EXIT_SUCCESS; +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *buf, size_t len) { + char filename[256]; + snprintf(filename, 255, "/tmp/libfuzzer.%d", getpid()); + + // Save input file to temporary file so that we can read it. + FILE *fp = fopen(filename, "wb"); + if (!fp) { + return 0; + } + fwrite(buf, len, 1, fp); + fclose(fp); + + return LLVMFuzzerTestOneInputByFile(filename); +} \ No newline at end of file From a3ab24ec0778c63c3adf9a35914d7774bbe1c2af Mon Sep 17 00:00:00 2001 From: Anthony Papetti Date: Thu, 3 Oct 2024 12:48:45 -0400 Subject: [PATCH 2/6] Got rid of the memory leaks for issue 70585 --- api/bag_metadata_import.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/api/bag_metadata_import.cpp b/api/bag_metadata_import.cpp index 69cfa6df8a..ec05792435 100644 --- a/api/bag_metadata_import.cpp +++ b/api/bag_metadata_import.cpp @@ -859,6 +859,7 @@ bool decodeDataQualityInfo( else if (schemaVersion == 2) { //gmd:DQ_DataQuality/gmd:scope/gmd:DQ_Scope/gmd:level + delete[] dataQualityInfo.scope; dataQualityInfo.scope = getContentsAsCharStar(node, "gmd:DQ_DataQuality/gmd:scope/gmd:DQ_Scope/gmd:level/gmd:MD_ScopeCode"); @@ -986,6 +987,7 @@ bool decodeSpatialRepresentationInfo( "gmd:MD_Georectified/gmd:axisDimensionProperties/gmd:MD_Dimension/gmd:dimensionName/gmd:MD_DimensionNameTypeCode[@codeListValue='column']/parent::*/parent::*/gmd:resolution/gco:Measure", "uom"); //gmd:MD_Georectified/gmd:cellGeometry + delete[] spatialRepresentationInfo.cellGeometry; spatialRepresentationInfo.cellGeometry = getContentsAsCharStar(node, "gmd:MD_Georectified/gmd:cellGeometry/gmd:MD_CellGeometryCode"); @@ -1660,18 +1662,22 @@ BagError bagImportMetadataFromXmlV2( return BAG_METADTA_NOT_INITIALIZED; //gmd:fileIdentifier + delete[] metadata.fileIdentifier; metadata.fileIdentifier = getContentsAsCharStar(*pRoot, "/gmi:MI_Metadata/gmd:fileIdentifier/gco:CharacterString"); //gmd:language + delete[] metadata.language; metadata.language = getContentsAsCharStar(*pRoot, "/gmi:MI_Metadata/gmd:language/gmd:LanguageCode"); //gmd:characterSet + delete[] metadata.characterSet; metadata.characterSet = getContentsAsCharStar(*pRoot, "/gmi:MI_Metadata/gmd:characterSet/gmd:MD_CharacterSetCode"); //gmd:hierarchyLevel + delete[] metadata.hierarchyLevel; metadata.hierarchyLevel = getContentsAsCharStar(*pRoot, "/gmi:MI_Metadata/gmd:hierarchyLevel/gmd:MD_ScopeCode"); @@ -1686,14 +1692,17 @@ BagError bagImportMetadataFromXmlV2( } //gmd:dateStamp + delete[] metadata.dateStamp; metadata.dateStamp = getContentsAsCharStar(*pRoot, "/gmi:MI_Metadata/gmd:dateStamp/gco:Date"); //gmd:metadataStandardName + delete[] metadata.metadataStandardName; metadata.metadataStandardName = getContentsAsCharStar(*pRoot, "/gmi:MI_Metadata/gmd:metadataStandardName/gco:CharacterString"); //gmd:metadataStandardVersion + delete[] metadata.metadataStandardVersion; metadata.metadataStandardVersion = getContentsAsCharStar(*pRoot, "/gmi:MI_Metadata/gmd:metadataStandardVersion/gco:CharacterString"); @@ -1849,7 +1858,9 @@ BagError bagImportMetadataFromXmlBuffer( if (!pDocument) return BAG_METADTA_NOT_INITIALIZED; - return bagImportMetadataFromXml(*pDocument, metadata, doValidation); + const BagError err = bagImportMetadataFromXml(*pDocument, metadata, doValidation); + xmlFreeDoc(pDocument); + return err; } //************************************************************************ @@ -1880,7 +1891,9 @@ BagError bagImportMetadataFromXmlFile( if (!pDocument) return BAG_METADTA_NOT_INITIALIZED; - return bagImportMetadataFromXml(*pDocument, metadata, doValidation); + const BagError err = bagImportMetadataFromXml(*pDocument, metadata, doValidation); + xmlFreeDoc(pDocument); + return err; } } // namespace BAG From fc6295715c139780d53e2fc252c43d6e836f2ef2 Mon Sep 17 00:00:00 2001 From: Anthony Papetti Date: Fri, 18 Oct 2024 12:22:53 -0400 Subject: [PATCH 3/6] Fixed segmentation faults by giving H5DOpen and H5GOpen error checking --- api/bag_dataset.cpp | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/api/bag_dataset.cpp b/api/bag_dataset.cpp index ab04c9276b..91372b342c 100644 --- a/api/bag_dataset.cpp +++ b/api/bag_dataset.cpp @@ -1055,6 +1055,26 @@ std::tuple Dataset::gridToGeo( return {x, y}; } +hid_t DopenProtector2(hid_t loc_id, const char *name, hid_t dapl_id) { + hid_t id = -1; + try { + id = H5Dopen2(loc_id, name, dapl_id); + } catch (std::exception& e) { + id = -1; + } + return id; +} + +hid_t GopenProtector2(hid_t loc_id, const char *name, hid_t dapl_id) { + hid_t id = -1; + try { + id = H5Dopen2(loc_id, name, dapl_id); + } catch (std::exception& e) { + id = -1; + } + return id; +} + //! Read an existing BAG. /*! \param fileName @@ -1088,8 +1108,7 @@ void Dataset::readDataset( const std::string internalPath = Layer::getInternalPath(layerType); if (internalPath.empty()) continue; - - const hid_t id = H5Dopen2(bagGroup.getLocId(), internalPath.c_str(), + hid_t id = DopenProtector2(bagGroup.getLocId(), internalPath.c_str(), H5P_DEFAULT); if (id < 0) continue; @@ -1105,7 +1124,7 @@ void Dataset::readDataset( // If the BAG is version 1.5+ ... if (bagVersion >= 1'005'000) { - hid_t id = H5Dopen2(bagGroup.getLocId(), NODE_GROUP_PATH, H5P_DEFAULT); + hid_t id = DopenProtector2(bagGroup.getLocId(), NODE_GROUP_PATH, H5P_DEFAULT); if (id >= 0) { H5Dclose(id); @@ -1120,8 +1139,7 @@ void Dataset::readDataset( NODE); this->addLayer(InterleavedLegacyLayer::open(*this, *layerDesc)); } - - id = H5Dopen2(bagGroup.getLocId(), ELEVATION_SOLUTION_GROUP_PATH, + id = DopenProtector2(bagGroup.getLocId(), ELEVATION_SOLUTION_GROUP_PATH, H5P_DEFAULT); if (id >= 0) { @@ -1146,7 +1164,7 @@ void Dataset::readDataset( } // Read optional VR - hid_t id = H5Dopen2(bagGroup.getLocId(), VR_TRACKING_LIST_PATH, H5P_DEFAULT); + hid_t id = DopenProtector2(bagGroup.getLocId(), VR_TRACKING_LIST_PATH, H5P_DEFAULT); if (id >= 0) { H5Dclose(id); @@ -1164,7 +1182,7 @@ void Dataset::readDataset( } // optional VRNodeLayer - id = H5Dopen2(bagGroup.getLocId(), VR_NODE_PATH, H5P_DEFAULT); + id = DopenProtector2(bagGroup.getLocId(), VR_NODE_PATH, H5P_DEFAULT); if (id >= 0) { H5Dclose(id); @@ -1177,7 +1195,7 @@ void Dataset::readDataset( m_pTrackingList = std::unique_ptr(new TrackingList{*this}); // Read optional Surface Corrections - id = H5Dopen2(bagGroup.getLocId(), VERT_DATUM_CORR_PATH, H5P_DEFAULT); + id = DopenProtector2(bagGroup.getLocId(), VERT_DATUM_CORR_PATH, H5P_DEFAULT); if (id >= 0) { H5Dclose(id); @@ -1190,7 +1208,7 @@ void Dataset::readDataset( if (bagVersion >= 2'000'000) { // Add all existing GeorefMetadataLayers - id = H5Gopen2(bagGroup.getLocId(), GEOREF_METADATA_PATH, H5P_DEFAULT); + id = GopenProtector2(bagGroup.getLocId(), GEOREF_METADATA_PATH, H5P_DEFAULT); if (id >= 0) { H5Gclose(id); From 768654599a9b83d1b3564b0d7533622b7538be09 Mon Sep 17 00:00:00 2001 From: Anthony Papetti Date: Fri, 18 Oct 2024 12:29:32 -0400 Subject: [PATCH 4/6] Revert "Fixed segmentation faults by giving H5DOpen and H5GOpen error checking" This reverts commit fc6295715c139780d53e2fc252c43d6e836f2ef2. --- api/bag_dataset.cpp | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/api/bag_dataset.cpp b/api/bag_dataset.cpp index 91372b342c..ab04c9276b 100644 --- a/api/bag_dataset.cpp +++ b/api/bag_dataset.cpp @@ -1055,26 +1055,6 @@ std::tuple Dataset::gridToGeo( return {x, y}; } -hid_t DopenProtector2(hid_t loc_id, const char *name, hid_t dapl_id) { - hid_t id = -1; - try { - id = H5Dopen2(loc_id, name, dapl_id); - } catch (std::exception& e) { - id = -1; - } - return id; -} - -hid_t GopenProtector2(hid_t loc_id, const char *name, hid_t dapl_id) { - hid_t id = -1; - try { - id = H5Dopen2(loc_id, name, dapl_id); - } catch (std::exception& e) { - id = -1; - } - return id; -} - //! Read an existing BAG. /*! \param fileName @@ -1108,7 +1088,8 @@ void Dataset::readDataset( const std::string internalPath = Layer::getInternalPath(layerType); if (internalPath.empty()) continue; - hid_t id = DopenProtector2(bagGroup.getLocId(), internalPath.c_str(), + + const hid_t id = H5Dopen2(bagGroup.getLocId(), internalPath.c_str(), H5P_DEFAULT); if (id < 0) continue; @@ -1124,7 +1105,7 @@ void Dataset::readDataset( // If the BAG is version 1.5+ ... if (bagVersion >= 1'005'000) { - hid_t id = DopenProtector2(bagGroup.getLocId(), NODE_GROUP_PATH, H5P_DEFAULT); + hid_t id = H5Dopen2(bagGroup.getLocId(), NODE_GROUP_PATH, H5P_DEFAULT); if (id >= 0) { H5Dclose(id); @@ -1139,7 +1120,8 @@ void Dataset::readDataset( NODE); this->addLayer(InterleavedLegacyLayer::open(*this, *layerDesc)); } - id = DopenProtector2(bagGroup.getLocId(), ELEVATION_SOLUTION_GROUP_PATH, + + id = H5Dopen2(bagGroup.getLocId(), ELEVATION_SOLUTION_GROUP_PATH, H5P_DEFAULT); if (id >= 0) { @@ -1164,7 +1146,7 @@ void Dataset::readDataset( } // Read optional VR - hid_t id = DopenProtector2(bagGroup.getLocId(), VR_TRACKING_LIST_PATH, H5P_DEFAULT); + hid_t id = H5Dopen2(bagGroup.getLocId(), VR_TRACKING_LIST_PATH, H5P_DEFAULT); if (id >= 0) { H5Dclose(id); @@ -1182,7 +1164,7 @@ void Dataset::readDataset( } // optional VRNodeLayer - id = DopenProtector2(bagGroup.getLocId(), VR_NODE_PATH, H5P_DEFAULT); + id = H5Dopen2(bagGroup.getLocId(), VR_NODE_PATH, H5P_DEFAULT); if (id >= 0) { H5Dclose(id); @@ -1195,7 +1177,7 @@ void Dataset::readDataset( m_pTrackingList = std::unique_ptr(new TrackingList{*this}); // Read optional Surface Corrections - id = DopenProtector2(bagGroup.getLocId(), VERT_DATUM_CORR_PATH, H5P_DEFAULT); + id = H5Dopen2(bagGroup.getLocId(), VERT_DATUM_CORR_PATH, H5P_DEFAULT); if (id >= 0) { H5Dclose(id); @@ -1208,7 +1190,7 @@ void Dataset::readDataset( if (bagVersion >= 2'000'000) { // Add all existing GeorefMetadataLayers - id = GopenProtector2(bagGroup.getLocId(), GEOREF_METADATA_PATH, H5P_DEFAULT); + id = H5Gopen2(bagGroup.getLocId(), GEOREF_METADATA_PATH, H5P_DEFAULT); if (id >= 0) { H5Gclose(id); From 55903bddf3d948b2c6cc188925450bbdb481e3c5 Mon Sep 17 00:00:00 2001 From: Anthony Papetti Date: Thu, 31 Oct 2024 12:52:47 -0400 Subject: [PATCH 5/6] Added error checking for HDF5 Aborts --- api/bag_dataset.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/bag_dataset.cpp b/api/bag_dataset.cpp index ab04c9276b..9cf0244195 100644 --- a/api/bag_dataset.cpp +++ b/api/bag_dataset.cpp @@ -32,6 +32,7 @@ #include #include #include +#include namespace BAG { @@ -1055,6 +1056,11 @@ std::tuple Dataset::gridToGeo( return {x, y}; } +void handleAbrt(int signum) { + std::cerr << "\nUnrecoverable HDF5 Error "; + exit(signum); +} + //! Read an existing BAG. /*! \param fileName @@ -1066,6 +1072,7 @@ void Dataset::readDataset( const std::string& fileName, OpenMode openMode) { + signal(SIGABRT, handleAbrt); m_pH5file = std::unique_ptr<::H5::H5File, DeleteH5File>(new ::H5::H5File{ fileName.c_str(), (openMode == BAG_OPEN_READONLY) ? H5F_ACC_RDONLY : H5F_ACC_RDWR}, From cad39437ea1b98ef73b8deb563284715f6955d20 Mon Sep 17 00:00:00 2001 From: Anthony Papetti Date: Thu, 31 Oct 2024 13:08:46 -0400 Subject: [PATCH 6/6] Added newline to error message --- api/bag_dataset.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/bag_dataset.cpp b/api/bag_dataset.cpp index 9cf0244195..02f332d900 100644 --- a/api/bag_dataset.cpp +++ b/api/bag_dataset.cpp @@ -1057,7 +1057,7 @@ std::tuple Dataset::gridToGeo( } void handleAbrt(int signum) { - std::cerr << "\nUnrecoverable HDF5 Error "; + std::cerr << "\nUnrecoverable HDF5 Error \n"; exit(signum); }