Merge branch 'master' into develop/update-externals

ReleaseNotes.md

@@ -1,20 +1,22 @@
-![alt tag](https://raw.github.com/ngageoint/six-library/master/docs/six_logo.png?raw=true)
-
-# SIX Release Notes
-
-## (WIP: Version 3.1.9); ????, ??, 2022
-* replace KDTree *`std::complex<float>` -> AMP8I_PHS8I conversion* with a
-  ["math based" approach](https://github.com/ngageoint/six-library/pull/537#issuecomment-1026453353).
+![alt tag](https://raw.github.com/ngageoint/six-library/master/docs/six_logo.png?raw=true)
+
+# SIX Release Notes
+
+## Version 3.1.9; February 22, 2022 (aka 2/22/22)
+* [coda-oss](https://github.com/mdaus/coda-oss) version [2022-02-22](https://github.com/mdaus/coda-oss/releases/tag/2022-02-22)
+* [nitro](https://github.com/mdaus/nitro) version [2.10.8](https://github.com/mdaus/nitro/releases/tag/NITRO-2.10.8)
+* replace KDTree *`std::complex<float>` -> AMP8I_PHS8I conversion* with a
+  ["math based" approach](https://github.com/ngageoint/six-library/pull/537#issuecomment-1026453353).
 * restore SIDD 2.0 `AngleMagnitudeType`, SIDD 3.0 is `AngleZeroToExclusive360MagnitudeType`
-* bugfix: "We found a bug/mistake/error in **six.sicd/source/RadarCollection.cpp**, specifically the function to rotate the Area\Plane\SegmentList block."
-
-## Version 3.1.8; December 13, 2021
-* [coda-oss](https://github.com/mdaus/coda-oss) version [2021-12-13](https://github.com/mdaus/coda-oss/releases/tag/2021-12-13)
-* [nitro](https://github.com/mdaus/nitro) version [2.10.7](https://github.com/mdaus/nitro/releases/tag/NITRO-2.10.7)
-* write [8AMPI_PHSI](https://github.com/ngageoint/six-library/tree/feature/8AMPI_PHSI) files
-* update schema for [SIDD 3.0](https://github.com/ngageoint/six-library/tree/feature/SIDD-3.0)
-
------
-
-## Contact
-February 2022, Dan <dot> Smith <at> Maxar <dot> <see><oh><em>
+* bugfix: "We found a bug/mistake/error in **six.sicd/source/RadarCollection.cpp**, specifically the function to rotate the Area\Plane\SegmentList block."
+
+## Version 3.1.8; December 13, 2021
+* [coda-oss](https://github.com/mdaus/coda-oss) version [2021-12-13](https://github.com/mdaus/coda-oss/releases/tag/2021-12-13)
+* [nitro](https://github.com/mdaus/nitro) version [2.10.7](https://github.com/mdaus/nitro/releases/tag/NITRO-2.10.7)
+* write [8AMPI_PHSI](https://github.com/ngageoint/six-library/tree/feature/8AMPI_PHSI) files
+* update schema for [SIDD 3.0](https://github.com/ngageoint/six-library/tree/feature/SIDD-3.0)
+
+-----
+
+## Contact
+February 2022, Dan <dot> Smith <at> Maxar <dot> <see><oh><em>

six/modules/c++/six.convert/include/six/convert/ConvertingReadControl.h

@@ -79,8 +79,8 @@ struct ConvertingReadControl : public ReadControl
      * \param pathname        Input filepath
      * \param schemaPathnames Not used
      */
-    virtual void load(const std::string& pathname,
-            const std::vector<std::string>& schemaPaths);
+    void load(const std::string& pathname, const std::vector<std::string>& schemaPaths) override;
+    void load(const std::filesystem::path& pathname, const std::vector<std::filesystem::path>* pSchemaPaths) override;
 
     /*!
      * \return type of file being converted

six/modules/c++/six.convert/source/ConvertingReadControl.cpp

@@ -58,6 +58,13 @@ void ConvertingReadControl::load(const std::string& pathname,
     mConverter->load(pathname);
     mContainer->addData(mConverter->convert().release());
 }
+void ConvertingReadControl::load(const std::filesystem::path& pathname,
+    const std::vector<std::filesystem::path>* /*pSchemaPaths*/)
+{
+    const std::vector<std::string> schemaPaths;
+    load(pathname.string(), schemaPaths);
+}
+
 
 std::string ConvertingReadControl::getFileType() const
 {

six/modules/c++/six.sicd/include/six/sicd/ImageData.h

@@ -41,6 +41,10 @@ namespace sicd
 {
 using cx_float = std::complex<float>;
 using AMP8I_PHS8I_t = std::pair<uint8_t, uint8_t>;
+//! Fixed size 256 element array of complex values.
+using input_values_t = std::array<std::complex<float>, UINT8_MAX + 1>;
+//! Fixed size 256 x 256 matrix of complex values.
+using input_amplitudes_t = std::array<input_values_t, UINT8_MAX + 1>;
 
 class GeoData;
 /*!
@@ -102,9 +106,20 @@ struct ImageData
     cx_float from_AMP8I_PHS8I(const AMP8I_PHS8I_t&) const; // for unit-tests
     static void to_AMP8I_PHS8I(const AmplitudeTable*, std::span<const cx_float>, std::span<AMP8I_PHS8I_t>, ptrdiff_t cutoff = -1); // for unit-tests
 
+    static void from_AMP8I_PHS8I(const input_amplitudes_t& lookup, std::span<const AMP8I_PHS8I_t>, std::span<cx_float>, ptrdiff_t cutoff = -1);
     void from_AMP8I_PHS8I(std::span<const AMP8I_PHS8I_t>, std::span<cx_float>, ptrdiff_t cutoff = -1) const;
     void to_AMP8I_PHS8I(std::span<const cx_float>, std::span<AMP8I_PHS8I_t>, ptrdiff_t cutoff = -1) const;
+
+    /*!
+     * Create a lookup table for converting from AMP8I_PHS8I to complex.
+     * @param pAmplitudeTable Input amplitude table. May be nullptr if no amplitude table is defined.
+     * @param pValues_ Output table's scope to keep it around past the function call. May be empty if there was no input amplitude table.
+     * @return reference to the output lookup table.
+     */
+    static const input_amplitudes_t& get_RE32F_IM32F_values(const six::AmplitudeTable* pAmplitudeTable,
+                                                            std::unique_ptr<input_amplitudes_t>& pValues_);
 };
+
 }
 }
 

six/modules/c++/six.sicd/include/six/sicd/NITFReadComplexXMLControl.h

@@ -65,21 +65,21 @@ namespace six
 			six::NITFReadControl& NITFReadControl() { return reader; }
 
 			void load(const std::string& fromFile, const std::vector<std::string>* pSchemaPaths);
-			void load(const std::string& fromFile, const std::vector<std::string>& schemaPaths)
-			{
-				load(fromFile, &schemaPaths);
-			}
 			void load(const std::filesystem::path& fromFile, const std::vector<std::filesystem::path>* pSchemaPaths);
-			void load(const std::filesystem::path& fromFile, const std::vector<std::filesystem::path>& schemaPaths)
+			template<typename TPath>
+			void load(const TPath& fromFile, const std::vector<TPath>& schemaPaths)
 			{
 				load(fromFile, &schemaPaths);
 			}
 			void load(const std::filesystem::path& fromFile)
 			{
 				load(fromFile, nullptr /*schemaPaths*/);
 			}
+
 			void load(io::FileInputStream&, const std::vector<std::string>* pSchemaPaths);
-			void load(io::FileInputStream& fis, const std::vector<std::string>& schemaPaths)
+			void load(io::FileInputStream&, const std::vector<std::filesystem::path>* pSchemaPaths);
+			template<typename TSchemaPath>
+			void load(io::FileInputStream& fis, const std::vector<TSchemaPath>& schemaPaths)
 			{
 				load(fis, &schemaPaths);
 			}

six/modules/c++/six.sicd/source/ImageData.cpp

@@ -155,9 +155,6 @@ static std::vector<KDNode_t> make_nodes(const six::AmplitudeTable* pAmplitudeTab
     return retval;
 }
 
-using input_values_t = std::array<std::complex<float>, UINT8_MAX + 1>;
-using input_amplitudes_t = std::array<input_values_t, UINT8_MAX + 1>;
-
 // input_amplitudes_t is too big for the stack
 static std::unique_ptr<input_amplitudes_t> AMP8I_PHS8I_to_RE32F_IM32F_(const six::AmplitudeTable* pAmplitudeTable)
 {
@@ -175,7 +172,7 @@ static std::unique_ptr<input_amplitudes_t> AMP8I_PHS8I_to_RE32F_IM32F_(const six
 }
 
 // This is a non-templatized function so that there is copy of the "static" data with a NULL AmplutdeTable.
-static const input_amplitudes_t* get_RE32F_IM32F_values(const six::AmplitudeTable* pAmplitudeTable)
+static const input_amplitudes_t* get_cached_RE32F_IM32F_values(const six::AmplitudeTable* pAmplitudeTable)
 {
     if (pAmplitudeTable == nullptr)
     {
@@ -193,7 +190,7 @@ std::complex<float> ImageData::from_AMP8I_PHS8I(const AMP8I_PHS8I_t& input) cons
     }
 
     auto const pAmplitudeTable = amplitudeTable.get();
-    auto const pValues = get_RE32F_IM32F_values(pAmplitudeTable);
+    auto const pValues = get_cached_RE32F_IM32F_values(pAmplitudeTable);
 
     // Do we have a cahced result to use (no amplitude table)?
     // Or must it be recomputed (have an amplutude table)?
@@ -206,10 +203,10 @@ std::complex<float> ImageData::from_AMP8I_PHS8I(const AMP8I_PHS8I_t& input) cons
     return std::complex<float>(gsl::narrow_cast<float>(S.real()), gsl::narrow_cast<float>(S.imag()));
 }
 
-static const input_amplitudes_t& get_RE32F_IM32F_values(const six::AmplitudeTable* pAmplitudeTable,
+const input_amplitudes_t& ImageData::get_RE32F_IM32F_values(const six::AmplitudeTable* pAmplitudeTable,
     std::unique_ptr<input_amplitudes_t>& pValues_)
 {
-    const input_amplitudes_t* pValues = get_RE32F_IM32F_values(pAmplitudeTable);
+    const input_amplitudes_t* pValues = get_cached_RE32F_IM32F_values(pAmplitudeTable);
     if (pValues == nullptr)
     {
         assert(pAmplitudeTable != nullptr);
@@ -230,6 +227,12 @@ void ImageData::from_AMP8I_PHS8I(std::span<const AMP8I_PHS8I_t> inputs, std::spa
 
     std::unique_ptr<input_amplitudes_t> pValues_;
     const auto& values = get_RE32F_IM32F_values(amplitudeTable.get(), pValues_);
+    from_AMP8I_PHS8I(values, inputs, results, cutoff_);
+}
+
+void ImageData::from_AMP8I_PHS8I(const input_amplitudes_t& values, std::span<const AMP8I_PHS8I_t> inputs, std::span<std::complex<float>> results,
+    ptrdiff_t cutoff_)
+{
     const auto get_RE32F_IM32F_value_f = [&values](const six::sicd::AMP8I_PHS8I_t& v)
     {
         return values[v.first][v.second];

six/modules/c++/six.sicd/source/NITFReadComplexXMLControl.cpp

@@ -46,19 +46,16 @@ void six::sicd::NITFReadComplexXMLControl::load(const std::string& fromFile,
 void six::sicd::NITFReadComplexXMLControl::load(const std::filesystem::path& fromFile,
     const std::vector<std::filesystem::path>* pSchemaPaths)
 {
-    const std::vector<std::string>* pSchemaPaths_ = nullptr;
-    std::vector<std::string> schemaPaths_;
-    if (pSchemaPaths != nullptr)
-    {
-        std::transform(pSchemaPaths->begin(), pSchemaPaths->end(), std::back_inserter(schemaPaths_), [](const fs::path& p) { return p.string(); });
-        pSchemaPaths_ = &schemaPaths_;
-    }
-    load(fromFile.string(), pSchemaPaths_);
+    reader.load(fromFile, pSchemaPaths);
 }
 void six::sicd::NITFReadComplexXMLControl::load(io::FileInputStream& fis, const std::vector<std::string>* pSchemaPaths)
 {
     reader.load(fis, pSchemaPaths);
 }
+void six::sicd::NITFReadComplexXMLControl::load(io::FileInputStream& fis, const std::vector<std::filesystem::path>* pSchemaPaths)
+{
+    reader.load(fis, pSchemaPaths);
+}
 
 
 std::shared_ptr<const six::Container> six::sicd::NITFReadComplexXMLControl::getContainer() const

six/modules/c++/six.sicd/source/Utilities.cpp

@@ -169,27 +169,29 @@ class SICD_readerAndConverter final
     {
         auto bufferPtr = buffer + ((row - offset.row) * (elementsPerRow / 2));
 
-        // Take each (uint8_t, uint8_t) out of the temp buffer and put it into the real buffer as a std::complex<float>
-        for (size_t index = 0; index < elementsPerRow * rowsToRead; index += 2)
-        {
-            // "For amplitude and phase components, the amplitude component is  stored first."
-            const auto& input_amplitude = tempVector[index];
-            const auto& input_value = tempVector[index + 1];
-
-            *bufferPtr = six::sicd::Utilities::from_AMP8I_PHS8I(input_amplitude, input_value, pAmplitudeTable);
-            bufferPtr++;
-        }
+        // There's type mangling going on here.
+        // We're taking std::vector<uint8_t> and saying it's packed with std::pair<uint8_t, uint8_t>.
+        static_assert(sizeof(uint8_t) * 2 == sizeof(six::sicd::AMP8I_PHS8I_t), "expected packed layout in pair");
+        auto packed = reinterpret_cast<const six::sicd::AMP8I_PHS8I_t*>(tempVector.data());
+
+        // Reuse image data's conversion to complex.
+        static const ptrdiff_t kDefaultCutoff = 0;
+        size_t count = (elementsPerRow * rowsToRead) / 2;
+        std::span<const six::sicd::AMP8I_PHS8I_t> input(packed, count);
+        std::span<std::complex<float>> output(bufferPtr, input.size());
+        six::sicd::ImageData::from_AMP8I_PHS8I(lookup, input, output, kDefaultCutoff);
     }
     const types::RowCol<size_t>& offset;
     std::complex<float>* buffer;
-    const six::AmplitudeTable* pAmplitudeTable = nullptr;
+    std::unique_ptr<six::sicd::input_amplitudes_t> lookupScope;
+    const six::sicd::input_amplitudes_t& lookup;
 
 public:
     SICD_readerAndConverter(six::NITFReadControl& reader, size_t imageNumber,
 			    const types::RowCol<size_t>& offset, const types::RowCol<size_t>& extent,
                 size_t elementsPerRow,
 			    std::complex<float>* buffer,  const six::AmplitudeTable* pAmplitudeTable = nullptr)
-      : offset(offset), buffer(buffer), pAmplitudeTable(pAmplitudeTable)
+      : offset(offset), buffer(buffer), lookupScope(nullptr), lookup(six::sicd::ImageData::get_RE32F_IM32F_values(pAmplitudeTable, lookupScope))
     {
         SICDreader<T>(reader, imageNumber, offset, extent, elementsPerRow,
             [&](size_t elementsPerRow, size_t row, size_t rowsToRead, const std::vector<T>& tempVector)

six/modules/c++/six.sicd/unittests/test_valid_six.cpp

@@ -30,6 +30,7 @@
 #include <std/optional>
 #include <cmath>
 #include <std/span>
+#include <algorithm>
 
 #include <io/FileInputStream.h>
 #include <logging/NullLogger.h>
@@ -98,6 +99,13 @@ static void setNitfPluginPath()
     sys::OS().setEnv("NITF_PLUGIN_PATH", path.string(), true /*overwrite*/);
 }
 
+static std::vector<fs::path> schemaPaths()
+{
+    const auto relativePath = fs::path("six") / "modules" / "c++" / "six.sicd" / "conf" / "schema";
+    auto path = six::testing::buildRootDir(argv0()) / relativePath;
+    TEST_ASSERT(fs::exists(path));
+    return { std::move(path) };
+}
 
 static std::shared_ptr<six::Container> getContainer(six::sicd::NITFReadComplexXMLControl& reader)
 {
@@ -187,8 +195,11 @@ TEST_CASE(valid_six_50x50)
 
     valid_six_50x50_(nullptr /*pSchemaPaths*/); // no XML validiaton
 
-    const std::vector<std::filesystem::path> schemaPaths;
-    valid_six_50x50_(&schemaPaths); // validate against schema
+    auto schemaPaths = ::schemaPaths();
+    valid_six_50x50_(&schemaPaths); // validate against schema (actual path)
+
+    schemaPaths.clear();
+    valid_six_50x50_(&schemaPaths); // "validate" against schema (use a default path)
 }
 
 const std::string classificationText_iso8859_1("NON CLASSIFI\xc9 / UNCLASSIFIED");  // ISO8859-1 "NON CLASSIFIÉ / UNCLASSIFIED"
@@ -200,9 +211,8 @@ TEST_CASE(sicd_French_xml)
 
     const auto inputPathname = getNitfPath("sicd_French_xml.nitf");
     std::unique_ptr<six::sicd::ComplexData> pComplexData;
-    //const std::vector<std::filesystem::path> schemaPaths;
-    //const auto image = six::sicd::readFromNITF(inputPathname, &schemaPaths, pComplexData);
-    const auto image = six::sicd::readFromNITF(inputPathname, pComplexData); // no validation
+    const auto schemaPaths = ::schemaPaths();
+    const auto image = six::sicd::readFromNITF(inputPathname, &schemaPaths, pComplexData);
     const six::Data* pData = pComplexData.get();
 
     const auto expectedCassificationText = sys::Platform == sys::PlatformType::Linux ? classificationText_utf_8 : classificationText_iso8859_1;
@@ -213,6 +223,27 @@ TEST_CASE(sicd_French_xml)
     test_nitf_image_info(*pComplexData, inputPathname, nitf::PixelValueType::Floating);
 }
 
+//TEST_CASE(sicd_French_legacy_xml)
+//{
+//    setNitfPluginPath();
+//
+//    const auto inputPathname = getNitfPath("sicd_French_xml.nitf");
+//    const auto pathname = inputPathname.string();
+//    const auto schemaPaths = ::schemaPaths();
+//
+//    // Use legacy APIs ... to test other XML processing path
+//    std::vector<std::string> schemaPaths_;
+//    std::transform(schemaPaths.begin(), schemaPaths.end(), std::back_inserter(schemaPaths_),
+//        [](const fs::path& p) { return p.string(); });
+//
+//    six::sicd::NITFReadComplexXMLControl reader;
+//    reader.setLogger();
+//    reader.load(pathname, schemaPaths_);
+//
+//    // For SICD, there's only one image (container->size() == 1)
+//    TEST_ASSERT(reader.getContainer()->size() == 1);
+//}
+
 static bool find_string(io::FileInputStream& stream, const std::string& s)
 {
     const auto pos = stream.tell();
@@ -255,7 +286,7 @@ static void sicd_French_xml_raw_(bool storeEncoding)
         TEST_ASSERT(encoding == PlatformEncoding);
     }
 
-    // UTF-8 characters in 50x50.nitf
+    // UTF-8 characters in sicd_French_xml.nitf
     std::string expectedCharData;
     size_t expectedLength;
     if (storeEncoding)
@@ -292,7 +323,6 @@ static void sicd_French_xml_raw_(bool storeEncoding)
 
     std::u8string u8_characterData;
     classificationXML.getCharacterData(u8_characterData);
-    std::clog << "'" << u8_characterData.length() << "' '" << expectedLength << "'\n";
     TEST_ASSERT_EQ(u8_characterData.length(), expectedLength);
     TEST_ASSERT(u8_characterData == u8_expectedCharData8);
 }
@@ -478,6 +508,7 @@ TEST_MAIN((void)argc; (void)argv;
     TEST_CHECK(valid_six_50x50);
     TEST_CHECK(sicd_French_xml_raw);
     TEST_CHECK(sicd_French_xml);
+    //TEST_CHECK(sicd_French_legacy_xml);    
     TEST_CHECK(test_readFromNITF_sicd_50x50);
     TEST_CHECK(test_read_sicd_50x50);
     TEST_CHECK(test_create_sicd_from_mem_32f);

six/modules/c++/six.sidd/conf/schema/SICommonTypes_V1.0.xsd

@@ -528,10 +528,19 @@
 			<xsd:documentation/>
 		</xsd:annotation>
 		<xsd:sequence>
-			<xsd:element name="Angle" type="ZeroToExclusive360Type"/>
+			<xsd:element name="Angle" type="Neg180To180Type"/>
 			<xsd:element name="Magnitude" type="xsd:double"/>
 		</xsd:sequence>
 	</xsd:complexType>
+	<xsd:complexType name="AngleZeroToExclusive360MagnitudeType">
+		<xsd:annotation>
+			<xsd:documentation/>
+		</xsd:annotation>
+		<xsd:sequence>
+			<xsd:element name="Angle" type="ZeroToExclusive360Type"/>
+			<xsd:element name="Magnitude" type="xsd:double"/>
+		</xsd:sequence>
+	</xsd:complexType>	
 	<xsd:complexType name="RangeAzimuthType">
 		<xsd:annotation>
 			<xsd:documentation>Represents range and azimuth</xsd:documentation>

six/modules/c++/six.sidd/conf/schema/SIDD_schema_V3.0.0.xsd

@@ -1315,12 +1315,12 @@
             <xs:documentation>Phenomenology related to both the geometry and the final product processing. All values computed at the center time of the full collection.</xs:documentation>
         </xs:annotation>
         <xs:sequence>
-            <xs:element name="Shadow" type="sicommon:AngleMagnitudeType" minOccurs="0">
+            <xs:element name="Shadow" type="sicommon:AngleZeroToExclusive360MagnitudeType" minOccurs="0">
                 <xs:annotation>
                     <xs:documentation>The phenomon where vertical objects occlude radar energy.</xs:documentation>
                 </xs:annotation>
             </xs:element>
-            <xs:element name="Layover" type="sicommon:AngleMagnitudeType" minOccurs="0">
+            <xs:element name="Layover" type="sicommon:AngleZeroToExclusive360MagnitudeType" minOccurs="0">
                 <xs:annotation>
                     <xs:documentation>The phenomenon where vertical objects appear as ground objects with the same range/range rate.</xs:documentation>
                 </xs:annotation>