diff --git a/include/api/CModelSnapshotJsonWriter.h b/include/api/CModelSnapshotJsonWriter.h index a4b2469283..a07e4377ee 100644 --- a/include/api/CModelSnapshotJsonWriter.h +++ b/include/api/CModelSnapshotJsonWriter.h @@ -32,6 +32,7 @@ class API_EXPORT CModelSnapshotJsonWriter //! Structure to store the model snapshot metadata struct SModelSnapshotReport { + std::string s_MinVersion; core_t::TTime s_SnapshotTimestamp; std::string s_Description; std::string s_SnapshotId; diff --git a/lib/api/CAnomalyJob.cc b/lib/api/CAnomalyJob.cc index 2c8b3c57b3..a636ab0e2d 100644 --- a/lib/api/CAnomalyJob.cc +++ b/lib/api/CAnomalyJob.cc @@ -77,6 +77,10 @@ const std::string HIERARCHICAL_RESULTS_TAG("f"); const std::string LATEST_RECORD_TIME_TAG("h"); const std::string MODEL_PLOT_TAG("i"); const std::string LAST_RESULTS_TIME_TAG("j"); + +//! The minimum version required to read the state corresponding to a model snapshot. +//! This should be updated every time there is a breaking change to the model state. +const std::string MODEL_SNAPSHOT_MIN_VERSION("6.3.0"); } // Statics @@ -1363,6 +1367,7 @@ bool CAnomalyJob::persistState(const std::string &descriptionPrefix, if (m_PersistCompleteFunc) { CModelSnapshotJsonWriter::SModelSnapshotReport modelSnapshotReport{ + MODEL_SNAPSHOT_MIN_VERSION, snapshotTimestamp, descriptionPrefix + core::CTimeUtils::toIso8601(snapshotTimestamp), snapShotId, diff --git a/lib/api/CModelSnapshotJsonWriter.cc b/lib/api/CModelSnapshotJsonWriter.cc index 3361079b22..7c994b39d5 100644 --- a/lib/api/CModelSnapshotJsonWriter.cc +++ b/lib/api/CModelSnapshotJsonWriter.cc @@ -17,6 +17,7 @@ namespace // JSON field names const std::string JOB_ID("job_id"); +const std::string MIN_VERSION("min_version"); const std::string TIMESTAMP("timestamp"); const std::string MODEL_SNAPSHOT("model_snapshot"); const std::string SNAPSHOT_ID("snapshot_id"); @@ -45,6 +46,8 @@ void CModelSnapshotJsonWriter::write(const SModelSnapshotReport &report) m_Writer.String(JOB_ID); m_Writer.String(m_JobId); + m_Writer.String(MIN_VERSION); + m_Writer.String(report.s_MinVersion); m_Writer.String(SNAPSHOT_ID); m_Writer.String(report.s_SnapshotId); diff --git a/lib/api/unittest/CModelSnapshotJsonWriterTest.cc b/lib/api/unittest/CModelSnapshotJsonWriterTest.cc new file mode 100644 index 0000000000..721639fe71 --- /dev/null +++ b/lib/api/unittest/CModelSnapshotJsonWriterTest.cc @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +#include "CModelSnapshotJsonWriterTest.h" + +#include +#include + +#include + +#include + +#include + +#include + +using namespace ml; +using namespace api; + +CppUnit::Test *CModelSnapshotJsonWriterTest::suite() +{ + CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CModelSnapshotJsonWriterTest"); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CModelSnapshotJsonWriterTest::testWrite", + &CModelSnapshotJsonWriterTest::testWrite) ); + return suiteOfTests; +} + +void CModelSnapshotJsonWriterTest::testWrite(void) +{ + std::ostringstream sstream; + + // The output writer won't close the JSON structures until is is destroyed + { + model::CResourceMonitor::SResults modelSizeStats{ + 10000, // bytes used + 3, // # by fields + 1, // # partition fields + 150, // # over fields + 4, // # allocation failures + model_t::E_MemoryStatusOk, // memory status + core_t::TTime(1521046309) // bucket start time + }; + + CModelSnapshotJsonWriter::SModelSnapshotReport report{ + "6.3.0", + core_t::TTime(1521046309), + "the snapshot description", + "test_snapshot_id", + size_t(15), // # docs + modelSizeStats, + "some normalizer state", + core_t::TTime(1521046409), // last record time + core_t::TTime(1521040000) // last result time + }; + + core::CJsonOutputStreamWrapper wrappedOutStream(sstream); + CModelSnapshotJsonWriter writer("job", wrappedOutStream); + writer.write(report); + } + + rapidjson::Document arrayDoc; + arrayDoc.Parse(sstream.str().c_str()); + + CPPUNIT_ASSERT(arrayDoc.IsArray()); + CPPUNIT_ASSERT_EQUAL(rapidjson::SizeType(1), arrayDoc.Size()); + + const rapidjson::Value &object = arrayDoc[rapidjson::SizeType(0)]; + CPPUNIT_ASSERT(object.IsObject()); + + CPPUNIT_ASSERT(object.HasMember("model_snapshot")); + const rapidjson::Value &snapshot = object["model_snapshot"]; + CPPUNIT_ASSERT(snapshot.HasMember("job_id")); + CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(snapshot["job_id"].GetString())); + CPPUNIT_ASSERT(snapshot.HasMember("min_version")); + CPPUNIT_ASSERT_EQUAL(std::string("6.3.0"), std::string(snapshot["min_version"].GetString())); + CPPUNIT_ASSERT(snapshot.HasMember("snapshot_id")); + CPPUNIT_ASSERT_EQUAL(std::string("test_snapshot_id"), std::string(snapshot["snapshot_id"].GetString())); + CPPUNIT_ASSERT(snapshot.HasMember("snapshot_doc_count")); + CPPUNIT_ASSERT_EQUAL(int64_t(15), snapshot["snapshot_doc_count"].GetInt64()); + CPPUNIT_ASSERT(snapshot.HasMember("timestamp")); + CPPUNIT_ASSERT_EQUAL(int64_t(1521046309000), snapshot["timestamp"].GetInt64()); + CPPUNIT_ASSERT(snapshot.HasMember("description")); + CPPUNIT_ASSERT_EQUAL(std::string("the snapshot description"), std::string(snapshot["description"].GetString())); + CPPUNIT_ASSERT(snapshot.HasMember("latest_record_time_stamp")); + CPPUNIT_ASSERT_EQUAL(int64_t(1521046409000), snapshot["latest_record_time_stamp"].GetInt64()); + CPPUNIT_ASSERT(snapshot.HasMember("latest_result_time_stamp")); + CPPUNIT_ASSERT_EQUAL(int64_t(1521040000000), snapshot["latest_result_time_stamp"].GetInt64()); + + CPPUNIT_ASSERT(snapshot.HasMember("model_size_stats")); + const rapidjson::Value &modelSizeStats = snapshot["model_size_stats"]; + CPPUNIT_ASSERT(modelSizeStats.HasMember("job_id")); + CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(modelSizeStats["job_id"].GetString())); + CPPUNIT_ASSERT(modelSizeStats.HasMember("model_bytes")); + CPPUNIT_ASSERT_EQUAL(int64_t(20000), modelSizeStats["model_bytes"].GetInt64()); + CPPUNIT_ASSERT(modelSizeStats.HasMember("total_by_field_count")); + CPPUNIT_ASSERT_EQUAL(int64_t(3), modelSizeStats["total_by_field_count"].GetInt64()); + CPPUNIT_ASSERT(modelSizeStats.HasMember("total_partition_field_count")); + CPPUNIT_ASSERT_EQUAL(int64_t(1), modelSizeStats["total_partition_field_count"].GetInt64()); + CPPUNIT_ASSERT(modelSizeStats.HasMember("total_over_field_count")); + CPPUNIT_ASSERT_EQUAL(int64_t(150), modelSizeStats["total_over_field_count"].GetInt64()); + CPPUNIT_ASSERT(modelSizeStats.HasMember("bucket_allocation_failures_count")); + CPPUNIT_ASSERT_EQUAL(int64_t(4), modelSizeStats["bucket_allocation_failures_count"].GetInt64()); + CPPUNIT_ASSERT(modelSizeStats.HasMember("memory_status")); + CPPUNIT_ASSERT_EQUAL(std::string("ok"), std::string(modelSizeStats["memory_status"].GetString())); + CPPUNIT_ASSERT(modelSizeStats.HasMember("timestamp")); + CPPUNIT_ASSERT_EQUAL(int64_t(1521046309000), modelSizeStats["timestamp"].GetInt64()); + CPPUNIT_ASSERT(modelSizeStats.HasMember("log_time")); + + CPPUNIT_ASSERT(snapshot.HasMember("quantiles")); + const rapidjson::Value &quantiles = snapshot["quantiles"]; + CPPUNIT_ASSERT(quantiles.HasMember("job_id")); + CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(quantiles["job_id"].GetString())); + CPPUNIT_ASSERT(quantiles.HasMember("quantile_state")); + CPPUNIT_ASSERT_EQUAL(std::string("some normalizer state"), std::string(quantiles["quantile_state"].GetString())); + CPPUNIT_ASSERT(quantiles.HasMember("timestamp")); + CPPUNIT_ASSERT_EQUAL(int64_t(1521040000000), quantiles["timestamp"].GetInt64()); +} diff --git a/lib/api/unittest/CModelSnapshotJsonWriterTest.h b/lib/api/unittest/CModelSnapshotJsonWriterTest.h new file mode 100644 index 0000000000..cce22c314f --- /dev/null +++ b/lib/api/unittest/CModelSnapshotJsonWriterTest.h @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +#ifndef INCLUDED_CModelSnapshotJsonWriterTest_h +#define INCLUDED_CModelSnapshotJsonWriterTest_h + +#include + + +class CModelSnapshotJsonWriterTest : public CppUnit::TestFixture +{ + public: + void testWrite(void); + + static CppUnit::Test *suite(); +}; + +#endif // INCLUDED_CModelSnapshotJsonWriterTest_h + diff --git a/lib/api/unittest/Main.cc b/lib/api/unittest/Main.cc index 8ef08eaa56..7380abfbb1 100644 --- a/lib/api/unittest/Main.cc +++ b/lib/api/unittest/Main.cc @@ -24,6 +24,7 @@ #include "CLineifiedJsonOutputWriterTest.h" #include "CLineifiedXmlInputParserTest.h" #include "CModelPlotDataJsonWriterTest.h" +#include "CModelSnapshotJsonWriterTest.h" #include "CMultiFileDataAdderTest.h" #include "COutputChainerTest.h" #include "CRestorePreviousStateTest.h" @@ -58,6 +59,7 @@ int main(int argc, const char **argv) runner.addTest( CLineifiedJsonOutputWriterTest::suite() ); runner.addTest( CLineifiedXmlInputParserTest::suite() ); runner.addTest( CModelPlotDataJsonWriterTest::suite() ); + runner.addTest( CModelSnapshotJsonWriterTest::suite() ); runner.addTest( CMultiFileDataAdderTest::suite() ); runner.addTest( COutputChainerTest::suite() ); runner.addTest( CRestorePreviousStateTest::suite() ); diff --git a/lib/api/unittest/Makefile b/lib/api/unittest/Makefile index f47854bee0..536ce20f16 100644 --- a/lib/api/unittest/Makefile +++ b/lib/api/unittest/Makefile @@ -42,6 +42,7 @@ SRCS=\ CMockDataProcessor.cc \ CMockSearcher.cc \ CModelPlotDataJsonWriterTest.cc \ + CModelSnapshotJsonWriterTest.cc \ CMultiFileDataAdderTest.cc \ COutputChainerTest.cc \ CRestorePreviousStateTest.cc \