Skip to content

Commit

Permalink
Improve restart integ test
Browse files Browse the repository at this point in the history
This commit removes randomization of locale for SimpleDateFormat, instead using explicit locales. The test framework randomizes locales, so the test can be ran with " ./gradlew ':integTest' --tests "org.opensearch.ad.e2e.DetectionResultEvalutationIT.testRestartHCADDetector" -Dtests.seed=3892A75781CC10EC -Dtests.security.manager=false -Dtests.locale=th-TH-u-nu-thai-x-lvariant-TH -Dtests.timezone=America/Merida -Druntime.java=14". This would cause the code to write the data string in thai locale and make bulk api fail with message "failed to parse date field [๒๕๖๕-๐๕-๒๓T๒๐:๕๖:๐๐Z] with format [strict_date_optional_time||epoch_millis]","caused_by":{"type":"date_time_parse_exception".

Signed-off-by: Kaituo Li <[email protected]>
  • Loading branch information
kaituo committed May 23, 2022
1 parent 899efba commit 1950e4b
Showing 1 changed file with 6 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,7 @@ private void bulkIndexTrainData(String datasetName, List<JsonObject> data, int t
bulkRequestBuilder.append("{ \"index\" : { \"_index\" : \"" + datasetName + "\", \"_id\" : \"" + i + "\" } }\n");
bulkRequestBuilder.append(data.get(i).toString()).append("\n");
}
TestHelpers
Response response = TestHelpers
.makeRequest(
client,
"POST",
Expand All @@ -370,6 +370,8 @@ private void bulkIndexTrainData(String datasetName, List<JsonObject> data, int t
toHttpEntity(bulkRequestBuilder.toString()),
ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana"))
);
JsonObject json = new JsonParser().parse(new InputStreamReader(response.getEntity().getContent())).getAsJsonObject();
LOG.info("Bulk response: " + json);
Thread.sleep(5_000);
waitAllSyncheticDataIngested(trainTestSplit, datasetName, client);
}
Expand All @@ -394,7 +396,7 @@ private void bulkIndexTestData(List<JsonObject> data, String datasetName, int tr
}

private void waitAllSyncheticDataIngested(int expectedSize, String datasetName, RestClient client) throws Exception {
int maxWaitCycles = 12;
int maxWaitCycles = 5;
do {
Request request = new Request("POST", String.format(Locale.ROOT, "/%s/_search", datasetName));
request
Expand All @@ -420,6 +422,7 @@ private void waitAllSyncheticDataIngested(int expectedSize, String datasetName,
// "_index":"synthetic","_type":"_doc","_id":"10080","_score":null,"_source":{"timestamp":"2019-11-08T00:00:00Z","Feature1":156.30028000000001,"Feature2":100.211205,"host":"host1"},"sort":[1573171200000]}
Response response = client.performRequest(request);
JsonObject json = new JsonParser().parse(new InputStreamReader(response.getEntity().getContent())).getAsJsonObject();
LOG.info("Search response: " + json);
JsonArray hits = json.getAsJsonObject("hits").getAsJsonArray("hits");
if (hits != null
&& hits.size() == 1
Expand Down Expand Up @@ -622,7 +625,7 @@ private void verifyRestart(String datasetName, int intervalMinutes, int shingleS

// e.g., 2019-11-01T00:03:00Z
String pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'";
SimpleDateFormat simpleDateFormat = new SimpleDateFormat(pattern);
SimpleDateFormat simpleDateFormat = new SimpleDateFormat(pattern, Locale.ROOT);
simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
// calculate the gap between current time and the beginning of last shingle
// the gap is used to adjust input training data's time so that the last
Expand Down

0 comments on commit 1950e4b

Please sign in to comment.