From 6ad765a9b58f511c98898a671d8fda71f460f97b Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 12 Jul 2019 17:42:45 +0100 Subject: [PATCH 01/12] [ML] Change response format of data frame stats endpoint This change adjusts the data frame stats endpoint to return the format discussed in #43767. Relates #43767 --- .../GetDataFrameTransformStatsResponse.java | 12 +- .../DataFrameTransformCheckpointStats.java | 71 ++++-- .../DataFrameTransformCheckpointingInfo.java | 33 ++- .../transforms/DataFrameTransformState.java | 169 ------------- .../DataFrameTransformStateAndStats.java | 104 -------- .../DataFrameTransformStateAndStatsInfo.java | 128 ++++++++++ .../client/DataFrameTransformIT.java | 28 +-- ...tDataFrameTransformStatsResponseTests.java | 12 +- ...ataFrameTransformCheckpointStatsTests.java | 33 ++- ...aFrameTransformCheckpointingInfoTests.java | 27 +-- ...aFrameTransformStateAndStatsInfoTests.java | 73 ++++++ .../DataFrameTransformStateAndStatsTests.java | 59 ----- .../DataFrameTransformStateTests.java | 78 ------ .../hlrc/DataFrameIndexerPositionTests.java | 6 +- .../DataFrameIndexerTransformStatsTests.java | 9 +- ...ataFrameTransformCheckpointStatsTests.java | 23 +- ...aFrameTransformCheckpointingInfoTests.java | 15 +- .../hlrc/DataFrameTransformProgressTests.java | 7 +- ...FrameTransformStateAndStatsInfoTests.java} | 126 +++++----- .../DataFrameTransformStateAndStatsTests.java | 73 ------ .../DataFrameTransformDocumentationIT.java | 18 +- .../xpack/core/dataframe/DataFrameField.java | 4 + .../GetDataFrameTransformsStatsAction.java | 36 +-- .../DataFrameTransformCheckpointStats.java | 125 ++++++++-- .../DataFrameTransformCheckpointingInfo.java | 56 ++--- .../DataFrameTransformStateAndStats.java | 26 +- .../DataFrameTransformStateAndStatsInfo.java | 227 ++++++++++++++++++ ...ameTransformsStatsActionResponseTests.java | 8 +- ...ataFrameTransformCheckpointStatsTests.java | 8 +- .../DataFrameTransformProgressTests.java | 1 + ...aFrameTransformStateAndStatsInfoTests.java | 56 +++++ ...portGetDataFrameTransformsStatsAction.java | 59 +++-- .../DataFrameTransformsConfigManager.java | 6 +- .../upgrades/DataFrameSurvivesUpgradeIT.java | 12 +- 34 files changed, 950 insertions(+), 778 deletions(-) delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformState.java delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStats.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsTests.java delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateTests.java rename client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/{DataFrameTransformStateTests.java => DataFrameTransformStateAndStatsInfoTests.java} (57%) delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponse.java index d3f854719a706..5f3829b846a18 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponse.java @@ -21,7 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.TaskOperationFailure; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -42,11 +42,11 @@ public class GetDataFrameTransformStatsResponse { @SuppressWarnings("unchecked") static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "get_data_frame_transform_stats_response", true, - args -> new GetDataFrameTransformStatsResponse((List) args[0], + args -> new GetDataFrameTransformStatsResponse((List) args[0], (List) args[1], (List) args[2])); static { - PARSER.declareObjectArray(constructorArg(), DataFrameTransformStateAndStats.PARSER::apply, TRANSFORMS); + PARSER.declareObjectArray(constructorArg(), DataFrameTransformStateAndStatsInfo.PARSER::apply, TRANSFORMS); // Discard the count field which is the size of the transforms array PARSER.declareInt((a, b) -> {}, COUNT); PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> TaskOperationFailure.fromXContent(p), @@ -59,11 +59,11 @@ public static GetDataFrameTransformStatsResponse fromXContent(final XContentPars return GetDataFrameTransformStatsResponse.PARSER.apply(parser, null); } - private final List transformsStateAndStats; + private final List transformsStateAndStats; private final List taskFailures; private final List nodeFailures; - public GetDataFrameTransformStatsResponse(List transformsStateAndStats, + public GetDataFrameTransformStatsResponse(List transformsStateAndStats, @Nullable List taskFailures, @Nullable List nodeFailures) { this.transformsStateAndStats = transformsStateAndStats; @@ -71,7 +71,7 @@ public GetDataFrameTransformStatsResponse(List this.nodeFailures = nodeFailures == null ? Collections.emptyList() : Collections.unmodifiableList(nodeFailures); } - public List getTransformsStateAndStats() { + public List getTransformsStateAndStats() { return transformsStateAndStats; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointStats.java index 723a38d61ea43..2239fe9f46c0b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointStats.java @@ -19,48 +19,86 @@ package org.elasticsearch.client.dataframe.transforms; +import org.elasticsearch.client.core.IndexerState; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + public class DataFrameTransformCheckpointStats { + + public static final ParseField CHECKPOINT = new ParseField("checkpoint"); + public static final ParseField INDEXER_STATE = new ParseField("indexer_state"); + public static final ParseField POSITION = new ParseField("position"); + public static final ParseField CHECKPOINT_PROGRESS = new ParseField("checkpoint_progress"); public static final ParseField TIMESTAMP_MILLIS = new ParseField("timestamp_millis"); public static final ParseField TIME_UPPER_BOUND_MILLIS = new ParseField("time_upper_bound_millis"); - public static DataFrameTransformCheckpointStats EMPTY = new DataFrameTransformCheckpointStats(0L, 0L); + public static final DataFrameTransformCheckpointStats EMPTY = new DataFrameTransformCheckpointStats(0L, null, null, null, 0L, 0L); + private final long checkpoint; + private final IndexerState indexerState; + private final DataFrameIndexerPosition position; + private final DataFrameTransformProgress checkpointProgress; private final long timestampMillis; private final long timeUpperBoundMillis; public static final ConstructingObjectParser LENIENT_PARSER = new ConstructingObjectParser<>( "data_frame_transform_checkpoint_stats", true, args -> { - long timestamp = args[0] == null ? 0L : (Long) args[0]; - long timeUpperBound = args[1] == null ? 0L : (Long) args[1]; + long checkpoint = args[0] == null ? 0L : (Long) args[0]; + IndexerState indexerState = (IndexerState) args[1]; + DataFrameIndexerPosition position = (DataFrameIndexerPosition) args[2]; + DataFrameTransformProgress checkpointProgress = (DataFrameTransformProgress) args[3]; + long timestamp = args[4] == null ? 0L : (Long) args[4]; + long timeUpperBound = args[5] == null ? 0L : (Long) args[5]; - return new DataFrameTransformCheckpointStats(timestamp, timeUpperBound); - }); + return new DataFrameTransformCheckpointStats(checkpoint, indexerState, position, checkpointProgress, timestamp, timeUpperBound); + }); static { - LENIENT_PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), TIMESTAMP_MILLIS); - LENIENT_PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), TIME_UPPER_BOUND_MILLIS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), CHECKPOINT); + LENIENT_PARSER.declareField(optionalConstructorArg(), p -> IndexerState.fromString(p.text()), INDEXER_STATE, + ObjectParser.ValueType.STRING); + LENIENT_PARSER.declareObject(optionalConstructorArg(), DataFrameIndexerPosition.PARSER, POSITION); + LENIENT_PARSER.declareObject(optionalConstructorArg(), DataFrameTransformProgress.PARSER, CHECKPOINT_PROGRESS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), TIMESTAMP_MILLIS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), TIME_UPPER_BOUND_MILLIS); } public static DataFrameTransformCheckpointStats fromXContent(XContentParser parser) throws IOException { return LENIENT_PARSER.parse(parser, null); } - public DataFrameTransformCheckpointStats(final long timestampMillis, final long timeUpperBoundMillis) { + public DataFrameTransformCheckpointStats(final long checkpoint, final IndexerState indexerState, + final DataFrameIndexerPosition position, final DataFrameTransformProgress checkpointProgress, + final long timestampMillis, final long timeUpperBoundMillis) { + this.checkpoint = checkpoint; + this.indexerState = indexerState; + this.position = position; + this.checkpointProgress = checkpointProgress; this.timestampMillis = timestampMillis; this.timeUpperBoundMillis = timeUpperBoundMillis; } - public DataFrameTransformCheckpointStats(StreamInput in) throws IOException { - this.timestampMillis = in.readLong(); - this.timeUpperBoundMillis = in.readLong(); + public long getCheckpoint() { + return checkpoint; + } + + public IndexerState getIndexerState() { + return indexerState; + } + + public DataFrameIndexerPosition getPosition() { + return position; + } + + public DataFrameTransformProgress getCheckpointProgress() { + return checkpointProgress; } public long getTimestampMillis() { @@ -73,7 +111,7 @@ public long getTimeUpperBoundMillis() { @Override public int hashCode() { - return Objects.hash(timestampMillis, timeUpperBoundMillis); + return Objects.hash(checkpoint, indexerState, position, checkpointProgress, timestampMillis, timeUpperBoundMillis); } @Override @@ -88,6 +126,11 @@ public boolean equals(Object other) { DataFrameTransformCheckpointStats that = (DataFrameTransformCheckpointStats) other; - return this.timestampMillis == that.timestampMillis && this.timeUpperBoundMillis == that.timeUpperBoundMillis; + return this.checkpoint == that.checkpoint + && Objects.equals(this.indexerState, that.indexerState) + && Objects.equals(this.position, that.position) + && Objects.equals(this.checkpointProgress, that.checkpointProgress) + && this.timestampMillis == that.timestampMillis + && this.timeUpperBoundMillis == that.timeUpperBoundMillis; } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfo.java index d00cf173e9e64..a5ef7a1ce0e05 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfo.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfo.java @@ -27,15 +27,14 @@ public class DataFrameTransformCheckpointingInfo { - public static final ParseField CURRENT_CHECKPOINT = new ParseField("current"); - public static final ParseField IN_PROGRESS_CHECKPOINT = new ParseField("in_progress"); + public static final ParseField LAST_CHECKPOINT = new ParseField("last", "current"); + public static final ParseField NEXT_CHECKPOINT = new ParseField("next", "in_progress"); public static final ParseField OPERATIONS_BEHIND = new ParseField("operations_behind"); - private final DataFrameTransformCheckpointStats current; - private final DataFrameTransformCheckpointStats inProgress; + private final DataFrameTransformCheckpointStats last; + private final DataFrameTransformCheckpointStats next; private final long operationsBehind; - private static final ConstructingObjectParser LENIENT_PARSER = new ConstructingObjectParser<>( "data_frame_transform_checkpointing_info", true, a -> { @@ -48,25 +47,25 @@ public class DataFrameTransformCheckpointingInfo { static { LENIENT_PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> DataFrameTransformCheckpointStats.fromXContent(p), CURRENT_CHECKPOINT); + (p, c) -> DataFrameTransformCheckpointStats.fromXContent(p), LAST_CHECKPOINT); LENIENT_PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> DataFrameTransformCheckpointStats.fromXContent(p), IN_PROGRESS_CHECKPOINT); + (p, c) -> DataFrameTransformCheckpointStats.fromXContent(p), NEXT_CHECKPOINT); LENIENT_PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), OPERATIONS_BEHIND); } - public DataFrameTransformCheckpointingInfo(DataFrameTransformCheckpointStats current, DataFrameTransformCheckpointStats inProgress, + public DataFrameTransformCheckpointingInfo(DataFrameTransformCheckpointStats last, DataFrameTransformCheckpointStats next, long operationsBehind) { - this.current = Objects.requireNonNull(current); - this.inProgress = Objects.requireNonNull(inProgress); + this.last = Objects.requireNonNull(last); + this.next = Objects.requireNonNull(next); this.operationsBehind = operationsBehind; } - public DataFrameTransformCheckpointStats getCurrent() { - return current; + public DataFrameTransformCheckpointStats getLast() { + return last; } - public DataFrameTransformCheckpointStats getInProgress() { - return inProgress; + public DataFrameTransformCheckpointStats getNext() { + return next; } public long getOperationsBehind() { @@ -79,7 +78,7 @@ public static DataFrameTransformCheckpointingInfo fromXContent(XContentParser p) @Override public int hashCode() { - return Objects.hash(current, inProgress, operationsBehind); + return Objects.hash(last, next, operationsBehind); } @Override @@ -94,8 +93,8 @@ public boolean equals(Object other) { DataFrameTransformCheckpointingInfo that = (DataFrameTransformCheckpointingInfo) other; - return Objects.equals(this.current, that.current) && - Objects.equals(this.inProgress, that.inProgress) && + return Objects.equals(this.last, that.last) && + Objects.equals(this.next, that.next) && this.operationsBehind == that.operationsBehind; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformState.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformState.java deleted file mode 100644 index 65216827f4837..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformState.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.dataframe.transforms; - -import org.elasticsearch.client.core.IndexerState; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser.ValueType; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; - -public class DataFrameTransformState { - - private static final ParseField INDEXER_STATE = new ParseField("indexer_state"); - private static final ParseField TASK_STATE = new ParseField("task_state"); - - // 7.3 BWC: current_position only exists in 7.2. In 7.3+ it is replaced by position. - private static final ParseField CURRENT_POSITION = new ParseField("current_position"); - private static final ParseField POSITION = new ParseField("position"); - private static final ParseField CHECKPOINT = new ParseField("checkpoint"); - private static final ParseField REASON = new ParseField("reason"); - private static final ParseField PROGRESS = new ParseField("progress"); - private static final ParseField NODE = new ParseField("node"); - - @SuppressWarnings("unchecked") - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("data_frame_transform_state", true, - args -> { - DataFrameTransformTaskState taskState = (DataFrameTransformTaskState) args[0]; - IndexerState indexerState = (IndexerState) args[1]; - Map bwcCurrentPosition = (Map) args[2]; - DataFrameIndexerPosition dataFrameIndexerPosition = (DataFrameIndexerPosition) args[3]; - - // BWC handling, translate current_position to position iff position isn't set - if (bwcCurrentPosition != null && dataFrameIndexerPosition == null) { - dataFrameIndexerPosition = new DataFrameIndexerPosition(bwcCurrentPosition, null); - } - - long checkpoint = (long) args[4]; - String reason = (String) args[5]; - DataFrameTransformProgress progress = (DataFrameTransformProgress) args[6]; - NodeAttributes node = (NodeAttributes) args[7]; - - return new DataFrameTransformState(taskState, indexerState, dataFrameIndexerPosition, checkpoint, reason, progress, - node); - }); - - static { - PARSER.declareField(constructorArg(), p -> DataFrameTransformTaskState.fromString(p.text()), TASK_STATE, ValueType.STRING); - PARSER.declareField(constructorArg(), p -> IndexerState.fromString(p.text()), INDEXER_STATE, ValueType.STRING); - PARSER.declareField(optionalConstructorArg(), (p, c) -> p.mapOrdered(), CURRENT_POSITION, ValueType.OBJECT); - PARSER.declareField(optionalConstructorArg(), DataFrameIndexerPosition::fromXContent, POSITION, ValueType.OBJECT); - PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), CHECKPOINT); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), REASON); - PARSER.declareField(optionalConstructorArg(), DataFrameTransformProgress::fromXContent, PROGRESS, ValueType.OBJECT); - PARSER.declareField(optionalConstructorArg(), NodeAttributes.PARSER::apply, NODE, ValueType.OBJECT); - } - - public static DataFrameTransformState fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - - private final DataFrameTransformTaskState taskState; - private final IndexerState indexerState; - private final long checkpoint; - private final DataFrameIndexerPosition position; - private final String reason; - private final DataFrameTransformProgress progress; - private final NodeAttributes node; - - public DataFrameTransformState(DataFrameTransformTaskState taskState, - IndexerState indexerState, - @Nullable DataFrameIndexerPosition position, - long checkpoint, - @Nullable String reason, - @Nullable DataFrameTransformProgress progress, - @Nullable NodeAttributes node) { - this.taskState = taskState; - this.indexerState = indexerState; - this.position = position; - this.checkpoint = checkpoint; - this.reason = reason; - this.progress = progress; - this.node = node; - } - - public IndexerState getIndexerState() { - return indexerState; - } - - public DataFrameTransformTaskState getTaskState() { - return taskState; - } - - @Nullable - public DataFrameIndexerPosition getPosition() { - return position; - } - - public long getCheckpoint() { - return checkpoint; - } - - @Nullable - public String getReason() { - return reason; - } - - @Nullable - public DataFrameTransformProgress getProgress() { - return progress; - } - - @Nullable - public NodeAttributes getNode() { - return node; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - - if (other == null || getClass() != other.getClass()) { - return false; - } - - DataFrameTransformState that = (DataFrameTransformState) other; - - return Objects.equals(this.taskState, that.taskState) && - Objects.equals(this.indexerState, that.indexerState) && - Objects.equals(this.position, that.position) && - Objects.equals(this.progress, that.progress) && - this.checkpoint == that.checkpoint && - Objects.equals(this.node, that.node) && - Objects.equals(this.reason, that.reason); - } - - @Override - public int hashCode() { - return Objects.hash(taskState, indexerState, position, checkpoint, reason, progress, node); - } - -} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStats.java deleted file mode 100644 index 9914a0e6331bc..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStats.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.dataframe.transforms; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; -import java.util.Objects; - -public class DataFrameTransformStateAndStats { - - public static final ParseField ID = new ParseField("id"); - public static final ParseField STATE_FIELD = new ParseField("state"); - public static final ParseField STATS_FIELD = new ParseField("stats"); - public static final ParseField CHECKPOINTING_INFO_FIELD = new ParseField("checkpointing"); - - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "data_frame_transform_state_and_stats", true, - a -> new DataFrameTransformStateAndStats((String) a[0], (DataFrameTransformState) a[1], (DataFrameIndexerTransformStats) a[2], - (DataFrameTransformCheckpointingInfo) a[3])); - - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), ID); - PARSER.declareObject(ConstructingObjectParser.constructorArg(), DataFrameTransformState.PARSER::apply, STATE_FIELD); - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> DataFrameIndexerTransformStats.fromXContent(p), - STATS_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> DataFrameTransformCheckpointingInfo.fromXContent(p), CHECKPOINTING_INFO_FIELD); - } - - public static DataFrameTransformStateAndStats fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - - private final String id; - private final DataFrameTransformState transformState; - private final DataFrameIndexerTransformStats transformStats; - private final DataFrameTransformCheckpointingInfo checkpointingInfo; - - public DataFrameTransformStateAndStats(String id, DataFrameTransformState state, DataFrameIndexerTransformStats stats, - DataFrameTransformCheckpointingInfo checkpointingInfo) { - this.id = id; - this.transformState = state; - this.transformStats = stats; - this.checkpointingInfo = checkpointingInfo; - } - - public String getId() { - return id; - } - - public DataFrameIndexerTransformStats getTransformStats() { - return transformStats; - } - - public DataFrameTransformState getTransformState() { - return transformState; - } - - public DataFrameTransformCheckpointingInfo getCheckpointingInfo() { - return checkpointingInfo; - } - - @Override - public int hashCode() { - return Objects.hash(id, transformState, transformStats, checkpointingInfo); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - - if (other == null || getClass() != other.getClass()) { - return false; - } - - DataFrameTransformStateAndStats that = (DataFrameTransformStateAndStats) other; - - return Objects.equals(this.id, that.id) && Objects.equals(this.transformState, that.transformState) - && Objects.equals(this.transformStats, that.transformStats) - && Objects.equals(this.checkpointingInfo, that.checkpointingInfo); - } -} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java new file mode 100644 index 0000000000000..4f07cb3b1b0eb --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.dataframe.transforms; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class DataFrameTransformStateAndStatsInfo { + + public static final ParseField ID = new ParseField("id"); + public static final ParseField TASK_STATE_FIELD = new ParseField("task_state"); + public static final ParseField REASON_FIELD = new ParseField("reason"); + public static final ParseField NODE_FIELD = new ParseField("node"); + public static final ParseField STATS_FIELD = new ParseField("stats"); + public static final ParseField CHECKPOINTING_INFO_FIELD = new ParseField("checkpointing"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "data_frame_transform_state_and_stats_info", true, + a -> new DataFrameTransformStateAndStatsInfo((String) a[0], (DataFrameTransformTaskState) a[1], (String) a[2], + (NodeAttributes) a[3], (DataFrameIndexerTransformStats) a[4], (DataFrameTransformCheckpointingInfo) a[5])); + + static { + PARSER.declareString(constructorArg(), ID); + PARSER.declareField(optionalConstructorArg(), p -> DataFrameTransformTaskState.fromString(p.text()), TASK_STATE_FIELD, + ObjectParser.ValueType.STRING); + PARSER.declareString(optionalConstructorArg(), REASON_FIELD); + PARSER.declareField(optionalConstructorArg(), NodeAttributes.PARSER::apply, NODE_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareObject(constructorArg(), (p, c) -> DataFrameIndexerTransformStats.fromXContent(p), STATS_FIELD); + PARSER.declareObject(optionalConstructorArg(), + (p, c) -> DataFrameTransformCheckpointingInfo.fromXContent(p), CHECKPOINTING_INFO_FIELD); + } + + public static DataFrameTransformStateAndStatsInfo fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private final String id; + private final String reason; + private final DataFrameTransformTaskState taskState; + private final NodeAttributes node; + private final DataFrameIndexerTransformStats transformStats; + private final DataFrameTransformCheckpointingInfo checkpointingInfo; + + public DataFrameTransformStateAndStatsInfo(String id, DataFrameTransformTaskState taskState, String reason, NodeAttributes node, + DataFrameIndexerTransformStats stats, + DataFrameTransformCheckpointingInfo checkpointingInfo) { + this.id = id; + this.taskState = taskState; + this.reason = reason; + this.node = node; + this.transformStats = stats; + this.checkpointingInfo = checkpointingInfo; + } + + public String getId() { + return id; + } + + public DataFrameTransformTaskState getTaskState() { + return taskState; + } + + public String getReason() { + return reason; + } + + public NodeAttributes getNode() { + return node; + } + + public DataFrameIndexerTransformStats getTransformStats() { + return transformStats; + } + + public DataFrameTransformCheckpointingInfo getCheckpointingInfo() { + return checkpointingInfo; + } + + @Override + public int hashCode() { + return Objects.hash(id, taskState, reason, node, transformStats, checkpointingInfo); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + DataFrameTransformStateAndStatsInfo that = (DataFrameTransformStateAndStatsInfo) other; + + return Objects.equals(this.id, that.id) + && Objects.equals(this.taskState, that.taskState) + && Objects.equals(this.reason, that.reason) + && Objects.equals(this.node, that.node) + && Objects.equals(this.transformStats, that.transformStats) + && Objects.equals(this.checkpointingInfo, that.checkpointingInfo); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java index d7371468b1506..64d4f7f57e524 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java @@ -41,7 +41,7 @@ import org.elasticsearch.client.dataframe.StopDataFrameTransformResponse; import org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.client.dataframe.transforms.DestConfig; import org.elasticsearch.client.dataframe.transforms.SourceConfig; @@ -259,7 +259,7 @@ public void testStartStop() throws IOException { GetDataFrameTransformStatsResponse statsResponse = execute(new GetDataFrameTransformStatsRequest(id), client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync); assertThat(statsResponse.getTransformsStateAndStats(), hasSize(1)); - DataFrameTransformTaskState taskState = statsResponse.getTransformsStateAndStats().get(0).getTransformState().getTaskState(); + DataFrameTransformTaskState taskState = statsResponse.getTransformsStateAndStats().get(0).getTaskState(); // Since we are non-continuous, the transform could auto-stop between being started earlier and us gathering the statistics assertThat(taskState, is(oneOf(DataFrameTransformTaskState.STARTED, DataFrameTransformTaskState.STOPPED))); @@ -274,7 +274,7 @@ public void testStartStop() throws IOException { // Calling stop with wait_for_completion assures that we will be in the `STOPPED` state for the transform task statsResponse = execute(new GetDataFrameTransformStatsRequest(id), client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync); - taskState = statsResponse.getTransformsStateAndStats().get(0).getTransformState().getTaskState(); + taskState = statsResponse.getTransformsStateAndStats().get(0).getTaskState(); assertThat(taskState, is(DataFrameTransformTaskState.STOPPED)); } @@ -354,9 +354,9 @@ public void testGetStats() throws Exception { client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync); assertEquals(1, statsResponse.getTransformsStateAndStats().size()); - DataFrameTransformStateAndStats stats = statsResponse.getTransformsStateAndStats().get(0); - assertEquals(DataFrameTransformTaskState.STOPPED, stats.getTransformState().getTaskState()); - assertEquals(IndexerState.STOPPED, stats.getTransformState().getIndexerState()); + DataFrameTransformStateAndStatsInfo stats = statsResponse.getTransformsStateAndStats().get(0); + assertEquals(DataFrameTransformTaskState.STOPPED, stats.getTaskState()); + assertEquals(IndexerState.STOPPED, stats.getCheckpointingInfo().getNext().getIndexerState()); DataFrameIndexerTransformStats zeroIndexerStats = new DataFrameIndexerTransformStats(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L); assertEquals(zeroIndexerStats, stats.getTransformStats()); @@ -369,17 +369,17 @@ public void testGetStats() throws Exception { assertBusy(() -> { GetDataFrameTransformStatsResponse response = execute(new GetDataFrameTransformStatsRequest(id), client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync); - DataFrameTransformStateAndStats stateAndStats = response.getTransformsStateAndStats().get(0); + DataFrameTransformStateAndStatsInfo stateAndStats = response.getTransformsStateAndStats().get(0); assertNotEquals(zeroIndexerStats, stateAndStats.getTransformStats()); - assertNotNull(stateAndStats.getTransformState().getProgress()); - assertThat(stateAndStats.getTransformState().getTaskState(), + assertThat(stateAndStats.getTaskState(), is(oneOf(DataFrameTransformTaskState.STARTED, DataFrameTransformTaskState.STOPPED))); - assertThat(stateAndStats.getTransformState().getIndexerState(), + assertThat(stateAndStats.getCheckpointingInfo().getNext().getIndexerState(), is(oneOf(IndexerState.STARTED, IndexerState.STOPPED))); - assertThat(stateAndStats.getTransformState().getProgress().getPercentComplete(), equalTo(100.0)); - assertThat(stateAndStats.getTransformState().getProgress().getTotalDocs(), greaterThan(0L)); - assertThat(stateAndStats.getTransformState().getProgress().getRemainingDocs(), equalTo(0L)); - assertThat(stateAndStats.getTransformState().getReason(), is(nullValue())); + assertNotNull(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress()); + assertThat(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress().getPercentComplete(), equalTo(100.0)); + assertThat(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress().getTotalDocs(), greaterThan(0L)); + assertThat(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress().getRemainingDocs(), equalTo(0L)); + assertThat(stateAndStats.getReason(), is(nullValue())); }); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponseTests.java index 89c0813eff8c6..ea808cf90b47a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponseTests.java @@ -21,8 +21,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.TaskOperationFailure; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsTests; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfoTests; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; @@ -48,9 +48,9 @@ public void testXContentParser() throws IOException { private static GetDataFrameTransformStatsResponse createTestInstance() { int count = randomIntBetween(1, 3); - List stats = new ArrayList<>(); + List stats = new ArrayList<>(); for (int i=0; i taskFailures = null; @@ -77,8 +77,8 @@ private static void toXContent(GetDataFrameTransformStatsResponse response, XCon builder.startObject(); { builder.startArray("transforms"); - for (DataFrameTransformStateAndStats stats : response.getTransformsStateAndStats()) { - DataFrameTransformStateAndStatsTests.toXContent(stats, builder); + for (DataFrameTransformStateAndStatsInfo stats : response.getTransformsStateAndStats()) { + DataFrameTransformStateAndStatsInfoTests.toXContent(stats, builder); } builder.endArray(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointStatsTests.java index 23019a0e58f95..4d4ba5967e74d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointStatsTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.dataframe.transforms; +import org.elasticsearch.client.core.IndexerState; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; @@ -30,22 +31,38 @@ public class DataFrameTransformCheckpointStatsTests extends ESTestCase { public void testFromXContent() throws IOException { xContentTester(this::createParser, - DataFrameTransformCheckpointStatsTests::randomDataFrameTransformCheckpointStats, - DataFrameTransformCheckpointStatsTests::toXContent, - DataFrameTransformCheckpointStats::fromXContent) + DataFrameTransformCheckpointStatsTests::randomDataFrameTransformCheckpointStats, + DataFrameTransformCheckpointStatsTests::toXContent, + DataFrameTransformCheckpointStats::fromXContent) .supportsUnknownFields(true) + .randomFieldsExcludeFilter(field -> field.startsWith("position")) .test(); } public static DataFrameTransformCheckpointStats randomDataFrameTransformCheckpointStats() { - return new DataFrameTransformCheckpointStats(randomLongBetween(1, 1_000_000), randomLongBetween(0, 1_000_000)); + return new DataFrameTransformCheckpointStats(randomLongBetween(1, 1_000_000), + randomBoolean() ? null : randomFrom(IndexerState.values()), + randomBoolean() ? null : DataFrameIndexerPositionTests.randomDataFrameIndexerPosition(), + randomBoolean() ? null : DataFrameTransformProgressTests.randomInstance(), + randomLongBetween(1, 1_000_000), randomLongBetween(0, 1_000_000)); } public static void toXContent(DataFrameTransformCheckpointStats stats, XContentBuilder builder) throws IOException { builder.startObject(); - builder.field("timestamp_millis", stats.getTimestampMillis()); - builder.field("time_upper_bound_millis", stats.getTimeUpperBoundMillis()); + builder.field(DataFrameTransformCheckpointStats.CHECKPOINT.getPreferredName(), stats.getCheckpoint()); + if (stats.getIndexerState() != null) { + builder.field(DataFrameTransformCheckpointStats.INDEXER_STATE.getPreferredName(), stats.getIndexerState().value()); + } + if (stats.getPosition() != null) { + builder.field(DataFrameTransformCheckpointStats.POSITION.getPreferredName()); + DataFrameIndexerPositionTests.toXContent(stats.getPosition(), builder); + } + if (stats.getCheckpointProgress() != null) { + builder.field(DataFrameTransformCheckpointStats.CHECKPOINT_PROGRESS.getPreferredName()); + DataFrameTransformProgressTests.toXContent(stats.getCheckpointProgress(), builder); + } + builder.field(DataFrameTransformCheckpointStats.TIMESTAMP_MILLIS.getPreferredName(), stats.getTimestampMillis()); + builder.field(DataFrameTransformCheckpointStats.TIME_UPPER_BOUND_MILLIS.getPreferredName(), stats.getTimeUpperBoundMillis()); builder.endObject(); } - -} \ No newline at end of file +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfoTests.java index 8f12d90a1c468..f70a853784a83 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfoTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfoTests.java @@ -30,32 +30,31 @@ public class DataFrameTransformCheckpointingInfoTests extends ESTestCase { public void testFromXContent() throws IOException { xContentTester(this::createParser, - DataFrameTransformCheckpointingInfoTests::randomDataFrameTransformCheckpointingInfo, - DataFrameTransformCheckpointingInfoTests::toXContent, - DataFrameTransformCheckpointingInfo::fromXContent) + DataFrameTransformCheckpointingInfoTests::randomDataFrameTransformCheckpointingInfo, + DataFrameTransformCheckpointingInfoTests::toXContent, + DataFrameTransformCheckpointingInfo::fromXContent) .supportsUnknownFields(false) .test(); } public static DataFrameTransformCheckpointingInfo randomDataFrameTransformCheckpointingInfo() { return new DataFrameTransformCheckpointingInfo( - DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(), - DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(), - randomLongBetween(0, 10000)); + DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(), + DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(), + randomLongBetween(0, 10000)); } public static void toXContent(DataFrameTransformCheckpointingInfo info, XContentBuilder builder) throws IOException { builder.startObject(); - if (info.getCurrent().getTimestampMillis() > 0) { - builder.field("current"); - DataFrameTransformCheckpointStatsTests.toXContent(info.getCurrent(), builder); + if (info.getLast().getTimestampMillis() > 0) { + builder.field(DataFrameTransformCheckpointingInfo.LAST_CHECKPOINT.getPreferredName()); + DataFrameTransformCheckpointStatsTests.toXContent(info.getLast(), builder); } - if (info.getInProgress().getTimestampMillis() > 0) { - builder.field("in_progress"); - DataFrameTransformCheckpointStatsTests.toXContent(info.getInProgress(), builder); + if (info.getNext().getTimestampMillis() > 0) { + builder.field(DataFrameTransformCheckpointingInfo.NEXT_CHECKPOINT.getPreferredName()); + DataFrameTransformCheckpointStatsTests.toXContent(info.getNext(), builder); } - builder.field("operations_behind", info.getOperationsBehind()); + builder.field(DataFrameTransformCheckpointingInfo.OPERATIONS_BEHIND.getPreferredName(), info.getOperationsBehind()); builder.endObject(); } - } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java new file mode 100644 index 0000000000000..73d71fe0e7690 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.dataframe.transforms; + +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class DataFrameTransformStateAndStatsInfoTests extends ESTestCase { + + public void testFromXContent() throws IOException { + xContentTester(this::createParser, + DataFrameTransformStateAndStatsInfoTests::randomInstance, + DataFrameTransformStateAndStatsInfoTests::toXContent, + DataFrameTransformStateAndStatsInfo::fromXContent) + .supportsUnknownFields(true) + .randomFieldsExcludeFilter(field -> field.equals("node.attributes") || field.contains("position")) + .test(); + } + + public static DataFrameTransformStateAndStatsInfo randomInstance() { + return new DataFrameTransformStateAndStatsInfo(randomAlphaOfLength(10), + randomBoolean() ? null : randomFrom(DataFrameTransformTaskState.values()), + randomBoolean() ? null : randomAlphaOfLength(100), + randomBoolean() ? null : NodeAttributesTests.createRandom(), + DataFrameIndexerTransformStatsTests.randomStats(), + randomBoolean() ? null : DataFrameTransformCheckpointingInfoTests.randomDataFrameTransformCheckpointingInfo()); + } + + public static void toXContent(DataFrameTransformStateAndStatsInfo stateAndStatsInfo, XContentBuilder builder) throws IOException { + builder.startObject(); + builder.field(DataFrameTransformStateAndStatsInfo.ID.getPreferredName(), stateAndStatsInfo.getId()); + if (stateAndStatsInfo.getTaskState() != null) { + builder.field(DataFrameTransformStateAndStatsInfo.TASK_STATE_FIELD.getPreferredName(), + stateAndStatsInfo.getTaskState().value()); + } + if (stateAndStatsInfo.getReason() != null) { + builder.field(DataFrameTransformStateAndStatsInfo.REASON_FIELD.getPreferredName(), stateAndStatsInfo.getReason()); + } + if (stateAndStatsInfo.getNode() != null) { + builder.field(DataFrameTransformStateAndStatsInfo.NODE_FIELD.getPreferredName()); + stateAndStatsInfo.getNode().toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.field(DataFrameTransformStateAndStatsInfo.STATS_FIELD.getPreferredName()); + DataFrameIndexerTransformStatsTests.toXContent(stateAndStatsInfo.getTransformStats(), builder); + if (stateAndStatsInfo.getCheckpointingInfo() != null) { + builder.field(DataFrameTransformStateAndStatsInfo.CHECKPOINTING_INFO_FIELD.getPreferredName()); + DataFrameTransformCheckpointingInfoTests.toXContent(stateAndStatsInfo.getCheckpointingInfo(), builder); + } + builder.endObject(); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsTests.java deleted file mode 100644 index 6ebdec5a69009..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsTests.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.dataframe.transforms; - -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; - -import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; - -public class DataFrameTransformStateAndStatsTests extends ESTestCase { - - public void testFromXContent() throws IOException { - xContentTester(this::createParser, - DataFrameTransformStateAndStatsTests::randomInstance, - DataFrameTransformStateAndStatsTests::toXContent, - DataFrameTransformStateAndStats::fromXContent) - .supportsUnknownFields(true) - .randomFieldsExcludeFilter(field -> field.startsWith("state")) - .test(); - } - - public static DataFrameTransformStateAndStats randomInstance() { - return new DataFrameTransformStateAndStats(randomAlphaOfLength(10), - DataFrameTransformStateTests.randomDataFrameTransformState(), - DataFrameIndexerTransformStatsTests.randomStats(), - DataFrameTransformCheckpointingInfoTests.randomDataFrameTransformCheckpointingInfo()); - } - - public static void toXContent(DataFrameTransformStateAndStats stateAndStats, XContentBuilder builder) throws IOException { - builder.startObject(); - builder.field(DataFrameTransformStateAndStats.ID.getPreferredName(), stateAndStats.getId()); - builder.field(DataFrameTransformStateAndStats.STATE_FIELD.getPreferredName()); - DataFrameTransformStateTests.toXContent(stateAndStats.getTransformState(), builder); - builder.field(DataFrameTransformStateAndStats.STATS_FIELD.getPreferredName()); - DataFrameIndexerTransformStatsTests.toXContent(stateAndStats.getTransformStats(), builder); - builder.field(DataFrameTransformStateAndStats.CHECKPOINTING_INFO_FIELD.getPreferredName()); - DataFrameTransformCheckpointingInfoTests.toXContent(stateAndStats.getCheckpointingInfo(), builder); - builder.endObject(); - } -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateTests.java deleted file mode 100644 index ef1cf3e89b6bf..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateTests.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.dataframe.transforms; - -import org.elasticsearch.client.core.IndexerState; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; - -import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; - -public class DataFrameTransformStateTests extends ESTestCase { - - public void testFromXContent() throws IOException { - xContentTester(this::createParser, - DataFrameTransformStateTests::randomDataFrameTransformState, - DataFrameTransformStateTests::toXContent, - DataFrameTransformState::fromXContent) - .supportsUnknownFields(true) - .randomFieldsExcludeFilter(field -> field.equals("position.indexer_position") || - field.equals("position.bucket_position") || - field.equals("node.attributes")) - .test(); - } - - public static DataFrameTransformState randomDataFrameTransformState() { - return new DataFrameTransformState(randomFrom(DataFrameTransformTaskState.values()), - randomFrom(IndexerState.values()), - randomBoolean() ? null : DataFrameIndexerPositionTests.randomDataFrameIndexerPosition(), - randomLongBetween(0,10), - randomBoolean() ? null : randomAlphaOfLength(10), - randomBoolean() ? null : DataFrameTransformProgressTests.randomInstance(), - randomBoolean() ? null : NodeAttributesTests.createRandom()); - } - - public static void toXContent(DataFrameTransformState state, XContentBuilder builder) throws IOException { - builder.startObject(); - builder.field("task_state", state.getTaskState().value()); - builder.field("indexer_state", state.getIndexerState().value()); - if (state.getPosition() != null) { - builder.field("position"); - DataFrameIndexerPositionTests.toXContent(state.getPosition(), builder); - } - builder.field("checkpoint", state.getCheckpoint()); - if (state.getReason() != null) { - builder.field("reason", state.getReason()); - } - if (state.getProgress() != null) { - builder.field("progress"); - DataFrameTransformProgressTests.toXContent(state.getProgress(), builder); - } - if (state.getNode() != null) { - builder.field("node"); - state.getNode().toXContent(builder, ToXContent.EMPTY_PARAMS); - } - builder.endObject(); - } - -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameIndexerPositionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameIndexerPositionTests.java index 9cf33e6500c72..bf0680a2dc247 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameIndexerPositionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameIndexerPositionTests.java @@ -40,9 +40,13 @@ public static DataFrameIndexerPosition fromHlrc( return new DataFrameIndexerPosition(instance.getIndexerPosition(), instance.getBucketsPosition()); } + public static DataFrameIndexerPosition randomDataFrameIndexerPosition() { + return new DataFrameIndexerPosition(randomPositionMap(), randomPositionMap()); + } + @Override protected DataFrameIndexerPosition createServerTestInstance() { - return new DataFrameIndexerPosition(randomPositionMap(), randomPositionMap()); + return randomDataFrameIndexerPosition(); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameIndexerTransformStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameIndexerTransformStatsTests.java index cb4b9e42838c9..a2626d357b499 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameIndexerTransformStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameIndexerTransformStatsTests.java @@ -48,9 +48,16 @@ public DataFrameIndexerTransformStats convertHlrcToInternal( return fromHlrc(instance); } + public static DataFrameIndexerTransformStats randomStats(String transformId) { + return new DataFrameIndexerTransformStats(transformId, randomLongBetween(10L, 10000L), + randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L)); + } + @Override protected DataFrameIndexerTransformStats createTestInstance() { - return DataFrameTransformStateTests.randomStats(DataFrameIndexerTransformStats.DEFAULT_TRANSFORM_ID); + return randomStats(DataFrameIndexerTransformStats.DEFAULT_TRANSFORM_ID); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointStatsTests.java index 9762aa4ab020b..0a41cfc85e903 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointStatsTests.java @@ -22,8 +22,10 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.client.AbstractHlrcXContentTestCase; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointStats; +import org.elasticsearch.xpack.core.indexing.IndexerState; import java.io.IOException; +import java.util.function.Predicate; public class DataFrameTransformCheckpointStatsTests extends AbstractHlrcXContentTestCase< DataFrameTransformCheckpointStats, @@ -31,7 +33,12 @@ public class DataFrameTransformCheckpointStatsTests extends AbstractHlrcXContent public static DataFrameTransformCheckpointStats fromHlrc( org.elasticsearch.client.dataframe.transforms.DataFrameTransformCheckpointStats instance) { - return new DataFrameTransformCheckpointStats(instance.getTimestampMillis(), instance.getTimeUpperBoundMillis()); + return new DataFrameTransformCheckpointStats(instance.getCheckpoint(), + (instance.getIndexerState() != null) ? IndexerState.fromString(instance.getIndexerState().value()) : null, + DataFrameIndexerPositionTests.fromHlrc(instance.getPosition()), + DataFrameTransformProgressTests.fromHlrc(instance.getCheckpointProgress()), + instance.getTimestampMillis(), + instance.getTimeUpperBoundMillis()); } @Override @@ -46,9 +53,17 @@ public DataFrameTransformCheckpointStats convertHlrcToInternal( return fromHlrc(instance); } + public static DataFrameTransformCheckpointStats randomDataFrameTransformCheckpointStats() { + return new DataFrameTransformCheckpointStats(randomLongBetween(1, 1_000_000), + randomBoolean() ? null : randomFrom(IndexerState.values()), + DataFrameIndexerPositionTests.randomDataFrameIndexerPosition(), + randomBoolean() ? null : DataFrameTransformProgressTests.randomDataFrameTransformProgress(), + randomLongBetween(1, 1_000_000), randomLongBetween(0, 1_000_000)); + } + @Override protected DataFrameTransformCheckpointStats createTestInstance() { - return DataFrameTransformStateTests.randomDataFrameTransformCheckpointStats(); + return DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(); } @Override @@ -61,4 +76,8 @@ protected boolean supportsUnknownFields() { return true; } + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> field.startsWith("position"); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java index a461d277c192e..84f096caf14e7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java @@ -32,14 +32,13 @@ public class DataFrameTransformCheckpointingInfoTests extends AbstractHlrcXConte public static DataFrameTransformCheckpointingInfo fromHlrc( org.elasticsearch.client.dataframe.transforms.DataFrameTransformCheckpointingInfo instance) { return new DataFrameTransformCheckpointingInfo( - DataFrameTransformCheckpointStatsTests.fromHlrc(instance.getCurrent()), - DataFrameTransformCheckpointStatsTests.fromHlrc(instance.getInProgress()), + DataFrameTransformCheckpointStatsTests.fromHlrc(instance.getLast()), + DataFrameTransformCheckpointStatsTests.fromHlrc(instance.getNext()), instance.getOperationsBehind()); } @Override - public org.elasticsearch.client.dataframe.transforms.DataFrameTransformCheckpointingInfo doHlrcParseInstance(XContentParser parser) - throws IOException { + public org.elasticsearch.client.dataframe.transforms.DataFrameTransformCheckpointingInfo doHlrcParseInstance(XContentParser parser) { return org.elasticsearch.client.dataframe.transforms.DataFrameTransformCheckpointingInfo.fromXContent(parser); } @@ -49,9 +48,14 @@ public DataFrameTransformCheckpointingInfo convertHlrcToInternal( return fromHlrc(instance); } + public static DataFrameTransformCheckpointingInfo randomDataFrameTransformCheckpointingInfo() { + return new DataFrameTransformCheckpointingInfo(DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(), + DataFrameTransformCheckpointStatsTests.randomDataFrameTransformCheckpointStats(), randomNonNegativeLong()); + } + @Override protected DataFrameTransformCheckpointingInfo createTestInstance() { - return DataFrameTransformStateTests.randomDataFrameTransformCheckpointingInfo(); + return randomDataFrameTransformCheckpointingInfo(); } @Override @@ -63,5 +67,4 @@ protected DataFrameTransformCheckpointingInfo doParseInstance(XContentParser par protected boolean supportsUnknownFields() { return true; } - } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformProgressTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformProgressTests.java index be589a63248b8..a9dd1eaecab07 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformProgressTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformProgressTests.java @@ -37,9 +37,14 @@ public static DataFrameTransformProgress fromHlrc( return new DataFrameTransformProgress(instance.getTotalDocs(), instance.getRemainingDocs()); } + public static DataFrameTransformProgress randomDataFrameTransformProgress() { + long totalDocs = randomNonNegativeLong(); + return new DataFrameTransformProgress(totalDocs, randomBoolean() ? null : randomLongBetween(0, totalDocs)); + } + @Override protected DataFrameTransformProgress createServerTestInstance() { - return DataFrameTransformStateTests.randomDataFrameTransformProgress(); + return randomDataFrameTransformProgress(); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsInfoTests.java similarity index 57% rename from client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateTests.java rename to client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsInfoTests.java index 6d378bca5f818..8ef5bba8c86d6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsInfoTests.java @@ -21,13 +21,11 @@ import org.elasticsearch.client.AbstractHlrcXContentTestCase; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerPosition; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.xpack.core.dataframe.transforms.NodeAttributes; import org.elasticsearch.xpack.core.indexing.IndexerState; @@ -37,18 +35,8 @@ import java.util.Map; import java.util.function.Predicate; -public class DataFrameTransformStateTests extends AbstractHlrcXContentTestCase { - - public static DataFrameTransformState fromHlrc(org.elasticsearch.client.dataframe.transforms.DataFrameTransformState instance) { - return new DataFrameTransformState(DataFrameTransformTaskState.fromString(instance.getTaskState().value()), - IndexerState.fromString(instance.getIndexerState().value()), - DataFrameIndexerPositionTests.fromHlrc(instance.getPosition()), - instance.getCheckpoint(), - instance.getReason(), - DataFrameTransformProgressTests.fromHlrc(instance.getProgress()), - fromHlrc(instance.getNode())); - } +public class DataFrameTransformStateAndStatsInfoTests extends AbstractHlrcXContentTestCase { public static NodeAttributes fromHlrc(org.elasticsearch.client.dataframe.transforms.NodeAttributes attributes) { return attributes == null ? null : new NodeAttributes(attributes.getId(), @@ -58,48 +46,65 @@ public static NodeAttributes fromHlrc(org.elasticsearch.client.dataframe.transfo attributes.getAttributes()); } - @Override - public org.elasticsearch.client.dataframe.transforms.DataFrameTransformState doHlrcParseInstance(XContentParser parser) - throws IOException { - return org.elasticsearch.client.dataframe.transforms.DataFrameTransformState.fromXContent(parser); + public static DataFrameTransformStateAndStatsInfo + fromHlrc(org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo instance) { + + return new DataFrameTransformStateAndStatsInfo(instance.getId(), + DataFrameTransformTaskState.fromString(instance.getTaskState().value()), + instance.getReason(), + fromHlrc(instance.getNode()), + DataFrameIndexerTransformStatsTests.fromHlrc(instance.getTransformStats()), + DataFrameTransformCheckpointingInfoTests.fromHlrc(instance.getCheckpointingInfo())); } @Override - public DataFrameTransformState convertHlrcToInternal(org.elasticsearch.client.dataframe.transforms.DataFrameTransformState instance) { - return fromHlrc(instance); + public org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo doHlrcParseInstance(XContentParser parser) + throws IOException { + return org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo.fromXContent(parser); } @Override - protected DataFrameTransformState createTestInstance() { - return randomDataFrameTransformState(); + public DataFrameTransformStateAndStatsInfo convertHlrcToInternal( + org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo instance) { + return new DataFrameTransformStateAndStatsInfo(instance.getId(), + DataFrameTransformTaskState.fromString(instance.getTaskState().value()), + instance.getReason(), + fromHlrc(instance.getNode()), + DataFrameIndexerTransformStatsTests.fromHlrc(instance.getTransformStats()), + DataFrameTransformCheckpointingInfoTests.fromHlrc(instance.getCheckpointingInfo())); + } + + public static DataFrameTransformStateAndStatsInfo randomDataFrameTransformStateAndStatsInfo() { + return new DataFrameTransformStateAndStatsInfo(randomAlphaOfLength(10), + randomFrom(DataFrameTransformTaskState.values()), + randomBoolean() ? null : randomAlphaOfLength(100), + randomBoolean() ? null : randomNodeAttributes(), + // On the server side the stats has transform ID "_all" when embedded in another doc + // TODO: change this so that the outer document sets the correct transform ID during parsing + // It's very confusing and could cause subtle errors that the inner object has a surprising ID + randomStats("_all"), + DataFrameTransformCheckpointingInfoTests.randomDataFrameTransformCheckpointingInfo()); } @Override - protected DataFrameTransformState doParseInstance(XContentParser parser) throws IOException { - return DataFrameTransformState.fromXContent(parser); + protected DataFrameTransformStateAndStatsInfo createTestInstance() { + return randomDataFrameTransformStateAndStatsInfo(); } @Override - protected boolean supportsUnknownFields() { - return true; + protected DataFrameTransformStateAndStatsInfo doParseInstance(XContentParser parser) throws IOException { + return DataFrameTransformStateAndStatsInfo.PARSER.apply(parser, null); } @Override protected Predicate getRandomFieldsExcludeFilter() { - return field -> field.equals("position.indexer_position") || - field.equals("position.bucket_position") || - field.equals("node.attributes"); - } - - public static DataFrameTransformStateAndStats randomDataFrameTransformStateAndStats(String id) { - return new DataFrameTransformStateAndStats(id, - randomDataFrameTransformState(), - randomStats(id), - randomDataFrameTransformCheckpointingInfo()); + return field -> field.contains("position") || field.equals("node.attributes"); } - public static DataFrameIndexerPosition randomDataFrameIndexerPosition() { - return new DataFrameIndexerPosition(randomPosition(), randomPosition()); + public static DataFrameTransformProgress randomDataFrameTransformProgress() { + long totalDocs = randomNonNegativeLong(); + Long remainingDocs = randomBoolean() ? null : randomLongBetween(0, totalDocs); + return new DataFrameTransformProgress(totalDocs, remainingDocs); } public static DataFrameTransformCheckpointingInfo randomDataFrameTransformCheckpointingInfo() { @@ -108,13 +113,11 @@ public static DataFrameTransformCheckpointingInfo randomDataFrameTransformCheckp } public static DataFrameTransformCheckpointStats randomDataFrameTransformCheckpointStats() { - return new DataFrameTransformCheckpointStats(randomNonNegativeLong(), randomNonNegativeLong()); - } - - public static DataFrameTransformProgress randomDataFrameTransformProgress() { - long totalDocs = randomNonNegativeLong(); - Long remainingDocs = randomBoolean() ? null : randomLongBetween(0, totalDocs); - return new DataFrameTransformProgress(totalDocs, remainingDocs); + return new DataFrameTransformCheckpointStats(randomLongBetween(1, 1_000_000), + randomBoolean() ? null : randomFrom(IndexerState.values()), + DataFrameIndexerPositionTests.randomDataFrameIndexerPosition(), + randomBoolean() ? null : DataFrameTransformProgressTests.randomDataFrameTransformProgress(), + randomLongBetween(1, 1_000_000), randomLongBetween(0, 1_000_000)); } public static NodeAttributes randomNodeAttributes() { @@ -138,31 +141,14 @@ public static DataFrameIndexerTransformStats randomStats(String transformId) { randomLongBetween(0L, 10000L)); } - public static DataFrameTransformState randomDataFrameTransformState() { - return new DataFrameTransformState(randomFrom(DataFrameTransformTaskState.values()), - randomFrom(IndexerState.values()), - randomDataFrameIndexerPosition(), - randomLongBetween(0,10), - randomBoolean() ? null : randomAlphaOfLength(10), - randomBoolean() ? null : randomDataFrameTransformProgress(), - randomBoolean() ? null : randomNodeAttributes()); + @Override + protected boolean supportsUnknownFields() { + return true; } - private static Map randomPosition() { - if (randomBoolean()) { - return null; - } - int numFields = randomIntBetween(1, 5); - Map position = new HashMap<>(); - for (int i = 0; i < numFields; i++) { - Object value; - if (randomBoolean()) { - value = randomLong(); - } else { - value = randomAlphaOfLengthBetween(1, 10); - } - position.put(randomAlphaOfLengthBetween(3, 10), value); - } - return position; + @Override + protected String[] getShuffleFieldsExceptions() { + return new String[] { "position" }; } } + diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsTests.java deleted file mode 100644 index 7b54ab538c30d..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsTests.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.dataframe.transforms.hlrc; - -import org.elasticsearch.client.AbstractHlrcXContentTestCase; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; - -import java.io.IOException; -import java.util.function.Predicate; - -public class DataFrameTransformStateAndStatsTests extends AbstractHlrcXContentTestCase { - - @Override - public org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats doHlrcParseInstance(XContentParser parser) - throws IOException { - return org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats.fromXContent(parser); - } - - @Override - public DataFrameTransformStateAndStats convertHlrcToInternal( - org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats instance) { - return new DataFrameTransformStateAndStats(instance.getId(), - DataFrameTransformStateTests.fromHlrc(instance.getTransformState()), - DataFrameIndexerTransformStatsTests.fromHlrc(instance.getTransformStats()), - DataFrameTransformCheckpointingInfoTests.fromHlrc(instance.getCheckpointingInfo())); - } - - @Override - protected DataFrameTransformStateAndStats createTestInstance() { - // the transform id is not part of HLRC as it's only to a field for internal storage, therefore use a default id - return DataFrameTransformStateTests - .randomDataFrameTransformStateAndStats(DataFrameIndexerTransformStats.DEFAULT_TRANSFORM_ID); - } - - @Override - protected DataFrameTransformStateAndStats doParseInstance(XContentParser parser) throws IOException { - return DataFrameTransformStateAndStats.PARSER.apply(parser, null); - } - - @Override - protected boolean supportsUnknownFields() { - return true; - } - - @Override - protected Predicate getRandomFieldsExcludeFilter() { - return field -> field.equals("state.position.indexer_position") || - field.equals("state.position.bucket_position") || - field.equals("state.node") || - field.equals("state.node.attributes"); - } -} - diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java index 048f383ce3a4f..5886c639a9f9f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java @@ -42,7 +42,7 @@ import org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformProgress; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.client.dataframe.transforms.DestConfig; import org.elasticsearch.client.dataframe.transforms.NodeAttributes; @@ -525,18 +525,20 @@ public void testGetStats() throws IOException, InterruptedException { assertThat(response.getTransformsStateAndStats(), hasSize(1)); // tag::get-data-frame-transform-stats-response - DataFrameTransformStateAndStats stateAndStats = - response.getTransformsStateAndStats().get(0); // <1> + DataFrameTransformStateAndStatsInfo stateAndStatsInfo = + response.getTransformsStateAndStats().get(0); // <1> DataFrameTransformTaskState taskState = - stateAndStats.getTransformState().getTaskState(); // <2> + stateAndStatsInfo.getTaskState(); // <2> IndexerState indexerState = - stateAndStats.getTransformState().getIndexerState(); // <3> + stateAndStatsInfo.getCheckpointingInfo() + .getNext().getIndexerState(); // <3> DataFrameIndexerTransformStats transformStats = - stateAndStats.getTransformStats(); // <4> + stateAndStatsInfo.getTransformStats(); // <4> DataFrameTransformProgress progress = - stateAndStats.getTransformState().getProgress(); // <5> + stateAndStatsInfo.getCheckpointingInfo() + .getNext().getCheckpointProgress(); // <5> NodeAttributes node = - stateAndStats.getTransformState().getNode(); // <6> + stateAndStatsInfo.getNode(); // <6> // end::get-data-frame-transform-stats-response assertEquals(IndexerState.STOPPED, indexerState); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java index 9d5db1e5022e3..d94bde3e57be1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java @@ -39,6 +39,10 @@ public final class DataFrameField { * Fields for checkpointing */ // the timestamp of the checkpoint, mandatory + public static final ParseField CHECKPOINT = new ParseField("checkpoint"); + public static final ParseField INDEXER_STATE = new ParseField("indexer_state"); + public static final ParseField POSITION = new ParseField("position"); + public static final ParseField CHECKPOINT_PROGRESS = new ParseField("checkpoint_progress"); public static final ParseField TIMESTAMP_MILLIS = new ParseField("timestamp_millis"); public static final ParseField TIMESTAMP = new ParseField("timestamp"); // checkpoint for for time based sync diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java index 8c4438e09db2c..0526ba09498ff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.action.util.PageParams; import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.dataframe.DataFrameField; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfo; import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper; import java.io.IOException; @@ -152,51 +152,51 @@ public boolean equals(Object obj) { } public static class Response extends BaseTasksResponse implements ToXContentObject { - private final QueryPage transformsStateAndStats; + private final QueryPage transformsStateAndStatsInfo; - public Response(List transformStateAndStats, long count) { + public Response(List transformStateAndStats, long count) { this(new QueryPage<>(transformStateAndStats, count, DataFrameField.TRANSFORMS)); } - public Response(List transformStateAndStats, + public Response(List transformStateAndStats, long count, List taskFailures, List nodeFailures) { this(new QueryPage<>(transformStateAndStats, count, DataFrameField.TRANSFORMS), taskFailures, nodeFailures); } - private Response(QueryPage transformsStateAndStats) { - this(transformsStateAndStats, Collections.emptyList(), Collections.emptyList()); + private Response(QueryPage transformsStateAndStatsInfo) { + this(transformsStateAndStatsInfo, Collections.emptyList(), Collections.emptyList()); } - private Response(QueryPage transformsStateAndStats, + private Response(QueryPage transformsStateAndStatsInfo, List taskFailures, List nodeFailures) { super(taskFailures, nodeFailures); - this.transformsStateAndStats = ExceptionsHelper.requireNonNull(transformsStateAndStats, "transformsStateAndStats"); + this.transformsStateAndStatsInfo = ExceptionsHelper.requireNonNull(transformsStateAndStatsInfo, "transformsStateAndStatsInfo"); } public Response(StreamInput in) throws IOException { super(in); if (in.getVersion().onOrAfter(Version.V_7_3_0)) { - transformsStateAndStats = new QueryPage<>(in, DataFrameTransformStateAndStats::new); + transformsStateAndStatsInfo = new QueryPage<>(in, DataFrameTransformStateAndStatsInfo::new); } else { - List stats = in.readList(DataFrameTransformStateAndStats::new); - transformsStateAndStats = new QueryPage<>(stats, stats.size(), DataFrameField.TRANSFORMS); + List stats = in.readList(DataFrameTransformStateAndStatsInfo::new); + transformsStateAndStatsInfo = new QueryPage<>(stats, stats.size(), DataFrameField.TRANSFORMS); } } - public List getTransformsStateAndStats() { - return transformsStateAndStats.results(); + public List getTransformsStateAndStatsInfo() { + return transformsStateAndStatsInfo.results(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); if (out.getVersion().onOrAfter(Version.V_7_3_0)) { - transformsStateAndStats.writeTo(out); + transformsStateAndStatsInfo.writeTo(out); } else { - out.writeList(transformsStateAndStats.results()); + out.writeList(transformsStateAndStatsInfo.results()); } } @@ -209,14 +209,14 @@ public void readFrom(StreamInput in) { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); toXContentCommon(builder, params); - transformsStateAndStats.doXContentBody(builder, params); + transformsStateAndStatsInfo.doXContentBody(builder, params); builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(transformsStateAndStats); + return Objects.hash(transformsStateAndStatsInfo); } @Override @@ -230,7 +230,7 @@ public boolean equals(Object other) { } final Response that = (Response) other; - return Objects.equals(this.transformsStateAndStats, that.transformsStateAndStats); + return Objects.equals(this.transformsStateAndStatsInfo, that.transformsStateAndStatsInfo); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java index 4dd4b68905fef..439e380d3272c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java @@ -6,18 +6,23 @@ package org.elasticsearch.xpack.core.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.dataframe.DataFrameField; +import org.elasticsearch.xpack.core.indexing.IndexerState; import java.io.IOException; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + /** * Checkpoint stats data for 1 checkpoint * @@ -25,34 +30,92 @@ */ public class DataFrameTransformCheckpointStats implements Writeable, ToXContentObject { - public static DataFrameTransformCheckpointStats EMPTY = new DataFrameTransformCheckpointStats(0L, 0L); + public static final DataFrameTransformCheckpointStats EMPTY = new DataFrameTransformCheckpointStats(0L, null, null, null, 0L, 0L); + private final long checkpoint; + private final IndexerState indexerState; + private final DataFrameIndexerPosition position; + private final DataFrameTransformProgress checkpointProgress; private final long timestampMillis; private final long timeUpperBoundMillis; - private static final ConstructingObjectParser LENIENT_PARSER = new ConstructingObjectParser<>( + static final ConstructingObjectParser LENIENT_PARSER = new ConstructingObjectParser<>( "data_frame_transform_checkpoint_stats", true, args -> { - long timestamp = args[0] == null ? 0L : (Long) args[0]; - long timeUpperBound = args[1] == null ? 0L : (Long) args[1]; + long checkpoint = args[0] == null ? 0L : (Long) args[0]; + IndexerState indexerState = (IndexerState) args[1]; + DataFrameIndexerPosition position = (DataFrameIndexerPosition) args[2]; + DataFrameTransformProgress checkpointProgress = (DataFrameTransformProgress) args[3]; + long timestamp = args[4] == null ? 0L : (Long) args[4]; + long timeUpperBound = args[5] == null ? 0L : (Long) args[5]; - return new DataFrameTransformCheckpointStats(timestamp, timeUpperBound); - }); + return new DataFrameTransformCheckpointStats(checkpoint, indexerState, position, checkpointProgress, timestamp, timeUpperBound); + }); static { - LENIENT_PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), DataFrameField.TIMESTAMP_MILLIS); - LENIENT_PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), DataFrameField.TIME_UPPER_BOUND_MILLIS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DataFrameField.CHECKPOINT); + LENIENT_PARSER.declareField(optionalConstructorArg(), p -> IndexerState.fromString(p.text()), DataFrameField.INDEXER_STATE, + ObjectParser.ValueType.STRING); + LENIENT_PARSER.declareObject(optionalConstructorArg(), DataFrameIndexerPosition.PARSER, DataFrameField.POSITION); + LENIENT_PARSER.declareObject(optionalConstructorArg(), DataFrameTransformProgress.PARSER, DataFrameField.CHECKPOINT_PROGRESS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DataFrameField.TIMESTAMP_MILLIS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DataFrameField.TIME_UPPER_BOUND_MILLIS); } - public DataFrameTransformCheckpointStats(final long timestampMillis, final long timeUpperBoundMillis) { + public DataFrameTransformCheckpointStats(final long checkpoint, final IndexerState indexerState, + final DataFrameIndexerPosition position, final DataFrameTransformProgress checkpointProgress, + final long timestampMillis, final long timeUpperBoundMillis) { + this.checkpoint = checkpoint; + this.indexerState = indexerState; + this.position = position; + this.checkpointProgress = checkpointProgress; this.timestampMillis = timestampMillis; this.timeUpperBoundMillis = timeUpperBoundMillis; } public DataFrameTransformCheckpointStats(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + this.checkpoint = in.readVLong(); + if (in.readBoolean()) { + this.indexerState = in.readEnum(IndexerState.class); + } else { + this.indexerState = null; + } + if (in.readBoolean()) { + this.position = new DataFrameIndexerPosition(in); + } else { + this.position = null; + } + if (in.readBoolean()) { + this.checkpointProgress = new DataFrameTransformProgress(in); + } else { + this.checkpointProgress = null; + } + } else { + this.checkpoint = 0; + this.indexerState = null; + this.position = null; + this.checkpointProgress = null; + } this.timestampMillis = in.readLong(); this.timeUpperBoundMillis = in.readLong(); } + public long getCheckpoint() { + return checkpoint; + } + + public IndexerState getIndexerState() { + return indexerState; + } + + public DataFrameIndexerPosition getPosition() { + return position; + } + + public DataFrameTransformProgress getCheckpointProgress() { + return checkpointProgress; + } + public long getTimestampMillis() { return timestampMillis; } @@ -64,6 +127,16 @@ public long getTimeUpperBoundMillis() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + builder.field(DataFrameField.CHECKPOINT.getPreferredName(), checkpoint); + if (indexerState != null) { + builder.field(DataFrameField.INDEXER_STATE.getPreferredName(), indexerState.value()); + } + if (position != null) { + builder.field(DataFrameField.POSITION.getPreferredName(), position); + } + if (checkpointProgress != null) { + builder.field(DataFrameField.CHECKPOINT_PROGRESS.getPreferredName(), checkpointProgress); + } builder.timeField(DataFrameField.TIMESTAMP_MILLIS.getPreferredName(), DataFrameField.TIMESTAMP.getPreferredName(), getTimestampMillis()); if (timeUpperBoundMillis > 0) { @@ -76,13 +149,34 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + out.writeVLong(checkpoint); + if (indexerState != null) { + out.writeBoolean(true); + out.writeEnum(indexerState); + } else { + out.writeBoolean(false); + } + if (position != null) { + out.writeBoolean(true); + position.writeTo(out); + } else { + out.writeBoolean(false); + } + if (checkpointProgress != null) { + out.writeBoolean(true); + checkpointProgress.writeTo(out); + } else { + out.writeBoolean(false); + } + } out.writeLong(timestampMillis); out.writeLong(timeUpperBoundMillis); } @Override public int hashCode() { - return Objects.hash(timestampMillis, timeUpperBoundMillis); + return Objects.hash(checkpoint, indexerState, position, checkpointProgress, timestampMillis, timeUpperBoundMillis); } @Override @@ -97,12 +191,15 @@ public boolean equals(Object other) { DataFrameTransformCheckpointStats that = (DataFrameTransformCheckpointStats) other; - return this.timestampMillis == that.timestampMillis && - this.timeUpperBoundMillis == that.timeUpperBoundMillis; + return this.checkpoint == that.checkpoint + && Objects.equals(this.indexerState, that.indexerState) + && Objects.equals(this.position, that.position) + && Objects.equals(this.checkpointProgress, that.checkpointProgress) + && this.timestampMillis == that.timestampMillis + && this.timeUpperBoundMillis == that.timeUpperBoundMillis; } public static DataFrameTransformCheckpointStats fromXContent(XContentParser p) { return LENIENT_PARSER.apply(p, null); } - -} \ No newline at end of file +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java index 1b66be41add09..b6c1357691c0b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java @@ -26,17 +26,17 @@ */ public class DataFrameTransformCheckpointingInfo implements Writeable, ToXContentObject { - public static DataFrameTransformCheckpointingInfo EMPTY = new DataFrameTransformCheckpointingInfo( + public static final DataFrameTransformCheckpointingInfo EMPTY = new DataFrameTransformCheckpointingInfo( DataFrameTransformCheckpointStats.EMPTY, DataFrameTransformCheckpointStats.EMPTY, 0L); - public static final ParseField CURRENT_CHECKPOINT = new ParseField("current"); - public static final ParseField IN_PROGRESS_CHECKPOINT = new ParseField("in_progress"); + public static final ParseField LAST_CHECKPOINT = new ParseField("last"); + public static final ParseField NEXT_CHECKPOINT = new ParseField("next"); public static final ParseField OPERATIONS_BEHIND = new ParseField("operations_behind"); - private final DataFrameTransformCheckpointStats current; - private final DataFrameTransformCheckpointStats inProgress; + private final DataFrameTransformCheckpointStats last; + private final DataFrameTransformCheckpointStats next; private final long operationsBehind; private static final ConstructingObjectParser LENIENT_PARSER = @@ -51,39 +51,39 @@ public class DataFrameTransformCheckpointingInfo implements Writeable, ToXConten static { LENIENT_PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> DataFrameTransformCheckpointStats.fromXContent(p), CURRENT_CHECKPOINT); + DataFrameTransformCheckpointStats.LENIENT_PARSER::apply, LAST_CHECKPOINT); LENIENT_PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> DataFrameTransformCheckpointStats.fromXContent(p), IN_PROGRESS_CHECKPOINT); + DataFrameTransformCheckpointStats.LENIENT_PARSER::apply, NEXT_CHECKPOINT); LENIENT_PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), OPERATIONS_BEHIND); } /** - * Create checkpoint stats object with checkpoint information about the current and in progress checkpoint as well as the current state + * Create checkpoint stats object with checkpoint information about the last and next checkpoint as well as the current state * of source. * - * @param current stats of the current checkpoint - * @param inProgress stats of the in progress checkpoint + * @param last stats of the last checkpoint + * @param next stats of the next checkpoint * @param operationsBehind counter of operations the current checkpoint is behind source */ - public DataFrameTransformCheckpointingInfo(DataFrameTransformCheckpointStats current, DataFrameTransformCheckpointStats inProgress, + public DataFrameTransformCheckpointingInfo(DataFrameTransformCheckpointStats last, DataFrameTransformCheckpointStats next, long operationsBehind) { - this.current = Objects.requireNonNull(current); - this.inProgress = Objects.requireNonNull(inProgress); + this.last = Objects.requireNonNull(last); + this.next = Objects.requireNonNull(next); this.operationsBehind = operationsBehind; } public DataFrameTransformCheckpointingInfo(StreamInput in) throws IOException { - current = new DataFrameTransformCheckpointStats(in); - inProgress = new DataFrameTransformCheckpointStats(in); + last = new DataFrameTransformCheckpointStats(in); + next = new DataFrameTransformCheckpointStats(in); operationsBehind = in.readLong(); } - public DataFrameTransformCheckpointStats getCurrent() { - return current; + public DataFrameTransformCheckpointStats getLast() { + return last; } - public DataFrameTransformCheckpointStats getInProgress() { - return inProgress; + public DataFrameTransformCheckpointStats getNext() { + return next; } public long getOperationsBehind() { @@ -93,11 +93,11 @@ public long getOperationsBehind() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (current.getTimestampMillis() > 0) { - builder.field(CURRENT_CHECKPOINT.getPreferredName(), current); + if (last.getTimestampMillis() > 0) { + builder.field(LAST_CHECKPOINT.getPreferredName(), last); } - if (inProgress.getTimestampMillis() > 0) { - builder.field(IN_PROGRESS_CHECKPOINT.getPreferredName(), inProgress); + if (next.getTimestampMillis() > 0) { + builder.field(NEXT_CHECKPOINT.getPreferredName(), next); } builder.field(OPERATIONS_BEHIND.getPreferredName(), operationsBehind); @@ -107,8 +107,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { - current.writeTo(out); - inProgress.writeTo(out); + last.writeTo(out); + next.writeTo(out); out.writeLong(operationsBehind); } @@ -118,7 +118,7 @@ public static DataFrameTransformCheckpointingInfo fromXContent(XContentParser p) @Override public int hashCode() { - return Objects.hash(current, inProgress, operationsBehind); + return Objects.hash(last, next, operationsBehind); } @Override @@ -133,8 +133,8 @@ public boolean equals(Object other) { DataFrameTransformCheckpointingInfo that = (DataFrameTransformCheckpointingInfo) other; - return Objects.equals(this.current, that.current) && - Objects.equals(this.inProgress, that.inProgress) && + return Objects.equals(this.last, that.last) && + Objects.equals(this.next, that.next) && this.operationsBehind == that.operationsBehind; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java index d28d64bdb1e82..5534c0df27bf4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.core.dataframe.transforms; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -17,11 +16,14 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.dataframe.DataFrameField; -import org.elasticsearch.xpack.core.indexing.IndexerState; import java.io.IOException; import java.util.Objects; +/** + * A wrapper for grouping transform state and stats when persisting to an index. + * Not intended to be returned in endpoint responses. + */ public class DataFrameTransformStateAndStats implements Writeable, ToXContentObject { public static final String NAME = "data_frame_transform_state_and_stats"; @@ -53,17 +55,6 @@ public static DataFrameTransformStateAndStats fromXContent(XContentParser parser return PARSER.parse(parser, null); } - public static DataFrameTransformStateAndStats initialStateAndStats(String id) { - return initialStateAndStats(id, new DataFrameIndexerTransformStats(id)); - } - - public static DataFrameTransformStateAndStats initialStateAndStats(String id, DataFrameIndexerTransformStats indexerTransformStats) { - return new DataFrameTransformStateAndStats(id, - new DataFrameTransformState(DataFrameTransformTaskState.STOPPED, IndexerState.STOPPED, null, 0L, null, null), - indexerTransformStats, - DataFrameTransformCheckpointingInfo.EMPTY); - } - /** * Get the persisted state and stats document name from the Data Frame Transform Id. * @@ -88,11 +79,6 @@ public DataFrameTransformStateAndStats(StreamInput in) throws IOException { this.checkpointingInfo = new DataFrameTransformCheckpointingInfo(in); } - @Nullable - public String getTransformId() { - return transformStats.getTransformId(); - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -100,9 +86,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(STATE_FIELD.getPreferredName(), transformState, params); builder.field(DataFrameField.STATS_FIELD.getPreferredName(), transformStats, params); builder.field(CHECKPOINTING_INFO_FIELD.getPreferredName(), checkpointingInfo, params); - if (params.paramAsBoolean(DataFrameField.FOR_INTERNAL_STORAGE, false)) { - builder.field(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), NAME); - } + builder.field(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), NAME); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java new file mode 100644 index 0000000000000..4ae17125aeabd --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java @@ -0,0 +1,227 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.dataframe.transforms; + +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Used as a wrapper for the objects returned from the stats endpoint. + * Objects of this class are expected to be ephermeral. + * Do not persist objects of this class to cluster state or an index. + */ +public class DataFrameTransformStateAndStatsInfo implements Writeable, ToXContentObject { + + public static final String NAME = "data_frame_transform_state_and_stats_info"; + public static final ParseField TASK_STATE_FIELD = new ParseField("task_state"); + public static final ParseField REASON_FIELD = new ParseField("reason"); + public static final ParseField NODE_FIELD = new ParseField("node"); + public static final ParseField CHECKPOINTING_INFO_FIELD = new ParseField("checkpointing"); + + private final String id; + private final DataFrameTransformTaskState taskState; + @Nullable + private String reason; + @Nullable + private NodeAttributes node; + private final DataFrameIndexerTransformStats transformStats; + private final DataFrameTransformCheckpointingInfo checkpointingInfo; + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + NAME, + true, + a -> new DataFrameTransformStateAndStatsInfo((String) a[0], + (DataFrameTransformTaskState) a[1], + (String) a[2], + (NodeAttributes) a[3], + (DataFrameIndexerTransformStats) a[4], + (DataFrameTransformCheckpointingInfo) a[5])); + + static { + PARSER.declareString(constructorArg(), DataFrameField.ID); + PARSER.declareField(constructorArg(), p -> DataFrameTransformTaskState.fromString(p.text()), TASK_STATE_FIELD, + ObjectParser.ValueType.STRING); + PARSER.declareString(optionalConstructorArg(), REASON_FIELD); + PARSER.declareField(optionalConstructorArg(), NodeAttributes.PARSER::apply, NODE_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareObject(constructorArg(), (p, c) -> DataFrameIndexerTransformStats.fromXContent(p), + DataFrameField.STATS_FIELD); + PARSER.declareObject(constructorArg(), + (p, c) -> DataFrameTransformCheckpointingInfo.fromXContent(p), CHECKPOINTING_INFO_FIELD); + } + + public static DataFrameTransformStateAndStatsInfo fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + public static DataFrameTransformStateAndStatsInfo initialStateAndStatsInfo(String id) { + return stoppedStateAndStatsInfo(id, new DataFrameIndexerTransformStats(id)); + } + + public static DataFrameTransformStateAndStatsInfo stoppedStateAndStatsInfo(String id, + DataFrameIndexerTransformStats indexerTransformStats) { + return new DataFrameTransformStateAndStatsInfo(id, + DataFrameTransformTaskState.STOPPED, + null, + null, + indexerTransformStats, + DataFrameTransformCheckpointingInfo.EMPTY); + } + + + public DataFrameTransformStateAndStatsInfo(String id, DataFrameTransformTaskState taskState, @Nullable String reason, + @Nullable NodeAttributes node, DataFrameIndexerTransformStats stats, + DataFrameTransformCheckpointingInfo checkpointingInfo) { + this.id = Objects.requireNonNull(id); + this.taskState = Objects.requireNonNull(taskState); + this.reason = reason; + this.node = node; + this.transformStats = Objects.requireNonNull(stats); + this.checkpointingInfo = Objects.requireNonNull(checkpointingInfo); + } + + public DataFrameTransformStateAndStatsInfo(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + this.id = in.readString(); + this.taskState = in.readEnum(DataFrameTransformTaskState.class); + if (in.readBoolean()) { + this.node = new NodeAttributes(in); + } else { + this.node = null; + } + this.transformStats = new DataFrameIndexerTransformStats(in); + this.checkpointingInfo = new DataFrameTransformCheckpointingInfo(in); + + } else { + // In 7.2 DataFrameTransformStateAndStatsInfo didn't exist, and we have to do + // the best we can of reading from a DataFrameTransformStateAndStats object + this.id = in.readString(); + this.taskState = new DataFrameTransformState(in).getTaskState(); + this.node = null; + this.transformStats = new DataFrameIndexerTransformStats(in); + this.checkpointingInfo = new DataFrameTransformCheckpointingInfo(in); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DataFrameField.ID.getPreferredName(), id); + builder.field(TASK_STATE_FIELD.getPreferredName(), taskState); + if (node != null) { + builder.field(NODE_FIELD.getPreferredName(), node); + } + builder.field(DataFrameField.STATS_FIELD.getPreferredName(), transformStats, params); + builder.field(CHECKPOINTING_INFO_FIELD.getPreferredName(), checkpointingInfo, params); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + out.writeString(id); + out.writeEnum(taskState); + if (node != null) { + out.writeBoolean(true); + node.writeTo(out); + } else { + out.writeBoolean(false); + } + transformStats.writeTo(out); + checkpointingInfo.writeTo(out); + } else { + // In 7.2 DataFrameTransformStateAndStatsInfo didn't exist, and we have to do + // the best we can of writing to a DataFrameTransformStateAndStats object + out.writeString(id); + new DataFrameTransformState(taskState, + checkpointingInfo.getNext().getIndexerState(), + checkpointingInfo.getNext().getPosition(), + checkpointingInfo.getLast().getCheckpoint(), + reason, + checkpointingInfo.getNext().getCheckpointProgress()).writeTo(out); + out.writeBoolean(false); + transformStats.writeTo(out); + checkpointingInfo.writeTo(out); + } + } + + @Override + public int hashCode() { + return Objects.hash(id, taskState, node, transformStats, checkpointingInfo); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + DataFrameTransformStateAndStatsInfo that = (DataFrameTransformStateAndStatsInfo) other; + + return Objects.equals(this.id, that.id) + && Objects.equals(this.taskState, that.taskState) + && Objects.equals(this.node, that.node) + && Objects.equals(this.transformStats, that.transformStats) + && Objects.equals(this.checkpointingInfo, that.checkpointingInfo); + } + + public String getId() { + return id; + } + + public DataFrameTransformTaskState getTaskState() { + return taskState; + } + + @Nullable + public String getReason() { + return reason; + } + + @Nullable + public NodeAttributes getNode() { + return node; + } + + public void setNode(NodeAttributes node) { + this.node = node; + } + + public DataFrameIndexerTransformStats getTransformStats() { + return transformStats; + } + + public DataFrameTransformCheckpointingInfo getCheckpointingInfo() { + return checkpointingInfo; + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java index aa5c1a03ef2ea..c6943d05f7e2a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java @@ -11,8 +11,8 @@ import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction.Response; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsTests; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfoTests; import java.util.ArrayList; import java.util.List; @@ -20,10 +20,10 @@ public class GetDataFrameTransformsStatsActionResponseTests extends AbstractWireSerializingDataFrameTestCase { @Override protected Response createTestInstance() { - List stats = new ArrayList<>(); + List stats = new ArrayList<>(); int totalStats = randomInt(10); for (int i = 0; i < totalStats; ++i) { - stats.add(DataFrameTransformStateAndStatsTests.randomDataFrameTransformStateAndStats()); + stats.add(DataFrameTransformStateAndStatsInfoTests.randomDataFrameTransformStateAndStatsInfo()); } int totalErrors = randomInt(10); List taskFailures = new ArrayList<>(totalErrors); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStatsTests.java index 88eab817d069d..4ea93c4c51938 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStatsTests.java @@ -8,13 +8,18 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.indexing.IndexerState; import java.io.IOException; public class DataFrameTransformCheckpointStatsTests extends AbstractSerializingDataFrameTestCase { public static DataFrameTransformCheckpointStats randomDataFrameTransformCheckpointStats() { - return new DataFrameTransformCheckpointStats(randomNonNegativeLong(), randomNonNegativeLong()); + return new DataFrameTransformCheckpointStats(randomLongBetween(1, 1_000_000), + randomBoolean() ? null : randomFrom(IndexerState.values()), + DataFrameIndexerPositionTests.randomDataFrameIndexerPosition(), + randomBoolean() ? null : DataFrameTransformProgressTests.randomDataFrameTransformProgress(), + randomLongBetween(1, 1_000_000), randomLongBetween(0, 1_000_000)); } @Override @@ -31,5 +36,4 @@ protected DataFrameTransformCheckpointStats createTestInstance() { protected Reader instanceReader() { return DataFrameTransformCheckpointStats::new; } - } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformProgressTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformProgressTests.java index 8339669057ec6..1d4c308e86b2a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformProgressTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformProgressTests.java @@ -15,6 +15,7 @@ import static org.hamcrest.Matchers.equalTo; public class DataFrameTransformProgressTests extends AbstractSerializingDataFrameTestCase { + public static DataFrameTransformProgress randomDataFrameTransformProgress() { long totalDocs = randomNonNegativeLong(); return new DataFrameTransformProgress(totalDocs, randomBoolean() ? null : randomLongBetween(0, totalDocs)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java new file mode 100644 index 0000000000000..9e8f0c81ec5a9 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.dataframe.transforms; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.function.Predicate; + +public class DataFrameTransformStateAndStatsInfoTests extends AbstractSerializingTestCase { + + public static DataFrameTransformStateAndStatsInfo randomDataFrameTransformStateAndStatsInfo() { + return new DataFrameTransformStateAndStatsInfo(randomAlphaOfLength(10), + randomFrom(DataFrameTransformTaskState.values()), + randomBoolean() ? null : randomAlphaOfLength(100), + randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(), + DataFrameIndexerTransformStatsTests.randomStats(DataFrameIndexerTransformStats.DEFAULT_TRANSFORM_ID), + DataFrameTransformCheckpointingInfoTests.randomDataFrameTransformCheckpointingInfo()); + } + + @Override + protected DataFrameTransformStateAndStatsInfo doParseInstance(XContentParser parser) throws IOException { + return DataFrameTransformStateAndStatsInfo.fromXContent(parser); + } + + @Override + protected DataFrameTransformStateAndStatsInfo createTestInstance() { + return randomDataFrameTransformStateAndStatsInfo(); + } + + @Override + protected Reader instanceReader() { + return DataFrameTransformStateAndStatsInfo::new; + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected String[] getShuffleFieldsExceptions() { + return new String[] { "position" }; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> !field.isEmpty(); + } +} diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java index 99b5704c7fd53..6abf6fd4bd838 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java @@ -27,11 +27,10 @@ import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction.Request; import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction.Response; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.xpack.core.dataframe.transforms.NodeAttributes; -import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.dataframe.checkpoint.DataFrameTransformsCheckpointService; import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager; import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformTask; @@ -69,9 +68,9 @@ public TransportGetDataFrameTransformsStatsAction(TransportService transportServ @Override protected Response newResponse(Request request, List tasks, List taskOperationFailures, List failedNodeExceptions) { - List responses = tasks.stream() - .flatMap(r -> r.getTransformsStateAndStats().stream()) - .sorted(Comparator.comparing(DataFrameTransformStateAndStats::getId)) + List responses = tasks.stream() + .flatMap(r -> r.getTransformsStateAndStatsInfo().stream()) + .sorted(Comparator.comparing(DataFrameTransformStateAndStatsInfo::getId)) .collect(Collectors.toList()); List allFailedNodeExceptions = new ArrayList<>(failedNodeExceptions); allFailedNodeExceptions.addAll(tasks.stream().flatMap(r -> r.getNodeFailures().stream()).collect(Collectors.toList())); @@ -86,14 +85,18 @@ protected void taskOperation(Request request, DataFrameTransformTask task, Actio if (task.isCancelled() == false) { transformsCheckpointService.getCheckpointStats(task.getTransformId(), task.getCheckpoint(), task.getInProgressCheckpoint(), ActionListener.wrap(checkpointStats -> listener.onResponse(new Response( - Collections.singletonList(new DataFrameTransformStateAndStats(task.getTransformId(), - task.getState(), + Collections.singletonList(new DataFrameTransformStateAndStatsInfo(task.getTransformId(), + task.getState().getTaskState(), + task.getState().getReason(), + null, task.getStats(), checkpointStats)), 1L)), e -> listener.onResponse(new Response( - Collections.singletonList(new DataFrameTransformStateAndStats(task.getTransformId(), - task.getState(), + Collections.singletonList(new DataFrameTransformStateAndStatsInfo(task.getTransformId(), + task.getState().getTaskState(), + task.getState().getReason(), + null, task.getStats(), DataFrameTransformCheckpointingInfo.EMPTY)), 1L, @@ -119,10 +122,10 @@ protected void doExecute(Task task, Request request, ActionListener fi PersistentTasksCustomMetaData tasksInProgress = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasksInProgress != null) { // Mutates underlying state object with the assigned node attributes - response.getTransformsStateAndStats().forEach(dtsas -> setNodeAttributes(dtsas, tasksInProgress, state)); + response.getTransformsStateAndStatsInfo().forEach(dtsasi -> setNodeAttributes(dtsasi, tasksInProgress, state)); } collectStatsForTransformsWithoutTasks(request, response, ActionListener.wrap( - finalResponse -> finalListener.onResponse(new Response(finalResponse.getTransformsStateAndStats(), + finalResponse -> finalListener.onResponse(new Response(finalResponse.getTransformsStateAndStatsInfo(), hitsAndIds.v1(), finalResponse.getTaskFailures(), finalResponse.getNodeFailures())), @@ -143,13 +146,11 @@ protected void doExecute(Task task, Request request, ActionListener fi )); } - private static void setNodeAttributes(DataFrameTransformStateAndStats dataFrameTransformStateAndStats, - PersistentTasksCustomMetaData persistentTasksCustomMetaData, - ClusterState state) { - var pTask = persistentTasksCustomMetaData.getTask(dataFrameTransformStateAndStats.getTransformId()); + private static void setNodeAttributes(DataFrameTransformStateAndStatsInfo dataFrameTransformStateAndStatsInfo, + PersistentTasksCustomMetaData persistentTasksCustomMetaData, ClusterState state) { + var pTask = persistentTasksCustomMetaData.getTask(dataFrameTransformStateAndStatsInfo.getId()); if (pTask != null) { - dataFrameTransformStateAndStats.getTransformState() - .setNode(NodeAttributes.fromDiscoveryNode(state.nodes().get(pTask.getExecutorNode()))); + dataFrameTransformStateAndStatsInfo.setNode(NodeAttributes.fromDiscoveryNode(state.nodes().get(pTask.getExecutorNode()))); } } @@ -157,13 +158,13 @@ private void collectStatsForTransformsWithoutTasks(Request request, Response response, ActionListener listener) { // We gathered all there is, no need to continue - if (request.getExpandedIds().size() == response.getTransformsStateAndStats().size()) { + if (request.getExpandedIds().size() == response.getTransformsStateAndStatsInfo().size()) { listener.onResponse(response); return; } Set transformsWithoutTasks = new HashSet<>(request.getExpandedIds()); - transformsWithoutTasks.removeAll(response.getTransformsStateAndStats().stream().map(DataFrameTransformStateAndStats::getId) + transformsWithoutTasks.removeAll(response.getTransformsStateAndStatsInfo().stream().map(DataFrameTransformStateAndStatsInfo::getId) .collect(Collectors.toList())); // Small assurance that we are at least below the max. Terms search has a hard limit of 10k, we should at least be below that. @@ -171,32 +172,28 @@ private void collectStatsForTransformsWithoutTasks(Request request, ActionListener> searchStatsListener = ActionListener.wrap( stats -> { - List allStateAndStats = response.getTransformsStateAndStats(); + List allStateAndStats = response.getTransformsStateAndStatsInfo(); // If the persistent task does NOT exist, it is STOPPED // There is a potential race condition where the saved document does not actually have a STOPPED state // as the task is cancelled before we persist state. stats.forEach(stat -> - allStateAndStats.add(new DataFrameTransformStateAndStats( + allStateAndStats.add(new DataFrameTransformStateAndStatsInfo( stat.getId(), - new DataFrameTransformState(DataFrameTransformTaskState.STOPPED, - IndexerState.STOPPED, - stat.getTransformState().getPosition(), - stat.getTransformState().getCheckpoint(), - stat.getTransformState().getReason(), - stat.getTransformState().getProgress()), + DataFrameTransformTaskState.STOPPED, + null, + null, stat.getTransformStats(), stat.getCheckpointingInfo())) ); - transformsWithoutTasks.removeAll( - stats.stream().map(DataFrameTransformStateAndStats::getId).collect(Collectors.toSet())); + transformsWithoutTasks.removeAll(stats.stream().map(DataFrameTransformStateAndStats::getId).collect(Collectors.toSet())); // Transforms that have not been started and have no state or stats. transformsWithoutTasks.forEach(transformId -> - allStateAndStats.add(DataFrameTransformStateAndStats.initialStateAndStats(transformId))); + allStateAndStats.add(DataFrameTransformStateAndStatsInfo.initialStateAndStatsInfo(transformId))); // Any transform in collection could NOT have a task, so, even though the list is initially sorted // it can easily become arbitrarily ordered based on which transforms don't have a task or stats docs - allStateAndStats.sort(Comparator.comparing(DataFrameTransformStateAndStats::getId)); + allStateAndStats.sort(Comparator.comparing(DataFrameTransformStateAndStatsInfo::getId)); listener.onResponse(new Response(allStateAndStats, allStateAndStats.size(), diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java index 5d3a664054a4b..b6d543163ad13 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java @@ -286,19 +286,19 @@ public void putOrUpdateTransformStats(DataFrameTransformStateAndStats stats, Act IndexRequest indexRequest = new IndexRequest(DataFrameInternalIndex.INDEX_NAME) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .id(DataFrameTransformStateAndStats.documentId(stats.getTransformId())) + .id(DataFrameTransformStateAndStats.documentId(stats.getId())) .source(source); executeAsyncWithOrigin(client, DATA_FRAME_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( r -> listener.onResponse(true), e -> listener.onFailure(new RuntimeException( - DataFrameMessages.getMessage(DataFrameMessages.DATA_FRAME_FAILED_TO_PERSIST_STATS, stats.getTransformId()), + DataFrameMessages.getMessage(DataFrameMessages.DATA_FRAME_FAILED_TO_PERSIST_STATS, stats.getId()), e)) )); } catch (IOException e) { // not expected to happen but for the sake of completeness listener.onFailure(new ElasticsearchParseException( - DataFrameMessages.getMessage(DataFrameMessages.DATA_FRAME_FAILED_TO_PERSIST_STATS, stats.getTransformId()), + DataFrameMessages.getMessage(DataFrameMessages.DATA_FRAME_FAILED_TO_PERSIST_STATS, stats.getId()), e)); } } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java index 1aee9130ddc50..11d493a611389 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java @@ -13,7 +13,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsResponse; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.client.dataframe.transforms.DestConfig; import org.elasticsearch.client.dataframe.transforms.SourceConfig; @@ -135,7 +135,7 @@ private void createAndStartContinuousDataFrame() throws Exception { startTransform(CONTINUOUS_DATA_FRAME_ID); waitUntilAfterCheckpoint(CONTINUOUS_DATA_FRAME_ID, 0L); - DataFrameTransformStateAndStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); + DataFrameTransformStateAndStatsInfo stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); assertThat(stateAndStats.getTransformStats().getOutputDocuments(), equalTo((long)ENTITIES.size())); assertThat(stateAndStats.getTransformStats().getNumDocuments(), equalTo(totalDocsWritten)); @@ -147,13 +147,13 @@ private void verifyContinuousDataFrameHandlesData(long expectedLastCheckpoint) t // A continuous data frame should automatically become started when it gets assigned to a node // if it was assigned to the node that was removed from the cluster assertBusy(() -> { - DataFrameTransformStateAndStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); + DataFrameTransformStateAndStatsInfo stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); assertThat(stateAndStats.getTransformState().getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); }, 120, TimeUnit.SECONDS); - DataFrameTransformStateAndStats previousStateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); + DataFrameTransformStateAndStatsInfo previousStateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); // Add a new user and write data to it // This is so we can have more reliable data counts, as writing to existing entities requires @@ -172,7 +172,7 @@ private void verifyContinuousDataFrameHandlesData(long expectedLastCheckpoint) t greaterThanOrEqualTo(docs + previousStateAndStats.getTransformStats().getNumDocuments())), 120, TimeUnit.SECONDS); - DataFrameTransformStateAndStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); + DataFrameTransformStateAndStatsInfo stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); assertThat(stateAndStats.getTransformState().getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); @@ -207,7 +207,7 @@ private void stopTransform(String id) throws IOException { assertEquals(200, response.getStatusLine().getStatusCode()); } - private DataFrameTransformStateAndStats getTransformStats(String id) throws IOException { + private DataFrameTransformStateAndStatsInfo getTransformStats(String id) throws IOException { final Request getStats = new Request("GET", DATAFRAME_ENDPOINT + id + "/_stats"); Response response = client().performRequest(getStats); assertEquals(200, response.getStatusLine().getStatusCode()); From ba3fa6861c120b824cafd81dd8d5b0e2254df398 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 15 Jul 2019 15:51:59 +0100 Subject: [PATCH 02/12] Fixing some HLRC unit tests --- .../hlrc/DataFrameTransformCheckpointingInfoTests.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java index 84f096caf14e7..e95eb7bdedcfd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformCheckpointingInfoTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; import java.io.IOException; +import java.util.function.Predicate; public class DataFrameTransformCheckpointingInfoTests extends AbstractHlrcXContentTestCase< DataFrameTransformCheckpointingInfo, @@ -67,4 +68,9 @@ protected DataFrameTransformCheckpointingInfo doParseInstance(XContentParser par protected boolean supportsUnknownFields() { return true; } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> field.contains("position"); + } } From 711e1f00ae673c1ea4ed8257f7fff188b8a905f4 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 16 Jul 2019 10:46:05 +0100 Subject: [PATCH 03/12] Adjusting docs --- .../apis/get-transform-stats.asciidoc | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/docs/reference/data-frames/apis/get-transform-stats.asciidoc b/docs/reference/data-frames/apis/get-transform-stats.asciidoc index 2abb50407fdaf..88536b93f9ab1 100644 --- a/docs/reference/data-frames/apis/get-transform-stats.asciidoc +++ b/docs/reference/data-frames/apis/get-transform-stats.asciidoc @@ -126,16 +126,7 @@ The API returns the following results: "transforms" : [ { "id" : "ecommerce_transform", - "state" : { - "task_state" : "started", - "indexer_state" : "started", - "checkpoint" : 1, - "progress" : { - "total_docs" : 1220, - "docs_remaining" : 0, - "percent_complete" : 100.0 - } - }, + "task_state" : "started", "stats" : { "pages_processed" : 2, "documents_processed" : 1220, @@ -149,10 +140,31 @@ The API returns the following results: "search_failures" : 0 }, "checkpointing" : { - "current" : { - "timestamp_millis" : 1557474786393 + "last" : { + "checkpoint" : 100, + "timestamp_millis" : 1561740252497, + "time_upper_bound_millis" : 1561740192497 + }, + "next" : { + "checkpoint" : 101, + "indexer_state" : "started", + "position" : { + "indexer_position" : { + "hashtag" : "abcd1234" + }, + "buckets_position" : { + "hashtag" : "abcd5678" + } + }, + "checkpoint_progress" : { + "total_docs" : 1900883, + "docs_remaining" : 1722762, + "percent_complete" : 9.370434687458408 + }, + "timestamp_millis" : 1561740629172, + "time_upper_bound_millis" : 1561740569172 }, - "operations_behind" : 0 + "operations_behind": 27000 } } ] From 2e97454057da7b9acf0f9e526f298114f73f11f1 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 16 Jul 2019 17:02:45 +0100 Subject: [PATCH 04/12] Further iteration This still doesn't work as the checkpointing info is always empty in the state-and-stats docs. This commit makes that clearer by not actually persisting empty checkpointing info in the state-and-stats docs, but instead hardcoding this at the point of use. The next step is then to get non-empty checkpointing info when required... --- .../DataFrameIndexerTransformStats.java | 2 -- .../DataFrameTransformCheckpointStats.java | 4 +-- .../DataFrameTransformStateAndStats.java | 33 ++++++++----------- .../DataFrameTransformStateAndStatsInfo.java | 12 +++---- .../DataFrameTransformStateAndStatsTests.java | 3 +- .../integration/DataFrameIntegTestCase.java | 3 +- .../integration/DataFrameTransformIT.java | 7 ++-- ...portGetDataFrameTransformsStatsAction.java | 5 +-- .../DataFrameTransformsCheckpointService.java | 26 ++++++++++++--- .../persistence/DataFrameInternalIndex.java | 4 ++- .../DataFrameTransformsConfigManager.java | 6 ++-- ...FrameTransformPersistentTasksExecutor.java | 2 +- .../transforms/DataFrameTransformTask.java | 6 ++-- ...meTransformCheckpointServiceNodeTests.java | 28 +++++++++++----- ...DataFrameTransformsConfigManagerTests.java | 13 ++++---- .../upgrades/DataFrameSurvivesUpgradeIT.java | 8 ++--- 16 files changed, 90 insertions(+), 72 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameIndexerTransformStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameIndexerTransformStats.java index 8f83fd375490d..2926a88768625 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameIndexerTransformStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameIndexerTransformStats.java @@ -55,7 +55,6 @@ public class DataFrameIndexerTransformStats extends IndexerJobStats { LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL); LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES); LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES); - LENIENT_PARSER.declareString(optionalConstructorArg(), DataFrameField.INDEX_DOC_TYPE); } private final String transformId; @@ -133,7 +132,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws throw new IllegalArgumentException("when storing transform statistics, a valid transform id must be provided"); } builder.field(DataFrameField.ID.getPreferredName(), transformId); - builder.field(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), NAME); } builder.endObject(); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java index 439e380d3272c..f0116056832a4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java @@ -73,7 +73,7 @@ public DataFrameTransformCheckpointStats(final long checkpoint, final IndexerSta } public DataFrameTransformCheckpointStats(StreamInput in) throws IOException { - if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport this.checkpoint = in.readVLong(); if (in.readBoolean()) { this.indexerState = in.readEnum(IndexerState.class); @@ -149,7 +149,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport out.writeVLong(checkpoint); if (indexerState != null) { out.writeBoolean(true); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java index 5534c0df27bf4..7d0981865e917 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -28,27 +29,22 @@ public class DataFrameTransformStateAndStats implements Writeable, ToXContentObj public static final String NAME = "data_frame_transform_state_and_stats"; public static final ParseField STATE_FIELD = new ParseField("state"); - public static final ParseField CHECKPOINTING_INFO_FIELD = new ParseField("checkpointing"); private final String id; private final DataFrameTransformState transformState; private final DataFrameIndexerTransformStats transformStats; - private final DataFrameTransformCheckpointingInfo checkpointingInfo; public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, true, a -> new DataFrameTransformStateAndStats((String) a[0], (DataFrameTransformState) a[1], - (DataFrameIndexerTransformStats) a[2], - (DataFrameTransformCheckpointingInfo) a[3])); + (DataFrameIndexerTransformStats) a[2])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), DataFrameField.ID); PARSER.declareObject(ConstructingObjectParser.constructorArg(), DataFrameTransformState.PARSER::apply, STATE_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> DataFrameIndexerTransformStats.fromXContent(p), DataFrameField.STATS_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> DataFrameTransformCheckpointingInfo.fromXContent(p), CHECKPOINTING_INFO_FIELD); } public static DataFrameTransformStateAndStats fromXContent(XContentParser parser) throws IOException { @@ -64,19 +60,19 @@ public static String documentId(String transformId) { return NAME + "-" + transformId; } - public DataFrameTransformStateAndStats(String id, DataFrameTransformState state, DataFrameIndexerTransformStats stats, - DataFrameTransformCheckpointingInfo checkpointingInfo) { + public DataFrameTransformStateAndStats(String id, DataFrameTransformState state, DataFrameIndexerTransformStats stats) { this.id = Objects.requireNonNull(id); this.transformState = Objects.requireNonNull(state); this.transformStats = Objects.requireNonNull(stats); - this.checkpointingInfo = Objects.requireNonNull(checkpointingInfo); } public DataFrameTransformStateAndStats(StreamInput in) throws IOException { this.id = in.readString(); this.transformState = new DataFrameTransformState(in); this.transformStats = new DataFrameIndexerTransformStats(in); - this.checkpointingInfo = new DataFrameTransformCheckpointingInfo(in); + if (in.getVersion().before(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport + new DataFrameTransformCheckpointingInfo(in); + } } @Override @@ -85,7 +81,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DataFrameField.ID.getPreferredName(), id); builder.field(STATE_FIELD.getPreferredName(), transformState, params); builder.field(DataFrameField.STATS_FIELD.getPreferredName(), transformStats, params); - builder.field(CHECKPOINTING_INFO_FIELD.getPreferredName(), checkpointingInfo, params); builder.field(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), NAME); builder.endObject(); return builder; @@ -96,12 +91,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); transformState.writeTo(out); transformStats.writeTo(out); - checkpointingInfo.writeTo(out); + if (out.getVersion().before(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport + DataFrameTransformCheckpointingInfo.EMPTY.writeTo(out); + } } @Override public int hashCode() { - return Objects.hash(id, transformState, transformStats, checkpointingInfo); + return Objects.hash(id, transformState, transformStats); } @Override @@ -116,9 +113,9 @@ public boolean equals(Object other) { DataFrameTransformStateAndStats that = (DataFrameTransformStateAndStats) other; - return Objects.equals(this.id, that.id) && Objects.equals(this.transformState, that.transformState) - && Objects.equals(this.transformStats, that.transformStats) - && Objects.equals(this.checkpointingInfo, that.checkpointingInfo); + return Objects.equals(this.id, that.id) + && Objects.equals(this.transformState, that.transformState) + && Objects.equals(this.transformStats, that.transformStats); } public String getId() { @@ -133,10 +130,6 @@ public DataFrameTransformState getTransformState() { return transformState; } - public DataFrameTransformCheckpointingInfo getCheckpointingInfo() { - return checkpointingInfo; - } - @Override public String toString() { return Strings.toString(this); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java index 4ae17125aeabd..ef97e70e06fd4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java @@ -101,7 +101,7 @@ public DataFrameTransformStateAndStatsInfo(String id, DataFrameTransformTaskStat } public DataFrameTransformStateAndStatsInfo(StreamInput in) throws IOException { - if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport this.id = in.readString(); this.taskState = in.readEnum(DataFrameTransformTaskState.class); if (in.readBoolean()) { @@ -113,8 +113,8 @@ public DataFrameTransformStateAndStatsInfo(StreamInput in) throws IOException { this.checkpointingInfo = new DataFrameTransformCheckpointingInfo(in); } else { - // In 7.2 DataFrameTransformStateAndStatsInfo didn't exist, and we have to do - // the best we can of reading from a DataFrameTransformStateAndStats object + // Prior to version 7.4 DataFrameTransformStateAndStatsInfo didn't exist, and we have + // to do the best we can of reading from a DataFrameTransformStateAndStats object this.id = in.readString(); this.taskState = new DataFrameTransformState(in).getTaskState(); this.node = null; @@ -139,7 +139,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport out.writeString(id); out.writeEnum(taskState); if (node != null) { @@ -151,8 +151,8 @@ public void writeTo(StreamOutput out) throws IOException { transformStats.writeTo(out); checkpointingInfo.writeTo(out); } else { - // In 7.2 DataFrameTransformStateAndStatsInfo didn't exist, and we have to do - // the best we can of writing to a DataFrameTransformStateAndStats object + // Prior to version 7.4 DataFrameTransformStateAndStatsInfo didn't exist, and we have + // to do the best we can of writing to a DataFrameTransformStateAndStats object out.writeString(id); new DataFrameTransformState(taskState, checkpointingInfo.getNext().getIndexerState(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsTests.java index 89ce1e6d84002..4f80d0d0b453c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsTests.java @@ -22,8 +22,7 @@ public class DataFrameTransformStateAndStatsTests extends AbstractSerializingDat public static DataFrameTransformStateAndStats randomDataFrameTransformStateAndStats(String id) { return new DataFrameTransformStateAndStats(id, DataFrameTransformStateTests.randomDataFrameTransformState(), - DataFrameIndexerTransformStatsTests.randomStats(id), - DataFrameTransformCheckpointingInfoTests.randomDataFrameTransformCheckpointingInfo()); + DataFrameIndexerTransformStatsTests.randomStats(id)); } public static DataFrameTransformStateAndStats randomDataFrameTransformStateAndStats() { diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java index 09b34f7ed7b20..e1219fd7541b0 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java @@ -135,7 +135,8 @@ protected void waitUntilCheckpoint(String id, long checkpoint, TimeValue waitTim assertEquals(checkpoint, getDataFrameTransformStats(id) .getTransformsStateAndStats() .get(0) - .getTransformState() + .getCheckpointingInfo() + .getNext() .getCheckpoint()), waitTime.getMillis(), TimeUnit.MILLISECONDS); diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java index 456ba91ef4381..5cf161785149d 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java @@ -65,9 +65,8 @@ public void testDataFrameTransformCrud() throws Exception { waitUntilCheckpoint(config.getId(), 1L); // It will eventually be stopped - assertBusy(() -> - assertThat(getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0).getTransformState().getIndexerState(), - equalTo(IndexerState.STOPPED))); + assertBusy(() -> assertThat(getDataFrameTransformStats(config.getId()) + .getTransformsStateAndStats().get(0).getCheckpointingInfo().getNext().getIndexerState(), equalTo(IndexerState.STOPPED))); stopDataFrameTransform(config.getId()); DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0); @@ -103,7 +102,7 @@ public void testContinuousDataFrameTransformCrud() throws Exception { assertTrue(startDataFrameTransform(config.getId(), RequestOptions.DEFAULT).isAcknowledged()); waitUntilCheckpoint(config.getId(), 1L); - assertThat(getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0).getTransformState().getTaskState(), + assertThat(getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0).getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); long docsIndexed = getDataFrameTransformStats(config.getId()) diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java index 6abf6fd4bd838..99cfb6101fd8e 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java @@ -84,6 +84,7 @@ protected void taskOperation(Request request, DataFrameTransformTask task, Actio String nodeId = state.nodes().getLocalNode().getId(); if (task.isCancelled() == false) { transformsCheckpointService.getCheckpointStats(task.getTransformId(), task.getCheckpoint(), task.getInProgressCheckpoint(), + task.getState().getIndexerState(), task.getState().getPosition(), ActionListener.wrap(checkpointStats -> listener.onResponse(new Response( Collections.singletonList(new DataFrameTransformStateAndStatsInfo(task.getTransformId(), task.getState().getTaskState(), @@ -183,7 +184,7 @@ private void collectStatsForTransformsWithoutTasks(Request request, null, null, stat.getTransformStats(), - stat.getCheckpointingInfo())) + DataFrameTransformCheckpointingInfo.EMPTY)) // TODO !!! ); transformsWithoutTasks.removeAll(stats.stream().map(DataFrameTransformStateAndStats::getId).collect(Collectors.toSet())); @@ -209,6 +210,6 @@ private void collectStatsForTransformsWithoutTasks(Request request, } ); - dataFrameTransformsConfigManager.getTransformStats(transformsWithoutTasks, searchStatsListener); + dataFrameTransformsConfigManager.getTransformStateAndStats(transformsWithoutTasks, searchStatsListener); } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java index d028d3248c8f6..373eeeb029933 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java @@ -18,12 +18,14 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerPosition; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig; import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig; +import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager; import java.util.Arrays; @@ -43,17 +45,30 @@ public class DataFrameTransformsCheckpointService { private static class Checkpoints { + long currentCheckpointNumber; + long inProgressCheckpointNumber; + IndexerState inProgressIndexerState; + DataFrameIndexerPosition inProgressPosition; DataFrameTransformCheckpoint currentCheckpoint = DataFrameTransformCheckpoint.EMPTY; DataFrameTransformCheckpoint inProgressCheckpoint = DataFrameTransformCheckpoint.EMPTY; DataFrameTransformCheckpoint sourceCheckpoint = DataFrameTransformCheckpoint.EMPTY; + Checkpoints(long currentCheckpointNumber, long inProgressCheckpointNumber, IndexerState inProgressIndexerState, + DataFrameIndexerPosition inProgressPosition) { + this.currentCheckpointNumber = currentCheckpointNumber; + this.inProgressCheckpointNumber = inProgressCheckpointNumber; + this.inProgressIndexerState = inProgressIndexerState; + this.inProgressPosition = inProgressPosition; + } + DataFrameTransformCheckpointingInfo buildInfo() { return new DataFrameTransformCheckpointingInfo( - new DataFrameTransformCheckpointStats(currentCheckpoint.getTimestamp(), currentCheckpoint.getTimeUpperBound()), - new DataFrameTransformCheckpointStats(inProgressCheckpoint.getTimestamp(), inProgressCheckpoint.getTimeUpperBound()), + new DataFrameTransformCheckpointStats(currentCheckpointNumber, null, null, null, + currentCheckpoint.getTimestamp(), currentCheckpoint.getTimeUpperBound()), + new DataFrameTransformCheckpointStats(inProgressCheckpointNumber, inProgressIndexerState, inProgressPosition, null, + inProgressCheckpoint.getTimestamp(), inProgressCheckpoint.getTimeUpperBound()), DataFrameTransformCheckpoint.getBehind(currentCheckpoint, sourceCheckpoint)); } - } private static final Logger logger = LogManager.getLogger(DataFrameTransformsCheckpointService.class); @@ -135,7 +150,6 @@ public void getCheckpoint(DataFrameTransformConfig transformConfig, long checkpo /** * Get checkpointing stats for a data frame * - * * @param transformId The data frame task * @param currentCheckpoint the current checkpoint * @param inProgressCheckpoint in progress checkpoint @@ -145,9 +159,11 @@ public void getCheckpointStats( String transformId, long currentCheckpoint, long inProgressCheckpoint, + IndexerState inProgressIndexerState, + DataFrameIndexerPosition inProgressPosition, ActionListener listener) { - Checkpoints checkpoints = new Checkpoints(); + Checkpoints checkpoints = new Checkpoints(currentCheckpoint, inProgressCheckpoint, inProgressIndexerState, inProgressPosition); // <3> notify the user once we have the current checkpoint ActionListener currentCheckpointListener = ActionListener.wrap( diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java index e28f8005448d9..c7169695d0118 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java @@ -211,7 +211,9 @@ private static XContentBuilder addDataFrameTransformStateAndStatsMappings(XConte .endObject() .endObject() .endObject() - .startObject(DataFrameTransformStateAndStats.CHECKPOINTING_INFO_FIELD.getPreferredName()) + // This is obsolete and can be removed for future versions of the index, but is left here as a warning/reminder that + // we cannot declare this field differently in version 1 of the internal index as it would cause a mapping clash + .startObject("checkpointing") .field(ENABLED, false) .endObject(); } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java index b6d543163ad13..99cfec4e3339a 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java @@ -280,7 +280,7 @@ public void deleteTransform(String transformId, ActionListener listener })); } - public void putOrUpdateTransformStats(DataFrameTransformStateAndStats stats, ActionListener listener) { + public void putOrUpdateTransformStateAndStats(DataFrameTransformStateAndStats stats, ActionListener listener) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) { XContentBuilder source = stats.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); @@ -303,7 +303,7 @@ public void putOrUpdateTransformStats(DataFrameTransformStateAndStats stats, Act } } - public void getTransformStats(String transformId, ActionListener resultListener) { + public void getTransformStateAndStats(String transformId, ActionListener resultListener) { GetRequest getRequest = new GetRequest(DataFrameInternalIndex.INDEX_NAME, DataFrameTransformStateAndStats.documentId(transformId)); executeAsyncWithOrigin(client, DATA_FRAME_ORIGIN, GetAction.INSTANCE, getRequest, ActionListener.wrap(getResponse -> { @@ -332,7 +332,7 @@ public void getTransformStats(String transformId, ActionListener transformIds, ActionListener> listener) { + public void getTransformStateAndStats(Collection transformIds, ActionListener> listener) { QueryBuilder builder = QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery() .filter(QueryBuilders.termsQuery(DataFrameField.ID.getPreferredName(), transformIds)) diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java index 5b02c0836dfe1..85a58531e0563 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java @@ -215,7 +215,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable DataFrameTr ActionListener> getFieldMappingsListener = ActionListener.wrap( fieldMappings -> { indexerBuilder.setFieldMappings(fieldMappings); - transformsConfigManager.getTransformStats(transformId, transformStatsActionListener); + transformsConfigManager.getTransformStateAndStats(transformId, transformStatsActionListener); }, error -> { String msg = DataFrameMessages.getMessage(DataFrameMessages.DATA_FRAME_UNABLE_TO_GATHER_FIELD_MAPPINGS, diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java index 8542222e4bd59..ef381a7026e5b 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java @@ -34,7 +34,6 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; @@ -647,9 +646,8 @@ protected void doSaveState(IndexerState indexerState, DataFrameIndexerPosition p // Persisting stats when we call `doSaveState` should be ok as we only call it on a state transition and // only every-so-often when doing the bulk indexing calls. See AsyncTwoPhaseIndexer#onBulkResponse for current periodicity - transformsConfigManager.putOrUpdateTransformStats( - new DataFrameTransformStateAndStats(transformId, state, getStats(), - DataFrameTransformCheckpointingInfo.EMPTY), // TODO should this be null + transformsConfigManager.putOrUpdateTransformStateAndStats( + new DataFrameTransformStateAndStats(transformId, state, getStats()), ActionListener.wrap( r -> { // for auto stop shutdown the task diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java index 166ec2de3febf..cbf9bb9260b5c 100644 --- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java +++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java @@ -39,10 +39,13 @@ import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerPosition; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerPositionTests; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfigTests; +import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.dataframe.DataFrameSingleNodeTestCase; import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager; import org.junit.After; @@ -172,6 +175,7 @@ public void testCreateReadDeleteCheckpoint() throws InterruptedException { public void testGetCheckpointStats() throws InterruptedException { String transformId = randomAlphaOfLengthBetween(3, 10); long timestamp = 1000; + DataFrameIndexerPosition position = DataFrameIndexerPositionTests.randomDataFrameIndexerPosition(); // create transform assertAsync( @@ -191,26 +195,32 @@ public void testGetCheckpointStats() throws InterruptedException { mockClientForCheckpointing.setShardStats(createShardStats(createCheckPointMap(transformId, 20, 20, 20))); DataFrameTransformCheckpointingInfo checkpointInfo = new DataFrameTransformCheckpointingInfo( - new DataFrameTransformCheckpointStats(timestamp, 0L), - new DataFrameTransformCheckpointStats(timestamp + 100L, 0L), + new DataFrameTransformCheckpointStats(1, null, null, null, timestamp, 0L), + new DataFrameTransformCheckpointStats(2, IndexerState.STARTED, position, null, timestamp + 100L, 0L), 30L); - assertAsync(listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, listener), checkpointInfo, null, null); + assertAsync( + listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.STARTED, position, listener), + checkpointInfo, null, null); mockClientForCheckpointing.setShardStats(createShardStats(createCheckPointMap(transformId, 10, 50, 33))); checkpointInfo = new DataFrameTransformCheckpointingInfo( - new DataFrameTransformCheckpointStats(timestamp, 0L), - new DataFrameTransformCheckpointStats(timestamp + 100L, 0L), + new DataFrameTransformCheckpointStats(1, null, null, null, timestamp, 0L), + new DataFrameTransformCheckpointStats(2, IndexerState.INDEXING, position, null, timestamp + 100L, 0L), 63L); - assertAsync(listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, listener), checkpointInfo, null, null); + assertAsync( + listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.INDEXING, position, listener), + checkpointInfo, null, null); // same as current mockClientForCheckpointing.setShardStats(createShardStats(createCheckPointMap(transformId, 10, 10, 10))); checkpointInfo = new DataFrameTransformCheckpointingInfo( - new DataFrameTransformCheckpointStats(timestamp, 0L), - new DataFrameTransformCheckpointStats(timestamp + 100L, 0L), + new DataFrameTransformCheckpointStats(1, null, null, null, timestamp, 0L), + new DataFrameTransformCheckpointStats(2, IndexerState.STOPPING, position, null, timestamp + 100L, 0L), 0L); - assertAsync(listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, listener), checkpointInfo, null, null); + assertAsync( + listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.STOPPING, position, listener), + checkpointInfo, null, null); } private static Map createCheckPointMap(String index, long checkpointShard1, long checkpointShard2, diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java index e7485e02e5b91..e860d4843a5f4 100644 --- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java +++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java @@ -251,13 +251,14 @@ public void testStateAndStats() throws InterruptedException { DataFrameTransformStateAndStats stateAndStats = DataFrameTransformStateAndStatsTests.randomDataFrameTransformStateAndStats(transformId); - assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStats(stateAndStats, listener), Boolean.TRUE, null, null); - assertAsync(listener -> transformsConfigManager.getTransformStats(transformId, listener), stateAndStats, null, null); + assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStateAndStats(stateAndStats, listener), + Boolean.TRUE, null, null); + assertAsync(listener -> transformsConfigManager.getTransformStateAndStats(transformId, listener), stateAndStats, null, null); DataFrameTransformStateAndStats updated = DataFrameTransformStateAndStatsTests.randomDataFrameTransformStateAndStats(transformId); - assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStats(updated, listener), Boolean.TRUE, null, null); - assertAsync(listener -> transformsConfigManager.getTransformStats(transformId, listener), updated, null, null); + assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStateAndStats(updated, listener), Boolean.TRUE, null, null); + assertAsync(listener -> transformsConfigManager.getTransformStateAndStats(transformId, listener), updated, null, null); } public void testGetStateAndStatsMultiple() throws InterruptedException { @@ -267,7 +268,7 @@ public void testGetStateAndStatsMultiple() throws InterruptedException { DataFrameTransformStateAndStats stat = DataFrameTransformStateAndStatsTests.randomDataFrameTransformStateAndStats(randomAlphaOfLength(6)); expectedStats.add(stat); - assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStats(stat, listener), Boolean.TRUE, null, null); + assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStateAndStats(stat, listener), Boolean.TRUE, null, null); } // remove one of the put stats so we don't retrieve all @@ -278,6 +279,6 @@ public void testGetStateAndStatsMultiple() throws InterruptedException { // get stats will be ordered by id expectedStats.sort(Comparator.comparing(DataFrameTransformStateAndStats::getId)); - assertAsync(listener -> transformsConfigManager.getTransformStats(ids, listener), expectedStats, null, null); + assertAsync(listener -> transformsConfigManager.getTransformStateAndStats(ids, listener), expectedStats, null, null); } } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java index 11d493a611389..508340aed2ea4 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java @@ -139,7 +139,7 @@ private void createAndStartContinuousDataFrame() throws Exception { assertThat(stateAndStats.getTransformStats().getOutputDocuments(), equalTo((long)ENTITIES.size())); assertThat(stateAndStats.getTransformStats().getNumDocuments(), equalTo(totalDocsWritten)); - assertThat(stateAndStats.getTransformState().getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); + assertThat(stateAndStats.getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); } private void verifyContinuousDataFrameHandlesData(long expectedLastCheckpoint) throws Exception { @@ -148,7 +148,7 @@ private void verifyContinuousDataFrameHandlesData(long expectedLastCheckpoint) t // if it was assigned to the node that was removed from the cluster assertBusy(() -> { DataFrameTransformStateAndStatsInfo stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); - assertThat(stateAndStats.getTransformState().getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); + assertThat(stateAndStats.getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); }, 120, TimeUnit.SECONDS); @@ -174,7 +174,7 @@ private void verifyContinuousDataFrameHandlesData(long expectedLastCheckpoint) t TimeUnit.SECONDS); DataFrameTransformStateAndStatsInfo stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); - assertThat(stateAndStats.getTransformState().getTaskState(), + assertThat(stateAndStats.getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); assertThat(stateAndStats.getTransformStats().getOutputDocuments(), greaterThan(previousStateAndStats.getTransformStats().getOutputDocuments())); @@ -222,7 +222,7 @@ private DataFrameTransformStateAndStatsInfo getTransformStats(String id) throws } private void waitUntilAfterCheckpoint(String id, long currentCheckpoint) throws Exception { - assertBusy(() -> assertThat(getTransformStats(id).getTransformState().getCheckpoint(), greaterThan(currentCheckpoint)), + assertBusy(() -> assertThat(getTransformStats(id).getCheckpointingInfo().getNext().getCheckpoint(), greaterThan(currentCheckpoint)), 60, TimeUnit.SECONDS); } From 83c30339258b640a743c9f948f4b296ae00d0a1c Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 17 Jul 2019 11:50:21 +0100 Subject: [PATCH 05/12] Renaming DataFrameTransformStateAndStatsInfo -> DataFrameTransformStats (user facing) DataFrameTransformStateAndStats -> DataFrameTransformStoredDoc (internal) --- .../GetDataFrameTransformStatsResponse.java | 20 +++--- ...Info.java => DataFrameTransformStats.java} | 28 ++++----- .../client/DataFrameTransformIT.java | 18 +++--- ...tDataFrameTransformStatsResponseTests.java | 14 ++--- ...java => DataFrameTransformStatsTests.java} | 42 ++++++------- ...java => DataFrameTransformStatsTests.java} | 39 ++++++------ .../DataFrameTransformDocumentationIT.java | 10 +-- .../dataframe/get_data_frame_stats.asciidoc | 4 +- .../GetDataFrameTransformsStatsAction.java | 36 +++++------ ...Info.java => DataFrameTransformStats.java} | 63 ++++++++++--------- ....java => DataFrameTransformStoredDoc.java} | 14 ++--- ...ameTransformsStatsActionResponseTests.java | 8 +-- ...java => DataFrameTransformStatsTests.java} | 18 +++--- ... => DataFrameTransformStoredDocTests.java} | 23 ++++--- .../integration/DataFrameIntegTestCase.java | 2 +- .../integration/DataFrameTransformIT.java | 12 ++-- .../integration/DataFrameUsageIT.java | 4 +- .../DataFrameInfoTransportAction.java | 4 +- ...portGetDataFrameTransformsStatsAction.java | 42 ++++++------- .../persistence/DataFrameInternalIndex.java | 8 +-- .../DataFrameTransformsConfigManager.java | 20 +++--- ...FrameTransformPersistentTasksExecutor.java | 6 +- .../transforms/DataFrameTransformTask.java | 6 +- ...DataFrameTransformsConfigManagerTests.java | 49 +++++++-------- .../upgrades/DataFrameSurvivesUpgradeIT.java | 32 +++++----- 25 files changed, 259 insertions(+), 263 deletions(-) rename client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/{DataFrameTransformStateAndStatsInfo.java => DataFrameTransformStats.java} (78%) rename client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/{DataFrameTransformStateAndStatsInfoTests.java => DataFrameTransformStatsTests.java} (53%) rename client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/{DataFrameTransformStateAndStatsInfoTests.java => DataFrameTransformStatsTests.java} (83%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/{DataFrameTransformStateAndStatsInfo.java => DataFrameTransformStats.java} (75%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/{DataFrameTransformStateAndStats.java => DataFrameTransformStoredDoc.java} (88%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/{DataFrameTransformStateAndStatsInfoTests.java => DataFrameTransformStatsTests.java} (64%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/{DataFrameTransformStateAndStatsTests.java => DataFrameTransformStoredDocTests.java} (59%) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponse.java index 5f3829b846a18..fe4698257ea88 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponse.java @@ -21,7 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.TaskOperationFailure; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -42,11 +42,11 @@ public class GetDataFrameTransformStatsResponse { @SuppressWarnings("unchecked") static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "get_data_frame_transform_stats_response", true, - args -> new GetDataFrameTransformStatsResponse((List) args[0], + args -> new GetDataFrameTransformStatsResponse((List) args[0], (List) args[1], (List) args[2])); static { - PARSER.declareObjectArray(constructorArg(), DataFrameTransformStateAndStatsInfo.PARSER::apply, TRANSFORMS); + PARSER.declareObjectArray(constructorArg(), DataFrameTransformStats.PARSER::apply, TRANSFORMS); // Discard the count field which is the size of the transforms array PARSER.declareInt((a, b) -> {}, COUNT); PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> TaskOperationFailure.fromXContent(p), @@ -59,20 +59,20 @@ public static GetDataFrameTransformStatsResponse fromXContent(final XContentPars return GetDataFrameTransformStatsResponse.PARSER.apply(parser, null); } - private final List transformsStateAndStats; + private final List transformsStats; private final List taskFailures; private final List nodeFailures; - public GetDataFrameTransformStatsResponse(List transformsStateAndStats, + public GetDataFrameTransformStatsResponse(List transformsStats, @Nullable List taskFailures, @Nullable List nodeFailures) { - this.transformsStateAndStats = transformsStateAndStats; + this.transformsStats = transformsStats; this.taskFailures = taskFailures == null ? Collections.emptyList() : Collections.unmodifiableList(taskFailures); this.nodeFailures = nodeFailures == null ? Collections.emptyList() : Collections.unmodifiableList(nodeFailures); } - public List getTransformsStateAndStats() { - return transformsStateAndStats; + public List getTransformsStats() { + return transformsStats; } public List getNodeFailures() { @@ -85,7 +85,7 @@ public List getTaskFailures() { @Override public int hashCode() { - return Objects.hash(transformsStateAndStats, nodeFailures, taskFailures); + return Objects.hash(transformsStats, nodeFailures, taskFailures); } @Override @@ -99,7 +99,7 @@ public boolean equals(Object other) { } final GetDataFrameTransformStatsResponse that = (GetDataFrameTransformStatsResponse) other; - return Objects.equals(this.transformsStateAndStats, that.transformsStateAndStats) + return Objects.equals(this.transformsStats, that.transformsStats) && Objects.equals(this.nodeFailures, that.nodeFailures) && Objects.equals(this.taskFailures, that.taskFailures); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStats.java similarity index 78% rename from client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java rename to client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStats.java index 4f07cb3b1b0eb..4d83d36c10986 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStats.java @@ -30,7 +30,7 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class DataFrameTransformStateAndStatsInfo { +public class DataFrameTransformStats { public static final ParseField ID = new ParseField("id"); public static final ParseField TASK_STATE_FIELD = new ParseField("task_state"); @@ -39,9 +39,9 @@ public class DataFrameTransformStateAndStatsInfo { public static final ParseField STATS_FIELD = new ParseField("stats"); public static final ParseField CHECKPOINTING_INFO_FIELD = new ParseField("checkpointing"); - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "data_frame_transform_state_and_stats_info", true, - a -> new DataFrameTransformStateAndStatsInfo((String) a[0], (DataFrameTransformTaskState) a[1], (String) a[2], + a -> new DataFrameTransformStats((String) a[0], (DataFrameTransformTaskState) a[1], (String) a[2], (NodeAttributes) a[3], (DataFrameIndexerTransformStats) a[4], (DataFrameTransformCheckpointingInfo) a[5])); static { @@ -55,7 +55,7 @@ public class DataFrameTransformStateAndStatsInfo { (p, c) -> DataFrameTransformCheckpointingInfo.fromXContent(p), CHECKPOINTING_INFO_FIELD); } - public static DataFrameTransformStateAndStatsInfo fromXContent(XContentParser parser) throws IOException { + public static DataFrameTransformStats fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } @@ -63,17 +63,17 @@ public static DataFrameTransformStateAndStatsInfo fromXContent(XContentParser pa private final String reason; private final DataFrameTransformTaskState taskState; private final NodeAttributes node; - private final DataFrameIndexerTransformStats transformStats; + private final DataFrameIndexerTransformStats indexerStats; private final DataFrameTransformCheckpointingInfo checkpointingInfo; - public DataFrameTransformStateAndStatsInfo(String id, DataFrameTransformTaskState taskState, String reason, NodeAttributes node, - DataFrameIndexerTransformStats stats, - DataFrameTransformCheckpointingInfo checkpointingInfo) { + public DataFrameTransformStats(String id, DataFrameTransformTaskState taskState, String reason, NodeAttributes node, + DataFrameIndexerTransformStats stats, + DataFrameTransformCheckpointingInfo checkpointingInfo) { this.id = id; this.taskState = taskState; this.reason = reason; this.node = node; - this.transformStats = stats; + this.indexerStats = stats; this.checkpointingInfo = checkpointingInfo; } @@ -93,8 +93,8 @@ public NodeAttributes getNode() { return node; } - public DataFrameIndexerTransformStats getTransformStats() { - return transformStats; + public DataFrameIndexerTransformStats getIndexerStats() { + return indexerStats; } public DataFrameTransformCheckpointingInfo getCheckpointingInfo() { @@ -103,7 +103,7 @@ public DataFrameTransformCheckpointingInfo getCheckpointingInfo() { @Override public int hashCode() { - return Objects.hash(id, taskState, reason, node, transformStats, checkpointingInfo); + return Objects.hash(id, taskState, reason, node, indexerStats, checkpointingInfo); } @Override @@ -116,13 +116,13 @@ public boolean equals(Object other) { return false; } - DataFrameTransformStateAndStatsInfo that = (DataFrameTransformStateAndStatsInfo) other; + DataFrameTransformStats that = (DataFrameTransformStats) other; return Objects.equals(this.id, that.id) && Objects.equals(this.taskState, that.taskState) && Objects.equals(this.reason, that.reason) && Objects.equals(this.node, that.node) - && Objects.equals(this.transformStats, that.transformStats) + && Objects.equals(this.indexerStats, that.indexerStats) && Objects.equals(this.checkpointingInfo, that.checkpointingInfo); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java index 64d4f7f57e524..8c6778464867f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java @@ -41,7 +41,7 @@ import org.elasticsearch.client.dataframe.StopDataFrameTransformResponse; import org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.client.dataframe.transforms.DestConfig; import org.elasticsearch.client.dataframe.transforms.SourceConfig; @@ -258,8 +258,8 @@ public void testStartStop() throws IOException { GetDataFrameTransformStatsResponse statsResponse = execute(new GetDataFrameTransformStatsRequest(id), client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync); - assertThat(statsResponse.getTransformsStateAndStats(), hasSize(1)); - DataFrameTransformTaskState taskState = statsResponse.getTransformsStateAndStats().get(0).getTaskState(); + assertThat(statsResponse.getTransformsStats(), hasSize(1)); + DataFrameTransformTaskState taskState = statsResponse.getTransformsStats().get(0).getTaskState(); // Since we are non-continuous, the transform could auto-stop between being started earlier and us gathering the statistics assertThat(taskState, is(oneOf(DataFrameTransformTaskState.STARTED, DataFrameTransformTaskState.STOPPED))); @@ -274,7 +274,7 @@ public void testStartStop() throws IOException { // Calling stop with wait_for_completion assures that we will be in the `STOPPED` state for the transform task statsResponse = execute(new GetDataFrameTransformStatsRequest(id), client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync); - taskState = statsResponse.getTransformsStateAndStats().get(0).getTaskState(); + taskState = statsResponse.getTransformsStats().get(0).getTaskState(); assertThat(taskState, is(DataFrameTransformTaskState.STOPPED)); } @@ -353,13 +353,13 @@ public void testGetStats() throws Exception { GetDataFrameTransformStatsResponse statsResponse = execute(new GetDataFrameTransformStatsRequest(id), client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync); - assertEquals(1, statsResponse.getTransformsStateAndStats().size()); - DataFrameTransformStateAndStatsInfo stats = statsResponse.getTransformsStateAndStats().get(0); + assertEquals(1, statsResponse.getTransformsStats().size()); + DataFrameTransformStats stats = statsResponse.getTransformsStats().get(0); assertEquals(DataFrameTransformTaskState.STOPPED, stats.getTaskState()); assertEquals(IndexerState.STOPPED, stats.getCheckpointingInfo().getNext().getIndexerState()); DataFrameIndexerTransformStats zeroIndexerStats = new DataFrameIndexerTransformStats(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L); - assertEquals(zeroIndexerStats, stats.getTransformStats()); + assertEquals(zeroIndexerStats, stats.getIndexerStats()); // start the transform StartDataFrameTransformResponse startTransformResponse = execute(new StartDataFrameTransformRequest(id), @@ -369,8 +369,8 @@ public void testGetStats() throws Exception { assertBusy(() -> { GetDataFrameTransformStatsResponse response = execute(new GetDataFrameTransformStatsRequest(id), client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync); - DataFrameTransformStateAndStatsInfo stateAndStats = response.getTransformsStateAndStats().get(0); - assertNotEquals(zeroIndexerStats, stateAndStats.getTransformStats()); + DataFrameTransformStats stateAndStats = response.getTransformsStats().get(0); + assertNotEquals(zeroIndexerStats, stateAndStats.getIndexerStats()); assertThat(stateAndStats.getTaskState(), is(oneOf(DataFrameTransformTaskState.STARTED, DataFrameTransformTaskState.STOPPED))); assertThat(stateAndStats.getCheckpointingInfo().getNext().getIndexerState(), diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponseTests.java index ea808cf90b47a..3a10a162ea94b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsResponseTests.java @@ -21,8 +21,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.TaskOperationFailure; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfoTests; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStatsTests; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; @@ -48,9 +48,9 @@ public void testXContentParser() throws IOException { private static GetDataFrameTransformStatsResponse createTestInstance() { int count = randomIntBetween(1, 3); - List stats = new ArrayList<>(); + List stats = new ArrayList<>(); for (int i=0; i taskFailures = null; @@ -77,8 +77,8 @@ private static void toXContent(GetDataFrameTransformStatsResponse response, XCon builder.startObject(); { builder.startArray("transforms"); - for (DataFrameTransformStateAndStatsInfo stats : response.getTransformsStateAndStats()) { - DataFrameTransformStateAndStatsInfoTests.toXContent(stats, builder); + for (DataFrameTransformStats stats : response.getTransformsStats()) { + DataFrameTransformStatsTests.toXContent(stats, builder); } builder.endArray(); @@ -92,7 +92,7 @@ private static void toXContent(GetDataFrameTransformStatsResponse response, XCon // the object so use a custom compare method rather than Object.equals private static void assertEqualInstances(GetDataFrameTransformStatsResponse expected, GetDataFrameTransformStatsResponse actual) { - assertEquals(expected.getTransformsStateAndStats(), actual.getTransformsStateAndStats()); + assertEquals(expected.getTransformsStats(), actual.getTransformsStats()); AcknowledgedTasksResponseTests.assertTaskOperationFailuresEqual(expected.getTaskFailures(), actual.getTaskFailures()); AcknowledgedTasksResponseTests.assertNodeFailuresEqual(expected.getNodeFailures(), actual.getNodeFailures()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStatsTests.java similarity index 53% rename from client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java rename to client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStatsTests.java index 73d71fe0e7690..af3bf53704ade 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStatsTests.java @@ -27,20 +27,20 @@ import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; -public class DataFrameTransformStateAndStatsInfoTests extends ESTestCase { +public class DataFrameTransformStatsTests extends ESTestCase { public void testFromXContent() throws IOException { xContentTester(this::createParser, - DataFrameTransformStateAndStatsInfoTests::randomInstance, - DataFrameTransformStateAndStatsInfoTests::toXContent, - DataFrameTransformStateAndStatsInfo::fromXContent) + DataFrameTransformStatsTests::randomInstance, + DataFrameTransformStatsTests::toXContent, + DataFrameTransformStats::fromXContent) .supportsUnknownFields(true) .randomFieldsExcludeFilter(field -> field.equals("node.attributes") || field.contains("position")) .test(); } - public static DataFrameTransformStateAndStatsInfo randomInstance() { - return new DataFrameTransformStateAndStatsInfo(randomAlphaOfLength(10), + public static DataFrameTransformStats randomInstance() { + return new DataFrameTransformStats(randomAlphaOfLength(10), randomBoolean() ? null : randomFrom(DataFrameTransformTaskState.values()), randomBoolean() ? null : randomAlphaOfLength(100), randomBoolean() ? null : NodeAttributesTests.createRandom(), @@ -48,25 +48,25 @@ public static DataFrameTransformStateAndStatsInfo randomInstance() { randomBoolean() ? null : DataFrameTransformCheckpointingInfoTests.randomDataFrameTransformCheckpointingInfo()); } - public static void toXContent(DataFrameTransformStateAndStatsInfo stateAndStatsInfo, XContentBuilder builder) throws IOException { + public static void toXContent(DataFrameTransformStats stats, XContentBuilder builder) throws IOException { builder.startObject(); - builder.field(DataFrameTransformStateAndStatsInfo.ID.getPreferredName(), stateAndStatsInfo.getId()); - if (stateAndStatsInfo.getTaskState() != null) { - builder.field(DataFrameTransformStateAndStatsInfo.TASK_STATE_FIELD.getPreferredName(), - stateAndStatsInfo.getTaskState().value()); + builder.field(DataFrameTransformStats.ID.getPreferredName(), stats.getId()); + if (stats.getTaskState() != null) { + builder.field(DataFrameTransformStats.TASK_STATE_FIELD.getPreferredName(), + stats.getTaskState().value()); } - if (stateAndStatsInfo.getReason() != null) { - builder.field(DataFrameTransformStateAndStatsInfo.REASON_FIELD.getPreferredName(), stateAndStatsInfo.getReason()); + if (stats.getReason() != null) { + builder.field(DataFrameTransformStats.REASON_FIELD.getPreferredName(), stats.getReason()); } - if (stateAndStatsInfo.getNode() != null) { - builder.field(DataFrameTransformStateAndStatsInfo.NODE_FIELD.getPreferredName()); - stateAndStatsInfo.getNode().toXContent(builder, ToXContent.EMPTY_PARAMS); + if (stats.getNode() != null) { + builder.field(DataFrameTransformStats.NODE_FIELD.getPreferredName()); + stats.getNode().toXContent(builder, ToXContent.EMPTY_PARAMS); } - builder.field(DataFrameTransformStateAndStatsInfo.STATS_FIELD.getPreferredName()); - DataFrameIndexerTransformStatsTests.toXContent(stateAndStatsInfo.getTransformStats(), builder); - if (stateAndStatsInfo.getCheckpointingInfo() != null) { - builder.field(DataFrameTransformStateAndStatsInfo.CHECKPOINTING_INFO_FIELD.getPreferredName()); - DataFrameTransformCheckpointingInfoTests.toXContent(stateAndStatsInfo.getCheckpointingInfo(), builder); + builder.field(DataFrameTransformStats.STATS_FIELD.getPreferredName()); + DataFrameIndexerTransformStatsTests.toXContent(stats.getIndexerStats(), builder); + if (stats.getCheckpointingInfo() != null) { + builder.field(DataFrameTransformStats.CHECKPOINTING_INFO_FIELD.getPreferredName()); + DataFrameTransformCheckpointingInfoTests.toXContent(stats.getCheckpointingInfo(), builder); } builder.endObject(); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStatsTests.java similarity index 83% rename from client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsInfoTests.java rename to client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStatsTests.java index 8ef5bba8c86d6..18b057883f407 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsInfoTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStatsTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.xpack.core.dataframe.transforms.NodeAttributes; import org.elasticsearch.xpack.core.indexing.IndexerState; @@ -35,8 +35,8 @@ import java.util.Map; import java.util.function.Predicate; -public class DataFrameTransformStateAndStatsInfoTests extends AbstractHlrcXContentTestCase { +public class DataFrameTransformStatsTests extends AbstractHlrcXContentTestCase { public static NodeAttributes fromHlrc(org.elasticsearch.client.dataframe.transforms.NodeAttributes attributes) { return attributes == null ? null : new NodeAttributes(attributes.getId(), @@ -46,36 +46,36 @@ public static NodeAttributes fromHlrc(org.elasticsearch.client.dataframe.transfo attributes.getAttributes()); } - public static DataFrameTransformStateAndStatsInfo - fromHlrc(org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo instance) { + public static DataFrameTransformStats + fromHlrc(org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats instance) { - return new DataFrameTransformStateAndStatsInfo(instance.getId(), + return new DataFrameTransformStats(instance.getId(), DataFrameTransformTaskState.fromString(instance.getTaskState().value()), instance.getReason(), fromHlrc(instance.getNode()), - DataFrameIndexerTransformStatsTests.fromHlrc(instance.getTransformStats()), + DataFrameIndexerTransformStatsTests.fromHlrc(instance.getIndexerStats()), DataFrameTransformCheckpointingInfoTests.fromHlrc(instance.getCheckpointingInfo())); } @Override - public org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo doHlrcParseInstance(XContentParser parser) + public org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats doHlrcParseInstance(XContentParser parser) throws IOException { - return org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo.fromXContent(parser); + return org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats.fromXContent(parser); } @Override - public DataFrameTransformStateAndStatsInfo convertHlrcToInternal( - org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo instance) { - return new DataFrameTransformStateAndStatsInfo(instance.getId(), + public DataFrameTransformStats convertHlrcToInternal( + org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats instance) { + return new DataFrameTransformStats(instance.getId(), DataFrameTransformTaskState.fromString(instance.getTaskState().value()), instance.getReason(), fromHlrc(instance.getNode()), - DataFrameIndexerTransformStatsTests.fromHlrc(instance.getTransformStats()), + DataFrameIndexerTransformStatsTests.fromHlrc(instance.getIndexerStats()), DataFrameTransformCheckpointingInfoTests.fromHlrc(instance.getCheckpointingInfo())); } - public static DataFrameTransformStateAndStatsInfo randomDataFrameTransformStateAndStatsInfo() { - return new DataFrameTransformStateAndStatsInfo(randomAlphaOfLength(10), + public static DataFrameTransformStats randomDataFrameTransformStats() { + return new DataFrameTransformStats(randomAlphaOfLength(10), randomFrom(DataFrameTransformTaskState.values()), randomBoolean() ? null : randomAlphaOfLength(100), randomBoolean() ? null : randomNodeAttributes(), @@ -87,13 +87,13 @@ public static DataFrameTransformStateAndStatsInfo randomDataFrameTransformStateA } @Override - protected DataFrameTransformStateAndStatsInfo createTestInstance() { - return randomDataFrameTransformStateAndStatsInfo(); + protected DataFrameTransformStats createTestInstance() { + return randomDataFrameTransformStats(); } @Override - protected DataFrameTransformStateAndStatsInfo doParseInstance(XContentParser parser) throws IOException { - return DataFrameTransformStateAndStatsInfo.PARSER.apply(parser, null); + protected DataFrameTransformStats doParseInstance(XContentParser parser) throws IOException { + return DataFrameTransformStats.PARSER.apply(parser, null); } @Override @@ -151,4 +151,3 @@ protected String[] getShuffleFieldsExceptions() { return new String[] { "position" }; } } - diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java index 5886c639a9f9f..13a495a0d1fd9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java @@ -42,7 +42,7 @@ import org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformProgress; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.client.dataframe.transforms.DestConfig; import org.elasticsearch.client.dataframe.transforms.NodeAttributes; @@ -522,18 +522,18 @@ public void testGetStats() throws IOException, InterruptedException { .getDataFrameTransformStats(request, RequestOptions.DEFAULT); // end::get-data-frame-transform-stats-execute - assertThat(response.getTransformsStateAndStats(), hasSize(1)); + assertThat(response.getTransformsStats(), hasSize(1)); // tag::get-data-frame-transform-stats-response - DataFrameTransformStateAndStatsInfo stateAndStatsInfo = - response.getTransformsStateAndStats().get(0); // <1> + DataFrameTransformStats stateAndStatsInfo = + response.getTransformsStats().get(0); // <1> DataFrameTransformTaskState taskState = stateAndStatsInfo.getTaskState(); // <2> IndexerState indexerState = stateAndStatsInfo.getCheckpointingInfo() .getNext().getIndexerState(); // <3> DataFrameIndexerTransformStats transformStats = - stateAndStatsInfo.getTransformStats(); // <4> + stateAndStatsInfo.getIndexerStats(); // <4> DataFrameTransformProgress progress = stateAndStatsInfo.getCheckpointingInfo() .getNext().getCheckpointProgress(); // <5> diff --git a/docs/java-rest/high-level/dataframe/get_data_frame_stats.asciidoc b/docs/java-rest/high-level/dataframe/get_data_frame_stats.asciidoc index 8a3e1a96acb14..dbed7971ff277 100644 --- a/docs/java-rest/high-level/dataframe/get_data_frame_stats.asciidoc +++ b/docs/java-rest/high-level/dataframe/get_data_frame_stats.asciidoc @@ -47,10 +47,10 @@ The returned +{response}+ contains the requested {dataframe-transform} statistic -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-response] -------------------------------------------------- -<1> The response contains a list of `DataFrameTransformStateAndStats` objects +<1> The response contains a list of `DataFrameTransformStats` objects <2> The running state of the transform task e.g `started` <3> The running state of the transform indexer e.g `started`, `indexing`, etc. <4> The overall transform statistics recording the number of documents indexed etc. <5> The progress of the current run in the transform. Supplies the number of docs left until the next checkpoint and the total number of docs expected. -<6> The assigned node information if the task is currently assigned to a node and running. \ No newline at end of file +<6> The assigned node information if the task is currently assigned to a node and running. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java index 0526ba09498ff..a8eff918b3d4c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.action.util.PageParams; import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.dataframe.DataFrameField; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStats; import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper; import java.io.IOException; @@ -152,51 +152,51 @@ public boolean equals(Object obj) { } public static class Response extends BaseTasksResponse implements ToXContentObject { - private final QueryPage transformsStateAndStatsInfo; + private final QueryPage transformsStats; - public Response(List transformStateAndStats, long count) { + public Response(List transformStateAndStats, long count) { this(new QueryPage<>(transformStateAndStats, count, DataFrameField.TRANSFORMS)); } - public Response(List transformStateAndStats, + public Response(List transformStateAndStats, long count, List taskFailures, List nodeFailures) { this(new QueryPage<>(transformStateAndStats, count, DataFrameField.TRANSFORMS), taskFailures, nodeFailures); } - private Response(QueryPage transformsStateAndStatsInfo) { - this(transformsStateAndStatsInfo, Collections.emptyList(), Collections.emptyList()); + private Response(QueryPage transformsStats) { + this(transformsStats, Collections.emptyList(), Collections.emptyList()); } - private Response(QueryPage transformsStateAndStatsInfo, + private Response(QueryPage transformsStats, List taskFailures, List nodeFailures) { super(taskFailures, nodeFailures); - this.transformsStateAndStatsInfo = ExceptionsHelper.requireNonNull(transformsStateAndStatsInfo, "transformsStateAndStatsInfo"); + this.transformsStats = ExceptionsHelper.requireNonNull(transformsStats, "transformsStats"); } public Response(StreamInput in) throws IOException { super(in); if (in.getVersion().onOrAfter(Version.V_7_3_0)) { - transformsStateAndStatsInfo = new QueryPage<>(in, DataFrameTransformStateAndStatsInfo::new); + transformsStats = new QueryPage<>(in, DataFrameTransformStats::new); } else { - List stats = in.readList(DataFrameTransformStateAndStatsInfo::new); - transformsStateAndStatsInfo = new QueryPage<>(stats, stats.size(), DataFrameField.TRANSFORMS); + List stats = in.readList(DataFrameTransformStats::new); + transformsStats = new QueryPage<>(stats, stats.size(), DataFrameField.TRANSFORMS); } } - public List getTransformsStateAndStatsInfo() { - return transformsStateAndStatsInfo.results(); + public List getTransformsStats() { + return transformsStats.results(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); if (out.getVersion().onOrAfter(Version.V_7_3_0)) { - transformsStateAndStatsInfo.writeTo(out); + transformsStats.writeTo(out); } else { - out.writeList(transformsStateAndStatsInfo.results()); + out.writeList(transformsStats.results()); } } @@ -209,14 +209,14 @@ public void readFrom(StreamInput in) { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); toXContentCommon(builder, params); - transformsStateAndStatsInfo.doXContentBody(builder, params); + transformsStats.doXContentBody(builder, params); builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(transformsStateAndStatsInfo); + return Objects.hash(transformsStats); } @Override @@ -230,7 +230,7 @@ public boolean equals(Object other) { } final Response that = (Response) other; - return Objects.equals(this.transformsStateAndStatsInfo, that.transformsStateAndStatsInfo); + return Objects.equals(this.transformsStats, that.transformsStats); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java similarity index 75% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java index ef97e70e06fd4..144c8d684bd79 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java @@ -28,12 +28,12 @@ /** * Used as a wrapper for the objects returned from the stats endpoint. - * Objects of this class are expected to be ephermeral. + * Objects of this class are expected to be ephemeral. * Do not persist objects of this class to cluster state or an index. */ -public class DataFrameTransformStateAndStatsInfo implements Writeable, ToXContentObject { +public class DataFrameTransformStats implements Writeable, ToXContentObject { - public static final String NAME = "data_frame_transform_state_and_stats_info"; + public static final String NAME = "data_frame_transform_stats"; public static final ParseField TASK_STATE_FIELD = new ParseField("task_state"); public static final ParseField REASON_FIELD = new ParseField("reason"); public static final ParseField NODE_FIELD = new ParseField("node"); @@ -45,13 +45,13 @@ public class DataFrameTransformStateAndStatsInfo implements Writeable, ToXConten private String reason; @Nullable private NodeAttributes node; - private final DataFrameIndexerTransformStats transformStats; + private final DataFrameIndexerTransformStats indexerStats; private final DataFrameTransformCheckpointingInfo checkpointingInfo; - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, true, - a -> new DataFrameTransformStateAndStatsInfo((String) a[0], + a -> new DataFrameTransformStats((String) a[0], (DataFrameTransformTaskState) a[1], (String) a[2], (NodeAttributes) a[3], @@ -70,17 +70,16 @@ public class DataFrameTransformStateAndStatsInfo implements Writeable, ToXConten (p, c) -> DataFrameTransformCheckpointingInfo.fromXContent(p), CHECKPOINTING_INFO_FIELD); } - public static DataFrameTransformStateAndStatsInfo fromXContent(XContentParser parser) throws IOException { + public static DataFrameTransformStats fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } - public static DataFrameTransformStateAndStatsInfo initialStateAndStatsInfo(String id) { - return stoppedStateAndStatsInfo(id, new DataFrameIndexerTransformStats(id)); + public static DataFrameTransformStats initialStats(String id) { + return stoppedStats(id, new DataFrameIndexerTransformStats(id)); } - public static DataFrameTransformStateAndStatsInfo stoppedStateAndStatsInfo(String id, - DataFrameIndexerTransformStats indexerTransformStats) { - return new DataFrameTransformStateAndStatsInfo(id, + public static DataFrameTransformStats stoppedStats(String id, DataFrameIndexerTransformStats indexerTransformStats) { + return new DataFrameTransformStats(id, DataFrameTransformTaskState.STOPPED, null, null, @@ -89,18 +88,18 @@ public static DataFrameTransformStateAndStatsInfo stoppedStateAndStatsInfo(Strin } - public DataFrameTransformStateAndStatsInfo(String id, DataFrameTransformTaskState taskState, @Nullable String reason, - @Nullable NodeAttributes node, DataFrameIndexerTransformStats stats, - DataFrameTransformCheckpointingInfo checkpointingInfo) { + public DataFrameTransformStats(String id, DataFrameTransformTaskState taskState, @Nullable String reason, + @Nullable NodeAttributes node, DataFrameIndexerTransformStats stats, + DataFrameTransformCheckpointingInfo checkpointingInfo) { this.id = Objects.requireNonNull(id); this.taskState = Objects.requireNonNull(taskState); this.reason = reason; this.node = node; - this.transformStats = Objects.requireNonNull(stats); + this.indexerStats = Objects.requireNonNull(stats); this.checkpointingInfo = Objects.requireNonNull(checkpointingInfo); } - public DataFrameTransformStateAndStatsInfo(StreamInput in) throws IOException { + public DataFrameTransformStats(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport this.id = in.readString(); this.taskState = in.readEnum(DataFrameTransformTaskState.class); @@ -109,16 +108,17 @@ public DataFrameTransformStateAndStatsInfo(StreamInput in) throws IOException { } else { this.node = null; } - this.transformStats = new DataFrameIndexerTransformStats(in); + this.indexerStats = new DataFrameIndexerTransformStats(in); this.checkpointingInfo = new DataFrameTransformCheckpointingInfo(in); } else { - // Prior to version 7.4 DataFrameTransformStateAndStatsInfo didn't exist, and we have - // to do the best we can of reading from a DataFrameTransformStateAndStats object + // Prior to version 7.4 DataFrameTransformStats didn't exist, and we have + // to do the best we can of reading from a DataFrameTransformStoredDoc object + // (which is called DataFrameTransformStateAndStats in 7.2/7.3) this.id = in.readString(); this.taskState = new DataFrameTransformState(in).getTaskState(); this.node = null; - this.transformStats = new DataFrameIndexerTransformStats(in); + this.indexerStats = new DataFrameIndexerTransformStats(in); this.checkpointingInfo = new DataFrameTransformCheckpointingInfo(in); } } @@ -131,7 +131,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (node != null) { builder.field(NODE_FIELD.getPreferredName(), node); } - builder.field(DataFrameField.STATS_FIELD.getPreferredName(), transformStats, params); + builder.field(DataFrameField.STATS_FIELD.getPreferredName(), indexerStats, params); builder.field(CHECKPOINTING_INFO_FIELD.getPreferredName(), checkpointingInfo, params); builder.endObject(); return builder; @@ -148,11 +148,12 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } - transformStats.writeTo(out); + indexerStats.writeTo(out); checkpointingInfo.writeTo(out); } else { - // Prior to version 7.4 DataFrameTransformStateAndStatsInfo didn't exist, and we have - // to do the best we can of writing to a DataFrameTransformStateAndStats object + // Prior to version 7.4 DataFrameTransformStats didn't exist, and we have + // to do the best we can of writing to a DataFrameTransformStoredDoc object + // (which is called DataFrameTransformStateAndStats in 7.2/7.3) out.writeString(id); new DataFrameTransformState(taskState, checkpointingInfo.getNext().getIndexerState(), @@ -161,14 +162,14 @@ public void writeTo(StreamOutput out) throws IOException { reason, checkpointingInfo.getNext().getCheckpointProgress()).writeTo(out); out.writeBoolean(false); - transformStats.writeTo(out); + indexerStats.writeTo(out); checkpointingInfo.writeTo(out); } } @Override public int hashCode() { - return Objects.hash(id, taskState, node, transformStats, checkpointingInfo); + return Objects.hash(id, taskState, node, indexerStats, checkpointingInfo); } @Override @@ -181,12 +182,12 @@ public boolean equals(Object other) { return false; } - DataFrameTransformStateAndStatsInfo that = (DataFrameTransformStateAndStatsInfo) other; + DataFrameTransformStats that = (DataFrameTransformStats) other; return Objects.equals(this.id, that.id) && Objects.equals(this.taskState, that.taskState) && Objects.equals(this.node, that.node) - && Objects.equals(this.transformStats, that.transformStats) + && Objects.equals(this.indexerStats, that.indexerStats) && Objects.equals(this.checkpointingInfo, that.checkpointingInfo); } @@ -212,8 +213,8 @@ public void setNode(NodeAttributes node) { this.node = node; } - public DataFrameIndexerTransformStats getTransformStats() { - return transformStats; + public DataFrameIndexerTransformStats getIndexerStats() { + return indexerStats; } public DataFrameTransformCheckpointingInfo getCheckpointingInfo() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStoredDoc.java similarity index 88% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStoredDoc.java index 7d0981865e917..78bd2346688a6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStoredDoc.java @@ -25,7 +25,7 @@ * A wrapper for grouping transform state and stats when persisting to an index. * Not intended to be returned in endpoint responses. */ -public class DataFrameTransformStateAndStats implements Writeable, ToXContentObject { +public class DataFrameTransformStoredDoc implements Writeable, ToXContentObject { public static final String NAME = "data_frame_transform_state_and_stats"; public static final ParseField STATE_FIELD = new ParseField("state"); @@ -34,9 +34,9 @@ public class DataFrameTransformStateAndStats implements Writeable, ToXContentObj private final DataFrameTransformState transformState; private final DataFrameIndexerTransformStats transformStats; - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, true, - a -> new DataFrameTransformStateAndStats((String) a[0], + a -> new DataFrameTransformStoredDoc((String) a[0], (DataFrameTransformState) a[1], (DataFrameIndexerTransformStats) a[2])); @@ -47,7 +47,7 @@ public class DataFrameTransformStateAndStats implements Writeable, ToXContentObj DataFrameField.STATS_FIELD); } - public static DataFrameTransformStateAndStats fromXContent(XContentParser parser) throws IOException { + public static DataFrameTransformStoredDoc fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } @@ -60,13 +60,13 @@ public static String documentId(String transformId) { return NAME + "-" + transformId; } - public DataFrameTransformStateAndStats(String id, DataFrameTransformState state, DataFrameIndexerTransformStats stats) { + public DataFrameTransformStoredDoc(String id, DataFrameTransformState state, DataFrameIndexerTransformStats stats) { this.id = Objects.requireNonNull(id); this.transformState = Objects.requireNonNull(state); this.transformStats = Objects.requireNonNull(stats); } - public DataFrameTransformStateAndStats(StreamInput in) throws IOException { + public DataFrameTransformStoredDoc(StreamInput in) throws IOException { this.id = in.readString(); this.transformState = new DataFrameTransformState(in); this.transformStats = new DataFrameIndexerTransformStats(in); @@ -111,7 +111,7 @@ public boolean equals(Object other) { return false; } - DataFrameTransformStateAndStats that = (DataFrameTransformStateAndStats) other; + DataFrameTransformStoredDoc that = (DataFrameTransformStoredDoc) other; return Objects.equals(this.id, that.id) && Objects.equals(this.transformState, that.transformState) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java index c6943d05f7e2a..007401a4db82d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java @@ -11,8 +11,8 @@ import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction.Response; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfo; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfoTests; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStatsTests; import java.util.ArrayList; import java.util.List; @@ -20,10 +20,10 @@ public class GetDataFrameTransformsStatsActionResponseTests extends AbstractWireSerializingDataFrameTestCase { @Override protected Response createTestInstance() { - List stats = new ArrayList<>(); + List stats = new ArrayList<>(); int totalStats = randomInt(10); for (int i = 0; i < totalStats; ++i) { - stats.add(DataFrameTransformStateAndStatsInfoTests.randomDataFrameTransformStateAndStatsInfo()); + stats.add(DataFrameTransformStatsTests.randomDataFrameTransformStats()); } int totalErrors = randomInt(10); List taskFailures = new ArrayList<>(totalErrors); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStatsTests.java similarity index 64% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStatsTests.java index 9e8f0c81ec5a9..8c708e67eacab 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsInfoTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStatsTests.java @@ -13,10 +13,10 @@ import java.io.IOException; import java.util.function.Predicate; -public class DataFrameTransformStateAndStatsInfoTests extends AbstractSerializingTestCase { +public class DataFrameTransformStatsTests extends AbstractSerializingTestCase { - public static DataFrameTransformStateAndStatsInfo randomDataFrameTransformStateAndStatsInfo() { - return new DataFrameTransformStateAndStatsInfo(randomAlphaOfLength(10), + public static DataFrameTransformStats randomDataFrameTransformStats() { + return new DataFrameTransformStats(randomAlphaOfLength(10), randomFrom(DataFrameTransformTaskState.values()), randomBoolean() ? null : randomAlphaOfLength(100), randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(), @@ -25,18 +25,18 @@ public static DataFrameTransformStateAndStatsInfo randomDataFrameTransformStateA } @Override - protected DataFrameTransformStateAndStatsInfo doParseInstance(XContentParser parser) throws IOException { - return DataFrameTransformStateAndStatsInfo.fromXContent(parser); + protected DataFrameTransformStats doParseInstance(XContentParser parser) throws IOException { + return DataFrameTransformStats.fromXContent(parser); } @Override - protected DataFrameTransformStateAndStatsInfo createTestInstance() { - return randomDataFrameTransformStateAndStatsInfo(); + protected DataFrameTransformStats createTestInstance() { + return randomDataFrameTransformStats(); } @Override - protected Reader instanceReader() { - return DataFrameTransformStateAndStatsInfo::new; + protected Reader instanceReader() { + return DataFrameTransformStats::new; } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStoredDocTests.java similarity index 59% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStoredDocTests.java index 4f80d0d0b453c..4998295a14b05 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStoredDocTests.java @@ -14,24 +14,24 @@ import java.io.IOException; import java.util.Collections; -public class DataFrameTransformStateAndStatsTests extends AbstractSerializingDataFrameTestCase { +public class DataFrameTransformStoredDocTests extends AbstractSerializingDataFrameTestCase { protected static ToXContent.Params TO_XCONTENT_PARAMS = new ToXContent.MapParams( Collections.singletonMap(DataFrameField.FOR_INTERNAL_STORAGE, "true")); - public static DataFrameTransformStateAndStats randomDataFrameTransformStateAndStats(String id) { - return new DataFrameTransformStateAndStats(id, + public static DataFrameTransformStoredDoc randomDataFrameTransformStoredDoc(String id) { + return new DataFrameTransformStoredDoc(id, DataFrameTransformStateTests.randomDataFrameTransformState(), DataFrameIndexerTransformStatsTests.randomStats(id)); } - public static DataFrameTransformStateAndStats randomDataFrameTransformStateAndStats() { - return randomDataFrameTransformStateAndStats(randomAlphaOfLengthBetween(1, 10)); + public static DataFrameTransformStoredDoc randomDataFrameTransformStoredDoc() { + return randomDataFrameTransformStoredDoc(randomAlphaOfLengthBetween(1, 10)); } @Override - protected DataFrameTransformStateAndStats doParseInstance(XContentParser parser) throws IOException { - return DataFrameTransformStateAndStats.PARSER.apply(parser, null); + protected DataFrameTransformStoredDoc doParseInstance(XContentParser parser) throws IOException { + return DataFrameTransformStoredDoc.PARSER.apply(parser, null); } @Override @@ -42,13 +42,12 @@ protected ToXContent.Params getToXContentParams() { } @Override - protected DataFrameTransformStateAndStats createTestInstance() { - return randomDataFrameTransformStateAndStats(); + protected DataFrameTransformStoredDoc createTestInstance() { + return randomDataFrameTransformStoredDoc(); } @Override - protected Reader instanceReader() { - return DataFrameTransformStateAndStats::new; + protected Reader instanceReader() { + return DataFrameTransformStoredDoc::new; } - } diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java index e1219fd7541b0..2ad9fde92d408 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java @@ -133,7 +133,7 @@ protected void waitUntilCheckpoint(String id, long checkpoint) throws Exception protected void waitUntilCheckpoint(String id, long checkpoint, TimeValue waitTime) throws Exception { assertBusy(() -> assertEquals(checkpoint, getDataFrameTransformStats(id) - .getTransformsStateAndStats() + .getTransformsStats() .get(0) .getCheckpointingInfo() .getNext() diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java index 5cf161785149d..17c4f9d900467 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java @@ -66,7 +66,7 @@ public void testDataFrameTransformCrud() throws Exception { // It will eventually be stopped assertBusy(() -> assertThat(getDataFrameTransformStats(config.getId()) - .getTransformsStateAndStats().get(0).getCheckpointingInfo().getNext().getIndexerState(), equalTo(IndexerState.STOPPED))); + .getTransformsStats().get(0).getCheckpointingInfo().getNext().getIndexerState(), equalTo(IndexerState.STOPPED))); stopDataFrameTransform(config.getId()); DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0); @@ -102,13 +102,13 @@ public void testContinuousDataFrameTransformCrud() throws Exception { assertTrue(startDataFrameTransform(config.getId(), RequestOptions.DEFAULT).isAcknowledged()); waitUntilCheckpoint(config.getId(), 1L); - assertThat(getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0).getTaskState(), + assertThat(getDataFrameTransformStats(config.getId()).getTransformsStats().get(0).getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); long docsIndexed = getDataFrameTransformStats(config.getId()) - .getTransformsStateAndStats() + .getTransformsStats() .get(0) - .getTransformStats() + .getIndexerStats() .getNumDocuments(); DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0); @@ -147,9 +147,9 @@ public void testContinuousDataFrameTransformCrud() throws Exception { // Assert that we wrote the new docs assertThat(getDataFrameTransformStats(config.getId()) - .getTransformsStateAndStats() + .getTransformsStats() .get(0) - .getTransformStats() + .getIndexerStats() .getNumDocuments(), greaterThan(docsIndexed)); stopDataFrameTransform(config.getId()); diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameUsageIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameUsageIT.java index 9295fd908233d..f195005165b43 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameUsageIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameUsageIT.java @@ -11,7 +11,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDoc; import org.elasticsearch.xpack.dataframe.persistence.DataFrameInternalIndex; import org.junit.Before; @@ -56,7 +56,7 @@ public void testUsage() throws Exception { Request statsExistsRequest = new Request("GET", DataFrameInternalIndex.INDEX_NAME+"/_search?q=" + INDEX_DOC_TYPE.getPreferredName() + ":" + - DataFrameTransformStateAndStats.NAME); + DataFrameTransformStoredDoc.NAME); // Verify that we have one stat document assertBusy(() -> { Map hasStatsMap = entityAsMap(client().performRequest(statsExistsRequest)); diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportAction.java index 4988d0e71df97..31d5fd860e446 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportAction.java @@ -30,7 +30,7 @@ import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; import org.elasticsearch.xpack.core.dataframe.DataFrameField; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDoc; import org.elasticsearch.xpack.dataframe.persistence.DataFrameInternalIndex; import java.util.ArrayList; @@ -108,7 +108,7 @@ static DataFrameIndexerTransformStats parseSearchAggs(SearchResponse searchRespo static void getStatisticSummations(Client client, ActionListener statsListener) { QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), - DataFrameTransformStateAndStats.NAME))); + DataFrameTransformStoredDoc.NAME))); SearchRequestBuilder requestBuilder = client.prepareSearch(DataFrameInternalIndex.INDEX_NAME) .setSize(0) diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java index 99cfb6101fd8e..40ade96a52e6e 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java @@ -27,8 +27,8 @@ import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction.Request; import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction.Response; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDoc; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.xpack.core.dataframe.transforms.NodeAttributes; import org.elasticsearch.xpack.dataframe.checkpoint.DataFrameTransformsCheckpointService; @@ -68,9 +68,9 @@ public TransportGetDataFrameTransformsStatsAction(TransportService transportServ @Override protected Response newResponse(Request request, List tasks, List taskOperationFailures, List failedNodeExceptions) { - List responses = tasks.stream() - .flatMap(r -> r.getTransformsStateAndStatsInfo().stream()) - .sorted(Comparator.comparing(DataFrameTransformStateAndStatsInfo::getId)) + List responses = tasks.stream() + .flatMap(r -> r.getTransformsStats().stream()) + .sorted(Comparator.comparing(DataFrameTransformStats::getId)) .collect(Collectors.toList()); List allFailedNodeExceptions = new ArrayList<>(failedNodeExceptions); allFailedNodeExceptions.addAll(tasks.stream().flatMap(r -> r.getNodeFailures().stream()).collect(Collectors.toList())); @@ -86,7 +86,7 @@ protected void taskOperation(Request request, DataFrameTransformTask task, Actio transformsCheckpointService.getCheckpointStats(task.getTransformId(), task.getCheckpoint(), task.getInProgressCheckpoint(), task.getState().getIndexerState(), task.getState().getPosition(), ActionListener.wrap(checkpointStats -> listener.onResponse(new Response( - Collections.singletonList(new DataFrameTransformStateAndStatsInfo(task.getTransformId(), + Collections.singletonList(new DataFrameTransformStats(task.getTransformId(), task.getState().getTaskState(), task.getState().getReason(), null, @@ -94,7 +94,7 @@ protected void taskOperation(Request request, DataFrameTransformTask task, Actio checkpointStats)), 1L)), e -> listener.onResponse(new Response( - Collections.singletonList(new DataFrameTransformStateAndStatsInfo(task.getTransformId(), + Collections.singletonList(new DataFrameTransformStats(task.getTransformId(), task.getState().getTaskState(), task.getState().getReason(), null, @@ -123,10 +123,10 @@ protected void doExecute(Task task, Request request, ActionListener fi PersistentTasksCustomMetaData tasksInProgress = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasksInProgress != null) { // Mutates underlying state object with the assigned node attributes - response.getTransformsStateAndStatsInfo().forEach(dtsasi -> setNodeAttributes(dtsasi, tasksInProgress, state)); + response.getTransformsStats().forEach(dtsasi -> setNodeAttributes(dtsasi, tasksInProgress, state)); } collectStatsForTransformsWithoutTasks(request, response, ActionListener.wrap( - finalResponse -> finalListener.onResponse(new Response(finalResponse.getTransformsStateAndStatsInfo(), + finalResponse -> finalListener.onResponse(new Response(finalResponse.getTransformsStats(), hitsAndIds.v1(), finalResponse.getTaskFailures(), finalResponse.getNodeFailures())), @@ -147,11 +147,11 @@ protected void doExecute(Task task, Request request, ActionListener fi )); } - private static void setNodeAttributes(DataFrameTransformStateAndStatsInfo dataFrameTransformStateAndStatsInfo, + private static void setNodeAttributes(DataFrameTransformStats dataFrameTransformStats, PersistentTasksCustomMetaData persistentTasksCustomMetaData, ClusterState state) { - var pTask = persistentTasksCustomMetaData.getTask(dataFrameTransformStateAndStatsInfo.getId()); + var pTask = persistentTasksCustomMetaData.getTask(dataFrameTransformStats.getId()); if (pTask != null) { - dataFrameTransformStateAndStatsInfo.setNode(NodeAttributes.fromDiscoveryNode(state.nodes().get(pTask.getExecutorNode()))); + dataFrameTransformStats.setNode(NodeAttributes.fromDiscoveryNode(state.nodes().get(pTask.getExecutorNode()))); } } @@ -159,26 +159,26 @@ private void collectStatsForTransformsWithoutTasks(Request request, Response response, ActionListener listener) { // We gathered all there is, no need to continue - if (request.getExpandedIds().size() == response.getTransformsStateAndStatsInfo().size()) { + if (request.getExpandedIds().size() == response.getTransformsStats().size()) { listener.onResponse(response); return; } Set transformsWithoutTasks = new HashSet<>(request.getExpandedIds()); - transformsWithoutTasks.removeAll(response.getTransformsStateAndStatsInfo().stream().map(DataFrameTransformStateAndStatsInfo::getId) + transformsWithoutTasks.removeAll(response.getTransformsStats().stream().map(DataFrameTransformStats::getId) .collect(Collectors.toList())); // Small assurance that we are at least below the max. Terms search has a hard limit of 10k, we should at least be below that. assert transformsWithoutTasks.size() <= Request.MAX_SIZE_RETURN; - ActionListener> searchStatsListener = ActionListener.wrap( + ActionListener> searchStatsListener = ActionListener.wrap( stats -> { - List allStateAndStats = response.getTransformsStateAndStatsInfo(); + List allStateAndStats = response.getTransformsStats(); // If the persistent task does NOT exist, it is STOPPED // There is a potential race condition where the saved document does not actually have a STOPPED state // as the task is cancelled before we persist state. stats.forEach(stat -> - allStateAndStats.add(new DataFrameTransformStateAndStatsInfo( + allStateAndStats.add(new DataFrameTransformStats( stat.getId(), DataFrameTransformTaskState.STOPPED, null, @@ -186,15 +186,15 @@ private void collectStatsForTransformsWithoutTasks(Request request, stat.getTransformStats(), DataFrameTransformCheckpointingInfo.EMPTY)) // TODO !!! ); - transformsWithoutTasks.removeAll(stats.stream().map(DataFrameTransformStateAndStats::getId).collect(Collectors.toSet())); + transformsWithoutTasks.removeAll(stats.stream().map(DataFrameTransformStoredDoc::getId).collect(Collectors.toSet())); // Transforms that have not been started and have no state or stats. transformsWithoutTasks.forEach(transformId -> - allStateAndStats.add(DataFrameTransformStateAndStatsInfo.initialStateAndStatsInfo(transformId))); + allStateAndStats.add(DataFrameTransformStats.initialStats(transformId))); // Any transform in collection could NOT have a task, so, even though the list is initially sorted // it can easily become arbitrarily ordered based on which transforms don't have a task or stats docs - allStateAndStats.sort(Comparator.comparing(DataFrameTransformStateAndStatsInfo::getId)); + allStateAndStats.sort(Comparator.comparing(DataFrameTransformStats::getId)); listener.onResponse(new Response(allStateAndStats, allStateAndStats.size(), @@ -210,6 +210,6 @@ private void collectStatsForTransformsWithoutTasks(Request request, } ); - dataFrameTransformsConfigManager.getTransformStateAndStats(transformsWithoutTasks, searchStatsListener); + dataFrameTransformsConfigManager.getTransformStoredDoc(transformsWithoutTasks, searchStatsListener); } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java index c7169695d0118..f3aafc2b2b4a4 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDoc; import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig; import org.elasticsearch.xpack.core.dataframe.transforms.SourceConfig; @@ -132,7 +132,7 @@ private static XContentBuilder mappings() throws IOException { // add the schema for transform configurations addDataFrameTransformsConfigMappings(builder); // add the schema for transform stats - addDataFrameTransformStateAndStatsMappings(builder); + addDataFrameTransformStoredDocMappings(builder); // end type builder.endObject(); // end properties @@ -143,9 +143,9 @@ private static XContentBuilder mappings() throws IOException { } - private static XContentBuilder addDataFrameTransformStateAndStatsMappings(XContentBuilder builder) throws IOException { + private static XContentBuilder addDataFrameTransformStoredDocMappings(XContentBuilder builder) throws IOException { return builder - .startObject(DataFrameTransformStateAndStats.STATE_FIELD.getPreferredName()) + .startObject(DataFrameTransformStoredDoc.STATE_FIELD.getPreferredName()) .startObject(PROPERTIES) .startObject(DataFrameTransformState.TASK_STATE.getPreferredName()) .field(TYPE, KEYWORD) diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java index 99cfec4e3339a..f729cf025c89a 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java @@ -47,7 +47,7 @@ import org.elasticsearch.xpack.core.dataframe.DataFrameMessages; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDoc; import java.io.IOException; import java.io.InputStream; @@ -280,13 +280,13 @@ public void deleteTransform(String transformId, ActionListener listener })); } - public void putOrUpdateTransformStateAndStats(DataFrameTransformStateAndStats stats, ActionListener listener) { + public void putOrUpdateTransformStoredDoc(DataFrameTransformStoredDoc stats, ActionListener listener) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) { XContentBuilder source = stats.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); IndexRequest indexRequest = new IndexRequest(DataFrameInternalIndex.INDEX_NAME) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .id(DataFrameTransformStateAndStats.documentId(stats.getId())) + .id(DataFrameTransformStoredDoc.documentId(stats.getId())) .source(source); executeAsyncWithOrigin(client, DATA_FRAME_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( @@ -303,8 +303,8 @@ public void putOrUpdateTransformStateAndStats(DataFrameTransformStateAndStats st } } - public void getTransformStateAndStats(String transformId, ActionListener resultListener) { - GetRequest getRequest = new GetRequest(DataFrameInternalIndex.INDEX_NAME, DataFrameTransformStateAndStats.documentId(transformId)); + public void getTransformStoredDoc(String transformId, ActionListener resultListener) { + GetRequest getRequest = new GetRequest(DataFrameInternalIndex.INDEX_NAME, DataFrameTransformStoredDoc.documentId(transformId)); executeAsyncWithOrigin(client, DATA_FRAME_ORIGIN, GetAction.INSTANCE, getRequest, ActionListener.wrap(getResponse -> { if (getResponse.isExists() == false) { @@ -316,7 +316,7 @@ public void getTransformStateAndStats(String transformId, ActionListener transformIds, ActionListener> listener) { + public void getTransformStoredDoc(Collection transformIds, ActionListener> listener) { QueryBuilder builder = QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery() .filter(QueryBuilders.termsQuery(DataFrameField.ID.getPreferredName(), transformIds)) - .filter(QueryBuilders.termQuery(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), DataFrameTransformStateAndStats.NAME))); + .filter(QueryBuilders.termQuery(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), DataFrameTransformStoredDoc.NAME))); SearchRequest searchRequest = client.prepareSearch(DataFrameInternalIndex.INDEX_NAME) .addSort(DataFrameField.ID.getPreferredName(), SortOrder.ASC) @@ -347,13 +347,13 @@ public void getTransformStateAndStats(Collection transformIds, ActionLis executeAsyncWithOrigin(client.threadPool().getThreadContext(), DATA_FRAME_ORIGIN, searchRequest, ActionListener.wrap( searchResponse -> { - List stats = new ArrayList<>(); + List stats = new ArrayList<>(); for (SearchHit hit : searchResponse.getHits().getHits()) { BytesReference source = hit.getSourceRef(); try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentType.JSON) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - stats.add(DataFrameTransformStateAndStats.fromXContent(parser)); + stats.add(DataFrameTransformStoredDoc.fromXContent(parser)); } catch (IOException e) { listener.onFailure( new ElasticsearchParseException("failed to parse data frame stats from search hit", e)); diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java index 85a58531e0563..7b7ebdc183ac9 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java @@ -32,7 +32,7 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDoc; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; @@ -176,7 +176,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable DataFrameTr // <3> Set the previous stats (if they exist), initialize the indexer, start the task (If it is STOPPED) // Since we don't create the task until `_start` is called, if we see that the task state is stopped, attempt to start // Schedule execution regardless - ActionListener transformStatsActionListener = ActionListener.wrap( + ActionListener transformStatsActionListener = ActionListener.wrap( stateAndStats -> { logger.trace("[{}] initializing state and stats: [{}]", transformId, stateAndStats.toString()); indexerBuilder.setInitialStats(stateAndStats.getTransformStats()) @@ -215,7 +215,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable DataFrameTr ActionListener> getFieldMappingsListener = ActionListener.wrap( fieldMappings -> { indexerBuilder.setFieldMappings(fieldMappings); - transformsConfigManager.getTransformStateAndStats(transformId, transformStatsActionListener); + transformsConfigManager.getTransformStoredDoc(transformId, transformStatsActionListener); }, error -> { String msg = DataFrameMessages.getMessage(DataFrameMessages.DATA_FRAME_UNABLE_TO_GATHER_FIELD_MAPPINGS, diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java index 34f5cec537739..2abfb8d5b081f 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java @@ -37,7 +37,7 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDoc; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.indexing.IndexerState; @@ -656,8 +656,8 @@ protected void doSaveState(IndexerState indexerState, DataFrameIndexerPosition p // Persisting stats when we call `doSaveState` should be ok as we only call it on a state transition and // only every-so-often when doing the bulk indexing calls. See AsyncTwoPhaseIndexer#onBulkResponse for current periodicity - transformsConfigManager.putOrUpdateTransformStateAndStats( - new DataFrameTransformStateAndStats(transformId, state, getStats()), + transformsConfigManager.putOrUpdateTransformStoredDoc( + new DataFrameTransformStoredDoc(transformId, state, getStats()), ActionListener.wrap( r -> { // for auto stop shutdown the task diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java index e860d4843a5f4..45c792f8d1166 100644 --- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java +++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java @@ -15,8 +15,8 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointTests; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfigTests; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; -import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsTests; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDoc; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStoredDocTests; import org.elasticsearch.xpack.dataframe.DataFrameSingleNodeTestCase; import org.junit.Before; @@ -245,40 +245,37 @@ public void testExpandIds() throws Exception { } - public void testStateAndStats() throws InterruptedException { - String transformId = "transform_test_stats_create_read_update"; + public void testStoredDoc() throws InterruptedException { + String transformId = "transform_test_stored_doc_create_read_update"; - DataFrameTransformStateAndStats stateAndStats = - DataFrameTransformStateAndStatsTests.randomDataFrameTransformStateAndStats(transformId); + DataFrameTransformStoredDoc storedDocs = DataFrameTransformStoredDocTests.randomDataFrameTransformStoredDoc(transformId); - assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStateAndStats(stateAndStats, listener), - Boolean.TRUE, null, null); - assertAsync(listener -> transformsConfigManager.getTransformStateAndStats(transformId, listener), stateAndStats, null, null); + assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStoredDoc(storedDocs, listener), Boolean.TRUE, null, null); + assertAsync(listener -> transformsConfigManager.getTransformStoredDoc(transformId, listener), storedDocs, null, null); - DataFrameTransformStateAndStats updated = - DataFrameTransformStateAndStatsTests.randomDataFrameTransformStateAndStats(transformId); - assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStateAndStats(updated, listener), Boolean.TRUE, null, null); - assertAsync(listener -> transformsConfigManager.getTransformStateAndStats(transformId, listener), updated, null, null); + DataFrameTransformStoredDoc updated = DataFrameTransformStoredDocTests.randomDataFrameTransformStoredDoc(transformId); + assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStoredDoc(updated, listener), Boolean.TRUE, null, null); + assertAsync(listener -> transformsConfigManager.getTransformStoredDoc(transformId, listener), updated, null, null); } - public void testGetStateAndStatsMultiple() throws InterruptedException { + public void testGetStoredDocMultiple() throws InterruptedException { int numStats = randomIntBetween(10, 15); - List expectedStats = new ArrayList<>(); + List expectedDocs = new ArrayList<>(); for (int i=0; i transformsConfigManager.putOrUpdateTransformStateAndStats(stat, listener), Boolean.TRUE, null, null); + DataFrameTransformStoredDoc stat = + DataFrameTransformStoredDocTests.randomDataFrameTransformStoredDoc(randomAlphaOfLength(6)); + expectedDocs.add(stat); + assertAsync(listener -> transformsConfigManager.putOrUpdateTransformStoredDoc(stat, listener), Boolean.TRUE, null, null); } - // remove one of the put stats so we don't retrieve all - if (expectedStats.size() > 1) { - expectedStats.remove(expectedStats.size() -1); + // remove one of the put docs so we don't retrieve all + if (expectedDocs.size() > 1) { + expectedDocs.remove(expectedDocs.size() - 1); } - List ids = expectedStats.stream().map(DataFrameTransformStateAndStats::getId).collect(Collectors.toList()); + List ids = expectedDocs.stream().map(DataFrameTransformStoredDoc::getId).collect(Collectors.toList()); - // get stats will be ordered by id - expectedStats.sort(Comparator.comparing(DataFrameTransformStateAndStats::getId)); - assertAsync(listener -> transformsConfigManager.getTransformStateAndStats(ids, listener), expectedStats, null, null); + // returned docs will be ordered by id + expectedDocs.sort(Comparator.comparing(DataFrameTransformStoredDoc::getId)); + assertAsync(listener -> transformsConfigManager.getTransformStoredDoc(ids, listener), expectedDocs, null, null); } } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java index 508340aed2ea4..d4ebbb83a26bc 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java @@ -13,7 +13,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsResponse; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; -import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStatsInfo; +import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStats; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.client.dataframe.transforms.DestConfig; import org.elasticsearch.client.dataframe.transforms.SourceConfig; @@ -135,10 +135,10 @@ private void createAndStartContinuousDataFrame() throws Exception { startTransform(CONTINUOUS_DATA_FRAME_ID); waitUntilAfterCheckpoint(CONTINUOUS_DATA_FRAME_ID, 0L); - DataFrameTransformStateAndStatsInfo stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); + DataFrameTransformStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); - assertThat(stateAndStats.getTransformStats().getOutputDocuments(), equalTo((long)ENTITIES.size())); - assertThat(stateAndStats.getTransformStats().getNumDocuments(), equalTo(totalDocsWritten)); + assertThat(stateAndStats.getIndexerStats().getOutputDocuments(), equalTo((long)ENTITIES.size())); + assertThat(stateAndStats.getIndexerStats().getNumDocuments(), equalTo(totalDocsWritten)); assertThat(stateAndStats.getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); } @@ -147,13 +147,13 @@ private void verifyContinuousDataFrameHandlesData(long expectedLastCheckpoint) t // A continuous data frame should automatically become started when it gets assigned to a node // if it was assigned to the node that was removed from the cluster assertBusy(() -> { - DataFrameTransformStateAndStatsInfo stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); + DataFrameTransformStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); assertThat(stateAndStats.getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); }, 120, TimeUnit.SECONDS); - DataFrameTransformStateAndStatsInfo previousStateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); + DataFrameTransformStats previousStateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); // Add a new user and write data to it // This is so we can have more reliable data counts, as writing to existing entities requires @@ -168,18 +168,18 @@ private void verifyContinuousDataFrameHandlesData(long expectedLastCheckpoint) t waitUntilAfterCheckpoint(CONTINUOUS_DATA_FRAME_ID, expectedLastCheckpoint); assertBusy(() -> assertThat( - getTransformStats(CONTINUOUS_DATA_FRAME_ID).getTransformStats().getNumDocuments(), - greaterThanOrEqualTo(docs + previousStateAndStats.getTransformStats().getNumDocuments())), + getTransformStats(CONTINUOUS_DATA_FRAME_ID).getIndexerStats().getNumDocuments(), + greaterThanOrEqualTo(docs + previousStateAndStats.getIndexerStats().getNumDocuments())), 120, TimeUnit.SECONDS); - DataFrameTransformStateAndStatsInfo stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); + DataFrameTransformStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID); assertThat(stateAndStats.getTaskState(), equalTo(DataFrameTransformTaskState.STARTED)); - assertThat(stateAndStats.getTransformStats().getOutputDocuments(), - greaterThan(previousStateAndStats.getTransformStats().getOutputDocuments())); - assertThat(stateAndStats.getTransformStats().getNumDocuments(), - greaterThanOrEqualTo(docs + previousStateAndStats.getTransformStats().getNumDocuments())); + assertThat(stateAndStats.getIndexerStats().getOutputDocuments(), + greaterThan(previousStateAndStats.getIndexerStats().getOutputDocuments())); + assertThat(stateAndStats.getIndexerStats().getNumDocuments(), + greaterThanOrEqualTo(docs + previousStateAndStats.getIndexerStats().getNumDocuments())); } private void putTransform(String id, DataFrameTransformConfig config) throws IOException { @@ -207,7 +207,7 @@ private void stopTransform(String id) throws IOException { assertEquals(200, response.getStatusLine().getStatusCode()); } - private DataFrameTransformStateAndStatsInfo getTransformStats(String id) throws IOException { + private DataFrameTransformStats getTransformStats(String id) throws IOException { final Request getStats = new Request("GET", DATAFRAME_ENDPOINT + id + "/_stats"); Response response = client().performRequest(getStats); assertEquals(200, response.getStatusLine().getStatusCode()); @@ -216,8 +216,8 @@ private DataFrameTransformStateAndStatsInfo getTransformStats(String id) throws NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, response.getEntity().getContent())) { GetDataFrameTransformStatsResponse resp = GetDataFrameTransformStatsResponse.fromXContent(parser); - assertThat(resp.getTransformsStateAndStats(), hasSize(1)); - return resp.getTransformsStateAndStats().get(0); + assertThat(resp.getTransformsStats(), hasSize(1)); + return resp.getTransformsStats().get(0); } } From c6e35f200169cf5c507d33ca950696e325824a38 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 18 Jul 2019 15:10:26 +0100 Subject: [PATCH 06/12] Get Java-based single/multi-node integration tests working Note: YAML tests are still TODO --- .../DataFrameTransformCheckpointStats.java | 8 +- .../DataFrameTransformCheckpointingInfo.java | 7 +- .../transforms/DataFrameTransformStats.java | 16 ++- .../integration/DataFrameIntegTestCase.java | 2 +- .../DataFrameGetAndGetStatsIT.java | 35 ++++--- .../integration/DataFramePivotRestIT.java | 2 +- .../integration/DataFrameRestTestCase.java | 12 ++- ...portGetDataFrameTransformsStatsAction.java | 97 +++++++++++++------ .../DataFrameTransformsCheckpointService.java | 2 + 9 files changed, 123 insertions(+), 58 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java index f0116056832a4..2d11c7bdafece 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointStats.java @@ -137,11 +137,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (checkpointProgress != null) { builder.field(DataFrameField.CHECKPOINT_PROGRESS.getPreferredName(), checkpointProgress); } - builder.timeField(DataFrameField.TIMESTAMP_MILLIS.getPreferredName(), DataFrameField.TIMESTAMP.getPreferredName(), - getTimestampMillis()); + if (timestampMillis > 0) { + builder.timeField(DataFrameField.TIMESTAMP_MILLIS.getPreferredName(), DataFrameField.TIMESTAMP.getPreferredName(), + timestampMillis); + } if (timeUpperBoundMillis > 0) { builder.timeField(DataFrameField.TIME_UPPER_BOUND_MILLIS.getPreferredName(), DataFrameField.TIME_UPPER_BOUND.getPreferredName(), - timeUpperBoundMillis); + timeUpperBoundMillis); } builder.endObject(); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java index b6c1357691c0b..2dbab97aa5ceb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java @@ -93,13 +93,10 @@ public long getOperationsBehind() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (last.getTimestampMillis() > 0) { - builder.field(LAST_CHECKPOINT.getPreferredName(), last); - } - if (next.getTimestampMillis() > 0) { + builder.field(LAST_CHECKPOINT.getPreferredName(), last); + if (next.getCheckpoint() > 0) { builder.field(NEXT_CHECKPOINT.getPreferredName(), next); } - builder.field(OPERATIONS_BEHIND.getPreferredName(), operationsBehind); builder.endObject(); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java index 144c8d684bd79..088f291905e9e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java @@ -42,7 +42,7 @@ public class DataFrameTransformStats implements Writeable, ToXContentObject { private final String id; private final DataFrameTransformTaskState taskState; @Nullable - private String reason; + private final String reason; @Nullable private NodeAttributes node; private final DataFrameIndexerTransformStats indexerStats; @@ -103,6 +103,7 @@ public DataFrameTransformStats(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport this.id = in.readString(); this.taskState = in.readEnum(DataFrameTransformTaskState.class); + this.reason = in.readOptionalString(); if (in.readBoolean()) { this.node = new NodeAttributes(in); } else { @@ -116,7 +117,9 @@ public DataFrameTransformStats(StreamInput in) throws IOException { // to do the best we can of reading from a DataFrameTransformStoredDoc object // (which is called DataFrameTransformStateAndStats in 7.2/7.3) this.id = in.readString(); - this.taskState = new DataFrameTransformState(in).getTaskState(); + DataFrameTransformState state = new DataFrameTransformState(in); + this.taskState = state.getTaskState(); + this.reason = state.getReason(); this.node = null; this.indexerStats = new DataFrameIndexerTransformStats(in); this.checkpointingInfo = new DataFrameTransformCheckpointingInfo(in); @@ -127,7 +130,10 @@ public DataFrameTransformStats(StreamInput in) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(DataFrameField.ID.getPreferredName(), id); - builder.field(TASK_STATE_FIELD.getPreferredName(), taskState); + builder.field(TASK_STATE_FIELD.getPreferredName(), taskState.value()); + if (reason != null) { + builder.field(REASON_FIELD.getPreferredName(), reason); + } if (node != null) { builder.field(NODE_FIELD.getPreferredName(), node); } @@ -142,6 +148,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO change to V_7_4_0 after backport out.writeString(id); out.writeEnum(taskState); + out.writeOptionalString(reason); if (node != null) { out.writeBoolean(true); node.writeTo(out); @@ -169,7 +176,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public int hashCode() { - return Objects.hash(id, taskState, node, indexerStats, checkpointingInfo); + return Objects.hash(id, taskState, reason, node, indexerStats, checkpointingInfo); } @Override @@ -186,6 +193,7 @@ public boolean equals(Object other) { return Objects.equals(this.id, that.id) && Objects.equals(this.taskState, that.taskState) + && Objects.equals(this.reason, that.reason) && Objects.equals(this.node, that.node) && Objects.equals(this.indexerStats, that.indexerStats) && Objects.equals(this.checkpointingInfo, that.checkpointingInfo); diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java index 2ad9fde92d408..2f96b512d481e 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java @@ -136,7 +136,7 @@ protected void waitUntilCheckpoint(String id, long checkpoint, TimeValue waitTim .getTransformsStats() .get(0) .getCheckpointingInfo() - .getNext() + .getLast() .getCheckpoint()), waitTime.getMillis(), TimeUnit.MILLISECONDS); diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java index fd6d21db045b5..b680a50b6621e 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java @@ -98,10 +98,13 @@ public void testGetAndGetStats() throws Exception { assertThat("documents_processed is not > 0.", ((Integer)stat.get("documents_processed")), greaterThan(0)); assertThat("search_total is not > 0.", ((Integer)stat.get("search_total")), greaterThan(0)); assertThat("pages_processed is not > 0.", ((Integer)stat.get("pages_processed")), greaterThan(0)); - Map progress = (Map)XContentMapValues.extractValue("state.progress", transformStats); + /* TODO progress is now checkpoint progress and it may be that no checkpoint is in progress here + Map progress = + (Map)XContentMapValues.extractValue("checkpointing.next.checkpoint_progress", transformStats); assertThat("total_docs is not 1000", progress.get("total_docs"), equalTo(1000)); assertThat("docs_remaining is not 0", progress.get("docs_remaining"), equalTo(0)); assertThat("percent_complete is not 100.0", progress.get("percent_complete"), equalTo(100.0)); + */ } // only pivot_1 @@ -111,10 +114,9 @@ public void testGetAndGetStats() throws Exception { transformsStats = (List>)XContentMapValues.extractValue("transforms", stats); assertEquals(1, transformsStats.size()); - Map state = (Map) XContentMapValues.extractValue("state", transformsStats.get(0)); - assertEquals("stopped", XContentMapValues.extractValue("task_state", state)); - assertEquals(null, XContentMapValues.extractValue("current_position", state)); - assertEquals(1, XContentMapValues.extractValue("checkpoint", state)); + assertEquals("stopped", XContentMapValues.extractValue("task_state", transformsStats.get(0))); + assertNull(XContentMapValues.extractValue("checkpointing.next.position", transformsStats.get(0))); + assertEquals(1, XContentMapValues.extractValue("checkpointing.last.checkpoint", transformsStats.get(0))); // only continuous getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "pivot_continuous/_stats", authHeader); @@ -123,9 +125,8 @@ public void testGetAndGetStats() throws Exception { transformsStats = (List>)XContentMapValues.extractValue("transforms", stats); assertEquals(1, transformsStats.size()); - state = (Map) XContentMapValues.extractValue("state", transformsStats.get(0)); - assertEquals("started", XContentMapValues.extractValue("task_state", state)); - assertEquals(1, XContentMapValues.extractValue("checkpoint", state)); + assertEquals("started", XContentMapValues.extractValue("task_state", transformsStats.get(0))); + assertEquals(1, XContentMapValues.extractValue("checkpointing.last.checkpoint", transformsStats.get(0))); // check all the different ways to retrieve all transforms @@ -198,10 +199,13 @@ public void testGetProgressStatsWithPivotQuery() throws Exception { assertThat("documents_processed is not > 0.", ((Integer)stat.get("documents_processed")), greaterThan(0)); assertThat("search_total is not > 0.", ((Integer)stat.get("search_total")), greaterThan(0)); assertThat("pages_processed is not > 0.", ((Integer)stat.get("pages_processed")), greaterThan(0)); - Map progress = (Map)XContentMapValues.extractValue("state.progress", transformStats); + /* TODO progress is now checkpoint progress and it may be that no checkpoint is in progress here + Map progress = + (Map)XContentMapValues.extractValue("checkpointing.next.checkpoint_progress", transformStats); assertThat("total_docs is not 37", progress.get("total_docs"), equalTo(37)); assertThat("docs_remaining is not 0", progress.get("docs_remaining"), equalTo(0)); assertThat("percent_complete is not 100.0", progress.get("percent_complete"), equalTo(100.0)); + */ } } @@ -239,13 +243,16 @@ public void testGetProgressResetWithContinuous() throws Exception { Map stats = entityAsMap(client().performRequest(getRequest)); List> transformsStats = (List>)XContentMapValues.extractValue("transforms", stats); assertEquals(1, transformsStats.size()); - // Verify that the transform's progress + /* TODO progress is now checkpoint progress and it may be that no checkpoint is in progress here + // Verify that the transforms progress for (Map transformStats : transformsStats) { - Map progress = (Map)XContentMapValues.extractValue("state.progress", transformStats); + Map progress = + (Map)XContentMapValues.extractValue("checkpointing.next.checkpoint_progress", transformStats); assertThat("total_docs is not 1000", progress.get("total_docs"), equalTo(1000)); assertThat("docs_remaining is not 0", progress.get("docs_remaining"), equalTo(0)); assertThat("percent_complete is not 100.0", progress.get("percent_complete"), equalTo(100.0)); } + */ // add more docs to verify total_docs gets updated with continuous int numDocs = 10; @@ -278,13 +285,15 @@ public void testGetProgressResetWithContinuous() throws Exception { Map statsResponse = entityAsMap(client().performRequest(getRequest)); List> contStats = (List>)XContentMapValues.extractValue("transforms", statsResponse); assertEquals(1, contStats.size()); - // add more docs to verify total_docs is the number of new docs added to the index + /* TODO progress is now checkpoint progress and it may be that no checkpoint is in progress here for (Map transformStats : contStats) { - Map progress = (Map)XContentMapValues.extractValue("state.progress", transformStats); + Map progress = + (Map)XContentMapValues.extractValue("checkpointing.next.checkpoint_progress", transformStats); assertThat("total_docs is not 10", progress.get("total_docs"), equalTo(numDocs)); assertThat("docs_remaining is not 0", progress.get("docs_remaining"), equalTo(0)); assertThat("percent_complete is not 100.0", progress.get("percent_complete"), equalTo(100.0)); } + */ }, 60, TimeUnit.SECONDS); } } diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java index 0a6ebe0b9ae74..e0b0f114190fc 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java @@ -760,7 +760,7 @@ public void testBulkIndexFailuresCauseTaskToFail() throws Exception { TimeUnit.SECONDS); Map state = getDataFrameState(transformId); - assertThat((String) XContentMapValues.extractValue("state.reason", state), + assertThat((String) XContentMapValues.extractValue("reason", state), containsString("task encountered more than 10 failures; latest failure: Bulk index experienced failures.")); // Force stop the transform as bulk indexing caused it to go into a failed state diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameRestTestCase.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameRestTestCase.java index 8245413804a11..12f8f893bd219 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameRestTestCase.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameRestTestCase.java @@ -325,13 +325,17 @@ private static List> getDataFrameTransforms() throws IOExcep protected static String getDataFrameIndexerState(String transformId) throws IOException { Map transformStatsAsMap = getDataFrameState(transformId); - return transformStatsAsMap == null ? null : - (String) XContentMapValues.extractValue("state.indexer_state", transformStatsAsMap); + if (transformStatsAsMap == null) { + return null; + } + String indexerState = (String) XContentMapValues.extractValue("checkpointing.next.indexer_state", transformStatsAsMap); + // If the transform is stopped then it might not have an indexer state, but logically that's the same as the indexer being stopped + return indexerState == null ? "stopped" : indexerState; } protected static String getDataFrameTaskState(String transformId) throws IOException { Map transformStatsAsMap = getDataFrameState(transformId); - return transformStatsAsMap == null ? null : (String) XContentMapValues.extractValue("state.task_state", transformStatsAsMap); + return transformStatsAsMap == null ? null : (String) XContentMapValues.extractValue("task_state", transformStatsAsMap); } protected static Map getDataFrameState(String transformId) throws IOException { @@ -422,7 +426,7 @@ static int getDataFrameCheckpoint(String transformId) throws IOException { Response statsResponse = client().performRequest(new Request("GET", DATAFRAME_ENDPOINT + transformId + "/_stats")); Map transformStatsAsMap = (Map) ((List) entityAsMap(statsResponse).get("transforms")).get(0); - return (int) XContentMapValues.extractValue("state.checkpoint", transformStatsAsMap); + return (int) XContentMapValues.extractValue("checkpointing.last.checkpoint", transformStatsAsMap); } protected void setupDataAccessRole(String role, String... indices) throws IOException { diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java index 40ade96a52e6e..0778b05f79b85 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java @@ -41,6 +41,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; public class TransportGetDataFrameTransformsStatsAction extends @@ -171,35 +172,32 @@ private void collectStatsForTransformsWithoutTasks(Request request, // Small assurance that we are at least below the max. Terms search has a hard limit of 10k, we should at least be below that. assert transformsWithoutTasks.size() <= Request.MAX_SIZE_RETURN; + // If the persistent task does NOT exist, it is STOPPED + // There is a potential race condition where the saved document does not actually have a STOPPED state + // as the task is cancelled before we persist state. ActionListener> searchStatsListener = ActionListener.wrap( - stats -> { + statsForTransformsWithoutTasks -> { List allStateAndStats = response.getTransformsStats(); - // If the persistent task does NOT exist, it is STOPPED - // There is a potential race condition where the saved document does not actually have a STOPPED state - // as the task is cancelled before we persist state. - stats.forEach(stat -> - allStateAndStats.add(new DataFrameTransformStats( - stat.getId(), - DataFrameTransformTaskState.STOPPED, - null, - null, - stat.getTransformStats(), - DataFrameTransformCheckpointingInfo.EMPTY)) // TODO !!! - ); - transformsWithoutTasks.removeAll(stats.stream().map(DataFrameTransformStoredDoc::getId).collect(Collectors.toSet())); - - // Transforms that have not been started and have no state or stats. - transformsWithoutTasks.forEach(transformId -> - allStateAndStats.add(DataFrameTransformStats.initialStats(transformId))); - - // Any transform in collection could NOT have a task, so, even though the list is initially sorted - // it can easily become arbitrarily ordered based on which transforms don't have a task or stats docs - allStateAndStats.sort(Comparator.comparing(DataFrameTransformStats::getId)); - - listener.onResponse(new Response(allStateAndStats, - allStateAndStats.size(), - response.getTaskFailures(), - response.getNodeFailures())); + addCheckpointingInfoForTransformsWithoutTasks(allStateAndStats, statsForTransformsWithoutTasks, + ActionListener.wrap( + aVoid -> { + transformsWithoutTasks.removeAll(statsForTransformsWithoutTasks.stream() + .map(DataFrameTransformStoredDoc::getId).collect(Collectors.toSet())); + + // Transforms that have not been started and have no state or stats. + transformsWithoutTasks.forEach( + transformId -> allStateAndStats.add(DataFrameTransformStats.initialStats(transformId))); + + // Any transform in collection could NOT have a task, so, even though the list is initially sorted + // it can easily become arbitrarily ordered based on which transforms don't have a task or stats docs + allStateAndStats.sort(Comparator.comparing(DataFrameTransformStats::getId)); + + listener.onResponse(new Response(allStateAndStats, + allStateAndStats.size(), + response.getTaskFailures(), + response.getNodeFailures())); + }, + listener::onFailure)); }, e -> { if (e instanceof IndexNotFoundException) { @@ -212,4 +210,49 @@ private void collectStatsForTransformsWithoutTasks(Request request, dataFrameTransformsConfigManager.getTransformStoredDoc(transformsWithoutTasks, searchStatsListener); } + + private void populateSingleStoppedTransformStat(DataFrameTransformStoredDoc transform, + ActionListener listener) { + transformsCheckpointService.getCheckpointStats(transform.getId(), transform.getTransformState().getCheckpoint(), + transform.getTransformState().getCheckpoint() + 1, transform.getTransformState().getIndexerState(), + transform.getTransformState().getPosition(), + ActionListener.wrap( + listener::onResponse, + e -> { + logger.warn("Failed to retrieve checkpointing info for transform [" + transform.getId() + "]", e); + listener.onResponse(DataFrameTransformCheckpointingInfo.EMPTY); + } + )); + } + + private void addCheckpointingInfoForTransformsWithoutTasks(List allStateAndStats, + List statsForTransformsWithoutTasks, + ActionListener listener) { + + if (statsForTransformsWithoutTasks.isEmpty()) { + // No work to do, but we must respond to the listener + listener.onResponse(null); + } + + AtomicInteger numberRemaining = new AtomicInteger(statsForTransformsWithoutTasks.size()); + + statsForTransformsWithoutTasks.forEach(stat -> populateSingleStoppedTransformStat(stat, + ActionListener.wrap( + checkpointingInfo -> { + synchronized (allStateAndStats) { + allStateAndStats.add(new DataFrameTransformStats( + stat.getId(), + DataFrameTransformTaskState.STOPPED, + null, + null, + stat.getTransformStats(), + checkpointingInfo)); + } + if (numberRemaining.decrementAndGet() == 0) { + listener.onResponse(null); + } + }, + listener::onFailure + ))); + } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java index f427c5c096f4f..fb4c67c9e31bd 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java @@ -151,6 +151,8 @@ public void getCheckpoint(DataFrameTransformConfig transformConfig, long checkpo * @param transformId The data frame task * @param currentCheckpoint the current checkpoint * @param inProgressCheckpoint in progress checkpoint + * @param inProgressIndexerState indexer state for in progress checkpoint + * @param inProgressPosition position for in progress checkpoint * @param listener listener to retrieve the result */ public void getCheckpointStats( From 9b96fca762bda1b9c42ebe451efde5efc39804f8 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 18 Jul 2019 16:36:43 +0100 Subject: [PATCH 07/12] Fix data frame YAML tests Now that indexer_state has moved into checkpointing.next it is unreliable to assert on the value of indexer_state in a YAML test because the tests are so quick that the next checkpoint can easily move to last before any assertion is made. --- .../test/data_frame/transforms_start_stop.yml | 33 ++++------ .../test/data_frame/transforms_stats.yml | 29 ++++---- .../mixed_cluster/80_data_frame_jobs_crud.yml | 66 ++++++++++++++----- .../old_cluster/80_data_frame_jobs_crud.yml | 9 --- .../80_data_frame_jobs_crud.yml | 34 ++++------ 5 files changed, 84 insertions(+), 87 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml index e4ff3c813ce0f..df54a90d61087 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml @@ -100,8 +100,7 @@ teardown: transform_id: "airline-transform-start-stop" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-stop" } - - match: { transforms.0.state.indexer_state: "/started|indexing/" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -114,8 +113,7 @@ teardown: transform_id: "airline-transform-start-stop" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-stop" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } - do: data_frame.start_data_frame_transform: @@ -127,8 +125,7 @@ teardown: transform_id: "airline-transform-start-stop" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-stop" } - - match: { transforms.0.state.indexer_state: "/started|indexing/" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } --- "Test start/stop/start continuous transform": - do: @@ -154,8 +151,7 @@ teardown: transform_id: "airline-transform-start-stop-continuous" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-stop-continuous" } - - match: { transforms.0.state.indexer_state: "/started|indexing/" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -168,8 +164,7 @@ teardown: transform_id: "airline-transform-start-stop-continuous" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-stop-continuous" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } - do: data_frame.start_data_frame_transform: @@ -181,8 +176,7 @@ teardown: transform_id: "airline-transform-start-stop-continuous" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-stop-continuous" } - - match: { transforms.0.state.indexer_state: "/started|indexing/" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -244,16 +238,14 @@ teardown: transform_id: "airline-transform-start-stop" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-stop" } - - match: { transforms.0.state.indexer_state: "/started|indexing/" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.get_data_frame_transform_stats: transform_id: "airline-transform-start-later" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-later" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } - do: data_frame.start_data_frame_transform: @@ -272,8 +264,7 @@ teardown: transform_id: "airline-transform-start-later" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-later" } - - match: { transforms.0.state.indexer_state: "/started|indexing/" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -319,10 +310,8 @@ teardown: data_frame.get_data_frame_transform_stats: transform_id: "*" - match: { count: 2 } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } - - match: { transforms.1.state.indexer_state: "stopped" } - - match: { transforms.1.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } + - match: { transforms.1.task_state: "stopped" } - do: data_frame.delete_data_frame_transform: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml index a6066aceb969b..8609159f02a28 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml @@ -47,9 +47,8 @@ teardown: transform_id: "airline-transform-stats" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-stats" } - - match: { transforms.0.state.indexer_state: "/started|indexing|stopped/" } - - match: { transforms.0.state.task_state: "/started|stopped/" } - - lte: { transforms.0.state.checkpoint: 1 } + - match: { transforms.0.task_state: "/started|stopped/" } + - lte: { transforms.0.checkpointing.last.checkpoint: 1 } - lte: { transforms.0.stats.pages_processed: 1 } - match: { transforms.0.stats.documents_processed: 0 } - match: { transforms.0.stats.documents_indexed: 0 } @@ -163,18 +162,14 @@ teardown: transform_id: "*" - match: { count: 2 } - match: { transforms.0.id: "airline-transform-stats" } - - match: { transforms.0.state.indexer_state: "/started|indexing|stopped/" } - match: { transforms.1.id: "airline-transform-stats-dos" } - - match: { transforms.1.state.indexer_state: "stopped" } - do: data_frame.get_data_frame_transform_stats: transform_id: "_all" - match: { count: 2 } - match: { transforms.0.id: "airline-transform-stats" } - - match: { transforms.0.state.indexer_state: "/started|indexing|stopped/" } - match: { transforms.1.id: "airline-transform-stats-dos" } - - match: { transforms.1.state.indexer_state: "stopped" } --- "Test get single transform stats when it does not have a task": @@ -196,9 +191,8 @@ teardown: transform_id: "airline-transform-stats-dos" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-stats-dos" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.checkpoint: 0 } - - is_false: transforms.0.state.progress + - match: { transforms.0.checkpointing.last.checkpoint: 0 } + - is_false: transforms.0.checkpointing.next.progress - match: { transforms.0.stats.pages_processed: 0 } - match: { transforms.0.stats.documents_processed: 0 } - match: { transforms.0.stats.documents_indexed: 0 } @@ -232,17 +226,16 @@ teardown: transform_id: "airline-transform-stats-continuous" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-stats-continuous" } - - match: { transforms.0.state.indexer_state: "/started|indexing|stopped/" } # Since this is continuous, there is no worry of it automatically stopping - - match: { transforms.0.state.task_state: "started" } - - lte: { transforms.0.state.checkpoint: 1 } + - match: { transforms.0.task_state: "started" } + - lte: { transforms.0.checkpointing.last.checkpoint: 1 } # Since this is continuous, and _start does not return until it is assigned # we should see a node assignment - - is_true: transforms.0.state.node - - is_true: transforms.0.state.node.id - - is_true: transforms.0.state.node.name - - is_true: transforms.0.state.node.ephemeral_id - - is_true: transforms.0.state.node.transport_address + - is_true: transforms.0.node + - is_true: transforms.0.node.id + - is_true: transforms.0.node.name + - is_true: transforms.0.node.ephemeral_id + - is_true: transforms.0.node.transport_address - lte: { transforms.0.stats.pages_processed: 1 } - match: { transforms.0.stats.documents_processed: 0 } - match: { transforms.0.stats.documents_indexed: 0 } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml index 9454423e98953..97d3bd75d65dd 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml @@ -29,7 +29,10 @@ transform_id: "mixed-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -42,8 +45,10 @@ transform_id: "mixed-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "stopped" } - do: data_frame.put_data_frame_transform: @@ -90,7 +95,10 @@ transform_id: "mixed-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-complex-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -103,8 +111,10 @@ transform_id: "mixed-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-complex-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "stopped" } --- "Test put continuous data frame transform on mixed cluster": @@ -153,7 +163,10 @@ transform_id: "mixed-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-continuous-transform" } - - match: { transforms.0.state.task_state: "started" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -166,8 +179,10 @@ transform_id: "mixed-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-continuous-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "stopped" } --- "Test GET, start, and stop old cluster batch transforms": @@ -196,7 +211,10 @@ transform_id: "old-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -208,8 +226,10 @@ transform_id: "old-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "stopped" } - do: data_frame.get_data_frame_transform: @@ -233,7 +253,10 @@ transform_id: "old-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "old-complex-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -245,8 +268,10 @@ transform_id: "old-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "old-complex-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.state.task_state: "stopped" } --- "Test GET, stop, start, old continuous transforms": @@ -275,7 +300,10 @@ transform_id: "old-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-continuous-transform" } - - match: { transforms.0.state.task_state: "started" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -288,5 +316,7 @@ transform_id: "old-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-continuous-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we + # cannot assert on task_state in the mixed cluster as it could be at the top level or under state + # TODO: uncomment this assertion in master + #- match: { transforms.0.task_state: "stopped" } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml index 7b666c2caa35f..bf2b1f6b939f8 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml @@ -65,7 +65,6 @@ transform_id: "old-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -78,8 +77,6 @@ transform_id: "old-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } - do: data_frame.put_data_frame_transform: @@ -124,7 +121,6 @@ transform_id: "old-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "old-complex-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -137,8 +133,6 @@ transform_id: "old-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "old-complex-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } --- "Test put continuous data frame transform on old cluster": @@ -201,7 +195,6 @@ transform_id: "old-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-continuous-transform" } - - match: { transforms.0.state.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -214,5 +207,3 @@ transform_id: "old-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-continuous-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml index ea63950d7fc64..e149a249fff78 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml @@ -27,7 +27,7 @@ setup: transform_id: "old-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } + - match: { transforms.0.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -39,8 +39,7 @@ setup: transform_id: "old-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } - do: data_frame.get_data_frame_transform: transform_id: "old-complex-transform" @@ -63,7 +62,7 @@ setup: transform_id: "old-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "old-complex-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } + - match: { transforms.0.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -75,8 +74,7 @@ setup: transform_id: "old-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "old-complex-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } # Simple and complex Mixed cluster transforms - do: @@ -98,7 +96,7 @@ setup: transform_id: "mixed-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } + - match: { transforms.0.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -110,8 +108,7 @@ setup: transform_id: "mixed-simple-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } - do: data_frame.get_data_frame_transform: @@ -135,7 +132,7 @@ setup: transform_id: "mixed-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-complex-transform" } - - match: { transforms.0.state.task_state: "/started|stopped/" } + - match: { transforms.0.task_state: "/started|stopped/" } - do: data_frame.stop_data_frame_transform: @@ -147,8 +144,7 @@ setup: transform_id: "mixed-complex-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-complex-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } # Delete all old and mixed transforms - do: @@ -185,14 +181,14 @@ setup: transform_id: "old-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-continuous-transform" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.get_data_frame_transform_stats: transform_id: "old-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-continuous-transform" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -205,8 +201,7 @@ setup: transform_id: "old-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "old-simple-continuous-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } - do: data_frame.delete_data_frame_transform: @@ -231,14 +226,14 @@ setup: transform_id: "mixed-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-continuous-transform" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.get_data_frame_transform_stats: transform_id: "mixed-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-continuous-transform" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: data_frame.stop_data_frame_transform: @@ -251,8 +246,7 @@ setup: transform_id: "mixed-simple-continuous-transform" - match: { count: 1 } - match: { transforms.0.id: "mixed-simple-continuous-transform" } - - match: { transforms.0.state.indexer_state: "stopped" } - - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.0.task_state: "stopped" } - do: data_frame.delete_data_frame_transform: From b83f44f23d41ab48d207bbf25b0a29ef5f5f16f7 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 18 Jul 2019 17:39:48 +0100 Subject: [PATCH 08/12] More places where we can't assert on indexer state now --- .../java/org/elasticsearch/client/DataFrameTransformIT.java | 4 ---- .../documentation/DataFrameTransformDocumentationIT.java | 1 - 2 files changed, 5 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java index 8c6778464867f..2fd365153b77e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.core.AcknowledgedResponse; -import org.elasticsearch.client.core.IndexerState; import org.elasticsearch.client.core.PageParams; import org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest; import org.elasticsearch.client.dataframe.GetDataFrameTransformRequest; @@ -356,7 +355,6 @@ public void testGetStats() throws Exception { assertEquals(1, statsResponse.getTransformsStats().size()); DataFrameTransformStats stats = statsResponse.getTransformsStats().get(0); assertEquals(DataFrameTransformTaskState.STOPPED, stats.getTaskState()); - assertEquals(IndexerState.STOPPED, stats.getCheckpointingInfo().getNext().getIndexerState()); DataFrameIndexerTransformStats zeroIndexerStats = new DataFrameIndexerTransformStats(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L); assertEquals(zeroIndexerStats, stats.getIndexerStats()); @@ -373,8 +371,6 @@ public void testGetStats() throws Exception { assertNotEquals(zeroIndexerStats, stateAndStats.getIndexerStats()); assertThat(stateAndStats.getTaskState(), is(oneOf(DataFrameTransformTaskState.STARTED, DataFrameTransformTaskState.STOPPED))); - assertThat(stateAndStats.getCheckpointingInfo().getNext().getIndexerState(), - is(oneOf(IndexerState.STARTED, IndexerState.STOPPED))); assertNotNull(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress()); assertThat(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress().getPercentComplete(), equalTo(100.0)); assertThat(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress().getTotalDocs(), greaterThan(0L)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java index 13a495a0d1fd9..2ec0d6e1789a7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java @@ -541,7 +541,6 @@ public void testGetStats() throws IOException, InterruptedException { stateAndStatsInfo.getNode(); // <6> // end::get-data-frame-transform-stats-response - assertEquals(IndexerState.STOPPED, indexerState); assertEquals(DataFrameTransformTaskState.STOPPED, taskState); assertNotNull(transformStats); assertNull(progress); From 3426afb34f450d54518a2f7beb577a86c944e770 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 19 Jul 2019 13:59:45 +0100 Subject: [PATCH 09/12] Add progress object back to next checkpoint stats Also renamed some more internal variables to match the user facing stats output --- .../DataFrameTransformCheckpointingInfo.java | 5 ++ ...portGetDataFrameTransformsStatsAction.java | 12 ++- .../DataFrameTransformsCheckpointService.java | 87 ++++++++++--------- ...meTransformCheckpointServiceNodeTests.java | 23 ++--- 4 files changed, 73 insertions(+), 54 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java index 2dbab97aa5ceb..4e33c178649d6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpointingInfo.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.dataframe.transforms; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -135,4 +136,8 @@ public boolean equals(Object other) { this.operationsBehind == that.operationsBehind; } + @Override + public String toString() { + return Strings.toString(this); + } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java index 0778b05f79b85..59f20333b5a53 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java @@ -41,6 +41,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -85,7 +86,7 @@ protected void taskOperation(Request request, DataFrameTransformTask task, Actio String nodeId = state.nodes().getLocalNode().getId(); if (task.isCancelled() == false) { transformsCheckpointService.getCheckpointStats(task.getTransformId(), task.getCheckpoint(), task.getInProgressCheckpoint(), - task.getState().getIndexerState(), task.getState().getPosition(), + task.getState().getIndexerState(), task.getState().getPosition(), task.getProgress(), ActionListener.wrap(checkpointStats -> listener.onResponse(new Response( Collections.singletonList(new DataFrameTransformStats(task.getTransformId(), task.getState().getTaskState(), @@ -215,7 +216,7 @@ private void populateSingleStoppedTransformStat(DataFrameTransformStoredDoc tran ActionListener listener) { transformsCheckpointService.getCheckpointStats(transform.getId(), transform.getTransformState().getCheckpoint(), transform.getTransformState().getCheckpoint() + 1, transform.getTransformState().getIndexerState(), - transform.getTransformState().getPosition(), + transform.getTransformState().getPosition(), transform.getTransformState().getProgress(), ActionListener.wrap( listener::onResponse, e -> { @@ -235,6 +236,7 @@ private void addCheckpointingInfoForTransformsWithoutTasks(List populateSingleStoppedTransformStat(stat, ActionListener.wrap( @@ -252,7 +254,11 @@ private void addCheckpointingInfoForTransformsWithoutTasks(List { + if (isExceptionReported.compareAndSet(false, true)) { + listener.onFailure(e); + } + } ))); } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java index fb4c67c9e31bd..3ef41bfc6e3b8 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig; import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig; import org.elasticsearch.xpack.core.indexing.IndexerState; @@ -44,29 +45,31 @@ public class DataFrameTransformsCheckpointService { private static class Checkpoints { - long currentCheckpointNumber; - long inProgressCheckpointNumber; - IndexerState inProgressIndexerState; - DataFrameIndexerPosition inProgressPosition; - DataFrameTransformCheckpoint currentCheckpoint = DataFrameTransformCheckpoint.EMPTY; - DataFrameTransformCheckpoint inProgressCheckpoint = DataFrameTransformCheckpoint.EMPTY; + long lastCheckpointNumber; + long nextCheckpointNumber; + IndexerState nextCheckpointIndexerState; + DataFrameIndexerPosition nextCheckpointPosition; + DataFrameTransformProgress nextCheckpointProgress; + DataFrameTransformCheckpoint lastCheckpoint = DataFrameTransformCheckpoint.EMPTY; + DataFrameTransformCheckpoint nextCheckpoint = DataFrameTransformCheckpoint.EMPTY; DataFrameTransformCheckpoint sourceCheckpoint = DataFrameTransformCheckpoint.EMPTY; - Checkpoints(long currentCheckpointNumber, long inProgressCheckpointNumber, IndexerState inProgressIndexerState, - DataFrameIndexerPosition inProgressPosition) { - this.currentCheckpointNumber = currentCheckpointNumber; - this.inProgressCheckpointNumber = inProgressCheckpointNumber; - this.inProgressIndexerState = inProgressIndexerState; - this.inProgressPosition = inProgressPosition; + Checkpoints(long lastCheckpointNumber, long nextCheckpointNumber, IndexerState nextCheckpointIndexerState, + DataFrameIndexerPosition nextCheckpointPosition, DataFrameTransformProgress nextCheckpointProgress) { + this.lastCheckpointNumber = lastCheckpointNumber; + this.nextCheckpointNumber = nextCheckpointNumber; + this.nextCheckpointIndexerState = nextCheckpointIndexerState; + this.nextCheckpointPosition = nextCheckpointPosition; + this.nextCheckpointProgress = nextCheckpointProgress; } DataFrameTransformCheckpointingInfo buildInfo() { return new DataFrameTransformCheckpointingInfo( - new DataFrameTransformCheckpointStats(currentCheckpointNumber, null, null, null, - currentCheckpoint.getTimestamp(), currentCheckpoint.getTimeUpperBound()), - new DataFrameTransformCheckpointStats(inProgressCheckpointNumber, inProgressIndexerState, inProgressPosition, null, - inProgressCheckpoint.getTimestamp(), inProgressCheckpoint.getTimeUpperBound()), - DataFrameTransformCheckpoint.getBehind(currentCheckpoint, sourceCheckpoint)); + new DataFrameTransformCheckpointStats(lastCheckpointNumber, null, null, null, + lastCheckpoint.getTimestamp(), lastCheckpoint.getTimeUpperBound()), + new DataFrameTransformCheckpointStats(nextCheckpointNumber, nextCheckpointIndexerState, nextCheckpointPosition, + nextCheckpointProgress, nextCheckpoint.getTimestamp(), nextCheckpoint.getTimeUpperBound()), + DataFrameTransformCheckpoint.getBehind(lastCheckpoint, sourceCheckpoint)); } } @@ -149,51 +152,53 @@ public void getCheckpoint(DataFrameTransformConfig transformConfig, long checkpo * Get checkpointing stats for a data frame * * @param transformId The data frame task - * @param currentCheckpoint the current checkpoint - * @param inProgressCheckpoint in progress checkpoint - * @param inProgressIndexerState indexer state for in progress checkpoint - * @param inProgressPosition position for in progress checkpoint + * @param lastCheckpoint the last checkpoint + * @param nextCheckpoint the next checkpoint + * @param nextCheckpointIndexerState indexer state for the next checkpoint + * @param nextCheckpointPosition position for the next checkpoint + * @param nextCheckpointProgress progress for the next checkpoint * @param listener listener to retrieve the result */ - public void getCheckpointStats( - String transformId, - long currentCheckpoint, - long inProgressCheckpoint, - IndexerState inProgressIndexerState, - DataFrameIndexerPosition inProgressPosition, - ActionListener listener) { + public void getCheckpointStats(String transformId, + long lastCheckpoint, + long nextCheckpoint, + IndexerState nextCheckpointIndexerState, + DataFrameIndexerPosition nextCheckpointPosition, + DataFrameTransformProgress nextCheckpointProgress, + ActionListener listener) { - Checkpoints checkpoints = new Checkpoints(currentCheckpoint, inProgressCheckpoint, inProgressIndexerState, inProgressPosition); + Checkpoints checkpoints = + new Checkpoints(lastCheckpoint, nextCheckpoint, nextCheckpointIndexerState, nextCheckpointPosition, nextCheckpointProgress); // <3> notify the user once we have the current checkpoint ActionListener currentCheckpointListener = ActionListener.wrap( currentCheckpointObj -> { - checkpoints.currentCheckpoint = currentCheckpointObj; + checkpoints.lastCheckpoint = currentCheckpointObj; listener.onResponse(checkpoints.buildInfo()); }, e -> { - logger.debug("Failed to retrieve current checkpoint [" + - currentCheckpoint + "] for data frame [" + transformId + "]", e); - listener.onFailure(new CheckpointException("Failure during current checkpoint info retrieval", e)); + logger.debug("Failed to retrieve last checkpoint [" + + lastCheckpoint + "] for data frame [" + transformId + "]", e); + listener.onFailure(new CheckpointException("Failure during last checkpoint info retrieval", e)); } ); // <2> after the in progress checkpoint, get the current checkpoint ActionListener inProgressCheckpointListener = ActionListener.wrap( inProgressCheckpointObj -> { - checkpoints.inProgressCheckpoint = inProgressCheckpointObj; - if (currentCheckpoint != 0) { + checkpoints.nextCheckpoint = inProgressCheckpointObj; + if (lastCheckpoint != 0) { dataFrameTransformsConfigManager.getTransformCheckpoint(transformId, - currentCheckpoint, + lastCheckpoint, currentCheckpointListener); } else { currentCheckpointListener.onResponse(DataFrameTransformCheckpoint.EMPTY); } }, e -> { - logger.debug("Failed to retrieve in progress checkpoint [" + - inProgressCheckpoint + "] for data frame [" + transformId + "]", e); - listener.onFailure(new CheckpointException("Failure during in progress checkpoint info retrieval", e)); + logger.debug("Failed to retrieve next checkpoint [" + + nextCheckpoint + "] for data frame [" + transformId + "]", e); + listener.onFailure(new CheckpointException("Failure during next checkpoint info retrieval", e)); } ); @@ -201,9 +206,9 @@ public void getCheckpointStats( ActionListener sourceCheckpointListener = ActionListener.wrap( sourceCheckpoint -> { checkpoints.sourceCheckpoint = sourceCheckpoint; - if (inProgressCheckpoint != 0) { + if (nextCheckpoint != 0) { dataFrameTransformsConfigManager.getTransformCheckpoint(transformId, - inProgressCheckpoint, + nextCheckpoint, inProgressCheckpointListener); } else { inProgressCheckpointListener.onResponse(DataFrameTransformCheckpoint.EMPTY); diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java index cbf9bb9260b5c..27714f40e2576 100644 --- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java +++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java @@ -45,6 +45,8 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointStats; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfigTests; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgressTests; import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.dataframe.DataFrameSingleNodeTestCase; import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager; @@ -78,7 +80,7 @@ private class MockClientForCheckpointing extends NoOpClient { super(testName); } - public void setShardStats(ShardStats[] shardStats) { + void setShardStats(ShardStats[] shardStats) { this.shardStats = shardStats; Set indices = new HashSet<>(); @@ -176,6 +178,7 @@ public void testGetCheckpointStats() throws InterruptedException { String transformId = randomAlphaOfLengthBetween(3, 10); long timestamp = 1000; DataFrameIndexerPosition position = DataFrameIndexerPositionTests.randomDataFrameIndexerPosition(); + DataFrameTransformProgress progress = DataFrameTransformProgressTests.randomDataFrameTransformProgress(); // create transform assertAsync( @@ -196,30 +199,30 @@ public void testGetCheckpointStats() throws InterruptedException { mockClientForCheckpointing.setShardStats(createShardStats(createCheckPointMap(transformId, 20, 20, 20))); DataFrameTransformCheckpointingInfo checkpointInfo = new DataFrameTransformCheckpointingInfo( new DataFrameTransformCheckpointStats(1, null, null, null, timestamp, 0L), - new DataFrameTransformCheckpointStats(2, IndexerState.STARTED, position, null, timestamp + 100L, 0L), + new DataFrameTransformCheckpointStats(2, IndexerState.STARTED, position, progress, timestamp + 100L, 0L), 30L); - assertAsync( - listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.STARTED, position, listener), + assertAsync(listener -> + transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.STARTED, position, progress, listener), checkpointInfo, null, null); mockClientForCheckpointing.setShardStats(createShardStats(createCheckPointMap(transformId, 10, 50, 33))); checkpointInfo = new DataFrameTransformCheckpointingInfo( new DataFrameTransformCheckpointStats(1, null, null, null, timestamp, 0L), - new DataFrameTransformCheckpointStats(2, IndexerState.INDEXING, position, null, timestamp + 100L, 0L), + new DataFrameTransformCheckpointStats(2, IndexerState.INDEXING, position, progress, timestamp + 100L, 0L), 63L); - assertAsync( - listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.INDEXING, position, listener), + assertAsync(listener -> + transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.INDEXING, position, progress, listener), checkpointInfo, null, null); // same as current mockClientForCheckpointing.setShardStats(createShardStats(createCheckPointMap(transformId, 10, 10, 10))); checkpointInfo = new DataFrameTransformCheckpointingInfo( new DataFrameTransformCheckpointStats(1, null, null, null, timestamp, 0L), - new DataFrameTransformCheckpointStats(2, IndexerState.STOPPING, position, null, timestamp + 100L, 0L), + new DataFrameTransformCheckpointStats(2, IndexerState.STOPPING, position, progress, timestamp + 100L, 0L), 0L); - assertAsync( - listener -> transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.STOPPING, position, listener), + assertAsync(listener -> + transformsCheckpointService.getCheckpointStats(transformId, 1, 2, IndexerState.STOPPING, position, progress, listener), checkpointInfo, null, null); } From d61ca6a200184acc942ec158bcdbe8980cb78558 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 19 Jul 2019 16:15:03 +0100 Subject: [PATCH 10/12] Address one review comment, more naming consistency and remove warnings --- .../DataFrameTransformCheckpoint.java | 16 +++++------ ...portGetDataFrameTransformsStatsAction.java | 1 + .../DataFrameTransformsCheckpointService.java | 27 +++++++++---------- ...FrameTransformsCheckpointServiceTests.java | 7 +++-- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpoint.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpoint.java index ec99827fdcdab..81a06eb4524d1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpoint.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformCheckpoint.java @@ -65,16 +65,16 @@ private static ConstructingObjectParser crea ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, args -> { String id = (String) args[0]; - Long timestamp = (Long) args[1]; - Long checkpoint = (Long) args[2]; + long timestamp = (Long) args[1]; + long checkpoint = (Long) args[2]; @SuppressWarnings("unchecked") Map checkpoints = (Map) args[3]; - Long timestamp_checkpoint = (Long) args[4]; + Long timeUpperBound = (Long) args[4]; // ignored, only for internal storage: String docType = (String) args[5]; - return new DataFrameTransformCheckpoint(id, timestamp, checkpoint, checkpoints, timestamp_checkpoint); + return new DataFrameTransformCheckpoint(id, timestamp, checkpoint, checkpoints, timeUpperBound); }); parser.declareString(constructorArg(), DataFrameField.ID); @@ -108,13 +108,13 @@ private static ConstructingObjectParser crea return parser; } - public DataFrameTransformCheckpoint(String transformId, Long timestamp, Long checkpoint, Map checkpoints, + public DataFrameTransformCheckpoint(String transformId, long timestamp, long checkpoint, Map checkpoints, Long timeUpperBound) { - this.transformId = transformId; - this.timestampMillis = timestamp.longValue(); + this.transformId = Objects.requireNonNull(transformId); + this.timestampMillis = timestamp; this.checkpoint = checkpoint; this.indicesCheckpoints = Collections.unmodifiableMap(checkpoints); - this.timeUpperBoundMillis = timeUpperBound == null ? 0 : timeUpperBound.longValue(); + this.timeUpperBoundMillis = timeUpperBound == null ? 0 : timeUpperBound; } public DataFrameTransformCheckpoint(StreamInput in) throws IOException { diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java index 59f20333b5a53..173f9a9714542 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java @@ -233,6 +233,7 @@ private void addCheckpointingInfoForTransformsWithoutTasks(List listener) { + ActionListener listener) { long timestamp = System.currentTimeMillis(); // for time based synchronization @@ -145,7 +145,6 @@ public void getCheckpoint(DataFrameTransformConfig transformConfig, long checkpo }, e -> listener.onFailure(new CheckpointException("Failed to create checkpoint", e)) )); - } /** @@ -170,10 +169,10 @@ public void getCheckpointStats(String transformId, Checkpoints checkpoints = new Checkpoints(lastCheckpoint, nextCheckpoint, nextCheckpointIndexerState, nextCheckpointPosition, nextCheckpointProgress); - // <3> notify the user once we have the current checkpoint - ActionListener currentCheckpointListener = ActionListener.wrap( - currentCheckpointObj -> { - checkpoints.lastCheckpoint = currentCheckpointObj; + // <3> notify the user once we have the last checkpoint + ActionListener lastCheckpointListener = ActionListener.wrap( + lastCheckpointObj -> { + checkpoints.lastCheckpoint = lastCheckpointObj; listener.onResponse(checkpoints.buildInfo()); }, e -> { @@ -183,16 +182,16 @@ public void getCheckpointStats(String transformId, } ); - // <2> after the in progress checkpoint, get the current checkpoint - ActionListener inProgressCheckpointListener = ActionListener.wrap( - inProgressCheckpointObj -> { - checkpoints.nextCheckpoint = inProgressCheckpointObj; + // <2> after the next checkpoint, get the last checkpoint + ActionListener nextCheckpointListener = ActionListener.wrap( + nextCheckpointObj -> { + checkpoints.nextCheckpoint = nextCheckpointObj; if (lastCheckpoint != 0) { dataFrameTransformsConfigManager.getTransformCheckpoint(transformId, lastCheckpoint, - currentCheckpointListener); + lastCheckpointListener); } else { - currentCheckpointListener.onResponse(DataFrameTransformCheckpoint.EMPTY); + lastCheckpointListener.onResponse(DataFrameTransformCheckpoint.EMPTY); } }, e -> { @@ -209,9 +208,9 @@ public void getCheckpointStats(String transformId, if (nextCheckpoint != 0) { dataFrameTransformsConfigManager.getTransformCheckpoint(transformId, nextCheckpoint, - inProgressCheckpointListener); + nextCheckpointListener); } else { - inProgressCheckpointListener.onResponse(DataFrameTransformCheckpoint.EMPTY); + nextCheckpointListener.onResponse(DataFrameTransformCheckpoint.EMPTY); } }, e -> { diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointServiceTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointServiceTests.java index c8d9be89611d4..6ced3fb25324e 100644 --- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointServiceTests.java +++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointServiceTests.java @@ -35,7 +35,6 @@ import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -61,7 +60,7 @@ public void testExtractIndexCheckpoints() { // low-level compare for (Entry entry : expectedCheckpoints.entrySet()) { - assertTrue(Arrays.equals(entry.getValue(), checkpoints.get(entry.getKey()))); + assertArrayEquals(entry.getValue(), checkpoints.get(entry.getKey())); } } @@ -78,7 +77,7 @@ public void testExtractIndexCheckpointsMissingSeqNoStats() { // low-level compare for (Entry entry : expectedCheckpoints.entrySet()) { - assertTrue(Arrays.equals(entry.getValue(), checkpoints.get(entry.getKey()))); + assertArrayEquals(entry.getValue(), checkpoints.get(entry.getKey())); } } @@ -95,7 +94,7 @@ public void testExtractIndexCheckpointsLostPrimaries() { // low-level compare for (Entry entry : expectedCheckpoints.entrySet()) { - assertTrue(Arrays.equals(entry.getValue(), checkpoints.get(entry.getKey()))); + assertArrayEquals(entry.getValue(), checkpoints.get(entry.getKey())); } } From 260cea506e86f904696cdb0b24d0bfa58934fb05 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 22 Jul 2019 14:09:36 +0100 Subject: [PATCH 11/12] Alter some comments --- .../transforms/hlrc/DataFrameTransformStatsTests.java | 4 +--- .../xpack/dataframe/transforms/DataFrameTransformTask.java | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStatsTests.java index 18b057883f407..19ddb8be1abbf 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStatsTests.java @@ -79,9 +79,7 @@ public static DataFrameTransformStats randomDataFrameTransformStats() { randomFrom(DataFrameTransformTaskState.values()), randomBoolean() ? null : randomAlphaOfLength(100), randomBoolean() ? null : randomNodeAttributes(), - // On the server side the stats has transform ID "_all" when embedded in another doc - // TODO: change this so that the outer document sets the correct transform ID during parsing - // It's very confusing and could cause subtle errors that the inner object has a surprising ID + // TODO: remove this ID field from the server side as it's no longer needed randomStats("_all"), DataFrameTransformCheckpointingInfoTests.randomDataFrameTransformCheckpointingInfo()); } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java index 3e1ba7e7e9f01..a735f2f40a9e9 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java @@ -676,8 +676,9 @@ protected void doSaveState(IndexerState indexerState, DataFrameIndexerPosition p getProgress()); logger.debug("Updating persistent state of transform [{}] to [{}]", transformConfig.getId(), state.toString()); - // Persisting stats when we call `doSaveState` should be ok as we only call it on a state transition and - // only every-so-often when doing the bulk indexing calls. See AsyncTwoPhaseIndexer#onBulkResponse for current periodicity + // Persist the current state and stats in the internal index. The interval of this method being + // called is controlled by AsyncTwoPhaseIndexer#onBulkResponse which calls doSaveState every so + // often when doing bulk indexing calls or at the end of one indexing run. transformsConfigManager.putOrUpdateTransformStoredDoc( new DataFrameTransformStoredDoc(transformId, state, getStats()), ActionListener.wrap( From c95594ac3c463969108c43a7bff3a9820ceba7ab Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 23 Jul 2019 08:46:43 +0100 Subject: [PATCH 12/12] Update test that was merged to PR branch --- .../rest-api-spec/test/data_frame/transforms_crud.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml index eac2490543605..881cddc710210 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml @@ -619,8 +619,7 @@ setup: transform_id: "airline-transform-start-delete" - match: { count: 1 } - match: { transforms.0.id: "airline-transform-start-delete" } - - match: { transforms.0.state.indexer_state: "/started|indexing/" } - - match: { transforms.0.state.task_state: "started" } + - match: { transforms.0.task_state: "started" } - do: catch: /Cannot delete data frame \[airline-transform-start-delete\] as the task is running/