From 31d938815c12b74dc48a026fcf51b5bfeeda40f7 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Wed, 26 Jun 2019 09:28:19 -0400 Subject: [PATCH 01/10] Add Cumulative Cardinality agg (and Data Science plugin) This adds a pipeline aggregation that calculates the cumulative cardinality of a field. It does this by iteratively merging in the HLL sketch from consecutive buckets and emitting the cardinality up to that point. This is useful for things like finding the total "new" users that have visited a website (as opposed to "repeat" visitors). This is a Basic+ aggregation and adds a new Data Science plugin to house it and future advanced analytics/data science aggregations. --- distribution/build.gradle | 1 + ...umulative-cardinality-aggregation.asciidoc | 230 ++++++++++++++++ .../metrics/InternalCardinality.java | 2 +- .../license/XPackLicenseState.java | 12 + .../elasticsearch/xpack/core/XPackField.java | 2 + x-pack/plugin/data-science/build.gradle | 26 ++ .../DataScienceAggregationBuilders.java | 15 ++ .../xpack/datascience/DataSciencePlugin.java | 50 ++++ ...CardinalityPipelineAggregationBuilder.java | 146 ++++++++++ ...mulativeCardinalityPipelineAggregator.java | 124 +++++++++ .../InternalSimpleLongValue.java | 90 +++++++ .../plugin-metadata/plugin-security.policy | 0 .../datascience/TestAggregatorFactory.java | 52 ++++ .../CumulativeCardinalityAggregatorTests.java | 249 ++++++++++++++++++ .../data_science/cumulative_cardinality.yml | 86 ++++++ 15 files changed, 1084 insertions(+), 1 deletion(-) create mode 100644 docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc create mode 100644 x-pack/plugin/data-science/build.gradle create mode 100644 x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceAggregationBuilders.java create mode 100644 x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java create mode 100644 x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java create mode 100644 x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java create mode 100644 x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/InternalSimpleLongValue.java create mode 100644 x-pack/plugin/data-science/src/main/plugin-metadata/plugin-security.policy create mode 100644 x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java create mode 100644 x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/data_science/cumulative_cardinality.yml diff --git a/distribution/build.gradle b/distribution/build.gradle index 1d081a2755f85..29c1c79a1ab85 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -409,6 +409,7 @@ task run(type: RunTask) { setting 'xpack.monitoring.enabled', 'true' setting 'xpack.sql.enabled', 'true' setting 'xpack.rollup.enabled', 'true' + setting 'xpack.data-science.enabled', 'true' keystoreSetting 'bootstrap.password', 'password' } } diff --git a/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc b/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc new file mode 100644 index 0000000000000..732ee1c521b14 --- /dev/null +++ b/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc @@ -0,0 +1,230 @@ +[role="xpack"] +[testenv="basic"] +[[search-aggregations-pipeline-cumulative-cardinality-aggregation]] +=== Cumulative Cardinality Aggregation + +A parent pipeline aggregation which calculates the Cumulative Cardinality in a parent histogram (or date_histogram) +aggregation. The specified metric must be a cardinality aggregation and the enclosing histogram +must have `min_doc_count` set to `0` (default for `histogram` aggregations). + +The `cumulative_cardinality` agg is useful for finding "total new items", like the number of new visitors to your +website each day. A regular cardinality aggregation will tell you how many unique visitors came each day, but doesn't +differentiate between "new" or "repeat" visitors. The Cumulative Cardinality aggregation can be used to determine +how many of each day's unique visitors are "new". + +==== Syntax + +A `cumulative_cardinality` aggregation looks like this in isolation: + +[source,js] +-------------------------------------------------- +{ + "cumulative_cardinality": { + "buckets_path": "my_cardinality_agg" + } +} +-------------------------------------------------- +// NOTCONSOLE + +[[cumulative-cardinality-params]] +.`cumulative_cardinality` Parameters +[options="header"] +|=== +|Parameter Name |Description |Required |Default Value +|`buckets_path` |The path to the cardinality aggregation we wish to find the cumulative cardinality for (see <> for more + details) |Required | +|`format` |format to apply to the output value of this aggregation |Optional |`null` +|=== + +The following snippet calculates the cumulative cardinality of the total monthly `sales`: + +[source,js] +-------------------------------------------------- +POST /sales/_search +{ + "size": 0, + "aggs" : { + "sales_per_month" : { + "date_histogram" : { + "field" : "date", + "calendar_interval" : "month" + }, + "aggs": { + "distinct_sale_types": { + "cardinality": { + "field": "type" + } + }, + "total_new_types": { + "cumulative_cardinality": { + "buckets_path": "distinct_sale_types" <1> + } + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:sales] + +<1> `buckets_path` instructs this aggregation to use the output of the `distinct_sale_types` aggregation for the cumulative cardinality + +And the following may be the response: + +[source,js] +-------------------------------------------------- +{ + "took": 11, + "timed_out": false, + "_shards": ..., + "hits": ..., + "aggregations": { + "sales_per_month": { + "buckets": [ + { + "key_as_string": "2015/01/01 00:00:00", + "key": 1420070400000, + "doc_count": 3, + "distinct_sale_types": { + "value": 3 + }, + "total_new_types": { + "value": 3 + } + }, + { + "key_as_string": "2015/02/01 00:00:00", + "key": 1422748800000, + "doc_count": 2, + "distinct_sale_types": { + "value": 2 + }, + "total_new_types": { + "value": 3 + } + }, + { + "key_as_string": "2015/03/01 00:00:00", + "key": 1425168000000, + "doc_count": 2, + "distinct_sale_types": { + "value": 2 + }, + "total_new_types": { + "value": 3 + } + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"took": 11/"took": $body.took/] +// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] + + +==== Incremental cumulative cardinality + +The `cumulative_cardinality` agg will show you the total, distinct count since the beginning of the time period +being queried. Sometimes, however, it is useful to see the "incremental" count. Meaning, how many new users +are added each day, rather than the total cumulative count. + +This can be accomplished by adding a `derivative` aggregation to our query: + +[source,js] +-------------------------------------------------- +POST /sales/_search +{ + "size": 0, + "aggs" : { + "sales_per_month" : { + "date_histogram" : { + "field" : "date", + "calendar_interval" : "month" + }, + "aggs": { + "distinct_sale_types": { + "cardinality": { + "field": "type" + } + }, + "total_new_types": { + "cumulative_cardinality": { + "buckets_path": "distinct_sale_types" + } + }, + "incremental_new_types": { + "derivative": { + "buckets_path": "total_new_types" + } + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:sales] + + +And the following may be the response: + +[source,js] +-------------------------------------------------- +{ + "took": 11, + "timed_out": false, + "_shards": ..., + "hits": ..., + "aggregations": { + "sales_per_month": { + "buckets": [ + { + "key_as_string": "2015/01/01 00:00:00", + "key": 1420070400000, + "doc_count": 3, + "distinct_sale_types": { + "value": 3 + }, + "total_new_types": { + "value": 3 + } + }, + { + "key_as_string": "2015/02/01 00:00:00", + "key": 1422748800000, + "doc_count": 2, + "distinct_sale_types": { + "value": 2 + }, + "total_new_types": { + "value": 3 + }, + "incremental_new_types": { + "value": 0.0 + } + }, + { + "key_as_string": "2015/03/01 00:00:00", + "key": 1425168000000, + "doc_count": 2, + "distinct_sale_types": { + "value": 2 + }, + "total_new_types": { + "value": 3 + }, + "incremental_new_types": { + "value": 0.0 + } + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"took": 11/"took": $body.took/] +// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCardinality.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCardinality.java index bfe82c6bef659..c3132a299042e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCardinality.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCardinality.java @@ -80,7 +80,7 @@ public long getValue() { return counts == null ? 0 : counts.cardinality(0); } - HyperLogLogPlusPlus getCounts() { + public HyperLogLogPlusPlus getCounts() { return counts; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 99c1e2e91715e..48d124e897d0f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -70,6 +70,9 @@ public class XPackLicenseState { "Creating and Starting rollup jobs will no longer be allowed.", "Stopping/Deleting existing jobs, RollupCaps API and RollupSearch continue to function." }); + messages.put(XPackField.DATA_SCIENCE, new String[] { + "Aggregations provided by Data Science plugin are no longer usable." + }); EXPIRATION_MESSAGES = Collections.unmodifiableMap(messages); } @@ -719,6 +722,15 @@ public synchronized boolean isOdbcAllowed() { return licensed && localStatus.active; } + /** + * Rollup is always available as long as there is a valid license + * + * @return true if the license is active + */ + public synchronized boolean isDataScienceAllowed() { + return status.active; + } + public synchronized boolean isTrialLicense() { return status.mode == OperationMode.TRIAL; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 54e0f58ae2867..0b95fbce380ac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -41,6 +41,8 @@ public final class XPackField { public static final String VECTORS = "vectors"; /** Name constant for the voting-only-node feature. */ public static final String VOTING_ONLY = "voting_only"; + /** Name constant for the data science plugin. */ + public static final String DATA_SCIENCE = "data_science"; private XPackField() {} diff --git a/x-pack/plugin/data-science/build.gradle b/x-pack/plugin/data-science/build.gradle new file mode 100644 index 0000000000000..815491451fd7d --- /dev/null +++ b/x-pack/plugin/data-science/build.gradle @@ -0,0 +1,26 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' +esplugin { + name 'x-pack-data-science' + description 'Elasticsearch Expanded Pack Plugin - Data Science' + classname 'org.elasticsearch.xpack.datascience.DataSciencePlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-data-science' + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" +compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" + + +dependencies { + compileOnly project(":server") + + compileOnly project(path: xpackModule('core'), configuration: 'default') + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + if (isEclipse) { + testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts') + } +} + +integTest.enabled = false diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceAggregationBuilders.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceAggregationBuilders.java new file mode 100644 index 0000000000000..fa2365db1b82c --- /dev/null +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceAggregationBuilders.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience; + +import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder; + +public class DataScienceAggregationBuilders { + + public static CumulativeCardinalityPipelineAggregationBuilder cumulativeCaardinality(String name, String bucketsPath) { + return new CumulativeCardinalityPipelineAggregationBuilder(name, bucketsPath); + } +} diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java new file mode 100644 index 0000000000000..4ad533d1c091c --- /dev/null +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder; +import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregator; + +import java.util.List; + +import static java.util.Collections.singletonList; + +public class DataSciencePlugin extends Plugin implements SearchPlugin { + + // volatile so all threads can see changes + protected static volatile boolean isDataScienceAllowed; + + public DataSciencePlugin() { + registerLicenseListener(); + } + + /** + * Protected for test over-riding + */ + protected void registerLicenseListener() { + // Add a listener to the license state and cache it when there is a change. + // Aggs could be called in high numbers so we don't want them contending on + // the synchronized isFooAllowed() methods + XPackPlugin.getSharedLicenseState() + .addListener(() -> isDataScienceAllowed = XPackPlugin.getSharedLicenseState().isDataScienceAllowed()); + } + + public static boolean isIsDataScienceAllowed() { + return isDataScienceAllowed; + } + + @Override + public List getPipelineAggregations() { + return singletonList(new PipelineAggregationSpec( + CumulativeCardinalityPipelineAggregationBuilder.NAME, + CumulativeCardinalityPipelineAggregationBuilder::new, + CumulativeCardinalityPipelineAggregator::new, + CumulativeCardinalityPipelineAggregationBuilder::parse)); + } +} diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java new file mode 100644 index 0000000000000..f9ced5016e333 --- /dev/null +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience.cumulativecardinality; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.BucketMetricsParser; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.datascience.DataSciencePlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH; +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT; + +public class CumulativeCardinalityPipelineAggregationBuilder + extends AbstractPipelineAggregationBuilder { + public static final String NAME = "cumulative_cardinality"; + + private String format; + + private static final Function> PARSER + = name -> { + ConstructingObjectParser parser = new ConstructingObjectParser<>( + CumulativeCardinalityPipelineAggregationBuilder.NAME, + false, + o -> new CumulativeCardinalityPipelineAggregationBuilder(name, (String) o[0])); + + parser.declareString(ConstructingObjectParser.constructorArg(), BUCKETS_PATH_FIELD); + parser.declareString(CumulativeCardinalityPipelineAggregationBuilder::format, FORMAT); + return parser; + }; + + public CumulativeCardinalityPipelineAggregationBuilder(String name, String bucketsPath) { + super(name, NAME, new String[] { bucketsPath }); + if (DataSciencePlugin.isIsDataScienceAllowed() == false) { + throw LicenseUtils.newComplianceException(XPackField.DATA_SCIENCE); + } + } + + /** + * Read from a stream. + */ + public CumulativeCardinalityPipelineAggregationBuilder(StreamInput in) throws IOException { + super(in, NAME); + format = in.readOptionalString(); + if (DataSciencePlugin.isIsDataScienceAllowed() == false) { + throw LicenseUtils.newComplianceException(XPackField.DATA_SCIENCE); + } + } + + @Override + protected final void doWriteTo(StreamOutput out) throws IOException { + out.writeOptionalString(format); + } + + /** + * Sets the format to use on the output of this aggregation. + */ + public CumulativeCardinalityPipelineAggregationBuilder format(String format) { + if (format == null) { + throw new IllegalArgumentException("[format] must not be null: [" + name + "]"); + } + this.format = format; + return this; + } + + /** + * Gets the format to use on the output of this aggregation. + */ + public String format() { + return format; + } + + protected DocValueFormat formatter() { + if (format != null) { + return new DocValueFormat.Decimal(format); + } else { + return DocValueFormat.RAW; + } + } + + @Override + protected PipelineAggregator createInternal(Map metaData) { + return new CumulativeCardinalityPipelineAggregator(name, bucketsPaths, formatter(), metaData); + } + + @Override + public void doValidate(AggregatorFactory parent, Collection aggFactories, + Collection pipelineAggregatorFactories) { + if (bucketsPaths.length != 1) { + throw new IllegalStateException(BUCKETS_PATH.getPreferredName() + + " must contain a single entry for aggregation [" + name + "]"); + } + + validateSequentiallyOrderedParentAggs(parent, NAME, name); + } + + @Override + protected final XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { + if (format != null) { + builder.field(BucketMetricsParser.FORMAT.getPreferredName(), format); + } + return builder; + } + + public static CumulativeCardinalityPipelineAggregationBuilder parse(String aggName, XContentParser parser) { + return PARSER.apply(aggName).apply(parser, null); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), format); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (super.equals(obj) == false) return false; + CumulativeCardinalityPipelineAggregationBuilder other = (CumulativeCardinalityPipelineAggregationBuilder) obj; + return Objects.equals(format, other.format); + } + + @Override + public String getWriteableName() { + return NAME; + } +} diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java new file mode 100644 index 0000000000000..bdbd13f51f7d8 --- /dev/null +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience.cumulativecardinality; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramFactory; +import org.elasticsearch.search.aggregations.metrics.HyperLogLogPlusPlus; +import org.elasticsearch.search.aggregations.metrics.InternalCardinality; +import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregationPath; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class CumulativeCardinalityPipelineAggregator extends PipelineAggregator { + private final DocValueFormat formatter; + + CumulativeCardinalityPipelineAggregator(String name, String[] bucketsPaths, DocValueFormat formatter, Map metadata) { + super(name, bucketsPaths, metadata); + this.formatter = formatter; + } + + /** + * Read from a stream. + */ + public CumulativeCardinalityPipelineAggregator(StreamInput in) throws IOException { + super(in); + formatter = in.readNamedWriteable(DocValueFormat.class); + } + + @Override + public void doWriteTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(formatter); + } + + @Override + public String getWriteableName() { + return CumulativeCardinalityPipelineAggregationBuilder.NAME; + } + + @Override + public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) { + InternalMultiBucketAggregation + histo = (InternalMultiBucketAggregation) aggregation; + List buckets = histo.getBuckets(); + HistogramFactory factory = (HistogramFactory) histo; + List newBuckets = new ArrayList<>(buckets.size()); + HyperLogLogPlusPlus hll = null; + + try { + long cardinality = 0; + for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { + HyperLogLogPlusPlus bucketHll = resolveBucketValue(histo, bucket, bucketsPaths()[0]); + if (hll == null && bucketHll != null) { + // We have to clone the HLL because otherwise it will alter the + // existing cardinality sketch and bucket value + hll = new HyperLogLogPlusPlus(bucketHll.precision(), reduceContext.bigArrays(), 1); + } + if (bucketHll != null) { + hll.merge(0, bucketHll, 0); + cardinality = hll.cardinality(0); + } + + List aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false) + .map((p) -> (InternalAggregation) p) + .collect(Collectors.toList()); + aggs.add(new InternalSimpleLongValue(name(), cardinality, formatter, new ArrayList<>(), metaData())); + Bucket newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs)); + newBuckets.add(newBucket); + } + return factory.createAggregation(newBuckets); + } finally { + if (hll != null) { + hll.close(); + } + } + } + + private HyperLogLogPlusPlus resolveBucketValue(MultiBucketsAggregation agg, + InternalMultiBucketAggregation.InternalBucket bucket, + String aggPath) { + List aggPathsList = AggregationPath.parse(aggPath).getPathElementsAsStringList(); + Object propertyValue = bucket.getProperty(agg.getName(), aggPathsList); + if (propertyValue == null) { + throw new AggregationExecutionException(AbstractPipelineAggregationBuilder.BUCKETS_PATH_FIELD.getPreferredName() + + " must reference a cardinality aggregation"); + } + + if (propertyValue instanceof InternalCardinality) { + return ((InternalCardinality) propertyValue).getCounts(); + } + + String currentAggName; + if (aggPathsList.isEmpty()) { + currentAggName = agg.getName(); + } else { + currentAggName = aggPathsList.get(0); + } + + throw new AggregationExecutionException(AbstractPipelineAggregationBuilder.BUCKETS_PATH_FIELD.getPreferredName() + + " must reference a cardinality aggregation, got: [" + + propertyValue.getClass().getSimpleName() + "] at aggregation [" + currentAggName + "]"); + } + +} diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/InternalSimpleLongValue.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/InternalSimpleLongValue.java new file mode 100644 index 0000000000000..2b3414ad3ddc9 --- /dev/null +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/InternalSimpleLongValue.java @@ -0,0 +1,90 @@ +package org.elasticsearch.xpack.datascience.cumulativecardinality; + + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.pipeline.SimpleValue; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class InternalSimpleLongValue extends InternalNumericMetricsAggregation.SingleValue implements SimpleValue { + public static final String NAME = "simple_long_value"; + protected final long value; + + public InternalSimpleLongValue(String name, long value, DocValueFormat formatter, List pipelineAggregators, + Map metaData) { + super(name, pipelineAggregators, metaData); + this.format = formatter; + this.value = value; + } + + /** + * Read from a stream. + */ + public InternalSimpleLongValue(StreamInput in) throws IOException { + super(in); + format = in.readNamedWriteable(DocValueFormat.class); + value = in.readZLong(); + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(format); + out.writeZLong(value); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public double value() { + return value; + } + + public long getValue() { + return value; + } + + DocValueFormat formatter() { + return format; + } + + @Override + public InternalSimpleLongValue doReduce(List aggregations, ReduceContext reduceContext) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + boolean hasValue = !(Double.isInfinite(value) || Double.isNaN(value)); + builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null); + if (hasValue && format != DocValueFormat.RAW) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value).toString()); + } + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), value); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (super.equals(obj) == false) return false; + InternalSimpleLongValue other = (InternalSimpleLongValue) obj; + return Objects.equals(value, other.value); + } +} diff --git a/x-pack/plugin/data-science/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/data-science/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java new file mode 100644 index 0000000000000..f54fad6221d57 --- /dev/null +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.datascience; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.MockPageCacheRecycler; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test implementation for AggregatorFactory. + */ +public class TestAggregatorFactory extends AggregatorFactory { + + private final Aggregator aggregator; + + private TestAggregatorFactory(SearchContext context, Aggregator aggregator) throws IOException { + super("_name", context, null, new AggregatorFactories.Builder(), Collections.emptyMap()); + this.aggregator = aggregator; + } + + @Override + protected Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List list, Map metaData) throws IOException { + return aggregator; + } + + public static TestAggregatorFactory createInstance() throws IOException { + BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); + SearchContext searchContext = mock(SearchContext.class); + when(searchContext.bigArrays()).thenReturn(bigArrays); + + Aggregator aggregator = mock(Aggregator.class); + + return new TestAggregatorFactory(searchContext, aggregator); + } +} diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java new file mode 100644 index 0000000000000..8e355cecf7dc6 --- /dev/null +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java @@ -0,0 +1,249 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.datascience.cumulativecardinality; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.datascience.TestAggregatorFactory; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalOrder; +import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregatorFactory; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorFactory; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.datascience.DataSciencePlugin; +import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder; +import org.elasticsearch.xpack.datascience.cumulativecardinality.InternalSimpleLongValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + +public class CumulativeCardinalityAggregatorTests extends AggregatorTestCase { + + private static final String HISTO_FIELD = "histo"; + private static final String VALUE_FIELD = "value_field"; + + private static final List datasetTimes = Arrays.asList( + "2017-01-01T01:07:45", //1 + "2017-01-01T03:43:34", //1 + "2017-01-03T04:11:00", //3 + "2017-01-03T05:11:31", //1 + "2017-01-05T08:24:05", //5 + "2017-01-05T13:09:32", //1 + "2017-01-07T13:47:43", //7 + "2017-01-08T16:14:34", //1 + "2017-01-09T17:09:50", //9 + "2017-01-09T22:55:46");//10 + + private static final List datasetValues = Arrays.asList(1,1,3,1,5,1,7,1,9,10); + private static final List cumulativeCardinality = Arrays.asList(1.0,1.0,2.0,2.0,3.0,3.0,4.0,4.0,6.0); + + // Initialize plugin so we can set license state + static DataSciencePlugin PLUGIN = new DataSciencePlugin() { + @Override + protected void registerLicenseListener() { + DataSciencePlugin.isDataScienceAllowed = true; + } + }; + + public void testSimple() throws IOException { + + Query query = new MatchAllDocsQuery(); + + DateHistogramAggregationBuilder aggBuilder = new DateHistogramAggregationBuilder("histo"); + aggBuilder.calendarInterval(DateHistogramInterval.DAY).field(HISTO_FIELD); + aggBuilder.subAggregation(new CardinalityAggregationBuilder("the_cardinality", ValueType.NUMERIC).field(VALUE_FIELD)); + aggBuilder.subAggregation(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "the_cardinality")); + + executeTestCase(query, aggBuilder, histogram -> { + assertEquals(9, ((Histogram)histogram).getBuckets().size()); + List buckets = ((Histogram)histogram).getBuckets(); + int counter = 0; + for (Histogram.Bucket bucket : buckets) { + assertThat(((InternalSimpleLongValue) (bucket.getAggregations().get("cumulative_card"))).value(), + equalTo(cumulativeCardinality.get(counter))); + counter += 1; + } + }); + } + + public void testAllNull() throws IOException { + Query query = new MatchAllDocsQuery(); + + DateHistogramAggregationBuilder aggBuilder = new DateHistogramAggregationBuilder("histo"); + aggBuilder.calendarInterval(DateHistogramInterval.DAY).field(HISTO_FIELD); + aggBuilder.subAggregation(new CardinalityAggregationBuilder("the_cardinality", ValueType.NUMERIC).field("foo")); + aggBuilder.subAggregation(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "the_cardinality")); + + executeTestCase(query, aggBuilder, histogram -> { + assertEquals(9, ((Histogram)histogram).getBuckets().size()); + List buckets = ((Histogram)histogram).getBuckets(); + for (Histogram.Bucket bucket : buckets) { + assertThat(((InternalSimpleLongValue) (bucket.getAggregations().get("cumulative_card"))).value(), equalTo(0.0)); + } + }); + } + + /** + * The validation should verify the parent aggregation is allowed. + */ + public void testValidate() throws IOException { + final Set aggBuilders = new HashSet<>(); + aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cusum", "sum")); + + final CumulativeCardinalityPipelineAggregationBuilder builder + = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid"); + builder.validate(getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders); + } + + /** + * The validation should throw an IllegalArgumentException, since parent + * aggregation is not a type of HistogramAggregatorFactory, + * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory. + */ + public void testValidateException() throws IOException { + final Set aggBuilders = new HashSet<>(); + aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum")); + TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance(); + + final CumulativeCardinalityPipelineAggregationBuilder builder + = new CumulativeCardinalityPipelineAggregationBuilder("name", "invalid_agg>metric"); + IllegalStateException ex = expectThrows(IllegalStateException.class, + () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders)); + assertEquals("cumulative_cardinality aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent", + ex.getMessage()); + } + + public void testNonCardinalityAgg() { + Query query = new MatchAllDocsQuery(); + + DateHistogramAggregationBuilder aggBuilder = new DateHistogramAggregationBuilder("histo"); + aggBuilder.calendarInterval(DateHistogramInterval.DAY).field(HISTO_FIELD); + aggBuilder.subAggregation(new SumAggregationBuilder("the_sum").field("foo")); + aggBuilder.subAggregation(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "the_sum")); + + AggregationExecutionException e = expectThrows(AggregationExecutionException.class, + () -> executeTestCase(query, aggBuilder, histogram -> fail("Test should not have executed"))); + assertThat(e.getMessage(), equalTo("buckets_path must reference a cardinality aggregation, " + + "got: [InternalSum] at aggregation [the_sum]")); + } + + private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consumer verify) throws IOException { + executeTestCase(query, aggBuilder, verify, indexWriter -> { + Document document = new Document(); + int counter = 0; + for (String date : datasetTimes) { + if (frequently()) { + indexWriter.commit(); + } + + long instant = asLong(date); + document.add(new SortedNumericDocValuesField(HISTO_FIELD, instant)); + document.add(new NumericDocValuesField(VALUE_FIELD, datasetValues.get(counter))); + indexWriter.addDocument(document); + document.clear(); + counter += 1; + } + }); + } + + private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consumer verify, + CheckedConsumer setup) throws IOException { + + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + setup.accept(indexWriter); + } + + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + + DateFieldMapper.Builder builder = new DateFieldMapper.Builder("_name"); + DateFieldMapper.DateFieldType fieldType = builder.fieldType(); + fieldType.setHasDocValues(true); + fieldType.setName(HISTO_FIELD); + + MappedFieldType valueFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + valueFieldType.setHasDocValues(true); + valueFieldType.setName("value_field"); + + InternalAggregation histogram; + histogram = searchAndReduce(indexSearcher, query, aggBuilder, fieldType, valueFieldType); + verify.accept(histogram); + } + } + } + + private static long asLong(String dateTime) { + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + } + + + private static AggregatorFactory getRandomSequentiallyOrderedParentAgg() throws IOException { + AggregatorFactory factory; + ValuesSourceConfig numericVS = new ValuesSourceConfig<>(ValuesSourceType.NUMERIC); + switch (randomIntBetween(0, 2)) { + case 0: + factory = new HistogramAggregatorFactory("name", numericVS, 0.0d, 0.0d, + mock(InternalOrder.class), false, 0L, 0.0d, 1.0d, mock(SearchContext.class), null, + new AggregatorFactories.Builder(), Collections.emptyMap()); + break; + case 1: + factory = new DateHistogramAggregatorFactory("name", numericVS, 0L, + mock(InternalOrder.class), false, 0L, mock(Rounding.class), mock(Rounding.class), + mock(ExtendedBounds.class), mock(SearchContext.class), mock(AggregatorFactory.class), + new AggregatorFactories.Builder(), Collections.emptyMap()); + break; + case 2: + default: + AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = new AutoDateHistogramAggregationBuilder.RoundingInfo[1]; + factory = new AutoDateHistogramAggregatorFactory("name", numericVS, + 1, roundings, + mock(SearchContext.class), null, new AggregatorFactories.Builder(), Collections.emptyMap()); + } + + return factory; + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_science/cumulative_cardinality.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_science/cumulative_cardinality.yml new file mode 100644 index 0000000000000..b59912e86f2a5 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_science/cumulative_cardinality.yml @@ -0,0 +1,86 @@ +setup: + - skip: + features: headers + - do: + indices.create: + index: foo + body: + mappings: + properties: + timestamp: + type: date + user: + type: keyword + + + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + bulk: + refresh: true + body: + - index: + _index: "foo" + - timestamp: "2017-01-01T05:00:00Z" + user: "a" + + - index: + _index: "foo" + - timestamp: "2017-01-01T05:00:00Z" + user: "b" + + - index: + _index: "foo" + - timestamp: "2017-01-01T05:00:00Z" + user: "c" + + - index: + _index: "foo" + - timestamp: "2017-01-02T05:00:00Z" + user: "a" + + - index: + _index: "foo" + - timestamp: "2017-01-02T05:00:00Z" + user: "b" + + - index: + _index: "foo" + - timestamp: "2017-01-03T05:00:00Z" + user: "d" + +--- +"Basic Search": + + - do: + search: + index: "foo" + body: + size: 0 + aggs: + histo: + date_histogram: + field: "timestamp" + calendar_interval: "day" + aggs: + distinct_users: + cardinality: + field: "user" + total_users: + cumulative_cardinality: + buckets_path: "distinct_users" + + - length: { aggregations.histo.buckets: 3 } + - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T00:00:00.000Z" } + - match: { aggregations.histo.buckets.0.doc_count: 3 } + - match: { aggregations.histo.buckets.0.distinct_users.value: 3 } + - match: { aggregations.histo.buckets.0.total_users.value: 3 } + - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-02T00:00:00.000Z" } + - match: { aggregations.histo.buckets.1.doc_count: 2 } + - match: { aggregations.histo.buckets.1.distinct_users.value: 2 } + - match: { aggregations.histo.buckets.1.total_users.value: 3 } + - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-03T00:00:00.000Z" } + - match: { aggregations.histo.buckets.2.doc_count: 1 } + - match: { aggregations.histo.buckets.2.distinct_users.value: 1 } + - match: { aggregations.histo.buckets.2.total_users.value: 4 } + From 1cc0fa809dc27199283ab14a8bfe16359ae327a5 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 27 Jun 2019 09:36:20 -0400 Subject: [PATCH 02/10] checkstyle --- .../CumulativeCardinalityPipelineAggregator.java | 1 - .../cumulativecardinality/InternalSimpleLongValue.java | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java index bdbd13f51f7d8..07aa2339bc8e0 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java @@ -19,7 +19,6 @@ import org.elasticsearch.search.aggregations.metrics.HyperLogLogPlusPlus; import org.elasticsearch.search.aggregations.metrics.InternalCardinality; import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationPath; diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/InternalSimpleLongValue.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/InternalSimpleLongValue.java index 2b3414ad3ddc9..bd9c7903f4a99 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/InternalSimpleLongValue.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/InternalSimpleLongValue.java @@ -1,6 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ package org.elasticsearch.xpack.datascience.cumulativecardinality; - import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; From c754b0fe2fb73c1ff6e92c0a74d4254209bfbc07 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Wed, 31 Jul 2019 19:01:22 -0400 Subject: [PATCH 03/10] Address review comments --- docs/build.gradle | 36 +++++ ...umulative-cardinality-aggregation.asciidoc | 133 +++++++++--------- .../license/XPackLicenseState.java | 2 +- ...mulativeCardinalityPipelineAggregator.java | 2 +- .../CumulativeCardinalityAggregatorTests.java | 55 +++++--- 5 files changed, 143 insertions(+), 85 deletions(-) diff --git a/docs/build.gradle b/docs/build.gradle index 08cb2de971320..874abd51652b3 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -214,6 +214,42 @@ buildRestTests.setups['sales'] = ''' {"index":{}} {"date": "2015/03/01 00:00:00", "price": 175, "promoted": false, "rating": 2, "type": "t-shirt"}''' +// Used by cumulative cardinality aggregation docs +buildRestTests.setups['user_hits'] = ''' + - do: + indices.create: + index: user_hits + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + user_id: + type: keyword + timestamp: + type: date + - do: + bulk: + index: user_hits + refresh: true + body: | + {"index":{}} + {"timestamp": "2019-01-01T13:00:00", "user_id": "1"} + {"index":{}} + {"timestamp": "2019-01-01T13:00:00", "user_id": "2"} + {"index":{}} + {"timestamp": "2019-01-02T13:00:00", "user_id": "1"} + {"index":{}} + {"timestamp": "2019-01-02T13:00:00", "user_id": "3"} + {"index":{}} + {"timestamp": "2019-01-03T13:00:00", "user_id": "1"} + {"index":{}} + {"timestamp": "2019-01-03T13:00:00", "user_id": "2"} + {"index":{}} + {"timestamp": "2019-01-03T13:00:00", "user_id": "4"}''' + + // Dummy bank account data used by getting-started.asciidoc buildRestTests.setups['bank'] = ''' - do: diff --git a/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc b/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc index 732ee1c521b14..2e316164d64b9 100644 --- a/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc @@ -36,28 +36,28 @@ A `cumulative_cardinality` aggregation looks like this in isolation: |`format` |format to apply to the output value of this aggregation |Optional |`null` |=== -The following snippet calculates the cumulative cardinality of the total monthly `sales`: +The following snippet calculates the cumulative cardinality of the total daily `users`: [source,js] -------------------------------------------------- -POST /sales/_search +GET /user_hits/_search { "size": 0, "aggs" : { - "sales_per_month" : { + "users_per_day" : { "date_histogram" : { - "field" : "date", - "calendar_interval" : "month" + "field" : "timestamp", + "calendar_interval" : "day" }, "aggs": { - "distinct_sale_types": { + "distinct_users": { "cardinality": { - "field": "type" + "field": "user_id" } }, - "total_new_types": { + "total_new_users": { "cumulative_cardinality": { - "buckets_path": "distinct_sale_types" <1> + "buckets_path": "distinct_users" <1> } } } @@ -66,9 +66,9 @@ POST /sales/_search } -------------------------------------------------- // CONSOLE -// TEST[setup:sales] +// TEST[setup:user_hits] -<1> `buckets_path` instructs this aggregation to use the output of the `distinct_sale_types` aggregation for the cumulative cardinality +<1> `buckets_path` instructs this aggregation to use the output of the `distinct_users` aggregation for the cumulative cardinality And the following may be the response: @@ -80,39 +80,39 @@ And the following may be the response: "_shards": ..., "hits": ..., "aggregations": { - "sales_per_month": { + "users_per_day": { "buckets": [ { - "key_as_string": "2015/01/01 00:00:00", - "key": 1420070400000, - "doc_count": 3, - "distinct_sale_types": { - "value": 3 + "key_as_string": "2019-01-01T00:00:00.000Z", + "key": 1546300800000, + "doc_count": 2, + "distinct_users": { + "value": 2 }, - "total_new_types": { - "value": 3 + "total_new_users": { + "value": 2 } }, { - "key_as_string": "2015/02/01 00:00:00", - "key": 1422748800000, + "key_as_string": "2019-01-02T00:00:00.000Z", + "key": 1546387200000, "doc_count": 2, - "distinct_sale_types": { + "distinct_users": { "value": 2 }, - "total_new_types": { + "total_new_users": { "value": 3 } }, { - "key_as_string": "2015/03/01 00:00:00", - "key": 1425168000000, - "doc_count": 2, - "distinct_sale_types": { - "value": 2 - }, - "total_new_types": { + "key_as_string": "2019-01-03T00:00:00.000Z", + "key": 1546473600000, + "doc_count": 3, + "distinct_users": { "value": 3 + }, + "total_new_users": { + "value": 4 } } ] @@ -125,6 +125,11 @@ And the following may be the response: // TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] +Note how the second day, `2019-01-02`, has two distinct users but the `total_new_users` metric generated by the +cumulative pipeline agg only increments to three. This means that only one of the two users that day were +new, the other had already been seen in the previous day. This happens again on the third day, where only +one of three users is completely new. + ==== Incremental cumulative cardinality The `cumulative_cardinality` agg will show you the total, distinct count since the beginning of the time period @@ -135,29 +140,29 @@ This can be accomplished by adding a `derivative` aggregation to our query: [source,js] -------------------------------------------------- -POST /sales/_search +GET /user_hits/_search { "size": 0, "aggs" : { - "sales_per_month" : { + "users_per_day" : { "date_histogram" : { - "field" : "date", - "calendar_interval" : "month" + "field" : "timestamp", + "calendar_interval" : "day" }, "aggs": { - "distinct_sale_types": { + "distinct_users": { "cardinality": { - "field": "type" + "field": "user_id" } }, - "total_new_types": { + "total_new_users": { "cumulative_cardinality": { - "buckets_path": "distinct_sale_types" + "buckets_path": "distinct_users" } }, - "incremental_new_types": { + "incremental_new_users": { "derivative": { - "buckets_path": "total_new_types" + "buckets_path": "total_new_users" } } } @@ -166,7 +171,7 @@ POST /sales/_search } -------------------------------------------------- // CONSOLE -// TEST[setup:sales] +// TEST[setup:user_hits] And the following may be the response: @@ -179,45 +184,45 @@ And the following may be the response: "_shards": ..., "hits": ..., "aggregations": { - "sales_per_month": { + "users_per_day": { "buckets": [ { - "key_as_string": "2015/01/01 00:00:00", - "key": 1420070400000, - "doc_count": 3, - "distinct_sale_types": { - "value": 3 + "key_as_string": "2019-01-01T00:00:00.000Z", + "key": 1546300800000, + "doc_count": 2, + "distinct_users": { + "value": 2 }, - "total_new_types": { - "value": 3 + "total_new_users": { + "value": 2 } }, { - "key_as_string": "2015/02/01 00:00:00", - "key": 1422748800000, + "key_as_string": "2019-01-02T00:00:00.000Z", + "key": 1546387200000, "doc_count": 2, - "distinct_sale_types": { + "distinct_users": { "value": 2 }, - "total_new_types": { + "total_new_users": { "value": 3 }, - "incremental_new_types": { - "value": 0.0 + "incremental_new_users": { + "value": 1.0 } }, { - "key_as_string": "2015/03/01 00:00:00", - "key": 1425168000000, - "doc_count": 2, - "distinct_sale_types": { - "value": 2 - }, - "total_new_types": { + "key_as_string": "2019-01-03T00:00:00.000Z", + "key": 1546473600000, + "doc_count": 3, + "distinct_users": { "value": 3 }, - "incremental_new_types": { - "value": 0.0 + "total_new_users": { + "value": 4 + }, + "incremental_new_users": { + "value": 1.0 } } ] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 48d124e897d0f..6dee11ac7f1a5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -723,7 +723,7 @@ public synchronized boolean isOdbcAllowed() { } /** - * Rollup is always available as long as there is a valid license + * Datascience is always available as long as there is a valid license * * @return true if the license is active */ diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java index 07aa2339bc8e0..5e79c1f3095ad 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregator.java @@ -70,7 +70,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { HyperLogLogPlusPlus bucketHll = resolveBucketValue(histo, bucket, bucketsPaths()[0]); if (hll == null && bucketHll != null) { - // We have to clone the HLL because otherwise it will alter the + // We have to create a new HLL because otherwise it will alter the // existing cardinality sketch and bucket value hll = new HyperLogLogPlusPlus(bucketHll.precision(), reduceContext.bigArrays(), 1); } diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java index 8e355cecf7dc6..d98e6c728ea2b 100644 --- a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java @@ -126,32 +126,48 @@ public void testAllNull() throws IOException { }); } - /** - * The validation should verify the parent aggregation is allowed. - */ - public void testValidate() throws IOException { - final Set aggBuilders = new HashSet<>(); - aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cusum", "sum")); - - final CumulativeCardinalityPipelineAggregationBuilder builder + public void testParentValidations() throws IOException { + ValuesSourceConfig numericVS = new ValuesSourceConfig<>(ValuesSourceType.NUMERIC); + + // Histogram + Set aggBuilders = new HashSet<>(); + aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum")); + AggregatorFactory parent = new HistogramAggregatorFactory("name", numericVS, 0.0d, 0.0d, + mock(InternalOrder.class), false, 0L, 0.0d, 1.0d, mock(SearchContext.class), null, + new AggregatorFactories.Builder(), Collections.emptyMap()); + CumulativeCardinalityPipelineAggregationBuilder builder = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid"); - builder.validate(getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders); - } + builder.validate(parent, Collections.emptySet(), aggBuilders); - /** - * The validation should throw an IllegalArgumentException, since parent - * aggregation is not a type of HistogramAggregatorFactory, - * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory. - */ - public void testValidateException() throws IOException { - final Set aggBuilders = new HashSet<>(); + // Date Histogram + aggBuilders.clear(); + aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum")); + parent = new DateHistogramAggregatorFactory("name", numericVS, 0L, + mock(InternalOrder.class), false, 0L, mock(Rounding.class), mock(Rounding.class), + mock(ExtendedBounds.class), mock(SearchContext.class), mock(AggregatorFactory.class), + new AggregatorFactories.Builder(), Collections.emptyMap()); + builder = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid"); + builder.validate(parent, Collections.emptySet(), aggBuilders); + + // Auto Date Histogram + aggBuilders.clear(); + aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum")); + AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = new AutoDateHistogramAggregationBuilder.RoundingInfo[1]; + parent = new AutoDateHistogramAggregatorFactory("name", numericVS, + 1, roundings, + mock(SearchContext.class), null, new AggregatorFactories.Builder(), Collections.emptyMap()); + builder = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid"); + builder.validate(parent, Collections.emptySet(), aggBuilders); + + // Mocked "test" agg, should fail validation + aggBuilders.clear(); aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum")); TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance(); - final CumulativeCardinalityPipelineAggregationBuilder builder + CumulativeCardinalityPipelineAggregationBuilder failBuilder = new CumulativeCardinalityPipelineAggregationBuilder("name", "invalid_agg>metric"); IllegalStateException ex = expectThrows(IllegalStateException.class, - () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders)); + () -> failBuilder.validate(parentFactory, Collections.emptySet(), aggBuilders)); assertEquals("cumulative_cardinality aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent", ex.getMessage()); } @@ -192,6 +208,7 @@ private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consume private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consumer verify, CheckedConsumer setup) throws IOException { + try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { setup.accept(indexWriter); From faaf31cc73dc8218072d3ad7a570d08838795c3c Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 1 Aug 2019 16:25:24 -0400 Subject: [PATCH 04/10] Fix merge conflicts --- .../CumulativeCardinalityPipelineAggregationBuilder.java | 2 +- .../org/elasticsearch/datascience/TestAggregatorFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java index f9ced5016e333..bda623b67f0a7 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java @@ -103,7 +103,7 @@ protected PipelineAggregator createInternal(Map metaData) { } @Override - public void doValidate(AggregatorFactory parent, Collection aggFactories, + public void doValidate(AggregatorFactory parent, Collection aggFactories, Collection pipelineAggregatorFactories) { if (bucketsPaths.length != 1) { throw new IllegalStateException(BUCKETS_PATH.getPreferredName() diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java index f54fad6221d57..0ed1099276a4f 100644 --- a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java @@ -26,7 +26,7 @@ /** * Test implementation for AggregatorFactory. */ -public class TestAggregatorFactory extends AggregatorFactory { +public class TestAggregatorFactory extends AggregatorFactory { private final Aggregator aggregator; From 7f434462e1d2d3d5a81d410548f2026f71651545 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Mon, 12 Aug 2019 12:12:34 -0400 Subject: [PATCH 05/10] Add basic usage/info functionality to data science plugin --- .../xpack/core/XPackSettings.java | 4 + .../core/action/XPackInfoFeatureAction.java | 3 +- .../core/action/XPackUsageFeatureAction.java | 3 +- .../DataScienceFeatureSetUsage.java | 49 +++++++++++++ .../DataScienceInfoTransportAction.java | 46 ++++++++++++ .../xpack/datascience/DataSciencePlugin.java | 15 +++- .../DataScienceUsageTransportAction.java | 49 +++++++++++++ .../DataScienceInfoTransportActionTests.java | 73 +++++++++++++++++++ .../datascience/TestAggregatorFactory.java | 2 +- .../CumulativeCardinalityAggregatorTests.java | 6 +- 10 files changed, 242 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java create mode 100644 x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportAction.java create mode 100644 x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceUsageTransportAction.java create mode 100644 x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportActionTests.java rename x-pack/plugin/data-science/src/test/java/org/elasticsearch/{ => xpack}/datascience/TestAggregatorFactory.java (97%) rename x-pack/plugin/data-science/src/test/java/org/elasticsearch/{ => xpack}/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java (97%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 618b4c86a9126..055c80bfc1f6f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -121,6 +121,10 @@ private XPackSettings() { /** Setting for enabling or disabling vectors. Defaults to true. */ public static final Setting VECTORS_ENABLED = Setting.boolSetting("xpack.vectors.enabled", true, Setting.Property.NodeScope); + /** Setting for enabling or disabling data science plugin. Defaults to true. */ + public static final Setting DATA_SCIENCE_ENABLED = Setting.boolSetting("xpack.datascience.enabled", + true, Setting.Property.NodeScope); + /* * SSL settings. These are the settings that are specifically registered for SSL. Many are private as we do not explicitly use them * but instead parse based on a prefix (eg *.ssl.*) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java index 2019256bb27e3..6be11ea8aeae4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java @@ -38,10 +38,11 @@ public class XPackInfoFeatureAction extends ActionType public static final XPackInfoFeatureAction VOTING_ONLY = new XPackInfoFeatureAction(XPackField.VOTING_ONLY); public static final XPackInfoFeatureAction FROZEN_INDICES = new XPackInfoFeatureAction(XPackField.FROZEN_INDICES); public static final XPackInfoFeatureAction SPATIAL = new XPackInfoFeatureAction(XPackField.SPATIAL); + public static final XPackInfoFeatureAction DATA_SCIENCE = new XPackInfoFeatureAction(XPackField.DATA_SCIENCE); public static final List ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, CCR, DATA_FRAME, FLATTENED, - VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL + VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, DATA_SCIENCE ); private XPackInfoFeatureAction(String name) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index e5a9eca8f1fc5..c08453af03735 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -38,10 +38,11 @@ public class XPackUsageFeatureAction extends ActionType ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, CCR, DATA_FRAME, FLATTENED, - VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL + VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, DATA_SCIENCE ); private XPackUsageFeatureAction(String name) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java new file mode 100644 index 0000000000000..53c76e8ea0871 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.datascience; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.Objects; + +public class DataScienceFeatureSetUsage extends XPackFeatureSet.Usage { + + public DataScienceFeatureSetUsage(boolean available, boolean enabled) { + super(XPackField.DATA_SCIENCE, available, enabled); + } + + public DataScienceFeatureSetUsage(StreamInput input) throws IOException { + super(input); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash(available, enabled); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + DataScienceFeatureSetUsage other = (DataScienceFeatureSetUsage) obj; + return Objects.equals(available, other.available) && + Objects.equals(enabled, other.enabled); + } +} diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportAction.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportAction.java new file mode 100644 index 0000000000000..8d53d5cbeeae4 --- /dev/null +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportAction.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience; + +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; + +public class DataScienceInfoTransportAction extends XPackInfoFeatureTransportAction { + + private final boolean enabled; + private final XPackLicenseState licenseState; + + @Inject + public DataScienceInfoTransportAction(TransportService transportService, ActionFilters actionFilters, + Settings settings, XPackLicenseState licenseState) { + super(XPackInfoFeatureAction.DATA_SCIENCE.name(), transportService, actionFilters); + this.enabled = XPackSettings.DATA_SCIENCE_ENABLED.get(settings); + this.licenseState = licenseState; + } + + @Override + public String name() { + return XPackField.DATA_SCIENCE; + } + + @Override + public boolean available() { + return licenseState.isDataScienceAllowed(); + } + + @Override + public boolean enabled() { + return enabled; + } + +} diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java index 4ad533d1c091c..6005dc64f0896 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java @@ -5,17 +5,23 @@ */ package org.elasticsearch.xpack.datascience; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder; import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregator; +import java.util.Arrays; import java.util.List; import static java.util.Collections.singletonList; -public class DataSciencePlugin extends Plugin implements SearchPlugin { +public class DataSciencePlugin extends Plugin implements SearchPlugin, ActionPlugin { // volatile so all threads can see changes protected static volatile boolean isDataScienceAllowed; @@ -47,4 +53,11 @@ public List getPipelineAggregations() { CumulativeCardinalityPipelineAggregator::new, CumulativeCardinalityPipelineAggregationBuilder::parse)); } + + @Override + public List> getActions() { + return Arrays.asList( + new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.DATA_SCIENCE, DataScienceUsageTransportAction.class), + new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.DATA_SCIENCE, DataScienceInfoTransportAction.class)); + } } diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceUsageTransportAction.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceUsageTransportAction.java new file mode 100644 index 0000000000000..79ba45a0b84e2 --- /dev/null +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceUsageTransportAction.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; +import org.elasticsearch.xpack.core.datascience.DataScienceFeatureSetUsage; + +public class DataScienceUsageTransportAction extends XPackUsageFeatureTransportAction { + private final Settings settings; + private final XPackLicenseState licenseState; + + @Inject + public DataScienceUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + Settings settings, XPackLicenseState licenseState) { + super(XPackUsageFeatureAction.DATA_SCIENCE.name(), transportService, clusterService, + threadPool, actionFilters, indexNameExpressionResolver); + this.settings = settings; + this.licenseState = licenseState; + } + + @Override + protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state, + ActionListener listener) { + boolean available = licenseState.isDataScienceAllowed(); + + DataScienceFeatureSetUsage usage = + new DataScienceFeatureSetUsage(available, XPackSettings.DATA_SCIENCE_ENABLED.get(settings)); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + } +} diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportActionTests.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportActionTests.java new file mode 100644 index 0000000000000..97dab41169033 --- /dev/null +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportActionTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience; + +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.datascience.DataScienceFeatureSetUsage; +import org.junit.Before; + +import static org.hamcrest.core.Is.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DataScienceInfoTransportActionTests extends ESTestCase { + + private XPackLicenseState licenseState; + + @Before + public void init() { + licenseState = mock(XPackLicenseState.class); + } + + public void testAvailable() throws Exception { + DataScienceInfoTransportAction featureSet = new DataScienceInfoTransportAction( + mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState); + boolean available = randomBoolean(); + when(licenseState.isDataScienceAllowed()).thenReturn(available); + assertThat(featureSet.available(), is(available)); + + DataScienceUsageTransportAction usageAction = new DataScienceUsageTransportAction(mock(TransportService.class), null, null, + mock(ActionFilters.class), null, Settings.EMPTY, licenseState); + PlainActionFuture future = new PlainActionFuture<>(); + usageAction.masterOperation(null, null, null, future); + XPackFeatureSet.Usage usage = future.get().getUsage(); + assertThat(usage.available(), is(available)); + + BytesStreamOutput out = new BytesStreamOutput(); + usage.writeTo(out); + XPackFeatureSet.Usage serializedUsage = new DataScienceFeatureSetUsage(out.bytes().streamInput()); + assertThat(serializedUsage.available(), is(available)); + } + + public void testEnabled() throws Exception { + Settings.Builder settings = Settings.builder(); + DataScienceInfoTransportAction featureSet = new DataScienceInfoTransportAction( + mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState); + assertThat(featureSet.enabled(), is(true)); + assertTrue(featureSet.enabled()); + + DataScienceUsageTransportAction usageAction = new DataScienceUsageTransportAction(mock(TransportService.class), + null, null, mock(ActionFilters.class), null, settings.build(), licenseState); + PlainActionFuture future = new PlainActionFuture<>(); + usageAction.masterOperation(null, null, null, future); + XPackFeatureSet.Usage usage = future.get().getUsage(); + assertTrue(usage.enabled()); + + BytesStreamOutput out = new BytesStreamOutput(); + usage.writeTo(out); + XPackFeatureSet.Usage serializedUsage = new DataScienceFeatureSetUsage(out.bytes().streamInput()); + assertTrue(serializedUsage.enabled()); + } + +} diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/TestAggregatorFactory.java similarity index 97% rename from x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java rename to x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/TestAggregatorFactory.java index 0ed1099276a4f..4d69d128a701d 100644 --- a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/TestAggregatorFactory.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.datascience; +package org.elasticsearch.xpack.datascience; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java similarity index 97% rename from x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java rename to x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java index d98e6c728ea2b..31ebe8b320802 100644 --- a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.datascience.cumulativecardinality; +package org.elasticsearch.xpack.datascience.cumulativecardinality; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -18,7 +18,7 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Rounding; import org.elasticsearch.common.time.DateFormatters; -import org.elasticsearch.datascience.TestAggregatorFactory; +import org.elasticsearch.xpack.datascience.TestAggregatorFactory; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -46,8 +46,6 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.xpack.datascience.DataSciencePlugin; -import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder; -import org.elasticsearch.xpack.datascience.cumulativecardinality.InternalSimpleLongValue; import java.io.IOException; import java.util.Arrays; From 0194f82de65e602f50138425432d95d040a6fc33 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Mon, 12 Aug 2019 14:07:02 -0400 Subject: [PATCH 06/10] Add stats for cumulative cardinality agg --- .../action/DataScienceStatsAction.java | 140 ++++++++++++++++++ .../xpack/datascience/DataSciencePlugin.java | 13 +- .../DataScienceInfoTransportAction.java | 2 +- .../DataScienceUsageTransportAction.java | 2 +- .../TransportDataScienceStatsAction.java | 59 ++++++++ ...CardinalityPipelineAggregationBuilder.java | 3 + .../DataScienceInfoTransportActionTests.java | 4 +- .../TransportDataScienceStatsActionTests.java | 77 ++++++++++ 8 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/action/DataScienceStatsAction.java rename x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/{ => action}/DataScienceInfoTransportAction.java (96%) rename x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/{ => action}/DataScienceUsageTransportAction.java (97%) create mode 100644 x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/TransportDataScienceStatsAction.java rename x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/{ => action}/DataScienceInfoTransportActionTests.java (94%) create mode 100644 x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/action/TransportDataScienceStatsActionTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/action/DataScienceStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/action/DataScienceStatsAction.java new file mode 100644 index 0000000000000..f1ec3f5408ba2 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/action/DataScienceStatsAction.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.datascience.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodeRequest; +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; + +public class DataScienceStatsAction extends ActionType { + public static final DataScienceStatsAction INSTANCE = new DataScienceStatsAction(); + public static final String NAME = "cluster:monitor/xpack/datascience/stats"; + + private DataScienceStatsAction() { + super(NAME, Response::new); + } + + public static class Request extends BaseNodesRequest implements ToXContentObject { + + public Request() { + super((String[]) null); + } + + public Request(StreamInput in) throws IOException { + super(in); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return 19; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + return true; + } + } + + public static class NodeRequest extends BaseNodeRequest { + public NodeRequest(StreamInput in) throws IOException { + super(in); + } + + public NodeRequest(Request request) { + + } + } + + public static class Response extends BaseNodesResponse implements Writeable, ToXContentObject { + public Response(StreamInput in) throws IOException { + super(in); + } + + public Response(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(NodeResponse::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("stats"); + for (NodeResponse node : getNodes()) { + node.toXContent(builder, params); + } + builder.endArray(); + + return builder; + } + } + + public static class NodeResponse extends BaseNodeResponse implements ToXContentObject { + static ParseField CUMULATIVE_CARDINALITY_USAGE = new ParseField("cumulative_cardinality_usage"); + private long cumulativeCardinalityUsage; + + public NodeResponse(StreamInput in) throws IOException { + super(in); + cumulativeCardinalityUsage = in.readZLong(); + } + + public NodeResponse(DiscoveryNode node) { + super(node); + } + + public void setCumulativeCardinalityUsage(long cumulativeCardinalityUsage) { + this.cumulativeCardinalityUsage = cumulativeCardinalityUsage; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeZLong(cumulativeCardinalityUsage); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CUMULATIVE_CARDINALITY_USAGE.getPreferredName(), cumulativeCardinalityUsage); + builder.endObject(); + return builder; + } + } +} diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java index 6005dc64f0896..8ba07a6b3f522 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java @@ -13,11 +13,16 @@ import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.datascience.action.DataScienceStatsAction; +import org.elasticsearch.xpack.datascience.action.DataScienceInfoTransportAction; +import org.elasticsearch.xpack.datascience.action.DataScienceUsageTransportAction; +import org.elasticsearch.xpack.datascience.action.TransportDataScienceStatsAction; import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder; import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregator; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import static java.util.Collections.singletonList; @@ -26,6 +31,9 @@ public class DataSciencePlugin extends Plugin implements SearchPlugin, ActionPlu // volatile so all threads can see changes protected static volatile boolean isDataScienceAllowed; + // TODO this should probably become more structured once DataScience plugin has more than just one agg + public static AtomicLong cumulativeCardUsage = new AtomicLong(0); + public DataSciencePlugin() { registerLicenseListener(); } @@ -57,7 +65,8 @@ public List getPipelineAggregations() { @Override public List> getActions() { return Arrays.asList( - new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.DATA_SCIENCE, DataScienceUsageTransportAction.class), - new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.DATA_SCIENCE, DataScienceInfoTransportAction.class)); + new ActionHandler<>(XPackUsageFeatureAction.DATA_SCIENCE, DataScienceUsageTransportAction.class), + new ActionHandler<>(XPackInfoFeatureAction.DATA_SCIENCE, DataScienceInfoTransportAction.class), + new ActionHandler<>(DataScienceStatsAction.INSTANCE, TransportDataScienceStatsAction.class)); } } diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportAction.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/DataScienceInfoTransportAction.java similarity index 96% rename from x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportAction.java rename to x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/DataScienceInfoTransportAction.java index 8d53d5cbeeae4..4a5f7440db9ea 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportAction.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/DataScienceInfoTransportAction.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.datascience; +package org.elasticsearch.xpack.datascience.action; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.common.inject.Inject; diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceUsageTransportAction.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/DataScienceUsageTransportAction.java similarity index 97% rename from x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceUsageTransportAction.java rename to x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/DataScienceUsageTransportAction.java index 79ba45a0b84e2..c71eb1ae28d9a 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceUsageTransportAction.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/DataScienceUsageTransportAction.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.datascience; +package org.elasticsearch.xpack.datascience.action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/TransportDataScienceStatsAction.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/TransportDataScienceStatsAction.java new file mode 100644 index 0000000000000..33fb47ac6050f --- /dev/null +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/action/TransportDataScienceStatsAction.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience.action; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.datascience.action.DataScienceStatsAction; +import org.elasticsearch.xpack.datascience.DataSciencePlugin; + +import java.io.IOException; +import java.util.List; + +public class TransportDataScienceStatsAction extends TransportNodesAction { + + + @Inject + public TransportDataScienceStatsAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters) { + super(DataScienceStatsAction.NAME, threadPool, clusterService, transportService, actionFilters, + DataScienceStatsAction.Request::new, DataScienceStatsAction.NodeRequest::new, ThreadPool.Names.MANAGEMENT, + DataScienceStatsAction.NodeResponse.class); + } + + @Override + protected DataScienceStatsAction.Response newResponse(DataScienceStatsAction.Request request, + List nodes, + List failures) { + return new DataScienceStatsAction.Response(clusterService.getClusterName(), nodes, failures); + } + + @Override + protected DataScienceStatsAction.NodeRequest newNodeRequest(DataScienceStatsAction.Request request) { + return new DataScienceStatsAction.NodeRequest(request); + } + + @Override + protected DataScienceStatsAction.NodeResponse newNodeResponse(StreamInput in) throws IOException { + return new DataScienceStatsAction.NodeResponse(in); + } + + @Override + protected DataScienceStatsAction.NodeResponse nodeOperation(DataScienceStatsAction.NodeRequest request, Task task) { + DataScienceStatsAction.NodeResponse statsResponse = new DataScienceStatsAction.NodeResponse(clusterService.localNode()); + statsResponse.setCumulativeCardinalityUsage(DataSciencePlugin.cumulativeCardUsage.get()); + return statsResponse; + } + +} diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java index bda623b67f0a7..cb033fe108d3d 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java @@ -122,6 +122,9 @@ protected final XContentBuilder internalXContent(XContentBuilder builder, Params } public static CumulativeCardinalityPipelineAggregationBuilder parse(String aggName, XContentParser parser) { + // Increment usage here since it is a good boundary between internal and external, and should correlate 1:1 with + // usage and not internal instantiations + DataSciencePlugin.cumulativeCardUsage.incrementAndGet(); return PARSER.apply(aggName).apply(parser, null); } diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportActionTests.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/action/DataScienceInfoTransportActionTests.java similarity index 94% rename from x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportActionTests.java rename to x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/action/DataScienceInfoTransportActionTests.java index 97dab41169033..053565bdedc5a 100644 --- a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportActionTests.java +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/action/DataScienceInfoTransportActionTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.datascience; +package org.elasticsearch.xpack.datascience.action; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; @@ -15,6 +15,8 @@ import org.elasticsearch.xpack.core.XPackFeatureSet; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; import org.elasticsearch.xpack.core.datascience.DataScienceFeatureSetUsage; +import org.elasticsearch.xpack.datascience.action.DataScienceInfoTransportAction; +import org.elasticsearch.xpack.datascience.action.DataScienceUsageTransportAction; import org.junit.Before; import static org.hamcrest.core.Is.is; diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/action/TransportDataScienceStatsActionTests.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/action/TransportDataScienceStatsActionTests.java new file mode 100644 index 0000000000000..25b705596015f --- /dev/null +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/action/TransportDataScienceStatsActionTests.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datascience.action; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.yaml.ObjectPath; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.datascience.action.DataScienceStatsAction; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportDataScienceStatsActionTests extends ESTestCase { + + private TransportDataScienceStatsAction action; + + @Before + public void setupTransportAction() { + TransportService transportService = mock(TransportService.class); + ThreadPool threadPool = mock(ThreadPool.class); + + ClusterService clusterService = mock(ClusterService.class); + DiscoveryNode discoveryNode = new DiscoveryNode("nodeId", buildNewFakeTransportAddress(), Version.CURRENT); + when(clusterService.localNode()).thenReturn(discoveryNode); + + ClusterName clusterName = new ClusterName("cluster_name"); + when(clusterService.getClusterName()).thenReturn(clusterName); + + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.getMetaData()).thenReturn(MetaData.EMPTY_META_DATA); + when(clusterService.state()).thenReturn(clusterState); + + + action = new TransportDataScienceStatsAction(transportService, clusterService, threadPool, new + ActionFilters(Collections.emptySet())); + } + + public void testCumulativeCardStats() throws Exception { + DataScienceStatsAction.Request request = new DataScienceStatsAction.Request(); + DataScienceStatsAction.NodeResponse nodeResponse1 = action.nodeOperation(new DataScienceStatsAction.NodeRequest(request), null); + DataScienceStatsAction.NodeResponse nodeResponse2 = action.nodeOperation(new DataScienceStatsAction.NodeRequest(request), null); + + DataScienceStatsAction.Response response = action.newResponse(request, + Arrays.asList(nodeResponse1, nodeResponse2), Collections.emptyList()); + + try (XContentBuilder builder = jsonBuilder()) { + builder.startObject(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + ObjectPath objectPath = ObjectPath.createFromXContent(JsonXContent.jsonXContent, BytesReference.bytes(builder)); + assertThat(objectPath.evaluate("stats.0.cumulative_cardinality_usage"), equalTo(0)); + assertThat(objectPath.evaluate("stats.1.cumulative_cardinality_usage"), equalTo(0)); + } + } +} From 2a62fdc44b2f68883822749fb1828536afabaa2d Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Fri, 16 Aug 2019 16:23:03 -0400 Subject: [PATCH 07/10] Remove unneeded override --- .../xpack/core/datascience/DataScienceFeatureSetUsage.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java index 53c76e8ea0871..7b4bab9084d4c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java @@ -24,11 +24,6 @@ public DataScienceFeatureSetUsage(StreamInput input) throws IOException { super(input); } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - @Override public int hashCode() { return Objects.hash(available, enabled); From 2e3772d63c7d8a69730b71b613cdde9939660938 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Fri, 16 Aug 2019 16:50:59 -0400 Subject: [PATCH 08/10] checkstyle --- .../xpack/core/datascience/DataScienceFeatureSetUsage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java index 7b4bab9084d4c..e461360d205d7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.core.datascience; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.core.XPackFeatureSet; import org.elasticsearch.xpack.core.XPackField; From afbb0e89c6173993e7984b6af9ef3d8f9791af99 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 20 Aug 2019 09:38:01 -0400 Subject: [PATCH 09/10] Test fixes --- docs/reference/rest-api/info.asciidoc | 4 ++++ .../java/org/elasticsearch/xpack/core/XPackClientPlugin.java | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index d91be315e86e3..741918ca2d38f 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -71,6 +71,10 @@ Example response: "available" : true, "enabled" : true }, + "data_science" : { + "available" : true, + "enabled" : true + }, "flattened" : { "available" : true, "enabled" : true diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 8e2f3414d58b1..66629870d56b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -49,6 +49,7 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig; import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig; +import org.elasticsearch.xpack.core.datascience.DataScienceFeatureSetUsage; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage; import org.elasticsearch.xpack.core.frozen.FrozenIndicesFeatureSetUsage; @@ -509,7 +510,9 @@ public List getNamedWriteables() { // Frozen indices new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FROZEN_INDICES, FrozenIndicesFeatureSetUsage::new), // Spatial - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new) + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new), + // data science + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_SCIENCE, DataScienceFeatureSetUsage::new) ); } From 2a914a6c21e9c7bb05624cd049cf43a78babfbac Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 22 Aug 2019 13:21:25 -0400 Subject: [PATCH 10/10] Review comments, remove license caching, add some tests --- .../action/DataScienceStatsAction.java | 4 +++- .../xpack/datascience/DataSciencePlugin.java | 23 +++---------------- ...CardinalityPipelineAggregationBuilder.java | 10 ++++---- ...actory.java => StubAggregatorFactory.java} | 8 +++---- .../CumulativeCardinalityAggregatorTests.java | 13 ++--------- .../rest-api-spec/test/xpack/10_basic.yml | 1 + .../rest-api-spec/test/xpack/15_basic.yml | 5 ++++ 7 files changed, 22 insertions(+), 42 deletions(-) rename x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/{TestAggregatorFactory.java => StubAggregatorFactory.java} (87%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/action/DataScienceStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/action/DataScienceStatsAction.java index f1ec3f5408ba2..6a06dc3922779 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/action/DataScienceStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/action/DataScienceStatsAction.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; public class DataScienceStatsAction extends ActionType { public static final DataScienceStatsAction INSTANCE = new DataScienceStatsAction(); @@ -50,7 +51,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { - return 19; + // Nothing to hash atm, so just use the action name + return Objects.hashCode(NAME); } @Override diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java index 8ba07a6b3f522..3fc409ec0ec6d 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java @@ -7,6 +7,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; @@ -28,30 +29,12 @@ public class DataSciencePlugin extends Plugin implements SearchPlugin, ActionPlugin { - // volatile so all threads can see changes - protected static volatile boolean isDataScienceAllowed; - // TODO this should probably become more structured once DataScience plugin has more than just one agg public static AtomicLong cumulativeCardUsage = new AtomicLong(0); - public DataSciencePlugin() { - registerLicenseListener(); - } + public DataSciencePlugin() { } - /** - * Protected for test over-riding - */ - protected void registerLicenseListener() { - // Add a listener to the license state and cache it when there is a change. - // Aggs could be called in high numbers so we don't want them contending on - // the synchronized isFooAllowed() methods - XPackPlugin.getSharedLicenseState() - .addListener(() -> isDataScienceAllowed = XPackPlugin.getSharedLicenseState().isDataScienceAllowed()); - } - - public static boolean isIsDataScienceAllowed() { - return isDataScienceAllowed; - } + public static XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } @Override public List getPipelineAggregations() { diff --git a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java index cb033fe108d3d..f24f09ccba49c 100644 --- a/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java +++ b/x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java @@ -50,9 +50,6 @@ public class CumulativeCardinalityPipelineAggregationBuilder public CumulativeCardinalityPipelineAggregationBuilder(String name, String bucketsPath) { super(name, NAME, new String[] { bucketsPath }); - if (DataSciencePlugin.isIsDataScienceAllowed() == false) { - throw LicenseUtils.newComplianceException(XPackField.DATA_SCIENCE); - } } /** @@ -61,9 +58,6 @@ public CumulativeCardinalityPipelineAggregationBuilder(String name, String bucke public CumulativeCardinalityPipelineAggregationBuilder(StreamInput in) throws IOException { super(in, NAME); format = in.readOptionalString(); - if (DataSciencePlugin.isIsDataScienceAllowed() == false) { - throw LicenseUtils.newComplianceException(XPackField.DATA_SCIENCE); - } } @Override @@ -122,6 +116,10 @@ protected final XContentBuilder internalXContent(XContentBuilder builder, Params } public static CumulativeCardinalityPipelineAggregationBuilder parse(String aggName, XContentParser parser) { + if (DataSciencePlugin.getLicenseState().isDataScienceAllowed() == false) { + throw LicenseUtils.newComplianceException(XPackField.DATA_SCIENCE); + } + // Increment usage here since it is a good boundary between internal and external, and should correlate 1:1 with // usage and not internal instantiations DataSciencePlugin.cumulativeCardUsage.incrementAndGet(); diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/TestAggregatorFactory.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/StubAggregatorFactory.java similarity index 87% rename from x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/TestAggregatorFactory.java rename to x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/StubAggregatorFactory.java index 4d69d128a701d..fd45a15c7659b 100644 --- a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/TestAggregatorFactory.java +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/StubAggregatorFactory.java @@ -26,11 +26,11 @@ /** * Test implementation for AggregatorFactory. */ -public class TestAggregatorFactory extends AggregatorFactory { +public class StubAggregatorFactory extends AggregatorFactory { private final Aggregator aggregator; - private TestAggregatorFactory(SearchContext context, Aggregator aggregator) throws IOException { + private StubAggregatorFactory(SearchContext context, Aggregator aggregator) throws IOException { super("_name", context, null, new AggregatorFactories.Builder(), Collections.emptyMap()); this.aggregator = aggregator; } @@ -40,13 +40,13 @@ protected Aggregator createInternal(Aggregator parent, boolean collectsFromSingl return aggregator; } - public static TestAggregatorFactory createInstance() throws IOException { + public static StubAggregatorFactory createInstance() throws IOException { BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); SearchContext searchContext = mock(SearchContext.class); when(searchContext.bigArrays()).thenReturn(bigArrays); Aggregator aggregator = mock(Aggregator.class); - return new TestAggregatorFactory(searchContext, aggregator); + return new StubAggregatorFactory(searchContext, aggregator); } } diff --git a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java index 31ebe8b320802..9cf2c9846b223 100644 --- a/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java +++ b/x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Rounding; import org.elasticsearch.common.time.DateFormatters; -import org.elasticsearch.xpack.datascience.TestAggregatorFactory; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -45,7 +44,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.xpack.datascience.DataSciencePlugin; +import org.elasticsearch.xpack.datascience.StubAggregatorFactory; import java.io.IOException; import java.util.Arrays; @@ -78,14 +77,6 @@ public class CumulativeCardinalityAggregatorTests extends AggregatorTestCase { private static final List datasetValues = Arrays.asList(1,1,3,1,5,1,7,1,9,10); private static final List cumulativeCardinality = Arrays.asList(1.0,1.0,2.0,2.0,3.0,3.0,4.0,4.0,6.0); - // Initialize plugin so we can set license state - static DataSciencePlugin PLUGIN = new DataSciencePlugin() { - @Override - protected void registerLicenseListener() { - DataSciencePlugin.isDataScienceAllowed = true; - } - }; - public void testSimple() throws IOException { Query query = new MatchAllDocsQuery(); @@ -160,7 +151,7 @@ public void testParentValidations() throws IOException { // Mocked "test" agg, should fail validation aggBuilders.clear(); aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum")); - TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance(); + StubAggregatorFactory parentFactory = StubAggregatorFactory.createInstance(); CumulativeCardinalityPipelineAggregationBuilder failBuilder = new CumulativeCardinalityPipelineAggregationBuilder("name", "invalid_agg>metric"); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/10_basic.yml index 514ba61824067..0b593a87d7eb3 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/10_basic.yml @@ -25,3 +25,4 @@ - contains: { nodes.$master.modules: { name: x-pack-security } } - contains: { nodes.$master.modules: { name: x-pack-sql } } - contains: { nodes.$master.modules: { name: x-pack-watcher } } + - contains: { nodes.$master.modules: { name: x-pack-data-science } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml index 1f2e5ce9625e8..d0e1a9b773ed6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml @@ -28,6 +28,8 @@ - is_true: features.monitoring - is_true: features.monitoring.enabled # - is_false: features.monitoring.available TODO fix once licensing is fixed + - is_true: features.data_science + - is_true: features.data_science.enabled - do: license.post: @@ -77,6 +79,8 @@ - is_true: features.monitoring - is_true: features.monitoring.enabled - is_true: features.monitoring.available + - is_true: features.data_science.enabled + - is_true: features.data_science.available - is_true: tagline - do: @@ -89,6 +93,7 @@ - is_true: graph.available - is_true: monitoring.enabled - is_true: monitoring.available + - is_true: data_science.available - do: xpack.info: