diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index a0596bb8..bf33e0db 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -30,20 +30,20 @@ jobs: uses: actions/checkout@v2 with: repository: 'opensearch-project/OpenSearch' - ref: '1.0.0-beta1' + ref: '1.x' path: OpenSearch - name: Build OpenSearch working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=beta1 -Dbuild.snapshot=false + run: ./gradlew publishToMavenLocal -Dbuild.snapshot=false # job-scheduler - name: Build and Test run: | - ./gradlew build -Dopensearch.version=1.0.0-beta1 -Dbuild.snapshot=false + ./gradlew build -Dopensearch.version=1.0.0 -Dbuild.snapshot=false - name: Publish to Maven Local run: | - ./gradlew publishToMavenLocal -Dopensearch.version=1.0.0-beta1 -Dbuild.snapshot=false + ./gradlew publishToMavenLocal -Dopensearch.version=1.0.0 -Dbuild.snapshot=false - name: Upload Coverage Report uses: codecov/codecov-action@v1 diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerPlugin.java b/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerPlugin.java index 3ded3b88..e6ec0b93 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerPlugin.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerPlugin.java @@ -105,6 +105,12 @@ public Collection createComponents(Client client, ClusterService cluster @Override public List> getSettings() { List> settingList = new ArrayList<>(); + settingList.add(LegacyOpenDistroJobSchedulerSettings.SWEEP_PAGE_SIZE); + settingList.add(LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT); + settingList.add(LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_MILLIS); + settingList.add(LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT); + settingList.add(LegacyOpenDistroJobSchedulerSettings.SWEEP_PERIOD); + settingList.add(LegacyOpenDistroJobSchedulerSettings.JITTER_LIMIT); settingList.add(JobSchedulerSettings.SWEEP_PAGE_SIZE); settingList.add(JobSchedulerSettings.REQUEST_TIMEOUT); settingList.add(JobSchedulerSettings.SWEEP_BACKOFF_MILLIS); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerSettings.java b/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerSettings.java index b6d11034..7f9af216 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerSettings.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerSettings.java @@ -31,34 +31,32 @@ public class JobSchedulerSettings { public static final Setting REQUEST_TIMEOUT = Setting.positiveTimeSetting( - "opendistro.jobscheduler.request_timeout", - TimeValue.timeValueSeconds(10), + "opensearch.jobscheduler.request_timeout", + LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT, Setting.Property.NodeScope, Setting.Property.Dynamic); public static final Setting SWEEP_BACKOFF_MILLIS = Setting.positiveTimeSetting( - "opendistro.jobscheduler.sweeper.backoff_millis", - TimeValue.timeValueMillis(50), + "opensearch.jobscheduler.sweeper.backoff_millis", + LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_MILLIS, Setting.Property.NodeScope, Setting.Property.Dynamic); public static final Setting SWEEP_BACKOFF_RETRY_COUNT = Setting.intSetting( - "opendistro.jobscheduler.retry_count", - 3, + "opensearch.jobscheduler.retry_count", + LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT, Setting.Property.NodeScope, Setting.Property.Dynamic); public static final Setting SWEEP_PERIOD = Setting.positiveTimeSetting( - "opendistro.jobscheduler.sweeper.period", - TimeValue.timeValueMinutes(5), + "opensearch.jobscheduler.sweeper.period", + LegacyOpenDistroJobSchedulerSettings.SWEEP_PERIOD, Setting.Property.NodeScope, Setting.Property.Dynamic); public static final Setting SWEEP_PAGE_SIZE = Setting.intSetting( - "opendistro.jobscheduler.sweeper.page_size", - 100, + "opensearch.jobscheduler.sweeper.page_size", + LegacyOpenDistroJobSchedulerSettings.SWEEP_PAGE_SIZE, Setting.Property.NodeScope, Setting.Property.Dynamic); public static final Setting JITTER_LIMIT = Setting.doubleSetting( - "opendistro.jobscheduler.jitter_limit", - 0.60, 0, 0.95, + "opensearch.jobscheduler.jitter_limit", + LegacyOpenDistroJobSchedulerSettings.JITTER_LIMIT, Setting.Property.NodeScope, Setting.Property.Dynamic); - - } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/LegacyOpenDistroJobSchedulerSettings.java b/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/LegacyOpenDistroJobSchedulerSettings.java new file mode 100644 index 00000000..eed7628d --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/LegacyOpenDistroJobSchedulerSettings.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.jobscheduler; + +import org.opensearch.common.settings.Setting; +import org.opensearch.common.unit.TimeValue; + +public class LegacyOpenDistroJobSchedulerSettings { + public static final Setting REQUEST_TIMEOUT = Setting.positiveTimeSetting( + "opendistro.jobscheduler.request_timeout", + TimeValue.timeValueSeconds(10), + Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); + + public static final Setting SWEEP_BACKOFF_MILLIS = Setting.positiveTimeSetting( + "opendistro.jobscheduler.sweeper.backoff_millis", + TimeValue.timeValueMillis(50), + Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); + + public static final Setting SWEEP_BACKOFF_RETRY_COUNT = Setting.intSetting( + "opendistro.jobscheduler.retry_count", + 3, + Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); + + public static final Setting SWEEP_PERIOD = Setting.positiveTimeSetting( + "opendistro.jobscheduler.sweeper.period", + TimeValue.timeValueMinutes(5), + Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); + + public static final Setting SWEEP_PAGE_SIZE = Setting.intSetting( + "opendistro.jobscheduler.sweeper.page_size", + 100, + Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); + + public static final Setting JITTER_LIMIT = Setting.doubleSetting( + "opendistro.jobscheduler.jitter_limit", + 0.60, 0, 0.95, + Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/sweeper/JobSweeper.java b/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/sweeper/JobSweeper.java index a103d74c..bfe39296 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/sweeper/JobSweeper.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/sweeper/JobSweeper.java @@ -148,25 +148,36 @@ private void addConfigListeners() { clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.SWEEP_PERIOD, timeValue -> { sweepPeriod = timeValue; - log.debug("Reinitializing background full sweep with period: " + sweepPeriod.getMinutes()); + log.debug("Reinitializing background full sweep with period: {}", this.sweepPeriod.getMinutes()); initBackgroundSweep(); }); clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.SWEEP_PAGE_SIZE, - intValue -> sweepPageMaxSize = intValue); + intValue -> { + sweepPageMaxSize = intValue; + log.debug("Setting background sweep page size: {}", this.sweepPageMaxSize); + }); clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.REQUEST_TIMEOUT, - timeValue -> this.sweepSearchTimeout = timeValue); + timeValue -> { + this.sweepSearchTimeout = timeValue; + log.debug("Setting background sweep search timeout: {}", this.sweepSearchTimeout.getMinutes()); + }); clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.SWEEP_BACKOFF_MILLIS, timeValue -> { this.sweepSearchBackoffMillis = timeValue; this.sweepSearchBackoff = this.updateRetryPolicy(); + log.debug("Setting background sweep search backoff: {}", this.sweepSearchBackoffMillis.getMillis()); }); clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT, intValue -> { this.sweepSearchBackoffRetryCount = intValue; this.sweepSearchBackoff = this.updateRetryPolicy(); + log.debug("Setting background sweep search backoff retry count: {}", this.sweepSearchBackoffRetryCount); }); clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.JITTER_LIMIT, - doubleValue -> this.jitterLimit = doubleValue); + doubleValue -> { + this.jitterLimit = doubleValue; + log.debug("Setting background sweep jitter limit: {}", this.jitterLimit); + }); } private BackoffPolicy updateRetryPolicy() { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerSettingsTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerSettingsTests.java new file mode 100644 index 00000000..a3c8d8fb --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/jobscheduler/JobSchedulerSettingsTests.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.jobscheduler; + +import java.util.Arrays; +import java.util.List; + +import org.junit.Before; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.test.OpenSearchTestCase; + +@SuppressWarnings({"rawtypes"}) +public class JobSchedulerSettingsTests extends OpenSearchTestCase { + + JobSchedulerPlugin plugin; + + @Before + public void setup() { + this.plugin = new JobSchedulerPlugin(); + } + + public void testAllLegacyOpenDistroSettingsReturned() { + List> settings = plugin.getSettings(); + assertTrue("legacy setting must be returned from settings", + settings.containsAll( + Arrays.asList( + LegacyOpenDistroJobSchedulerSettings.JITTER_LIMIT, + LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT, + LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_MILLIS, + LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT, + LegacyOpenDistroJobSchedulerSettings.SWEEP_PAGE_SIZE, + LegacyOpenDistroJobSchedulerSettings.SWEEP_PERIOD + ) + ) + ); + } + + public void testAllOpenSearchSettingsReturned() { + List> settings = plugin.getSettings(); + assertTrue("legacy setting must be returned from settings", + settings.containsAll( + Arrays.asList( + JobSchedulerSettings.JITTER_LIMIT, + JobSchedulerSettings.REQUEST_TIMEOUT, + JobSchedulerSettings.SWEEP_BACKOFF_MILLIS, + JobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT, + JobSchedulerSettings.SWEEP_PAGE_SIZE, + JobSchedulerSettings.SWEEP_PERIOD + ) + ) + ); + } + + public void testLegacyOpenDistroSettingsFallback() { + assertEquals( + JobSchedulerSettings.REQUEST_TIMEOUT.get(Settings.EMPTY), + LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT.get(Settings.EMPTY) + ); + } + + public void testSettingsGetValue() { + Settings settings = Settings.builder().put("opensearch.jobscheduler.request_timeout", "42s").build(); + assertEquals(JobSchedulerSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(42)); + assertEquals(LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(10)); + } + + public void testSettingsGetValueWithLegacyFallback() { + Settings settings = Settings.builder() + .put("opendistro.jobscheduler.request_timeout", "1s") + .put("opendistro.jobscheduler.sweeper.backoff_millis", "2ms") + .put("opendistro.jobscheduler.retry_count", 3) + .put("opendistro.jobscheduler.sweeper.period", "4s") + .put("opendistro.jobscheduler.sweeper.page_size", 5) + .put("opendistro.jobscheduler.jitter_limit", 6) + .build(); + + assertEquals(JobSchedulerSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(1)); + assertEquals(JobSchedulerSettings.SWEEP_BACKOFF_MILLIS.get(settings), TimeValue.timeValueMillis(2)); + assertEquals(JobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT.get(settings), Integer.valueOf(3)); + assertEquals(JobSchedulerSettings.SWEEP_PERIOD.get(settings), TimeValue.timeValueSeconds(4)); + assertEquals(JobSchedulerSettings.SWEEP_PAGE_SIZE.get(settings), Integer.valueOf(5)); + assertEquals(JobSchedulerSettings.JITTER_LIMIT.get(settings), Double.valueOf(6.0)); + + assertSettingDeprecationsAndWarnings(new Setting[]{ + LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT, + LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_MILLIS, + LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT, + LegacyOpenDistroJobSchedulerSettings.SWEEP_PERIOD, + LegacyOpenDistroJobSchedulerSettings.SWEEP_PAGE_SIZE, + LegacyOpenDistroJobSchedulerSettings.JITTER_LIMIT + }); + } +}