Skip to content

Commit

Permalink
Rename settings from OpenDistro and OpenSearch, with backwards compat…
Browse files Browse the repository at this point in the history
…ibility, using fallback (opensearch-project#20)
  • Loading branch information
dblock authored May 7, 2021
1 parent 5798f39 commit 0a1b790
Show file tree
Hide file tree
Showing 6 changed files with 218 additions and 22 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/test-and-build-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,20 @@ jobs:
uses: actions/checkout@v2
with:
repository: 'opensearch-project/OpenSearch'
ref: '1.0.0-beta1'
ref: '1.x'
path: OpenSearch
- name: Build OpenSearch
working-directory: ./OpenSearch
run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=beta1 -Dbuild.snapshot=false
run: ./gradlew publishToMavenLocal -Dbuild.snapshot=false

# job-scheduler
- name: Build and Test
run: |
./gradlew build -Dopensearch.version=1.0.0-beta1 -Dbuild.snapshot=false
./gradlew build -Dopensearch.version=1.0.0 -Dbuild.snapshot=false
- name: Publish to Maven Local
run: |
./gradlew publishToMavenLocal -Dopensearch.version=1.0.0-beta1 -Dbuild.snapshot=false
./gradlew publishToMavenLocal -Dopensearch.version=1.0.0 -Dbuild.snapshot=false
- name: Upload Coverage Report
uses: codecov/codecov-action@v1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,12 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
@Override
public List<Setting<?>> getSettings() {
List<Setting<?>> settingList = new ArrayList<>();
settingList.add(LegacyOpenDistroJobSchedulerSettings.SWEEP_PAGE_SIZE);
settingList.add(LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT);
settingList.add(LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_MILLIS);
settingList.add(LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT);
settingList.add(LegacyOpenDistroJobSchedulerSettings.SWEEP_PERIOD);
settingList.add(LegacyOpenDistroJobSchedulerSettings.JITTER_LIMIT);
settingList.add(JobSchedulerSettings.SWEEP_PAGE_SIZE);
settingList.add(JobSchedulerSettings.REQUEST_TIMEOUT);
settingList.add(JobSchedulerSettings.SWEEP_BACKOFF_MILLIS);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,34 +31,32 @@

public class JobSchedulerSettings {
public static final Setting<TimeValue> REQUEST_TIMEOUT = Setting.positiveTimeSetting(
"opendistro.jobscheduler.request_timeout",
TimeValue.timeValueSeconds(10),
"opensearch.jobscheduler.request_timeout",
LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT,
Setting.Property.NodeScope, Setting.Property.Dynamic);

public static final Setting<TimeValue> SWEEP_BACKOFF_MILLIS = Setting.positiveTimeSetting(
"opendistro.jobscheduler.sweeper.backoff_millis",
TimeValue.timeValueMillis(50),
"opensearch.jobscheduler.sweeper.backoff_millis",
LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_MILLIS,
Setting.Property.NodeScope, Setting.Property.Dynamic);

public static final Setting<Integer> SWEEP_BACKOFF_RETRY_COUNT = Setting.intSetting(
"opendistro.jobscheduler.retry_count",
3,
"opensearch.jobscheduler.retry_count",
LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT,
Setting.Property.NodeScope, Setting.Property.Dynamic);

public static final Setting<TimeValue> SWEEP_PERIOD = Setting.positiveTimeSetting(
"opendistro.jobscheduler.sweeper.period",
TimeValue.timeValueMinutes(5),
"opensearch.jobscheduler.sweeper.period",
LegacyOpenDistroJobSchedulerSettings.SWEEP_PERIOD,
Setting.Property.NodeScope, Setting.Property.Dynamic);

public static final Setting<Integer> SWEEP_PAGE_SIZE = Setting.intSetting(
"opendistro.jobscheduler.sweeper.page_size",
100,
"opensearch.jobscheduler.sweeper.page_size",
LegacyOpenDistroJobSchedulerSettings.SWEEP_PAGE_SIZE,
Setting.Property.NodeScope, Setting.Property.Dynamic);

public static final Setting<Double> JITTER_LIMIT = Setting.doubleSetting(
"opendistro.jobscheduler.jitter_limit",
0.60, 0, 0.95,
"opensearch.jobscheduler.jitter_limit",
LegacyOpenDistroJobSchedulerSettings.JITTER_LIMIT,
Setting.Property.NodeScope, Setting.Property.Dynamic);


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.jobscheduler;

import org.opensearch.common.settings.Setting;
import org.opensearch.common.unit.TimeValue;

public class LegacyOpenDistroJobSchedulerSettings {
public static final Setting<TimeValue> REQUEST_TIMEOUT = Setting.positiveTimeSetting(
"opendistro.jobscheduler.request_timeout",
TimeValue.timeValueSeconds(10),
Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated);

public static final Setting<TimeValue> SWEEP_BACKOFF_MILLIS = Setting.positiveTimeSetting(
"opendistro.jobscheduler.sweeper.backoff_millis",
TimeValue.timeValueMillis(50),
Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated);

public static final Setting<Integer> SWEEP_BACKOFF_RETRY_COUNT = Setting.intSetting(
"opendistro.jobscheduler.retry_count",
3,
Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated);

public static final Setting<TimeValue> SWEEP_PERIOD = Setting.positiveTimeSetting(
"opendistro.jobscheduler.sweeper.period",
TimeValue.timeValueMinutes(5),
Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated);

public static final Setting<Integer> SWEEP_PAGE_SIZE = Setting.intSetting(
"opendistro.jobscheduler.sweeper.page_size",
100,
Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated);

public static final Setting<Double> JITTER_LIMIT = Setting.doubleSetting(
"opendistro.jobscheduler.jitter_limit",
0.60, 0, 0.95,
Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated);
}
Original file line number Diff line number Diff line change
Expand Up @@ -148,25 +148,36 @@ private void addConfigListeners() {
clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.SWEEP_PERIOD,
timeValue -> {
sweepPeriod = timeValue;
log.debug("Reinitializing background full sweep with period: " + sweepPeriod.getMinutes());
log.debug("Reinitializing background full sweep with period: {}", this.sweepPeriod.getMinutes());
initBackgroundSweep();
});
clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.SWEEP_PAGE_SIZE,
intValue -> sweepPageMaxSize = intValue);
intValue -> {
sweepPageMaxSize = intValue;
log.debug("Setting background sweep page size: {}", this.sweepPageMaxSize);
});
clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.REQUEST_TIMEOUT,
timeValue -> this.sweepSearchTimeout = timeValue);
timeValue -> {
this.sweepSearchTimeout = timeValue;
log.debug("Setting background sweep search timeout: {}", this.sweepSearchTimeout.getMinutes());
});
clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.SWEEP_BACKOFF_MILLIS,
timeValue -> {
this.sweepSearchBackoffMillis = timeValue;
this.sweepSearchBackoff = this.updateRetryPolicy();
log.debug("Setting background sweep search backoff: {}", this.sweepSearchBackoffMillis.getMillis());
});
clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT,
intValue -> {
this.sweepSearchBackoffRetryCount = intValue;
this.sweepSearchBackoff = this.updateRetryPolicy();
log.debug("Setting background sweep search backoff retry count: {}", this.sweepSearchBackoffRetryCount);
});
clusterService.getClusterSettings().addSettingsUpdateConsumer(JobSchedulerSettings.JITTER_LIMIT,
doubleValue -> this.jitterLimit = doubleValue);
doubleValue -> {
this.jitterLimit = doubleValue;
log.debug("Setting background sweep jitter limit: {}", this.jitterLimit);
});
}

private BackoffPolicy updateRetryPolicy() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.jobscheduler;

import java.util.Arrays;
import java.util.List;

import org.junit.Before;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.test.OpenSearchTestCase;

@SuppressWarnings({"rawtypes"})
public class JobSchedulerSettingsTests extends OpenSearchTestCase {

JobSchedulerPlugin plugin;

@Before
public void setup() {
this.plugin = new JobSchedulerPlugin();
}

public void testAllLegacyOpenDistroSettingsReturned() {
List<Setting<?>> settings = plugin.getSettings();
assertTrue("legacy setting must be returned from settings",
settings.containsAll(
Arrays.asList(
LegacyOpenDistroJobSchedulerSettings.JITTER_LIMIT,
LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT,
LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_MILLIS,
LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT,
LegacyOpenDistroJobSchedulerSettings.SWEEP_PAGE_SIZE,
LegacyOpenDistroJobSchedulerSettings.SWEEP_PERIOD
)
)
);
}

public void testAllOpenSearchSettingsReturned() {
List<Setting<?>> settings = plugin.getSettings();
assertTrue("legacy setting must be returned from settings",
settings.containsAll(
Arrays.asList(
JobSchedulerSettings.JITTER_LIMIT,
JobSchedulerSettings.REQUEST_TIMEOUT,
JobSchedulerSettings.SWEEP_BACKOFF_MILLIS,
JobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT,
JobSchedulerSettings.SWEEP_PAGE_SIZE,
JobSchedulerSettings.SWEEP_PERIOD
)
)
);
}

public void testLegacyOpenDistroSettingsFallback() {
assertEquals(
JobSchedulerSettings.REQUEST_TIMEOUT.get(Settings.EMPTY),
LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT.get(Settings.EMPTY)
);
}

public void testSettingsGetValue() {
Settings settings = Settings.builder().put("opensearch.jobscheduler.request_timeout", "42s").build();
assertEquals(JobSchedulerSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(42));
assertEquals(LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(10));
}

public void testSettingsGetValueWithLegacyFallback() {
Settings settings = Settings.builder()
.put("opendistro.jobscheduler.request_timeout", "1s")
.put("opendistro.jobscheduler.sweeper.backoff_millis", "2ms")
.put("opendistro.jobscheduler.retry_count", 3)
.put("opendistro.jobscheduler.sweeper.period", "4s")
.put("opendistro.jobscheduler.sweeper.page_size", 5)
.put("opendistro.jobscheduler.jitter_limit", 6)
.build();

assertEquals(JobSchedulerSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(1));
assertEquals(JobSchedulerSettings.SWEEP_BACKOFF_MILLIS.get(settings), TimeValue.timeValueMillis(2));
assertEquals(JobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT.get(settings), Integer.valueOf(3));
assertEquals(JobSchedulerSettings.SWEEP_PERIOD.get(settings), TimeValue.timeValueSeconds(4));
assertEquals(JobSchedulerSettings.SWEEP_PAGE_SIZE.get(settings), Integer.valueOf(5));
assertEquals(JobSchedulerSettings.JITTER_LIMIT.get(settings), Double.valueOf(6.0));

assertSettingDeprecationsAndWarnings(new Setting[]{
LegacyOpenDistroJobSchedulerSettings.REQUEST_TIMEOUT,
LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_MILLIS,
LegacyOpenDistroJobSchedulerSettings.SWEEP_BACKOFF_RETRY_COUNT,
LegacyOpenDistroJobSchedulerSettings.SWEEP_PERIOD,
LegacyOpenDistroJobSchedulerSettings.SWEEP_PAGE_SIZE,
LegacyOpenDistroJobSchedulerSettings.JITTER_LIMIT
});
}
}

0 comments on commit 0a1b790

Please sign in to comment.