Skip to content

Commit

Permalink
Merge remote-tracking branch 'elastic/master' into remove-warn-date
Browse files Browse the repository at this point in the history
* elastic/master:
  Remove Watcher Account "unsecure" settings (elastic#36736)
  Add cache cleaning task for ML snapshot (elastic#37505)
  Update jdk used by the docker builds (elastic#37621)
  Remove an unused constant in PutMappingRequest.
  Update get users to allow unknown fields (elastic#37593)
  Do not add index event listener if CCR disabled (elastic#37432)
  Add local session timeouts to leader node (elastic#37438)
  • Loading branch information
jasontedor committed Jan 20, 2019
2 parents 2472b0a + 5308746 commit 696da03
Show file tree
Hide file tree
Showing 38 changed files with 542 additions and 344 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

package org.elasticsearch.client.indices;

import com.carrotsearch.hppc.ObjectHashSet;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.support.IndicesOptions;
Expand All @@ -43,11 +42,6 @@
*/
public class PutMappingRequest extends TimedRequest implements IndicesRequest, ToXContentObject {

private static ObjectHashSet<String> RESERVED_FIELDS = ObjectHashSet.from(
"_uid", "_id", "_type", "_source", "_all", "_analyzer", "_parent", "_routing", "_index",
"_size", "_timestamp", "_ttl", "_field_names"
);

private final String[] indices;
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public int hashCode() {
public static final ParseField ENABLED = new ParseField("enabled");

@SuppressWarnings("unchecked")
public static final ConstructingObjectParser<ParsedUser, String> USER_PARSER = new ConstructingObjectParser<>("user_info",
public static final ConstructingObjectParser<ParsedUser, String> USER_PARSER = new ConstructingObjectParser<>("user_info", true,
(constructorObjects) -> {
int i = 0;
final String username = (String) constructorObjects[i++];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,58 +19,97 @@
package org.elasticsearch.client.security;

import org.elasticsearch.client.security.user.User;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.EqualsHashCodeTestUtils;
import org.elasticsearch.test.XContentTestUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import static org.hamcrest.Matchers.equalTo;
import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;

/** tests the Response for getting users from the security HLRC */
public class GetUsersResponseTests extends ESTestCase {

public void testFromXContent() throws IOException {
String json =
"{\n" +
" \"jacknich\": {\n" +
" \"username\": \"jacknich\",\n" +
" \"roles\": [\n" +
" \"admin\", \"other_role1\"\n" +
" ],\n" +
" \"full_name\": \"Jack Nicholson\",\n" +
" \"email\": \"[email protected]\",\n" +
" \"metadata\": { \"intelligence\" : 7 },\n" +
" \"enabled\": true\n" +
" }\n" +
"}";
final GetUsersResponse response = GetUsersResponse.fromXContent((XContentType.JSON.xContent().createParser(
new NamedXContentRegistry(Collections.emptyList()), new DeprecationHandler() {
@Override
public void usedDeprecatedName(String usedName, String modernName) {
}
xContentTester(this::createParser,
GetUsersResponseTests::createTestInstance,
this::toXContent,
GetUsersResponse::fromXContent)
.supportsUnknownFields(false)
.assertToXContentEquivalence(false)
.test();
}

private XContentBuilder toXContentUser(User user, boolean enabled, XContentBuilder builder) throws IOException {
XContentBuilder tempBuilder = JsonXContent.contentBuilder();
tempBuilder.startObject();
tempBuilder.field("username", user.getUsername());
tempBuilder.array("roles", user.getRoles().toArray());
tempBuilder.field("full_name", user.getFullName());
tempBuilder.field("email", user.getEmail());
tempBuilder.field("metadata", user.getMetadata());
tempBuilder.field("enabled", enabled);
tempBuilder.endObject();

// This sub object should support unknown fields, but metadata cannot contain complex extra objects or it will fail
Predicate<String> excludeFilter = path -> path.equals("metadata");
BytesReference newBytes = XContentTestUtils.insertRandomFields(XContentType.JSON, BytesReference.bytes(tempBuilder),
excludeFilter, random());
builder.rawValue(newBytes.streamInput(), XContentType.JSON);
return builder;
}

private XContentBuilder toXContent(GetUsersResponse response, XContentBuilder builder) throws IOException {
builder.startObject();

List<User> disabledUsers = new ArrayList<>(response.getUsers());
disabledUsers.removeAll(response.getEnabledUsers());

for (User user : disabledUsers) {
builder.field(user.getUsername());
toXContentUser(user, false, builder);
}
for (User user : response.getEnabledUsers()) {
builder.field(user.getUsername());
toXContentUser(user, true, builder);
}
builder.endObject();
return builder;
}

private static GetUsersResponse createTestInstance() {
final Set<User> users = new HashSet<>();
final Set<User> enabledUsers = new HashSet<>();
Map<String, Object> metadata = new HashMap<>();
metadata.put(randomAlphaOfLengthBetween(1, 5), randomInt());

@Override
public void usedDeprecatedField(String usedName, String replacedWith) {
}
}, json)));
assertThat(response.getUsers().size(), equalTo(1));
final User user = response.getUsers().iterator().next();
assertThat(user.getUsername(), equalTo("jacknich"));
assertThat(user.getRoles().size(), equalTo(2));
assertThat(user.getFullName(), equalTo("Jack Nicholson"));
assertThat(user.getEmail(), equalTo("[email protected]"));
final Map<String, Object> metadata = new HashMap<>();
metadata.put("intelligence", 7);
assertThat(metadata, equalTo(user.getMetadata()));
final User user1 = new User(randomAlphaOfLength(8),
Arrays.asList(new String[] {randomAlphaOfLength(5), randomAlphaOfLength(5)}),
metadata, randomAlphaOfLength(10), null);
users.add(user1);
enabledUsers.add(user1);
Map<String, Object> metadata2 = new HashMap<>();
metadata2.put(randomAlphaOfLengthBetween(1, 5), randomInt());
metadata2.put(randomAlphaOfLengthBetween(1, 5), randomBoolean());

final User user2 = new User(randomAlphaOfLength(8),
Arrays.asList(new String[] {randomAlphaOfLength(5), randomAlphaOfLength(5)}),
metadata2, randomAlphaOfLength(10), null);
users.add(user2);
return new GetUsersResponse(users, enabledUsers);
}

public void testEqualsHashCode() {
Expand Down
7 changes: 4 additions & 3 deletions distribution/docker/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ dependencies {
ext.expansions = { oss ->
return [
'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz",
'jdkUrl' : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
'jdkVersion' : '11.0.1',
'jdkUrl' : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz',
'jdkVersion' : '11.0.2',
'license': oss ? 'Apache-2.0' : 'Elastic License',
'version' : VersionProperties.elasticsearch
]
Expand Down Expand Up @@ -58,6 +58,7 @@ void addCopyDockerContextTask(final boolean oss) {

void addCopyDockerfileTask(final boolean oss) {
task(taskName("copy", oss, "Dockerfile"), type: Copy) {
inputs.properties(expansions(oss)) // ensure task is run when ext.expansions is changed
mustRunAfter(taskName("copy", oss, "DockerContext"))
into files(oss)

Expand All @@ -82,7 +83,7 @@ void addBuildDockerImage(final boolean oss) {
]
}
executable 'docker'
final List<String> dockerArgs = ['build', files(oss), '--pull']
final List<String> dockerArgs = ['build', files(oss), '--pull', '--no-cache']
for (final String tag : tags) {
dockerArgs.add('--tag')
dockerArgs.add(tag)
Expand Down
23 changes: 23 additions & 0 deletions docs/reference/migration/migrate_7_0/settings.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -131,3 +131,26 @@ The removal of these default settings also removes the ability for a component t
fallback to a default configuration when using TLS. Each component (realm, transport, http,
http client, etc) must now be configured with their own settings for TLS if it is being
used.

[float]
[[watcher-notifications-account-settings]]
==== Watcher notifications account settings

The following settings have been removed in favor of the secure variants.
The <<secure-settings, secure settings>> have to be defined inside each cluster
node's keystore, i.e., they are not to be specified via the cluster settings API.

- `xpack.notification.email.account.<id>.smtp.password`, instead use
`xpack.notification.email.account.<id>.smtp.secure_password`
- `xpack.notification.hipchat.account.<id>.auth_token`, instead use
`xpack.notification.hipchat.account.<id>.secure_auth_token`
- `xpack.notification.jira.account.<id>.url`, instead use
`xpack.notification.jira.account.<id>.secure_url`
- `xpack.notification.jira.account.<id>.user`, instead use
`xpack.notification.jira.account.<id>.secure_user`
- `xpack.notification.jira.account.<id>.password`, instead use
`xpack.notification.jira.account.<id>.secure_password`
- `xpack.notification.pagerduty.account.<id>.service_api_key`, instead use
`xpack.notification.pagerduty.account.<id>.secure_service_api_key`
- `xpack.notification.slack.account.<id>.url`, instead use
`xpack.notification.slack.account.<id>.secure_url`
20 changes: 9 additions & 11 deletions docs/reference/settings/notification-settings.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ can specify the following email account attributes:
`smtp.user` (<<cluster-update-settings,Dynamic>>);;
The user name for SMTP. Required.

`smtp.password` (<<cluster-update-settings,Dynamic>>);;
`smtp.secure_password` (<<secure-settings,Secure>>);;
The password for the specified SMTP user.

`smtp.starttls.enable` (<<cluster-update-settings,Dynamic>>);;
Expand Down Expand Up @@ -222,9 +222,8 @@ via HipChat. You can specify the following HipChat account attributes:
The HipChat account profile to use: `integration`,
`user`, or `v1`. Required.

`auth_token`;;
The authentication token to use to access
the HipChat API. Required.
`secure_auth_token` (<<secure-settings,Secure>>);;
The authentication token to use to access the HipChat API. Required.

`host`;;
The HipChat server hostname. Defaults to `api.hipchat.com`.
Expand Down Expand Up @@ -268,9 +267,8 @@ via Slack. You can specify the following Slack account attributes:

[[slack-account-attributes]]

`url`;;
The Incoming Webhook URL to use to post
messages to Slack. Required.
`secure_url` (<<secure-settings,Secure>>);;
The Incoming Webhook URL to use to post messages to Slack. Required.

`message_defaults.from`;;
The sender name to display in the
Expand Down Expand Up @@ -309,13 +307,13 @@ issues in Jira. You can specify the following Jira account attributes:

[[jira-account-attributes]]

`url`;;
`secure_url` (<<secure-settings,Secure>>);;
The URL of the Jira Software server. Required.

`user`;;
`secure_user` (<<secure-settings,Secure>>);;
The name of the user to connect to the Jira Software server. Required.

`password`;;
`secure_password` (<<secure-settings,Secure>>);;
The password of the user to connect to the Jira Software server. Required.

`issue_defaults`;;
Expand All @@ -341,7 +339,7 @@ via PagerDuty. You can specify the following PagerDuty account attributes:
A name for the PagerDuty account associated with the API key you
are using to access PagerDuty. Required.

`service_api_key`;;
`secure_service_api_key` (<<secure-settings,Secure>>);;
The https://developer.pagerduty.com/documentation/rest/authentication[
PagerDuty API key] to use to access PagerDuty. Required.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,10 @@ public Collection<Object> createComponents(
return emptyList();
}

CcrRestoreSourceService restoreSourceService = new CcrRestoreSourceService();
this.restoreSourceService.set(restoreSourceService);
CcrSettings ccrSettings = new CcrSettings(settings, clusterService.getClusterSettings());
this.ccrSettings.set(ccrSettings);
CcrRestoreSourceService restoreSourceService = new CcrRestoreSourceService(threadPool, ccrSettings);
this.restoreSourceService.set(restoreSourceService);
return Arrays.asList(
ccrLicenseChecker,
restoreSourceService,
Expand Down Expand Up @@ -306,7 +306,9 @@ public Map<String, Repository.Factory> getInternalRepositories(Environment env,

@Override
public void onIndexModule(IndexModule indexModule) {
indexModule.addIndexEventListener(this.restoreSourceService.get());
if (enabled) {
indexModule.addIndexEventListener(this.restoreSourceService.get());
}
}

protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ public final class CcrSettings {
Setting.byteSizeSetting("ccr.indices.recovery.max_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB),
Setting.Property.Dynamic, Setting.Property.NodeScope);

/**
* The leader must open resources for a ccr recovery. If there is no activity for this interval of time,
* the leader will close the restore session.
*/
public static final Setting<TimeValue> INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING =
Setting.timeSetting("ccr.indices.recovery.recovery_activity_timeout", TimeValue.timeValueSeconds(60),
Setting.Property.Dynamic, Setting.Property.NodeScope);

/**
* The settings defined by CCR.
*
Expand All @@ -53,22 +61,33 @@ static List<Setting<?>> getSettings() {
XPackSettings.CCR_ENABLED_SETTING,
CCR_FOLLOWING_INDEX_SETTING,
RECOVERY_MAX_BYTES_PER_SECOND,
INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING,
CCR_AUTO_FOLLOW_WAIT_FOR_METADATA_TIMEOUT);
}

private final CombinedRateLimiter ccrRateLimiter;
private volatile TimeValue recoveryActivityTimeout;

public CcrSettings(Settings settings, ClusterSettings clusterSettings) {
this.recoveryActivityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings);
this.ccrRateLimiter = new CombinedRateLimiter(RECOVERY_MAX_BYTES_PER_SECOND.get(settings));
clusterSettings.addSettingsUpdateConsumer(RECOVERY_MAX_BYTES_PER_SECOND, this::setMaxBytesPerSec);
clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setRecoveryActivityTimeout);
}

private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) {
ccrRateLimiter.setMBPerSec(maxBytesPerSec);
}

private void setRecoveryActivityTimeout(TimeValue recoveryActivityTimeout) {
this.recoveryActivityTimeout = recoveryActivityTimeout;
}

public CombinedRateLimiter getRateLimiter() {
return ccrRateLimiter;
}

public TimeValue getRecoveryActivityTimeout() {
return recoveryActivityTimeout;
}
}
Loading

0 comments on commit 696da03

Please sign in to comment.