Skip to content

Commit

Permalink
Remove unnecessary gradle task for segrep
Browse files Browse the repository at this point in the history 
Signed-off-by: Suraj Singh <[email protected]>
  • Loading branch information
@dreamer-89
dreamer-89 committed May 11, 2023
1 parent 1cc09cd commit effdbd9
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 46 deletions.
30 changes: 0 additions & 30 deletions qa/mixed-cluster/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
}

String baseName = "v${bwcVersion}"
String segRepCluster = "${baseName}segrep"

/* This project runs the core REST tests against a 4 node cluster where two of
the nodes has a different minor. */
Expand All @@ -66,12 +65,6 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {

setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
}

"$segRepCluster" {
versions = [bwcVersion.toString(), project.version]
numberOfNodes = 2
setting 'path.repo', "${buildDir}/cluster/shared/repo/${segRepCluster}"
}
}

tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) {
Expand All @@ -96,30 +89,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
onlyIf { project.bwc_tests_enabled }
}

/**
* Defines a new task for segment replication where tests utilizes system property to set boolean param, consumed
* in tests.
*/
tasks.register("${baseName}#segrepMixedClusterTest", StandaloneRestIntegTestTask) {
useCluster testClusters."${segRepCluster}"
doFirst {
delete("${buildDir}/cluster/shared/repo/${segRepCluster}")
// Getting the endpoints causes a wait for the cluster
println "Test cluster endpoints are: ${-> testClusters."${segRepCluster}".allHttpSocketURI.join(",")}"
println "Upgrading one node to create a mixed cluster"
testClusters."${segRepCluster}".nextNodeToNextVersion()
// Getting the endpoints causes a wait for the cluster
println "Upgrade complete, endpoints are: ${-> testClusters."${segRepCluster}".allHttpSocketURI.join(",")}"
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${segRepCluster}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${segRepCluster}".getName()}")
}
systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${segRepCluster}"
systemProperty 'tests.segrep_enabled', true
onlyIf { project.bwc_tests_enabled }
}

tasks.register(bwcTaskName(bwcVersion)) {
dependsOn "${baseName}#mixedClusterTest"
dependsOn "${baseName}#segrepMixedClusterTest"
}
}
33 changes: 17 additions & 16 deletions qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,6 @@

public class IndexingIT extends OpenSearchRestTestCase {

protected static final Boolean SEGREP_ENABLED = Boolean.parseBoolean(System.getProperty("tests.segrep_enabled"));


private int indexDocs(String index, final int idStart, final int numDocs) throws IOException {
for (int i = 0; i < numDocs; i++) {
Expand Down Expand Up @@ -118,37 +116,37 @@ private void printIndexSettings(String index) throws IOException, ParseException
}

/**
* This test verifies that segment replication does not break when primary shards are on higher version.
* This test verifies that segment replication does not break when primary shards are on lower OS version. It does this
* by verifying replica shards contains same number of documents as primary's.
*
* @throws Exception
*/
public void testIndexingWithPrimaryOnBwcNodes() throws Exception {
if (SEGREP_ENABLED == false) return;
Nodes nodes = buildNodeAndVersions();
assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty());
logger.info("cluster discovered:\n {}", nodes.toString());
final List<String> bwcNamesList = nodes.getBWCNodes().stream().map(Node::getNodeName).collect(Collectors.toList());
logger.info("--> bwc nodes {}", bwcNamesList);
final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(","));
// Exclude bwc nodes from allocation so that primaries gets allocated on current version
Settings.Builder settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put("index.routing.allocation.include._name", bwcNames);
final String index = "test-index";
createIndex(index, settings.build());
ensureYellow(index);
printClusterRouting();

printIndexSettings(index);

int docCount = 200;
int docCount = 20;
try (RestClient nodeClient = buildClient(restClientSettings(),
nodes.values().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) {
nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) {

logger.info("allowing replica shards assignment on bwc nodes");
updateIndexSettings(index, Settings.builder().putNull("index.routing.allocation.include._name"));
// Add replicas so that it can be assigned on higher OS version nodes.
updateIndexSettings(index, Settings.builder().put("index.number_of_replicas", 2));

printClusterRouting();
printIndexSettings(index);
ensureGreen(index);
Expand All @@ -166,22 +164,22 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception {


/**
* This test creates a cluster with primary on older version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider} which prevents replica shard allocation on lower OpenSearch version
* This test creates a cluster with primary on older version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider};
* replica shard allocation on lower OpenSearch version is prevented. Thus, this test though cover the use case where
* primary shard containing nodes are running on higher OS version while replicas are unassigned.
*
* @throws Exception
*/
public void testIndexingWithReplicaOnBwcNodes() throws Exception {
if (SEGREP_ENABLED == false) return;
Nodes nodes = buildNodeAndVersions();
assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty());
logger.info("cluster discovered:\n {}", nodes.toString());
final List<String> bwcNamesList = nodes.getBWCNodes().stream().map(Node::getNodeName).collect(Collectors.toList());
logger.info("--> bwc nodes {}", bwcNamesList);
final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(","));
// Exclude bwc nodes from allocation so that primaries gets allocated on current version
// Exclude bwc nodes from allocation so that primaries gets allocated on current/higher version
Settings.Builder settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put("index.routing.allocation.exclude._name", bwcNames);
final String index = "test-index";
Expand All @@ -196,6 +194,8 @@ public void testIndexingWithReplicaOnBwcNodes() throws Exception {

logger.info("allowing replica shards assignment on bwc nodes");
updateIndexSettings(index, Settings.builder().putNull("index.routing.allocation.exclude._name"));
// Add replicas so that it can be assigned on lower OS version nodes, but it doesn't work as called out in test overview
updateIndexSettings(index, Settings.builder().put("index.number_of_replicas", 2));
printClusterRouting();

// Index docs
Expand Down Expand Up @@ -507,10 +507,11 @@ private void assertSeqNoOnShards(String index, Nodes nodes, int numDocs, RestCli
});
}

private List<Shard> buildShards(String index, Nodes nodes, RestClient client) throws IOException {
private List<Shard> buildShards(String index, Nodes nodes, RestClient client) throws IOException, ParseException {
Request request = new Request("GET", index + "/_stats");
request.addParameter("level", "shards");
Response response = client.performRequest(request);
logger.info("_stats response --> {}", EntityUtils.toString(client().performRequest(request).getEntity()).trim());
List<Object> shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + index + ".shards.0");
ArrayList<Shard> shards = new ArrayList<>();
for (Object shard : shardStats) {
Expand Down

0 comments on commit effdbd9

Please sign in to comment.