diff --git a/model/common_datatypes.smithy b/model/common_datatypes.smithy index 81255e03d..812ab4257 100644 --- a/model/common_datatypes.smithy +++ b/model/common_datatypes.smithy @@ -28,6 +28,10 @@ enum SettingType { DEFAULTS = "defaults" } +list IndexNameList { + member: IndexName +} + list UserDefinedValueList{ member: String } diff --git a/model/opensearch.smithy b/model/opensearch.smithy index ca7be76f3..a9f06319e 100644 --- a/model/opensearch.smithy +++ b/model/opensearch.smithy @@ -17,5 +17,23 @@ use aws.protocols#restJson1 @restJson1 service OpenSearch { version: "2021-11-23", - operations: [PutCreateIndex, PutIndexMappingWithIndex, GetCatIndices, GetCatIndicesWithIndex, GetCatNodes, PostSearch, PostSearchWithIndex, DeleteIndex, GetDocumentDoc, GetDocumentSource, GetClusterInfo, PutUpdateClusterSettings, GetClusterSettings, PostAliases, GetSettingsIndex, GetSettingsIndexSetting] + operations: [ + PutCreateIndex, + PutIndexMappingWithIndex, + GetCatIndices, + GetCatIndicesWithIndex, + GetCatNodes, + PostSearch, + PostSearchWithIndex, + DeleteIndex, + GetDocumentDoc, + GetDocumentSource, + GetClusterInfo, + PutUpdateClusterSettings, + GetClusterSettings, + PostAliases, + GetSettingsIndex, + GetSettingsIndexSetting, + PostRemoteStoreRestore + ] } diff --git a/model/remotestore/restore/operations.smithy b/model/remotestore/restore/operations.smithy new file mode 100644 index 000000000..4937e3071 --- /dev/null +++ b/model/remotestore/restore/operations.smithy @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@unstable +@externalDocumentation( + "OpenSearch Documentation": "https://opensearch.org/docs/latest/opensearch/remote/#restoring-from-a-backup" +) + +@suppress(["HttpUriConflict"]) +@http(method: "POST", uri: "/_remotestore/_restore") +@documentation("Restore one or more indices from a remote backup.") +operation PostRemoteStoreRestore{ + input: PostRemoteStoreRestoreInput, + output: PostRemoteStoreRestoreOutput +} diff --git a/model/remotestore/restore/structures.smithy b/model/remotestore/restore/structures.smithy new file mode 100644 index 000000000..802196981 --- /dev/null +++ b/model/remotestore/restore/structures.smithy @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@unstable +structure PostRemoteStoreRestoreInput { + @httpQuery("cluster_manager_timeout") + cluster_manager_timeout: Time, + + @httpQuery("wait_for_completion") + wait_for_completion: Boolean, + + @required + indices: IndexNameList +} + +@unstable +structure PostRemoteStoreRestoreOutput { + accepted: Boolean, + remote_store: RemoteStoreRestoreInfo +} + +@unstable +structure RemoteStoreRestoreInfo { + snapshot: String, + indices: IndexNameList, + shards: RemoteStoreRestoreShardsInfo +} + +@unstable +structure RemoteStoreRestoreShardsInfo { + total: Integer, + failed: Integer, + successful: Integer +} + +apply PostRemoteStoreRestore @examples([ + { + title: "Examples for Post Remote Storage Restore Operation.", + input: { + indices: ["books"] + }, + output: { + remote_store: { + indices: ["books"], + shards: { + total: 1, + failed: 0, + successful: 1 + } + } + } + } +]) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index edc99e395..588c76de4 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: opensearch-node1: - image: opensearchproject/opensearch:latest + build: ./opensearch container_name: opensearch-node1 environment: - cluster.name=opensearch-cluster @@ -9,7 +9,8 @@ services: - discovery.seed_hosts=opensearch-node1,opensearch-node2 - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - path.repo=/mnt/snapshots + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dopensearch.experimental.feature.replication_type.enabled=true -Dopensearch.experimental.feature.remote_store.enabled=true" ulimits: memlock: soft: -1 @@ -19,6 +20,7 @@ services: hard: 65536 volumes: - opensearch-data1:/usr/share/opensearch/data + - opensearch-snapshots:/mnt/snapshots ports: - 9200:9200 - 9600:9600 # required for Performance Analyzer @@ -26,7 +28,7 @@ services: - opensearch-net opensearch-node2: - image: opensearchproject/opensearch:latest + build: ./opensearch container_name: opensearch-node2 environment: - cluster.name=opensearch-cluster @@ -34,7 +36,8 @@ services: - discovery.seed_hosts=opensearch-node1,opensearch-node2 - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - path.repo=/mnt/snapshots + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dopensearch.experimental.feature.replication_type.enabled=true -Dopensearch.experimental.feature.remote_store.enabled=true" ulimits: memlock: soft: -1 @@ -44,12 +47,14 @@ services: hard: 65536 volumes: - opensearch-data2:/usr/share/opensearch/data + - opensearch-snapshots:/mnt/snapshots networks: - opensearch-net volumes: opensearch-data1: opensearch-data2: + opensearch-snapshots: networks: opensearch-net: diff --git a/test/models/remotestore/restore/OpenSearchModel.json b/test/models/remotestore/restore/OpenSearchModel.json new file mode 100644 index 000000000..170fb39e4 --- /dev/null +++ b/test/models/remotestore/restore/OpenSearchModel.json @@ -0,0 +1,169 @@ +{ + "openapi": "3.0.2", + "info": { + "title": "OpenSearch", + "version": "2021-11-23" + }, + "paths": { + "/_remotestore/_restore": { + "post": { + "description": "Restore one or more indices from a remote backup.", + "operationId": "PostRemoteStoreRestore", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PostRemoteStoreRestoreRequestContent" + }, + "examples": { + "PostRemoteStoreRestore_example1": { + "summary": "Examples for Post Remote Storage Restore Operation.", + "description": "", + "value": { + "indices": [ + "books" + ] + } + } + } + } + }, + "required": true + }, + "parameters": [ + { + "name": "cluster_manager_timeout", + "in": "query", + "schema": { + "type": "string", + "pattern": "^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$" + } + }, + { + "name": "wait_for_completion", + "in": "query", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "PostRemoteStoreRestore 200 response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PostRemoteStoreRestoreResponseContent" + }, + "examples": { + "PostRemoteStoreRestore_example1": { + "summary": "Examples for Post Remote Storage Restore Operation.", + "description": "", + "value": { + "remote_store": { + "indices": [ + "books" + ], + "shards": { + "total": 1, + "failed": 0, + "successful": 1 + } + } + } + } + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "PostRemoteStoreRestoreRequestContent": { + "type": "object", + "properties": { + "indices": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[^+_\\-\\.][^\\\\, /*?\"<>| ,#\\nA-Z]+$" + } + } + }, + "required": [ + "indices" + ] + }, + "PostRemoteStoreRestoreResponseContent": { + "type": "object", + "properties": { + "accepted": { + "type": "boolean" + }, + "remote_store": { + "$ref": "#/components/schemas/RemoteStoreRestoreInfo" + } + }, + "example": { + "remote_store": { + "indices": [ + "books" + ], + "shards": { + "total": 1, + "failed": 0, + "successful": 1 + } + } + } + }, + "RemoteStoreRestoreInfo": { + "type": "object", + "properties": { + "snapshot": { + "type": "string" + }, + "indices": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[^+_\\-\\.][^\\\\, /*?\"<>| ,#\\nA-Z]+$" + } + }, + "shards": { + "$ref": "#/components/schemas/RemoteStoreRestoreShardsInfo" + } + } + }, + "RemoteStoreRestoreShardsInfo": { + "type": "object", + "properties": { + "total": { + "type": "number" + }, + "failed": { + "type": "number" + }, + "successful": { + "type": "number" + } + } + } + }, + "securitySchemes": { + "smithy.api.httpBasicAuth": { + "type": "http", + "description": "HTTP Basic authentication", + "scheme": "Basic" + } + } + }, + "security": [ + { + "smithy.api.httpBasicAuth": [] + } + ] + } \ No newline at end of file diff --git a/test/models/remotestore/restore/hooks.js b/test/models/remotestore/restore/hooks.js new file mode 100644 index 000000000..8a5ecadea --- /dev/null +++ b/test/models/remotestore/restore/hooks.js @@ -0,0 +1,98 @@ +const https = require('https'); +const fetch = require('node-fetch') +const hooks = require('hooks'); +const fs = require('fs'); + +var host = ""; +var protocol = "https"; +var auth = ""; + +// Reading .txt file to set URL +const data = fs.readFileSync('url.txt', {encoding:'utf8', flag:'r'}); +function address() +{ + text = data.toString(); + text = text.split(" "); + host = text[0].substring(8,text[0].length); + auth = text[1]; + return (protocol + "://" + auth + "@" + host); +} + +// Remote Storage Restore + +hooks.before("/_remotestore/_restore > POST > 200 > application/json", function(transactions, done) { + transactions.expected.headers['Content-Type'] = "application/json; charset=UTF-8"; + + const request = async () => { + + var url = address(); + + // Register repository for remote store + await fetch(url + '/_snapshot/books-repo', { + method: 'PUT', + body: JSON.stringify({ + type: 'fs', + settings: { + location: '/mnt/snapshots' + } + }), + headers: { + "content-type": "application/json; charset=UTF-8" + } + }); + + // Create an index configured for remote store + await fetch(url + '/books', { + method: 'PUT', + body: JSON.stringify({ + settings : { + index: { + number_of_shards: 1, + number_of_replicas: 0, + replication: { + type: 'SEGMENT' + }, + remote_store: { + enabled: true, + repository: 'books-repo' + } + } + } + }), + headers: { + "content-type": "application/json; charset=UTF-8" + } + }); + + // Close the index + await fetch(url + '/books/_close', { + method: 'POST' + }); + + done(); + } + + request(); +}); + +hooks.after("/_remotestore/_restore > POST > 200 > application/json", function(transactions, done) { + + const request = async () => { + + var url = address(); + + // Delete index + await fetch(url + '/books', { + method: 'DELETE' + }); + + // Delete repository + await fetch(url + '/_snapshot/books-repo', { + method: 'DELETE' + }); + + done(); + } + request(); +}); + diff --git a/test/opensearch/Dockerfile b/test/opensearch/Dockerfile new file mode 100644 index 000000000..569375bfb --- /dev/null +++ b/test/opensearch/Dockerfile @@ -0,0 +1,7 @@ +FROM opensearchproject/opensearch:latest + +USER root +RUN mkdir -p /mnt/snapshots && chown -R opensearch:opensearch /mnt/snapshots + +USER opensearch +VOLUME /mnt/snapshots \ No newline at end of file diff --git a/test/scripts/operation-filter.py b/test/scripts/operation-filter.py index faead58f5..5339847e9 100644 --- a/test/scripts/operation-filter.py +++ b/test/scripts/operation-filter.py @@ -22,7 +22,7 @@ def filter_operation(self): shutil.copyfile(original, target) - command = 'sed "s/operations:.*$/operations: [' + self.operations + \ + command = 'sed -e \'1h;2,$H;$!d;g\' -e "s/operations: \[[^]]*\]/operations: [' + self.operations + \ ']/" opensearch_temp.smithy > ../../model/opensearch.smithy' os.system(command)