From 2d469d441d2ba3d592eefd2b619895b9eaf9c2ce Mon Sep 17 00:00:00 2001 From: Jakob Date: Thu, 23 Mar 2023 16:19:55 +0100 Subject: [PATCH] SnapshotDeleteRequest use []string instead of string (#237) * SnapshotDeleteRequest use []string instead of string Signed-off-by: Jakob Hahn * Add opensearchapi snapshot ingegration test Signed-off-by: Jakob Hahn * Add path.repo value to test-integration-unreleased workflow Signed-off-by: Jakob Hahn * Add UPGRADING.md Signed-off-by: Jakob Hahn * opensearchapi: Fix integration test with new error handling Signed-off-by: Jakob Hahn --------- Signed-off-by: Jakob Hahn Co-authored-by: Daniel (dB.) Doubrovkine --- .ci/opensearch/docker-compose.yml | 1 + .../workflows/test-integration-unreleased.yml | 2 +- CHANGELOG.md | 41 +++--- UPGRADING.md | 66 ++++++++-- opensearchapi/api.snapshot.delete.go | 10 +- .../opensearchapi_integration_test.go | 121 ++++++++++++++++++ 6 files changed, 199 insertions(+), 42 deletions(-) diff --git a/.ci/opensearch/docker-compose.yml b/.ci/opensearch/docker-compose.yml index 9fbd391e7..b27694e98 100755 --- a/.ci/opensearch/docker-compose.yml +++ b/.ci/opensearch/docker-compose.yml @@ -12,6 +12,7 @@ services: environment: - discovery.type=single-node - bootstrap.memory_lock=true + - path.repo=/usr/share/opensearch/mnt ports: - "9200:9200" user: opensearch diff --git a/.github/workflows/test-integration-unreleased.yml b/.github/workflows/test-integration-unreleased.yml index 05c67992b..3f272d03b 100644 --- a/.github/workflows/test-integration-unreleased.yml +++ b/.github/workflows/test-integration-unreleased.yml @@ -51,7 +51,7 @@ jobs: working-directory: opensearch/distribution/archives/linux-tar/build/distributions run: | tar xf opensearch-min-* - ./opensearch-*/bin/opensearch & + ./opensearch-*/bin/opensearch -Epath.repo=/usr/share/opensearch/mnt & for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done - name: Checkout Go Client diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb9905ad..fe9e9ab2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,47 +6,36 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Dependencies +- Bumps `github.com/aws/aws-sdk-go` from 1.44.45 to 1.44.225 - Bumps `github.com/aws/aws-sdk-go-v2` from 1.17.1 to 1.17.6 -- Bumps `github.com/aws/aws-sdk-go-v2/config` from 1.18.10 to 1.18.18 -- Bumps `github.com/aws/aws-sdk-go-v2/config` from 1.18.9 to 1.18.12 -- Bumps `github.com/aws/aws-sdk-go-v2/config` from 1.18.8 to 1.18.10 -- Bumps `github.com/aws/aws-sdk-go-v2/config` from 1.17.10 to 1.18.9 -- Bumps `github.com/aws/aws-sdk-go` from 1.44.195 to 1.44.225 -- Bumps `github.com/aws/aws-sdk-go` from 1.44.185 to 1.44.205 -- Bumps `github.com/aws/aws-sdk-go` from 1.44.180 to 1.44.195 -- Bumps `github.com/aws/aws-sdk-go` from 1.44.176 to 1.44.185 -- Bumps `github.com/aws/aws-sdk-go` from 1.44.132 to 1.44.180 -- Bumps `github.com/stretchr/testify` from 1.8.1 to 1.8.2 +- Bumps `github.com/aws/aws-sdk-go-v2/config` from 1.18.8 to 1.18.18 +- Bumps `github.com/stretchr/testify` from 1.8.0 to 1.8.2 ### Added -- Github workflow for changelog verification ([#172](https://github.com/opensearch-project/opensearch-go/pull/172)) -- Add Go Documentation link for the client ([#182](https://github.com/opensearch-project/opensearch-go/pull/182)) -- Add implementation of Data Streams API ([#257](https://github.com/opensearch-project/opensearch-go/pull/257) -- Support for Amazon OpenSearch Serverless ([#216](https://github.com/opensearch-project/opensearch-go/pull/216), [#259](https://github.com/opensearch-project/opensearch-go/pull/259)) -- Add Err() function to Response for detailed errors ([#246](https://github.com/opensearch-project/opensearch-go/pull/246)) - -### Dependencies - -- Bumps `github.com/stretchr/testify` from 1.8.0 to 1.8.1 -- Bumps `github.com/aws/aws-sdk-go` from 1.44.45 to 1.44.132 +- Adds support for Amazon OpenSearch Serverless ([#216](https://github.com/opensearch-project/opensearch-go/pull/216), [#259](https://github.com/opensearch-project/opensearch-go/pull/259)) +- Adds Github workflow for changelog verification ([#172](https://github.com/opensearch-project/opensearch-go/pull/172)) +- Adds Go Documentation link for the client ([#182](https://github.com/opensearch-project/opensearch-go/pull/182)) +- Adds implementation of Data Streams API ([#257](https://github.com/opensearch-project/opensearch-go/pull/257) +- Adds `Err()` function to Response for detailed errors ([#246](https://github.com/opensearch-project/opensearch-go/pull/246)) ### Changed -- Workflow improvements ([#242](https://github.com/opensearch-project/opensearch-go/pull/242)) -- Opensearchapi check the response for errors ([#246](https://github.com/opensearch-project/opensearch-go/pull/246)) +- Uses `[]string` instead of `string` in `SnapshotDeleteRequest` ([#237](https://github.com/opensearch-project/opensearch-go/pull/237)) +- Removes the need for double error checking ([#246](https://github.com/opensearch-project/opensearch-go/pull/246)) +- Updates workflows to reduce CI time, consolidate OpenSearch versions, update compatibility matrix ([#242](https://github.com/opensearch-project/opensearch-go/pull/242)) ### Deprecated ### Removed -- Remove info call before performing every request ([#219](https://github.com/opensearch-project/opensearch-go/pull/219)) +- Removes info call before performing every request ([#219](https://github.com/opensearch-project/opensearch-go/pull/219)) ### Fixed -- Renamed the sequence number struct tag to if_seq_no to fix optimistic concurrency control ([#166](https://github.com/opensearch-project/opensearch-go/pull/166)) -- Fix `RetryOnConflict` on bulk indexer ([#215](https://github.com/opensearch-project/opensearch-go/pull/215)) -- Correct curl logging to emit the correct URL destination ([#101](https://github.com/opensearch-project/opensearch-go/pull/101)) +- Renames the sequence number struct tag to `if_seq_no` to fix optimistic concurrency control ([#166](https://github.com/opensearch-project/opensearch-go/pull/166)) +- Fixes `RetryOnConflict` on bulk indexer ([#215](https://github.com/opensearch-project/opensearch-go/pull/215)) +- Corrects curl logging to emit the correct URL destination ([#101](https://github.com/opensearch-project/opensearch-go/pull/101)) ### Security diff --git a/UPGRADING.md b/UPGRADING.md index 48dc9818a..e0d86616b 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,25 +1,71 @@ - [Upgrading Opensearch GO Client](#upgrading-opensearch-go-client) - [Upgraading to >= 3.0.0](#upgrading-to->=-3.0.0) + - [opensearchapi](#opensearchapi-snapshot-delete) - [opensearchapi](#opensearchapi-error-handling) # Upgrading Opensearch GO Client ## Upgrading to >= 3.0.0 +### opensearchapi snapshot delete + +`SnapshotDeleteRequest` and `SnapshotDelete` changed the argument `Snapshot` type from `string` to `[]string`. + +Before: + +```go +// If you have a string containing your snapshot +stringSnapshotsToDelete := "snapshot-1,snapshot-2" +reqSnapshots := &opensearchapi.SnapshotDeleteRequest{ + Repository: repo, + Snapshot: stringSnapshotsToDelete, +} + +// If you have a slice of strings containing your snapshot +sliceSnapshotToDelete := []string{"snapshot-1","snapshot-2"} +reqSnapshots := &opensearchapi.SnapshotDeleteRequest{ + Repository: repo, + Snapshot: strings.Join(sliceSnapshotsToDelete, ","), +} +``` + +After: + +```go +// If you have a string containing your snapshots +stringSnapshotsToDelete := strings.Split("snapshot-1,snapshot-2", ",") +reqSnapshots := &opensearchapi.SnapshotDeleteRequest{ + Repository: repo, + Snapshot: stringSnapshotsToDelete, +} + +// If you have a slice of strings containing your snapshots +sliceSnapshotToDelete := []string{"snapshot-1", "snapshot-2"} +reqSnapshots := &opensearchapi.SnapshotDeleteRequest{ + Repository: repo, + Snapshot: sliceSnapshotsToDelete, +``` + ### opensearchapi error handling -With opensearch-go >= 3.0.0 opensearchapi responses are now checked for errors. -Prior versions only returned an error if the request failed to execute. For example if the client can't reach the server or the TLS handshake failed. + +With opensearch-go >= 3.0.0 opensearchapi responses are now checked for errors. Checking for errors twice is no longer needed. + +Prior versions only returned an error if the request failed to execute. For example if the client can't reach the server or the TLS handshake failed. With opensearch-go >= 3.0.0 each opensearchapi requests will return an error if the response http status code is > 299. -The error can be parsed into the new opensearchapi.Error type by using `errors.As` to match for exceptions and get a more detailed view. See the example below. +The error can be parsed into the new `opensearchapi.Error` type by using `errors.As` to match for exceptions and get a more detailed view. + +Example: ```go - createIndex := opensearchapi.IndicesCreateRequest{ - Index: IndexName, - Body: mapping, - } - ctx := context.Background() - var opensearchError *opensearchapi.Error - createIndexResponse, err := createIndex.Do(ctx, client) +createIndex := opensearchapi.IndicesCreateRequest{ + Index: IndexName, + Body: mapping, +} + +ctx := context.Background() +var opensearchError *opensearchapi.Error + +createIndexResponse, err := createIndex.Do(ctx, client) // Load err into opensearchapi.Error to access the fields and tolerate if the index already exists if err != nil { if errors.As(err, &opensearchError) { diff --git a/opensearchapi/api.snapshot.delete.go b/opensearchapi/api.snapshot.delete.go index dcceef74f..1d2021ad2 100644 --- a/opensearchapi/api.snapshot.delete.go +++ b/opensearchapi/api.snapshot.delete.go @@ -34,7 +34,7 @@ import ( ) func newSnapshotDeleteFunc(t Transport) SnapshotDelete { - return func(repository string, snapshot string, o ...func(*SnapshotDeleteRequest)) (*Response, error) { + return func(repository string, snapshot []string, o ...func(*SnapshotDeleteRequest)) (*Response, error) { var r = SnapshotDeleteRequest{Repository: repository, Snapshot: snapshot} for _, f := range o { f(&r) @@ -48,13 +48,13 @@ func newSnapshotDeleteFunc(t Transport) SnapshotDelete { // SnapshotDelete deletes a snapshot. // // -type SnapshotDelete func(repository string, snapshot string, o ...func(*SnapshotDeleteRequest)) (*Response, error) +type SnapshotDelete func(repository string, snapshot []string, o ...func(*SnapshotDeleteRequest)) (*Response, error) // SnapshotDeleteRequest configures the Snapshot Delete API request. // type SnapshotDeleteRequest struct { Repository string - Snapshot string + Snapshot []string MasterTimeout time.Duration ClusterManagerTimeout time.Duration @@ -80,13 +80,13 @@ func (r SnapshotDeleteRequest) Do(ctx context.Context, transport Transport) (*Re method = "DELETE" - path.Grow(1 + len("_snapshot") + 1 + len(r.Repository) + 1 + len(r.Snapshot)) + path.Grow(1 + len("_snapshot") + 1 + len(r.Repository) + 1 + len(strings.Join(r.Snapshot, ","))) path.WriteString("/") path.WriteString("_snapshot") path.WriteString("/") path.WriteString(r.Repository) path.WriteString("/") - path.WriteString(r.Snapshot) + path.WriteString(strings.Join(r.Snapshot, ",")) params = make(map[string]string) diff --git a/opensearchapi/opensearchapi_integration_test.go b/opensearchapi/opensearchapi_integration_test.go index 8690b112b..a4ad291c0 100644 --- a/opensearchapi/opensearchapi_integration_test.go +++ b/opensearchapi/opensearchapi_integration_test.go @@ -31,7 +31,9 @@ package opensearchapi_test import ( "bytes" + "context" "encoding/json" + "errors" "fmt" "strings" "testing" @@ -192,4 +194,123 @@ func TestAPI(t *testing.T) { } } }) + t.Run("Snapshot", func(t *testing.T) { + // Functio to perform requests + // + opensearchDo := func(ctx context.Context, client *opensearch.Client, req opensearchapi.Request, msg string, t *testing.T) { + _, err := req.Do(ctx, client) + if err != nil { + var opensearchError *opensearchapi.Error + if errors.As(err, &opensearchError) { + if opensearchError.Err.Type == "snapshot_missing_exception" { + return + } + t.Fatalf("Failed to %s: %s", msg, err) + } + } + } + + // Create Client + // + client, err := opensearch.NewDefaultClient() + if err != nil { + t.Fatalf("Error creating the client: %s\n", err) + } + + // Pre Cleanup indices + // + iDeleteReq := &opensearchapi.IndicesDeleteRequest{ + Index: []string{"test", "test_restored"}, + IgnoreUnavailable: opensearchapi.BoolPtr(true), + } + ctx := context.Background() + opensearchDo(ctx, client, iDeleteReq, "index data", t) + + // Index data + // + var buf bytes.Buffer + for j := 1; j <= 1000; j++ { + meta := []byte(fmt.Sprintf(`{ "index" : { "_id" : "%d" } }%s`, j, "\n")) + data := []byte(`{"content":"` + strings.Repeat("ABC", 100) + `"}`) + data = append(data, "\n"...) + + buf.Grow(len(meta) + len(data)) + buf.Write(meta) + buf.Write(data) + } + + bulkReq := &opensearchapi.BulkRequest{ + Body: bytes.NewReader(buf.Bytes()), + Index: "test", + Refresh: "true", + } + opensearchDo(ctx, client, bulkReq, "index data", t) + + // Test Snapshot functions + // + sRepoCreateReq := &opensearchapi.SnapshotCreateRepositoryRequest{ + Body: bytes.NewBufferString(`{"type":"fs","settings":{"location":"/usr/share/opensearch/mnt"}}`), + Repository: "snapshot-test", + } + opensearchDo(ctx, client, sRepoCreateReq, "create Snapshot Repository", t) + + sRepoVerifyReq := &opensearchapi.SnapshotVerifyRepositoryRequest{ + Repository: "snapshot-test", + } + opensearchDo(ctx, client, sRepoVerifyReq, "verify Snapshot Repository", t) + + sDeleteReq := &opensearchapi.SnapshotDeleteRequest{ + Snapshot: []string{"test", "clone-test"}, + Repository: "snapshot-test", + } + opensearchDo(ctx, client, sDeleteReq, "delete Snapshots", t) + + sCreateReq := &opensearchapi.SnapshotCreateRequest{ + Body: bytes.NewBufferString(`{"indices":"test","ignore_unavailable":true,"include_global_state":false,"partial":false}`), + Snapshot: "test", + Repository: "snapshot-test", + WaitForCompletion: opensearchapi.BoolPtr(true), + } + opensearchDo(ctx, client, sCreateReq, "create Snapshot", t) + + sCloneReq := &opensearchapi.SnapshotCloneRequest{ + Body: bytes.NewBufferString(`{"indices":"*"}`), + Snapshot: "test", + TargetSnapshot: "clone-test", + Repository: "snapshot-test", + } + opensearchDo(ctx, client, sCloneReq, "clone Snapshot", t) + + sGetReq := &opensearchapi.SnapshotGetRequest{ + Snapshot: []string{"test", "clone-test"}, + Repository: "snapshot-test", + } + opensearchDo(ctx, client, sGetReq, "get Snapshots", t) + + sStatusReq := &opensearchapi.SnapshotGetRequest{ + Snapshot: []string{"test", "clone-test"}, + Repository: "snapshot-test", + } + opensearchDo(ctx, client, sStatusReq, "get Snapshot status", t) + + sRestoreReq := &opensearchapi.SnapshotRestoreRequest{ + Body: bytes.NewBufferString( + `{ + "indices":"test", + "ignore_unavailable":true, + "include_global_state":false, + "partial":false, + "rename_pattern": "(.+)", + "rename_replacement":"$1_restored" + }`, + ), + Snapshot: "clone-test", + Repository: "snapshot-test", + WaitForCompletion: opensearchapi.BoolPtr(true), + } + opensearchDo(ctx, client, sRestoreReq, "restore Snapshot", t) + + opensearchDo(ctx, client, sDeleteReq, "delete Snapshots", t) + opensearchDo(ctx, client, iDeleteReq, "index data", t) + }) }