add peering_commontopo tests [NET-3700] (#17951)

Co-authored-by: R.B. Boyer <[email protected]> Co-authored-by: R.B. Boyer <[email protected]> Co-authored-by: Freddy <[email protected]> Co-authored-by: NiniOak <[email protected]>
hashicorp · Jul 18, 2023 · cd3fc9e · cd3fc9e
1 parent 548a5ca
commit cd3fc9e
Show file tree

Hide file tree

Showing 27 changed files with 3,933 additions and 27 deletions.
diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml
@@ -422,7 +422,7 @@ jobs:
               -tags "${{ env.GOTAGS }}" \
               -timeout=30m \
               -json \
-              `go list ./... | grep -v upgrade` \
+              `go list -tags "${{ env.GOTAGS }}" ./... | grep -v upgrade | grep -v peering_commontopo` \
               --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
               --target-version local \
               --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
@@ -591,6 +591,98 @@ jobs:
           DD_ENV: ci
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
 
+  peering_commontopo-integration-test:
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    needs:
+      - setup
+      - dev-build
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read
+    strategy:
+      fail-fast: false
+    env:
+      ENVOY_VERSION: "1.24.6"
+    steps:
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
+      - name: Setup Git
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
+      - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+        with:
+          go-version-file: 'go.mod'
+      - run: go env
+
+      # Get go binary from workspace
+      - name: fetch binary
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
+          path: .
+      - name: restore mode+x
+        run: chmod +x consul
+      - name: Build consul:local image
+        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Peering commonTopo Integration Tests
+        run: |
+          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
+          cd ./test-integ/peering_commontopo
+          docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+            --raw-command \
+            --format=short-verbose \
+            --debug \
+            --packages="./..." \
+            -- \
+            go test \
+            -tags "${{ env.GOTAGS }}" \
+            -timeout=30m \
+            -json . \
+            --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+            --target-version local \
+            --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+            --latest-version latest
+          ls -lrt
+        env:
+          # this is needed because of incompatibility between RYUK container and GHA
+          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
+          GOTESTSUM_FORMAT: standard-verbose
+          COMPOSE_INTERACTIVE_NO_CLI: 1
+          # tput complains if this isn't set to something.
+          TERM: ansi
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Authenticate to Vault
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: vault-auth
+        run: vault-auth
+
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Fetch Secrets
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+              kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
+
+      - name: prepare datadog-ci
+        if: ${{ !endsWith(github.repository, '-enterprise') }}
+        run: |
+          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
+          chmod +x /usr/local/bin/datadog-ci
+
+      - name: upload coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
+          DD_ENV: ci
+        run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
+
   test-integrations-success:
     needs: 
     - setup
@@ -600,6 +692,7 @@ jobs:
     - generate-envoy-job-matrices
     - envoy-integration-test
     - compatibility-integration-test
+    - peering_commontopo-integration-test
     - upgrade-integration-test
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
     if: ${{ always() }}

diff --git a/.gitignore b/.gitignore
@@ -10,6 +10,7 @@
 .vagrant/
 /pkg
 bin/
+workdir/
 changelog.tmp
 exit-code
 Thumbs.db
@@ -68,3 +69,4 @@ override.tf.json
 terraform.rc
 /go.work
 /go.work.sum
+.docker
diff --git a/api/prepared_query.go b/api/prepared_query.go
@@ -32,11 +32,11 @@ type QueryFailoverTarget struct {
 
 	// Partition specifies a partition to try during failover
 	// Note: Partition are available only in Consul Enterprise
-	Partition string
+	Partition string `json:",omitempty"`
 
 	// Namespace specifies a namespace to try during failover
 	// Note: Namespaces are available only in Consul Enterprise
-	Namespace string
+	Namespace string `json:",omitempty"`
 }
 
 // QueryDNSOptions controls settings when query results are served over DNS.

diff --git a/test-integ/README.md b/test-integ/README.md
@@ -0,0 +1,3 @@
+# test-integ
+
+Go integration tests for consul. `/test/integration` also holds integration tests; they need migrating.
diff --git a/test-integ/go.mod b/test-integ/go.mod
@@ -0,0 +1,105 @@
+module github.com/hashicorp/consul/test-integ
+
+go 1.20
+
+require (
+	github.com/hashicorp/consul/api v1.22.0
+	github.com/hashicorp/consul/sdk v0.14.0
+	github.com/hashicorp/consul/test/integration/consul-container v0.0.0-20230628201853-bdf4fad7c5a5
+	github.com/hashicorp/consul/testing/deployer v0.0.0-00010101000000-000000000000
+	github.com/hashicorp/go-cleanhttp v0.5.2
+	github.com/itchyny/gojq v0.12.13
+	github.com/mitchellh/copystructure v1.2.0
+	github.com/stretchr/testify v1.8.4
+)
+
+require (
+	fortio.org/dflag v1.5.2 // indirect
+	fortio.org/fortio v1.54.0 // indirect
+	fortio.org/log v1.3.0 // indirect
+	fortio.org/sets v1.0.2 // indirect
+	fortio.org/version v1.0.2 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/agext/levenshtein v1.2.1 // indirect
+	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/avast/retry-go v3.0.0+incompatible // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/containerd/containerd v1.7.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/docker v23.0.6+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/fatih/color v1.14.1 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-msgpack v0.5.5 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/hashicorp/go-version v1.2.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/hcl/v2 v2.16.2 // indirect
+	github.com/hashicorp/memberlist v0.5.0 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/itchyny/timefmt-go v0.1.5 // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/miekg/dns v1.1.50 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/patternmatcher v0.5.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
+	github.com/opencontainers/runc v1.1.7 // indirect
+	github.com/otiai10/copy v1.10.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/rboyer/safeio v0.2.2 // indirect
+	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 // indirect
+	github.com/testcontainers/testcontainers-go v0.20.1 // indirect
+	github.com/zclconf/go-cty v1.12.1 // indirect
+	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/grpc v1.55.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools/v3 v3.4.0 // indirect
+)
+
+replace (
+	github.com/hashicorp/consul => ../
+	github.com/hashicorp/consul/api => ../api
+	github.com/hashicorp/consul/envoyextensions => ../envoyextensions
+	github.com/hashicorp/consul/proto-public => ../proto-public
+	github.com/hashicorp/consul/sdk => ../sdk
+	github.com/hashicorp/consul/test/integration/consul-container => ../test/integration/consul-container
+	github.com/hashicorp/consul/testing/deployer => ../testing/deployer
+)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# test-integ

		Go integration tests for consul. `/test/integration` also holds integration tests; they need migrating.