Implement e2e tests for the CLI

overmindtech · Jun 24, 2024 · a99a1b5 · a99a1b5
1 parent df88f3f
commit a99a1b5
Show file tree

Hide file tree

Showing 5 changed files with 232 additions and 11 deletions.
diff --git a/.github/e2eapply.tape b/.github/e2eapply.tape
@@ -0,0 +1,18 @@
+Output apply.mp4
+
+Set Width 1920
+Set Height 1080
+
+Set FontSize 12
+
+Hide
+Type "export PATH=$PWD:$PATH"
+Enter
+Type "clear"
+Enter
+Show
+
+Type@1ms "overmind terraform apply -- tfplan"
+Enter
+Sleep 60
+Screenshot apply.png
diff --git a/.github/e2eplan.tape b/.github/e2eplan.tape
@@ -0,0 +1,20 @@
+Output plan.mp4
+
+Set Width 1920
+Set Height 1080
+
+Set FontSize 12
+
+Hide
+Type "export PATH=$PWD:$PATH"
+Enter
+Type "clear"
+Enter
+Show
+
+Type@1ms "overmind terraform plan -- -out tfplan"
+Enter
+Sleep 2
+Enter
+Sleep 60
+Screenshot plan.png
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -0,0 +1,171 @@
+name: E2E Tests
+on:
+  push:
+    branches:
+      - main
+      - e2e-tests
+
+jobs:
+  actions:
+    runs-on: depot-ubuntu-22.04-4
+    env:
+      CGO_ENABLED: 0
+    permissions:
+      contents: read # required for checkout
+      id-token: write # mint AWS credentials through OIDC
+
+    steps:
+      - name: Cache Terraform Providers
+        id: cache-terraform
+        uses: actions/cache@v3
+        with:
+          path: .terraform
+          key: ${{ runner.os }}-${{ inputs.environment }}-${{ hashFiles('**/.terraform.lock.hcl') }}
+
+      - uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.5.7
+          terraform_wrapper: false
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com # set default value to work around https://github.com/aws-actions/configure-aws-credentials/issues/271#issuecomment-1401481855
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::211125377101:role/terraform-ci-test
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Go Init
+        uses: ./.github/actions/go_init
+
+      - name: Build the binary
+        run: |
+          set -xe
+          go get -v -t -d ./...
+          go build -o overmind main.go
+
+      - name: Run E2E Tests
+        env:
+          OVM_API_KEY: ${{ secrets.OVM_API_KEY }}
+          HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY }}
+        run: |
+          set -xe
+
+          ./overmind -v
+
+          terraform init
+          terraform plan -out=tfplan
+          terraform show -json tfplan > tfplan.json
+
+          ticket_link='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'
+
+          ./overmind changes submit-plan \
+            --title "Test Change for e2e tests" \
+            --description "some description" \
+            --ticket-link "$ticket_link" \
+            tfplan.json \
+            > ./overmindtech-change-url
+
+          ./overmind changes get-change \
+            --change "$(< ./overmindtech-change-url)" \
+            --format markdown \
+            > ./overmindtech-message
+
+          ./overmind changes start-change \
+            --ticket-link "$ticket_link"
+
+          ./overmind changes end-change \
+            --ticket-link "$ticket_link"
+
+          echo "E2E Tests Complete"
+
+  interactive:
+    runs-on: depot-ubuntu-22.04-4
+    env:
+      CGO_ENABLED: 0
+    permissions:
+      contents: read # required for checkout
+      id-token: write # mint AWS credentials through OIDC
+
+    steps:
+      - name: Cache Terraform Providers
+        id: cache-terraform
+        uses: actions/cache@v3
+        with:
+          path: .terraform
+          key: ${{ runner.os }}-${{ inputs.environment }}-${{ hashFiles('**/.terraform.lock.hcl') }}
+
+      - uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.5.7
+          terraform_wrapper: false
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com # set default value to work around https://github.com/aws-actions/configure-aws-credentials/issues/271#issuecomment-1401481855
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::211125377101:role/terraform-ci-test
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Go Init
+        uses: ./.github/actions/go_init
+
+      - name: Build the binary
+        run: |
+          set -xe
+          go get -v -t -d ./...
+          go build -o overmind main.go
+
+      - name: Run E2E Tests
+        env:
+          OVM_API_KEY: ${{ secrets.OVM_API_KEY }}
+          HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY }}
+          TEABUG: pls
+        run: |
+          set -xe
+
+          go build -o overmind main.go
+
+          ./overmind -v
+
+          # avoid inheriting permissions from the containers below
+          terraform init
+          terraform plan
+          touch teabug.log
+
+          cp -a $(which terraform) . # provide a terraform binary to the containers below
+
+          docker run --rm -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e AWS_REGION -e AWS_SESSION_TOKEN -e OVM_API_KEY -e TEABUG -v $PWD:/vhs -v ~/.aws:/root/.aws ghcr.io/charmbracelet/vhs /vhs/.github/e2eplan.tape
+          docker run --rm -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e AWS_REGION -e AWS_SESSION_TOKEN -e OVM_API_KEY -e TEABUG -v $PWD:/vhs -v ~/.aws:/root/.aws ghcr.io/charmbracelet/vhs /vhs/.github/e2eapply.tape
+
+          terraform apply -destroy -auto-approve
+
+          echo "E2E Tests Complete"
+
+      - name: debug output
+        if: always()
+        run: |
+          cat teabug.log
+
+      - name: Upload plan results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: plan.mp4
+          path: plan.*
+
+      - name: Upload plan results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: apply.mp4
+          path: apply.*
diff --git a/cmd/tea.go b/cmd/tea.go
@@ -107,7 +107,13 @@ func CmdWrapper(action string, requiredScopes []string, commandModel func(args [
 				tasks:          map[string]tea.Model{},
 			}
 			m.cmd = commandModel(args, &m, m.width)
-			p := tea.NewProgram(&m)
+
+			options := []tea.ProgramOption{}
+			if os.Getenv("CI") != "" {
+				// See https://github.com/charmbracelet/bubbletea/issues/761#issuecomment-1625863769
+				options = append(options, tea.WithInput(nil))
+			}
+			p := tea.NewProgram(&m, options...)
 			result, err := p.Run()
 			if err != nil {
 				return fmt.Errorf("could not start program: %w", err)

diff --git a/cmd/terraform_apply.go b/cmd/terraform_apply.go
@@ -69,7 +69,9 @@ type changeIdentifiedMsg struct {
 }
 
 type runTfApplyMsg struct{}
-type tfApplyFinishedMsg struct{}
+type tfApplyFinishedMsg struct {
+	err error
+}
 
 func NewTfApplyModel(args []string, parent *cmdModel, width int) tea.Model {
 	hasPlanSet := false
@@ -305,20 +307,24 @@ func (m tfApplyModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 				defer span.End()
 
 				if err != nil {
-					return fatalError{err: fmt.Errorf("failed to run terraform apply: %w", err)}
+					return tfApplyFinishedMsg{err: fmt.Errorf("failed to run terraform apply: %w", err)}
 				}
 
-				return tfApplyFinishedMsg{}
+				return tfApplyFinishedMsg{err: nil}
 			})
 	case tfApplyFinishedMsg:
 		m.runTfApply = false
-		m.isEnding = true
-		cmds = append(cmds,
-			func() tea.Msg { return hideStartupStatusMsg{} },
-			m.endingChangeSnapshot.Init(),
-			m.startEndChangeCmd(),
-			m.waitForEndingActivity,
-		)
+		cmds = append(cmds, func() tea.Msg { return hideStartupStatusMsg{} })
+		if msg.err != nil {
+			cmds = append(cmds, func() tea.Msg { return fatalError{err: msg.err} })
+		} else {
+			m.isEnding = true
+			cmds = append(cmds,
+				m.endingChangeSnapshot.Init(),
+				m.startEndChangeCmd(),
+				m.waitForEndingActivity,
+			)
+		}
 	}
 
 	mdl, cmd := m.startingChangeSnapshot.Update(msg)