Golang Re-Write MVP (#293)

* Creating a Golang version of reckoner * do snapshots * remove docker build using rok8s * build python 3.9 * different image
FairwindsOps · Aug 6, 2021 · 7e626e7 · 7e626e7
1 parent 66a8464
commit 7e626e7
Show file tree

Hide file tree

Showing 34 changed files with 2,754 additions and 66 deletions.
diff --git a/.circleci/build.config b/.circleci/build.config
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -24,26 +24,56 @@ executors:
   python-3-8:
     docker:
       - image: circleci/python:3.8
+  python-3-9:
+    docker:
+      - image: cimg/python:3.9
 
 references:
-  e2e_configuration: &e2e_configuration
-    pre_script: end_to_end_testing/pre.sh
-    script: end_to_end_testing/run.sh
+  e2e_configuration_python: &e2e_configuration_python
+    pre_script: end_to_end_testing/pre_python.sh
+    script: end_to_end_testing/run_python.sh
     command_runner_image: quay.io/reactiveops/ci-images:v11-buster
-    enable_docker_layer_caching: true
+    enable_docker_layer_caching: false
     store-test-results: /tmp/test-results
-    attach-workspace: true
-    workspace-location: /tmp/binaries
-    kind_version: 0.9.0
+    kind_version: 0.10.0
     requires:
-      - build-3-8
       - build-3-7
+      - build-3-8
+      - build-3-9
     filters:
       branches:
         only: /.*/
       tags:
-        ignore: /v.*/
-
+        ignore: /.*/
+  e2e_configuration_go: &e2e_configuration_go
+    pre_script: end_to_end_testing/pre_go.sh
+    script: end_to_end_testing/run_go.sh
+    command_runner_image: quay.io/reactiveops/ci-images:v11-buster
+    enable_docker_layer_caching: false
+    store-test-results: /tmp/test-results
+    kind_version: 0.10.0
+    requires:
+      - test-go
+    filters:
+      branches:
+        only: /.*/
+      tags:
+        ignore: /.*/
+  install_goreleaser: &install_goreleaser
+    run:
+      name: Install Goreleaser
+      command: |
+        curl -fsSLo goreleaser_amd64.deb https://github.com/goreleaser/goreleaser/releases/download/v0.164.0/goreleaser_amd64.deb
+        echo "577c88019cca787161ee1ff853d45be7f635ad7d8a204bd7a2735ab04abcc255  goreleaser_amd64.deb" | sha256sum -c -
+        sudo dpkg -i goreleaser_amd64.deb
+        rm goreleaser_amd64.deb
+  install_go: &install_go
+    run:
+      name: Install Golang
+      command: |
+        curl -LO https://golang.org/dl/go1.16.4.linux-amd64.tar.gz
+        tar -C /usr/local -xzf go1.16.4.linux-amd64.tar.gz
+        echo "PATH=$PATH:/usr/local/go/bin" >> ${BASH_ENV}
 jobs:
   build-3-7:
     executor: python-3-7
@@ -76,9 +106,46 @@ jobs:
             pip install --user -e .
             reckoner --version
             pytest --cov ./
-
+  build-3-9:
+    executor: python-3-9
+    working_directory: ~/reckoner
+    steps:
+      - run:
+          name: Setup PATH to support pip user installs
+          command: echo 'export PATH=$PATH:/home/circleci/.local/bin' >> $BASH_ENV
+      - checkout
+      - run:
+          name: Unit Tests
+          command: |
+            pip install --user -r development-requirements.txt
+            pip install --user -e .
+            reckoner --version
+            pytest --cov ./
+  test-go:
+    working_directory: /go/src/github.com/fairwindsops/reckoner
+    docker:
+      - image: circleci/golang:1.16
+        environment:
+          GO111MODULE: "on"
+    steps:
+      - checkout
+      - run: make test
+  snapshot-go:
+    docker:
+      - image: quay.io/reactiveops/ci-images:v11-buster
+    steps:
+      - checkout
+      - *install_go
+      - *install_goreleaser
+      - setup_remote_docker
+      - run:
+          name: Goreleaser Snapshot
+          command: goreleaser --snapshot
+      - store_artifacts:
+          path: dist
+          destination: snapshot
   release:
-    executor: python-3-8
+    executor: python-3-9
     environment:
       GITHUB_ORGANIZATION: $CIRCLE_PROJECT_USERNAME
       GITHUB_REPOSITORY: $CIRCLE_PROJECT_REPONAME
@@ -122,50 +189,59 @@ jobs:
             cd ./dist
             aws s3 sync ./ s3://reckoner.docs.fairwinds.com --delete
 
+
 workflows:
   version: 2
   build_and_test:
     jobs:
-      - build-3-7:
+      - test-go:
           filters:
             tags:
+              ignore: /.*/
+            branches:
               only: /.*/
+      - build-3-7:
+          filters:
+            tags:
+              ignore: /.*/
             branches:
               only: /.*/
       - build-3-8:
           filters:
             tags:
+              ignore: /.*/
+            branches:
               only: /.*/
+      - build-3-9:
+          filters:
+            tags:
+              ignore: /.*/
             branches:
               only: /.*/
-      - rok8s/kubernetes_e2e_tests:
-          name: "End-To-End Kubernetes 1.18.8"
-          kind_node_image: "kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb"
-          <<: *e2e_configuration
-      - rok8s/kubernetes_e2e_tests:
-          name: "End-To-End Kubernetes 1.19.1"
-          kind_node_image: "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600"
-          <<: *e2e_configuration
-      - rok8s/docker_build_and_push:
-          name: build-and-push-container
-          docker-push: true
-          enable_docker_layer_caching: true
-          config_file: .circleci/build.config
-          context: org-global
-          docker-login: true
-          password-variable: "fairwinds_quay_token"
-          registry: quay.io
-          username: fairwinds+circleci
+      - snapshot-go:
           requires:
-            - build-3-7
-            - build-3-8
-            - "End-To-End Kubernetes 1.18.8"
-            - "End-To-End Kubernetes 1.19.1"
+            - test-go
           filters:
-            branches:
-              ignore: /pull\/[0-9]+/
             tags:
               ignore: /.*/
+            branches:
+              only: /.*/
+      - rok8s/kubernetes_e2e_tests:
+          name: "End-To-End Kubernetes 1.19 - Python"
+          kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca"
+          <<: *e2e_configuration_python
+      - rok8s/kubernetes_e2e_tests:
+          name: "End-To-End Kubernetes 1.20 - Python"
+          kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab"
+          <<: *e2e_configuration_python
+      - rok8s/kubernetes_e2e_tests:
+          name: "End-To-End Kubernetes 1.19 - Go"
+          kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca"
+          <<: *e2e_configuration_go
+      - rok8s/kubernetes_e2e_tests:
+          name: "End-To-End Kubernetes 1.20 - Go"
+          kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab"
+          <<: *e2e_configuration_go
   release:
     jobs:
       - release:
@@ -174,21 +250,6 @@ workflows:
               only: /.*/
             branches:
               ignore: /.*/
-      - rok8s/docker_build_and_push:
-          name: build-and-push-container
-          docker-push: true
-          enable_docker_layer_caching: false
-          config_file: .circleci/build.config
-          context: org-global
-          docker-login: true
-          password-variable: "fairwinds_quay_token"
-          registry: quay.io
-          username: fairwinds+circleci
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: /.*/
       - publish_docs:
           filters:
             branches:

diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,7 @@ reckoner/version.txt
 installer/reckoner.spec
 node_modules
 /dist
+/bin
+
+coverage.txt
+/reckoner-go
diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -0,0 +1,21 @@
+builds:
+  - ldflags:
+      - -X main.version={{.Version}} -X main.commit={{.Commit}} -s -w
+    goarch:
+      - amd64
+      - arm
+      - arm64
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarm:
+      - 6
+      - 7
+dockers:
+- image_templates:
+  - "quay.io/fairwinds/reckoner:go-{{ .Tag }}"
+  - "quay.io/fairwinds/reckoner:go-v{{ .Major }}"
+  - "quay.io/fairwinds/reckoner:go-v{{ .Major }}.{{ .Minor }}"
+  dockerfile: Dockerfile-go
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,48 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+     - id: check-json
+     - id: detect-private-key
+     - id: trailing-whitespace
+       exclude: >
+         (?x)^(
+           docs/.+
+         )$
+     - id: check-added-large-files
+       args: ['--maxkb=500']
+     - id: check-byte-order-marker
+     - id: check-merge-conflict
+     - id: check-symlinks
+     - id: end-of-file-fixer
+       exclude: >
+         (?x)^(
+           docs/.+
+         )$
+     - id: check-executables-have-shebangs
+     - id: flake8
+     - id: no-commit-to-branch
+       args: [--branch, master]
+     - id: pretty-format-json
+       args: ['--autofix']
+  - repo: https://github.com/jumanjihouse/pre-commit-hooks
+    rev: 2.1.5
+    hooks:
+      - id: forbid-binary
+        exclude: >
+          (?x)^(
+            .+\.png|
+            .+\.woff|
+            .+\.woff2|
+            .+\.tff|
+            .+\.ico
+          )$
+      - id: shellcheck
+  - repo: https://github.com/dnephin/pre-commit-golang.git
+    rev: v0.3.5
+    hooks:
+      - id: go-fmt
+      - id: golangci-lint
+      - id: go-vet
+      - id: go-unit-tests
diff --git a/DESIGN.md b/DESIGN.md
@@ -24,7 +24,41 @@ Reckoner is intended to provide a declarative syntax to install and manage multi
 * Overwriting portions of a chart's output (eg Kustomize or Ship)
 * Managing installation versions of Helm and Tiller
 
+## Golang Re-Write and Schema Changes
 
-## Architecture
+We will be rewriting this project in Go in order to provide a nicer UX via a pre-compiled binary. At the same time, the course file schema will be changed in order to facilitate this rewrite.
 
-TBD
+The re-write will occur in [This Pull Request](https://github.com/FairwindsOps/reckoner/pull/293). Any new branches related to the rewrite should come from here in the short term.
+
+### Guiding principles of the re-write
+
+- No new features will be added that are not directly related to the schema change
+- Feature parity should be maintained
+- The old end-to-end test suite must run with the new binary with minimal changes (pre-converting the schema may be necessary)
+- A conversion path from the old schema to the new will be provided
+- Unit Test coverage should be relatively high. I'm thinking >60%
+
+### Un-answered Questions
+
+- Should we automatically detect the old schema and convert it for the end-user?
+- Should we instead just error and point them at the conversion utility?
+- Do we care about ordering? If so, this may get much uglier. In the past we have maintained strict ordering of the releases.
+- Can we import the helm packages instead of relying on exec to the installed helm binary?
+
+### New Schema Changes
+
+The new schema will align better with Go structs, rather than constantly using `map[string]interface`.
+
+[See the new schema here, along with the conversion function](https://github.com/FairwindsOps/reckoner/blob/golang/pkg/course/course.go)
+
+Some major changes to the functionality:
+
+#### Repositories will _only_ be defined in the header
+
+Each release will reference the name of the repository from the header section.
+
+This includes git repositories. This should prevent confusion in the future, as well as allow us to not need a dynamic field in the release struct (or in the repositories section of the header)
+
+#### Charts will be renamed Releases
+
+This makes a lot more sense. It's what they are.
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 FROM python:3.8
 
-ADD . /reckoner
+ADD . /bin/reckoner
 RUN pip install ./reckoner
 
 ENTRYPOINT ["reckoner"]

diff --git a/Dockerfile-go b/Dockerfile-go
@@ -0,0 +1,6 @@
+FROM scratch
+
+USER nobody
+COPY reckoner /
+WORKDIR /
+ENTRYPOINT ["/reckoner"]