Prepare monorepo import other repos

.github/workflows/checks.yml

@@ -0,0 +1,109 @@
+name: Build components images
+on:
+  # TODO: enable push
+  # push:
+  #   - master
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to be released - Just for testing will be removed prior to merging'
+        required: true
+        type: string
+      go-version:
+        description: 'Go version to install'
+        default: '1.19'
+        type: string
+jobs:
+  run-unit-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - flytepropeller
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ matrix.component }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ inputs.go-version }}
+      - name: Run unit tests
+        # TODO: update this to use test_unit_codecov
+        # run: make install && make test_unit_codecov
+        run: make install && make test_unit
+  build-image:
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - flytepropeller
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ matrix.component }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ inputs.go-version }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+          password: "${{ secrets.FLYTE_BOT_PAT }}"
+
+  push_docker_image:
+    name: Build & Push Flytepropeller Image
+    # TODO: enable this
+    # needs: [ bump_version ]
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - flytepropeller
+    uses: eapolinario/flytetools/.github/workflows/publish.yml@monorepo--publish-using-components
+    with:
+      # TODO: get version from bump_version step
+      # version: ${{ needs.bump_version.outputs.version }}
+      version: ${{ inputs.version }}
+      component: ${{ matrix.component }}
+      dockerfile: Dockerfile
+      push: true
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
+  #
+  # push_docker_image:
+  #   name: Build & Push Flytepropeller Image
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       component:
+  #         - flytepropeller
+  #   runs-on: ubuntu-latest
+  #   defaults:
+  #     run:
+  #       working-directory: ${{ matrix.component }}
+  #   steps:
+  #
+  #   uses: flyteorg/flytetools/.github/workflows/publish.yml@master
+  #   with:
+  #     # TODO: get version from bump_version step
+  #     # version: ${{ needs.bump_version.outputs.version }}
+  #     version: ${{ inputs.version }}
+  #     dockerfile: Dockerfile
+  #     push: true
+  #     repository: ${{ inputs.organization }}
+  #   secrets:
+  #     # FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+  #     # TODO: can we use the default github token?
+  #     FLYTE_BOT_PAT: ${{ secrets.GITHUB_TOKEN }}
+  #     # FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
+  #     FLYTE_BOT_USERNAME: ${{ inputs.organization }}

.github/workflows/create_release.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Tag image to release version
         run: |
           for release in latest ${{ needs.bump-version.outputs.version }}; do
-            docker buildx imagetools create --tag "ghcr.io/flyteorg/flyte-binary-release:${release}" "ghcr.io/flyteorg/flyte-binary:sha-${{ github.sha }}"
+            docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/flyte-binary-release:${release}" "ghcr.io/${{ github.repository_owner }}/flyte-binary:sha-${{ github.sha }}"
           done
 
   publish-flyte-component-image:
@@ -93,7 +93,7 @@ jobs:
       - name: Tag Image to release version
         run: |
           for release in latest ${{ needs.bump-version.outputs.version }}; do
-            docker buildx imagetools create --tag "ghcr.io/flyteorg/${{ matrix.component }}-release:${release}" "ghcr.io/flyteorg/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
+            docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}-release:${release}" "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
           done
 
   helm-release:
@@ -105,7 +105,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: "0"
-
       - name: Install Helm
         uses: azure/setup-helm@v3
         with:
@@ -116,10 +115,10 @@ jobs:
         run: |
           git config user.name "${{ github.actor }}"
           git config user.email "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
-      - name: Preprare Flyte Helm Release
+      - name: Prepare Flyte Helm Release
         env:
           VERSION: ${{ needs.bump-version.outputs.version }}
-          REPOSITORY: "https://flyteorg.github.io/flyte"
+          REPOSITORY: "https://${{ github.repository_owner }}.github.io/flyte"
         run: |
           make prepare_artifacts
       - name: Run chart-releaser

.github/workflows/generate_flyte_manifest.yml

@@ -45,6 +45,4 @@ jobs:
           labels: |
             kustomize
             helm
-          team-reviewers: |
-            flyte-maintainers
           draft: false

.github/workflows/helm-charts.yaml

@@ -32,4 +32,4 @@ jobs:
             --app-version=${{ github.sha }} \
             --version=0.0-${{ github.sha }} \
             ${{ matrix.chart }}
-          helm push ${{ matrix.chart }}-*.tgz oci://ghcr.io/flyteorg/helm-charts
+          helm push ${{ matrix.chart }}-*.tgz oci://ghcr.io/${{ github.repository_owner }}/helm-charts

.github/workflows/tests.yml

@@ -17,21 +17,22 @@ jobs:
       - name: Compile
         run: make compile
 
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Fetch the code
-        uses: actions/checkout@v2
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.8"
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          if [ -f doc-requirements.txt ]; then pip install -r doc-requirements.txt; fi
-      - name: Build the documentation
-        run: make docs
+  # TODO(monorepo) reenable docs
+  # docs:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Fetch the code
+  #       uses: actions/checkout@v2
+  #     - name: Set up Python 3.8
+  #       uses: actions/setup-python@v2
+  #       with:
+  #         python-version: "3.8"
+  #     - name: Install dependencies
+  #       run: |
+  #         python -m pip install --upgrade pip
+  #         if [ -f doc-requirements.txt ]; then pip install -r doc-requirements.txt; fi
+  #     - name: Build the documentation
+  #       run: make docs
 
   generate_kustomize:
     runs-on: ubuntu-latest

Dockerfile

@@ -9,6 +9,10 @@ ENV GOARCH "${TARGETARCH}"
 ENV GOOS linux
 
 WORKDIR /flyteorg/build
+
+# TODO(monorepo): copy other components
+COPY flytepropeller flytepropeller
+
 COPY go.mod go.sum ./
 RUN go mod download
 COPY cmd cmd

datacalog/.github/config.yml

@@ -0,0 +1,15 @@
+# Comment to be posted on PRs from first-time contributors in your repository
+newPRWelcomeComment: >
+  Thank you for opening this pull request! 🙌
+
+  These tips will help get your PR across the finish line:
+  - Most of the repos have a PR template; if not, fill it out to the best of your knowledge.
+  - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)).
+
+# Comment to be posted to on pull requests merged by a first time user
+firstPRMergeComment: >
+  Congrats on merging your first pull request! 🎉
+
+# Comment to be posted on first-time issues
+newIssueWelcomeComment: >
+  Thank you for opening your first issue here! 🛠

datacalog/.github/workflows/checks.yml

@@ -0,0 +1,66 @@
+name: Datacatalog Checks
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  lint:
+    name: Lint
+    uses: flyteorg/flytetools/.github/workflows/lint.yml@master
+    with:
+      go-version: "1.19"
+
+  tests:
+    name: Unit Tests
+    uses: flyteorg/flytetools/.github/workflows/tests.yml@master
+    with:
+      go-version: "1.19"
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+
+  docker_build:
+    name: Docker Build Images
+    uses: flyteorg/flytetools/.github/workflows/docker_build.yml@master
+
+  generate:
+    name: Check Go Gennerate
+    uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master
+    with:
+      go-version: "1.19"
+
+  bump_version:
+    name: Bump Version
+    if: ${{ github.event_name != 'pull_request' }}
+    needs: [ docker_build, lint, tests, generate ] # Only to ensure it can successfully build
+    uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+
+  goreleaser:
+    name: Goreleaser
+    needs: [ bump_version ] # Only to ensure it can successfully build
+    uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+    with:
+      go-version: "1.19"
+
+  push_docker_image:
+    name: Build & Push Datacatalog Image
+    needs: [ bump_version ]
+    uses: flyteorg/flytetools/.github/workflows/publish.yml@master
+    with:
+      version: ${{ needs.bump_version.outputs.version }}
+      dockerfile: Dockerfile
+      push: true
+      repository: ${{ github.repository }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
+

datacalog/.github/workflows/upgrade_automation.yml

@@ -0,0 +1,21 @@
+name: Upgrade Automation
+on:
+  workflow_dispatch:
+    inputs:
+      component:
+        description: "Name of Flyte component"
+        required: true
+        default: "boilerplate"
+        type: choice
+        options:
+          - boilerplate
+          - flyteidl
+jobs:
+  trigger-upgrade:
+    name: ${{ github.event.inputs.component }} Upgrade
+    uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master
+    with:
+      component: ${{ github.event.inputs.component }}
+      go-version: "1.19"
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}

datacalog/.gitignore

@@ -0,0 +1,5 @@
+.idea/
+vendor/
+vendor-new/
+.DS_Store
+bin/

datacalog/.golangci.yml

@@ -0,0 +1,30 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+run:
+  skip-dirs:
+    - pkg/client
+
+linters:
+  disable-all: true
+  enable:
+    - deadcode
+    - errcheck
+    - gas
+    - goconst
+    - goimports
+    - golint
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - nakedret
+    - staticcheck
+    - structcheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck

datacalog/.goreleaser.yml

@@ -0,0 +1,35 @@
+project_name: datacatalog
+before:
+  hooks:
+    - go mod download
+builds:
+  - id: datacatalog
+    env:
+      - CGO_ENABLED=0
+    main: ./cmd/main.go
+    binary: datacatalog
+    goos:
+      - linux
+      - windows
+      - darwin
+archives:
+  - id: datacatalog-archive
+    name_template: |-
+      datacatalog_{{ .Tag }}_{{ .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+    builds:
+      - datacatalog
+    format_overrides:
+      - goos: windows
+        format: zip
+
+checksum:
+  name_template: 'checksums.txt'
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'

datacalog/CODEOWNERS

@@ -0,0 +1,3 @@
+# These are the assigned owners of the datacatalog repo.
+# When a PR is opened they will be notified.
+*       @kumare3 @wild-endeavor

datacalog/CODE_OF_CONDUCT.md

@@ -0,0 +1,2 @@
+This project is governed by LF AI Foundation's [code of conduct](https://lfprojects.org/policies/code-of-conduct/). 
+All contributors and participants agree to abide by its terms.