Merge pull request #387 from NEONScience/choim-location-loader

Choim location loader
NEONScience · Oct 2, 2024 · e5bd6ed · e5bd6ed
2 parents 6eec7f0 + 0f038e5
commit e5bd6ed
Show file tree

Hide file tree

Showing 20 changed files with 152 additions and 63 deletions.
diff --git a/.github/workflows/build_push_location_loader.yml b/.github/workflows/build_push_location_loader.yml
@@ -0,0 +1,47 @@
+name: "Build-push_location_loader"
+
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'modules/location_loader/*'
+  workflow_dispatch: {} # Allows trigger of workflow from web interface
+
+env:
+  GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN }}
+  # Use github and google registries
+  GHCR_REGISTRY: ghcr.io
+  GCP_ARTIFACT_HOST: ${{ vars.SHARED_WIF_LOCATON }}-docker.pkg.dev
+  GCP_REGISTRY: ${{ vars.SHARED_WIF_LOCATON }}-docker.pkg.dev/${{ vars.SHARED_WIF_PROJECT }}/${{ vars.SHARED_WIF_REPO }}
+  GCP_PROVIDER:  ${{ vars.SHARED_WIF_PROVIDER }}
+  GCP_SERVICE_ACCOUNT: ${{ vars.SHARED_WIF_SERVICE_ACCOUNT }}
+  GHCR_NS: battelleecology
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # Get just the repo name from the event, i.e., NEON-IS-data-processing
+  REPO_NAME: ${{ github.event.repository.name }}
+  # IS module name
+  MODULE_PATH: ./modules/location_loader
+  IMAGE_NAME: neon-is-loc-loader
+
+jobs:
+  build-push:
+    runs-on: ubuntu-latest
+    permissions:
+        contents: 'write'
+        id-token: 'write'
+    steps:
+      - name: "Checkout"
+        uses: "actions/[email protected]"
+        with:
+          fetch-depth: '0'
+
+      - name: Get short SHA
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: ./.github/actions/build-push
+        with:
+            image-tag: "${short_sha}"
+
diff --git a/.github/workflows/build_push_update_location_loader.yml b/.github/workflows/build_push_update_location_loader.yml
@@ -0,0 +1,51 @@
+name: "Build-push-update_location_loader"
+
+on:
+  push:
+    # branches:
+    #   - 'master'
+    tags:
+      - "location_loader/v*.*.*"
+env:
+  GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN }}
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  # Use github and google registries
+  GHCR_REGISTRY: ghcr.io
+  GCP_ARTIFACT_HOST: ${{ vars.SHARED_WIF_LOCATON }}-docker.pkg.dev
+  GCP_REGISTRY: ${{ vars.SHARED_WIF_LOCATON }}-docker.pkg.dev/${{ vars.SHARED_WIF_PROJECT }}/${{ vars.SHARED_WIF_REPO }}
+  GCP_PROVIDER:  ${{ vars.SHARED_WIF_PROVIDER }}
+  GCP_SERVICE_ACCOUNT: ${{ vars.SHARED_WIF_SERVICE_ACCOUNT }}
+  GHCR_NS: battelleecology
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  # IMAGE_NAME: ${{ github.repository }}
+  # Get just the repo name from the event, i.e., NEON-IS-data-processing
+  REPO_NAME: ${{ github.event.repository.name }}
+  # IS module name
+  MODULE_PATH: ./modules/location_loader
+  IMAGE_NAME: neon-is-loc-loader
+
+jobs:
+  build-push-update:
+    runs-on: ubuntu-latest
+    permissions:
+        contents: 'write'
+        id-token: 'write'
+        pull-requests: 'write'
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/[email protected]"
+        with:
+          ref: 'master'
+          fetch-depth: '0'
+
+      - name: Get short SHA
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Get semver
+        run:  echo "semver=$(git describe --tags --abbrev=0 | awk -F "/" '{print $2}')" >> $GITHUB_ENV 
+
+      - name: Build push and update
+        uses: ./.github/actions/build-push-update
diff --git a/modules/location_loader/Dockerfile b/modules/location_loader/Dockerfile
@@ -7,6 +7,7 @@
 ###
 FROM registry.access.redhat.com/ubi8/ubi-minimal
 
+ARG MODULE_DIR="./modules"
 ARG APP_DIR="location_loader"
 ARG COMMON_DIR="common"
 ARG DATA_ACCESS_DIR="data_access"
@@ -15,8 +16,8 @@ ENV PYTHONPATH="${PYTHONPATH}:${CONTAINER_APP_DIR}"
 
 WORKDIR ${CONTAINER_APP_DIR}
 
-COPY ${APP_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${APP_DIR}/app-requirements.txt
-COPY ${DATA_ACCESS_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${APP_DIR}/data-access-requirements.txt
+COPY ${MODULE_DIR}/${APP_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${APP_DIR}/app-requirements.txt
+COPY ${MODULE_DIR}/${DATA_ACCESS_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${APP_DIR}/data-access-requirements.txt
 
 RUN update-ca-trust && \
     microdnf update -y --disableplugin=subscription-manager && \
@@ -37,8 +38,8 @@ RUN update-ca-trust && \
     groupadd -g 9999 appuser && \
     useradd -r -u 9999 -g appuser appuser
 
-COPY ${APP_DIR} ${CONTAINER_APP_DIR}/${APP_DIR}
-COPY ${COMMON_DIR} ${CONTAINER_APP_DIR}/${COMMON_DIR}
-COPY ${DATA_ACCESS_DIR} ${CONTAINER_APP_DIR}/${DATA_ACCESS_DIR}
+COPY ${MODULE_DIR}/${APP_DIR} ${CONTAINER_APP_DIR}/${APP_DIR}
+COPY ${MODULE_DIR}/${COMMON_DIR} ${CONTAINER_APP_DIR}/${COMMON_DIR}
+COPY ${MODULE_DIR}/${DATA_ACCESS_DIR} ${CONTAINER_APP_DIR}/${DATA_ACCESS_DIR}
 
 USER appuser
diff --git a/modules/location_loader/build_tag_push_update.sh b/modules/location_loader/build_tag_push_update.sh
diff --git a/pipe/aepg600m/aepg600m_location_loader.yaml b/pipe/aepg600m/aepg600m_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: aepg600m_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:c0ce8112
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/csat3/csat3_location_loader.yaml b/pipe/csat3/csat3_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: csat3_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/dualfan/dualfan_location_loader.yaml b/pipe/dualfan/dualfan_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: dualfan_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/hmp155/hmp155_location_loader.yaml b/pipe/hmp155/hmp155_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: hmp155_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/li191r/li191r_location_loader.yaml b/pipe/li191r/li191r_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: li191r_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/li7200/li7200_location_loader.yaml b/pipe/li7200/li7200_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: li7200_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/mcseries/mcseries_location_loader.yaml b/pipe/mcseries/mcseries_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: mcseries_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/metone370380/metone370380_location_loader.yaml b/pipe/metone370380/metone370380_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: metone370380_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:c0ce8112
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/mti300ahrs/mti300ahrs_location_loader.yaml b/pipe/mti300ahrs/mti300ahrs_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: mti300ahrs_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/pqs1/pqs1_location_loader.yaml b/pipe/pqs1/pqs1_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: pqs1_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/pressuretransducer/pressuretransducer_location_loader.yaml b/pipe/pressuretransducer/pressuretransducer_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: pressuretransducer_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/prt/prt_location_loader.yaml b/pipe/prt/prt_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: prt_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/ptb330a/ptb330a_location_loader.yaml b/pipe/ptb330a/ptb330a_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: ptb330a_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/tchain/location_loader_tchain.yaml b/pipe/tchain/location_loader_tchain.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: location_loader_tchain
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/troll/troll_location_loader.yaml b/pipe/troll/troll_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: troll_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin:

diff --git a/pipe/windobserverii/windobserverii_location_loader.yaml b/pipe/windobserverii/windobserverii_location_loader.yaml
@@ -2,9 +2,9 @@
 pipeline:
   name: windobserverii_location_loader
 transform:
-  image_pull_secrets:
-  - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/location_loader:ee44d9db
+  # image_pull_secrets:
+  # - battelleecology-quay-read-all-pull-secret
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-loader:sha-46b4e16
   cmd:
   - /bin/bash
   stdin: