don't copy errored datums to pub bucket

NEONScience · Nov 28, 2023 · 97a636c · 97a636c
1 parent 02414fb
commit 97a636c
Show file tree

Hide file tree

Showing 7 changed files with 111 additions and 63 deletions.
diff --git a/pipe/groundwaterPhysical/groundwaterPhysical_pub_egress_and_publish.yaml b/pipe/groundwaterPhysical/groundwaterPhysical_pub_egress_and_publish.yaml
@@ -17,17 +17,23 @@ transform:
       python3 -m pub_egress.pub_egress_main
     fi
     # If there is output, egress it
-    dirs=$(find $OUT_PATH/* -type d)
+    dirs=$(find $OUT_PATH/NEON.DOM.SITE* -maxdepth 0 -type d)
     if [[ ${dirs} ]]; then
       echo "Syncing files to bucket $BUCKET_NAME"
-      rclone \
-        --no-check-dest \
-        --copy-links \
-        --gcs-bucket-policy-only \
-        --gcs-no-check-bucket \
-        copy \
-        "${OUT_PATH}" \
-        ":gcs://${BUCKET_NAME}"
+      for DIR in $dirs; do
+        echo "Syncing $DIR to bucket $BUCKET_NAME"
+        # Parse the product
+        [[ "$DIR" =~ ^$OUT_PATH/(.*)$ ]]
+        PRODUCT="${BASH_REMATCH[1]}"
+        rclone \
+          --no-check-dest \
+          --copy-links \
+          --gcs-bucket-policy-only \
+          --gcs-no-check-bucket \
+          copy \
+          "${OUT_PATH}/${PRODUCT}" \
+          ":gcs://${BUCKET_NAME}/${PRODUCT}"
+      done
       echo "Done"
     fi
     # Set some environment variables for the second module

diff --git a/pipe/parQuantumLine/parQuantumLine_pub_egress_and_publish.yaml b/pipe/parQuantumLine/parQuantumLine_pub_egress_and_publish.yaml
@@ -17,17 +17,23 @@ transform:
       python3 -m pub_egress.pub_egress_main
     fi
     # If there is output, egress it
-    dirs=$(find $OUT_PATH/* -type d)
+    dirs=$(find $OUT_PATH/NEON.DOM.SITE* -maxdepth 0 -type d)
     if [[ ${dirs} ]]; then
       echo "Syncing files to bucket $BUCKET_NAME"
-      rclone \
-        --no-check-dest \
-        --copy-links \
-        --gcs-bucket-policy-only \
-        --gcs-no-check-bucket \
-        copy \
-        "${OUT_PATH}" \
-        ":gcs://${BUCKET_NAME}"
+      for DIR in $dirs; do
+        echo "Syncing $DIR to bucket $BUCKET_NAME"
+        # Parse the product
+        [[ "$DIR" =~ ^$OUT_PATH/(.*)$ ]]
+        PRODUCT="${BASH_REMATCH[1]}"
+        rclone \
+          --no-check-dest \
+          --copy-links \
+          --gcs-bucket-policy-only \
+          --gcs-no-check-bucket \
+          copy \
+          "${OUT_PATH}/${PRODUCT}" \
+          ":gcs://${BUCKET_NAME}/${PRODUCT}"
+      done
       echo "Done"
     fi
     # Set some environment variables for the second module

diff --git a/pipe/parWaterSurface/parWaterSurface_pub_egress_and_publish.yaml b/pipe/parWaterSurface/parWaterSurface_pub_egress_and_publish.yaml
@@ -9,22 +9,31 @@ transform:
   - "-c"
   - |-
     /bin/bash <<'EOF'
+    # Use bash-scrict mode. See http://redsymbol.net/articles/unofficial-bash-strict-mode/
+    set -euo pipefail
+    IFS=$'\n\t'
     # Run first module - pub_egress (using environment variables below as input parameters)
     if [[ $(echo $DATA_PATH) ]]; then
       python3 -m pub_egress.pub_egress_main
     fi
     # If there is output, egress it
-    dirs=$(find $OUT_PATH/* -type d)
+    dirs=$(find $OUT_PATH/NEON.DOM.SITE* -maxdepth 0 -type d)
     if [[ ${dirs} ]]; then
       echo "Syncing files to bucket $BUCKET_NAME"
-      rclone \
-        --no-check-dest \
-        --copy-links \
-        --gcs-bucket-policy-only \
-        --gcs-no-check-bucket \
-        copy \
-        "${OUT_PATH}" \
-        ":gcs://${BUCKET_NAME}"
+      for DIR in $dirs; do
+        echo "Syncing $DIR to bucket $BUCKET_NAME"
+        # Parse the product
+        [[ "$DIR" =~ ^$OUT_PATH/(.*)$ ]]
+        PRODUCT="${BASH_REMATCH[1]}"
+        rclone \
+          --no-check-dest \
+          --copy-links \
+          --gcs-bucket-policy-only \
+          --gcs-no-check-bucket \
+          copy \
+          "${OUT_PATH}/${PRODUCT}" \
+          ":gcs://${BUCKET_NAME}/${PRODUCT}"
+      done
       echo "Done"
     fi
     # Set some environment variables for the second module

diff --git a/pipe/relHumidity/relHumidity_pub_egress_and_publish.yaml b/pipe/relHumidity/relHumidity_pub_egress_and_publish.yaml
@@ -17,17 +17,23 @@ transform:
       python3 -m pub_egress.pub_egress_main
     fi
     # If there is output, egress it
-    dirs=$(find $OUT_PATH/* -type d)
+    dirs=$(find $OUT_PATH/NEON.DOM.SITE* -maxdepth 0 -type d)
     if [[ ${dirs} ]]; then
       echo "Syncing files to bucket $BUCKET_NAME"
-      rclone \
-        --no-check-dest \
-        --copy-links \
-        --gcs-bucket-policy-only \
-        --gcs-no-check-bucket \
-        copy \
-        "${OUT_PATH}" \
-        ":gcs://${BUCKET_NAME}"
+      for DIR in $dirs; do
+        echo "Syncing $DIR to bucket $BUCKET_NAME"
+        # Parse the product
+        [[ "$DIR" =~ ^$OUT_PATH/(.*)$ ]]
+        PRODUCT="${BASH_REMATCH[1]}"
+        rclone \
+          --no-check-dest \
+          --copy-links \
+          --gcs-bucket-policy-only \
+          --gcs-no-check-bucket \
+          copy \
+          "${OUT_PATH}/${PRODUCT}" \
+          ":gcs://${BUCKET_NAME}/${PRODUCT}"
+      done
       echo "Done"
     fi
     # Set some environment variables for the second module

diff --git a/pipe/surfacewaterPhysical/surfacewaterPhysical_pub_egress_and_publish.yaml b/pipe/surfacewaterPhysical/surfacewaterPhysical_pub_egress_and_publish.yaml
@@ -17,17 +17,23 @@ transform:
       python3 -m pub_egress.pub_egress_main
     fi
     # If there is output, egress it
-    dirs=$(find $OUT_PATH/* -type d)
+    dirs=$(find $OUT_PATH/NEON.DOM.SITE* -maxdepth 0 -type d)
     if [[ ${dirs} ]]; then
       echo "Syncing files to bucket $BUCKET_NAME"
-      rclone \
-        --no-check-dest \
-        --copy-links \
-        --gcs-bucket-policy-only \
-        --gcs-no-check-bucket \
-        copy \
-        "${OUT_PATH}" \
-        ":gcs://${BUCKET_NAME}"
+      for DIR in $dirs; do
+        echo "Syncing $DIR to bucket $BUCKET_NAME"
+        # Parse the product
+        [[ "$DIR" =~ ^$OUT_PATH/(.*)$ ]]
+        PRODUCT="${BASH_REMATCH[1]}"
+        rclone \
+          --no-check-dest \
+          --copy-links \
+          --gcs-bucket-policy-only \
+          --gcs-no-check-bucket \
+          copy \
+          "${OUT_PATH}/${PRODUCT}" \
+          ":gcs://${BUCKET_NAME}/${PRODUCT}"
+      done
       echo "Done"
     fi
     # Set some environment variables for the second module

diff --git a/pipe/tempSoil/tempSoil_pub_egress_and_publish.yaml b/pipe/tempSoil/tempSoil_pub_egress_and_publish.yaml
@@ -17,17 +17,23 @@ transform:
       python3 -m pub_egress.pub_egress_main
     fi
     # If there is output, egress it
-    dirs=$(find $OUT_PATH/* -type d)
+    dirs=$(find $OUT_PATH/NEON.DOM.SITE* -maxdepth 0 -type d)
     if [[ ${dirs} ]]; then
       echo "Syncing files to bucket $BUCKET_NAME"
-      rclone \
-        --no-check-dest \
-        --copy-links \
-        --gcs-bucket-policy-only \
-        --gcs-no-check-bucket \
-        copy \
-        "${OUT_PATH}" \
-        ":gcs://${BUCKET_NAME}"
+      for DIR in $dirs; do
+        echo "Syncing $DIR to bucket $BUCKET_NAME"
+        # Parse the product
+        [[ "$DIR" =~ ^$OUT_PATH/(.*)$ ]]
+        PRODUCT="${BASH_REMATCH[1]}"
+        rclone \
+          --no-check-dest \
+          --copy-links \
+          --gcs-bucket-policy-only \
+          --gcs-no-check-bucket \
+          copy \
+          "${OUT_PATH}/${PRODUCT}" \
+          ":gcs://${BUCKET_NAME}/${PRODUCT}"
+      done
       echo "Done"
     fi
     # Set some environment variables for the second module

diff --git a/pipe/testprod/testprod_pub_egress_and_publish.yaml b/pipe/testprod/testprod_pub_egress_and_publish.yaml
@@ -9,22 +9,31 @@ transform:
   - "-c"
   - |-
     /bin/bash <<'EOF'
+    # Use bash-scrict mode. See http://redsymbol.net/articles/unofficial-bash-strict-mode/
+    set -euo pipefail
+    IFS=$'\n\t'
     # Run first module - pub_egress (using environment variables below as input parameters)
     if [[ $(echo $DATA_PATH) ]]; then
       python3 -m pub_egress.pub_egress_main
     fi
     # If there is output, egress it
-    dirs=$(find $OUT_PATH/* -type d)
+    dirs=$(find $OUT_PATH/NEON.DOM.SITE* -maxdepth 0 -type d)
     if [[ ${dirs} ]]; then
       echo "Syncing files to bucket $BUCKET_NAME"
-      rclone \
-        --no-check-dest \
-        --copy-links \
-        --gcs-bucket-policy-only \
-        --gcs-no-check-bucket \
-        copy \
-        "${OUT_PATH}" \
-        ":gcs://${BUCKET_NAME}"
+      for DIR in $dirs; do
+        echo "Syncing $DIR to bucket $BUCKET_NAME"
+        # Parse the product
+        [[ "$DIR" =~ ^$OUT_PATH/(.*)$ ]]
+        PRODUCT="${BASH_REMATCH[1]}"
+        rclone \
+          --no-check-dest \
+          --copy-links \
+          --gcs-bucket-policy-only \
+          --gcs-no-check-bucket \
+          copy \
+          "${OUT_PATH}/${PRODUCT}" \
+          ":gcs://${BUCKET_NAME}/${PRODUCT}"
+      done
       echo "Done"
     fi
     # Set some environment variables for the second module