From 8796a228727492c629fb78a0b331600076608fbd Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 11 Sep 2024 12:42:54 -0500
Subject: [PATCH 01/23] add opentelemery span reporter wrapper

---
 tools/rapids-conda-retry           |  2 +-
 tools/rapids-get-pr-conda-artifact |  7 +++++++
 tools/rapids-mamba-retry           |  2 +-
 tools/rapids-otel-wrap             | 20 ++++++++++++++++++++
 tools/rapids-upload-conda-to-s3    |  3 ++-
 tools/rapids-upload-wheels-to-s3   |  3 ++-
 6 files changed, 33 insertions(+), 4 deletions(-)
 create mode 100644 tools/rapids-otel-wrap

diff --git a/tools/rapids-conda-retry b/tools/rapids-conda-retry
index 9ce59ca..78e25b2 100755
--- a/tools/rapids-conda-retry
+++ b/tools/rapids-conda-retry
@@ -67,7 +67,7 @@ condaCmd=${RAPIDS_CONDA_EXE:=conda}
 #    needToRetry: 1 if the command should be retried, 0 if it should not be
 function runConda {
     # shellcheck disable=SC2086
-    ${condaCmd} ${args} 2>&1| tee "${outfile}"
+    rapids-otel-wrap ${condaCmd} ${args} 2>&1| tee "${outfile}"
     exitcode=$?
     needToRetry=0
     needToClean=0
diff --git a/tools/rapids-get-pr-conda-artifact b/tools/rapids-get-pr-conda-artifact
index 3c3605d..ce2367f 100755
--- a/tools/rapids-get-pr-conda-artifact
+++ b/tools/rapids-get-pr-conda-artifact
@@ -33,4 +33,11 @@ if [[ -z "${commit}" ]]; then
     commit=$(git ls-remote https://github.com/rapidsai/"${repo}".git refs/heads/pull-request/"${pr}" | cut -c1-7)
 fi
 
+if rapids-check-otel-cli; then
+
+fi
+rapids-otel-start-span "$(basename "$0")"
+
 rapids-get-artifact "ci/${repo}/pull-request/${pr}/${commit}/${artifact_name}"
+
+rapids-otel-end-span "$(basename "$0")"
\ No newline at end of file
diff --git a/tools/rapids-mamba-retry b/tools/rapids-mamba-retry
index 5db5d00..e9f2644 100755
--- a/tools/rapids-mamba-retry
+++ b/tools/rapids-mamba-retry
@@ -46,4 +46,4 @@ for arg in "$@"; do
    fi
 done
 
-rapids-conda-retry "$@"
+rapids-otel-wrap rapids-conda-retry "$@"
diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
new file mode 100644
index 0000000..ac94940
--- /dev/null
+++ b/tools/rapids-otel-wrap
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Wraps arbitrary commands with arbitrary args. Emits an OpenTelemetry span for tracing the command
+#
+# To add metadata tags (attrs), set (or append to) the RAPIDS_OTEL_ATTRS str.
+# See https://github.com/equinix-labs/otel-cli?tab=readme-ov-file#header-and-attribute-formatting for details
+#
+#
+type otel-cli >/dev/null 2>&1 && [ -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" ] && [ -n "${TRACEPARENT}" ] {
+    start=$(date --rfc-3339=ns)
+}
+# Run the script
+RETURN_STATUS=$("${@:2}")
+
+# Trim leading commas from RAPIDS_OTEL_ATTRS from clumsy concatentation
+RAPIDS_OTEL_ATTRS=$(echo "${RAPIDS_OTEL_ATTRS}" | sed 's/,$//')
+
+type otel-cli >/dev/null 2>&1 && {
+    otel-cli span -n "$1" -s some-interesting-program --start $start --end $(date --rfc-3339=ns) --attr "${RAPIDS_OTEL_ATTRS}" --status-code ${RETURN_STATUS}
+}
+exit ${RETURN_STATUS}
\ No newline at end of file
diff --git a/tools/rapids-upload-conda-to-s3 b/tools/rapids-upload-conda-to-s3
index 91a049f..17b478e 100755
--- a/tools/rapids-upload-conda-to-s3
+++ b/tools/rapids-upload-conda-to-s3
@@ -30,4 +30,5 @@ pkg_name="$(rapids-package-name "$pkg_type")"
 # Where conda build artifacts are output
 path_to_tar_up="${RAPIDS_CONDA_BLD_OUTPUT_DIR}"
 
-rapids-upload-to-s3 "${pkg_name}" "${path_to_tar_up}"
+RAPIDS_OTEL_ATTRS="${RAPIDS_OTEL_ATTRS},rapids_task=upload,rapids_package_name=${pkg_name}" \
+    rapids-otel-wrap rapids-upload-to-s3 "${pkg_name}" "${path_to_tar_up}"
diff --git a/tools/rapids-upload-wheels-to-s3 b/tools/rapids-upload-wheels-to-s3
index 9b35b39..1b096ac 100755
--- a/tools/rapids-upload-wheels-to-s3
+++ b/tools/rapids-upload-wheels-to-s3
@@ -20,4 +20,5 @@ if [ "${CI:-false}" = "false" ]; then
   exit 0
 fi
 
-rapids-upload-to-s3 "${pkg_name}" "$@"
+RAPIDS_OTEL_ATTRS="${RAPIDS_OTEL_ATTRS},rapids_task=upload,rapids_package_name=${pkg_name}" \
+    rapids-otel-wrap rapids-upload-to-s3 "${pkg_name}" "$@"

From b0086f4ad2875d8d1e13c5c1895e2d0fb70427f6 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 11 Sep 2024 13:47:37 -0500
Subject: [PATCH 02/23] set program name to be RAPIDS_JOB_ID variable (to be
 set in gha job)

---
 tools/rapids-otel-wrap | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index ac94940..c4e5621 100644
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -15,6 +15,6 @@ RETURN_STATUS=$("${@:2}")
 RAPIDS_OTEL_ATTRS=$(echo "${RAPIDS_OTEL_ATTRS}" | sed 's/,$//')
 
 type otel-cli >/dev/null 2>&1 && {
-    otel-cli span -n "$1" -s some-interesting-program --start $start --end $(date --rfc-3339=ns) --attr "${RAPIDS_OTEL_ATTRS}" --status-code ${RETURN_STATUS}
+    otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" --start $start --end $(date --rfc-3339=ns) --attr "${RAPIDS_OTEL_ATTRS}" --status-code ${RETURN_STATUS}
 }
 exit ${RETURN_STATUS}
\ No newline at end of file

From 53b8f69b333096de9d89aaf61836a47c7965276b Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Tue, 17 Sep 2024 09:17:05 -0500
Subject: [PATCH 03/23] add debug echo for whether instrumentation is run

---
 tools/rapids-get-pr-conda-artifact |  9 +--------
 tools/rapids-otel-wrap             | 16 +++++++++++++---
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/tools/rapids-get-pr-conda-artifact b/tools/rapids-get-pr-conda-artifact
index ce2367f..2e9885b 100755
--- a/tools/rapids-get-pr-conda-artifact
+++ b/tools/rapids-get-pr-conda-artifact
@@ -33,11 +33,4 @@ if [[ -z "${commit}" ]]; then
     commit=$(git ls-remote https://github.com/rapidsai/"${repo}".git refs/heads/pull-request/"${pr}" | cut -c1-7)
 fi
 
-if rapids-check-otel-cli; then
-
-fi
-rapids-otel-start-span "$(basename "$0")"
-
-rapids-get-artifact "ci/${repo}/pull-request/${pr}/${commit}/${artifact_name}"
-
-rapids-otel-end-span "$(basename "$0")"
\ No newline at end of file
+rapids-otel-wrap rapids-get-artifact "ci/${repo}/pull-request/${pr}/${commit}/${artifact_name}"
diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index c4e5621..d0d5ddd 100644
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -5,11 +5,21 @@
 # See https://github.com/equinix-labs/otel-cli?tab=readme-ov-file#header-and-attribute-formatting for details
 #
 #
-type otel-cli >/dev/null 2>&1 && [ -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" ] && [ -n "${TRACEPARENT}" ] {
+type otel-cli >/dev/null 2>&1 && ([ -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" ] && [ -n "${TRACEPARENT}" ] {
     start=$(date --rfc-3339=ns)
+    echo "Running command with OpenTelemetry instrumentation"
+    # Run the script
+    RETURN_STATUS=$(opentelemetry-instrument \
+        --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \
+        --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER:-"console"} \
+        --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER:-"console"} \
+        "${@:2}")
+}) || {
+    # Run the script (no telemetry auto-instrumentation)
+    echo "Skipping instrumentation"
+    RETURN_STATUS=$("${@:2}")
 }
-# Run the script
-RETURN_STATUS=$("${@:2}")
+
 
 # Trim leading commas from RAPIDS_OTEL_ATTRS from clumsy concatentation
 RAPIDS_OTEL_ATTRS=$(echo "${RAPIDS_OTEL_ATTRS}" | sed 's/,$//')

From 291d850506fb98eee55863d498e73c460f2bb4db Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Tue, 17 Sep 2024 09:29:02 -0500
Subject: [PATCH 04/23] chmod +x otel wrap

---
 tools/rapids-otel-wrap | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tools/rapids-otel-wrap

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
old mode 100644
new mode 100755

From d86e9d792986461612d932cf5c12ba456d57f7d3 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 18 Sep 2024 10:34:17 -0500
Subject: [PATCH 05/23] pass output from subshells

---
 tools/rapids-otel-wrap | 54 +++++++++++++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 14 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index d0d5ddd..f077416 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -5,26 +5,52 @@
 # See https://github.com/equinix-labs/otel-cli?tab=readme-ov-file#header-and-attribute-formatting for details
 #
 #
-type otel-cli >/dev/null 2>&1 && ([ -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" ] && [ -n "${TRACEPARENT}" ] {
-    start=$(date --rfc-3339=ns)
-    echo "Running command with OpenTelemetry instrumentation"
-    # Run the script
-    RETURN_STATUS=$(opentelemetry-instrument \
-        --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \
-        --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER:-"console"} \
-        --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER:-"console"} \
-        "${@:2}")
-}) || {
-    # Run the script (no telemetry auto-instrumentation)
-    echo "Skipping instrumentation"
-    RETURN_STATUS=$("${@:2}")
+TIME_FORMAT="--rfc-3339=ns"
+
+# this cleanly sets up a tempdir for the fifo that we use to pass data out from subshell into our currrent one
+# https://unix.stackexchange.com/a/29918/34459
+tmpdir=
+cleanup () {
+  trap - EXIT
+  if [ -n "$tmpdir" ] ; then rm -rf "$tmpdir"; fi
+  if [ -n "$1" ]; then trap - $1; kill -$1 $$; fi
 }
+tmpdir=$(mktemp -d)
+trap 'cleanup' EXIT
+trap 'cleanup HUP' HUP
+trap 'cleanup TERM' TERM
+trap 'cleanup INT' INT
+mkfifo "$tmpdir/pipe"
+mkfifo "$tmpdir/status_pipe"
+
+
+if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 && -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" && -n "${TRACEPARENT}" ]]; then
+    echo "Running command with OpenTelemetry instrumentation";
+    (
+        start=$(date ${TIME_FORMAT});
+        opentelemetry-instrument \
+            --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \
+            --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER:-"console"} \
+            --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER:-"console"} \
+            bash -c "$@" 2>&1 > $tmpdir/pipe;
+        echo $? > $tmpdir/status_pipe;
+    ) &
+else
+    echo "Skipping instrumentation, running \"${@}\"";
+    (
+        eval "$@" 2>&1 > $tmpdir/pipe;
+        echo $? > $tmpdir/status_pipe;
+    ) &
+fi
 
+echo "stdout/stderr from command:"
+echo $(<$tmpdir/pipe)
+RETURN_STATUS=$(<$tmpdir/status_pipe)
 
 # Trim leading commas from RAPIDS_OTEL_ATTRS from clumsy concatentation
 RAPIDS_OTEL_ATTRS=$(echo "${RAPIDS_OTEL_ATTRS}" | sed 's/,$//')
 
 type otel-cli >/dev/null 2>&1 && {
-    otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" --start $start --end $(date --rfc-3339=ns) --attr "${RAPIDS_OTEL_ATTRS}" --status-code ${RETURN_STATUS}
+    otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" --start $start --end $(date ${TIME_FORMAT}) --attr "${RAPIDS_OTEL_ATTRS}" --status-code ${RETURN_STATUS}
 }
 exit ${RETURN_STATUS}
\ No newline at end of file

From 003bf1d1cdaae0d78db08e51fd3bd6d6c9c39721 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 18 Sep 2024 10:53:16 -0500
Subject: [PATCH 06/23] add output with while read; fix cmd running

---
 tools/rapids-otel-wrap | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index f077416..3fd125d 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -5,6 +5,7 @@
 # See https://github.com/equinix-labs/otel-cli?tab=readme-ov-file#header-and-attribute-formatting for details
 #
 #
+
 TIME_FORMAT="--rfc-3339=ns"
 
 # this cleanly sets up a tempdir for the fifo that we use to pass data out from subshell into our currrent one
@@ -32,25 +33,32 @@ if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 && -n "${OTEL_EXPORTER_OTLP_ENDPOIN
             --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \
             --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER:-"console"} \
             --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER:-"console"} \
-            bash -c "$@" 2>&1 > $tmpdir/pipe;
+            "$@" 2>&1 > $tmpdir/pipe;
         echo $? > $tmpdir/status_pipe;
     ) &
 else
     echo "Skipping instrumentation, running \"${@}\"";
     (
-        eval "$@" 2>&1 > $tmpdir/pipe;
+        "$@" 2>&1 > $tmpdir/pipe;
         echo $? > $tmpdir/status_pipe;
     ) &
 fi
 
+set +x
 echo "stdout/stderr from command:"
-echo $(<$tmpdir/pipe)
+while IFS= read -r line; do
+    echo "$line"
+done < $tmpdir/pipe
 RETURN_STATUS=$(<$tmpdir/status_pipe)
+set -x
 
 # Trim leading commas from RAPIDS_OTEL_ATTRS from clumsy concatentation
 RAPIDS_OTEL_ATTRS=$(echo "${RAPIDS_OTEL_ATTRS}" | sed 's/,$//')
 
 type otel-cli >/dev/null 2>&1 && {
-    otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" --start $start --end $(date ${TIME_FORMAT}) --attr "${RAPIDS_OTEL_ATTRS}" --status-code ${RETURN_STATUS}
+    otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" \
+        --start $start --end $(date ${TIME_FORMAT}) \
+        --attrs "${RAPIDS_OTEL_ATTRS}" \
+        --status-code ${RETURN_STATUS}
 }
 exit ${RETURN_STATUS}
\ No newline at end of file

From 57a1c76024536baef184392a9c6ba691d5e8c927 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 18 Sep 2024 13:38:10 -0500
Subject: [PATCH 07/23] don't add attrs unless they are non-null

---
 tools/rapids-otel-wrap | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 3fd125d..30b4f00 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -56,9 +56,12 @@ set -x
 RAPIDS_OTEL_ATTRS=$(echo "${RAPIDS_OTEL_ATTRS}" | sed 's/,$//')
 
 type otel-cli >/dev/null 2>&1 && {
+    if [ -n "${RAPIDS_OTEL_ATTRS}" ]; then  \
+        attrs="--attrs ${RAPIDS_OTEL_ATTRS}"; \
+    fi
     otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" \
         --start $start --end $(date ${TIME_FORMAT}) \
-        --attrs "${RAPIDS_OTEL_ATTRS}" \
+        "${attrs}" \
         --status-code ${RETURN_STATUS}
 }
 exit ${RETURN_STATUS}
\ No newline at end of file

From 2d01c7d6f6021624e90a43eb5288b3ef0b69c41a Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 18 Sep 2024 14:07:02 -0500
Subject: [PATCH 08/23] debug instrumentation command

---
 tools/rapids-otel-wrap | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 30b4f00..ab4843a 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -28,13 +28,17 @@ mkfifo "$tmpdir/status_pipe"
 if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 && -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" && -n "${TRACEPARENT}" ]]; then
     echo "Running command with OpenTelemetry instrumentation";
     (
+        set -x
         start=$(date ${TIME_FORMAT});
         opentelemetry-instrument \
+            --service-name ${RAPIDS_OTEL_SERVICE_NAME:-"rapids-build-test"} \
             --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \
             --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER:-"console"} \
             --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER:-"console"} \
+            --exporter_otlp_protocol http/protobuf \
             "$@" 2>&1 > $tmpdir/pipe;
         echo $? > $tmpdir/status_pipe;
+        set +x
     ) &
 else
     echo "Skipping instrumentation, running \"${@}\"";

From 5469dcd350d3c5ad0395e7d997c645c713cd1b40 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 18 Sep 2024 14:09:38 -0500
Subject: [PATCH 09/23] wrong flag format

---
 tools/rapids-otel-wrap | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index ab4843a..a7d653c 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -31,7 +31,7 @@ if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 && -n "${OTEL_EXPORTER_OTLP_ENDPOIN
         set -x
         start=$(date ${TIME_FORMAT});
         opentelemetry-instrument \
-            --service-name ${RAPIDS_OTEL_SERVICE_NAME:-"rapids-build-test"} \
+            --service_name ${RAPIDS_OTEL_SERVICE_NAME:-"rapids-build-test"} \
             --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \
             --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER:-"console"} \
             --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER:-"console"} \

From 57e0c38930b3e7534ce6629446f9eae12264e4dc Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 18 Sep 2024 14:17:04 -0500
Subject: [PATCH 10/23] move start time higher so that it isn't out of scope

---
 tools/rapids-otel-wrap | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index a7d653c..f52072f 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -24,12 +24,12 @@ trap 'cleanup INT' INT
 mkfifo "$tmpdir/pipe"
 mkfifo "$tmpdir/status_pipe"
 
+start=$(date ${TIME_FORMAT});
 
 if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 && -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" && -n "${TRACEPARENT}" ]]; then
     echo "Running command with OpenTelemetry instrumentation";
     (
         set -x
-        start=$(date ${TIME_FORMAT});
         opentelemetry-instrument \
             --service_name ${RAPIDS_OTEL_SERVICE_NAME:-"rapids-build-test"} \
             --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \

From d4eddf8efeadf00624630accaa2ae30e774d5947 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 18 Sep 2024 14:52:16 -0500
Subject: [PATCH 11/23] quote start and end span times

---
 tools/rapids-otel-wrap | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index f52072f..6360f56 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -64,7 +64,7 @@ type otel-cli >/dev/null 2>&1 && {
         attrs="--attrs ${RAPIDS_OTEL_ATTRS}"; \
     fi
     otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" \
-        --start $start --end $(date ${TIME_FORMAT}) \
+        --start "$start" --end "$(date ${TIME_FORMAT})" \
         "${attrs}" \
         --status-code ${RETURN_STATUS}
 }

From 7e23539ef46370d40c069cfca45e5b3bed385bae Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 18 Sep 2024 14:59:20 -0500
Subject: [PATCH 12/23] specify endpoints in instrumentation CLI

---
 tools/rapids-otel-wrap | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 6360f56..7776d27 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -26,7 +26,7 @@ mkfifo "$tmpdir/status_pipe"
 
 start=$(date ${TIME_FORMAT});
 
-if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 && -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" && -n "${TRACEPARENT}" ]]; then
+if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 ]]; then
     echo "Running command with OpenTelemetry instrumentation";
     (
         set -x
@@ -35,6 +35,9 @@ if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 && -n "${OTEL_EXPORTER_OTLP_ENDPOIN
             --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \
             --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER:-"console"} \
             --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER:-"console"} \
+            --exporter_otlp_traces_endpoint ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} \
+            --exporter_otlp_metrics_endpoint ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT} \
+            --exporter_otlp_logs_endpoint ${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT} \
             --exporter_otlp_protocol http/protobuf \
             "$@" 2>&1 > $tmpdir/pipe;
         echo $? > $tmpdir/status_pipe;

From 2ded7f94d591d4a631abd4812324af38c474a25f Mon Sep 17 00:00:00 2001
From: Mike Sarahan <msarahan@gmail.com>
Date: Thu, 19 Sep 2024 13:48:23 -0500
Subject: [PATCH 13/23] unquote attrs to avoid empty quotes as positional arg

---
 tools/rapids-otel-wrap | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 7776d27..19ac94a 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -68,7 +68,7 @@ type otel-cli >/dev/null 2>&1 && {
     fi
     otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" \
         --start "$start" --end "$(date ${TIME_FORMAT})" \
-        "${attrs}" \
-        --status-code ${RETURN_STATUS}
+        --status-code ${RETURN_STATUS} \
+        ${attrs}
 }
-exit ${RETURN_STATUS}
\ No newline at end of file
+exit ${RETURN_STATUS}

From eb0071d5d16a2dbe067c04ffb23890fcf42272a3 Mon Sep 17 00:00:00 2001
From: Mike Sarahan <msarahan@gmail.com>
Date: Thu, 19 Sep 2024 14:06:24 -0500
Subject: [PATCH 14/23] add otlp endpoint and headers for otel-cli calls

---
 tools/rapids-otel-wrap | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 19ac94a..53f1fd4 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -66,7 +66,8 @@ type otel-cli >/dev/null 2>&1 && {
     if [ -n "${RAPIDS_OTEL_ATTRS}" ]; then  \
         attrs="--attrs ${RAPIDS_OTEL_ATTRS}"; \
     fi
-    otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" \
+    otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" --endpoint ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} \
+        --otlp-headers "${OTEL_EXPORTER_OTLP_HEADERS}" \
         --start "$start" --end "$(date ${TIME_FORMAT})" \
         --status-code ${RETURN_STATUS} \
         ${attrs}

From 214d3a327c3d08c53d0a4a8540d68aa36dad5a80 Mon Sep 17 00:00:00 2001
From: Mike Sarahan <msarahan@gmail.com>
Date: Thu, 19 Sep 2024 14:07:59 -0500
Subject: [PATCH 15/23] add protocol to otel-cli call

---
 tools/rapids-otel-wrap | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 53f1fd4..903d774 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -69,6 +69,7 @@ type otel-cli >/dev/null 2>&1 && {
     otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" --endpoint ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} \
         --otlp-headers "${OTEL_EXPORTER_OTLP_HEADERS}" \
         --start "$start" --end "$(date ${TIME_FORMAT})" \
+        --protocol http/protobuf \
         --status-code ${RETURN_STATUS} \
         ${attrs}
 }

From 696906d061b2afe39a323c73ca0da89fc2433974 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 25 Sep 2024 13:37:03 -0500
Subject: [PATCH 16/23] consistently use RAPIDS_SERVICE_NAME for service_name
 across tools

---
 tools/rapids-otel-wrap | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 903d774..f76f140 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -26,15 +26,23 @@ mkfifo "$tmpdir/status_pipe"
 
 start=$(date ${TIME_FORMAT});
 
+RAPIDS_SERVICE_NAME=${RAPIDS_SERVICE_NAME:-${GITHUB_JOB:-"default-telemetry-service-name"}}
+RAPIDS_OTEL_TRACES_EXPORTER=${RAPIDS_OTEL_TRACES_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}
+RAPIDS_OTEL_METRICS_EXPORTER=${RAPIDS_OTEL_METRICS_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}
+RAPIDS_OTEL_LOGS_EXPORTER=${RAPIDS_OTEL_LOGS_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}
+OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces}
+OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/metrics}
+OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/logs}
+
 if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 ]]; then
     echo "Running command with OpenTelemetry instrumentation";
     (
         set -x
         opentelemetry-instrument \
-            --service_name ${RAPIDS_OTEL_SERVICE_NAME:-"rapids-build-test"} \
-            --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER:-"console"} \
-            --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER:-"console"} \
-            --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER:-"console"} \
+            --service_name ${RAPIDS_SERVICE_NAME} \
+            --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER} \
+            --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER} \
+            --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER} \
             --exporter_otlp_traces_endpoint ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} \
             --exporter_otlp_metrics_endpoint ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT} \
             --exporter_otlp_logs_endpoint ${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT} \
@@ -66,9 +74,12 @@ type otel-cli >/dev/null 2>&1 && {
     if [ -n "${RAPIDS_OTEL_ATTRS}" ]; then  \
         attrs="--attrs ${RAPIDS_OTEL_ATTRS}"; \
     fi
-    otel-cli span -n "$1" -s "${RAPIDS_JOB_ID}" --endpoint ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} \
-        --otlp-headers "${OTEL_EXPORTER_OTLP_HEADERS}" \
+    # TRACEPARENT gets picked up as an env var automatically
+    # https://github.com/equinix-labs/otel-cli?tab=readme-ov-file#examples
+    otel-cli span -n "$1" -s "${RAPIDS_SERVICE_NAME}" \
+        --endpoint ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} \
         --start "$start" --end "$(date ${TIME_FORMAT})" \
+        --otlp-headers "${OTEL_EXPORTER_OTLP_HEADERS}" \
         --protocol http/protobuf \
         --status-code ${RETURN_STATUS} \
         ${attrs}

From da342399190dcc5536c0c6ce7ce184e5f4df3db1 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 25 Sep 2024 14:07:27 -0500
Subject: [PATCH 17/23] shellcheck

---
 tools/rapids-otel-wrap | 68 ++++++++++++++++++++++--------------------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index f76f140..95e93e4 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -14,7 +14,6 @@ tmpdir=
 cleanup () {
   trap - EXIT
   if [ -n "$tmpdir" ] ; then rm -rf "$tmpdir"; fi
-  if [ -n "$1" ]; then trap - $1; kill -$1 $$; fi
 }
 tmpdir=$(mktemp -d)
 trap 'cleanup' EXIT
@@ -26,49 +25,51 @@ mkfifo "$tmpdir/status_pipe"
 
 start=$(date ${TIME_FORMAT});
 
-RAPIDS_SERVICE_NAME=${RAPIDS_SERVICE_NAME:-${GITHUB_JOB:-"default-telemetry-service-name"}}
-RAPIDS_OTEL_TRACES_EXPORTER=${RAPIDS_OTEL_TRACES_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}
-RAPIDS_OTEL_METRICS_EXPORTER=${RAPIDS_OTEL_METRICS_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}
-RAPIDS_OTEL_LOGS_EXPORTER=${RAPIDS_OTEL_LOGS_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}
-OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces}
-OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/metrics}
-OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/logs}
+RAPIDS_SERVICE_NAME="${RAPIDS_SERVICE_NAME:-${GITHUB_JOB:-"default-telemetry-service-name"}}"
+RAPIDS_OTEL_TRACES_EXPORTER="${RAPIDS_OTEL_TRACES_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}"
+RAPIDS_OTEL_METRICS_EXPORTER="${RAPIDS_OTEL_METRICS_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}"
+RAPIDS_OTEL_LOGS_EXPORTER="${RAPIDS_OTEL_LOGS_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}"
+OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces}"
+OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/metrics}"
+OTEL_EXPORTER_OTLP_LOGS_ENDPOINT="${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/logs}"
 
 if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 ]]; then
     echo "Running command with OpenTelemetry instrumentation";
     (
-        set -x
-        opentelemetry-instrument \
-            --service_name ${RAPIDS_SERVICE_NAME} \
-            --traces_exporter ${RAPIDS_OTEL_TRACES_EXPORTER} \
-            --metrics_exporter ${RAPIDS_OTEL_METRICS_EXPORTER} \
-            --logs_exporter ${RAPIDS_OTEL_LOGS_EXPORTER} \
-            --exporter_otlp_traces_endpoint ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} \
-            --exporter_otlp_metrics_endpoint ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT} \
-            --exporter_otlp_logs_endpoint ${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT} \
-            --exporter_otlp_protocol http/protobuf \
-            "$@" 2>&1 > $tmpdir/pipe;
-        echo $? > $tmpdir/status_pipe;
-        set +x
+        (
+            set -x
+            opentelemetry-instrument \
+                --service_name "${RAPIDS_SERVICE_NAME}" \
+                --traces_exporter "${RAPIDS_OTEL_TRACES_EXPORTER}" \
+                --metrics_exporter "${RAPIDS_OTEL_METRICS_EXPORTER}" \
+                --logs_exporter "${RAPIDS_OTEL_LOGS_EXPORTER}" \
+                --exporter_otlp_traces_endpoint "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}" \
+                --exporter_otlp_metrics_endpoint "${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT}" \
+                --exporter_otlp_logs_endpoint "${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT}" \
+                --exporter_otlp_protocol http/protobuf \
+                "$@" > "$tmpdir/pipe" 2>&1;
+        )
+        echo $? > "$tmpdir/status_pipe";
     ) &
 else
-    echo "Skipping instrumentation, running \"${@}\"";
+    echo "Skipping instrumentation, running \"${*}\"";
     (
-        "$@" 2>&1 > $tmpdir/pipe;
-        echo $? > $tmpdir/status_pipe;
+        (
+            set -x
+            "$*" > "$tmpdir/pipe" 2>&1 ;
+        )
+        echo $? > "$tmpdir/status_pipe";
     ) &
 fi
 
-set +x
 echo "stdout/stderr from command:"
 while IFS= read -r line; do
     echo "$line"
-done < $tmpdir/pipe
-RETURN_STATUS=$(<$tmpdir/status_pipe)
-set -x
+done < "$tmpdir/pipe"
+RETURN_STATUS=$(<"$tmpdir/status_pipe")
 
 # Trim leading commas from RAPIDS_OTEL_ATTRS from clumsy concatentation
-RAPIDS_OTEL_ATTRS=$(echo "${RAPIDS_OTEL_ATTRS}" | sed 's/,$//')
+RAPIDS_OTEL_ATTRS="${RAPIDS_OTEL_ATTRS//^,/}"
 
 type otel-cli >/dev/null 2>&1 && {
     if [ -n "${RAPIDS_OTEL_ATTRS}" ]; then  \
@@ -77,11 +78,12 @@ type otel-cli >/dev/null 2>&1 && {
     # TRACEPARENT gets picked up as an env var automatically
     # https://github.com/equinix-labs/otel-cli?tab=readme-ov-file#examples
     otel-cli span -n "$1" -s "${RAPIDS_SERVICE_NAME}" \
-        --endpoint ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} \
+        --endpoint "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}" \
         --start "$start" --end "$(date ${TIME_FORMAT})" \
         --otlp-headers "${OTEL_EXPORTER_OTLP_HEADERS}" \
         --protocol http/protobuf \
-        --status-code ${RETURN_STATUS} \
-        ${attrs}
+        --status-code "${RETURN_STATUS}" \
+        "${attrs}"
 }
-exit ${RETURN_STATUS}
+cleanup "$@"
+exit "${RETURN_STATUS}"

From 01de8e259d6a127a650edfec4530eac0c1caf368 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Wed, 2 Oct 2024 20:15:33 -0500
Subject: [PATCH 18/23] fix service name

---
 tools/rapids-otel-wrap | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 95e93e4..43ebe6b 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -25,7 +25,7 @@ mkfifo "$tmpdir/status_pipe"
 
 start=$(date ${TIME_FORMAT});
 
-RAPIDS_SERVICE_NAME="${RAPIDS_SERVICE_NAME:-${GITHUB_JOB:-"default-telemetry-service-name"}}"
+RAPIDS_OTEL_SERVICE_NAME="${RAPIDS_OTEL_SERVICE_NAME:-${GITHUB_JOB:-"default-telemetry-service-name"}}"
 RAPIDS_OTEL_TRACES_EXPORTER="${RAPIDS_OTEL_TRACES_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}"
 RAPIDS_OTEL_METRICS_EXPORTER="${RAPIDS_OTEL_METRICS_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}"
 RAPIDS_OTEL_LOGS_EXPORTER="${RAPIDS_OTEL_LOGS_EXPORTER:-${RAPIDS_OTEL_EXPORTER:-"console"}}"
@@ -39,7 +39,7 @@ if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 ]]; then
         (
             set -x
             opentelemetry-instrument \
-                --service_name "${RAPIDS_SERVICE_NAME}" \
+                --service_name "${RAPIDS_OTEL_SERVICE_NAME}" \
                 --traces_exporter "${RAPIDS_OTEL_TRACES_EXPORTER}" \
                 --metrics_exporter "${RAPIDS_OTEL_METRICS_EXPORTER}" \
                 --logs_exporter "${RAPIDS_OTEL_LOGS_EXPORTER}" \

From 46916e9f3c00288bc8fbc3c0ff9bdc4291139cff Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Thu, 3 Oct 2024 09:24:13 -0500
Subject: [PATCH 19/23] fix handling of empty headers in otel-cli call

---
 tools/rapids-otel-wrap | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 43ebe6b..7f895bf 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -75,15 +75,17 @@ type otel-cli >/dev/null 2>&1 && {
     if [ -n "${RAPIDS_OTEL_ATTRS}" ]; then  \
         attrs="--attrs ${RAPIDS_OTEL_ATTRS}"; \
     fi
+    if [ -n "OTEL_EXPORTER_OTLP_HEADERS" ]; then \
+        headers="--otlp-headers ${OTEL_EXPORTER_OTLP_HEADERS}"; \
+    fi
     # TRACEPARENT gets picked up as an env var automatically
     # https://github.com/equinix-labs/otel-cli?tab=readme-ov-file#examples
     otel-cli span -n "$1" -s "${RAPIDS_SERVICE_NAME}" \
         --endpoint "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}" \
         --start "$start" --end "$(date ${TIME_FORMAT})" \
-        --otlp-headers "${OTEL_EXPORTER_OTLP_HEADERS}" \
         --protocol http/protobuf \
         --status-code "${RETURN_STATUS}" \
-        "${attrs}"
+        ${attrs} ${headers}
 }
 cleanup "$@"
 exit "${RETURN_STATUS}"

From 897830311e638ecad74c95eb6766efae261b2433 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Thu, 3 Oct 2024 09:31:34 -0500
Subject: [PATCH 20/23] handle RAPIDS_OTEL_ATTRS in upload conda script

---
 tools/rapids-otel-wrap          | 3 +++
 tools/rapids-upload-conda-to-s3 | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 7f895bf..6a7bf3c 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -85,6 +85,9 @@ type otel-cli >/dev/null 2>&1 && {
         --start "$start" --end "$(date ${TIME_FORMAT})" \
         --protocol http/protobuf \
         --status-code "${RETURN_STATUS}" \
+        --tls-ca-cert ${OTEL_EXPORTER_OTLP_CERTIFICATE} \
+        --tls-client-cert ${OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE} \
+        --tls-client-key ${OTEL_EXPORTER_OTLP_CLIENT_KEY} \
         ${attrs} ${headers}
 }
 cleanup "$@"
diff --git a/tools/rapids-upload-conda-to-s3 b/tools/rapids-upload-conda-to-s3
index 17b478e..7db378d 100755
--- a/tools/rapids-upload-conda-to-s3
+++ b/tools/rapids-upload-conda-to-s3
@@ -30,5 +30,5 @@ pkg_name="$(rapids-package-name "$pkg_type")"
 # Where conda build artifacts are output
 path_to_tar_up="${RAPIDS_CONDA_BLD_OUTPUT_DIR}"
 
-RAPIDS_OTEL_ATTRS="${RAPIDS_OTEL_ATTRS},rapids_task=upload,rapids_package_name=${pkg_name}" \
-    rapids-otel-wrap rapids-upload-to-s3 "${pkg_name}" "${path_to_tar_up}"
+export RAPIDS_OTEL_ATTRS="${RAPIDS_OTEL_ATTRS:-},rapids_task=upload,rapids_package_name=${pkg_name}"
+rapids-otel-wrap rapids-upload-to-s3 "${pkg_name}" "${path_to_tar_up}"

From 6b515678f1e2be9f4982255f7c6c3115881e5cdf Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Fri, 4 Oct 2024 15:02:30 -0500
Subject: [PATCH 21/23] fix incorrect replacement expression for leading comma

---
 tools/rapids-otel-wrap | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index 6a7bf3c..cabc8a0 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -69,7 +69,7 @@ done < "$tmpdir/pipe"
 RETURN_STATUS=$(<"$tmpdir/status_pipe")
 
 # Trim leading commas from RAPIDS_OTEL_ATTRS from clumsy concatentation
-RAPIDS_OTEL_ATTRS="${RAPIDS_OTEL_ATTRS//^,/}"
+RAPIDS_OTEL_ATTRS="${RAPIDS_OTEL_ATTRS/^,//}"
 
 type otel-cli >/dev/null 2>&1 && {
     if [ -n "${RAPIDS_OTEL_ATTRS}" ]; then  \

From 16d67291b96625164ef512d71ca00f874d850ab0 Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Fri, 4 Oct 2024 16:42:30 -0500
Subject: [PATCH 22/23] print traceparent

---
 tools/rapids-otel-wrap | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index cabc8a0..b28edb0 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -88,6 +88,7 @@ type otel-cli >/dev/null 2>&1 && {
         --tls-ca-cert ${OTEL_EXPORTER_OTLP_CERTIFICATE} \
         --tls-client-cert ${OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE} \
         --tls-client-key ${OTEL_EXPORTER_OTLP_CLIENT_KEY} \
+        --tp-print \
         ${attrs} ${headers}
 }
 cleanup "$@"

From ae304e33eb2057e1ffd87cd0d950e889d029caad Mon Sep 17 00:00:00 2001
From: Michael Sarahan <msarahan@nvidia.com>
Date: Fri, 4 Oct 2024 17:09:04 -0500
Subject: [PATCH 23/23] debugging traceparent

---
 tools/rapids-otel-wrap | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/tools/rapids-otel-wrap b/tools/rapids-otel-wrap
index b28edb0..565adb4 100755
--- a/tools/rapids-otel-wrap
+++ b/tools/rapids-otel-wrap
@@ -68,18 +68,16 @@ while IFS= read -r line; do
 done < "$tmpdir/pipe"
 RETURN_STATUS=$(<"$tmpdir/status_pipe")
 
-# Trim leading commas from RAPIDS_OTEL_ATTRS from clumsy concatentation
-RAPIDS_OTEL_ATTRS="${RAPIDS_OTEL_ATTRS/^,//}"
-
 type otel-cli >/dev/null 2>&1 && {
-    if [ -n "${RAPIDS_OTEL_ATTRS}" ]; then  \
-        attrs="--attrs ${RAPIDS_OTEL_ATTRS}"; \
+    if [ "${RAPIDS_OTEL_ATTRS}" != "" ]; then  \
+        attrs="--attrs $(sed "s/,//" <<< ${RAPIDS_OTEL_ATTRS})"; \
     fi
-    if [ -n "OTEL_EXPORTER_OTLP_HEADERS" ]; then \
+    if [ "${OTEL_EXPORTER_OTLP_HEADERS}" != "" ]; then \
         headers="--otlp-headers ${OTEL_EXPORTER_OTLP_HEADERS}"; \
     fi
     # TRACEPARENT gets picked up as an env var automatically
     # https://github.com/equinix-labs/otel-cli?tab=readme-ov-file#examples
+    set -x
     otel-cli span -n "$1" -s "${RAPIDS_SERVICE_NAME}" \
         --endpoint "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}" \
         --start "$start" --end "$(date ${TIME_FORMAT})" \
@@ -90,6 +88,7 @@ type otel-cli >/dev/null 2>&1 && {
         --tls-client-key ${OTEL_EXPORTER_OTLP_CLIENT_KEY} \
         --tp-print \
         ${attrs} ${headers}
+    set +x
 }
 cleanup "$@"
 exit "${RETURN_STATUS}"