From 2dce857e2ea7f9ae12e86e7f49d4d473267139a7 Mon Sep 17 00:00:00 2001 From: Max Isom Date: Tue, 31 Dec 2024 13:34:30 -0800 Subject: [PATCH] [BLD]: save OTEL traces in artifact during CI (#3378) --- .github/actions/export-tilt-logs/action.yaml | 19 ++++++++++++ .github/actions/tilt/action.yaml | 1 + .github/workflows/_go-tests.yml | 5 +++ .github/workflows/_python-tests.yml | 22 +++++--------- .github/workflows/_rust-tests.yml | 6 ++++ .github/workflows/nightly-tests.yml | 18 +++-------- bin/get-logs.sh | 32 ++++++++------------ 7 files changed, 55 insertions(+), 48 deletions(-) create mode 100644 .github/actions/export-tilt-logs/action.yaml diff --git a/.github/actions/export-tilt-logs/action.yaml b/.github/actions/export-tilt-logs/action.yaml new file mode 100644 index 00000000000..7298ea6b0ec --- /dev/null +++ b/.github/actions/export-tilt-logs/action.yaml @@ -0,0 +1,19 @@ +name: Export Tilt logs +description: Exports logs and traces from services in Tilt to an artifact +inputs: + artifact-name: + description: "The name of the artifact containing logs and traces from services in Tilt" + required: true +runs: + using: "composite" + steps: + - name: Get logs of all services + id: get-logs + run: | + bin/get-logs.sh ${{ inputs.artifact-name }}.zip + shell: bash + - name: Upload logs as artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: "${{ inputs.artifact-name }}.zip" diff --git a/.github/actions/tilt/action.yaml b/.github/actions/tilt/action.yaml index 84d8d710b4c..8735952a3da 100644 --- a/.github/actions/tilt/action.yaml +++ b/.github/actions/tilt/action.yaml @@ -31,3 +31,4 @@ runs: kubectl -n chroma port-forward svc/query-service 50053:50051 & kubectl -n chroma port-forward svc/frontend-service 8000:8000 & kubectl -n chroma port-forward svc/minio 9000:9000 & + kubectl -n chroma port-forward svc/jaeger 16686:16686 & diff --git a/.github/workflows/_go-tests.yml b/.github/workflows/_go-tests.yml index 1d57d703392..2f212b7a958 100644 --- a/.github/workflows/_go-tests.yml +++ b/.github/workflows/_go-tests.yml @@ -44,3 +44,8 @@ jobs: uses: ./.github/actions/go - uses: ./.github/actions/tilt - run: bin/cluster-test.sh bash -c 'cd go && go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma-core/chroma/go/pkg/memberlist_manager' + - name: Save service logs to artifact + if: always() + uses: ./.github/actions/export-tilt-logs + with: + artifact-name: "go-cluster-test" diff --git a/.github/workflows/_python-tests.yml b/.github/workflows/_python-tests.yml index d4441d7cc97..363de49a4b4 100644 --- a/.github/workflows/_python-tests.yml +++ b/.github/workflows/_python-tests.yml @@ -118,21 +118,15 @@ jobs: shell: bash env: PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }} - - name: Get logs of all services - id: get-logs - if: success() || failure() - run: | - bin/get-logs.sh "${{ matrix.test-globs }}" "${{ matrix.python }}" - # Output the logs zip file path as a job output - echo "artifact_name=cluster_logs_$(basename "${{ matrix.test-globs }}" .py)_${{ matrix.python }}" >> $GITHUB_OUTPUT - echo "logs_zip_path=$(pwd)/$(basename "${{ matrix.test-globs }}" .py)_${{ matrix.python }}_logs.zip" >> $GITHUB_OUTPUT - shell: bash - - name: Upload logs as artifact - if: success() || failure() - uses: actions/upload-artifact@v4 + - name: Compute artifact name + if: always() + id: compute-artifact-name + run: echo "artifact_name=cluster_logs_$(basename "${{ matrix.test-globs }}" .py)_${{ matrix.python }}" >> $GITHUB_OUTPUT + - name: Save service logs to artifact + if: always() + uses: ./.github/actions/export-tilt-logs with: - name: ${{ steps.get-logs.outputs.artifact_name }} - path: ${{ steps.get-logs.outputs.logs_zip_path }} + artifact-name: ${{ steps.compute-artifact-name.outputs.artifact_name }} merge-cluster-logs: runs-on: ubuntu-latest diff --git a/.github/workflows/_rust-tests.yml b/.github/workflows/_rust-tests.yml index c0a58200a77..d9318688691 100644 --- a/.github/workflows/_rust-tests.yml +++ b/.github/workflows/_rust-tests.yml @@ -36,6 +36,12 @@ jobs: uses: ./.github/actions/tilt - name: Run tests run: cargo nextest run --profile k8s_integration + - name: Save service logs to artifact + if: always() + uses: ./.github/actions/export-tilt-logs + with: + artifact-name: "rust-integration-test" + test-benches: strategy: matrix: diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index cfcc4f8d786..4812749701b 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -27,21 +27,11 @@ jobs: shell: bash env: PROPERTY_TESTING_PRESET: slow - - name: Get logs of all services - id: get-logs - if: success() || failure() - run: | - bin/get-logs.sh "${{ matrix.test-globs }}" "3.12" - # Output the logs zip file path as a job output - echo "artifact_name=cluster_logs_$(basename "${{ matrix.test-globs }}" .py)_3.12" >> $GITHUB_OUTPUT - echo "logs_zip_path=$(pwd)/$(basename "${{ matrix.test-globs }}" .py)_3.12_logs.zip" >> $GITHUB_OUTPUT - shell: bash - - name: Upload logs as artifact - if: success() || failure() - uses: actions/upload-artifact@v4 + - name: Save service logs to artifact + if: always() + uses: ./.github/actions/export-tilt-logs with: - name: ${{ steps.get-logs.outputs.artifact_name }} - path: ${{ steps.get-logs.outputs.logs_zip_path }} + artifact-name: "tilt-logs" - name: Send PagerDuty alert on failure if: ${{ failure() }} uses: Entle/action-pagerduty-alert@0.2.0 diff --git a/bin/get-logs.sh b/bin/get-logs.sh index b3aee3e2b05..8e67cd24e1b 100755 --- a/bin/get-logs.sh +++ b/bin/get-logs.sh @@ -6,24 +6,16 @@ NAMESPACE=chroma echo "Namespace: $NAMESPACE" # Check if the test-name and version-number are provided as arguments -if [ -z "$1" ] || [ -z "$2" ]; then - echo "Usage: $0 " +if [ -z "$1" ]; then + echo "Usage: $0 " exit 1 # Exit with code 1 indicating an error fi -TEST_FILE_PATH=$1 -TEST_VERSION=$2 +OUTPUT_FILE_PATH=$(readlink -m $1) +TEMP_DIR=$(mktemp -d) -# Extract the test file name from the path and remove the ".py" extension -TEST_NAME=$(basename "$TEST_FILE_PATH" .py) -echo "Test name: $TEST_NAME" -echo "Test version: $TEST_VERSION" - -# Create a directory with the test name to store logs -LOGS_DIR="./logs/${TEST_NAME}_${TEST_VERSION}" -echo "Logs directory: $LOGS_DIR" -mkdir -p "$LOGS_DIR" -echo "Created logs dir: $LOGS_DIR" +mkdir "$TEMP_DIR/logs" +mkdir "$TEMP_DIR/traces" # Get the list of all pods in the namespace PODS=$(kubectl get pods -n $NAMESPACE -o jsonpath='{.items[*].metadata.name}') @@ -33,14 +25,14 @@ echo "Got all the pods: $PODS" for POD in $PODS; do echo "Getting logs for pod: $POD" # Save the logs to a file named after the pod and test name - kubectl logs $POD -n $NAMESPACE --since=0s > "${LOGS_DIR}/${POD}_logs.txt" + kubectl logs $POD -n $NAMESPACE --since=0s > "${TEMP_DIR}/logs/${POD}.txt" || true done +# Get traces from Jaeger for all services +curl "http://localhost:16686/api/services" | jq -r '.data[]' | while read -r service; do curl "http://localhost:16686/api/traces?limit=100&lookback=1h&maxDuration&minDuration&service=$service" > "$TEMP_DIR/traces/$service.json" || true; done + # Zip all log files -zip -r "${TEST_NAME}_${TEST_VERSION}_logs.zip" "$LOGS_DIR" +cd $TEMP_DIR &&zip -r "$OUTPUT_FILE_PATH" . && cd - # Print confirmation message -echo "Logs have been zipped to ${TEST_NAME}_${TEST_VERSION}_logs.zip" - -# Output the path to the zip file -echo "logs_zip_path=$(pwd)/${TEST_NAME}_${TEST_VERSION}_logs.zip" >> $GITHUB_OUTPUT +echo "Logs have been zipped to $OUTPUT_FILE_PATH"