From 51e0dac82d4f8f7d9bd201db8cb4adcca16c275f Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Mon, 9 Dec 2024 00:56:57 -0800 Subject: [PATCH] actions: source reference cores from gDrive --- .github/scripts/download-dumps-from-gdrive.sh | 10 ++++++++++ .../{download-dump-from-s3.sh => extract-dump.sh} | 10 +++------- .github/workflows/main.yml | 5 +++-- README.md | 10 ++++------ 4 files changed, 20 insertions(+), 15 deletions(-) create mode 100755 .github/scripts/download-dumps-from-gdrive.sh rename .github/scripts/{download-dump-from-s3.sh => extract-dump.sh} (86%) diff --git a/.github/scripts/download-dumps-from-gdrive.sh b/.github/scripts/download-dumps-from-gdrive.sh new file mode 100755 index 0000000..1c9a9e0 --- /dev/null +++ b/.github/scripts/download-dumps-from-gdrive.sh @@ -0,0 +1,10 @@ +#!/bin/bash -eux + +# +# Download all the reference core dumps from the public gDrive folder +# and place them in the root directory. +# +pip install gdown +gdown --folder 1fdPVuGXbxNKcMEVwhuda04hZTPCcYJms +mv SDB-Public/* . +rmdir SDB-Public diff --git a/.github/scripts/download-dump-from-s3.sh b/.github/scripts/extract-dump.sh similarity index 86% rename from .github/scripts/download-dump-from-s3.sh rename to .github/scripts/extract-dump.sh index 8c7754f..e927e5c 100755 --- a/.github/scripts/download-dump-from-s3.sh +++ b/.github/scripts/extract-dump.sh @@ -3,7 +3,7 @@ # # Assumptions for this to work: # [1] This script is executed from the root of the SDB repo -# [2] The archive downloaded from S3 has one of the following profiles: +# [2] The archive downloaded from gdrive has one of the following profiles: # Profile A - It is lzma-compressed and has the following file structure: # dump-data # ├── dump.201912060006 @@ -33,12 +33,8 @@ if [ ! -d $DATA_DIR ]; then exit 1 fi -if [ -f "$1" ]; then - echo "Found $1 locally, skip download ..." -else - echo "downloading of $1 from S3 ..." - wget https://sdb-testing-bucket.s3.us-west-2.amazonaws.com/$1 - [ $? -eq 0 ] || exit 1 +if [ ! -f "$1" ]; then + echo "error: $1: file not found" fi if [[ $1 == *.tar.lzma ]]; then diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9d74583..efebe3c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,6 @@ jobs: strategy: matrix: python-version: [3.8, 3.9] - dump: [dump.201912060006.tar.lzma, dump.202303131823.tar.gz] env: AWS_DEFAULT_REGION: 'us-west-2' steps: @@ -67,7 +66,9 @@ jobs: - run: python3 -m pip install aws python-config pytest pytest-cov - run: ./.github/scripts/install-libkdumpfile.sh - run: ./.github/scripts/install-drgn.sh - - run: ./.github/scripts/download-dump-from-s3.sh ${{ matrix.dump }} + - run: ./.github/scripts/download-dumps-from-gdrive.sh + - run: ./.github/scripts/extract-dump.sh dump.201912060006.tar.lzma + - run: ./.github/scripts/extract-dump.sh dump.202303131823.tar.gz - run: pytest -v --cov sdb --cov-report xml tests - uses: codecov/codecov-action@v1 with: diff --git a/README.md b/README.md index f78056b..ae8b3d8 100644 --- a/README.md +++ b/README.md @@ -92,15 +92,13 @@ $ python3 -m yapf -i --style google --recursive tests Regression testing is currently done by downloading a refererence crash/core dump and running a predetermined set of commads on it, ensuring that they -return the same reference output before and after your changes. To see the list -of reference crash dumps refer to the testing matrix of the `pytest` Github -action in `.github/workflows/main.yml`. Here is an example of running the -regression test commands against `dump.201912060006.tar.lzma` with code -coverage and verbose output: +return the same reference output before and after your changes. ``` $ python3 -m pip install python-config pytest pytest-cov -$ .github/scripts/download-dump-from-s3.sh dump.201912060006.tar.lzma +$ .github/scripts/download-dumps-from-gdrive.sh +$ .github/scripts/extract-dump.sh dump.201912060006.tar.lzma +$ .github/scripts/extract-dump.sh dump.202303131823.tar.gz $ python3 -m pytest -v --cov sdb --cov-report xml tests ```