From c3e82c1f0b68a56a16d95bbcedeab94b96cd38dd Mon Sep 17 00:00:00 2001 From: peixuanzuo Date: Thu, 20 Oct 2022 00:31:42 +0000 Subject: [PATCH] use env variable --- orttraining/tools/ci_test/download_azure_blob_archive.py | 8 +++++--- .../orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml | 2 +- .../azure-pipelines/orttraining-pai-ci-pipeline.yml | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/orttraining/tools/ci_test/download_azure_blob_archive.py b/orttraining/tools/ci_test/download_azure_blob_archive.py index 58a8d8f7f9279..6fa875a1d2373 100755 --- a/orttraining/tools/ci_test/download_azure_blob_archive.py +++ b/orttraining/tools/ci_test/download_azure_blob_archive.py @@ -50,7 +50,6 @@ def _check_file_sha256_digest(path, expected_digest): def main(): parser = argparse.ArgumentParser(description="Downloads an Azure blob archive.") parser.add_argument("--azure_blob_url", required=True, help="The Azure blob URL.") - parser.add_argument("--azure_blob_sas_token", type=str, default="", help="The Azure blob SAS Token") parser.add_argument("--target_dir", required=True, help="The destination directory.") parser.add_argument("--archive_sha256_digest", help="The SHA256 digest of the archive. Verified if provided.") args = parser.parse_args() @@ -58,9 +57,12 @@ def main(): with tempfile.TemporaryDirectory() as temp_dir, get_azcopy() as azcopy_path: archive_path = os.path.join(temp_dir, "archive.zip") print("Downloading archive from '{}'...".format(args.azure_blob_url)) + azure_blob_url = args.azure_blob_url - if args.azure_blob_sas_token != "": - azure_blob_url = azure_blob_url + "?" + args.azure_blob_sas_token + azure_blob_sas_token = os.getenv("AZURE_BLOB_SAS_TOKEN", None) + if azure_blob_sas_token and azure_blob_sas_token != "": + azure_blob_url = azure_blob_url + "?" + azure_blob_sas_token + _download(azcopy_path, azure_blob_url, archive_path) if args.archive_sha256_digest: _check_file_sha256_digest(archive_path, args.archive_sha256_digest) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml index f59e7e9c7abe6..8207a336880ff 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml @@ -23,9 +23,9 @@ jobs: # update these if the E2E test data changes - script: |- + export AZURE_BLOB_SAS_TOKEN="$(onnxruntimetestdata-storage-training-container-sas-token)" python orttraining/tools/ci_test/download_azure_blob_archive.py \ --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip \ - --azure_blob_sas_token "$(onnxruntimetestdata-storage-training-container-sas-token)" \ --target_dir training_e2e_test_data \ --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 displayName: 'Download onnxruntime_training_data.zip data' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index ad7ef4516109f..153411db84e45 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -284,9 +284,9 @@ jobs: - task: CmdLine@2 inputs: script: |- + export AZURE_BLOB_SAS_TOKEN="$(onnxruntimetestdata-storage-training-container-sas-token)" python orttraining/tools/ci_test/download_azure_blob_archive.py \ --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip \ - --azure_blob_sas_token "$(onnxruntimetestdata-storage-training-container-sas-token)" \ --target_dir training_e2e_test_data \ --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed