From 5b625623872b89107584959f227aa804c23b448d Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Wed, 8 Mar 2023 14:02:16 -0500 Subject: [PATCH] Fix `moto` env vars & pass `AWS_SESSION_TOKEN` to conda builds (#12902) This PR adds some additional AWS related environment variable mocks to the `cudf` `pytest`s. These environment variables are documented by `moto` here: https://docs.getmoto.org/en/latest/docs/getting_started.html#recommended-usage. This is important because of some recent CI changes that we made which now populate the `AWS_SESSION_TOKEN` environment variable, which was not mocked prior to the changes in this PR. Additionally, this PR adds the `AWS_SESSION_TOKEN` value to the `script_env` lists for our conda recipes so that it's value is passed through to builds. I also alphabetized the `script_env` lists. Authors: - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Bradley Dice (https://github.com/bdice) - Jordan Jacobelli (https://github.com/jjacobelli) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/12902 --- conda/recipes/cudf/meta.yaml | 17 +++++++++-------- conda/recipes/cudf_kafka/meta.yaml | 17 +++++++++-------- conda/recipes/custreamz/meta.yaml | 17 +++++++++-------- conda/recipes/dask-cudf/meta.yaml | 17 +++++++++-------- conda/recipes/libcudf/meta.yaml | 19 ++++++++++--------- python/cudf/cudf/tests/test_s3.py | 11 +++++++---- .../dask_cudf/dask_cudf/io/tests/test_s3.py | 11 +++++++---- 7 files changed, 60 insertions(+), 49 deletions(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 1ce401abb81..8cf3fc61e7c 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -18,18 +18,19 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda_{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script_env: - - PARALLEL_LEVEL - - CMAKE_GENERATOR + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=cudf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=cudf-linux64 # [linux64] + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_GENERATOR + - PARALLEL_LEVEL - SCCACHE_BUCKET - - SCCACHE_REGION - SCCACHE_IDLE_TIMEOUT - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY + - SCCACHE_REGION + - SCCACHE_S3_KEY_PREFIX=cudf-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=cudf-linux64 # [linux64] ignore_run_exports: # libcudf's run_exports pinning is looser than we would like - libcudf diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index 5cbea78e82b..ab92220d525 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -17,18 +17,19 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script_env: - - PARALLEL_LEVEL - - CMAKE_GENERATOR + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=cudf-kafka-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=cudf-kafka-linux64 # [linux64] + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_GENERATOR + - PARALLEL_LEVEL - SCCACHE_BUCKET - - SCCACHE_REGION - SCCACHE_IDLE_TIMEOUT - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY + - SCCACHE_REGION + - SCCACHE_S3_KEY_PREFIX=cudf-kafka-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=cudf-kafka-linux64 # [linux64] requirements: build: diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml index af5705341e6..277c5d6cc17 100644 --- a/conda/recipes/custreamz/meta.yaml +++ b/conda/recipes/custreamz/meta.yaml @@ -17,18 +17,19 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script_env: - - PARALLEL_LEVEL - - CMAKE_GENERATOR + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=custreamz-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=custreamz-linux64 # [linux64] + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_GENERATOR + - PARALLEL_LEVEL - SCCACHE_BUCKET - - SCCACHE_REGION - SCCACHE_IDLE_TIMEOUT - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY + - SCCACHE_REGION + - SCCACHE_S3_KEY_PREFIX=custreamz-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=custreamz-linux64 # [linux64] requirements: host: diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index 3ee3d4d3952..85dd3231635 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -18,18 +18,19 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda_{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script_env: - - PARALLEL_LEVEL - - CMAKE_GENERATOR + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=dask-cudf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=dask-cudf-linux64 # [linux64] + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_GENERATOR + - PARALLEL_LEVEL - SCCACHE_BUCKET - - SCCACHE_REGION - SCCACHE_IDLE_TIMEOUT - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY + - SCCACHE_REGION + - SCCACHE_S3_KEY_PREFIX=dask-cudf-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=dask-cudf-linux64 # [linux64] requirements: host: diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index fbfcf6e71a2..890ab9f6632 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -15,19 +15,20 @@ source: build: script_env: - - PARALLEL_LEVEL - - CMAKE_GENERATOR + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=libcudf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libcudf-linux64 # [linux64] + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_GENERATOR + - PARALLEL_LEVEL + - RAPIDS_ARTIFACTS_DIR - SCCACHE_BUCKET - - SCCACHE_REGION - SCCACHE_IDLE_TIMEOUT - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - RAPIDS_ARTIFACTS_DIR + - SCCACHE_REGION + - SCCACHE_S3_KEY_PREFIX=libcudf-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=libcudf-linux64 # [linux64] requirements: build: diff --git a/python/cudf/cudf/tests/test_s3.py b/python/cudf/cudf/tests/test_s3.py index de3bba25223..d54a2eabf22 100644 --- a/python/cudf/cudf/tests/test_s3.py +++ b/python/cudf/cudf/tests/test_s3.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import os import socket @@ -59,9 +59,12 @@ def s3_base(endpoint_ip, endpoint_port): with ensure_safe_environment_variables(): # Fake aws credentials exported to prevent botocore looking for # system aws credentials, https://github.com/spulec/moto/issues/1793 - os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - os.environ.setdefault("S3FS_LOGGING_LEVEL", "DEBUG") + os.environ["AWS_ACCESS_KEY_ID"] = "foobar_key" + os.environ["AWS_SECRET_ACCESS_KEY"] = "foobar_secret" + os.environ["S3FS_LOGGING_LEVEL"] = "DEBUG" + os.environ["AWS_SECURITY_TOKEN"] = "foobar_security_token" + os.environ["AWS_SESSION_TOKEN"] = "foobar_session_token" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" # Launching moto in server mode, i.e., as a separate process # with an S3 endpoint on localhost diff --git a/python/dask_cudf/dask_cudf/io/tests/test_s3.py b/python/dask_cudf/dask_cudf/io/tests/test_s3.py index 5be0cf7c887..7614ea38d6a 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_s3.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_s3.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import os import socket @@ -55,9 +55,12 @@ def s3_base(endpoint_ip, endpoint_port): with ensure_safe_environment_variables(): # Fake aws credentials exported to prevent botocore looking for # system aws credentials, https://github.com/spulec/moto/issues/1793 - os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - os.environ.setdefault("S3FS_LOGGING_LEVEL", "DEBUG") + os.environ["AWS_ACCESS_KEY_ID"] = "foobar_key" + os.environ["AWS_SECRET_ACCESS_KEY"] = "foobar_secret" + os.environ["S3FS_LOGGING_LEVEL"] = "DEBUG" + os.environ["AWS_SECURITY_TOKEN"] = "foobar_security_token" + os.environ["AWS_SESSION_TOKEN"] = "foobar_session_token" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" # Launching moto in server mode, i.e., as a separate process # with an S3 endpoint on localhost