From 0899b277c02082ffc24bb732e8a7cf3ef4333948 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 8 Jul 2020 19:54:17 +0300 Subject: [PATCH] tests: launch oss/azure containers from fixtures (#4178) * tests: launch oss/azure containers from fixtures Related to #4054 * ci: hdfs: remove unneeded fix We are using pyarrow 0.17.0 already. --- scripts/ci/before_install.sh | 2 -- scripts/ci/install_azurite.sh | 14 -------- scripts/ci/install_hadoop.sh | 7 ---- scripts/ci/install_oss.sh | 10 ------ setup.py | 1 + tests/docker-compose.yml | 11 ++++++ tests/remotes/__init__.py | 14 ++++++-- tests/remotes/azure.py | 63 +++++++++++++++++++++++---------- tests/remotes/oss.py | 57 +++++++++++++++++------------ tests/unit/remote/test_azure.py | 12 ++----- 10 files changed, 106 insertions(+), 85 deletions(-) delete mode 100755 scripts/ci/install_azurite.sh delete mode 100644 scripts/ci/install_oss.sh create mode 100644 tests/docker-compose.yml diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh index a2a1070883..62b799f166 100644 --- a/scripts/ci/before_install.sh +++ b/scripts/ci/before_install.sh @@ -32,8 +32,6 @@ if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then fi if [ "$TRAVIS_OS_NAME" == "linux" ]; then - bash "$scriptdir/install_azurite.sh" - bash "$scriptdir/install_oss.sh" bash "$scriptdir/install_hadoop.sh" fi diff --git a/scripts/ci/install_azurite.sh b/scripts/ci/install_azurite.sh deleted file mode 100755 index 48e263684d..0000000000 --- a/scripts/ci/install_azurite.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -# run azurite -sudo docker run -d --restart always -e executable=blob -p 10000:10000 \ - --tmpfs /opt/azurite/folder \ - mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \ - azurite -l /data --blobHost 0.0.0.0 - -# save secrets -echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >>env.sh -echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >>env.sh diff --git a/scripts/ci/install_hadoop.sh b/scripts/ci/install_hadoop.sh index 48c35e0c77..c44bd5e64d 100755 --- a/scripts/ci/install_hadoop.sh +++ b/scripts/ci/install_hadoop.sh @@ -24,13 +24,6 @@ echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh echo "export YARN_HOME=/usr/local/hadoop" >>env.sh echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh -# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841 -# retrieves native library from $HADOOP_HOME directory instead of -# `$HADOOP_HOME/lib/native`. -# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`. -# Note: not needed for PyArrow==0.17.0. -echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh - echo "export JAVA_HOME=/usr/" >>env.sh echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh diff --git a/scripts/ci/install_oss.sh b/scripts/ci/install_oss.sh deleted file mode 100644 index a56ac7cb79..0000000000 --- a/scripts/ci/install_oss.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -git clone https://github.com/iterative/oss-emulator.git -sudo docker image build -t oss:1.0 oss-emulator -sudo docker run --detach --restart always -p 8880:8880 --name oss-emulator oss:1.0 -echo "export OSS_ENDPOINT='localhost:8880'" >>env.sh -echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >>env.sh -echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >>env.sh diff --git a/setup.py b/setup.py index 6d4831effe..b49f57d19c 100644 --- a/setup.py +++ b/setup.py @@ -103,6 +103,7 @@ def run(self): "wheel>=0.31.1", # Test requirements: "pytest>=4.6.0", + "pytest-docker>=0.7.2", "pytest-timeout>=1.3.3", "pytest-cov>=2.6.1", "pytest-xdist>=1.26.1", diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000000..c06b60ce67 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,11 @@ +--- +version: '3.2' +services: + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.3.0-preview + ports: + - "10000" + oss: + image: rkuprieiev/oss-emulator + ports: + - "8880" diff --git a/tests/remotes/__init__.py b/tests/remotes/__init__.py index 7ac0e58cb8..877b1f74dd 100644 --- a/tests/remotes/__init__.py +++ b/tests/remotes/__init__.py @@ -1,10 +1,12 @@ +import subprocess + import pytest -from .azure import Azure, azure # noqa: F401 +from .azure import Azure, azure, azure_server # noqa: F401 from .hdfs import HDFS, hdfs # noqa: F401 from .http import HTTP, http, http_server # noqa: F401 from .local import Local, local_cloud, local_remote # noqa: F401 -from .oss import OSS, TEST_OSS_REPO_BUCKET, oss # noqa: F401 +from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_server # noqa: F401 from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3 # noqa: F401 TEST_REMOTE = "upstream" @@ -35,6 +37,14 @@ ) +@pytest.fixture(scope="session") +def docker_compose(): + try: + subprocess.check_output("docker-compose version", shell=True) + except (subprocess.CalledProcessError, OSError): + pytest.skip("no docker-compose installed") + + @pytest.fixture def remote(tmp_dir, dvc, request): cloud = request.param diff --git a/tests/remotes/azure.py b/tests/remotes/azure.py index bad780a682..8dae521d2d 100644 --- a/tests/remotes/azure.py +++ b/tests/remotes/azure.py @@ -1,36 +1,61 @@ # pylint:disable=abstract-method -import os import uuid import pytest from dvc.path_info import CloudURLInfo -from dvc.utils import env2bool from .base import Base +TEST_AZURE_CONTAINER = "tests" +TEST_AZURE_CONNECTION_STRING = ( + "DefaultEndpointsProtocol=http;" + "AccountName=devstoreaccount1;" + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSR" + "Z6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + "BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;" +) + class Azure(Base, CloudURLInfo): - @staticmethod - def should_test(): - do_test = env2bool("DVC_TEST_AZURE", undefined=None) - if do_test is not None: - return do_test + pass + + +@pytest.fixture(scope="session") +def azure_server(docker_compose, docker_services): + from azure.storage.blob import ( # pylint: disable=no-name-in-module + BlockBlobService, + ) + from azure.common import ( # pylint: disable=no-name-in-module + AzureException, + ) + + port = docker_services.port_for("azurite", 10000) + connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port) + + def _check(): + try: + BlockBlobService( + connection_string=connection_string, + ).list_containers() + return True + except AzureException: + return False - return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv( - "AZURE_STORAGE_CONNECTION_STRING" - ) + docker_services.wait_until_responsive( + timeout=60.0, pause=0.1, check=_check + ) - @staticmethod - def get_url(): - container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME") - assert container_name is not None - return "azure://{}/{}".format(container_name, str(uuid.uuid4())) + return connection_string @pytest.fixture -def azure(): - if not Azure.should_test(): - pytest.skip("no azure running") - yield Azure(Azure.get_url()) +def azure(azure_server): + url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}" + ret = Azure(url) + ret.config = { + "url": url, + "connection_string": azure_server, + } + return ret diff --git a/tests/remotes/oss.py b/tests/remotes/oss.py index c0749fa8a1..1a163cd3c8 100644 --- a/tests/remotes/oss.py +++ b/tests/remotes/oss.py @@ -1,41 +1,54 @@ # pylint:disable=abstract-method -import os import uuid import pytest from dvc.path_info import CloudURLInfo -from dvc.utils import env2bool from .base import Base TEST_OSS_REPO_BUCKET = "dvc-test" +TEST_OSS_ENDPOINT = "127.0.0.1:{port}" +TEST_OSS_ACCESS_KEY_ID = "AccessKeyID" +TEST_OSS_ACCESS_KEY_SECRET = "AccessKeySecret" class OSS(Base, CloudURLInfo): - @staticmethod - def should_test(): - do_test = env2bool("DVC_TEST_OSS", undefined=None) - if do_test is not None: - return do_test + pass - return ( - os.getenv("OSS_ENDPOINT") - and os.getenv("OSS_ACCESS_KEY_ID") - and os.getenv("OSS_ACCESS_KEY_SECRET") - ) - @staticmethod - def _get_storagepath(): - return f"{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}" +@pytest.fixture(scope="session") +def oss_server(docker_compose, docker_services): + import oss2 - @staticmethod - def get_url(): - return f"oss://{OSS._get_storagepath()}" + port = docker_services.port_for("oss", 8880) + endpoint = TEST_OSS_ENDPOINT.format(port=port) + + def _check(): + try: + auth = oss2.Auth( + TEST_OSS_ACCESS_KEY_ID, TEST_OSS_ACCESS_KEY_SECRET + ) + oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info() + return True + except oss2.exceptions.NoSuchBucket: + return True + except oss2.exceptions.OssError: + return False + + docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check) + + return endpoint @pytest.fixture -def oss(): - if not OSS.should_test(): - pytest.skip("no oss running") - yield OSS(OSS.get_url()) +def oss(oss_server): + url = f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}" + ret = OSS(url) + ret.config = { + "url": url, + "oss_key_id": TEST_OSS_ACCESS_KEY_ID, + "oss_key_secret": TEST_OSS_ACCESS_KEY_SECRET, + "oss_endpoint": oss_server, + } + return ret diff --git a/tests/unit/remote/test_azure.py b/tests/unit/remote/test_azure.py index 2c601c30d1..200639e117 100644 --- a/tests/unit/remote/test_azure.py +++ b/tests/unit/remote/test_azure.py @@ -1,8 +1,5 @@ -import pytest - from dvc.path_info import PathInfo from dvc.remote.azure import AzureRemoteTree -from tests.remotes import Azure container_name = "container-name" connection_string = ( @@ -32,14 +29,11 @@ def test_init(dvc): assert tree._conn_kwargs["connection_string"] == connection_string -def test_get_file_hash(tmp_dir): - if not Azure.should_test(): - pytest.skip("no azurite running") - +def test_get_file_hash(tmp_dir, azure): tmp_dir.gen("foo", "foo") - tree = AzureRemoteTree(None, {}) - to_info = tree.PATH_CLS(Azure.get_url()) + tree = AzureRemoteTree(None, azure.config) + to_info = azure tree.upload(PathInfo("foo"), to_info) assert tree.exists(to_info) hash_ = tree.get_file_hash(to_info)