Testing - Using Argo validation in Travis tests (kubeflow#3123)

* Testing - Using Argo validation in Travis tests :# Date: Wed Feb 19 13:14:09 2020 -0800 * Fixed typo
akartsky · May 29, 2020 · b7111c3 · b7111c3
1 parent b503050
commit b7111c3
Show file tree

Hide file tree

Showing 10 changed files with 43 additions and 439 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -39,6 +39,11 @@ matrix:
         # Visualization test dependencies
         - cd $TRAVIS_BUILD_DIR/backend/src/apiserver/visualization
         - pip3 install -r requirements-test.txt
+        # Using Argo to lint all compiled workflows
+        - export LOCAL_BIN="${HOME}/.local/bin"
+        - mkdir -p "$LOCAL_BIN"
+        - export PATH="${PATH}:$LOCAL_BIN" # Unnecessary - Travis already has it in PATH
+        - wget --quiet -O "${LOCAL_BIN}/argo" https://github.com/argoproj/argo/releases/download/v2.3.0/argo-linux-amd64 && chmod +x "${LOCAL_BIN}/argo"
       script: &1 # DSL tests
         - cd $TRAVIS_BUILD_DIR/sdk/python
         - python3 -m pip install -e .

diff --git a/components/aws/sagemaker/tests/integration_tests/Dockerfile b/components/aws/sagemaker/tests/integration_tests/Dockerfile
@@ -40,4 +40,17 @@ ENV KFP_NAMESPACE="kubeflow"
 
 COPY ./sagemaker/ .
 
+# copy sample pipelines from GitHub
+RUN mkdir ./tests/integration_tests/resources/definition/sample_pipelines
+RUN wget -O ./tests/integration_tests/resources/definition/sample_pipelines/training-pipeline.py \
+         https://raw.githubusercontent.com/akartsky/pipelines/integ_for_sample/samples/contrib/aws-samples/simple_train_pipeline/training-pipeline.py
+RUN wget -O ./tests/integration_tests/resources/definition/sample_pipelines/kmeans-hpo-pipeline.py \
+         https://raw.githubusercontent.com/kubeflow/pipelines/master/samples/contrib/aws-samples/mnist-kmeans-sagemaker/kmeans-hpo-pipeline.py
+RUN wget -O ./tests/integration_tests/resources/definition/sample_pipelines/mnist-classification-pipeline.py \
+         https://github.com/kubeflow/pipelines/blob/master/samples/contrib/aws-samples/mnist-kmeans-sagemaker/mnist-classification-pipeline.py
+RUN wget -O ./tests/integration_tests/resources/definition/sample_pipelines/mnist-classification-pipeline.py \
+         https://github.com/kubeflow/pipelines/blob/master/samples/contrib/aws-samples/mnist-kmeans-sagemaker/mnist-classification-pipeline.py
+RUN wget -O ./tests/integration_tests/resources/definition/sample_pipelines/mini-image-classification-pipeline.py \
+         https://github.com/kubeflow/pipelines/blob/master/samples/contrib/aws-samples/ground_truth_pipeline_demo/mini-image-classification-pipeline.py
+
 ENTRYPOINT [ "/bin/bash", "./tests/integration_tests/scripts/run_integration_tests" ]
diff --git a/...s/aws/sagemaker/tests/integration_tests/component_tests/test_batch_transform_component.py b/...s/aws/sagemaker/tests/integration_tests/component_tests/test_batch_transform_component.py
diff --git a/components/aws/sagemaker/tests/integration_tests/component_tests/test_deploy_component.py b/components/aws/sagemaker/tests/integration_tests/component_tests/test_deploy_component.py
diff --git a/...nents/aws/sagemaker/tests/integration_tests/component_tests/test_groundtruth_component.py b/...nents/aws/sagemaker/tests/integration_tests/component_tests/test_groundtruth_component.py
diff --git a/components/aws/sagemaker/tests/integration_tests/component_tests/test_model_component.py b/components/aws/sagemaker/tests/integration_tests/component_tests/test_model_component.py