forked from kubeflow/kubeflow
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
E2E test for kfctl should deploy kubeflow with basic auth (kubeflow#2705
) * The E2E test attempts to call delete but delete isn't working. * kfctl delete fails because there is no kubeconfig file created. * The delete step is marked as an expected failure. Improve logging in kfctl with go binary * Log the name and status of GCP DM operations * Print out DM creation errors so users see things like quota issues. Fix kubeflow#2706 - The coordinator commands should only invoke the commands apply/generate/delete for the resources specified. * There was a bug in the switch statements and we were always calling generate/apply/delete for platform & k8s and not respecting the parameter Attempt to fix kubeflow#2706 * We want to eliminate the need to talk to a K8s server when calling ks init * Hardcode the server address to 127.0.0.1 * Hardcode K8s version to 1.11.7 * The communication with the K8s server was coming because we were trying to talk to the K8s master to get the K8s version but we don't need to do that if we specify it. * Add filename log hook to kfctl so that we can emit the filename and line number where errors occur. * On init we Don't need to call GetConfig which tries to read the kubeconfig file on generate. Add retries to generate and apply because running under argo kfctl seems to randomly exit.
- Loading branch information
1 parent
d860215
commit e4c6f89
Showing
12 changed files
with
305 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| import pytest | ||
|
|
||
| def pytest_addoption(parser): | ||
| parser.addoption( | ||
| "--app_path", action="store", default="", | ||
| help="Path where the KF application should be stored") | ||
|
|
||
| parser.addoption( | ||
| "--kfctl_path", action="store", default="", | ||
| help="Path to kfctl.") | ||
|
|
||
| parser.addoption( | ||
| "--project", action="store", default="kubeflow-ci-deployment", | ||
| help="GCP project to deploy Kubeflow to") | ||
|
|
||
| @pytest.fixture | ||
| def app_path(request): | ||
| return request.config.getoption("--app_path") | ||
|
|
||
| @pytest.fixture | ||
| def kfctl_path(request): | ||
| return request.config.getoption("--kfctl_path") | ||
|
|
||
| @pytest.fixture | ||
| def project(request): | ||
| return request.config.getoption("--project") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| """Run kfctl delete as a pytest. | ||
| We use this in order to generate a junit_xml file. | ||
| """ | ||
| import datetime | ||
| import logging | ||
| import os | ||
| import subprocess | ||
| import tempfile | ||
| import uuid | ||
| from retrying import retry | ||
|
|
||
| import pytest | ||
|
|
||
| from kubeflow.testing import util | ||
|
|
||
| # TODO(): Need to make delete work with a KUBECONFIG file. | ||
| @pytest.mark.xfail | ||
| def test_kfctl_delete(kfctl_path, app_path, project): | ||
| if not kfctl_path: | ||
| raise ValueError("kfctl_path is required") | ||
|
|
||
| if not app_path: | ||
| raise ValueError("app_path is required") | ||
|
|
||
| logging.info("Using kfctl path %s", kfctl_path) | ||
| logging.info("Using app path %s", app_path) | ||
|
|
||
| util.run([kfctl_path, "delete", "-V", "all"], cwd=app_path) | ||
|
|
||
| # Delete the storage | ||
| app_name = os.path.basename(app_path) | ||
| util.run(["gcloud", "deployment-manager", "--project=" + project, | ||
| "deployments", "delete", app_name + "-storage", "--quiet"]) | ||
|
|
||
| if __name__ == "__main__": | ||
| logging.basicConfig(level=logging.INFO, | ||
| format=('%(levelname)s|%(asctime)s' | ||
| '|%(pathname)s|%(lineno)d| %(message)s'), | ||
| datefmt='%Y-%m-%dT%H:%M:%S', | ||
| ) | ||
| logging.getLogger().setLevel(logging.INFO) | ||
| pytest.main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,7 @@ | ||
| import datetime | ||
| import logging | ||
| import os | ||
| import subprocess | ||
| import tempfile | ||
| import uuid | ||
| from retrying import retry | ||
|
|
@@ -9,17 +10,44 @@ | |
|
|
||
| from kubeflow.testing import util | ||
|
|
||
| # We need to use retry builds because when building in the test cluster | ||
| # we see intermittent failures pulling dependencies | ||
| @retry(stop_max_attempt_number=7) | ||
| def build(build_dir): | ||
| util.run(["make", "build-kfctl"], cwd=build_dir) | ||
| def run_with_retries(*args, **kwargs): | ||
| util.run(*args, **kwargs) | ||
|
|
||
| def test_build_kfctl_go(): | ||
| def test_build_kfctl_go(app_path, project): | ||
| if not app_path: | ||
| logging.info("--app_path not specified") | ||
| stamp = datetime.datetime.now().strftime("%H%M") | ||
| app_path = os.path.join(tempfile.gettempdir(), | ||
| "kfctl-{0}-{1}".format(stamp, | ||
| uuid.uuid4().hex[0:4])) | ||
| logging.info("Using app path %s", app_path) | ||
| this_dir = os.path.dirname(__file__) | ||
| root = os.path.abspath(os.path.join(this_dir, "..", "..")) | ||
| build_dir = os.path.join(root, "bootstrap") | ||
| build(build_dir) | ||
|
|
||
| # We need to use retry builds because when building in the test cluster | ||
| # we see intermittent failures pulling dependencies | ||
| run_with_retries(["make", "build-kfctl"], cwd=build_dir) | ||
| kfctl_path = os.path.join(build_dir, "bin", "kfctl") | ||
|
|
||
| # We don't want the password to show up in the logs because the logs | ||
| # are public. So we use subprocess and not util.run | ||
| subprocess.check_call([kfctl_path, "init", app_path, "-V", "--platform=gcp", | ||
| "--use_basic_auth", "--skip-init-gcp-project", | ||
| "--project=" + project, | ||
| "--basic_auth_username=kf-test-user", | ||
| "--basic_auth_password=" + uuid.uuid4().hex]) | ||
|
|
||
| # TODO(jlewi): We need to specify a valid email otherwise we get an error | ||
| # when trying to apply the IAM policy. | ||
| run_with_retries([kfctl_path, "generate", "-V", "all", | ||
| "[email protected]"], | ||
| cwd=app_path) | ||
|
|
||
| # We need to use retries because if we don't we see random failures | ||
| # where kfctl just appears to die. | ||
| run_with_retries([kfctl_path, "apply", "-V", "all"], cwd=app_path) | ||
|
|
||
| if __name__ == "__main__": | ||
| logging.basicConfig(level=logging.INFO, | ||
|
|
||
Oops, something went wrong.