diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index f822529c..37ae2b91 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -26,10 +26,10 @@ blocks: - name: docker prologue: commands: + - sem-version python 3.9 - checkout - docker login --username "${DOCKER_USERNAME}" --password-stdin <<< "${DOCKER_PASSWORD}" - - export PATH="$PATH:$HOME/.local/bin" - - pip install --user .[ci] + - pip install .[ci] jobs: - name: "Unit tests + docker build" commands: @@ -44,9 +44,11 @@ blocks: task: prologue: commands: + - sem-version python 3.9 - checkout + - pip install .[ci] + - mkdir -p "$HOME/.local/bin" - export PATH="$PATH:$HOME/.local/bin" - - pip install --user .[ci] - curl -Lo $HOME/.local/bin/kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 && chmod +x $HOME/.local/bin/kind - cache restore "${SEMAPHORE_PROJECT_NAME}-${SEMAPHORE_WORKFLOW_ID}-build" jobs: diff --git a/Dockerfile b/Dockerfile index 9ac673d3..4af5531a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM python:2-alpine3.8 as common +FROM python:3.9-alpine3.16 as common MAINTAINER fiaas@googlegroups.com # Install any binary package dependencies here RUN apk --no-cache add \ @@ -29,9 +29,6 @@ COPY . /fiaas-deploy-daemon COPY .wheel_cache/*.whl /links/ WORKDIR /fiaas-deploy-daemon -# setuptools needs to be upgraded in order to perform the build of the wheels -# Note: setuptools 45.0.0 will drop the support for python 2 -RUN pip install -U setuptools==44.1.1 RUN pip wheel . --no-cache-dir --wheel-dir=/wheels/ --find-links=/links/ FROM common as production diff --git a/README.md b/README.md index 1e919c5d..dad01476 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,6 @@ Useful resources: - http://docs.python-guide.org/ - http://pytest.org/ -- http://www.voidspace.org.uk/python/mock - http://flask.pocoo.org/ Running fiaas-deploy-daemon against a local `minikube` or `kind` backed Kubernetes cluster diff --git a/bin/ingress_check.py b/bin/ingress_check.py index 42f4ba60..e1e460a7 100755 --- a/bin/ingress_check.py +++ b/bin/ingress_check.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals, print_function + import base64 import logging diff --git a/fiaas_deploy_daemon/__init__.py b/fiaas_deploy_daemon/__init__.py index 4a1c7122..3f52d231 100644 --- a/fiaas_deploy_daemon/__init__.py +++ b/fiaas_deploy_daemon/__init__.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import logging import os @@ -22,7 +22,7 @@ import sys import threading import traceback -from Queue import Queue +from queue import Queue import pinject import requests @@ -127,7 +127,7 @@ def thread_dump_logger(log): def _dump_threads(signum, frame): log.info("Received signal %s, dumping thread stacks", signum) thread_names = {t.ident: t.name for t in threading.enumerate()} - for thread_ident, frame in sys._current_frames().items(): + for thread_ident, frame in list(sys._current_frames().items()): log.info("Thread ident=0x%x name=%s", thread_ident, thread_names.get(thread_ident, "unknown")) log.info("".join(traceback.format_stack(frame))) diff --git a/fiaas_deploy_daemon/bootstrap/__init__.py b/fiaas_deploy_daemon/bootstrap/__init__.py index 4600bc21..4855a8b6 100644 --- a/fiaas_deploy_daemon/bootstrap/__init__.py +++ b/fiaas_deploy_daemon/bootstrap/__init__.py @@ -14,11 +14,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import logging import sys -from Queue import Queue +from queue import Queue import pinject import requests diff --git a/fiaas_deploy_daemon/bootstrap/bootstrapper.py b/fiaas_deploy_daemon/bootstrap/bootstrapper.py index bee8be39..0b5f3dd0 100644 --- a/fiaas_deploy_daemon/bootstrap/bootstrapper.py +++ b/fiaas_deploy_daemon/bootstrap/bootstrapper.py @@ -14,14 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import import logging import threading import time from blinker import signal -from monotonic import monotonic as time_monotonic +from time import monotonic as time_monotonic from yaml import YAMLError from ..config import InvalidConfigurationException @@ -48,10 +47,10 @@ def store_status(self, status, app_name, namespace): def values(self): with self._statuses_lock: - return self._statuses.values() + return list(self._statuses.values()) def items(self): - for key, status in self._statuses.iteritems(): + for key, status in self._statuses.items(): name, namespace = key yield name, namespace, status @@ -131,15 +130,15 @@ def _store_status(self, sender, status, subject): def _wait_for_readiness(self, wait_time_seconds, timeout_seconds): start = time_monotonic() while time_monotonic() < (start + timeout_seconds): - if all(status == STATUS_SUCCESS for status in self._status_collector.values()): - LOG.info("Bootstrapped {} applications".format(len(self._status_collector.values()))) + if all(status == STATUS_SUCCESS for status in list(self._status_collector.values())): + LOG.info("Bootstrapped {} applications".format(len(list(self._status_collector.values())))) return True else: time.sleep(wait_time_seconds) message = "Timed out after waiting {}s for applications to become ready.\n".format(timeout_seconds) message += "Applications which failed to become ready:\n" - for name, namespace, status in self._status_collector.items(): + for name, namespace, status in list(self._status_collector.items()): message += "{} in namespace {} had final state {}\n".format(name, namespace, status) LOG.error(message) return False diff --git a/fiaas_deploy_daemon/config.py b/fiaas_deploy_daemon/config.py index f58e9a02..0f6bc71a 100644 --- a/fiaas_deploy_daemon/config.py +++ b/fiaas_deploy_daemon/config.py @@ -209,7 +209,7 @@ def _parse_args(self, args): action="store_true") parser.add_argument("--deployment-max-surge", help="maximum number of extra pods that can be scheduled above the desired " "number of pods during an update", - default=u"25%", type=_int_or_unicode) + default="25%", type=_int_or_unicode) parser.add_argument("--deployment-max-unavailable", help="The maximum number of pods that can be unavailable during an update", default="0", type=_int_or_unicode) parser.add_argument("--enable-deprecated-tls-entry-per-host", help=TLS_ENTRY_PER_HOST_HELP, @@ -360,4 +360,4 @@ def _int_or_unicode(arg): try: return int(arg) except ValueError: - return unicode(arg) + return str(arg) diff --git a/fiaas_deploy_daemon/crd/__init__.py b/fiaas_deploy_daemon/crd/__init__.py index ceec2da4..9dc14bd0 100644 --- a/fiaas_deploy_daemon/crd/__init__.py +++ b/fiaas_deploy_daemon/crd/__init__.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import pinject diff --git a/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1.py b/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1.py index 5bce89b6..1474c5f4 100644 --- a/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1.py +++ b/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import import logging diff --git a/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1beta1.py b/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1beta1.py index c577a1a4..5fb4e108 100644 --- a/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1beta1.py +++ b/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1beta1.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import import logging diff --git a/fiaas_deploy_daemon/crd/status.py b/fiaas_deploy_daemon/crd/status.py index 05684468..a4dde998 100644 --- a/fiaas_deploy_daemon/crd/status.py +++ b/fiaas_deploy_daemon/crd/status.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import import logging import struct @@ -22,6 +21,7 @@ import pytz from blinker import signal +from py27hash.hash import hash27 from k8s.client import NotFound from k8s.models.common import ObjectMeta, OwnerReference @@ -85,7 +85,7 @@ def _save_status(result, subject): def _get_logs(app_name, namespace, deployment_id, result): - return get_running_logs(app_name, namespace, deployment_id) if result in [u"RUNNING", u"INITIATED"] else \ + return get_running_logs(app_name, namespace, deployment_id) if result in ["RUNNING", "INITIATED"] else \ get_final_logs(app_name, namespace, deployment_id) @@ -110,7 +110,7 @@ def create_name(name, deployment_id): By convention, the names of Kubernetes resources should be up to maximum length of 253 characters and consist of lower case alphanumeric characters, '-', and '.'. """ - suffix = b32encode(struct.pack('q', hash(deployment_id))).lower().strip("=") + suffix = b32encode(struct.pack('q', hash27(deployment_id))).decode("utf-8").lower().strip("=") return "{}-{}".format(name, suffix) diff --git a/fiaas_deploy_daemon/crd/types.py b/fiaas_deploy_daemon/crd/types.py index 0a359bd8..3ef8f7da 100644 --- a/fiaas_deploy_daemon/crd/types.py +++ b/fiaas_deploy_daemon/crd/types.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import six from k8s.base import Model diff --git a/fiaas_deploy_daemon/crd/watcher.py b/fiaas_deploy_daemon/crd/watcher.py index 86a0ff89..5773d422 100644 --- a/fiaas_deploy_daemon/crd/watcher.py +++ b/fiaas_deploy_daemon/crd/watcher.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import import logging diff --git a/fiaas_deploy_daemon/deployer/__init__.py b/fiaas_deploy_daemon/deployer/__init__.py index 487e1b91..b470c1de 100644 --- a/fiaas_deploy_daemon/deployer/__init__.py +++ b/fiaas_deploy_daemon/deployer/__init__.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + from collections import namedtuple diff --git a/fiaas_deploy_daemon/deployer/deploy.py b/fiaas_deploy_daemon/deployer/deploy.py index e7bd63a7..10395730 100644 --- a/fiaas_deploy_daemon/deployer/deploy.py +++ b/fiaas_deploy_daemon/deployer/deploy.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import import logging diff --git a/fiaas_deploy_daemon/deployer/kubernetes/__init__.py b/fiaas_deploy_daemon/deployer/kubernetes/__init__.py index c983326c..853d0d44 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/__init__.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/__init__.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import pinject diff --git a/fiaas_deploy_daemon/deployer/kubernetes/adapter.py b/fiaas_deploy_daemon/deployer/kubernetes/adapter.py index c839a03d..97ea6521 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/adapter.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/adapter.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals + import logging diff --git a/fiaas_deploy_daemon/deployer/kubernetes/autoscaler.py b/fiaas_deploy_daemon/deployer/kubernetes/autoscaler.py index 8aa78f21..33cc0b8f 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/autoscaler.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/autoscaler.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import logging @@ -41,7 +41,7 @@ def deploy(self, app_spec, labels): custom_labels = merge_dicts(app_spec.labels.horizontal_pod_autoscaler, labels) metadata = ObjectMeta(name=app_spec.name, namespace=app_spec.namespace, labels=custom_labels, annotations=app_spec.annotations.horizontal_pod_autoscaler) - scale_target_ref = CrossVersionObjectReference(kind=u"Deployment", name=app_spec.name, apiVersion="apps/v1") + scale_target_ref = CrossVersionObjectReference(kind="Deployment", name=app_spec.name, apiVersion="apps/v1") spec = HorizontalPodAutoscalerSpec(scaleTargetRef=scale_target_ref, minReplicas=app_spec.autoscaler.min_replicas, maxReplicas=app_spec.autoscaler.max_replicas, diff --git a/fiaas_deploy_daemon/deployer/kubernetes/deployment/__init__.py b/fiaas_deploy_daemon/deployer/kubernetes/deployment/__init__.py index 06da3f0d..45e0c175 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/deployment/__init__.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/deployment/__init__.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import pinject as pinject diff --git a/fiaas_deploy_daemon/deployer/kubernetes/deployment/deployer.py b/fiaas_deploy_daemon/deployer/kubernetes/deployment/deployer.py index 26fe50fd..bf688f06 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/deployment/deployer.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/deployment/deployer.py @@ -172,7 +172,7 @@ def _make_env(self, app_spec): # fiaas_managed_env overrides global_env overrides legacy_fiaas_env static_env = merge_dicts(self._legacy_fiaas_env, self._global_env, fiaas_managed_env) - env = [EnvVar(name=name, value=value) for name, value in static_env.items()] + env = [EnvVar(name=name, value=value) for name, value in list(static_env.items())] # FIAAS managed environment variables using the downward API env.extend([ @@ -222,7 +222,7 @@ def _make_probe(check_spec): if check_spec.http: probe.httpGet = HTTPGetAction(path=check_spec.http.path, port=check_spec.http.port, httpHeaders=[HTTPHeader(name=name, value=value) - for name, value in check_spec.http.http_headers.items()]) + for name, value in list(check_spec.http.http_headers.items())]) elif check_spec.tcp: probe.tcpSocket = TCPSocketAction(port=check_spec.tcp.port) elif check_spec.execute: @@ -254,5 +254,5 @@ def _build_global_env(global_env): global_env key/value are added as is and with the key prefix FIAAS_ """ _global_env_copy = global_env.copy() - _global_env_copy.update({'FIAAS_{}'.format(k): v for k, v in _global_env_copy.items()}) + _global_env_copy.update({'FIAAS_{}'.format(k): v for k, v in list(_global_env_copy.items())}) return _global_env_copy diff --git a/fiaas_deploy_daemon/deployer/kubernetes/deployment/secrets.py b/fiaas_deploy_daemon/deployer/kubernetes/deployment/secrets.py index 4c68894d..fcb83888 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/deployment/secrets.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/deployment/secrets.py @@ -189,7 +189,7 @@ def _make_secrets_init_container(self, app_spec, image, env_vars=None): if env_vars is None: env_vars = {} env_vars.update({"K8S_DEPLOYMENT": app_spec.name}) - environment = [EnvVar(name=k, value=v) for k, v in env_vars.items()] + environment = [EnvVar(name=k, value=v) for k, v in list(env_vars.items())] container = Container(name=self.SECRETS_INIT_CONTAINER_NAME, image=image, imagePullPolicy="IfNotPresent", diff --git a/fiaas_deploy_daemon/deployer/kubernetes/ingress.py b/fiaas_deploy_daemon/deployer/kubernetes/ingress.py index f111f50c..f00842c3 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/ingress.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/ingress.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import base64 import hashlib @@ -35,8 +35,8 @@ def __init__(self, config, default_app_spec, ingress_adapter): self._host_rewrite_rules = config.host_rewrite_rules self._ingress_adapter = ingress_adapter self._tls_issuer_type_default = config.tls_certificate_issuer_type_default - self._tls_issuer_type_overrides = sorted(config.tls_certificate_issuer_type_overrides.iteritems(), - key=lambda (k, v): len(k), reverse=True) + self._tls_issuer_type_overrides = sorted(iter(config.tls_certificate_issuer_type_overrides.items()), + key=lambda k_v: len(k_v[0]), reverse=True) def deploy(self, app_spec, labels): if self._should_have_ingress(app_spec): @@ -133,13 +133,13 @@ def _group_ingresses(self, app_spec): else: default_ingress.ingress_items.append(ingress_item) - ingresses.extend(i for i in override_issuer_ingresses.values()) + ingresses.extend(i for i in list(override_issuer_ingresses.values())) return ingresses def _generate_default_hosts(self, name): for suffix in self._ingress_suffixes: - yield u"{}.{}".format(name, suffix) + yield "{}.{}".format(name, suffix) def _apply_host_rewrite_rules(self, host): for rule in self._host_rewrite_rules: @@ -164,7 +164,7 @@ def _has_explicitly_set_host(ingress_items): def _has_http_port(app_spec): - return any(port.protocol == u"http" for port in app_spec.ports) + return any(port.protocol == "http" for port in app_spec.ports) def _has_ingress(app_spec): @@ -194,7 +194,7 @@ def apply(self, ingress, app_spec, hosts, issuer_type, use_suffixes=True): issuer = app_spec.ingress_tls.certificate_issuer if app_spec.ingress_tls.certificate_issuer else self._cert_issuer tls_annotations[issuer_type] = issuer else: - tls_annotations[u"kubernetes.io/tls-acme"] = u"true" + tls_annotations["kubernetes.io/tls-acme"] = "true" ingress.metadata.annotations = merge_dicts( ingress.metadata.annotations if ingress.metadata.annotations else {}, tls_annotations @@ -231,9 +231,9 @@ def _should_have_ingress_tls(self, app_spec): def _generate_short_host(self, app_spec): h = hashlib.sha1() - h.update(app_spec.name) - h.update(app_spec.namespace) - prefix = base64.b32encode(h.digest()).strip("=").lower() + h.update(app_spec.name.encode("utf-8")) + h.update(app_spec.namespace.encode("utf-8")) + prefix = base64.b32encode(h.digest()).decode("utf-8").strip("=").lower() short_prefix = prefix[:62 - len(self._shortest_suffix)] short_name = "{}.{}".format(short_prefix, self._shortest_suffix) if len(short_name) > 63 or short_name[0] == ".": diff --git a/fiaas_deploy_daemon/deployer/kubernetes/ingress_networkingv1.py b/fiaas_deploy_daemon/deployer/kubernetes/ingress_networkingv1.py index 82299bed..07418ae1 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/ingress_networkingv1.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/ingress_networkingv1.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + from k8s.client import NotFound from k8s.models.common import ObjectMeta @@ -37,7 +37,7 @@ def __init__(self, ingress_tls_deployer, owner_references, extension_hook): @retry_on_upsert_conflict def create_ingress(self, app_spec, annotated_ingress, labels): default_annotations = { - u"fiaas/expose": u"true" if annotated_ingress.explicit_host else u"false" + "fiaas/expose": "true" if annotated_ingress.explicit_host else "false" } annotations = merge_dicts(app_spec.annotations.ingress, annotated_ingress.annotations, default_annotations) diff --git a/fiaas_deploy_daemon/deployer/kubernetes/ingress_v1beta1.py b/fiaas_deploy_daemon/deployer/kubernetes/ingress_v1beta1.py index e1d7b44e..6d8cf11d 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/ingress_v1beta1.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/ingress_v1beta1.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + from k8s.client import NotFound from k8s.models.common import ObjectMeta @@ -36,7 +36,7 @@ def __init__(self, ingress_tls_deployer, owner_references, extension_hook): @retry_on_upsert_conflict def create_ingress(self, app_spec, annotated_ingress, labels): default_annotations = { - u"fiaas/expose": u"true" if annotated_ingress.explicit_host else u"false" + "fiaas/expose": "true" if annotated_ingress.explicit_host else "false" } annotations = merge_dicts(app_spec.annotations.ingress, annotated_ingress.annotations, default_annotations) diff --git a/fiaas_deploy_daemon/deployer/kubernetes/ready_check.py b/fiaas_deploy_daemon/deployer/kubernetes/ready_check.py index 5d3dc831..6b411e48 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/ready_check.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/ready_check.py @@ -22,7 +22,7 @@ from k8s.client import NotFound from k8s.models.certificate import Certificate from k8s.models.deployment import Deployment -from monotonic import monotonic as time_monotonic +from time import monotonic as time_monotonic LOG = logging.getLogger(__name__) diff --git a/fiaas_deploy_daemon/deployer/kubernetes/service.py b/fiaas_deploy_daemon/deployer/kubernetes/service.py index 18fef4be..d7c8246c 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/service.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/service.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import logging @@ -87,7 +87,7 @@ def _make_service_port(port_spec): @staticmethod def _make_tcp_port_annotation(app_spec): tcp_port_names = [port_spec.name for port_spec in app_spec.ports - if port_spec.protocol == u"tcp"] + if port_spec.protocol == "tcp"] return { 'fiaas/tcp_port_names': ','.join(map(str, tcp_port_names)) } if tcp_port_names else {} diff --git a/fiaas_deploy_daemon/deployer/kubernetes/service_account.py b/fiaas_deploy_daemon/deployer/kubernetes/service_account.py index 1df7ee18..c43ac0f5 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/service_account.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/service_account.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import logging diff --git a/fiaas_deploy_daemon/deployer/scheduler.py b/fiaas_deploy_daemon/deployer/scheduler.py index 23daf466..ad813082 100644 --- a/fiaas_deploy_daemon/deployer/scheduler.py +++ b/fiaas_deploy_daemon/deployer/scheduler.py @@ -15,9 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import time -from Queue import PriorityQueue +from queue import PriorityQueue -from monotonic import monotonic as time_monotonic +from time import monotonic as time_monotonic from ..base_thread import DaemonThread diff --git a/fiaas_deploy_daemon/extension_hook_caller.py b/fiaas_deploy_daemon/extension_hook_caller.py index ec2f0aef..8c9dd9b3 100644 --- a/fiaas_deploy_daemon/extension_hook_caller.py +++ b/fiaas_deploy_daemon/extension_hook_caller.py @@ -17,7 +17,7 @@ import json import logging import posixpath -import urlparse +import urllib.parse LOG = logging.getLogger(__name__) @@ -31,7 +31,7 @@ def __init__(self, config, session): def apply(self, obj, app_spec): if self._url is None: return obj - url = urlparse.urljoin(self._url, "fiaas/deploy/") + url = urllib.parse.urljoin(self._url, "fiaas/deploy/") url = posixpath.join(url, type(obj).__name__) dump = json.dumps({"object": obj.as_dict(), "application": app_spec.app_config}) response = self._session.post( diff --git a/fiaas_deploy_daemon/log_extras.py b/fiaas_deploy_daemon/log_extras.py index 733b37c6..c6c5b22b 100644 --- a/fiaas_deploy_daemon/log_extras.py +++ b/fiaas_deploy_daemon/log_extras.py @@ -21,8 +21,8 @@ _LOGS = defaultdict(list) _LOG_EXTRAS = threading.local() -_LOG_FORMAT = u"[%(asctime)s|%(levelname)7s] %(message)s " \ - u"[%(name)s|%(threadName)s|%(extras_namespace)s/%(extras_app_name)s]" +_LOG_FORMAT = "[%(asctime)s|%(levelname)7s] %(message)s " \ + "[%(name)s|%(threadName)s|%(extras_namespace)s/%(extras_app_name)s]" class ExtraFilter(logging.Filter): diff --git a/fiaas_deploy_daemon/logsetup.py b/fiaas_deploy_daemon/logsetup.py index a4e9b729..27ad7d5a 100644 --- a/fiaas_deploy_daemon/logsetup.py +++ b/fiaas_deploy_daemon/logsetup.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import datetime import json @@ -44,7 +44,7 @@ def format(self, record): fields["extras"] = getattr(record, "extras", {}) if "exc_info" in fields and fields["exc_info"]: fields["throwable"] = self.formatException(fields["exc_info"]) - for original, replacement in self.RENAME.iteritems(): + for original, replacement in self.RENAME.items(): fields[replacement] = fields.pop(original) for unwanted in self.UNWANTED: fields.pop(unwanted) diff --git a/fiaas_deploy_daemon/retry.py b/fiaas_deploy_daemon/retry.py index b13543c1..1b120482 100644 --- a/fiaas_deploy_daemon/retry.py +++ b/fiaas_deploy_daemon/retry.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import functools -import inspect -import sys import backoff from k8s.client import ClientError @@ -36,9 +34,7 @@ class UpsertConflict(Exception): - def __init__(self, cause, response): - self.traceback = sys.exc_info() - super(self.__class__, self).__init__(cause.message) + def __init__(self, response): self.response = response def __str__(self): @@ -67,13 +63,7 @@ def _count_failure(target, *args, **kwargs): def canonical_name(func): - if inspect.ismethod(func): - # method's class is im_class, classmethod's class is im_self - method_cls = func.im_class if func.im_self is None else func.im_self - for cls in inspect.getmro(method_cls): - if func.__name__ in cls.__dict__: - return "{}.{}.{}".format(cls.__module__, cls.__name__, func.__name__) - return "{}.{}".format(func.__module__, func.__name__) + return "{}.{}".format(func.__module__, func.__qualname__) def retry_on_upsert_conflict(_func=None, max_value_seconds=CONFLICT_MAX_VALUE, max_tries=CONFLICT_MAX_RETRIES): @@ -91,7 +81,7 @@ def _wrap(*args, **kwargs): return func(*args, **kwargs) except ClientError as e: if e.response.status_code == 409: # Conflict - raise UpsertConflict(e, e.response) + raise UpsertConflict(e.response) from e else: raise return _wrap diff --git a/fiaas_deploy_daemon/secrets.py b/fiaas_deploy_daemon/secrets.py index 8e2389f0..5e62dc82 100644 --- a/fiaas_deploy_daemon/secrets.py +++ b/fiaas_deploy_daemon/secrets.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals + import os from collections import namedtuple @@ -28,7 +28,7 @@ def resolve_secrets(secrets_directory): filename = field.replace("_", "-") fpath = os.path.join(secrets_directory, filename) if os.path.isfile(fpath): - with open(fpath) as fobj: + with open(fpath, 'rb') as fobj: kwargs[field] = fobj.read().strip() else: kwargs[field] = None diff --git a/fiaas_deploy_daemon/specs/__init__.py b/fiaas_deploy_daemon/specs/__init__.py index 5e8725ea..f7015c07 100644 --- a/fiaas_deploy_daemon/specs/__init__.py +++ b/fiaas_deploy_daemon/specs/__init__.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import importlib import pkgutil diff --git a/fiaas_deploy_daemon/specs/factory.py b/fiaas_deploy_daemon/specs/factory.py index 0db0a4a9..06ddf358 100644 --- a/fiaas_deploy_daemon/specs/factory.py +++ b/fiaas_deploy_daemon/specs/factory.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import logging @@ -28,13 +28,13 @@ def __init__(self, factory, transformers, config): self._factory = factory self._transformers = transformers self._config = config - self._supported_versions = [factory.version] + transformers.keys() + self._supported_versions = [factory.version] + list(transformers.keys()) self._fiaas_counter = Counter("fiaas_yml_version", "The version of fiaas.yml used", ["version", "app_name"]) def __call__(self, uid, name, image, app_config, teams, tags, deployment_id, namespace, additional_labels, additional_annotations): """Create an app_spec from app_config""" - fiaas_version = app_config.get(u"version", 1) + fiaas_version = app_config.get("version", 1) self._fiaas_counter.labels(fiaas_version, name).inc() LOG.info("Attempting to create app_spec for %s from fiaas.yml version %s", name, fiaas_version) try: @@ -49,14 +49,14 @@ def __call__(self, uid, name, image, app_config, teams, tags, deployment_id, nam return app_spec def transform(self, app_config, strip_defaults=False): - fiaas_version = app_config.get(u"version", 1) + fiaas_version = app_config.get("version", 1) if fiaas_version not in self._supported_versions: raise InvalidConfiguration("Requested version %s, but the only supported versions are: %r" % (fiaas_version, self._supported_versions)) current_version = fiaas_version while current_version < self._factory.version: app_config = self._transformers[current_version](app_config, strip_defaults=strip_defaults) - current_version = app_config.get(u"version", 1) + current_version = app_config.get("version", 1) return app_config def _validate(self, app_spec): diff --git a/fiaas_deploy_daemon/specs/lookup.py b/fiaas_deploy_daemon/specs/lookup.py index 81a08548..c6e18495 100644 --- a/fiaas_deploy_daemon/specs/lookup.py +++ b/fiaas_deploy_daemon/specs/lookup.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import collections -from itertools import izip_longest +from itertools import zip_longest from .factory import InvalidConfiguration @@ -91,7 +91,7 @@ def _get_value(self, col, idx): def __eq__(self, other): if not isinstance(other, collections.Sequence): return NotImplemented - for self_i, other_i in izip_longest(self, other, fillvalue=object()): + for self_i, other_i in zip_longest(self, other, fillvalue=object()): if self_i != other_i: return False return True diff --git a/fiaas_deploy_daemon/specs/v2/__init__.py b/fiaas_deploy_daemon/specs/v2/__init__.py index 35a5a81b..1fe4f7a8 100644 --- a/fiaas_deploy_daemon/specs/v2/__init__.py +++ b/fiaas_deploy_daemon/specs/v2/__init__.py @@ -14,6 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + from .transformer import Transformer # NOQA diff --git a/fiaas_deploy_daemon/specs/v2/transformer.py b/fiaas_deploy_daemon/specs/v2/transformer.py index 90396ad7..0859eb51 100644 --- a/fiaas_deploy_daemon/specs/v2/transformer.py +++ b/fiaas_deploy_daemon/specs/v2/transformer.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals + import collections import pkgutil @@ -70,7 +70,7 @@ def __call__(self, app_config, strip_defaults=False): "readiness": readiness, } } - for old, new in Transformer.COPY_MAPPING.iteritems(): + for old, new in Transformer.COPY_MAPPING.items(): value = _get(lookup, old) _set(new_config, new, value) if lookup["autoscaler"]["enabled"]: @@ -100,11 +100,11 @@ def _strip_v3_defaults(self, app_config): pass return dict( [("version", app_config["version"])] + - _remove_intersect(app_config, v3defaults).items()) + list(_remove_intersect(app_config, v3defaults).items())) @staticmethod def _health_check(lookup, ports_lookup): - value = {key: value for key, value in lookup.iteritems() if key not in ("execute", "http", "tcp")} + value = {key: value for key, value in lookup.items() if key not in ("execute", "http", "tcp")} for check in ("execute", "http", "tcp"): if lookup.get_config_value(check): value[check] = lookup[check] @@ -175,12 +175,12 @@ def _set(d, keys, value): def _flatten(d): if isinstance(d, collections.Mapping): - return {k: _flatten(v) for k, v in d.items()} + return {k: _flatten(v) for k, v in list(d.items())} return d def _remove_intersect(dict1, dict2): - map(dict1.pop, filter(lambda k: k in dict2 and dict2[k] == dict1[k], dict1)) + list(map(dict1.pop, [k for k in dict1 if k in dict2 and dict2[k] == dict1[k]])) keys_dict1 = set(dict1.keys()) keys_dict2 = set(dict2.keys()) for k in keys_dict1 & keys_dict2: @@ -188,7 +188,7 @@ def _remove_intersect(dict1, dict2): dict1[k].update(_remove_intersect(dict1[k], dict2[k])) elif all(_single_dict_list(x) for x in [dict1[k], dict2[k]]): dict1[k] = [item for item in [_remove_intersect(x, y) for x, y in zip(dict1[k], dict2[k])] if item] - map(dict1.pop, filter(lambda k: not dict1[k], dict1)) + list(map(dict1.pop, [k for k in dict1 if not dict1[k]])) return dict1 diff --git a/fiaas_deploy_daemon/specs/v3/__init__.py b/fiaas_deploy_daemon/specs/v3/__init__.py index 5eb1ef63..608b9765 100644 --- a/fiaas_deploy_daemon/specs/v3/__init__.py +++ b/fiaas_deploy_daemon/specs/v3/__init__.py @@ -14,6 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + from .factory import Factory # NOQA diff --git a/fiaas_deploy_daemon/specs/v3/factory.py b/fiaas_deploy_daemon/specs/v3/factory.py index 7589b0c9..78b3705b 100644 --- a/fiaas_deploy_daemon/specs/v3/factory.py +++ b/fiaas_deploy_daemon/specs/v3/factory.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals + import pkgutil @@ -41,7 +41,7 @@ def __call__(self, uid, name, image, teams, tags, app_config, deployment_id, nam additional_labels, additional_annotations): if app_config.get("extensions") and app_config["extensions"].get("tls") and type( app_config["extensions"]["tls"]) == bool: - app_config["extensions"]["tls"] = {u"enabled": app_config["extensions"]["tls"]} + app_config["extensions"]["tls"] = {"enabled": app_config["extensions"]["tls"]} lookup = LookupMapping(config=app_config, defaults=self._defaults) app_spec = AppSpec( uid=uid, @@ -211,7 +211,7 @@ def resolve_port_number(port): port_number = http_ports.get(port) if port_number: return port_number - elif port in http_ports.values(): + elif port in list(http_ports.values()): return port else: raise InvalidConfiguration("{} is not a valid port name or port number".format(port)) @@ -223,7 +223,7 @@ def ingress_item(host, paths, annotations): ] return IngressItemSpec(host=host, pathmappings=ingress_path_mapping_specs, annotations=annotations) - if len(http_ports.items()) > 0: + if len(list(http_ports.items())) > 0: return [ingress_item(host_path_mapping["host"], host_path_mapping["paths"], host_path_mapping["annotations"]) for host_path_mapping in ingress_lookup] else: @@ -242,7 +242,7 @@ def _strongbox(strongbox_lookup): def _secrets_specs(secrets_lookup): return [SecretsSpec(type=k, parameters=v["parameters"], - annotations=v["annotations"]) for (k, v) in secrets_lookup.iteritems()] + annotations=v["annotations"]) for (k, v) in secrets_lookup.items()] def _get_value(key, overrides): diff --git a/fiaas_deploy_daemon/tools.py b/fiaas_deploy_daemon/tools.py index 0dbfad5c..ae5a04fe 100644 --- a/fiaas_deploy_daemon/tools.py +++ b/fiaas_deploy_daemon/tools.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from Queue import Queue +from queue import Queue from collections import Iterator from k8s import config @@ -38,5 +38,5 @@ def log_request_response(resp, *args, **kwargs): class IterableQueue(Queue, Iterator): - def next(self): + def __next__(self): return self.get() diff --git a/fiaas_deploy_daemon/usage_reporting/dev_hose_auth.py b/fiaas_deploy_daemon/usage_reporting/dev_hose_auth.py index f641b4ae..478c5fb5 100644 --- a/fiaas_deploy_daemon/usage_reporting/dev_hose_auth.py +++ b/fiaas_deploy_daemon/usage_reporting/dev_hose_auth.py @@ -21,7 +21,7 @@ import json import time import uuid -from urllib import quote, quote_plus +from urllib.parse import quote, quote_plus from requests.auth import AuthBase @@ -29,7 +29,7 @@ class DevHoseAuth(AuthBase): def __init__(self, key, tenant): self._key = base64.b64decode(key.strip()) - self._auth_context = base64.b64encode(json.dumps({"type": tenant})) + self._auth_context = base64.b64encode(json.dumps({"type": tenant}).encode("utf-8")).decode("utf-8") def __call__(self, r): timestamp = time.time() @@ -43,8 +43,8 @@ def __call__(self, r): def _calculate_signature(self, r, timestamp, nonce): signature = hmac.new(self._key, digestmod=hashlib.sha256) string_to_sign = self._create_string_to_sign(r, timestamp, nonce) - signature.update(string_to_sign) - return base64.b64encode(signature.digest()) + signature.update(string_to_sign.encode("utf-8")) + return base64.b64encode(signature.digest()).decode("utf-8") def _create_string_to_sign(self, r, timestamp, nonce): return "\n".join(( diff --git a/fiaas_deploy_daemon/usage_reporting/transformer.py b/fiaas_deploy_daemon/usage_reporting/transformer.py index b8df6054..97174d1a 100644 --- a/fiaas_deploy_daemon/usage_reporting/transformer.py +++ b/fiaas_deploy_daemon/usage_reporting/transformer.py @@ -76,7 +76,7 @@ def __call__(self, status, app_name, namespace, deployment_id, repository): details={'environment': self._environment}, trigger=DevhoseDeploymentEventTransformer.FIAAS_TRIGGER, team=self._team) - return event.__dict__ + return event._asdict() def _environment(env): diff --git a/fiaas_deploy_daemon/usage_reporting/usage_reporter.py b/fiaas_deploy_daemon/usage_reporting/usage_reporter.py index 21956077..16b9a557 100644 --- a/fiaas_deploy_daemon/usage_reporting/usage_reporter.py +++ b/fiaas_deploy_daemon/usage_reporting/usage_reporter.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import import collections import logging diff --git a/fiaas_deploy_daemon/web/__init__.py b/fiaas_deploy_daemon/web/__init__.py index 8c82ed94..c8c1371e 100644 --- a/fiaas_deploy_daemon/web/__init__.py +++ b/fiaas_deploy_daemon/web/__init__.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + import logging import pkgutil diff --git a/setup.cfg b/setup.cfg index 01849c05..a91d38d6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [egg_info] -tag_build=-SNAPSHOT +# PEP440 compliant developmental version tag https://peps.python.org/pep-0440/#developmental-releases +tag_build=.dev [aliases] test=pytest diff --git a/setup.py b/setup.py index ec22acd5..f64eba32 100755 --- a/setup.py +++ b/setup.py @@ -31,18 +31,23 @@ def read(filename): "pyaml == 19.4.1", "pinject == 0.14.1", "decorator < 5.0.0", # 5.0.0 and later drops py2 support (transitive dep from pinject) - "six == 1.12.0", + "six >= 1.12.0", "dnspython == 1.16.0", "k8s == 0.21.4", - "monotonic == 1.6", "appdirs == 1.4.3", "requests-toolbelt == 0.9.1", "backoff == 1.8.0", + "py27hash == 1.1.0", ] WEB_REQ = [ + # TODO: upgrade flask and associated dependencies to 2.x "Flask == 1.1.1", "flask-talisman==0.7.0", + "jinja2 <= 2.11.3", # jinja 3.x requires also upgrading flask to 2.x + "markupsafe <= 1.1.1", # markupsafe 2.x requires also upgrading flask to 2.x + "itsdangerous <= 1.1.0", # markupsafe 2.x requires also upgrading flask to 2.x + "werkzeug <= 1.0.1", # markupsafe 2.x requires also upgrading flask to 2.x "blinker == 1.4", ] @@ -59,7 +64,6 @@ def read(filename): ] TESTS_REQ = [ - 'mock == 3.0.5', 'pytest-xdist == 1.27.0', 'pytest-sugar == 0.9.2', 'pytest-html == 1.22.0', diff --git a/tests/conftest.py b/tests/conftest.py index c5d9c982..537efc7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,7 +37,7 @@ def uuid(): def prometheus_registry(): from prometheus_client.core import REGISTRY yield REGISTRY - for c in REGISTRY._collector_to_names.keys(): + for c in list(REGISTRY._collector_to_names.keys()): REGISTRY.unregister(c) @@ -68,8 +68,8 @@ def assert_dicts(actual, expected): try: assert actual == expected - except AssertionError as ae: - raise AssertionError(ae.message + _add_argument_diff(actual, expected)) + except AssertionError: + raise AssertionError(_add_argument_diff(actual, expected)) def _add_useful_error_message(assertion, mockk, first, args): @@ -80,14 +80,14 @@ def _add_useful_error_message(assertion, mockk, first, args): __tracebackhide__ = True try: assertion() - except AssertionError as ae: + except AssertionError: other_calls = [call[0] for call in mockk.call_args_list if (first is None or call[0][0] == first)] if other_calls: extra_info = '\n\nURI {} got the following other calls:\n{}\n'.format(first, '\n'.join( _format_call(call) for call in other_calls)) if len(other_calls) == 1 and len(other_calls[0]) == 2 and args is not None: extra_info += _add_argument_diff(other_calls[0][1], args[0]) - raise AssertionError(ae.message + extra_info) + raise AssertionError(extra_info) else: raise @@ -100,14 +100,14 @@ def _add_argument_diff(actual, expected, indent=0, acc=None): if type(actual) != type(expected): acc.append("{}{!r} {} {!r}".format(" " * indent * 2, actual, "==" if actual == expected else "!=", expected)) elif isinstance(actual, dict): - for k in set(actual.keys() + expected.keys()): + for k in set(list(actual.keys()) + list(expected.keys())): acc.append("{}{}:".format(" " * indent * 2, k)) a = actual.get(k) e = expected.get(k) if a != e: _add_argument_diff(a, e, indent + 1, acc) elif isinstance(actual, list): - for a, e in itertools.izip_longest(actual, expected): + for a, e in itertools.zip_longest(actual, expected): acc.append("{}-".format(" " * indent * 2)) if a != e: _add_argument_diff(a, e, indent + 1, acc) @@ -137,17 +137,45 @@ def _split_scope(self, nodeid): scope = LoadScopeScheduling._split_scope(self, nodeid) else: fixture_values = m.group(1).split("-") - if "test_e2e" in nodeid: - scope = "-".join(fixture_values[:2]) + if "test_e2e.py" in nodeid: + scope = self._select_scope_e2e(nodeid, fixture_values) else: scope = self._select_scope(fixture_values) self._assigned_scope[nodeid] = scope return scope def _select_scope(self, fixture_values): - groups = itertools.izip_longest(fillvalue="", *([iter(fixture_values)] * 3)) + groups = itertools.zip_longest(fillvalue="", *([iter(fixture_values)] * 3)) return "-".join(next(groups)) + def _select_scope_e2e(self, nodeid, fixture_values): + """Kubernetes cluster startup time for the e2e tests in test_e2e.py is significant. To ensure tests that use + the same cluster run on the same worker, group tests from test_e2e.py by the cluster the test needs, by + setting the same scope for tests that use the same cluster. This should avoid two different workers spinning + up the same type of cluster to run tests against separately. + + There are currently 3 cluster types used by the e2e tests; + - kubernetes with NodePort service_type, + - kubernetes with ClusterIP service_type + - kubernetes_service_account + + Scopes: + - group tests with NodePort or ClusterIP in fixture_values to use kubernetes/NodePort or kubernetes/ClusterIP + respectively + - group tests which contain test_custom_resource_definition_deploy_with_service_account together to use + kubernetes_service_account + - if none of those apply, use the previous behavior of grouping by the two first fixture names (this is just + as a fallback and might lead to suboptimal scheduling). + """ + if 'test_custom_resource_definition_deploy_with_service_account' in nodeid: + return 'serviceaccount' + + for service_type in ("NodePort", "ClusterIP"): + if service_type in fixture_values: + return service_type + + return "-".join(fixture_values[:2]) + @pytest.mark.tryfirst def pytest_xdist_make_scheduler(config, log): diff --git a/tests/fiaas_deploy_daemon/conftest.py b/tests/fiaas_deploy_daemon/conftest.py index 1f39296a..0d79b7a3 100644 --- a/tests/fiaas_deploy_daemon/conftest.py +++ b/tests/fiaas_deploy_daemon/conftest.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pytest from k8s import config from k8s.client import NotFound @@ -56,8 +56,8 @@ def app_spec(): readiness=CheckSpec(http=HttpCheckSpec(path="/", port=8080, http_headers={}), tcp=None, execute=None, initial_delay_seconds=10, period_seconds=10, success_threshold=1, failure_threshold=3, timeout_seconds=1)), - teams=[u'foo'], - tags=[u'bar'], + teams=['foo'], + tags=['bar'], deployment_id="test_app_deployment_id", labels=LabelAndAnnotationSpec({}, {}, {}, {}, {}, {}, {}), annotations=LabelAndAnnotationSpec({}, {}, {}, {}, {}, {}, {}), @@ -117,8 +117,8 @@ def app_spec_teams_and_tags(app_spec): return app_spec._replace( ports=None, health_checks=None, - teams=[u'Order Produkt Betaling'], - tags=[u'høyt-i-stacken', u'ad-in', u'Anonnseinnlegging'] + teams=['Order Produkt Betaling'], + tags=['høyt-i-stacken', 'ad-in', 'Anonnseinnlegging'] ) @@ -184,6 +184,6 @@ def _mock_namespace_file_open(name, *args, **kwargs): else: return real_open(name, *args, **kwargs) - with mock.patch("__builtin__.open") as mock_open: + with mock.patch("builtins.open") as mock_open: mock_open.side_effect = _mock_namespace_file_open yield mock_open diff --git a/tests/fiaas_deploy_daemon/crd/test_crd_resources_syncer_apiextensionsv1.py b/tests/fiaas_deploy_daemon/crd/test_crd_resources_syncer_apiextensionsv1.py index 00b463b8..d903540b 100644 --- a/tests/fiaas_deploy_daemon/crd/test_crd_resources_syncer_apiextensionsv1.py +++ b/tests/fiaas_deploy_daemon/crd/test_crd_resources_syncer_apiextensionsv1.py @@ -15,9 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals -import mock +from unittest import mock from requests import Response from k8s.client import NotFound diff --git a/tests/fiaas_deploy_daemon/crd/test_crd_resources_syncer_apiextensionsv1beta1.py b/tests/fiaas_deploy_daemon/crd/test_crd_resources_syncer_apiextensionsv1beta1.py index 792d1460..c949093e 100644 --- a/tests/fiaas_deploy_daemon/crd/test_crd_resources_syncer_apiextensionsv1beta1.py +++ b/tests/fiaas_deploy_daemon/crd/test_crd_resources_syncer_apiextensionsv1beta1.py @@ -15,9 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals -import mock +from unittest import mock from requests import Response from k8s.client import NotFound diff --git a/tests/fiaas_deploy_daemon/crd/test_crd_status.py b/tests/fiaas_deploy_daemon/crd/test_crd_status.py index efacd08b..3cbd3a76 100644 --- a/tests/fiaas_deploy_daemon/crd/test_crd_status.py +++ b/tests/fiaas_deploy_daemon/crd/test_crd_status.py @@ -16,7 +16,7 @@ import re from collections import namedtuple -import mock +from unittest import mock import pytest from blinker import Namespace from k8s.client import ClientError, NotFound @@ -33,8 +33,8 @@ LAST_UPDATE = now() LOG_LINE = "This is a log line from a test." -DEPLOYMENT_ID = u"deployment_id" -NAME = u"name" +DEPLOYMENT_ID = "deployment_id" +NAME = "name" VALID_NAME = re.compile(r"^[a-z0-9.-]+$") @@ -78,10 +78,10 @@ def pytest_generate_tests(self, metafunc): TestData = namedtuple("TestData", ("signal_name", "action", "status", "result", "new", "called_mock", "ignored_mock")) name2result = { - STATUS_STARTED: u"RUNNING", - STATUS_FAILED: u"FAILED", - STATUS_SUCCESS: u"SUCCESS", - STATUS_INITIATED: u"INITIATED" + STATUS_STARTED: "RUNNING", + STATUS_FAILED: "FAILED", + STATUS_SUCCESS: "SUCCESS", + STATUS_INITIATED: "INITIATED" } action2data = { "create": (True, "post", "put"), @@ -186,10 +186,10 @@ def test_action_on_signal(self, request, get, app_spec, test_data, signal): ignored_mock.assert_not_called() @pytest.mark.parametrize("deployment_id", ( - u"fiaas/fiaas-deploy-daemon:latest", - u"1234123", - u"The Ultimate Deployment ID", - u"@${[]}!#%&/()=?" + "fiaas/fiaas-deploy-daemon:latest", + "1234123", + "The Ultimate Deployment ID", + "@${[]}!#%&/()=?" )) def test_create_name(self, deployment_id): final_name = status.create_name(NAME, deployment_id) @@ -279,4 +279,4 @@ def _subject_from_app_spec(app_spec): def _create_status(i, annotate=True): annotations = {LAST_UPDATED_KEY: "2020-12-12T23.59.{:02}".format(i)} if annotate else None metadata = ObjectMeta(name="name-{}".format(i), namespace="test", annotations=annotations) - return FiaasApplicationStatus(new=False, metadata=metadata, result=u"SUCCESS") + return FiaasApplicationStatus(new=False, metadata=metadata, result="SUCCESS") diff --git a/tests/fiaas_deploy_daemon/crd/test_crd_watcher.py b/tests/fiaas_deploy_daemon/crd/test_crd_watcher.py index 003a8fb7..59285f86 100644 --- a/tests/fiaas_deploy_daemon/crd/test_crd_watcher.py +++ b/tests/fiaas_deploy_daemon/crd/test_crd_watcher.py @@ -15,11 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals -from Queue import Queue +from queue import Queue -import mock +from unittest import mock import pytest from k8s.base import WatchEvent from k8s.client import NotFound diff --git a/tests/fiaas_deploy_daemon/crd/test_fiaasapplication.py b/tests/fiaas_deploy_daemon/crd/test_fiaasapplication.py index 43ac12a7..49f724e1 100644 --- a/tests/fiaas_deploy_daemon/crd/test_fiaasapplication.py +++ b/tests/fiaas_deploy_daemon/crd/test_fiaasapplication.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # coding: utf-8 -from __future__ import absolute_import -import mock + +from unittest import mock import pytest from k8s.client import NotFound from k8s.models.common import ObjectMeta diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/conftest.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/conftest.py index 290151e3..8aeb640c 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/conftest.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/conftest.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import pytest -import mock +from unittest import mock from fiaas_deploy_daemon.deployer.kubernetes.owner_references import OwnerReferences diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_datadog.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_datadog.py index 40ba1bcc..90ca1f92 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_datadog.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_datadog.py @@ -16,7 +16,7 @@ # limitations under the License. from copy import deepcopy -import mock +from unittest import mock import pytest from k8s.models.deployment import Deployment, DeploymentSpec from k8s.models.pod import PodTemplateSpec, PodSpec, Container, EnvVar diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_prometheus.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_prometheus.py index db04e1c9..c69eb3af 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_prometheus.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_prometheus.py @@ -16,7 +16,7 @@ # limitations under the License. from copy import deepcopy -import mock +from unittest import mock import pytest from k8s.models.common import ObjectMeta from k8s.models.deployment import Deployment, DeploymentSpec diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_secrets.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_secrets.py index 1ddb90f0..f8ff0a99 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_secrets.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/deployment/test_secrets.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pytest from k8s.models.common import ObjectMeta from k8s.models.deployment import Deployment, DeploymentSpec @@ -355,10 +355,12 @@ def test_environment(self, deployment, app_spec, strongbox_secrets, secrets_spec init_container = deployment.spec.template.spec.initContainers[0] assert init_container is not None - assert 3 == len(init_container.env) - self._assert_env_var(init_container.env[0], "K8S_DEPLOYMENT", app_spec.name) - self._assert_env_var(init_container.env[1], "AWS_REGION", self.AWS_REGION) - self._assert_env_var(init_container.env[2], "SECRET_GROUPS", self.GROUPS) + expected = [ + EnvVar(name="AWS_REGION", value=self.AWS_REGION), + EnvVar(name="SECRET_GROUPS", value=self.GROUPS), + EnvVar(name="K8S_DEPLOYMENT", value=app_spec.name), + ] + assert init_container.env == expected def test_annotations(self, deployment, app_spec, strongbox_secrets, secrets_spec): strongbox_secrets.apply(deployment, app_spec, secrets_spec) @@ -368,10 +370,3 @@ def test_annotations(self, deployment, app_spec, strongbox_secrets, secrets_spec CANARY_NAME: CANARY_VALUE, "iam.amazonaws.com/role": self.IAM_ROLE } - - @staticmethod - def _assert_env_var(env_var, name, value): - __tracebackhide__ = True - - assert name == env_var.name - assert value == env_var.value diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_adapter.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_adapter.py index 69bf5d69..41cedff6 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_adapter.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_adapter.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pytest from k8s.models.common import ObjectMeta from k8s.models.resourcequota import ResourceQuota, ResourceQuotaSpec, NotBestEffort, BestEffort @@ -27,8 +27,8 @@ from fiaas_deploy_daemon.specs.models import ResourcesSpec, ResourceRequirementSpec FIAAS_VERSION = "1" -TEAMS = u"foo" -TAGS = u"bar" +TEAMS = "foo" +TAGS = "bar" class TestK8s(object): diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_autoscaler.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_autoscaler.py index e7755ad0..89ed1bc5 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_autoscaler.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_autoscaler.py @@ -16,7 +16,7 @@ # limitations under the License. import pytest from k8s.models.autoscaler import HorizontalPodAutoscaler -from mock import create_autospec +from unittest.mock import create_autospec from requests import Response from utils import TypeMatcher diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_deployment_deploy.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_deployment_deploy.py index f45d8546..cbe9c710 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_deployment_deploy.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_deployment_deploy.py @@ -17,10 +17,10 @@ from collections import defaultdict import enum -import mock +from unittest import mock import pytest from k8s.models.deployment import Deployment -from mock import create_autospec +from unittest.mock import create_autospec from requests import Response from fiaas_deploy_daemon.extension_hook_caller import ExtensionHookCaller @@ -127,7 +127,7 @@ def config(self, request, environment): config.pre_stop_delay = 1 config.log_format = "json" config.use_in_memory_emptydirs = Feature.USE_IN_MEMORY_EMPTYDIRS in features - config.deployment_max_surge = u"25%" + config.deployment_max_surge = "25%" config.deployment_max_unavailable = 0 config.extension_hook_url = None config.disable_deprecated_managed_env_vars = Feature.DISABLE_DEPRECATED_MANAGED_ENV_VARS in features @@ -444,7 +444,7 @@ def create_expected_deployment(config, } if add_init_container_annotations else {} pod_annotations = _none_if_empty(merge_dicts(pod_annotations, init_container_annotations)) - max_surge = u"25%" + max_surge = "25%" max_unavailable = 0 if app_spec.autoscaler.max_replicas == 1 and app_spec.singleton: max_surge = 0 @@ -515,9 +515,9 @@ def create_environment_variables(config, global_env=None, version="version", env if global_env: _env_variables.update(global_env) - _env_variables.update({"FIAAS_{}".format(k): v for k, v in global_env.items()}) + _env_variables.update({"FIAAS_{}".format(k): v for k, v in list(global_env.items())}) - env = [{'name': k, 'value': v} for k, v in _env_variables.items()] + env = [{'name': k, 'value': v} for k, v in list(_env_variables.items())] env.append({'name': 'FIAAS_REQUESTS_CPU', 'valueFrom': {'resourceFieldRef': { 'containerName': 'testapp', 'resource': 'requests.cpu', 'divisor': 1}}}) diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_ingress_deploy.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_ingress_deploy.py index f76ec3fe..793f0143 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_ingress_deploy.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_ingress_deploy.py @@ -14,11 +14,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pytest from k8s.models.common import ObjectMeta from k8s.models.ingress import Ingress, IngressTLS -from mock import create_autospec, Mock +from unittest.mock import create_autospec, Mock from requests import Response from fiaas_deploy_daemon import ExtensionHookCaller @@ -62,8 +62,8 @@ def app_spec(**kwargs): readiness=CheckSpec(http=HttpCheckSpec(path="/", port=8080, http_headers={}), tcp=None, execute=None, initial_delay_seconds=10, period_seconds=10, success_threshold=1, failure_threshold=3, timeout_seconds=1)), - teams=[u'foo'], - tags=[u'bar'], + teams=['foo'], + tags=['bar'], deployment_id="test_app_deployment_id", labels=LabelAndAnnotationSpec({}, {}, {}, {}, {}, {}, {}), annotations=LabelAndAnnotationSpec({}, {}, ANNOTATIONS.copy(), {}, {}, {}, {}), @@ -640,11 +640,11 @@ def test_no_ingress(self, delete, post, deployer_no_suffix, app_spec): pytest.helpers.assert_any_call(delete, INGRESSES_URI, body=None, params=LABEL_SELECTOR_PARAMS) @pytest.mark.parametrize("app_spec, hosts", ( - (app_spec(), [u'testapp.svc.test.example.com', u'testapp.127.0.0.1.xip.io']), + (app_spec(), ['testapp.svc.test.example.com', 'testapp.127.0.0.1.xip.io']), (app_spec(ingresses=[ IngressItemSpec(host="foo.rewrite.example.com", pathmappings=[IngressPathMappingSpec(path="/", port=80)], annotations={})]), - [u'test.foo.rewrite.example.com', u'testapp.svc.test.example.com', u'testapp.127.0.0.1.xip.io']), + ['test.foo.rewrite.example.com', 'testapp.svc.test.example.com', 'testapp.127.0.0.1.xip.io']), )) @pytest.mark.usefixtures("delete") def test_applies_ingress_tls_deployer(self, deployer, ingress_tls_deployer, app_spec, hosts): diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_networking_v1_ingress.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_networking_v1_ingress.py index 64d5b986..a6d7265a 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_networking_v1_ingress.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_networking_v1_ingress.py @@ -14,10 +14,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pytest from k8s.models.networking_v1_ingress import Ingress, IngressTLS -from mock import create_autospec, Mock +from unittest.mock import create_autospec, Mock from requests import Response from fiaas_deploy_daemon import ExtensionHookCaller @@ -61,8 +61,8 @@ def app_spec(**kwargs): readiness=CheckSpec(http=HttpCheckSpec(path="/", port=8080, http_headers={}), tcp=None, execute=None, initial_delay_seconds=10, period_seconds=10, success_threshold=1, failure_threshold=3, timeout_seconds=1)), - teams=[u'foo'], - tags=[u'bar'], + teams=['foo'], + tags=['bar'], deployment_id="test_app_deployment_id", labels=LabelAndAnnotationSpec({}, {}, {}, {}, {}, {}, {}), annotations=LabelAndAnnotationSpec({}, {}, ANNOTATIONS.copy(), {}, {}, {}, {}), @@ -382,11 +382,11 @@ def test_no_ingress(self, delete, post, deployer_no_suffix, app_spec): pytest.helpers.assert_any_call(delete, INGRESSES_URI, body=None, params=LABEL_SELECTOR_PARAMS) @pytest.mark.parametrize("app_spec, hosts", ( - (app_spec(), [u'testapp.svc.test.example.com', u'testapp.127.0.0.1.xip.io']), + (app_spec(), ['testapp.svc.test.example.com', 'testapp.127.0.0.1.xip.io']), (app_spec(ingresses=[ IngressItemSpec(host="foo.rewrite.example.com", pathmappings=[IngressPathMappingSpec(path="/", port=80)], annotations={})]), - [u'test.foo.rewrite.example.com', u'testapp.svc.test.example.com', u'testapp.127.0.0.1.xip.io']), + ['test.foo.rewrite.example.com', 'testapp.svc.test.example.com', 'testapp.127.0.0.1.xip.io']), )) @pytest.mark.usefixtures("delete") def test_applies_ingress_tls_deployer(self, deployer, ingress_tls_deployer, app_spec, hosts): diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_ready_check.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_ready_check.py index 46604947..cb47c3b5 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_ready_check.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_ready_check.py @@ -16,9 +16,9 @@ # limitations under the License. from datetime import datetime, timedelta -import mock +from unittest import mock import pytest -from monotonic import monotonic as time_monotonic +from time import monotonic as time_monotonic from fiaas_deploy_daemon.deployer.bookkeeper import Bookkeeper from fiaas_deploy_daemon.deployer.kubernetes.ingress_v1beta1 import V1Beta1IngressAdapter diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_service_account_deploy.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_service_account_deploy.py index da8903ff..0af32d8a 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_service_account_deploy.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_service_account_deploy.py @@ -15,8 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pytest -import mock -from mock import create_autospec +from unittest import mock +from unittest.mock import create_autospec from requests import Response from k8s.models.service_account import ServiceAccount diff --git a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_service_deploy.py b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_service_deploy.py index 0ccd6226..d04c9f95 100644 --- a/tests/fiaas_deploy_daemon/deployer/kubernetes/test_service_deploy.py +++ b/tests/fiaas_deploy_daemon/deployer/kubernetes/test_service_deploy.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import pytest -from mock import create_autospec +from unittest.mock import create_autospec from requests import Response from k8s.models.service import Service diff --git a/tests/fiaas_deploy_daemon/deployer/test_deploy.py b/tests/fiaas_deploy_daemon/deployer/test_deploy.py index ca2f171d..ae8916de 100644 --- a/tests/fiaas_deploy_daemon/deployer/test_deploy.py +++ b/tests/fiaas_deploy_daemon/deployer/test_deploy.py @@ -14,9 +14,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from Queue import Queue +from queue import Queue -import mock +from unittest import mock import pytest from fiaas_deploy_daemon.config import Configuration diff --git a/tests/fiaas_deploy_daemon/e2e_expected/secrets-deployment.yml b/tests/fiaas_deploy_daemon/e2e_expected/secrets-deployment.yml index a44a2414..6f58be89 100644 --- a/tests/fiaas_deploy_daemon/e2e_expected/secrets-deployment.yml +++ b/tests/fiaas_deploy_daemon/e2e_expected/secrets-deployment.yml @@ -210,12 +210,12 @@ spec: command: [] args: [] env: - - name: 'K8S_DEPLOYMENT' - value: 'v3-data-examples-secrets' - name: 'AWS_REGION' value: 'eu-central-1' - name: 'SECRET_PATH' value: 'some-param' + - name: 'K8S_DEPLOYMENT' + value: 'v3-data-examples-secrets' imagePullPolicy: 'IfNotPresent' ports: [] imagePullSecrets: [] diff --git a/tests/fiaas_deploy_daemon/e2e_expected/strongbox-deployment.yml b/tests/fiaas_deploy_daemon/e2e_expected/strongbox-deployment.yml index a99692c3..bdaff6d2 100644 --- a/tests/fiaas_deploy_daemon/e2e_expected/strongbox-deployment.yml +++ b/tests/fiaas_deploy_daemon/e2e_expected/strongbox-deployment.yml @@ -209,12 +209,12 @@ spec: command: [] args: [] env: - - name: 'K8S_DEPLOYMENT' - value: 'v3-data-examples-strongbox' - name: 'AWS_REGION' value: 'eu-central-1' - name: 'SECRET_GROUPS' value: 'secretgroup1,secretgroup2,secretgroup3' + - name: 'K8S_DEPLOYMENT' + value: 'v3-data-examples-strongbox' imagePullPolicy: 'IfNotPresent' ports: [] imagePullSecrets: [] diff --git a/tests/fiaas_deploy_daemon/e2e_expected/v3full-deployment.yml b/tests/fiaas_deploy_daemon/e2e_expected/v3full-deployment.yml index 3889bda3..3a568b37 100644 --- a/tests/fiaas_deploy_daemon/e2e_expected/v3full-deployment.yml +++ b/tests/fiaas_deploy_daemon/e2e_expected/v3full-deployment.yml @@ -258,12 +258,12 @@ spec: command: [] args: [] env: - - name: K8S_DEPLOYMENT - value: v3-data-examples-full - name: AWS_REGION value: eu-central-1 - name: SECRET_GROUPS value: secretgroup1,secretgroup2 + - name: K8S_DEPLOYMENT + value: v3-data-examples-full imagePullPolicy: IfNotPresent ports: [] imagePullSecrets: [] diff --git a/tests/fiaas_deploy_daemon/specs/test_lookup.py b/tests/fiaas_deploy_daemon/specs/test_lookup.py index 4a0ad472..8ddcb98d 100644 --- a/tests/fiaas_deploy_daemon/specs/test_lookup.py +++ b/tests/fiaas_deploy_daemon/specs/test_lookup.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import + import pytest @@ -69,7 +69,7 @@ def test_len(self, lookup, type, expected): assert len(lookup[type]) == expected def test_items(self, lookup): - assert lookup["object"].items() == [("simple", 1), ("complex", {"first": 1, "second": 2})] + assert list(lookup["object"].items()) == [("simple", 1), ("complex", {"first": 1, "second": 2})] @pytest.mark.parametrize("config,defaults", ( (CONFIG, 1), diff --git a/tests/fiaas_deploy_daemon/specs/test_spec_factory.py b/tests/fiaas_deploy_daemon/specs/test_spec_factory.py index bac1316b..b3a1955a 100644 --- a/tests/fiaas_deploy_daemon/specs/test_spec_factory.py +++ b/tests/fiaas_deploy_daemon/specs/test_spec_factory.py @@ -15,13 +15,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import pytest -from mock import ANY, create_autospec +from unittest.mock import ANY, create_autospec from fiaas_deploy_daemon import Configuration from fiaas_deploy_daemon.specs.factory import SpecFactory, InvalidConfiguration, BaseFactory, BaseTransformer -IMAGE = u"finntech/docker-image:some-version" -NAME = u"application-name" +IMAGE = "finntech/docker-image:some-version" +NAME = "application-name" TEAMS = "IO" TAGS = "Foo" DEPLOYMENT_ID = "deployment_id" diff --git a/tests/fiaas_deploy_daemon/specs/v3/test_v3factory.py b/tests/fiaas_deploy_daemon/specs/v3/test_v3factory.py index 49d62a23..49f0a0ef 100644 --- a/tests/fiaas_deploy_daemon/specs/v3/test_v3factory.py +++ b/tests/fiaas_deploy_daemon/specs/v3/test_v3factory.py @@ -14,9 +14,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, unicode_literals -import mock + +from unittest import mock import pytest from fiaas_deploy_daemon import Configuration diff --git a/tests/fiaas_deploy_daemon/test_bootstrap_e2e.py b/tests/fiaas_deploy_daemon/test_bootstrap_e2e.py index 58b184d1..7d4d4e45 100644 --- a/tests/fiaas_deploy_daemon/test_bootstrap_e2e.py +++ b/tests/fiaas_deploy_daemon/test_bootstrap_e2e.py @@ -138,7 +138,7 @@ def custom_resource_definition_test_case(self, fiaas_path, namespace, labels, ex del expected[Ingress] elif expected.get(NetworkingV1Ingress): del expected[NetworkingV1Ingress] - expected = {kind: read_yml_if_exists(path) for kind, path in expected.items()} + expected = {kind: read_yml_if_exists(path) for kind, path in list(expected.items())} name = sanitize_resource_name(fiaas_path) @@ -174,7 +174,7 @@ def success(): wait_until(success, "CRD bootstrapping was successful", patience=PATIENCE) for name, namespace, app_uid, expected in expectations: - for kind in expected.keys(): + for kind in list(expected.keys()): try: kind.delete(name, namespace=namespace) except NotFound: @@ -182,13 +182,13 @@ def success(): def ensure_resources_not_exists(name, expected, namespace): - for kind in expected.keys(): + for kind in list(expected.keys()): with pytest.raises(NotFound): kind.get(name, namespace=namespace) def deploy_successful(name, namespace, app_uid, expected): - for kind, expected_result in expected.items(): + for kind, expected_result in list(expected.items()): if expected_result == SHOULD_NOT_EXIST: with pytest.raises(NotFound): kind.get(name, namespace=namespace) diff --git a/tests/fiaas_deploy_daemon/test_config.py b/tests/fiaas_deploy_daemon/test_config.py index 0518678f..17146d62 100644 --- a/tests/fiaas_deploy_daemon/test_config.py +++ b/tests/fiaas_deploy_daemon/test_config.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pyaml import pytest @@ -30,7 +30,7 @@ def getenv(self): @pytest.fixture def namespace_file_does_not_exist(self): - with mock.patch("__builtin__.open", new_callable=mock.mock_open) as _open: + with mock.patch("builtins.open", new_callable=mock.mock_open) as _open: _open.side_effect = IOError("does not exist") yield _open @@ -107,12 +107,12 @@ def test_config_from_file(self, key, attr, value, tmpdir): assert getattr(config, attr) == value def test_host_rewrite_rules(self): - args = ("pattern=value", r"(\d+)\.\example\.com=$1.example.net", "www.([a-z]+.com)={env}.$1") + args = ("pattern=value", r"(\d+)\.example\.com=$1.example.net", "www.([a-z]+.com)={env}.$1") config = Configuration(["--host-rewrite-rule=%s" % arg for arg in args]) assert config.host_rewrite_rules == [HostRewriteRule(arg) for arg in args] def test_host_rewrite_rules_from_file(self, tmpdir): - args = ("pattern=value", r"(\d+)\.\example\.com=$1.example.net", "www.([a-z]+.com)={env}.$1") + args = ("pattern=value", r"(\d+)\.example\.com=$1.example.net", "www.([a-z]+.com)={env}.$1") config_file = tmpdir.join("config.yaml") with config_file.open("w") as fobj: pyaml.dump({"host-rewrite-rule": args}, fobj, safe=True, default_style='"') diff --git a/tests/fiaas_deploy_daemon/test_e2e.py b/tests/fiaas_deploy_daemon/test_e2e.py index 59978be7..d8409166 100644 --- a/tests/fiaas_deploy_daemon/test_e2e.py +++ b/tests/fiaas_deploy_daemon/test_e2e.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function import os import subprocess @@ -41,10 +40,10 @@ AdditionalLabelsOrAnnotations from fiaas_deploy_daemon.tools import merge_dicts -IMAGE1 = u"finntech/application-name:123" -IMAGE2 = u"finntech/application-name:321" -DEPLOYMENT_ID1 = u"deployment_id_1" -DEPLOYMENT_ID2 = u"deployment_id_2" +IMAGE1 = "finntech/application-name:123" +IMAGE2 = "finntech/application-name:321" +DEPLOYMENT_ID1 = "deployment_id_1" +DEPLOYMENT_ID2 = "deployment_id_2" PATIENCE = 60 TIMEOUT = 5 SHOULD_NOT_EXIST = object() # Mapped to Kinds that should not exist in a test case @@ -76,7 +75,7 @@ def kubernetes(self, service_type, k8s_version): @pytest.fixture(scope="module") def kubernetes_service_account(self, k8s_version): try: - name = 'kind-{}-{}-{}'.format(k8s_version, "serviceaccount", uuid()) + name = 'kind-{}-{}-{}'.format(k8s_version, "sa", uuid()) kind = KindWrapper(k8s_version, name) try: yield kind.start() @@ -331,7 +330,7 @@ def custom_resource_definition_service_account(self, request, k8s_version): expected = self._construct_expected(expected, request) # modify the expected service account for this test - for k, _ in expected.items(): + for k, _ in list(expected.items()): if expected[k]['kind'] == 'Deployment': name = expected[k]['metadata']['name'] expected[k]['spec']['template']['spec']['serviceAccountName'] = name @@ -343,7 +342,7 @@ def custom_resource_definition_service_account(self, request, k8s_version): def _construct_expected(self, expected, request): new_expected = {} - for kind, path in expected.items(): + for kind, path in list(expected.items()): if path == SHOULD_NOT_EXIST: new_expected[kind] = SHOULD_NOT_EXIST else: @@ -382,8 +381,8 @@ def _end_popen(popen): @staticmethod def _select_kinds(expected): - if len(expected.keys()) > 0: - return expected.keys() + if len(list(expected.keys())) > 0: + return list(expected.keys()) else: return [Service, Deployment, Ingress] @@ -403,7 +402,7 @@ def run_crd_deploy(self, custom_resource_definition, service_type, service_accou # Check that deployment status is RUNNING def _assert_status(): status = FiaasApplicationStatus.get(create_name(name, DEPLOYMENT_ID1)) - assert status.result == u"RUNNING" + assert status.result == "RUNNING" assert len(status.logs) > 0 assert any("Saving result RUNNING for default/{}".format(name) in line for line in status.logs) @@ -413,7 +412,7 @@ def _assert_status(): status_labels = fiaas_application.spec.additional_labels.status if status_labels: status = FiaasApplicationStatus.get(create_name(name, DEPLOYMENT_ID1)) - label_difference = status_labels.viewitems() - status.metadata.labels.viewitems() + label_difference = status_labels.items() - status.metadata.labels.items() assert label_difference == set() # Check deploy success @@ -495,7 +494,7 @@ def test_multiple_ingresses(self, request, input, expected, k8s_version): k8s_ingress = NetworkingV1Ingress else: k8s_ingress = Ingress - expected = {k: read_yml(request.fspath.dirpath().join(v[k8s_ingress]).strpath) for (k, v) in expected.items()} + expected = {k: read_yml(request.fspath.dirpath().join(v[k8s_ingress]).strpath) for (k, v) in list(expected.items())} metadata = ObjectMeta(name=name, namespace="default", labels={"fiaas/deployment_id": DEPLOYMENT_ID1}) spec = FiaasApplicationSpec(application=name, image=IMAGE1, config=fiaas_yml) @@ -507,7 +506,7 @@ def test_multiple_ingresses(self, request, input, expected, k8s_version): # Check that deployment status is RUNNING def _assert_status(): status = FiaasApplicationStatus.get(create_name(name, DEPLOYMENT_ID1)) - assert status.result == u"RUNNING" + assert status.result == "RUNNING" assert len(status.logs) > 0 assert any("Saving result RUNNING for default/{}".format(name) in line for line in status.logs) @@ -517,7 +516,7 @@ def _check_two_ingresses(): assert k8s_ingress.get(name) assert k8s_ingress.get("{}-1".format(name)) - for ingress_name, expected_dict in expected.items(): + for ingress_name, expected_dict in list(expected.items()): actual = k8s_ingress.get(ingress_name) assert_k8s_resource_matches(actual, expected_dict, IMAGE1, None, DEPLOYMENT_ID1, None, app_uid) @@ -543,7 +542,7 @@ def _check_one_ingress(): FiaasApplication.delete(name) def cleanup_complete(): - for name, _ in expected.items(): + for name, _ in list(expected.items()): with pytest.raises(NotFound): k8s_ingress.get(name) @@ -552,7 +551,7 @@ def cleanup_complete(): def _deploy_success(name, service_type, image, expected, deployment_id, strongbox_groups=None, app_uid=None): def action(): - for kind, expected_dict in expected.items(): + for kind, expected_dict in list(expected.items()): if expected_dict == SHOULD_NOT_EXIST: with pytest.raises(NotFound): kind.get(name) diff --git a/tests/fiaas_deploy_daemon/test_extension_hook_caller.py b/tests/fiaas_deploy_daemon/test_extension_hook_caller.py index a0e3c89e..81b5f9f3 100644 --- a/tests/fiaas_deploy_daemon/test_extension_hook_caller.py +++ b/tests/fiaas_deploy_daemon/test_extension_hook_caller.py @@ -16,7 +16,7 @@ # limitations under the License. import copy -import mock +from unittest import mock import pytest from k8s.models.deployment import Deployment, DeploymentSpec from k8s.models.pod import Container, PodSpec, PodTemplateSpec, EnvVar diff --git a/tests/fiaas_deploy_daemon/test_healthcheck.py b/tests/fiaas_deploy_daemon/test_healthcheck.py index 54f68d8e..0e7439e3 100644 --- a/tests/fiaas_deploy_daemon/test_healthcheck.py +++ b/tests/fiaas_deploy_daemon/test_healthcheck.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pytest from fiaas_deploy_daemon import HealthCheck diff --git a/tests/fiaas_deploy_daemon/test_logsetup.py b/tests/fiaas_deploy_daemon/test_logsetup.py index 2b060bdf..d79554d6 100644 --- a/tests/fiaas_deploy_daemon/test_logsetup.py +++ b/tests/fiaas_deploy_daemon/test_logsetup.py @@ -17,9 +17,9 @@ import json import logging import sys -from cStringIO import StringIO +from io import StringIO -import mock +from unittest import mock import pytest from callee import InstanceOf, Attrs, List diff --git a/tests/fiaas_deploy_daemon/test_resolve_secrets.py b/tests/fiaas_deploy_daemon/test_resolve_secrets.py index 5ab54012..1259412e 100644 --- a/tests/fiaas_deploy_daemon/test_resolve_secrets.py +++ b/tests/fiaas_deploy_daemon/test_resolve_secrets.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import + import pytest @@ -52,6 +52,6 @@ def test_secrets_are_bytes(self, secrets_dir): assert isinstance(secrets.usage_reporting_key, bytes) def test_secrets_are_stripped(self, secrets_dir): - secrets_dir.join("usage-reporting-key").write(KEY + "\n") + secrets_dir.join("usage-reporting-key").write(KEY + b"\n") secrets = resolve_secrets(str(secrets_dir)) assert secrets.usage_reporting_key == KEY diff --git a/tests/fiaas_deploy_daemon/test_retry.py b/tests/fiaas_deploy_daemon/test_retry.py index e5c09822..807b2c7a 100644 --- a/tests/fiaas_deploy_daemon/test_retry.py +++ b/tests/fiaas_deploy_daemon/test_retry.py @@ -12,14 +12,30 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pytest from k8s.client import ClientError -from requests import Response +from requests import Response, Request from fiaas_deploy_daemon.retry import retry_on_upsert_conflict, UpsertConflict, canonical_name +def test_upsertconflict_str(): + expected = "409 Conflict for POST http://example.com. reason=bad conflict, message=conflict because of reasons" + + response = mock.MagicMock(spec=Response) + response.status_code = 409 # Conflict + request = mock.MagicMock(spec=Request) + request.method = "POST" + request.url = "http://example.com" + response.request = request + response.json.return_value = {"reason": "bad conflict", "message": "conflict because of reasons"} + + e = UpsertConflict(response) + + assert str(e) == expected + + @pytest.mark.parametrize("status", ( 400, # Bad Request 402, # Payment Required @@ -109,7 +125,7 @@ def name_test_function(): @pytest.mark.parametrize("func, expected", ( - (id, "__builtin__.id"), + (id, "builtins.id"), (name_test_function, "{}.name_test_function".format(__name__)), (NameTester.clsmethod, "{}.NameTester.clsmethod".format(__name__)), (NameTester.method, "{}.NameTester.method".format(__name__)), diff --git a/tests/fiaas_deploy_daemon/test_warn_env_variable_config.py b/tests/fiaas_deploy_daemon/test_warn_env_variable_config.py index 9902a2bf..90f6e156 100644 --- a/tests/fiaas_deploy_daemon/test_warn_env_variable_config.py +++ b/tests/fiaas_deploy_daemon/test_warn_env_variable_config.py @@ -3,7 +3,7 @@ import logging -import mock +from unittest import mock import pytest @@ -54,9 +54,9 @@ def config_flags(): ConfigFlag('CLIENT_CERT', 'client_cert'), ConfigFlag('CLIENT_KEY', 'client_key'), ConfigFlag('INGRESS_SUFFIXES', r'[1\.example.com,2.example.com]', config_value=[r'1\.example.com', '2.example.com']), - ConfigFlag('HOST_REWRITE_RULES', r'[pattern=value,(\d+)\.\example\.com=$1.example.net,www.([a-z]+.com)={env}.$1]', + ConfigFlag('HOST_REWRITE_RULES', r'[pattern=value,(\d+)\.example\.com=$1.example.net,www.([a-z]+.com)={env}.$1]', config_value=[HostRewriteRule('pattern=value'), - HostRewriteRule(r'(\d+)\.\example\.com=$1.example.net'), + HostRewriteRule(r'(\d+)\.example\.com=$1.example.net'), HostRewriteRule('www.([a-z]+.com)={env}.$1')]), ConfigFlag('FIAAS_GLOBAL_ENV', '[pattern=value,FIAAS_ENV=test]', config_key='global_env', config_value={'pattern': 'value', 'FIAAS_ENV': 'test'}), diff --git a/tests/fiaas_deploy_daemon/usage_reporting/test_dev_hose_auth.py b/tests/fiaas_deploy_daemon/usage_reporting/test_dev_hose_auth.py index 7a7452cb..3b9ce558 100644 --- a/tests/fiaas_deploy_daemon/usage_reporting/test_dev_hose_auth.py +++ b/tests/fiaas_deploy_daemon/usage_reporting/test_dev_hose_auth.py @@ -14,12 +14,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import + import base64 from datetime import datetime -import mock +from unittest import mock import pytest import requests diff --git a/tests/fiaas_deploy_daemon/usage_reporting/test_devhose_transformer.py b/tests/fiaas_deploy_daemon/usage_reporting/test_devhose_transformer.py index 330401ba..3e46bb9f 100644 --- a/tests/fiaas_deploy_daemon/usage_reporting/test_devhose_transformer.py +++ b/tests/fiaas_deploy_daemon/usage_reporting/test_devhose_transformer.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import pytest from fiaas_deploy_daemon import Configuration diff --git a/tests/fiaas_deploy_daemon/usage_reporting/test_usage_reporter.py b/tests/fiaas_deploy_daemon/usage_reporting/test_usage_reporter.py index a62a9856..d84a5e09 100644 --- a/tests/fiaas_deploy_daemon/usage_reporting/test_usage_reporter.py +++ b/tests/fiaas_deploy_daemon/usage_reporting/test_usage_reporter.py @@ -14,9 +14,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import -import mock + +from unittest import mock import pytest from blinker import signal from requests.auth import AuthBase diff --git a/tests/fiaas_deploy_daemon/utils.py b/tests/fiaas_deploy_daemon/utils.py index 99cab3ec..1e82295a 100644 --- a/tests/fiaas_deploy_daemon/utils.py +++ b/tests/fiaas_deploy_daemon/utils.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + import base64 import contextlib @@ -27,7 +27,7 @@ import uuid as uuidlib from copy import deepcopy from datetime import datetime -from urlparse import urljoin +from urllib.parse import urljoin import json import pytest @@ -37,7 +37,7 @@ from k8s.models.autoscaler import HorizontalPodAutoscaler from k8s.models.deployment import Deployment from k8s.models.service import Service -from monotonic import monotonic as time_monotonic +from time import monotonic as time_monotonic from fiaas_deploy_daemon.crd.types import FiaasApplication, FiaasApplicationStatus @@ -324,10 +324,10 @@ def _run_cmd(self, args): output = subprocess.check_output(args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: plog("vvvvvvvvvvvvvvvv Output from {} vvvvvvvvvvvvvvvv".format(args[0])) - plog(e.output) + plog(e.output.decode('utf-8')) plog("^^^^^^^^^^^^^^^^ Output from {} ^^^^^^^^^^^^^^^^".format(args[0])) raise e - return output.strip() + return output.decode('utf-8').strip() def _is_macos(): diff --git a/tox.ini b/tox.ini index 9ab10036..9e9e6e56 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,8 @@ envlist = codestyle,test,integration_test [testenv] -basepython=python2.7 -envdir={toxworkdir}/py27 +basepython=python3.9 +envdir={toxworkdir}/py39 usedevelop=True deps=.[dev] setenv =