From 3641bfcea4fffe62f410b32aac247bb228254577 Mon Sep 17 00:00:00 2001 From: Nick Blaskey Date: Tue, 18 Jun 2024 13:29:57 -0400 Subject: [PATCH] feat: support proxied Determined tasks on remote k8s clusters (#9469) - Enables Determined tasks running on remote Kubernetes clusters to be exposed to the Determined master and proxies. - Facilitates multi-resource manager setups by configuring a Gateway controller in the external Kubernetes cluster. Co-authored-by: Hamid Zare <12127420+hamidzr@users.noreply.github.com> --- .../devcluster/multi-k8s.devcluster.yaml | 8 + .circleci/real_config.yml | 9 +- docs/release-notes/gateway.rst | 20 + docs/setup-cluster/k8s/_index.rst | 2 + docs/setup-cluster/k8s/controller-reviews.rst | 192 +++++++++ .../k8s/install-on-kubernetes.rst | 20 +- .../k8s/internal-task-gateway.rst | 134 ++++++ .../k8s/setup-multiple-resource-managers.rst | 4 + e2e_tests/tests/cluster/test_proxy.py | 144 ++++++- e2e_tests/tests/command/test_notebook.py | 1 + e2e_tests/tests/command/test_shell.py | 1 + .../tests/fixtures/ports-proxy/config.yaml | 31 ++ e2e_tests/tests/fixtures/ports-proxy/start.py | 87 ++++ go.mod | 58 +-- go.sum | 192 ++++----- harness/determined/cli/proxy.py | 2 +- harness/determined/exec/prep_container.py | 6 + master/.mockery.yaml | 18 +- .../internal/config/resource_config_test.go | 36 ++ .../config/resource_manager_config.go | 69 +++- .../rm/kubernetesrm/gateway_service.go | 168 ++++++++ .../rm/kubernetesrm/gateway_service_test.go | 269 ++++++++++++ .../internal/rm/kubernetesrm/gateway_spec.go | 31 ++ master/internal/rm/kubernetesrm/job.go | 100 ++++- master/internal/rm/kubernetesrm/jobs.go | 385 +++++++++++++++--- .../kubernetes_resource_manager.go | 1 + .../rm/kubernetesrm/mock_client_test.go | 45 +- .../internal/rm/kubernetesrm/request_queue.go | 65 ++- .../rm/kubernetesrm/request_queue_test.go | 22 +- .../rm/kubernetesrm/request_workers.go | 96 ++++- .../kubernetesrm/resource_pool_intg_test.go | 1 + master/internal/rm/kubernetesrm/spec.go | 105 ++++- master/internal/rm/kubernetesrm/spec_test.go | 129 ++++++ master/internal/task/allocation.go | 2 +- master/pkg/check/check_numbers.go | 9 + master/pkg/check/check_string.go | 23 ++ master/pkg/check/check_test.go | 68 ++++ tools/k8s/launch-minikube-with-gateway.sh | 59 +++ 38 files changed, 2352 insertions(+), 260 deletions(-) create mode 100644 docs/release-notes/gateway.rst create mode 100644 docs/setup-cluster/k8s/controller-reviews.rst create mode 100644 docs/setup-cluster/k8s/internal-task-gateway.rst create mode 100644 e2e_tests/tests/fixtures/ports-proxy/config.yaml create mode 100644 e2e_tests/tests/fixtures/ports-proxy/start.py create mode 100644 master/internal/rm/kubernetesrm/gateway_service.go create mode 100644 master/internal/rm/kubernetesrm/gateway_service_test.go create mode 100644 master/internal/rm/kubernetesrm/gateway_spec.go create mode 100755 tools/k8s/launch-minikube-with-gateway.sh diff --git a/.circleci/devcluster/multi-k8s.devcluster.yaml b/.circleci/devcluster/multi-k8s.devcluster.yaml index 8eee412ed7f..6a035f6a3de 100644 --- a/.circleci/devcluster/multi-k8s.devcluster.yaml +++ b/.circleci/devcluster/multi-k8s.devcluster.yaml @@ -38,6 +38,14 @@ stages: kubeconfig_path: /tmp/defaultrm-kubeconf determined_master_ip: $DOCKER_LOCALHOST determined_master_port: 8080 + internal_task_gateway: + gateway_name: contour + gateway_namespace: projectcontour + gateway_ip: $GATEWAY_IP + gateway_port_range_start: 49152 + gateway_port_range_end: 65535 + + additional_resource_managers: - resource_manager: type: kubernetes diff --git a/.circleci/real_config.yml b/.circleci/real_config.yml index e301252b691..1aa481884f3 100644 --- a/.circleci/real_config.yml +++ b/.circleci/real_config.yml @@ -3273,6 +3273,9 @@ jobs: collect-det-job-logs: type: boolean default: true + k8s-version: + type: string + default: "1.29.5" machine: image: <> resource_class: <> @@ -3304,10 +3307,12 @@ jobs: curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && sudo install minikube-linux-amd64 /usr/local/bin/minikube - run: name: Start defaultrm minikube - command: minikube start --profile defaultrm + command: | + K8S_VERSION=<> source tools/k8s/launch-minikube-with-gateway.sh defaultrm + echo "export GATEWAY_IP=\"${GATEWAY_IP}\"" >> $BASH_ENV - run: name: Start additionalrm minikube - command: minikube start --profile additionalrm + command: minikube start --profile additionalrm --kubernetes-version <> - install-devcluster - unless: diff --git a/docs/release-notes/gateway.rst b/docs/release-notes/gateway.rst new file mode 100644 index 00000000000..3eff08b5a8e --- /dev/null +++ b/docs/release-notes/gateway.rst @@ -0,0 +1,20 @@ +:orphan: + +**New Features** + +- Kubernetes: The :ref:`Internal Task Gateway ` feature enables Determined + tasks running on remote Kubernetes clusters to be exposed to the Determined master and proxies. + This feature facilitates multi-resource manager setups by configuring a Gateway controller in the + external Kubernetes cluster. + +.. important:: + + Enabling this feature exposes Determined tasks to the outside world. It is crucial to implement + appropriate security measures to restrict access to exposed tasks and secure communication + between the external cluster and the main cluster. Recommended measures include: + + - Setting up a firewall + - Using a VPN + - Implementing IP whitelisting + - Configuring Kubernetes Network Policies + - Employing other security measures as needed diff --git a/docs/setup-cluster/k8s/_index.rst b/docs/setup-cluster/k8s/_index.rst index 694234e86ad..78145e58df2 100644 --- a/docs/setup-cluster/k8s/_index.rst +++ b/docs/setup-cluster/k8s/_index.rst @@ -151,4 +151,6 @@ for diagnosing any issues that arise during installation. custom-pod-specs helm-commands setup-multiple-resource-managers + internal-task-gateway + controller-reviews troubleshooting diff --git a/docs/setup-cluster/k8s/controller-reviews.rst b/docs/setup-cluster/k8s/controller-reviews.rst new file mode 100644 index 00000000000..7247ac094ad --- /dev/null +++ b/docs/setup-cluster/k8s/controller-reviews.rst @@ -0,0 +1,192 @@ +.. _controller-reviews: + +############################# + Gateway API Implementations +############################# + +This document is a survey of the Gateway API controllers that are available and listed by the `SIG +here `_. + +Based on the documentation provided by the projects, we've categorized the implementations into +three groups: + +- **Supported**: The project has implemented the TCPRoute resource and we have tested it. +- **Support Not Tested**: The project has indicated implementation of the TCPRoute resource but we + have not tested it. +- **Not Yet Supported**: The project either has not implemented the TCPRoute resource or has not + indicated support for it, or we have not found the documentation on it. + +********* + Contour +********* + +> Contour v1.29.0 implements Gateway API v1.0.0. All Standard channel v1 API group resources +(GatewayClass, Gateway, HTTPRoute, ReferenceGrant), plus most v1alpha2 API group resources +(TLSRoute, TCPRoute, GRPCRoute, ReferenceGrant, and BackendTLSPolicy) are supported. + +`Contour Gateway API Guide `_ + +*************** + Envoy Gateway +*************** + +`Envoy Gateway TCP Routing `_ + +#################### + Support Not Tested +#################### + +******** + Cilium +******** + +`Cilium Gateway API +`_ +Based on Envoy. + +******************************** + HAProxy K8s Ingress Controller +******************************** + +`HAProxy Kubernetes TCPRoute +`_ HAProxy +Enterprise Kubernetes Ingress Controller. + +****************** + Hashicorp Consul +****************** + +`Consul TCPRoute Reference +`_ + +********* + Traefik +********* + +`Traefik Kubernetes Gateway `_ +`Traefik Gateway Provider `_ > +Enabling The Experimental Kubernetes Gateway Provider > Since this provider is still experimental, +it needs to be activated in the experimental section of the static configuration. + +******************************************* + Kong Operator and Kong Ingress Controller +******************************************* + +`Kong Gateway API `_ + +****** + Kuma +****** + +Based on Envoy. `Kuma Mesh TCPRoute +`_ + +********* + Flomesh +********* + +`Flomesh Gateway API Compatibility +`_ Partial tcproute +support + +******* + Istio +******* + +`Istio Gateway API Differences +`_ + +################### + Not Yet Supported +################### + +************** + Acnodal Epic +************** + +Supports k8s v0.5 + +*************** + Apache Apisix +*************** + +`Apisix Ingress Controller `_ +Mainly ingress focused. + +******* + Azure +******* + +`Azure Application Gateway +`_ No TCPRoute +support. + +************ + VMWare Avi +************ + +Advertises level 4 load balancing but no TCPRoute support yet. Supports k8v1. `VMWare Avi Kubernetes +Guide +`_ + +*********** + Easegress +*********** + +No TCPRoute support. + +******************************* + Emissary Ingress - Ambassador +******************************* + +No TCPRoute support. `Ambassador Gateway API +`_ + +*********** + Gloo Solo +*********** + +Based on Envoy but no TCPRoute support. + +***************** + HAProxy Ingress +***************** + +No TCPRoute support. `HAProxy Ingress Gateway API +`_ + +********* + Linkerd +********* + +No TCPRoute support. `Linkerd HTTPRoute Reference `_ + +*********** + Litespeed +*********** + +No TCPRoute support. `Litespeed Kubernetes Gateway +`_ + +***************** + Nginx GW Fabric +***************** + +No TCPRoute support yet. `Nginx Gateway API Compatibility +`_ + +******* + Ngrok +******* + +No TCPRoute support. Only HTTRoutes are stable, the others are in an experimental channel. ngrok +supports edges for HTTP/S, TLS, and TCP. The ngrok Operator currently only supports the HTTPRoute. +TLSRoute and TCPRoute will be added after they become stable. `Ngrok Kubernetes Gateway API +`_ + +********** + WSO2 APK +********** + +No TCPRoute support. `WSO2 Kubernetes CRDs +`_ diff --git a/docs/setup-cluster/k8s/install-on-kubernetes.rst b/docs/setup-cluster/k8s/install-on-kubernetes.rst index a3c803e099b..5e4f885700f 100644 --- a/docs/setup-cluster/k8s/install-on-kubernetes.rst +++ b/docs/setup-cluster/k8s/install-on-kubernetes.rst @@ -18,16 +18,29 @@ using the `Determined Helm Chart `__. When the Determined Helm chart is installed, the following entities will be created: - Deployment of the Determined master. + - ConfigMap containing configurations for the Determined master. + - LoadBalancer service to make the Determined master accessible. Later in this guide, we describe how it is possible to replace this with a NodePort service. -- ServiceAcccount which will be used by the Determined master. + +- ServiceAccount which will be used by the Determined master. + - Deployment of a Postgres database. Later in this guide, we describe how an external database can be used instead. + - PersistentVolumeClaim for the Postgres database. Omitted if using an external database. + - Service to allow the Determined master to communicate with the Postgres database. Omitted if using an external database. +- In case of multiple Kubernetes clusters and in each external-to-master clusters: + + - Gateway service to allow north-south access to Determined proxied tasks in external-to-master + clusters. + - Service to expose proxied ports on Determined jobs. + - TCPRoute to attach the gateway service to the proxied ports service. + *************** Prerequisites *************** @@ -450,6 +463,11 @@ To set up multiple resource pools for Determined on your Kubernetes cluster: #. Add the appropriate resource pool name to namespace mappings in the ``resourcePools`` section of the ``values.yaml`` file in the Helm chart. +.. note:: + + To enable north-south access to Determined proxied tasks in external-to-master clusters, set up a + gateway as described in the docs :doc:`Internal Task Gateway ` + ******************** Install Determined ******************** diff --git a/docs/setup-cluster/k8s/internal-task-gateway.rst b/docs/setup-cluster/k8s/internal-task-gateway.rst new file mode 100644 index 00000000000..add53615ae1 --- /dev/null +++ b/docs/setup-cluster/k8s/internal-task-gateway.rst @@ -0,0 +1,134 @@ +.. _internal-task-gateway: + +####################### + Internal Task Gateway +####################### + +`Kubernetes (K8s) Gateway APIs `_ allow us to expose otherwise +internal Determined jobs running on remote Kubernetes (K8s) clusters to Determined master and +proxies. This is useful for multi-resource manager setups. + +The overall setup includes installing and configuring a Gateway controller in the Kubernetes (K8s) +cluster external to Determined and configuring the Determined master to use the Gateway controller. +Please refer to the sections below to see the configuration changes and controller requirements. + +.. warning:: + + Enabling this feature exposes Determined tasks to the outside world. It is crucial to implement + appropriate security measures to restrict access to exposed tasks and secure communication + between the external cluster and the main cluster. Recommended measures include: + + - Setting up a firewall + - Using a VPN + - Implementing IP whitelisting + - Configuring Kubernetes Network Policies + - Employing other security measures as needed + - Some tasks including JupyterLab notebooks and shells already have secure transport built-in. + - Since TensorBoards do not use TLS, you should consider a tunneling solution. + +Limitations: + +- Proxying to multi-node trials is not supported. Currently this only includes experiments running + distributed training that want to manually expose proxies to the outside world. + +################################### + Controller Support - Requirements +################################### + +High-level requirements from the controller implementations are: + +- Supports Gateway APIs > v1 + +- Supports `TCPRoute + `_ and level 4 + routing support + +Take a look at our current survey of the existing implementations here: :doc:`controller-reviews` + +In these docs, we'll be using `Contour from Project Contour `_. + +############################ + Sample Setup - Development +############################ + +For internal testing and development, we provide a simple setup script that uses a dynamic +provisioner provided by Project Contour. + +On a local dev machine, you can use Minikube and Contour as the controller. We provide a script to +simplify the process. This can be found in `tools/k8s/launch-minikube-with-gateway.sh`. + +After you have a working Kubernetes (K8s) cluster and a Gateway controller running, configure the +resource manager via master config and start the Determined cluster. + +############### + Configuration +############### + +This section describes how to configure your cluster and Determined master to use the Internal Task +Gateway. + +- Total active proxies will be limited by: maxItems set in the Gateway CRD and the portRange + configured for Determined (not exhaustive). + +********************** + Master Configuration +********************** + +To configure the optional InternalTaskGateway for a Kubernetes resource manager, add a struct under +``internal_task_gateway`` key under each desired resource manager configurations. + +The config is shown below. + +.. code:: yaml + + internal_task_gateway: + # GatewayName as defined in the k8s cluster. + gateway_name: + + # GatewayNamespace as defined in the k8s cluster. + gateway_namespace: + + # GatewayIP as defined in the k8s cluster. + # GatewayIP can also be a hostname if that is easier for your setup. + gateway_ip: + + # GWPortStart denotes the inclusive start of the available and exclusive port range to + # MLDE for InternalTaskGateway. + gateway_port_range_start: + + # GWPortEnd denotes the inclusive end of the available and exclusive port range to + # MLDE for InternalTaskGateway. + gateway_port_range_end: + +- Valid port range starts from 1025 to 65535, inclusive. +- GatewayIP is the IP address of the Gateway controller that is visible to the Determined master. + +********* + Gateway +********* + +In the CRD ``gateways.gateway.networking.k8s.io`` +``schema.openAPIV3Schema.properties.spec.properties.listeners.maxItems`` defines a max limit of how +many listeners can be active on a single gateway. This limit sets the upper bound on how many tasks +can be actively proxied. By default this limit is only 64. + +Note that when configuring this number you might hit Kubernetes (K8s) validation complexity +thresholds checks. This can be configured and is dependent on each Kubernetes (K8s) cluster's +requirements and setup. + +For example to up the number from allowed listeners to 128, you can modify the CRD at the given path +above and `kubectl apply -f `. Make sure to set the value for the version of the spec +that your Gateway API is going to use. + +############################################### + Running Determined Outside of the K8s Cluster +############################################### + +If you're running Determined outside of the Kubernetes (K8s) cluster, for example on your local +machine for testing and development, it's possible to test this feature using just a single +Kubernetes (K8s) cluster. All that is needed is for Det master to be sitting external to the target +cluster. + +For allowing Determined tasks to connect to master that's running locally on your machine, you can +use services like ngrok or a reverse SSH tunnel if you have access to a public IP like so: `ssh -R +8080:localhost:8080 aws-dev.prv -N -o ServerAliveInterval=60 -o ServerAliveCountMax=10` diff --git a/docs/setup-cluster/k8s/setup-multiple-resource-managers.rst b/docs/setup-cluster/k8s/setup-multiple-resource-managers.rst index bc6dc505889..65ac8f44065 100644 --- a/docs/setup-cluster/k8s/setup-multiple-resource-managers.rst +++ b/docs/setup-cluster/k8s/setup-multiple-resource-managers.rst @@ -24,6 +24,10 @@ service in one Kubernetes cluster and schedule workloads in the same or other Ku Any requests to resource pools not defined in the master configuration are routed to the default resource manager. Such requests are not routed to additional resource managers, if defined. +To enable use of Determined tasks that rely on Determined proxies in the external-to-master +clusters, set up a gateway as described in the docs :doc:Internal Task Gateway here :doc:`Internal +Task Gateway here `. + ********************************************* How to Configure Multiple Resource Managers ********************************************* diff --git a/e2e_tests/tests/cluster/test_proxy.py b/e2e_tests/tests/cluster/test_proxy.py index 3e77784c330..ce05159938d 100644 --- a/e2e_tests/tests/cluster/test_proxy.py +++ b/e2e_tests/tests/cluster/test_proxy.py @@ -2,8 +2,10 @@ import io import pathlib import re +import socket import subprocess import time +from typing import Callable, Optional, Tuple import pytest import requests @@ -28,19 +30,53 @@ def _experiment_task_id(sess: api.Session, exp_id: int) -> str: return task_id -def _probe_tunnel(proc: "subprocess.Popen[str]", port: int = 8265) -> None: - max_tunnel_time = 300 - start = time.time() - ctr = 0 - while time.time() - start < max_tunnel_time: +Check = Callable[[], bool] + + +def _ray_tunnel_check(port: int = 8265) -> Check: + def check() -> bool: try: r = requests.get(f"http://localhost:{port}", timeout=5) if r.status_code == 200: - break + return True except requests.exceptions.ConnectionError: pass except requests.exceptions.ReadTimeout: pass + return False + + return check + + +def _echo_server_check(port: int) -> Check: + host = "127.0.0.1" + test_message = "Hello, Server" + + def check() -> bool: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client_socket: + client_socket.connect((host, port)) + client_socket.sendall(test_message.encode()) + received_message = client_socket.recv(1024).decode() + assert ( + received_message == test_message + ), f"Expected '{test_message}', but got '{received_message}'" + return True + except ConnectionRefusedError: + pass + return False + + return check + + +def _probe_tunnel( + proc: "subprocess.Popen[str]", predicate: Check, max_tunnel_time: int = 300 +) -> None: + start = time.time() + ctr = 0 + while time.time() - start < max_tunnel_time: + if predicate(): + break if ctr + 1 % 10 == 0: print(f"Tunnel probe pending: {ctr} ticks...") time.sleep(1) @@ -57,8 +93,100 @@ def _ray_job_submit(exp_path: pathlib.Path, port: int = 8265) -> None: return ray_utils.ray_job_submit(exp_path, ["python", "ray_job.py"], port=port) +@pytest.mark.e2e_cpu +@pytest.mark.e2e_multi_k8s +@pytest.mark.timeout(600) +@pytest.mark.parametrize( + "exp_port, is_tcp", + [ + (8000, False), + (6000, True), + ], +) +def test_experiment_proxy_simple_zero_slot(exp_port: int, is_tcp: bool) -> None: + port_map = (exp_port, is_tcp) + return _test_experiment_proxy_simple(port_map, slots=0, max_conc_trials=2) + + +@pytest.mark.port_registry # has multiple slots +@pytest.mark.timeout(600) +@pytest.mark.parametrize( + "exp_port, is_tcp, max_conc_trials", + [ + (8000, False, 1), + (6000, True, 1), + ], +) +def test_experiment_proxy_simple_two_slots( + exp_port: int, + is_tcp: bool, + max_conc_trials: int, +) -> None: + port_map = (exp_port, is_tcp) + return _test_experiment_proxy_simple(port_map, slots=2, max_conc_trials=max_conc_trials) + + +def _test_experiment_proxy_simple( + port_map: Tuple[int, bool], slots: int, max_conc_trials: int +) -> None: + exp_port, is_tcp = port_map + listen_port = 23424 + sess = api_utils.user_session() + exp_path = pathlib.Path(conf.fixtures_path("ports-proxy")) + exp_id = exp.create_experiment( + sess, + str(exp_path / "config.yaml"), + str(exp_path), + [ + "--config", + f"resources.slots_per_trial={slots}", + "--config", + f"searcher.max_concurrent_trials={max_conc_trials}", + ], + ) + try: + exp.wait_for_experiment_state(sess, exp_id, bindings.experimentv1State.RUNNING) + task_id = _experiment_task_id(sess, exp_id) + + tl: Optional[bindings.v1TaskLogsResponse] = None + for tl in api.task_logs(sess, task_id, follow=True): + if "Servers started" in tl.message: + break + assert tl is not None, "Failed to find 'Servers started' in task logs" + print("server ready", tl) + proxy_url = f"/proxy/{task_id}:{exp_port}/" + if is_tcp: + proc = detproc.Popen( + sess, + [ + "python", + "-m", + "determined.cli.tunnel", + "--listener", + str(listen_port), + "--auth", + conf.make_master_url(), + f"{task_id}:{exp_port}", + ], + text=True, + ) + + try: + _probe_tunnel(proc, _echo_server_check(port=listen_port)) + finally: + proc.terminate() + proc.wait(10) + else: + resp = sess.get(proxy_url) + resp.raise_for_status() + assert "Hello" in resp.text + finally: + bindings.post_KillExperiment(sess, id=exp_id) + + @pytest.mark.e2e_cpu @pytest.mark.timeout(600) +@pytest.mark.e2e_multi_k8s def test_experiment_proxy_ray_tunnel() -> None: sess = api_utils.user_session() exp_path = conf.EXAMPLES_PATH / "features" / "ports" @@ -88,7 +216,7 @@ def test_experiment_proxy_ray_tunnel() -> None: ) try: - _probe_tunnel(proc) + _probe_tunnel(proc, _ray_tunnel_check(port=8265)) _ray_job_submit(exp_path) finally: proc.terminate() @@ -167,7 +295,7 @@ def test_experiment_proxy_ray_publish() -> None: try: exp.wait_for_experiment_state(sess, exp_id, bindings.experimentv1State.RUNNING) - _probe_tunnel(proc) + _probe_tunnel(proc, _ray_tunnel_check(port=8265)) _ray_job_submit(exp_path) finally: bindings.post_KillExperiment(sess, id=exp_id) diff --git a/e2e_tests/tests/command/test_notebook.py b/e2e_tests/tests/command/test_notebook.py index f5f533385ef..1ed5b222f56 100644 --- a/e2e_tests/tests/command/test_notebook.py +++ b/e2e_tests/tests/command/test_notebook.py @@ -26,6 +26,7 @@ def test_basic_notebook_start_and_kill() -> None: @pytest.mark.e2e_cpu +@pytest.mark.e2e_multi_k8s def test_notebook_proxy() -> None: session = api_utils.user_session() diff --git a/e2e_tests/tests/command/test_shell.py b/e2e_tests/tests/command/test_shell.py index b90d264f6f8..cf3a784de64 100644 --- a/e2e_tests/tests/command/test_shell.py +++ b/e2e_tests/tests/command/test_shell.py @@ -12,6 +12,7 @@ @pytest.mark.e2e_gpu @pytest.mark.e2e_slurm @pytest.mark.e2e_pbs +@pytest.mark.e2e_multi_k8s def test_start_and_write_to_shell(tmp_path: pathlib.Path) -> None: sess = api_utils.user_session() with cmd.interactive_command(sess, ["shell", "start"]) as shell: diff --git a/e2e_tests/tests/fixtures/ports-proxy/config.yaml b/e2e_tests/tests/fixtures/ports-proxy/config.yaml new file mode 100644 index 00000000000..eaf631afb93 --- /dev/null +++ b/e2e_tests/tests/fixtures/ports-proxy/config.yaml @@ -0,0 +1,31 @@ +name: experiment-port-proxy +entrypoint: python3 start.py + +resources: + slots_per_trial: 2 + +searcher: + max_length: 10000000 + name: grid + metric: x + max_concurrent_trials: 2 + +hyperparameters: + model_version: + type: categorical + vals: + - 1 + - 2 + - 3 + - 4 + +max_restarts: 0 + +# Hardcode the image because the new image has a bug. TODO fix this when the image bug is fixed. +environment: + image: determinedai/pytorch-tensorflow-cpu-dev:8b3bea3 + proxy_ports: + - proxy_port: 8000 + proxy_tcp: false + - proxy_port: 6000 + proxy_tcp: true diff --git a/e2e_tests/tests/fixtures/ports-proxy/start.py b/e2e_tests/tests/fixtures/ports-proxy/start.py new file mode 100644 index 00000000000..899f694ae23 --- /dev/null +++ b/e2e_tests/tests/fixtures/ports-proxy/start.py @@ -0,0 +1,87 @@ +import http.server +import logging +import socket +import threading +import time + +import determined as det + + +class HelloHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.send_header("Content-type", "text/plain") + self.end_headers() + self.wfile.write(b"Hello") + + def do_POST(self) -> None: + self.do_GET() + + +def start_http_server( + server_class=http.server.HTTPServer, handler_class=HelloHandler, port: int = 8000 +) -> None: + server_address = ("", port) + httpd = server_class(server_address, handler_class) + print(f"Starting HTTP server on port {port}") + httpd.serve_forever() + + +def handle_client(conn: socket.socket, addr: tuple) -> None: + print(f"Connected by {addr}") + with conn: + while True: + data = conn.recv(1024) + if not data: + break + try: + print(f"Received: {data.decode()}") + except UnicodeDecodeError: + print(f"Received: {data}") + conn.sendall(data) + + +def start_tcp_server(host: str = "0.0.0.0", port: int = 6000) -> None: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind((host, port)) + s.listen() + print(f"Server listening on {host}:{port}") + + while True: + conn, addr = s.accept() + threading.Thread(target=handle_client, args=(conn, addr)).start() + + +def run_servers() -> None: + http_thread = threading.Thread(target=start_http_server) + tcp_thread = threading.Thread(target=start_tcp_server) + + http_thread.start() + tcp_thread.start() + + print("Servers started") + http_thread.join() + tcp_thread.join() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format=det.LOG_FORMAT) + info = det.get_cluster_info() + if info is None: + run_servers() + exit() + slots_per_node = len(info.slot_ids) + num_nodes = len(info.container_addrs) + cross_rank = info.container_rank + chief_ip = info.container_addrs[0] + print("allocationID", info.allocation_id) + print("containerAddrs", info.container_addrs) + print("taskID", info.task_id) + print("agentID", info.agent_id) + print(f"cross_rank: {cross_rank}, chief_ip: {chief_ip}") + if cross_rank == 0: + run_servers() + else: + print("Not the chief, waiting around...") + while True: + time.sleep(1) diff --git a/go.mod b/go.mod index c64c9068295..5950275e00e 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/go-pg/pg/v10 v10.10.6 github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/protobuf v1.5.4 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.5.0 github.com/gorilla/websocket v1.5.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 @@ -38,12 +38,12 @@ require ( github.com/o1egl/paseto v1.0.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.11.1 + github.com/prometheus/client_golang v1.17.0 github.com/santhosh-tekuri/jsonschema/v2 v2.2.0 github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect github.com/sirupsen/logrus v1.9.0 github.com/soheilhy/cmux v0.1.4 - github.com/spf13/cobra v1.6.1 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.9.0 github.com/stretchr/testify v1.8.4 @@ -57,9 +57,9 @@ require ( gopkg.in/guregu/null.v3 v3.4.0 gopkg.in/segmentio/analytics-go.v3 v3.1.0 gotest.tools v2.2.0+incompatible - k8s.io/api v0.20.14 - k8s.io/apimachinery v0.20.14 - k8s.io/client-go v0.20.14 + k8s.io/api v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/client-go v0.28.3 ) require ( @@ -75,12 +75,6 @@ require ( ) require ( - github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.20 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/logger v0.2.1 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect @@ -89,7 +83,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect @@ -97,15 +91,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/googleapis/gax-go/v2 v2.11.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -117,7 +109,6 @@ require ( github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -126,9 +117,9 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/shopspring/decimal v1.2.0 github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect @@ -157,12 +148,12 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.3.0 // indirect - k8s.io/klog/v2 v2.30.0 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect mellium.im/sasl v0.3.1 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) require ( @@ -196,14 +187,23 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/gorilla/mux v1.8.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/russellhaering/goxmldsig v1.4.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect @@ -212,13 +212,15 @@ require ( golang.org/x/mod v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect ) require ( cloud.google.com/go/compute/metadata v0.2.3 golang.org/x/sync v0.5.0 google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 - k8s.io/component-helpers v0.20.14 + k8s.io/component-helpers v0.28.3 + sigs.k8s.io/gateway-api v1.0.0 ) // Determined AI's CircleCI doesn't have access to "github.hpe.com/hpe/hpc-ard-launcher-go", diff --git a/go.sum b/go.sum index 21b66ae7732..75b36336554 100644 --- a/go.sum +++ b/go.sum @@ -54,26 +54,6 @@ cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7Biccwk dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.20 h1:s8H1PbCZSqg/DH7JMlOz6YMig6htWLNPsjDdlLqCx3M= -github.com/Azure/go-autorest/autorest v0.11.20/go.mod h1:o3tqFY+QR40VOlk+pV4d77mORO64jOXSgEnPQgLK6JY= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.15 h1:X+p2GF0GWyOiSmqohIaEeuNFNDY4I4EOlVuUQvFdWMk= -github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= @@ -86,10 +66,7 @@ github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmy github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -118,7 +95,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.40.34 h1:SBYmodndE2d4AYucuuJnOXk4MD1SFbucoIdpwKVKeSA= @@ -201,8 +177,6 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 h1:90Ly+6UfUypEF6vvvW5rQIv9opIL8CbmW9FT20LDQoY= github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0/go.mod h1:V+Qd57rJe8gd4eiGzZyg4h54VLHmYVVw54iMnlAMrF8= @@ -212,8 +186,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M2y/YtBU2NC2M= github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= @@ -226,8 +200,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -236,18 +210,15 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gavv/httpexpect v2.0.0+incompatible h1:1X9kcRshkSKEjNJJxX9Y9mQ5BRfbxU5kORdjhlA1yX8= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= @@ -265,8 +236,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -275,14 +244,14 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-pg/migrations/v8 v8.1.0 h1:bc1wQwFoWRKvLdluXCRFRkeaw9xDU4qJ63uCAagh66w= github.com/go-pg/migrations/v8 v8.1.0/go.mod h1:o+CN1u572XHphEHZyK6tqyg2GDkRvL2bIoLNyGIewus= github.com/go-pg/pg/v10 v10.4.0/go.mod h1:BfgPoQnD2wXNd986RYEHzikqv9iE875PrFaZ9vXvtNM= @@ -295,6 +264,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -308,7 +279,6 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= @@ -353,6 +323,8 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -365,13 +337,14 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -392,6 +365,7 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= @@ -408,10 +382,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -425,7 +395,6 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= @@ -461,7 +430,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= @@ -481,15 +449,14 @@ github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -553,6 +520,8 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -588,7 +557,6 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -611,8 +579,8 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -641,8 +609,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -675,10 +643,10 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= @@ -694,21 +662,21 @@ github.com/o1egl/paseto v1.0.0/go.mod h1:5HxsZPmw/3RI2pAwGo1HhOOwSdvBpcuVzO7uDkm github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= +github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -730,7 +698,6 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -751,15 +718,16 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= @@ -767,16 +735,17 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= @@ -825,25 +794,22 @@ github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -1005,14 +971,12 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= @@ -1175,7 +1139,6 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1216,7 +1179,6 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201017003518-b09fb700fbb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1232,7 +1194,6 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1242,7 +1203,6 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1280,7 +1240,6 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1300,7 +1259,6 @@ golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1330,7 +1288,6 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1440,7 +1397,6 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1556,7 +1512,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1572,28 +1527,20 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.14 h1:vAI3AdDY0Ou9oAkOy/fQ3K0F+FOT+TyZqykKGKHJYQA= -k8s.io/api v0.20.14/go.mod h1:l/ErofD0cbemY3VGAuqcyQFu0+FoSYD1sOAi6PwUCis= -k8s.io/apimachinery v0.20.14 h1:LG7YY3R3ZRO5UxaIsInDk8adAb9J744CP2EfckAIM7w= -k8s.io/apimachinery v0.20.14/go.mod h1:4KFiDSxCoGviCiRk9kTXIROsIf4VSGkVYjVJjJln3pg= -k8s.io/client-go v0.20.14 h1:DAtFSq905IE49N/WOzI1PvwnifI6Vduti5v8A2xJEt8= -k8s.io/client-go v0.20.14/go.mod h1:NP3va0ehKLBNmXBUIQD6ddTvK7Pu/wioGuitv++pYow= -k8s.io/component-helpers v0.20.14 h1:MoO3BxPMvlJlgzyLxc85zYZnRIf+f7OPc29MfKq/TNI= -k8s.io/component-helpers v0.20.14/go.mod h1:IiAdi9DFhI9PLuPJJOz4zMIVQe+2+S28I8uHM4DHYAI= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/component-helpers v0.28.3 h1:te9ieTGzcztVktUs92X53P6BamAoP73MK0qQP0WmDqc= +k8s.io/component-helpers v0.28.3/go.mod h1:oJR7I9ist5UAQ3y/CTdbw6CXxdMZ1Lw2Ua/EZEwnVLs= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= @@ -1601,10 +1548,13 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/gateway-api v1.0.0 h1:iPTStSv41+d9p0xFydll6d7f7MOBGuqXM6p2/zVYMAs= +sigs.k8s.io/gateway-api v1.0.0/go.mod h1:4cUgr0Lnp5FZ0Cdq8FdRwCvpiWws7LVhLHGIudLlf4c= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/harness/determined/cli/proxy.py b/harness/determined/cli/proxy.py index 59d35e0c21d..04637b23c5e 100644 --- a/harness/determined/cli/proxy.py +++ b/harness/determined/cli/proxy.py @@ -95,7 +95,7 @@ def copy_from_websocket2( ): if isinstance(event, lomond.events.Rejected): print(event.response) - raise Exception("Connection failed: {}".format(event)) + raise Exception(f"Connection {ws.url} failed: {event}") elif isinstance(event, (lomond.events.Closing, lomond.events.Disconnected)): break finally: diff --git a/harness/determined/exec/prep_container.py b/harness/determined/exec/prep_container.py index 7f98e1497a8..18e27e76584 100644 --- a/harness/determined/exec/prep_container.py +++ b/harness/determined/exec/prep_container.py @@ -308,6 +308,12 @@ def set_proxy_address(sess: api.Session, allocation_id: str) -> None: def set_proxy_address_kubernetes(sess: api.Session, allocation_id: str) -> None: + # When using a gateway, we know the static IP in advance so we don't want to tell the + # master the pod IP. Handling this on the master side gets a little tricky with restore. + # For example we send this proxy address before master sends the allocation its address. + if bool(os.environ.get("DET_PROXY_THROUGH_GATEWAY", None)): + return + pod_ip_str = os.environ.get("DET_KUBERNETES_POD_IP") assert pod_ip_str, "Unable to complete rendezvous without DET_KUBERNETES_POD_IP" diff --git a/master/.mockery.yaml b/master/.mockery.yaml index 9c6367970ec..24a9a5ce210 100644 --- a/master/.mockery.yaml +++ b/master/.mockery.yaml @@ -84,6 +84,10 @@ packages: config: filename: node_iface.go mockname: NodeInterface + ServiceInterface: + config: + filename: service_iface.go + mockname: ServiceInterface CoreV1Interface: config: filename: k8s_corev1_iface.go @@ -103,4 +107,16 @@ packages: Interface: config: filename: k8s_clientset.go - mockname: K8sClientsetInterface \ No newline at end of file + mockname: K8sClientsetInterface + sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1: + interfaces: + GatewayInterface: + config: + filename: gateway_iface.go + mockname: GatewayInterface + sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1alpha2: + interfaces: + TCPRouteInterface: + config: + filename: tcproute_iface.go + mockname: TCPRouteInterface diff --git a/master/internal/config/resource_config_test.go b/master/internal/config/resource_config_test.go index 61064644d86..e3621dadf37 100644 --- a/master/internal/config/resource_config_test.go +++ b/master/internal/config/resource_config_test.go @@ -214,6 +214,42 @@ resource_manager: cpu: -10`, nil, "Check Failed! 1 errors found:\n\terror found at root.ResourceConfig." + "RootManagerInternal.KubernetesRM: slot_resource_requests.cpu " + "must be > 0: -10 is not greater than 0"}, + + {"k8s missing gateway_name", ` +resource_manager: + type: kubernetes + max_slots_per_pod: 1 + name: a + internal_task_gateway: + gateway_namespace: test + gateway_ip: 127.0.0.1 + `, nil, "Check Failed! 1 errors found:\n\terror found at " + + "root.ResourceConfig.RootManagerInternal.KubernetesRM.InternalTaskGateway: " + + "invalid gateway_name: must be non-empty"}, + + {"k8s missing gateway_namespace", ` +resource_manager: + type: kubernetes + max_slots_per_pod: 1 + name: a + internal_task_gateway: + gateway_name: test + gateway_ip: 127.0.0.1 + `, nil, "Check Failed! 1 errors found:\n\terror found at " + + "root.ResourceConfig.RootManagerInternal.KubernetesRM.InternalTaskGateway: " + + "invalid gateway_namespace: must be non-empty"}, + + {"k8s missing gateway_ip", ` +resource_manager: + type: kubernetes + max_slots_per_pod: 1 + name: a + internal_task_gateway: + gateway_name: test + gateway_namespace: abc + `, nil, "Check Failed! 1 errors found:\n\terror found at " + + "root.ResourceConfig.RootManagerInternal.KubernetesRM.InternalTaskGateway: " + + "invalid gateway_ip: must be non-empty"}, } RegisterAuthZType("basic") diff --git a/master/internal/config/resource_manager_config.go b/master/internal/config/resource_manager_config.go index a8f1fd656c3..4815cfbc208 100644 --- a/master/internal/config/resource_manager_config.go +++ b/master/internal/config/resource_manager_config.go @@ -11,7 +11,11 @@ import ( "github.com/determined-ai/determined/master/pkg/union" ) -const defaultResourcePoolName = "default" +const ( + defaultResourcePoolName = "default" + validGWPortRangeStart = 1025 + validGWPortRangeEnd = 65535 +) // ResourceManagerConfig hosts configuration fields for the resource manager. type ResourceManagerConfig struct { @@ -171,10 +175,73 @@ type KubernetesResourceManagerConfig struct { DefaultComputeResourcePool string `json:"default_compute_resource_pool"` NoDefaultResourcePools bool `json:"no_default_resource_pools"` + InternalTaskGateway *InternalTaskGatewayConfig `json:"internal_task_gateway"` + Name string `json:"name"` Metadata map[string]string `json:"metadata"` } +// InternalTaskGatewayConfig is config for exposing Determined tasks to outside of the cluster. +// Useful for multirm when we can only be running in a single cluster. +type InternalTaskGatewayConfig struct { + // GatewayName as defined in the k8s cluster. + GatewayName string `json:"gateway_name"` + // GatewayNamespace as defined in the k8s cluster. + GatewayNamespace string `json:"gateway_namespace"` + GatewayIP string `json:"gateway_ip"` + // GWPortStart denotes the inclusive start of the available and exclusive port range to + // MLDE for InternalTaskGateway. + GWPortStart int `json:"gateway_port_range_start"` + // GWPortEnd denotes the inclusive end of the available and exclusive port range to + // MLDE for InternalTaskGateway. + GWPortEnd int `json:"gateway_port_range_end"` +} + +var defaultInternalTaskGatewayConfig = InternalTaskGatewayConfig{ + GWPortStart: 32768, + GWPortEnd: 65535, +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (i *InternalTaskGatewayConfig) UnmarshalJSON(data []byte) error { + *i = defaultInternalTaskGatewayConfig + type DefaultParser *InternalTaskGatewayConfig + return json.Unmarshal(data, DefaultParser(i)) +} + +// Validate implements the check.Validatable interface. +func (i *InternalTaskGatewayConfig) Validate() []error { + var errs []error + + if err := check.IsValidK8sLabel(i.GatewayName); err != nil { + errs = append(errs, fmt.Errorf("invalid gateway_name: %w", err)) + } + + if err := check.IsValidK8sLabel(i.GatewayNamespace); err != nil { + errs = append(errs, fmt.Errorf("invalid gateway_namespace: %w", err)) + } + + // Don't validate the IP just check it is not empty so hostnames and the like can be used. + if err := check.NotEmpty(i.GatewayIP); err != nil { + errs = append(errs, fmt.Errorf("invalid gateway_ip: %w", err)) + } + + if err := check.BetweenInclusive( + i.GWPortStart, validGWPortRangeStart, validGWPortRangeEnd); err != nil { + errs = append(errs, fmt.Errorf("invalid gateway_port_range_start: %w", err)) + } + + if err := check.BetweenInclusive( + i.GWPortEnd, validGWPortRangeStart, validGWPortRangeEnd); err != nil { + errs = append(errs, fmt.Errorf("invalid gateway_port_range_end: %w", err)) + } + + if i.GWPortStart >= i.GWPortEnd { + errs = append(errs, fmt.Errorf("gateway_port_range_start must be less than or equal to gateway_port_range_end")) + } + return errs +} + var defaultKubernetesResourceManagerConfig = KubernetesResourceManagerConfig{ SlotType: device.CUDA, // default to CUDA-backed slots. } diff --git a/master/internal/rm/kubernetesrm/gateway_service.go b/master/internal/rm/kubernetesrm/gateway_service.go new file mode 100644 index 00000000000..295653842f0 --- /dev/null +++ b/master/internal/rm/kubernetesrm/gateway_service.go @@ -0,0 +1,168 @@ +package kubernetesrm + +import ( + "context" + "fmt" + "slices" + "sync" + + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + gatewayTyped "sigs.k8s.io/gateway-api/apis/v1" + gateway "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1" + + alphaGateway "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1alpha2" + + "github.com/determined-ai/determined/master/internal/config" + "github.com/determined-ai/determined/master/pkg/model" +) + +type gatewayService struct { + mu sync.Mutex + gatewayInterface gateway.GatewayInterface + tcpRouteInterfaces map[string]alphaGateway.TCPRouteInterface + gatewayName string + portRangeStart int + portRangeEnd int +} + +type gatewayResourceComm struct { + requestedPorts int + resourceDescriptor proxyResourceGenerator + allocationID model.AllocationID + reportResources func([]gatewayProxyResource) +} + +func newGatewayService( + gatewayInterface gateway.GatewayInterface, + tcpRouteInterfaces map[string]alphaGateway.TCPRouteInterface, + taskGWConfig config.InternalTaskGatewayConfig, +) (*gatewayService, error) { + g := &gatewayService{ + gatewayInterface: gatewayInterface, + tcpRouteInterfaces: tcpRouteInterfaces, + gatewayName: taskGWConfig.GatewayName, + portRangeStart: taskGWConfig.GWPortStart, + portRangeEnd: taskGWConfig.GWPortEnd, + } + return g, nil +} + +func (g *gatewayService) generateAndAddListeners(allocationID model.AllocationID, count int) ([]int, error) { + var ports []int + if err := g.updateGateway(func(gateway *gatewayTyped.Gateway) error { + var err error + listeners := make([]gatewayTyped.Listener, count) + ports, err = g.pickNFreePorts(gateway, len(listeners)) + if err != nil { + return err + } + + if gateway.Annotations == nil { + gateway.Annotations = make(map[string]string) + } + + for i := 0; i < count; i++ { + gateway.Annotations[portToAnnotationKey(ports[i])] = string(allocationID) + listeners[i] = createListenerForPod(ports[i]) + } + gateway.Spec.Listeners = append(gateway.Spec.Listeners, listeners...) + + return nil + }); err != nil { + return nil, fmt.Errorf("adding %d listeners to gateway: %w", count, err) + } + return ports, nil +} + +func (g *gatewayService) pickNFreePorts(gateway *gatewayTyped.Gateway, count int) ([]int, error) { + usedPorts := make(map[int]struct{}, len(gateway.Spec.Listeners)) + for _, listener := range gateway.Spec.Listeners { + usedPorts[int(listener.Port)] = struct{}{} + } + ports := make([]int, 0, count) + for port := g.portRangeStart; port <= g.portRangeEnd; port++ { + if _, used := usedPorts[port]; !used { + ports = append(ports, port) + if len(ports) == count { + return ports, nil + } + } + } + return nil, fmt.Errorf("not enough free ports in range %d-%d", g.portRangeStart, g.portRangeEnd) +} + +func (g *gatewayService) freePorts(ports []int) error { + if err := g.updateGateway(func(gateway *gatewayTyped.Gateway) error { + if gateway.Annotations == nil { + gateway.Annotations = make(map[string]string) + } + + var newListeners []gatewayTyped.Listener + for _, l := range gateway.Spec.Listeners { + if slices.Contains(ports, int(l.Port)) { + delete(gateway.Annotations, portToAnnotationKey(int(l.Port))) + } else { + newListeners = append(newListeners, l) + } + } + + gateway.Spec.Listeners = newListeners + return nil + }); err != nil { + return fmt.Errorf("freeing ports %v from gateway: %w", ports, err) + } + return nil +} + +func (g *gatewayService) getProxyPorts(allocationID *model.AllocationID) ([]int, error) { + g.mu.Lock() + defer g.mu.Unlock() + + gateway, err := g.gatewayInterface.Get(context.TODO(), g.gatewayName, metaV1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("getting gateway for proxy ports: %w", err) + } + + var ports []int + for _, listener := range gateway.Spec.Listeners { + portAllocationID := gateway.Annotations[portToAnnotationKey(int(listener.Port))] + if portAllocationID == "" { + continue // Don't touch ports that we didn't add. + } + if allocationID != nil && string(*allocationID) != portAllocationID { + continue + } + + ports = append(ports, int(listener.Port)) + } + + return ports, nil +} + +func (g *gatewayService) updateGateway(update func(*gatewayTyped.Gateway) error) error { + g.mu.Lock() + defer g.mu.Unlock() + + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + gateway, err := g.gatewayInterface.Get(context.TODO(), g.gatewayName, metaV1.GetOptions{}) + if err != nil { + return err + } + + if err = update(gateway); err != nil { + return err + } + + if _, err := g.gatewayInterface. + Update(context.TODO(), gateway, metaV1.UpdateOptions{}); err != nil { + return err + } + + return nil + }); err != nil { + return fmt.Errorf("updating gateway with name '%s': %w", g.gatewayName, err) + } + + return nil +} diff --git a/master/internal/rm/kubernetesrm/gateway_service_test.go b/master/internal/rm/kubernetesrm/gateway_service_test.go new file mode 100644 index 00000000000..2eb2818c887 --- /dev/null +++ b/master/internal/rm/kubernetesrm/gateway_service_test.go @@ -0,0 +1,269 @@ +package kubernetesrm + +import ( + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayTyped "sigs.k8s.io/gateway-api/apis/v1" + alphaGateway "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1alpha2" + + "github.com/determined-ai/determined/master/internal/config" + "github.com/determined-ai/determined/master/internal/mocks" +) + +func testGatewayConfig() config.InternalTaskGatewayConfig { + return config.InternalTaskGatewayConfig{ + GWPortStart: 1, GWPortEnd: 100, + GatewayName: "gatewayname", + } +} + +func TestGatewayServiceAddListeners(t *testing.T) { + gatewayMock := &mocks.GatewayInterface{} + tcpRouteMock := &mocks.TCPRouteInterface{} + tcpInterfaces := map[string]alphaGateway.TCPRouteInterface{ + "default": tcpRouteMock, + } + g, err := newGatewayService( + gatewayMock, tcpInterfaces, testGatewayConfig(), + ) + require.NoError(t, err) + + toReturn := &gatewayTyped.Gateway{ + ObjectMeta: metaV1.ObjectMeta{ + Annotations: map[string]string{ + portToAnnotationKey(1): "alloc", + }, + }, + Spec: gatewayTyped.GatewaySpec{ + Listeners: []gatewayTyped.Listener{ + createListenerForPod(1), + }, + }, + } + + expected := &gatewayTyped.Gateway{ + ObjectMeta: metaV1.ObjectMeta{ + Annotations: map[string]string{ + portToAnnotationKey(1): "alloc", + portToAnnotationKey(2): "alloc", + portToAnnotationKey(3): "alloc", + }, + }, + Spec: gatewayTyped.GatewaySpec{ + Listeners: []gatewayTyped.Listener{ + createListenerForPod(1), + createListenerForPod(2), + createListenerForPod(3), + }, + }, + } + + gatewayMock.On("Get", mock.Anything, "gatewayname", metaV1.GetOptions{}).Return(toReturn, nil) + gatewayMock.On("Update", mock.Anything, expected, metaV1.UpdateOptions{}).Return(nil, nil) + ports, err := g.generateAndAddListeners("alloc", 2) + require.Len(t, ports, 2) + require.Equal(t, []int{2, 3}, ports) + require.NoError(t, err) + gatewayMock.AssertExpectations(t) +} + +func TestGatewayServiceFreePorts(t *testing.T) { + gatewayMock := &mocks.GatewayInterface{} + tcpRouteMock := &mocks.TCPRouteInterface{} + tcpInterfaces := map[string]alphaGateway.TCPRouteInterface{ + "default": tcpRouteMock, + } + g, err := newGatewayService( + gatewayMock, tcpInterfaces, testGatewayConfig(), + ) + require.NoError(t, err) + + toReturn := &gatewayTyped.Gateway{ + ObjectMeta: metaV1.ObjectMeta{ + Annotations: map[string]string{ + portToAnnotationKey(1): "alloc", + portToAnnotationKey(2): "alloc", + portToAnnotationKey(3): "alloc", + }, + }, + Spec: gatewayTyped.GatewaySpec{ + Listeners: []gatewayTyped.Listener{ + { + Port: 1, + }, + { + Port: 2, + }, + { + Port: 3, + }, + }, + }, + } + + expected := &gatewayTyped.Gateway{ + ObjectMeta: metaV1.ObjectMeta{ + Annotations: map[string]string{ + portToAnnotationKey(2): "alloc", + }, + }, + Spec: gatewayTyped.GatewaySpec{ + Listeners: []gatewayTyped.Listener{ + { + Port: 2, + }, + }, + }, + } + + gatewayMock.On("Get", mock.Anything, "gatewayname", metaV1.GetOptions{}).Return(toReturn, nil) + gatewayMock.On("Update", mock.Anything, expected, metaV1.UpdateOptions{}).Return(nil, nil) + require.NoError(t, g.freePorts([]int{1, 3, 4})) + + gatewayMock.AssertExpectations(t) +} + +func TestGatewayServicePickPorts(t *testing.T) { + testCases := []struct { + name string + rangeStart int + rangeEnd int + usedPorts []int + requestedCount int + expectedPorts []int + }{ + { + name: "no ports available", + rangeStart: 1, + rangeEnd: 9, + usedPorts: []int{1, 2, 3, 4, 5, 6, 7, 8, 9}, + requestedCount: 1, + expectedPorts: nil, + }, + { + name: "some ports available", + rangeStart: 1, + rangeEnd: 9, + usedPorts: []int{1, 2, 3, 4, 5, 6}, + requestedCount: 2, + expectedPorts: []int{7, 8}, + }, + { + name: "exact number of ports available", + rangeStart: 1, + rangeEnd: 9, + usedPorts: []int{1, 2, 3, 4, 5, 6, 7}, + requestedCount: 2, + expectedPorts: []int{8, 9}, + }, + { + name: "more requested than available", + rangeStart: 1, + rangeEnd: 5, + usedPorts: []int{1, 2}, + requestedCount: 4, + expectedPorts: nil, + }, + { + name: "all ports available", + rangeStart: 10, + rangeEnd: 20, + usedPorts: []int{}, + requestedCount: 3, + expectedPorts: []int{10, 11, 12}, + }, + { + name: "some ports used in the middle of the range", + rangeStart: 1, + rangeEnd: 10, + usedPorts: []int{5, 6, 7}, + requestedCount: 3, + expectedPorts: []int{1, 2, 3}, + }, + { + name: "used ports at the end of range", + rangeStart: 1, + rangeEnd: 5, + usedPorts: []int{4, 5}, + requestedCount: 2, + expectedPorts: []int{1, 2}, + }, + { + name: "non-sequential used ports", + rangeStart: 1, + rangeEnd: 10, + usedPorts: []int{2, 4, 6, 8, 10}, + requestedCount: 3, + expectedPorts: []int{1, 3, 5}, + }, + { + name: "used ports outside range, enough ports available", + rangeStart: 10, + rangeEnd: 20, + usedPorts: []int{1, 2, 3, 11, 12}, + requestedCount: 3, + expectedPorts: []int{10, 13, 14}, + }, + { + name: "used ports outside range, not enough ports available", + rangeStart: 10, + rangeEnd: 15, + usedPorts: []int{1, 2, 3, 11, 12}, + requestedCount: 5, + expectedPorts: nil, + }, + { + name: "used ports within and outside range", + rangeStart: 20, + rangeEnd: 30, + usedPorts: []int{15, 18, 21, 25, 28, 35}, + requestedCount: 4, + expectedPorts: []int{20, 22, 23, 24}, + }, + { + name: "all used ports outside range", + rangeStart: 5, + rangeEnd: 10, + usedPorts: []int{1, 2, 3, 11, 12, 13}, + requestedCount: 3, + expectedPorts: []int{5, 6, 7}, + }, + { + name: "mixed ports inside and outside range, exact count available", + rangeStart: 100, + rangeEnd: 105, + usedPorts: []int{95, 96, 100, 101, 102}, + requestedCount: 3, + expectedPorts: []int{103, 104, 105}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + listeners := []gatewayTyped.Listener{} + for _, port := range tc.usedPorts { + listeners = append(listeners, createListenerForPod(port)) + } + gatewayGetReturn := &gatewayTyped.Gateway{ + Spec: gatewayTyped.GatewaySpec{ + Listeners: listeners, + }, + } + g := &gatewayService{ + portRangeStart: tc.rangeStart, + portRangeEnd: tc.rangeEnd, + } + ports, err := g.pickNFreePorts(gatewayGetReturn, tc.requestedCount) + if tc.expectedPorts == nil { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.expectedPorts, ports) + }) + } +} diff --git a/master/internal/rm/kubernetesrm/gateway_spec.go b/master/internal/rm/kubernetesrm/gateway_spec.go new file mode 100644 index 00000000000..1a3f129282d --- /dev/null +++ b/master/internal/rm/kubernetesrm/gateway_spec.go @@ -0,0 +1,31 @@ +package kubernetesrm + +import ( + "fmt" + + gatewayTyped "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/determined-ai/determined/master/pkg/ptrs" +) + +func generateListenerName(p int) string { + return fmt.Sprintf("det-%d", p) +} + +func createListenerForPod(gwPort int) gatewayTyped.Listener { + gatewayListener := gatewayTyped.Listener{ + Name: gatewayTyped.SectionName(generateListenerName(gwPort)), + Port: gatewayTyped.PortNumber(gwPort), + Protocol: "TCP", + AllowedRoutes: &gatewayTyped.AllowedRoutes{ + Namespaces: &gatewayTyped.RouteNamespaces{ + From: ptrs.Ptr(gatewayTyped.NamespacesFromAll), + }, + }, + } + return gatewayListener +} + +func portToAnnotationKey(port int) string { + return fmt.Sprintf("determined.ai/det-gateway-port-%d", port) +} diff --git a/master/internal/rm/kubernetesrm/job.go b/master/internal/rm/kubernetesrm/job.go index afcf1ee453c..fcaa5814385 100644 --- a/master/internal/rm/kubernetesrm/job.go +++ b/master/internal/rm/kubernetesrm/job.go @@ -17,6 +17,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sClient "k8s.io/client-go/kubernetes" typedV1 "k8s.io/client-go/kubernetes/typed/core/v1" + gatewayTyped "sigs.k8s.io/gateway-api/apis/v1" + alphaGatewayTyped "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/determined-ai/determined/master/internal/config" "github.com/determined-ai/determined/master/internal/rm/rmevents" @@ -30,6 +32,26 @@ import ( "github.com/determined-ai/determined/master/pkg/tasks" ) +type gatewayProxyResource struct { + serviceSpec *k8sV1.Service + tcpRouteSpec *alphaGatewayTyped.TCPRoute + gatewayListener gatewayTyped.Listener +} + +func (g gatewayProxyResource) PodPort() int { + return int(g.serviceSpec.Spec.Ports[0].Port) +} + +func (g gatewayProxyResource) GWPort() int { + return int(g.gatewayListener.Port) +} + +func (g gatewayProxyResource) SetGWPort(port int) { + gwPort := gatewayTyped.PortNumber(port) + g.gatewayListener.Port = gwPort + g.tcpRouteSpec.Spec.CommonRouteSpec.ParentRefs[0].Port = &gwPort +} + var successfulExit = exitReason{} // describes why a job failed. empty value indicates success. @@ -63,7 +85,12 @@ type job struct { masterTLSConfig model.TLSClientConfig jobName string configMapName string - allocationID model.AllocationID + + gatewayProxyResources []gatewayProxyResource + internalTaskGWConfig *config.InternalTaskGatewayConfig + gatewayService *gatewayService + + allocationID model.AllocationID // req.State is mutated, we should change this. req *sproto.AllocateRequest // Kubernetes-specific request information. @@ -114,6 +141,8 @@ func newJob( slotType device.Type, slotResourceRequests config.PodSlotResourceRequests, scheduler string, + internalTaskGWConfig *config.InternalTaskGatewayConfig, + gatewayService *gatewayService, ) *job { // The lifecycle of the containers specified in this map will be monitored. // As soon as one or more of them exits, the pod will be terminated. @@ -149,6 +178,8 @@ func newJob( scheduler: scheduler, slotType: slotType, slotResourceRequests: slotResourceRequests, + internalTaskGWConfig: internalTaskGWConfig, + gatewayService: gatewayService, syslog: logrus.WithField("component", "job").WithFields( logger.MergeContexts(msg.logContext, logger.Context{ "job": name, @@ -264,6 +295,43 @@ func (j *job) jobDeletedCallback() { j.informTaskResourcesStopped() } +func (j *job) makeGatewayComms(spec *tasks.TaskSpec) *gatewayResourceComm { + if j.internalTaskGWConfig == nil { + return nil + } + + updateResources := func(resources []gatewayProxyResource) { + j.mu.Lock() + defer j.mu.Unlock() + j.gatewayProxyResources = resources + } + + return &gatewayResourceComm{ + resourceDescriptor: j.configureProxyResources(spec), + reportResources: updateResources, + allocationID: j.req.AllocationID, + requestedPorts: len(j.req.ProxyPorts), + } +} + +func (j *job) getGatewayAddresses() []cproto.Address { + if j.internalTaskGWConfig == nil { + return nil + } + + var addresses []cproto.Address + for _, g := range j.gatewayProxyResources { + addresses = append(addresses, cproto.Address{ + ContainerIP: j.internalTaskGWConfig.GatewayIP, + HostIP: j.internalTaskGWConfig.GatewayIP, + ContainerPort: g.PodPort(), + HostPort: g.GWPort(), + }) + } + + return addresses +} + func (j *job) podUpdatedCallback(updatedPod k8sV1.Pod) error { j.mu.Lock() defer j.mu.Unlock() @@ -322,7 +390,11 @@ func (j *job) podUpdatedCallback(updatedPod k8sV1.Pod) error { if allPodsFound && allPodsAtLeastRunning && !j.sentRunningEvent { j.syslog.WithField("pod-name", podName).Info("pod is running") j.container.State = cproto.Running - j.informTaskResourcesStarted(sproto.ResourcesStarted{NativeResourcesID: j.jobName}) + j.informTaskResourcesStarted(sproto.ResourcesStarted{ + NativeResourcesID: j.jobName, + Addresses: j.getGatewayAddresses(), + }) + j.sentRunningEvent = true } @@ -419,8 +491,23 @@ func (j *job) kill() { return } + var serviceNames, tcpRouteNames []string + var gatewayPortsToFree []int + for _, g := range j.gatewayProxyResources { + serviceNames = append(serviceNames, g.serviceSpec.Name) + tcpRouteNames = append(tcpRouteNames, g.tcpRouteSpec.Name) + gatewayPortsToFree = append(gatewayPortsToFree, int(g.gatewayListener.Port)) + } + j.syslog.Infof("requesting to delete kubernetes resources %s", j.jobName) - j.resourceRequestQueue.deleteKubernetesResources(j.namespace, j.jobName, j.configMapName, "") + j.resourceRequestQueue.deleteKubernetesResources(deleteKubernetesResources{ + namespace: j.namespace, + jobName: j.jobName, + configMapName: j.configMapName, + serviceNames: serviceNames, + tcpRouteNames: tcpRouteNames, + gatewayPortsToFree: gatewayPortsToFree, + }) } func (j *job) killPod(name string) { @@ -429,7 +516,10 @@ func (j *job) killPod(name string) { } j.syslog.Infof("requesting to delete kubernetes resources %s", j.jobName) - j.resourceRequestQueue.deleteKubernetesResources(j.namespace, "", "", name) + j.resourceRequestQueue.deleteKubernetesResources(deleteKubernetesResources{ + namespace: j.namespace, + podName: name, + }) j.podKillSent[name] = true } @@ -484,7 +574,7 @@ func (j *job) createSpecAndSubmit(spec *tasks.TaskSpec) error { return err } - j.resourceRequestQueue.createKubernetesResources(jobSpec, configMapSpec) + j.resourceRequestQueue.createKubernetesResources(jobSpec, configMapSpec, j.makeGatewayComms(spec)) return nil } diff --git a/master/internal/rm/kubernetesrm/jobs.go b/master/internal/rm/kubernetesrm/jobs.go index 5582d74fb7d..314c8111a34 100644 --- a/master/internal/rm/kubernetesrm/jobs.go +++ b/master/internal/rm/kubernetesrm/jobs.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/exp/maps" @@ -30,6 +31,9 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + alphaGatewayTyped "sigs.k8s.io/gateway-api/apis/v1alpha2" + gateway "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1" + alphaGateway "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1alpha2" "github.com/determined-ai/determined/master/internal/config" "github.com/determined-ai/determined/master/internal/db" @@ -52,6 +56,7 @@ const ( determinedLabel = "determined" determinedPreemptionLabel = "determined-preemption" determinedSystemLabel = "determined-system" + jobNameAnnotation = "determined.ai/job-name" kubernetesJobNameLabel = "batch.kubernetes.io/job-name" @@ -92,12 +97,17 @@ type jobsService struct { detMasterPort int32 kubeconfigPath string + internalTaskGWConfig *config.InternalTaskGatewayConfig + // System dependencies. Also set in initialization and never modified after. - syslog *logrus.Entry - clientSet k8sClient.Interface - podInterfaces map[string]typedV1.PodInterface - configMapInterfaces map[string]typedV1.ConfigMapInterface - jobInterfaces map[string]typedBatchV1.JobInterface + syslog *logrus.Entry + clientSet k8sClient.Interface + podInterfaces map[string]typedV1.PodInterface + configMapInterfaces map[string]typedV1.ConfigMapInterface + jobInterfaces map[string]typedBatchV1.JobInterface + serviceInterfaces map[string]typedV1.ServiceInterface + tcpRouteInterfaces map[string]alphaGateway.TCPRouteInterface + resourceRequestQueue *requestQueue jobSchedulingStateCallback jobSchedulingStateCallback @@ -111,6 +121,8 @@ type jobsService struct { jobHandlerToMetadata map[*job]jobMetadata nodeToSystemResourceRequests map[string]int64 currentNodes map[string]*k8sV1.Node + gatewayService *gatewayService + // TODO(RM-236) make one cache and make this code more straightforward. summarizeCacheLock sync.RWMutex summarizeCache summarizeResult @@ -135,6 +147,7 @@ func newJobsService( detMasterPort int32, kubeconfigPath string, jobSchedulingStateCb jobSchedulingStateCallback, + internalTaskGWConfig *config.InternalTaskGatewayConfig, ) (*jobsService, error) { p := &jobsService{ wg: waitgroupx.WithContext(context.Background()), @@ -160,10 +173,13 @@ func newJobsService( podInterfaces: make(map[string]typedV1.PodInterface), configMapInterfaces: make(map[string]typedV1.ConfigMapInterface), jobInterfaces: make(map[string]typedBatchV1.JobInterface), + serviceInterfaces: make(map[string]typedV1.ServiceInterface), + tcpRouteInterfaces: make(map[string]alphaGateway.TCPRouteInterface), syslog: logrus.WithField("namespace", namespace), jobSchedulingStateCallback: jobSchedulingStateCb, - kubeconfigPath: kubeconfigPath, + internalTaskGWConfig: internalTaskGWConfig, + kubeconfigPath: kubeconfigPath, } if err := p.startClientSet(); err != nil { @@ -207,7 +223,7 @@ func newJobsService( factory := informers.NewSharedInformerFactoryWithOptions(p.clientSet, time.Hour, informers.WithNamespace(namespace)) jobsInformer := factory.Batch().V1().Jobs() - jobsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + if _, err := jobsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { p.mu.Lock() defer p.mu.Unlock() @@ -226,11 +242,13 @@ func newJobsService( defer p.mu.Unlock() p.jobDeletedCallback(obj) }, - }) + }); err != nil { + return nil, fmt.Errorf("adding job informer: %w", err) + } cacheSyncs = append(cacheSyncs, jobsInformer.Informer().HasSynced) podsInformer := factory.Core().V1().Pods() - podsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + if _, err := podsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { p.mu.Lock() defer p.mu.Unlock() @@ -249,7 +267,9 @@ func newJobsService( defer p.mu.Unlock() p.podDeletedCallback(obj) }, - }) + }); err != nil { + return nil, fmt.Errorf("adding pod informer: %w", err) + } cacheSyncs = append(cacheSyncs, podsInformer.Informer().HasSynced) factory.Start(nil) @@ -271,12 +291,40 @@ func (j *jobsService) startClientSet() error { return fmt.Errorf("failed to initialize kubernetes clientSet: %w", err) } + namespaces := append(maps.Keys(j.namespaceToPoolName), j.namespace) for _, ns := range append(maps.Keys(j.namespaceToPoolName), j.namespace) { j.podInterfaces[ns] = j.clientSet.CoreV1().Pods(ns) j.configMapInterfaces[ns] = j.clientSet.CoreV1().ConfigMaps(ns) j.jobInterfaces[ns] = j.clientSet.BatchV1().Jobs(ns) } + if taskGWConfig := j.internalTaskGWConfig; taskGWConfig != nil { + // Using the CoreV1 RESTClient for gateway resources will cause "resource not found" errors. + alphaGatewayClientSet, err := alphaGateway.NewForConfig(config) + if err != nil { + return fmt.Errorf("creating Kubernetes gateway clientSet: %w", err) + } + for _, ns := range namespaces { + j.serviceInterfaces[ns] = j.clientSet.CoreV1().Services(ns) + j.tcpRouteInterfaces[ns] = alphaGatewayClientSet.TCPRoutes(ns) + } + + // Using the alphaGateway clientSet will not work properly. + gatewayClientSet, err := gateway.NewForConfig(config) + if err != nil { + return fmt.Errorf("creating Kubernetes gateway clientSet: %w", err) + } + gwService, err := newGatewayService( + gatewayClientSet.Gateways(taskGWConfig.GatewayNamespace), + j.tcpRouteInterfaces, + *taskGWConfig, + ) + if err != nil { + return fmt.Errorf("creating gateway service: %w", err) + } + j.gatewayService = gwService + } + j.syslog.Infof("kubernetes clientSet initialized") return nil } @@ -365,9 +413,9 @@ func (j *jobsService) deleteDoomedKubernetesResources() error { return fmt.Errorf("error listing existing pods: %w", err) } - toKillJobs := &batchV1.JobList{} + var toKillJobs []batchV1.Job savedJobNames := make(set.Set[string]) - for _, job := range jobs.Items { + for _, job := range jobs { if _, ok := j.namespaceToPoolName[job.Namespace]; !ok { continue } @@ -375,14 +423,14 @@ func (j *jobsService) deleteDoomedKubernetesResources() error { resourcePool := job.Labels[resourcePoolLabel] if resourcePool == "" { j.syslog.Warnf("deleting job '%s' without resource pool label", job.Name) - toKillJobs.Items = append(toKillJobs.Items, job) + toKillJobs = append(toKillJobs, job) continue } allocationIDStr := job.Labels[allocationIDLabel] if allocationIDStr == "" { j.syslog.Warnf("deleting job '%s' without determined label (whose value is the allocation ID)", job.Name) - toKillJobs.Items = append(toKillJobs.Items, job) + toKillJobs = append(toKillJobs, job) continue } allocationID := model.AllocationID(allocationIDStr) @@ -391,32 +439,93 @@ func (j *jobsService) deleteDoomedKubernetesResources() error { j.syslog. WithField("allocation-id", allocationID). Warnf("deleting job '%s', did not find an open allocation for it", job.Name) - toKillJobs.Items = append(toKillJobs.Items, job) + toKillJobs = append(toKillJobs, job) continue } savedJobNames.Insert(job.Name) } + resourceIsSaved := func(namespace, jobName string) bool { + if _, ok := j.namespaceToPoolName[namespace]; !ok { + return true + } + + return savedJobNames.Contains(jobName) + } configMaps, err := j.listConfigMapsInAllNamespaces(context.TODO(), listOptions) if err != nil { return fmt.Errorf("error listing existing config maps: %w", err) } - toKillConfigMaps := &k8sV1.ConfigMapList{} - for _, cm := range configMaps.Items { - if _, ok := j.namespaceToPoolName[cm.Namespace]; !ok { + var toKillConfigMaps []k8sV1.ConfigMap + for _, cm := range configMaps { + // Config map name and job name are the same. + if resourceIsSaved(cm.Namespace, cm.Name) { continue } - if savedJobNames.Contains(cm.Name) { // Job name is same as config map name. - continue + j.syslog.Debugf("deleting config map '%s', did not find a matching job that will be restored", cm.Name) + toKillConfigMaps = append(toKillConfigMaps, cm) + } + + var toKillServices []k8sV1.Service + var toKillTCPRoutes []alphaGatewayTyped.TCPRoute + var toFreeGatewayPorts []int + if j.internalTaskGWConfig != nil { + services, err := j.listServicesInAllNamespaces(context.TODO(), listOptions) + if err != nil { + return fmt.Errorf("listing existing services: %w", err) + } + for _, s := range services { + if resourceIsSaved(s.Namespace, s.Annotations[jobNameAnnotation]) { + continue + } + + j.syslog.Debugf("deleting service '%s', did not find a matching job that will be restored", s.Name) + toKillServices = append(toKillServices, s) } - j.syslog.Debugf("deleting config map '%s', did not find a matching job that will be restored", cm.Name) - toKillConfigMaps.Items = append(toKillConfigMaps.Items, cm) + savedGatewayPorts := make(map[int]bool) + tcpRoutes, err := j.listTCPRoutesInAllNamespaces(context.TODO(), listOptions) + if err != nil { + return fmt.Errorf("listing existing services: %w", err) + } + for _, t := range tcpRoutes { + if resourceIsSaved(t.Namespace, t.Annotations[jobNameAnnotation]) { + for _, s := range t.Spec.ParentRefs { + if p := s.Port; p != nil { + savedGatewayPorts[int(*p)] = true + } + } + + continue + } + + j.syslog.Debugf("deleting TCPRoute '%s', did not find a matching job that will be restored", t.Name) + toKillTCPRoutes = append(toKillTCPRoutes, t) + } + + gatewayPorts, err := j.gatewayService.getProxyPorts(nil) + if err != nil { + return fmt.Errorf("listing gateway ports: %w", err) + } + for _, p := range gatewayPorts { + if savedGatewayPorts[p] { + continue + } + + j.syslog.Debugf("freeing Gateway port '%d', did not find a matching job that will be restored", p) + toFreeGatewayPorts = append(toFreeGatewayPorts, p) + } } - j.deleteKubernetesResources(toKillJobs, toKillConfigMaps) + j.deleteKubernetesResources( + toKillJobs, + toKillConfigMaps, + toKillServices, + toKillTCPRoutes, + toFreeGatewayPorts, + ) return nil } @@ -457,6 +566,8 @@ func (j *jobsService) startJob(msg startJob) error { j.slotType, j.slotResourceRequests, j.scheduler, + j.internalTaskGWConfig, + j.gatewayService, ) if _, alreadyExists := j.jobNameToJobHandler[newJobHandler.jobName]; alreadyExists { @@ -523,39 +634,84 @@ type reattachJobResponse struct { } func (j *jobsService) reattachJob(msg reattachJobRequest) (reattachJobResponse, error) { + // Get all expected resources for the job. listOptions := metaV1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s", determinedLabel, msg.allocationID), } + var errs *multierror.Error jobs, err := j.listJobsInAllNamespaces(context.TODO(), listOptions) - if err != nil { - return reattachJobResponse{}, fmt.Errorf("error listing pods checking if they can be restored: %w", err) - } + errs = multierror.Append(errs, err) configMaps, err := j.listConfigMapsInAllNamespaces(context.TODO(), listOptions) - if err != nil { - return reattachJobResponse{}, fmt.Errorf("error listing config maps checking if they can be restored: %w", err) + errs = multierror.Append(errs, err) + + var services []k8sV1.Service + var tcpRoutes []alphaGatewayTyped.TCPRoute + var gatewayPorts []int + if j.internalTaskGWConfig != nil { + services, err = j.listServicesInAllNamespaces(context.TODO(), listOptions) + errs = multierror.Append(errs, err) + + tcpRoutes, err = j.listTCPRoutesInAllNamespaces(context.TODO(), listOptions) + errs = multierror.Append(errs, err) + + gatewayPorts, err = j.gatewayService.getProxyPorts(&msg.allocationID) + errs = multierror.Append(errs, err) } - existingConfigMaps := make(set.Set[string]) - for _, cm := range configMaps.Items { - if _, ok := j.namespaceToPoolName[cm.Namespace]; !ok { - continue + + // Do a sanity check validate. Is this a job that can reattach? + // Err on the side of caution here. + if len(jobs) != 1 { + errs = multierror.Append(errs, fmt.Errorf("expected one job got %d", len(jobs))) + } + if len(configMaps) != 1 { + errs = multierror.Append(errs, fmt.Errorf("expected one config map got %d", len(configMaps))) + } + expectedProxyNum := len(msg.req.ProxyPorts) + if j.internalTaskGWConfig != nil && expectedProxyNum > 0 { + if len(services) != expectedProxyNum { + errs = multierror.Append(errs, + fmt.Errorf("expected %d services got %d", expectedProxyNum, len(services))) + } + if len(tcpRoutes) != expectedProxyNum { + errs = multierror.Append(errs, + fmt.Errorf("expected %d tcpRoutes got %d", expectedProxyNum, len(services))) + } + if len(gatewayPorts) != expectedProxyNum { + errs = multierror.Append(errs, + fmt.Errorf("expected %d gateway ports got %d", expectedProxyNum, len(gatewayPorts))) } - existingConfigMaps.Insert(cm.Name) } - if len(jobs.Items) == 0 { - return reattachJobResponse{}, fmt.Errorf("did not find job for allocation %s", msg.allocationID) - } else if len(jobs.Items) > 1 { - return reattachJobResponse{}, fmt.Errorf("found multiple allocation jobs for allocation %s", msg.allocationID) + // Cleanup the job if we don't get the format we expect. + cleanup := func() { + j.deleteKubernetesResources(jobs, configMaps, services, tcpRoutes, gatewayPorts) + } + if errs.Len() > 0 { + cleanup() + return reattachJobResponse{}, fmt.Errorf("reattach job: %w", errs) } - job := jobs.Items[0] + job := jobs[0] + if len(jobs) != 1 { // Unnecessary, but we should be careful here. + cleanup() + return reattachJobResponse{}, fmt.Errorf("expected one job") + } resourcePool, ok := job.Labels[resourcePoolLabel] if !ok { + cleanup() return reattachJobResponse{}, fmt.Errorf("could not recover resource pool for %s", msg.allocationID) } + gatewayResources, err := j.recreateGatewayProxyResources( + msg.allocationID, services, tcpRoutes, gatewayPorts, + ) + if err != nil { + cleanup() + return reattachJobResponse{}, err + } + resp, err := j.recreateJobHandler( job.Name, msg.req, @@ -564,15 +720,67 @@ func (j *jobsService) reattachJob(msg reattachJobRequest) (reattachJobResponse, &job, msg.slots, msg.numPods, + gatewayResources, msg.logContext, ) if err != nil { - j.deleteKubernetesResources(jobs, configMaps) + cleanup() return reattachJobResponse{}, fmt.Errorf("error restoring pod with allocation ID %s: %w", msg.allocationID, err) } + return resp, nil } +func (j *jobsService) recreateGatewayProxyResources( + allocationID model.AllocationID, + services []k8sV1.Service, + tcpRoutes []alphaGatewayTyped.TCPRoute, + gatewayPorts []int, +) ([]gatewayProxyResource, error) { + if j.internalTaskGWConfig == nil { + return nil, nil + } + + var resources []gatewayProxyResource + for _, port := range gatewayPorts { + var tcpRoute *alphaGatewayTyped.TCPRoute + for _, t := range tcpRoutes { + t := t + if len(t.Spec.ParentRefs) > 0 && + t.Spec.ParentRefs[0].Port != nil && + int(*t.Spec.ParentRefs[0].Port) == port { + tcpRoute = &t + break + } + } + if tcpRoute == nil { + return nil, fmt.Errorf("couldn't find tcpRoute for port %d", port) + } + + var service *k8sV1.Service + for _, s := range services { + s := s + if s.Name == tcpRoute.Name { + service = &s + break + } + } + if service == nil { + return nil, fmt.Errorf("couldn't find service matching %s", tcpRoute.Name) + } + if len(service.Spec.Ports) != 1 { + return nil, fmt.Errorf("expected service to have one port got %d", len(service.Spec.Ports)) + } + resources = append(resources, gatewayProxyResource{ + serviceSpec: service, + tcpRouteSpec: tcpRoute, + gatewayListener: createListenerForPod(port), + }) + } + + return resources, nil +} + func (j *jobsService) recreateJobHandler( name string, req *sproto.AllocateRequest, @@ -581,6 +789,7 @@ func (j *jobsService) recreateJobHandler( job *batchV1.Job, slots int, numPods int, + gatewayProxyResources []gatewayProxyResource, logContext logger.Context, ) (reattachJobResponse, error) { startMsg := startJob{ @@ -612,12 +821,16 @@ func (j *jobsService) recreateJobHandler( j.slotType, j.slotResourceRequests, j.scheduler, + j.internalTaskGWConfig, + j.gatewayService, ) newJobHandler.restore = true newJobHandler.jobName = job.Name newJobHandler.configMapName = job.Name + newJobHandler.gatewayProxyResources = gatewayProxyResources + err := newJobHandler.startPodLogStreamers() if err != nil { return reattachJobResponse{}, fmt.Errorf("reattaching pod: %w", err) @@ -636,14 +849,45 @@ func (j *jobsService) recreateJobHandler( } func (j *jobsService) deleteKubernetesResources( - jobs *batchV1.JobList, configMaps *k8sV1.ConfigMapList, + jobs []batchV1.Job, + configMaps []k8sV1.ConfigMap, + services []k8sV1.Service, + tcpRoutes []alphaGatewayTyped.TCPRoute, + gatewayPortsToFree []int, ) { - for _, job := range jobs.Items { - j.resourceRequestQueue.deleteKubernetesResources(job.Namespace, job.Name, "", "") + for _, job := range jobs { + j.resourceRequestQueue.deleteKubernetesResources(deleteKubernetesResources{ + namespace: job.Namespace, + jobName: job.Name, + }) } - for _, configMap := range configMaps.Items { - j.resourceRequestQueue.deleteKubernetesResources(configMap.Namespace, "", configMap.Name, "") + for _, configMap := range configMaps { + j.resourceRequestQueue.deleteKubernetesResources(deleteKubernetesResources{ + namespace: configMap.Namespace, + configMapName: configMap.Name, + }) + } + + for _, s := range services { + j.resourceRequestQueue.deleteKubernetesResources(deleteKubernetesResources{ + namespace: s.Namespace, + serviceNames: []string{s.Name}, + }) + } + + for _, r := range tcpRoutes { + j.resourceRequestQueue.deleteKubernetesResources(deleteKubernetesResources{ + namespace: r.Namespace, + tcpRouteNames: []string{r.Name}, + }) + } + + if len(gatewayPortsToFree) > 0 && j.internalTaskGWConfig != nil { + j.resourceRequestQueue.deleteKubernetesResources(deleteKubernetesResources{ + namespace: j.internalTaskGWConfig.GatewayNamespace, + gatewayPortsToFree: gatewayPortsToFree, + }) } } @@ -669,7 +913,7 @@ func (j *jobsService) refreshJobState(allocationID model.AllocationID) error { return fmt.Errorf("error listing pods checking if they can be restored: %w", err) } - for _, job := range jobs.Items { + for _, job := range jobs { if _, ok := j.namespaceToPoolName[job.Namespace]; !ok { continue } @@ -812,7 +1056,14 @@ func (j *jobsService) startPreemptionListeners() error { func (j *jobsService) startResourceRequestQueue() { failures := make(chan resourcesRequestFailure, 16) - j.resourceRequestQueue = startRequestQueue(j.jobInterfaces, j.podInterfaces, j.configMapInterfaces, failures) + j.resourceRequestQueue = startRequestQueue( + j.jobInterfaces, + j.podInterfaces, + j.configMapInterfaces, + j.serviceInterfaces, + j.gatewayService, + j.tcpRouteInterfaces, + failures) j.wg.Go(func(ctx context.Context) { for { select { @@ -1744,15 +1995,15 @@ func numSlots(slots model.SlotsSummary) int { func (j *jobsService) listJobsInAllNamespaces( ctx context.Context, opts metaV1.ListOptions, -) (*batchV1.JobList, error) { - res := &batchV1.JobList{} +) ([]batchV1.Job, error) { + var res []batchV1.Job for n, i := range j.jobInterfaces { pods, err := i.List(ctx, opts) if err != nil { return nil, fmt.Errorf("error listing pods for namespace %s: %w", n, err) } - res.Items = append(res.Items, pods.Items...) + res = append(res, pods.Items...) } return res, nil @@ -1776,14 +2027,44 @@ func (j *jobsService) listPodsInAllNamespaces( func (j *jobsService) listConfigMapsInAllNamespaces( ctx context.Context, opts metaV1.ListOptions, -) (*k8sV1.ConfigMapList, error) { - res := &k8sV1.ConfigMapList{} +) ([]k8sV1.ConfigMap, error) { + var res []k8sV1.ConfigMap for n, i := range j.configMapInterfaces { cms, err := i.List(ctx, opts) if err != nil { return nil, fmt.Errorf("error listing config maps for namespace %s: %w", n, err) } - res.Items = append(res.Items, cms.Items...) + res = append(res, cms.Items...) + } + + return res, nil +} + +func (j *jobsService) listServicesInAllNamespaces( + ctx context.Context, opts metaV1.ListOptions, +) ([]k8sV1.Service, error) { + var res []k8sV1.Service + for n, i := range j.serviceInterfaces { + services, err := i.List(ctx, opts) + if err != nil { + return nil, fmt.Errorf("listing services for namespace %s: %w", n, err) + } + res = append(res, services.Items...) + } + + return res, nil +} + +func (j *jobsService) listTCPRoutesInAllNamespaces( + ctx context.Context, opts metaV1.ListOptions, +) ([]alphaGatewayTyped.TCPRoute, error) { + var res []alphaGatewayTyped.TCPRoute + for n, i := range j.tcpRouteInterfaces { + routes, err := i.List(ctx, opts) + if err != nil { + return nil, fmt.Errorf("listing TCPRoutes for namespace %s: %w", n, err) + } + res = append(res, routes.Items...) } return res, nil diff --git a/master/internal/rm/kubernetesrm/kubernetes_resource_manager.go b/master/internal/rm/kubernetesrm/kubernetes_resource_manager.go index 82085fb1a2c..4a88cf8fe13 100644 --- a/master/internal/rm/kubernetesrm/kubernetes_resource_manager.go +++ b/master/internal/rm/kubernetesrm/kubernetes_resource_manager.go @@ -109,6 +109,7 @@ func New( k.config.DetMasterPort, k.config.KubeconfigPath, k.jobSchedulingStateCallback, + k.config.InternalTaskGateway, ) if err != nil { return nil, err diff --git a/master/internal/rm/kubernetesrm/mock_client_test.go b/master/internal/rm/kubernetesrm/mock_client_test.go index 895fd68ef92..b0cca6e2eab 100644 --- a/master/internal/rm/kubernetesrm/mock_client_test.go +++ b/master/internal/rm/kubernetesrm/mock_client_test.go @@ -14,10 +14,13 @@ import ( batchV1 "k8s.io/api/batch/v1" k8sV1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" "k8s.io/api/policy/v1beta1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" + applyBatchV1 "k8s.io/client-go/applyconfigurations/batch/v1" + corev1 "k8s.io/client-go/applyconfigurations/core/v1" "k8s.io/client-go/rest" ) @@ -99,6 +102,12 @@ func (m *mockConfigMapInterface) Patch( panic("implement me") } +func (m *mockConfigMapInterface) Apply( + ctx context.Context, configMap *corev1.ConfigMapApplyConfiguration, opts metaV1.ApplyOptions, +) (result *k8sV1.ConfigMap, err error) { + panic("implement me") +} + type mockPodInterface struct { pods map[string]*k8sV1.Pod // Simulates latency of the real k8 API server. @@ -189,16 +198,22 @@ func (m *mockPodInterface) Patch( panic("implement me") } -func (m *mockPodInterface) GetEphemeralContainers( - ctx context.Context, podName string, options metaV1.GetOptions, -) (*k8sV1.EphemeralContainers, error) { +func (m *mockPodInterface) Apply( + ctx context.Context, pod *corev1.PodApplyConfiguration, opts metaV1.ApplyOptions, +) (result *k8sV1.Pod, err error) { + panic("implement me") +} + +func (m *mockPodInterface) ApplyStatus( + ctx context.Context, pod *corev1.PodApplyConfiguration, opts metaV1.ApplyOptions, +) (result *k8sV1.Pod, err error) { panic("implement me") } func (m *mockPodInterface) UpdateEphemeralContainers( - ctx context.Context, podName string, ephemeralContainers *k8sV1.EphemeralContainers, + ctx context.Context, podName string, ephemeralContainers *k8sV1.Pod, opts metaV1.UpdateOptions, -) (*k8sV1.EphemeralContainers, error) { +) (*k8sV1.Pod, error) { panic("implement me") } @@ -210,6 +225,14 @@ func (m *mockPodInterface) Evict(ctx context.Context, eviction *v1beta1.Eviction panic("implement me") } +func (m *mockPodInterface) EvictV1(ctx context.Context, eviction *policyv1.Eviction) error { + panic("implement me") +} + +func (m *mockPodInterface) EvictV1beta1(ctx context.Context, eviction *v1beta1.Eviction) error { + panic("implement me") +} + func (m *mockPodInterface) GetLogs(name string, opts *k8sV1.PodLogOptions) *rest.Request { client := cleanhttp.DefaultClient() client.Transport = &mockRoundTripInterface{message: m.logMessage} @@ -324,3 +347,15 @@ func (m *mockJobInterface) Patch( ) (result *batchV1.Job, err error) { panic("implement me") } + +func (m *mockJobInterface) Apply( + ctx context.Context, job *applyBatchV1.JobApplyConfiguration, opts metaV1.ApplyOptions, +) (result *batchV1.Job, err error) { + panic("implement me") +} + +func (m *mockJobInterface) ApplyStatus( + ctx context.Context, job *applyBatchV1.JobApplyConfiguration, opts metaV1.ApplyOptions, +) (result *batchV1.Job, err error) { + panic("implement me") +} diff --git a/master/internal/rm/kubernetesrm/request_queue.go b/master/internal/rm/kubernetesrm/request_queue.go index 9dcb7247c92..7bd6fad3159 100644 --- a/master/internal/rm/kubernetesrm/request_queue.go +++ b/master/internal/rm/kubernetesrm/request_queue.go @@ -2,6 +2,7 @@ package kubernetesrm import ( "strconv" + "strings" "sync" batchV1 "k8s.io/api/batch/v1" @@ -10,6 +11,7 @@ import ( "github.com/sirupsen/logrus" k8sV1 "k8s.io/api/core/v1" typedV1 "k8s.io/client-go/kubernetes/typed/core/v1" + alphaGateway "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1alpha2" "github.com/determined-ai/determined/master/pkg/set" ) @@ -24,13 +26,17 @@ type ( createKubernetesResources struct { jobSpec *batchV1.Job configMapSpec *k8sV1.ConfigMap + gw *gatewayResourceComm } deleteKubernetesResources struct { - namespace string - jobName string - configMapName string - podName string + namespace string + jobName string + podName string + configMapName string + serviceNames []string + tcpRouteNames []string + gatewayPortsToFree []int } ) @@ -108,7 +114,11 @@ type requestQueue struct { jobInterfaces map[string]typedBatchV1.JobInterface podInterfaces map[string]typedV1.PodInterface configMapInterfaces map[string]typedV1.ConfigMapInterface - failures chan<- resourcesRequestFailure + serviceInterfaces map[string]typedV1.ServiceInterface + tcpRouteInterfaces map[string]alphaGateway.TCPRouteInterface + gatewayService *gatewayService + + failures chan<- resourcesRequestFailure mu sync.Mutex workerChan chan interface{} @@ -128,13 +138,20 @@ func startRequestQueue( jobInterfaces map[string]typedBatchV1.JobInterface, podInterfaces map[string]typedV1.PodInterface, configMapInterfaces map[string]typedV1.ConfigMapInterface, + serviceInterfaces map[string]typedV1.ServiceInterface, + gatewayService *gatewayService, + tcpRouteInterfaces map[string]alphaGateway.TCPRouteInterface, failures chan<- resourcesRequestFailure, ) *requestQueue { r := &requestQueue{ jobInterfaces: jobInterfaces, podInterfaces: podInterfaces, configMapInterfaces: configMapInterfaces, - failures: failures, + serviceInterfaces: serviceInterfaces, + gatewayService: gatewayService, + tcpRouteInterfaces: tcpRouteInterfaces, + + failures: failures, workerChan: make(chan interface{}), @@ -156,6 +173,9 @@ func (r *requestQueue) startWorkers() { r.jobInterfaces, r.podInterfaces, r.configMapInterfaces, + r.serviceInterfaces, + r.gatewayService, + r.tcpRouteInterfaces, strconv.Itoa(i), r.workerChan, r.workerReady, @@ -184,17 +204,33 @@ func keyForDelete(msg deleteKubernetesResources) requestID { if msg.configMapName != "" { return requestID(msg.namespace + "/" + msg.configMapName) } + if len(msg.serviceNames) > 0 { + return requestID(msg.namespace + "/" + strings.Join(msg.serviceNames, ",")) + } + if len(msg.tcpRouteNames) > 0 { + return requestID(msg.namespace + "/" + strings.Join(msg.tcpRouteNames, ",")) + } + if len(msg.gatewayPortsToFree) > 0 { + var stringPorts []string + for _, p := range msg.gatewayPortsToFree { + stringPorts = append(stringPorts, strconv.Itoa(p)) + } + + return requestID(msg.namespace + "/" + strings.Join(stringPorts, ",")) + } + panic("invalid deleteKubernetesResources message") } func (r *requestQueue) createKubernetesResources( jobSpec *batchV1.Job, configMapSpec *k8sV1.ConfigMap, + gwResources *gatewayResourceComm, ) { r.mu.Lock() defer r.mu.Unlock() - msg := createKubernetesResources{jobSpec, configMapSpec} + msg := createKubernetesResources{jobSpec, configMapSpec, gwResources} ref := keyForCreate(msg) if _, requestAlreadyExists := r.pendingResourceCreations[ref]; requestAlreadyExists { @@ -212,21 +248,10 @@ func (r *requestQueue) createKubernetesResources( } } -func (r *requestQueue) deleteKubernetesResources( - namespace string, - jobName string, - configMapName string, - podName string, -) { +func (r *requestQueue) deleteKubernetesResources(msg deleteKubernetesResources) { r.mu.Lock() defer r.mu.Unlock() - msg := deleteKubernetesResources{ - namespace: namespace, - jobName: jobName, - configMapName: configMapName, - podName: podName, - } ref := keyForDelete(msg) // If the request has not been processed yet, cancel it and inform the handler. @@ -234,7 +259,7 @@ func (r *requestQueue) deleteKubernetesResources( r.pendingResourceCreations[ref].createResources = nil delete(r.pendingResourceCreations, ref) r.failures <- resourceCreationCancelled{ - jobName: jobName, + jobName: msg.jobName, } r.syslog.Warnf("delete issued with pending create request for %s", ref) return diff --git a/master/internal/rm/kubernetesrm/request_queue_test.go b/master/internal/rm/kubernetesrm/request_queue_test.go index 1b0f283ced2..15a0c00710d 100644 --- a/master/internal/rm/kubernetesrm/request_queue_test.go +++ b/master/internal/rm/kubernetesrm/request_queue_test.go @@ -63,11 +63,15 @@ func (m *mockJob) create() { Name: m.name, Namespace: "default", }} - m.requestQueue.createKubernetesResources(&jobSpec, &cmSpec) + m.requestQueue.createKubernetesResources(&jobSpec, &cmSpec, nil) } func (m *mockJob) delete() { - m.requestQueue.deleteKubernetesResources("default", m.name, m.name, "") + m.requestQueue.deleteKubernetesResources(deleteKubernetesResources{ + namespace: "default", + jobName: m.name, + configMapName: m.name, + }) } func getNumberOfActiveJobs(jobInterface typedBatchV1.JobInterface) int { @@ -113,7 +117,7 @@ func TestRequestQueueCreatingManyPod(t *testing.T) { map[string]typedBatchV1.JobInterface{"default": jobInterface}, map[string]typedV1.PodInterface{"default": podInterface}, map[string]typedV1.ConfigMapInterface{"default": configMapInterface}, - failures, + nil, nil, nil, failures, ) ctx, cancel := context.WithCancel(context.Background()) @@ -139,7 +143,7 @@ func TestRequestQueueCreatingAndDeletingManyPod(t *testing.T) { map[string]typedBatchV1.JobInterface{"default": jobInterface}, map[string]typedV1.PodInterface{"default": podInterface}, map[string]typedV1.ConfigMapInterface{"default": configMapInterface}, - failures, + nil, nil, nil, failures, ) ctx, cancel := context.WithCancel(context.Background()) @@ -167,7 +171,7 @@ func TestRequestQueueCreatingThenDeletingManyPods(t *testing.T) { map[string]typedBatchV1.JobInterface{"default": jobInterface}, map[string]typedV1.PodInterface{"default": podInterface}, map[string]typedV1.ConfigMapInterface{"default": configMapInterface}, - failures, + nil, nil, nil, failures, ) ctx, cancel := context.WithCancel(context.Background()) @@ -202,7 +206,7 @@ func TestRequestQueueCreatingAndDeletingManyPodWithDelay(t *testing.T) { map[string]typedBatchV1.JobInterface{"default": jobInterface}, map[string]typedV1.PodInterface{"default": podInterface}, map[string]typedV1.ConfigMapInterface{"default": configMapInterface}, - failures, + nil, nil, nil, failures, ) ctx, cancel := context.WithCancel(context.Background()) @@ -233,7 +237,7 @@ func TestRequestQueueCreationCancelled(t *testing.T) { map[string]typedBatchV1.JobInterface{"default": jobInterface}, map[string]typedV1.PodInterface{"default": podInterface}, map[string]typedV1.ConfigMapInterface{"default": configMapInterface}, - failures, + nil, nil, nil, failures, ) for i := 0; i < numKubernetesWorkers; i++ { @@ -274,7 +278,7 @@ func TestRequestQueueCreationFailed(t *testing.T) { map[string]typedBatchV1.JobInterface{"default": jobInterface}, map[string]typedV1.PodInterface{"default": podInterface}, map[string]typedV1.ConfigMapInterface{"default": configMapInterface}, - failures, + nil, nil, nil, failures, ) var wg sync.WaitGroup @@ -312,7 +316,7 @@ func TestRequestQueueDeletionFailed(t *testing.T) { map[string]typedBatchV1.JobInterface{"default": jobInterface}, map[string]typedV1.PodInterface{"default": podInterface}, map[string]typedV1.ConfigMapInterface{"default": configMapInterface}, - failures, + nil, nil, nil, failures, ) var wg sync.WaitGroup diff --git a/master/internal/rm/kubernetesrm/request_workers.go b/master/internal/rm/kubernetesrm/request_workers.go index cd07c1fde38..568dd35916d 100644 --- a/master/internal/rm/kubernetesrm/request_workers.go +++ b/master/internal/rm/kubernetesrm/request_workers.go @@ -10,6 +10,8 @@ import ( batchV1 "k8s.io/client-go/kubernetes/typed/batch/v1" typedV1 "k8s.io/client-go/kubernetes/typed/core/v1" + alphaGateway "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1alpha2" + "github.com/determined-ai/determined/master/pkg/ptrs" ) @@ -17,6 +19,9 @@ type requestProcessingWorker struct { jobInterface map[string]batchV1.JobInterface podInterface map[string]typedV1.PodInterface configMapInterfaces map[string]typedV1.ConfigMapInterface + serviceInterfaces map[string]typedV1.ServiceInterface + tcpRouteInterfaces map[string]alphaGateway.TCPRouteInterface + gatewayService *gatewayService failures chan<- resourcesRequestFailure syslog *logrus.Entry } @@ -27,6 +32,9 @@ func startRequestProcessingWorker( jobInterface map[string]batchV1.JobInterface, podInterface map[string]typedV1.PodInterface, configMapInterfaces map[string]typedV1.ConfigMapInterface, + serviceInterfaces map[string]typedV1.ServiceInterface, + gatewayService *gatewayService, + tcpRouteInterfaces map[string]alphaGateway.TCPRouteInterface, id string, in <-chan interface{}, ready readyCallbackFunc, @@ -37,8 +45,12 @@ func startRequestProcessingWorker( jobInterface: jobInterface, podInterface: podInterface, configMapInterfaces: configMapInterfaces, - failures: failures, - syslog: syslog, + serviceInterfaces: serviceInterfaces, + gatewayService: gatewayService, + tcpRouteInterfaces: tcpRouteInterfaces, + + failures: failures, + syslog: syslog, } go r.receive(in, ready) return r @@ -85,6 +97,47 @@ func (r *requestProcessingWorker) receiveCreateKubernetesResources( return } r.syslog.Infof("created job %s", job.Name) + + var ports []int + var proxyResources []gatewayProxyResource + // TODO(RM-272) do we leak resources if the request queue fails? + // Do we / should we delete created resources? + if msg.gw != nil { + if msg.gw.requestedPorts > 0 { + if ports, err = r.gatewayService.generateAndAddListeners( + msg.gw.allocationID, msg.gw.requestedPorts, + ); err != nil { + r.syslog.WithError(err).Errorf("error patching gateway for job %s", msg.jobSpec.Name) + r.failures <- resourceCreationFailed{jobName: msg.jobSpec.Name, err: err} + return + } + r.syslog.Info("created gateway proxy listeners", msg.gw.requestedPorts, ports) + } + proxyResources = msg.gw.resourceDescriptor(ports) + } + + for _, proxyResource := range proxyResources { + r.syslog.Debugf("launching service with spec %v", *proxyResource.serviceSpec) + if _, err := r.serviceInterfaces[msg.jobSpec.Namespace].Create( + context.TODO(), proxyResource.serviceSpec, metaV1.CreateOptions{}, + ); err != nil { + r.syslog.WithError(err).Errorf("error creating service for pod %s", msg.jobSpec.Name) + r.failures <- resourceCreationFailed{jobName: msg.jobSpec.Name, err: err} + return + } + r.syslog.Debugf("launching tcproute with spec %v", *proxyResource.tcpRouteSpec) + if _, err := r.tcpRouteInterfaces[msg.jobSpec.Namespace].Create( + context.TODO(), proxyResource.tcpRouteSpec, metaV1.CreateOptions{}, + ); err != nil { + r.syslog.WithError(err).Errorf("error creating tcproute for pod %s", msg.jobSpec.Name) + r.failures <- resourceCreationFailed{jobName: msg.jobSpec.Name, err: err} + return + } + } + if msg.gw != nil && msg.gw.reportResources != nil { + msg.gw.reportResources(proxyResources) + r.syslog.Info("created gateway proxy resources") + } } func (r *requestProcessingWorker) receiveDeleteKubernetesResources( @@ -119,7 +172,7 @@ func (r *requestProcessingWorker) receiveDeleteKubernetesResources( case err != nil: r.syslog.WithError(err).Errorf("failed to delete pod %s", msg.jobName) default: - r.syslog.Infof("deleted pod %s", msg.jobName) + r.syslog.Infof("deleted pod %s", msg.podName) } } @@ -133,7 +186,42 @@ func (r *requestProcessingWorker) receiveDeleteKubernetesResources( case err != nil: r.syslog.WithError(err).Errorf("failed to delete configMap %s", msg.jobName) default: - r.syslog.Infof("deleted configMap %s", msg.jobName) + r.syslog.Infof("deleted configMap %s", msg.configMapName) + } + } + + for _, serviceName := range msg.serviceNames { + errDeletingService := r.serviceInterfaces[msg.namespace].Delete( + context.TODO(), serviceName, + metaV1.DeleteOptions{GracePeriodSeconds: &gracePeriod}) + if errDeletingService != nil { + r.syslog.WithError(errDeletingService). + Errorf("failed to delete service %s", serviceName) + err = errDeletingService + } else { + r.syslog.Infof("deleted service %s", serviceName) + } + } + + for _, tcpRouteName := range msg.tcpRouteNames { + errDeletingService := r.tcpRouteInterfaces[msg.namespace].Delete( + context.TODO(), tcpRouteName, + metaV1.DeleteOptions{GracePeriodSeconds: &gracePeriod}) + if errDeletingService != nil { + r.syslog.WithError(errDeletingService). + Errorf("failed to delete tcpRoute %s", tcpRouteName) + err = errDeletingService + } else { + r.syslog.Infof("deleted tcpRoute %s", tcpRouteName) + } + } + + if len(msg.gatewayPortsToFree) > 0 { + err = r.gatewayService.freePorts(msg.gatewayPortsToFree) + if err != nil { + r.syslog.WithError(err).Errorf("failed to free gateway ports %v", msg.gatewayPortsToFree) + } else { + r.syslog.Infof("freed gateway ports %v", msg.gatewayPortsToFree) } } diff --git a/master/internal/rm/kubernetesrm/resource_pool_intg_test.go b/master/internal/rm/kubernetesrm/resource_pool_intg_test.go index 9de247bd406..2b2dd165a1e 100644 --- a/master/internal/rm/kubernetesrm/resource_pool_intg_test.go +++ b/master/internal/rm/kubernetesrm/resource_pool_intg_test.go @@ -1277,6 +1277,7 @@ func newTestJobsService(t *testing.T) *jobsService { 8080, "~/.kube/config", nil, + nil, ) require.NoError(t, err) return j diff --git a/master/internal/rm/kubernetesrm/spec.go b/master/internal/rm/kubernetesrm/spec.go index b9721226440..a239d4565c0 100644 --- a/master/internal/rm/kubernetesrm/spec.go +++ b/master/internal/rm/kubernetesrm/spec.go @@ -33,6 +33,8 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" + + alphaGatewayTyped "sigs.k8s.io/gateway-api/apis/v1alpha2" ) const ( @@ -128,6 +130,10 @@ func (j *job) configureEnvVars( envVarsMap["DET_KUBERNETES_JOB_PARALLELISM"] = strconv.Itoa(j.numPods) + if j.internalTaskGWConfig != nil { + envVarsMap["DET_PROXY_THROUGH_GATEWAY"] = "true" + } + envVars := make([]k8sV1.EnvVar, 0, len(envVarsMap)) for envVarKey, envVarValue := range envVarsMap { envVars = append(envVars, k8sV1.EnvVar{Name: envVarKey, Value: envVarValue}) @@ -147,6 +153,102 @@ func (j *job) configureEnvVars( return envVars, nil } +// proxyResourceGenerator returns a configured list of proxy resources given a set of ports. +// We do this lazily to make port selection easier. If we created the resource before it is +// added to the request queue we risk that port being taken later on. +type proxyResourceGenerator func([]int) []gatewayProxyResource + +func (j *job) configureProxyResources(t *tasks.TaskSpec) proxyResourceGenerator { + if j.internalTaskGWConfig == nil { + return nil + } + + generator := proxyResourceGenerator(func(ports []int) []gatewayProxyResource { + var resources []gatewayProxyResource + if len(ports) != len(j.req.ProxyPorts) { + panic("proxy ports and ports must be the same length") + } + + for i, proxyPort := range j.req.ProxyPorts { + sharedName := fmt.Sprintf("%s-%d", j.jobName, i) + + gwPort := ports[i] + allocLabels := map[string]string{ + determinedLabel: t.AllocationID, + } + annotations := map[string]string{ + jobNameAnnotation: j.jobName, + } + + serviceSpec := &k8sV1.Service{ + ObjectMeta: metaV1.ObjectMeta{ + Name: sharedName, + Namespace: j.namespace, + Labels: allocLabels, + Annotations: annotations, + }, + Spec: k8sV1.ServiceSpec{ + Ports: []k8sV1.ServicePort{ + { + Protocol: k8sV1.ProtocolTCP, + Port: int32(proxyPort.Port), + }, + }, + Selector: allocLabels, + Type: k8sV1.ServiceTypeClusterIP, + }, + } + + tcpRouteSpec := &alphaGatewayTyped.TCPRoute{ + ObjectMeta: metaV1.ObjectMeta{ + Name: sharedName, + Namespace: j.namespace, + Labels: allocLabels, + Annotations: annotations, + }, + Spec: alphaGatewayTyped.TCPRouteSpec{ + CommonRouteSpec: alphaGatewayTyped.CommonRouteSpec{ + ParentRefs: []alphaGatewayTyped.ParentReference{ + { + Namespace: ptrs.Ptr(alphaGatewayTyped.Namespace(j.internalTaskGWConfig.GatewayNamespace)), + Name: alphaGatewayTyped.ObjectName(j.internalTaskGWConfig.GatewayName), + Port: ptrs.Ptr(alphaGatewayTyped.PortNumber(gwPort)), + SectionName: ptrs.Ptr(alphaGatewayTyped.SectionName( + generateListenerName(gwPort), + )), + }, + }, + }, + Rules: []alphaGatewayTyped.TCPRouteRule{ + { + BackendRefs: []alphaGatewayTyped.BackendRef{ + { + BackendObjectReference: alphaGatewayTyped.BackendObjectReference{ + Name: alphaGatewayTyped.ObjectName(serviceSpec.Name), + Kind: ptrs.Ptr(alphaGatewayTyped.Kind("Service")), + Port: ptrs.Ptr(alphaGatewayTyped.PortNumber(proxyPort.Port)), + }, + }, + }, + }, + }, + }, + } + + gatewayListener := createListenerForPod(gwPort) + + resources = append(resources, gatewayProxyResource{ + serviceSpec: serviceSpec, + tcpRouteSpec: tcpRouteSpec, + gatewayListener: gatewayListener, + }) + } + return resources + }) + + return generator +} + func (j *job) configureConfigMapSpec( taskSpec *tasks.TaskSpec, runArchives []cproto.RunArchive, @@ -638,7 +740,8 @@ func configureUniqueName(t tasks.TaskSpec) string { clusterIDPrefix = t.ClusterID } if clusterIDPrefix != "" { - name = fmt.Sprintf("%s-%s", clusterIDPrefix, name) + // Starting with clusterID is not a valid DNS name since it could be a number sometimes. + name = fmt.Sprintf("det-%s-%s", clusterIDPrefix, name) } return name diff --git a/master/internal/rm/kubernetesrm/spec_test.go b/master/internal/rm/kubernetesrm/spec_test.go index a8ce139904c..6428396773c 100644 --- a/master/internal/rm/kubernetesrm/spec_test.go +++ b/master/internal/rm/kubernetesrm/spec_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/determined-ai/determined/master/internal/config" "github.com/determined-ai/determined/master/internal/sproto" "github.com/determined-ai/determined/master/pkg/device" "github.com/determined-ai/determined/master/pkg/model" @@ -15,6 +16,8 @@ import ( "github.com/determined-ai/determined/master/pkg/tasks" k8sV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + alphaGatewayTyped "sigs.k8s.io/gateway-api/apis/v1alpha2" ) func TestGetDetContainerSecurityContext(t *testing.T) { @@ -61,6 +64,100 @@ func TestGetDetContainerSecurityContext(t *testing.T) { require.Equal(t, expectedCaps, secContext.Capabilities.Add) } +func TestConfigureProxyResources(t *testing.T) { + j := &job{ + namespace: "podnamespace", + jobName: "sharedName", + } + taskSpec := &tasks.TaskSpec{ + AllocationID: "allocID", + } + require.Nil(t, j.configureProxyResources(taskSpec)) + + j.req = &sproto.AllocateRequest{ + ProxyPorts: []*sproto.ProxyPortConfig{ + { + ServiceID: "test_unused", + Port: 12345, + ProxyTCP: false, // We still want a TCP proxy. + }, + }, + } + j.internalTaskGWConfig = &config.InternalTaskGatewayConfig{ + GatewayName: "gatewayname", + GatewayNamespace: "gatewaynamespace", + } + + svc := &k8sV1.Service{ + ObjectMeta: metaV1.ObjectMeta{ + Name: j.jobName + "-0", + Namespace: "podnamespace", + Labels: map[string]string{determinedLabel: "allocID"}, + Annotations: map[string]string{ + jobNameAnnotation: j.jobName, + }, + }, + Spec: k8sV1.ServiceSpec{ + Ports: []k8sV1.ServicePort{ + { + Protocol: k8sV1.ProtocolTCP, + Port: 12345, + }, + }, + Selector: map[string]string{determinedLabel: "allocID"}, + Type: k8sV1.ServiceTypeClusterIP, + }, + } + + tcp := &alphaGatewayTyped.TCPRoute{ + ObjectMeta: metaV1.ObjectMeta{ + Name: j.jobName + "-0", + Namespace: "podnamespace", + Labels: map[string]string{determinedLabel: "allocID"}, + Annotations: map[string]string{ + jobNameAnnotation: j.jobName, + }, + }, + Spec: alphaGatewayTyped.TCPRouteSpec{ + CommonRouteSpec: alphaGatewayTyped.CommonRouteSpec{ + ParentRefs: []alphaGatewayTyped.ParentReference{ + { + Namespace: ptrs.Ptr(alphaGatewayTyped.Namespace("gatewaynamespace")), + Name: alphaGatewayTyped.ObjectName("gatewayname"), + Port: ptrs.Ptr(alphaGatewayTyped.PortNumber(12345)), + SectionName: ptrs.Ptr(alphaGatewayTyped.SectionName( + generateListenerName(12345), + )), + }, + }, + }, + Rules: []alphaGatewayTyped.TCPRouteRule{ + { + BackendRefs: []alphaGatewayTyped.BackendRef{ + { + BackendObjectReference: alphaGatewayTyped.BackendObjectReference{ + Name: alphaGatewayTyped.ObjectName(j.jobName + "-0"), + Kind: ptrs.Ptr(alphaGatewayTyped.Kind("Service")), + Port: ptrs.Ptr(alphaGatewayTyped.PortNumber(12345)), + }, + }, + }, + }, + }, + }, + } + + listener := createListenerForPod(12345) + + require.Equal(t, []gatewayProxyResource{ + { + serviceSpec: svc, + tcpRouteSpec: tcp, + gatewayListener: listener, + }, + }, (j.configureProxyResources(taskSpec))([]int{12345})) +} + func TestAddNodeDisabledAffinityToPodSpec(t *testing.T) { hasDisabledLabel := func(p *k8sV1.Pod) { actualList := p.Spec.Affinity. @@ -176,6 +273,38 @@ func TestAddDisallowedNodesToPodSpec(t *testing.T) { } } +func TestDetProxyThroughGatewayEnv(t *testing.T) { + env := expconf.EnvironmentConfig{ + RawEnvironmentVariables: &expconf.EnvironmentVariablesMap{ + RawCPU: []string{}, + }, + } + + t.Run("with gateway", func(t *testing.T) { + j := job{ + internalTaskGWConfig: &config.InternalTaskGatewayConfig{}, + } + + actual, err := j.configureEnvVars(make(map[string]string), env, device.CPU) + require.NoError(t, err) + require.Contains(t, actual, k8sV1.EnvVar{Name: "DET_PROXY_THROUGH_GATEWAY", Value: "true"}) + }) + + t.Run("without gateway", func(t *testing.T) { + j := job{ + internalTaskGWConfig: nil, + } + + actual, err := j.configureEnvVars(make(map[string]string), env, device.CPU) + require.NoError(t, err) + var keys []string + for _, a := range actual { + keys = append(keys, a.Name) + } + require.NotContains(t, keys, "DET_PROXY_THROUGH_GATEWAY") + }) +} + func TestLaterEnvironmentVariablesGetSet(t *testing.T) { dontBe := k8sV1.EnvVar{Name: "var", Value: "dontbe"} shouldBe := k8sV1.EnvVar{Name: "var", Value: "shouldbe"} diff --git a/master/internal/task/allocation.go b/master/internal/task/allocation.go index eacb7e85cc6..6aaec4980ea 100644 --- a/master/internal/task/allocation.go +++ b/master/internal/task/allocation.go @@ -1084,7 +1084,7 @@ func (a *allocation) registerProxies(addresses []cproto.Address) { Scheme: urlScheme, Host: fmt.Sprintf("%s:%d", address.HostIP, address.HostPort), }, pcfg.ProxyTCP, pcfg.Unauthenticated) - a.syslog.Debugf("registered proxy id: %s, tcp: %v\n", pcfg.ServiceID, pcfg.ProxyTCP) + a.syslog.Debugf("registered proxy id: %s, cfg: %v\n", pcfg.ServiceID, pcfg) a.proxies = append(a.proxies, pcfg.ServiceID) } diff --git a/master/pkg/check/check_numbers.go b/master/pkg/check/check_numbers.go index a6a7960497a..3ebb04a081f 100644 --- a/master/pkg/check/check_numbers.go +++ b/master/pkg/check/check_numbers.go @@ -232,3 +232,12 @@ func LessThanOrEqualTo(actual, expected interface{}, msgAndArgs ...interface{}) return maybeCompare(actual, expected, func(comparison int) bool { return comparison <= 0 }, msgAndArgs, "%v is not less than or equal to %v", actual, expected) } + +// BetweenInclusive checks whether `actual` is between `lower` and `upper` inclusive. +func BetweenInclusive(actual, lower, upper interface{}, msgAndArgs ...interface{}) error { + err := GreaterThanOrEqualTo(actual, lower, msgAndArgs...) + if err != nil { + return err + } + return LessThanOrEqualTo(actual, upper, msgAndArgs...) +} diff --git a/master/pkg/check/check_string.go b/master/pkg/check/check_string.go index 428a83d6bae..73e75a64ff5 100644 --- a/master/pkg/check/check_string.go +++ b/master/pkg/check/check_string.go @@ -30,3 +30,26 @@ func Match(actual string, regex string, msgAndArgs ...interface{}) error { return check(compiled.FindString(actual) == actual, msgAndArgs, "%s doesn't match regex %s", actual, regex) } + +// LenBetween checks whether the length of the first argument is between the second and third arguments. +// The method returns an error with the provided message if the check fails. +func LenBetween(actual string, min, max int, msgAndArgs ...interface{}) error { + return BetweenInclusive(len(actual), min, max, msgAndArgs...) +} + +// IsValidK8sLabel checks whether the first argument is a valid Kubernetes label. The method returns +// an error with the provided message if the check fails. +func IsValidK8sLabel(actual string, msgAndArgs ...interface{}) error { + if err := NotEmpty(actual, msgAndArgs...); err != nil { + return err + } + if err := LenBetween(actual, 1, 63, msgAndArgs...); err != nil { + return err + } + if err := Match( + actual, `^[a-zA-Z0-9]([-a-zA-Z0-9_.]*[a-zA-Z0-9])?$`, msgAndArgs..., + ); err != nil { + return err + } + return nil +} diff --git a/master/pkg/check/check_test.go b/master/pkg/check/check_test.go index bbcb47f3482..63157f46350 100644 --- a/master/pkg/check/check_test.go +++ b/master/pkg/check/check_test.go @@ -1,6 +1,7 @@ package check import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -143,3 +144,70 @@ func TestGreaterThanOrEqualTo(t *testing.T) { runTestCase(t, tc) } } + +func TestIsValidK8sLabel(t *testing.T) { + tests := []struct { + label string + wantErr bool + }{ + {"valid-label", false}, + {"Valid-Label_123", false}, + {"a", false}, + {"a1", false}, + {"1a", false}, + {"a_b.c", false}, + {"-invalid", true}, + {"invalid#", true}, + {"invalid$", true}, + {"invalid%", true}, + {"invalid-", true}, + {"_invalid", true}, + {"invalid_", true}, + {".invalid", true}, + {"invalid.", true}, + {"", true}, + {"this-label-is-way-too-long-and-should-definitely-fail-because-it-is-over-sixty-three-characters", true}, + } + + for _, tt := range tests { + t.Run(tt.label, func(t *testing.T) { + err := IsValidK8sLabel(tt.label) + if (err != nil) != tt.wantErr { + t.Errorf("IsValidK8sLabel(%s) error = %v, wantErr %v", tt.label, err, tt.wantErr) + } + }) + } +} + +func TestBetweenInclusive(t *testing.T) { + tests := []struct { + actual interface{} + lower interface{} + upper interface{} + wantErr bool + }{ + {10, 5, 15, false}, + {10, 10, 15, false}, + {10, 5, 10, false}, + {10, 10, 10, false}, + {10, 11, 15, true}, + {10, 5, 9, true}, + {10, "5", 15, true}, + {int32(10), int32(5), int32(15), false}, + {int32(10), int32(10), int32(15), false}, + {int32(10), int32(5), int32(10), false}, + {int32(10), int32(10), int32(10), false}, + {int32(10), int32(11), int32(15), true}, + {int32(10), int32(5), int32(9), true}, + {int32(10), 5, int32(15), true}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%v between %v and %v", tt.actual, tt.lower, tt.upper), func(t *testing.T) { + err := BetweenInclusive(tt.actual, tt.lower, tt.upper) + if (err != nil) != tt.wantErr { + t.Errorf("BetweenInclusive(%v, %v, %v) error = %v, wantErr %v", tt.actual, tt.lower, tt.upper, err, tt.wantErr) + } + }) + } +} diff --git a/tools/k8s/launch-minikube-with-gateway.sh b/tools/k8s/launch-minikube-with-gateway.sh new file mode 100755 index 00000000000..15433d4f1b2 --- /dev/null +++ b/tools/k8s/launch-minikube-with-gateway.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -e + +K8S_VERSION=${K8S_VERSION:-1.29.5} # https://endoflife.date/kubernetes +minikube_profile=$1 +tools_k8s_dir=tools/k8s # $(dirname "$(realpath "$0")") + +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +minikube start --profile $minikube_profile --kubernetes-version $K8S_VERSION +kubectl apply -f https://raw.githubusercontent.com/projectcontour/contour/release-1.29/examples/render/contour-gateway-provisioner.yaml + +kubectl apply -f - </dev/null; then + # Either like have a smaller subnet so we don't conflict. Or like don't start it for the second one. + nohup minikube --profile $minikube_profile tunnel & # TODO won't work for users with sudo passwords. +else + echo "sudo password is required to start the tunnel." + echo "Please run the following command separately to start the tunnel:" + echo "minikube --profile $minikube_profile tunnel" + read -p "Press [Enter] once the tunnel has started..." +fi + +for ((i = 0; i < 60; i++)); do + export GATEWAY_IP=$(kubectl -n projectcontour get svc envoy-contour -o=jsonpath='{.status.loadBalancer.ingress[0].ip}') + if [ -n "$GATEWAY_IP" ]; then + echo "External IP address of envoy-contour service: $GATEWAY_IP" + break + fi + + sleep 1 +done