From 1e165b1f50274e4e7f0b33524abb8da3d5c3c957 Mon Sep 17 00:00:00 2001 From: Diego Bonfigli Date: Sun, 30 Jan 2022 12:48:17 +0100 Subject: [PATCH] Implement external gRPC Cloud Provider --- cluster-autoscaler/README.md | 2 + .../cloudprovider/builder/builder_all.go | 9 + .../builder/builder_externalgrpc.go | 43 + .../cloudprovider/cloud_provider.go | 2 + .../cloudprovider/externalgrpc/OWNERS | 4 + .../cloudprovider/externalgrpc/README.md | 80 + .../certmanager-manifests/ca-issuer.yaml | 8 + .../examples/certmanager-manifests/ca.yaml | 16 + .../clusterAutoscalerCert.yaml | 17 + .../clusterAutoscalerProviderCert.yaml | 17 + .../selfsigned-issuer.yaml | 7 + .../cluster-autoscaler-cm.yaml | 12 + .../cluster-autoscaler.yaml | 176 + .../external-grpc-provider-service.yaml | 16 + .../external-grpc-provider.yaml | 51 + .../.gitignore | 2 + .../Dockerfile.amd64 | 18 + .../Dockerfile.arm64 | 18 + .../Makefile | 73 + .../main.go | 142 + .../wrapper/wrapper.go | 374 +++ .../externalgrpc_cloud_provider.go | 386 +++ .../externalgrpc_cloud_provider_test.go | 507 +++ .../externalgrpc/externalgrpc_node_group.go | 284 ++ .../externalgrpc_node_group_test.go | 463 +++ .../externalgrpc/externalgrpc_utils_test.go | 132 + .../externalgrpc/protos/externalgrpc.pb.go | 2942 +++++++++++++++++ .../externalgrpc/protos/externalgrpc.proto | 371 +++ .../protos/externalgrpc_grpc.pb.go | 695 ++++ hack/verify-gofmt.sh | 1 + hack/verify-golint.sh | 1 + 31 files changed, 6869 insertions(+) create mode 100644 cluster-autoscaler/cloudprovider/builder/builder_externalgrpc.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/OWNERS create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/README.md create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/ca-issuer.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/ca.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/clusterAutoscalerCert.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/clusterAutoscalerProviderCert.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/selfsigned-issuer.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/cluster-autoscaler-manifests/cluster-autoscaler-cm.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/cluster-autoscaler-manifests/cluster-autoscaler.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service-manifests/external-grpc-provider-service.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service-manifests/external-grpc-provider.yaml create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/.gitignore create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Dockerfile.amd64 create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Dockerfile.arm64 create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Makefile create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/main.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider_test.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_utils_test.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto create mode 100644 cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go diff --git a/cluster-autoscaler/README.md b/cluster-autoscaler/README.md index 66f31e7628f5..cc6c8f6ba91b 100644 --- a/cluster-autoscaler/README.md +++ b/cluster-autoscaler/README.md @@ -20,6 +20,7 @@ You should also take a look at the notes and "gotchas" for your specific cloud p * [Brightbox](./cloudprovider/brightbox/README.md) * [CloudStack](./cloudprovider/cloudstack/README.md) * [HuaweiCloud](./cloudprovider/huaweicloud/README.md) +* [External gRPC](./cloudprovider/externalgrpc/README.md) * [Hetzner](./cloudprovider/hetzner/README.md) * [Equinix Metal](./cloudprovider/packet/README.md#notes) * [IonosCloud](./cloudprovider/ionoscloud/README.md) @@ -166,6 +167,7 @@ Supported cloud providers: * CloudStack https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/cloudstack/README.md * Exoscale https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/exoscale/README.md * Equinix Metal https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/packet/README.md +* External gRPC https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/externalgrpc/README.md * OVHcloud https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/ovhcloud/README.md * Linode https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/linode/README.md * OCI https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/oci/README.md diff --git a/cluster-autoscaler/cloudprovider/builder/builder_all.go b/cluster-autoscaler/cloudprovider/builder/builder_all.go index c6637031c22a..cf38a62be9db 100644 --- a/cluster-autoscaler/cloudprovider/builder/builder_all.go +++ b/cluster-autoscaler/cloudprovider/builder/builder_all.go @@ -1,5 +1,10 @@ +<<<<<<< HEAD //go:build !gce && !aws && !azure && !kubemark && !alicloud && !magnum && !digitalocean && !clusterapi && !huaweicloud && !ionoscloud && !linode && !hetzner && !bizflycloud && !brightbox && !packet && !oci && !vultr && !tencentcloud && !civo // +build !gce,!aws,!azure,!kubemark,!alicloud,!magnum,!digitalocean,!clusterapi,!huaweicloud,!ionoscloud,!linode,!hetzner,!bizflycloud,!brightbox,!packet,!oci,!vultr,!tencentcloud,!civo +======= +//go:build !gce && !aws && !azure && !kubemark && !alicloud && !magnum && !digitalocean && !clusterapi && !huaweicloud && !ionoscloud && !linode && !hetzner && !bizflycloud && !brightbox && !packet && !oci && !vultr && !tencentcloud && !externalgrpc +// +build !gce,!aws,!azure,!kubemark,!alicloud,!magnum,!digitalocean,!clusterapi,!huaweicloud,!ionoscloud,!linode,!hetzner,!bizflycloud,!brightbox,!packet,!oci,!vultr,!tencentcloud,!externalgrpc +>>>>>>> db5336d08 (Implement external gRPC Cloud Provider) /* Copyright 2018 The Kubernetes Authors. @@ -32,6 +37,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/clusterapi" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/digitalocean" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/exoscale" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/gce" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/huaweicloud" @@ -57,6 +63,7 @@ var AvailableCloudProviders = []string{ cloudprovider.MagnumProviderName, cloudprovider.DigitalOceanProviderName, cloudprovider.ExoscaleProviderName, + cloudprovider.ExternalGrpcProviderName, cloudprovider.HuaweicloudProviderName, cloudprovider.HetznerProviderName, cloudprovider.OracleCloudProviderName, @@ -97,6 +104,8 @@ func buildCloudProvider(opts config.AutoscalingOptions, do cloudprovider.NodeGro return digitalocean.BuildDigitalOcean(opts, do, rl) case cloudprovider.ExoscaleProviderName: return exoscale.BuildExoscale(opts, do, rl) + case cloudprovider.ExternalGrpcProviderName: + return externalgrpc.BuildExternalGrpc(opts, do, rl) case cloudprovider.MagnumProviderName: return magnum.BuildMagnum(opts, do, rl) case cloudprovider.HuaweicloudProviderName: diff --git a/cluster-autoscaler/cloudprovider/builder/builder_externalgrpc.go b/cluster-autoscaler/cloudprovider/builder/builder_externalgrpc.go new file mode 100644 index 000000000000..5c2e8b574d9a --- /dev/null +++ b/cluster-autoscaler/cloudprovider/builder/builder_externalgrpc.go @@ -0,0 +1,43 @@ +//go:build externalgrpc +// +build externalgrpc + +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc" + "k8s.io/autoscaler/cluster-autoscaler/config" +) + +// AvailableCloudProviders supported by the cloud provider builder. +var AvailableCloudProviders = []string{ + cloudprovider.ExternalGrpcProviderName, +} + +// DefaultCloudProvider for externalgrpc-only build is externalgrpc. +const DefaultCloudProvider = cloudprovider.ExternalGrpcProviderName + +func buildCloudProvider(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider { + switch opts.CloudProviderName { + case cloudprovider.ExternalGrpcProviderName: + return externalgrpc.BuildExternalGrpc(opts, do, rl) + } + + return nil +} diff --git a/cluster-autoscaler/cloudprovider/cloud_provider.go b/cluster-autoscaler/cloudprovider/cloud_provider.go index 22d773258fa3..78a21fca8779 100644 --- a/cluster-autoscaler/cloudprovider/cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/cloud_provider.go @@ -72,6 +72,8 @@ const ( PacketProviderName = "packet" // TencentcloudProviderName gets the provider name of tencentcloud TencentcloudProviderName = "tencentcloud" + // ExternalGrpcProviderName gets the provider name of the external grpc provider + ExternalGrpcProviderName = "externalgrpc" // CivoProviderName gets the provider name of civo CivoProviderName = "civo" ) diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/OWNERS b/cluster-autoscaler/cloudprovider/externalgrpc/OWNERS new file mode 100644 index 000000000000..318b2bb4ee85 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/OWNERS @@ -0,0 +1,4 @@ +approvers: +#- dbonfigli +reviewers: +#- dbonfigli diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/README.md b/cluster-autoscaler/cloudprovider/externalgrpc/README.md new file mode 100644 index 000000000000..7e95990e4b7c --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/README.md @@ -0,0 +1,80 @@ +# External gRPC Cloud Provider + +The Exteral gRPC Cloud Provider provides a plugin system to support out-of-tree cloud provider implementations. + +Cluster Autoscaler adds or removes nodes from the cluster by creating or deleting VMs. To separate the autoscaling logic (the same for all clouds) from the API calls required to execute it (different for each cloud), the latter are hidden behind an interface, `CloudProvider`. Each supported cloud has its own implementation in this repository and `--cloud-provider` flag determines which one will be used. + +The gRPC Cloud Provider acts as a client for a cloud provider that implements its custom logic separately from the cluster autoscaler, and serves it as a `CloudProvider` gRPC service (similar to the `CloudProvider` interface) without the need to fork this project, follow its development lifecyle, adhere to its rules (e.g. do not use additional external dependencies) or implement the Cluster API. + +## Configuration + +For the cluster autoscaler parameters, use the `--cloud-provider=externalgrpc` flag and define the cloud configuration file with `--cloud-config=`, this is yaml file with the following parameters: + +| Key | Value | Mandatory | Default | +|-----|-------|-----------|---------| +| address | external gRPC cloud provider service address of the form "host:port", "host%zone:port", "[host]:port" or "[host%zone]:port" | yes | none | +| key | path to file containing the tls key, if using mTLS | no | none | +| cert | path to file containing the tls certificate, if using mTLS | no | none | +| cacert | path to file containing the CA certificate, if using mTLS | no | none | + +The use of mTLS is recommended, since simple, non-authenticated calls to the external gRPC cloud provider service will result in the creation / deletion of nodes. + +Log levels of intertest for this provider are: +* 1 (flag: ```--v=1```): basic logging of errors; +* 5 (flag: ```--v=5```): detailed logging of every call; + +For the deployment and configuration of an external gRPC cloud provider of choice, see its specific documentation. + +## Examples + +You can find an example of external gRPC cloud provider service implementation on the [examples/external-grpc-cloud-provider-service](examples/external-grpc-cloud-provider-service) directory: it is actually a server that wraps all the in-tree cloud providers. + +A complete example: +* deploy `cert-manager` and the manifests in [examples/certmanager-manifests](examples/certmanager-manifests) to generate certificates for gRPC client and server; +* build the image for the example external gRPC cloud provider service as defined in [examples/external-grpc-cloud-provider-service](examples/external-grpc-cloud-provider-service); +* deploy the example external gRPC cloud provider service using the manifests at [examples/external-grpc-cloud-provider-service-manifests](examples/external-grpc-cloud-provider-service-manifests), change the parameters as needed and test whichever cloud provider you want; +* deploy the cluster autoscaler selecting the External gRPC Cloud Provider using the manifests at [examples/cluster-autoscaler-manifests](examples/cluster-autoscaler-manifests). + +## Development + +### External gRPC Cloud Provider service Implementation + +To build a cloud provider, create a gRPC server for the `CloudProvider` service defined in [protos/externalgrpc.proto](protos/externalgrpc.proto) that implements all its required RPCs. + +### Caching + +The `CloudProvider` interface was designed with the assumption that its implementation functions would be fast, this may not be true anymore with the added overhead of gRPC. In the interest of performance, some gRPC API responses are cached by this cloud provider: +* `NodeGroupForNode()` caches the node group for a node until `Refresh()` is called; +* `NodeGroups()` caches the current node groups until `Refresh()` is called; +* `GPULabel()` and `GetAvailableGPUTypes()` are cached at first call and never wiped; +* A `NodeGroup` caches `MaxSize()`, `MinSize()` and `Debug()` return values during its creation, and `TemplateNodeInfo()` at its first call, these values will be cached for the lifetime of the `NodeGroup` object. + +### Code Generation + +To regenerate the gRPC code: + +1. install `protoc` and `protoc-gen-go-grpc`: + +```bash +go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 +go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 +``` + +2. generate gRPC client and server code: + +```bash +protoc \ + -I ./cluster-autoscaler \ + -I ./cluster-autoscaler/vendor \ + --go_out=. \ + --go-grpc_out=. \ + ./cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto +``` + +### General considerations + +Abstractions used by Cluster Autoscaler assume nodes belong to "node groups". All node within a group must be of the same machine type (have the same amount of resources), have the same set of labels and taints, and be located in the same availability zone. This doesn't mean a cloud has to have a concept of such node groups, but it helps. + +There must be a way to delete a specific node. If your cloud supports instance groups, and you are only able to provide a method to decrease the size of a given group, without guaranteeing which instance will be killed, it won't work well. + +There must be a way to match a Kubernetes node to an instance it is running on. This is usually done by kubelet setting node's `ProviderId` field to an instance id which can be used in API calls to cloud. diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/ca-issuer.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/ca-issuer.yaml new file mode 100644 index 000000000000..9e5167391e2c --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/ca-issuer.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer + namespace: kube-system +spec: + ca: + secretName: ca-root-secret diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/ca.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/ca.yaml new file mode 100644 index 000000000000..c7c71bd24933 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/ca.yaml @@ -0,0 +1,16 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: selfsigned-ca + namespace: kube-system +spec: + isCA: true + commonName: selfsigned-ca + secretName: ca-root-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/clusterAutoscalerCert.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/clusterAutoscalerCert.yaml new file mode 100644 index 000000000000..a2ae6280d613 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/clusterAutoscalerCert.yaml @@ -0,0 +1,17 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cluster-autoscaler-grpc-client-cert + namespace: kube-system +spec: + secretName: cluster-autoscaler-grpc-client-cert + commonName: cluster-autoscaler-grpc-client + dnsNames: + - "cluster-autoscaler-grpc-client" + duration: 87600h + usages: + - client auth + issuerRef: + name: ca-issuer + kind: Issuer + group: cert-manager.io diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/clusterAutoscalerProviderCert.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/clusterAutoscalerProviderCert.yaml new file mode 100644 index 000000000000..7a421b2a31d6 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/clusterAutoscalerProviderCert.yaml @@ -0,0 +1,17 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cluster-autoscaler-grpc-server-cert + namespace: kube-system +spec: + secretName: cluster-autoscaler-grpc-server-cert + commonName: ca-external-grpc-cloud-provider-service + duration: 87600h + usages: + - server auth + dnsNames: + - "ca-external-grpc-cloud-provider-service" + issuerRef: + name: ca-issuer + kind: Issuer + group: cert-manager.io diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/selfsigned-issuer.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/selfsigned-issuer.yaml new file mode 100644 index 000000000000..50c536404496 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/certmanager-manifests/selfsigned-issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: kube-system +spec: + selfSigned: {} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/cluster-autoscaler-manifests/cluster-autoscaler-cm.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/cluster-autoscaler-manifests/cluster-autoscaler-cm.yaml new file mode 100644 index 000000000000..a0b8b6d11bb1 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/cluster-autoscaler-manifests/cluster-autoscaler-cm.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-autoscaler-cloud-config + namespace: kube-system +data: + cloud-config: |- + address: "ca-external-grpc-cloud-provider-service:8086" + key: "/etc/ssl/client-cert/tls.key" + cert: "/etc/ssl/client-cert/tls.crt" + cacert: "/etc/ssl/client-cert/ca.crt" diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/cluster-autoscaler-manifests/cluster-autoscaler.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/cluster-autoscaler-manifests/cluster-autoscaler.yaml new file mode 100644 index 000000000000..930db9a5bd63 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/cluster-autoscaler-manifests/cluster-autoscaler.yaml @@ -0,0 +1,176 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "namespaces" + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes", "csidrivers", "csistoragecapacities"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + serviceAccountName: cluster-autoscaler + containers: + - image: k8s.gcr.io/autoscaling/cluster-autoscaler:latest + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=5 + - --cloud-provider=externalgrpc + - --cloud-config=/config/cloud-config + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + - name: cloud-config + mountPath: /config + readOnly: true + - name: cluster-autoscaler-grpc-client-cert + mountPath: "/etc/ssl/client-cert" + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" #/etc/ssl/certs/ca-bundle.crt for Amazon Linux Worker Nodes + - name: cloud-config + configMap: + name: cluster-autoscaler-cloud-config + - name: cluster-autoscaler-grpc-client-cert + secret: + secretName: cluster-autoscaler-grpc-client-cert + defaultMode: 0400 diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service-manifests/external-grpc-provider-service.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service-manifests/external-grpc-provider-service.yaml new file mode 100644 index 000000000000..a223e612f2a6 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service-manifests/external-grpc-provider-service.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: ca-external-grpc-cloud-provider-service + namespace: kube-system + labels: + app: ca-external-grpc-cloud-provider +spec: + ports: + - name: grpc + port: 8086 + protocol: TCP + targetPort: 8086 + selector: + app: ca-external-grpc-cloud-provider diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service-manifests/external-grpc-provider.yaml b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service-manifests/external-grpc-provider.yaml new file mode 100644 index 000000000000..a8c6d808b604 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service-manifests/external-grpc-provider.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ca-external-grpc-cloud-provider + namespace: kube-system + labels: + app: ca-external-grpc-cloud-provider +spec: + replicas: 1 + selector: + matchLabels: + app: ca-external-grpc-cloud-provider + template: + metadata: + labels: + app: ca-external-grpc-cloud-provider + spec: + containers: + - image: ca-external-grpc-cloud-provider-service:dev + name: ca-external-grpc-cloud-provider + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./ca-external-grpc-cloud-provider + - --v=10 + - --key-cert=/etc/ssl/server-cert/tls.key + - --cert=/etc/ssl/server-cert/tls.crt + - --ca-cert=/etc/ssl/server-cert/ca.crt + - --cloud-provider=aws + - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + volumeMounts: + - name: cluster-autoscaler-grpc-server-cert + mountPath: "/etc/ssl/server-cert" + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: /etc/ssl/certs/ca-certificates.crt #/etc/ssl/certs/ca-bundle.crt for Amazon Linux Worker Nodes + - name: cluster-autoscaler-grpc-server-cert + secret: + secretName: cluster-autoscaler-grpc-server-cert + defaultMode: 0400 diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/.gitignore b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/.gitignore new file mode 100644 index 000000000000..9d82f00239da --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/.gitignore @@ -0,0 +1,2 @@ +ca-external-grpc-cloud-provider-amd64 +ca-external-grpc-cloud-provider-arm64 diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Dockerfile.amd64 b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Dockerfile.amd64 new file mode 100644 index 000000000000..00bddd92ce4c --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Dockerfile.amd64 @@ -0,0 +1,18 @@ +# Copyright 2022 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG BASEIMAGE=gcr.io/distroless/static:latest-amd64 +FROM $BASEIMAGE + +COPY ca-external-grpc-cloud-provider-amd64 /ca-external-grpc-cloud-provider +CMD ["/ca-external-grpc-provider"] diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Dockerfile.arm64 b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Dockerfile.arm64 new file mode 100644 index 000000000000..cdc86c8b0a5a --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Dockerfile.arm64 @@ -0,0 +1,18 @@ +# Copyright 2022 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG BASEIMAGE=gcr.io/distroless/static:latest-arm64 +FROM $BASEIMAGE + +COPY ca-external-grpc-cloud-provider-arm64 /ca-external-grpc-cloud-provider +CMD ["/ca-external-grpc-provider"] diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Makefile b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Makefile new file mode 100644 index 000000000000..af1d3d1c345c --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/Makefile @@ -0,0 +1,73 @@ +ALL_ARCH = amd64 arm64 +all: $(addprefix build-arch-,$(ALL_ARCH)) + +TAG?=dev +FLAGS= +LDFLAGS?=-s +ENVVAR=CGO_ENABLED=0 +GOOS?=linux +GOARCH?=$(shell go env GOARCH) +REGISTRY?=staging-k8s.gcr.io +DOCKER_NETWORK?=default +ifdef BUILD_TAGS + TAGS_FLAG=--tags ${BUILD_TAGS} + PROVIDER=-${BUILD_TAGS} + FOR_PROVIDER=" for ${BUILD_TAGS}" +else + TAGS_FLAG= + PROVIDER= + FOR_PROVIDER= +endif +ifdef LDFLAGS + LDFLAGS_FLAG=--ldflags "${LDFLAGS}" +else + LDFLAGS_FLAG= +endif +ifdef DOCKER_RM + RM_FLAG=--rm +else + RM_FLAG= +endif +IMAGE=$(REGISTRY)/ca-external-grpc-cloud-provider$(PROVIDER) + +export DOCKER_CLI_EXPERIMENTAL := enabled + +build: build-arch-$(GOARCH) + +build-arch-%: clean-arch-% + $(ENVVAR) GOOS=$(GOOS) GOARCH=$* go build -o ca-external-grpc-cloud-provider-$* ${LDFLAGS_FLAG} ${TAGS_FLAG} + +make-image: make-image-arch-$(GOARCH) + +make-image-arch-%: +ifdef BASEIMAGE + docker build --pull --build-arg BASEIMAGE=${BASEIMAGE} \ + -t ${IMAGE}-$*:${TAG} \ + -f Dockerfile.$* . +else + docker build --pull \ + -t ${IMAGE}-$*:${TAG} \ + -f Dockerfile.$* . +endif + @echo "Image ${TAG}${FOR_PROVIDER}-$* completed" + +clean: clean-arch-$(GOARCH) + +clean-arch-%: + rm -f ca-external-grpc-cloud-provider-$* + +docker-builder: + docker build --network=${DOCKER_NETWORK} -t autoscaling-builder ../../../../../builder + +build-in-docker: build-in-docker-arch-$(GOARCH) + +build-in-docker-arch-%: clean-arch-% docker-builder + docker run ${RM_FLAG} -v `pwd`/../../../../:/gopath/src/k8s.io/autoscaler/cluster-autoscaler/:Z autoscaling-builder:latest \ + bash -c 'cd /gopath/src/k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service && BUILD_TAGS=${BUILD_TAGS} LDFLAGS="${LDFLAGS}" make build-arch-$*' + +container: container-arch-$(GOARCH) + +container-arch-%: build-in-docker-arch-% make-image-arch-% + @echo "Full in-docker image ${TAG}${FOR_PROVIDER}-$* completed" + +.PHONY: all build clean docker-builder build-in-docker diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/main.go b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/main.go new file mode 100644 index 000000000000..ea4144f0fa1e --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/main.go @@ -0,0 +1,142 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + "crypto/x509" + "flag" + "io/ioutil" + "net" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + cloudBuilder "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/builder" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/protos" + "k8s.io/autoscaler/cluster-autoscaler/config" + kube_flag "k8s.io/component-base/cli/flag" + klog "k8s.io/klog/v2" +) + +// MultiStringFlag is a flag for passing multiple parameters using same flag +type MultiStringFlag []string + +// String returns string representation of the node groups. +func (flag *MultiStringFlag) String() string { + return "[" + strings.Join(*flag, " ") + "]" +} + +// Set adds a new configuration. +func (flag *MultiStringFlag) Set(value string) error { + *flag = append(*flag, value) + return nil +} + +func multiStringFlag(name string, usage string) *MultiStringFlag { + value := new(MultiStringFlag) + flag.Var(value, name, usage) + return value +} + +var ( + // flags needed by the external grpc provider service + address = flag.String("address", ":8086", "The address to expose the grpc service.") + keyCert = flag.String("key-cert", "", "The path to the certificate key file. Empty string for insecure communication.") + cert = flag.String("cert", "", "The path to the certificate file. Empty string for insecure communication.") + cacert = flag.String("ca-cert", "", "The path to the ca certificate file. Empty string for insecure communication.") + + // flags needed by the specific cloud provider + cloudProviderFlag = flag.String("cloud-provider", cloudBuilder.DefaultCloudProvider, + "Cloud provider type. Available values: ["+strings.Join(cloudBuilder.AvailableCloudProviders, ",")+"]") + cloudConfig = flag.String("cloud-config", "", "The path to the cloud provider configuration file. Empty string for no configuration file.") + clusterName = flag.String("cluster-name", "", "Autoscaled cluster name, if available") + nodeGroupsFlag = multiStringFlag( + "nodes", + "sets min,max size and other configuration data for a node group in a format accepted by cloud provider. Can be used multiple times. Format: ::") + nodeGroupAutoDiscoveryFlag = multiStringFlag( + "node-group-auto-discovery", + "One or more definition(s) of node group auto-discovery. "+ + "A definition is expressed `:[[=]]`. "+ + "The `aws` and `gce` cloud providers are currently supported. AWS matches by ASG tags, e.g. `asg:tag=tagKey,anotherTagKey`. "+ + "GCE matches by IG name prefix, and requires you to specify min and max nodes per IG, e.g. `mig:namePrefix=pfx,min=0,max=10` "+ + "Can be used multiple times.") +) + +func main() { + klog.InitFlags(nil) + kube_flag.InitFlags() + + var s *grpc.Server + + // tls config + var serverOpt grpc.ServerOption + if *keyCert == "" || *cert == "" || *cacert == "" { + klog.V(1).Info("no cert specified, using insecure") + s = grpc.NewServer() + } else { + + certificate, err := tls.LoadX509KeyPair(*cert, *keyCert) + if err != nil { + klog.Fatalf("failed to read certificate files: %s", err) + } + certPool := x509.NewCertPool() + bs, err := ioutil.ReadFile(*cacert) + if err != nil { + klog.Fatalf("failed to read client ca cert: %s", err) + } + ok := certPool.AppendCertsFromPEM(bs) + if !ok { + klog.Fatal("failed to append client certs") + } + transportCreds := credentials.NewTLS(&tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{certificate}, + ClientCAs: certPool, + }) + serverOpt = grpc.Creds(transportCreds) + s = grpc.NewServer(serverOpt) + } + + //cloud provider config + autoscalingOptions := config.AutoscalingOptions{ + CloudProviderName: *cloudProviderFlag, + CloudConfig: *cloudConfig, + NodeGroupAutoDiscovery: *nodeGroupAutoDiscoveryFlag, + NodeGroups: *nodeGroupsFlag, + ClusterName: *clusterName, + ConcurrentGceRefreshes: 1, + UserAgent: "user-agent", + } + cloudProvider := cloudBuilder.NewCloudProvider(autoscalingOptions) + srv := wrapper.NewCloudProviderGrpcWrapper(cloudProvider) + + // listen + lis, err := net.Listen("tcp", *address) + if err != nil { + klog.Fatalf("failed to listen: %s", err) + } + + // serve + protos.RegisterCloudProviderServer(s, srv) + klog.V(1).Infof("Server ready at: %s\n", *address) + if err := s.Serve(lis); err != nil { + klog.Fatalf("failed to serve: %v", err) + } + +} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go new file mode 100644 index 000000000000..33a139785e67 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go @@ -0,0 +1,374 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrapper + +import ( + "context" + "fmt" + + "google.golang.org/protobuf/types/known/anypb" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/protos" + "k8s.io/autoscaler/cluster-autoscaler/config" + klog "k8s.io/klog/v2" +) + +// Wrapper implements protos.CloudProviderServer. +type Wrapper struct { + protos.UnimplementedCloudProviderServer + + provider cloudprovider.CloudProvider +} + +// NewCloudProviderGrpcWrapper creates a grpc wrapper for a cloud provider implementation. +func NewCloudProviderGrpcWrapper(provider cloudprovider.CloudProvider) *Wrapper { + return &Wrapper{ + provider: provider, + } +} + +// apiv1Node converts a protos.ExternalGrpcNode to a apiv1.Node. +func apiv1Node(pbNode *protos.ExternalGrpcNode) *apiv1.Node { + apiv1Node := &apiv1.Node{} + apiv1Node.ObjectMeta = metav1.ObjectMeta{ + Name: pbNode.GetName(), + Annotations: pbNode.GetAnnotations(), + Labels: pbNode.GetLabels(), + } + apiv1Node.Spec = apiv1.NodeSpec{ + ProviderID: pbNode.GetProviderID(), + } + return apiv1Node +} + +// apiv1Node converts an apiv1.Node to a protos.ExternalGrpcNode. +func pbNodeGroup(ng cloudprovider.NodeGroup) *protos.NodeGroup { + return &protos.NodeGroup{ + Id: ng.Id(), + MaxSize: int32(ng.MaxSize()), + MinSize: int32(ng.MinSize()), + Debug: ng.Debug(), + } +} + +func debug(req fmt.Stringer) { + klog.V(10).Infof("got gRPC request: %T %s", req, req) +} + +// NodeGroups is the wrapper for the cloud provider NodeGroups method. +func (w *Wrapper) NodeGroups(_ context.Context, req *protos.NodeGroupsRequest) (*protos.NodeGroupsResponse, error) { + debug(req) + + pbNgs := make([]*protos.NodeGroup, 0) + for _, ng := range w.provider.NodeGroups() { + pbNgs = append(pbNgs, pbNodeGroup(ng)) + } + return &protos.NodeGroupsResponse{ + NodeGroups: pbNgs, + }, nil +} + +// NodeGroupForNode is the wrapper for the cloud provider NodeGroupForNode method. +func (w *Wrapper) NodeGroupForNode(_ context.Context, req *protos.NodeGroupForNodeRequest) (*protos.NodeGroupForNodeResponse, error) { + debug(req) + + pbNode := req.GetNode() + if pbNode == nil { + return nil, fmt.Errorf("request fields were nil") + } + node := apiv1Node(pbNode) + ng, err := w.provider.NodeGroupForNode(node) + if err != nil { + return nil, err + } + if ng == nil { + return &protos.NodeGroupForNodeResponse{ + NodeGroup: &protos.NodeGroup{}, //NodeGroup with id = "", meaning the node should not be processed by cluster autoscaler + }, nil + } + return &protos.NodeGroupForNodeResponse{ + NodeGroup: pbNodeGroup(ng), + }, nil +} + +// PricingNodePrice is the wrapper for the cloud provider Pricing NodePrice method. +func (w *Wrapper) PricingNodePrice(_ context.Context, req *protos.PricingNodePriceRequest) (*protos.PricingNodePriceResponse, error) { + debug(req) + + model, err := w.provider.Pricing() + if err != nil { + return nil, err + } + reqNode := req.GetNode() + reqStartTime := req.GetStartTime() + reqEndTime := req.GetEndTime() + if reqNode == nil || reqStartTime == nil || reqEndTime == nil { + return nil, fmt.Errorf("request fields were nil") + } + price, nodePriceErr := model.NodePrice(apiv1Node(reqNode), reqStartTime.Time, reqEndTime.Time) + if nodePriceErr != nil { + return nil, nodePriceErr + } + return &protos.PricingNodePriceResponse{ + Price: price, + }, nil +} + +// PricingPodPrice is the wrapper for the cloud provider Pricing PodPrice method. +func (w *Wrapper) PricingPodPrice(_ context.Context, req *protos.PricingPodPriceRequest) (*protos.PricingPodPriceResponse, error) { + debug(req) + + model, err := w.provider.Pricing() + if err != nil { + return nil, err + } + reqPod := req.GetPod() + reqStartTime := req.GetStartTime() + reqEndTime := req.GetEndTime() + if reqPod == nil || reqStartTime == nil || reqEndTime == nil { + return nil, fmt.Errorf("request fields were nil") + } + price, podPriceErr := model.PodPrice(reqPod, reqStartTime.Time, reqEndTime.Time) + if podPriceErr != nil { + return nil, podPriceErr + } + return &protos.PricingPodPriceResponse{ + Price: price, + }, nil +} + +// GPULabel is the wrapper for the cloud provider GPULabel method. +func (w *Wrapper) GPULabel(_ context.Context, req *protos.GPULabelRequest) (*protos.GPULabelResponse, error) { + debug(req) + + label := w.provider.GPULabel() + return &protos.GPULabelResponse{ + Label: label, + }, nil +} + +// GetAvailableGPUTypes is the wrapper for the cloud provider GetAvailableGPUTypes method. +func (w *Wrapper) GetAvailableGPUTypes(_ context.Context, req *protos.GetAvailableGPUTypesRequest) (*protos.GetAvailableGPUTypesResponse, error) { + debug(req) + + types := w.provider.GetAvailableGPUTypes() + pbGpuTypes := make(map[string]*anypb.Any) + for t := range types { + pbGpuTypes[t] = nil + } + return &protos.GetAvailableGPUTypesResponse{ + GpuTypes: pbGpuTypes, + }, nil +} + +// Cleanup is the wrapper for the cloud provider Cleanup method. +func (w *Wrapper) Cleanup(_ context.Context, req *protos.CleanupRequest) (*protos.CleanupResponse, error) { + debug(req) + + err := w.provider.Cleanup() + return &protos.CleanupResponse{}, err +} + +// Refresh is the wrapper for the cloud provider Refresh method. +func (w *Wrapper) Refresh(_ context.Context, req *protos.RefreshRequest) (*protos.RefreshResponse, error) { + debug(req) + + err := w.provider.Refresh() + return &protos.RefreshResponse{}, err +} + +// getNodeGroup retrieves the NodeGroup giving its id. +func (w *Wrapper) getNodeGroup(id string) cloudprovider.NodeGroup { + for _, n := range w.provider.NodeGroups() { + if n.Id() == id { + return n + } + } + return nil +} + +// NodeGroupTargetSize is the wrapper for the cloud provider NodeGroup TargetSize method. +func (w *Wrapper) NodeGroupTargetSize(_ context.Context, req *protos.NodeGroupTargetSizeRequest) (*protos.NodeGroupTargetSizeResponse, error) { + debug(req) + + id := req.GetId() + ng := w.getNodeGroup(id) + if ng == nil { + return nil, fmt.Errorf("NodeGroup %q, not found", id) + } + size, err := ng.TargetSize() + if err != nil { + return nil, err + } + return &protos.NodeGroupTargetSizeResponse{ + TargetSize: int32(size), + }, nil +} + +// NodeGroupIncreaseSize is the wrapper for the cloud provider NodeGroup IncreaseSize method. +func (w *Wrapper) NodeGroupIncreaseSize(_ context.Context, req *protos.NodeGroupIncreaseSizeRequest) (*protos.NodeGroupIncreaseSizeResponse, error) { + debug(req) + + id := req.GetId() + ng := w.getNodeGroup(id) + if ng == nil { + return nil, fmt.Errorf("NodeGroup %q, not found", id) + } + err := ng.IncreaseSize(int(req.GetDelta())) + if err != nil { + return nil, err + } + return &protos.NodeGroupIncreaseSizeResponse{}, nil +} + +// NodeGroupDeleteNodes is the wrapper for the cloud provider NodeGroup DeleteNodes method. +func (w *Wrapper) NodeGroupDeleteNodes(_ context.Context, req *protos.NodeGroupDeleteNodesRequest) (*protos.NodeGroupDeleteNodesResponse, error) { + debug(req) + + id := req.GetId() + ng := w.getNodeGroup(id) + if ng == nil { + return nil, fmt.Errorf("NodeGroup %q, not found", id) + } + nodes := make([]*apiv1.Node, 0) + for _, n := range req.GetNodes() { + nodes = append(nodes, apiv1Node(n)) + } + err := ng.DeleteNodes(nodes) + if err != nil { + return nil, err + } + return &protos.NodeGroupDeleteNodesResponse{}, nil +} + +// NodeGroupDecreaseTargetSize is the wrapper for the cloud provider NodeGroup DecreaseTargetSize method. +func (w *Wrapper) NodeGroupDecreaseTargetSize(_ context.Context, req *protos.NodeGroupDecreaseTargetSizeRequest) (*protos.NodeGroupDecreaseTargetSizeResponse, error) { + debug(req) + + id := req.GetId() + ng := w.getNodeGroup(id) + if ng == nil { + return nil, fmt.Errorf("NodeGroup %q, not found", id) + } + err := ng.DecreaseTargetSize(int(req.GetDelta())) + if err != nil { + return nil, err + } + return &protos.NodeGroupDecreaseTargetSizeResponse{}, nil +} + +// NodeGroupNodes is the wrapper for the cloud provider NodeGroup Nodes method. +func (w *Wrapper) NodeGroupNodes(_ context.Context, req *protos.NodeGroupNodesRequest) (*protos.NodeGroupNodesResponse, error) { + debug(req) + + id := req.GetId() + ng := w.getNodeGroup(id) + if ng == nil { + return nil, fmt.Errorf("NodeGroup %q, not found", id) + } + instances, err := ng.Nodes() + if err != nil { + return nil, err + } + pbInstances := make([]*protos.Instance, 0) + for _, i := range instances { + pbInstance := new(protos.Instance) + pbInstance.Id = i.Id + if i.Status == nil { + pbInstance.Status = &protos.InstanceStatus{ + InstanceState: protos.InstanceStatus_unspecified, + ErrorInfo: &protos.InstanceErrorInfo{}, + } + } else { + pbInstance.Status = new(protos.InstanceStatus) + pbInstance.Status.InstanceState = protos.InstanceStatus_InstanceState(i.Status.State) + if i.Status.ErrorInfo == nil { + pbInstance.Status.ErrorInfo = &protos.InstanceErrorInfo{} + } else { + pbInstance.Status.ErrorInfo = &protos.InstanceErrorInfo{ + ErrorCode: i.Status.ErrorInfo.ErrorCode, + ErrorMessage: i.Status.ErrorInfo.ErrorMessage, + InstanceErrorClass: int32(i.Status.ErrorInfo.ErrorClass), + } + } + } + pbInstances = append(pbInstances, pbInstance) + } + return &protos.NodeGroupNodesResponse{ + Instances: pbInstances, + }, nil +} + +// NodeGroupTemplateNodeInfo is the wrapper for the cloud provider NodeGroup TemplateNodeInfo method. +func (w *Wrapper) NodeGroupTemplateNodeInfo(_ context.Context, req *protos.NodeGroupTemplateNodeInfoRequest) (*protos.NodeGroupTemplateNodeInfoResponse, error) { + debug(req) + + id := req.GetId() + ng := w.getNodeGroup(id) + if ng == nil { + return nil, fmt.Errorf("NodeGroup %q, not found", id) + } + info, err := ng.TemplateNodeInfo() + if err != nil { + return nil, err + } + return &protos.NodeGroupTemplateNodeInfoResponse{ + NodeInfo: info.Node(), + }, nil +} + +// NodeGroupGetOptions is the wrapper for the cloud provider NodeGroup GetOptions method. +func (w *Wrapper) NodeGroupGetOptions(_ context.Context, req *protos.NodeGroupAutoscalingOptionsRequest) (*protos.NodeGroupAutoscalingOptionsResponse, error) { + debug(req) + + id := req.GetId() + ng := w.getNodeGroup(id) + if ng == nil { + return nil, fmt.Errorf("NodeGroup %q, not found", id) + } + pbDefaults := req.GetDefaults() + if pbDefaults == nil { + return nil, fmt.Errorf("request fields were nil") + } + defaults := config.NodeGroupAutoscalingOptions{ + ScaleDownUtilizationThreshold: pbDefaults.GetScaleDownGpuUtilizationThreshold(), + ScaleDownGpuUtilizationThreshold: pbDefaults.GetScaleDownGpuUtilizationThreshold(), + ScaleDownUnneededTime: pbDefaults.GetScaleDownUnneededTime().Duration, + ScaleDownUnreadyTime: pbDefaults.GetScaleDownUnneededTime().Duration, + } + opts, err := ng.GetOptions(defaults) + if err != nil { + return nil, err + } + if opts == nil { + return nil, fmt.Errorf("GetOptions not implemented") //make this explicitly so that grpc response is discarded + } + return &protos.NodeGroupAutoscalingOptionsResponse{ + NodeGroupAutoscalingOptions: &protos.NodeGroupAutoscalingOptions{ + ScaleDownUtilizationThreshold: opts.ScaleDownUtilizationThreshold, + ScaleDownGpuUtilizationThreshold: opts.ScaleDownGpuUtilizationThreshold, + ScaleDownUnneededTime: &metav1.Duration{ + Duration: opts.ScaleDownUnneededTime, + }, + ScaleDownUnreadyTime: &metav1.Duration{ + Duration: opts.ScaleDownUnreadyTime, + }, + }, + }, nil +} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider.go new file mode 100644 index 000000000000..d8743e918ba0 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider.go @@ -0,0 +1,386 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package externalgrpc + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "gopkg.in/yaml.v2" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/protos" + "k8s.io/autoscaler/cluster-autoscaler/config" + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + klog "k8s.io/klog/v2" +) + +const ( + grpcTimeout = 5 * time.Second +) + +// externalGrpcCloudProvider implements CloudProvider interface. +type externalGrpcCloudProvider struct { + resourceLimiter *cloudprovider.ResourceLimiter + client protos.CloudProviderClient + + mutex sync.Mutex + nodeGroupForNodeCache map[string]cloudprovider.NodeGroup // used to cache NodeGroupForNode grpc calls. Discarded at each Refresh() + nodeGroupsCache []cloudprovider.NodeGroup // used to cache NodeGroups grpc calls. Discarded at each Refresh() + gpuLabelCache *string // used to cache GPULabel grpc calls + gpuTypesCache map[string]struct{} // used to cache GetAvailableGPUTypes grpc calls +} + +// Name returns name of the cloud provider. +func (e *externalGrpcCloudProvider) Name() string { + return cloudprovider.ExternalGrpcProviderName +} + +// NodeGroups returns all node groups configured for this cloud provider. +func (e *externalGrpcCloudProvider) NodeGroups() []cloudprovider.NodeGroup { + e.mutex.Lock() + defer e.mutex.Unlock() + + if e.nodeGroupsCache != nil { + klog.V(5).Info("Returning cached NodeGroups") + return e.nodeGroupsCache + } + nodeGroups := make([]cloudprovider.NodeGroup, 0) + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Info("Performing gRPC call NodeGroups") + res, err := e.client.NodeGroups(ctx, &protos.NodeGroupsRequest{}) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroups: %v", err) + return nodeGroups + } + for _, pbNg := range res.GetNodeGroups() { + ng := &NodeGroup{ + id: pbNg.Id, + minSize: int(pbNg.MinSize), + maxSize: int(pbNg.MaxSize), + debug: pbNg.Debug, + client: e.client, + } + nodeGroups = append(nodeGroups, ng) + } + e.nodeGroupsCache = nodeGroups + return nodeGroups +} + +// NodeGroupForNode returns the node group for the given node, nil if the node +// should not be processed by cluster autoscaler, or non-nil error if such +// occurred. Must be implemented. +func (e *externalGrpcCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.NodeGroup, error) { + e.mutex.Lock() + defer e.mutex.Unlock() + + if node == nil { + return nil, fmt.Errorf("node in NodeGroupForNode call cannot be nil") + } + nodeID := node.Name + node.Spec.ProviderID //ProviderID is empty in some edge cases + // lookup cache + if ng, ok := e.nodeGroupForNodeCache[nodeID]; ok { + klog.V(5).Infof("Returning cached information for NodeGroupForNode for node %v - %v", node.Name, node.Spec.ProviderID) + return ng, nil + } + // perform grpc call + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call NodeGroupForNode for node %v - %v", node.Name, node.Spec.ProviderID) + res, err := e.client.NodeGroupForNode(ctx, &protos.NodeGroupForNodeRequest{ + Node: externalGrpcNode(node), + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroupForNode: %v", err) + return nil, err + } + pbNg := res.GetNodeGroup() + if pbNg.GetId() == "" { // if id == "" then the node should not be processed by cluster autoscaler, do not cache this + return nil, nil + } + ng := &NodeGroup{ + id: pbNg.GetId(), + maxSize: int(pbNg.GetMaxSize()), + minSize: int(pbNg.GetMinSize()), + debug: pbNg.GetDebug(), + client: e.client, + } + e.nodeGroupForNodeCache[nodeID] = ng + return ng, nil +} + +// pricingModel implements cloudprovider.PricingModel interface. +type pricingModel struct { + client protos.CloudProviderClient +} + +// NodePrice returns a price of running the given node for a given period of time. +func (m *pricingModel) NodePrice(node *apiv1.Node, startTime time.Time, endTime time.Time) (float64, error) { + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call PricingNodePrice for node %v", node.Name) + start := metav1.NewTime(startTime) + end := metav1.NewTime(endTime) + res, err := m.client.PricingNodePrice(ctx, &protos.PricingNodePriceRequest{ + Node: externalGrpcNode(node), + StartTime: &start, + EndTime: &end, + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call PricingNodePrice: %v", err) + return 0, err + } + return res.GetPrice(), nil +} + +// PodPrice returns a theoretical minimum price of running a pod for a given +// period of time on a perfectly matching machine. +func (m *pricingModel) PodPrice(pod *apiv1.Pod, startTime time.Time, endTime time.Time) (float64, error) { + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call PricingPodPrice for pod %v", pod.Name) + start := metav1.NewTime(startTime) + end := metav1.NewTime(endTime) + res, err := m.client.PricingPodPrice(ctx, &protos.PricingPodPriceRequest{ + Pod: pod, + StartTime: &start, + EndTime: &end, + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call PricingPodPrice: %v", err) + return 0, err + } + return res.GetPrice(), nil +} + +// Pricing returns pricing model for this cloud provider or error if not available. +// Implementation optional. +// +// The external gRPC provider will always return a pricing model without errors, +// even if a cloud provider does not actually support this feature, errors will be returned +// by subsequent calls to the pricing model if this is the case. +func (e *externalGrpcCloudProvider) Pricing() (cloudprovider.PricingModel, errors.AutoscalerError) { + return &pricingModel{ + client: e.client, + }, nil +} + +// GetAvailableMachineTypes get all machine types that can be requested from the cloud provider. +// Implementation optional. +func (e *externalGrpcCloudProvider) GetAvailableMachineTypes() ([]string, error) { + return []string{}, cloudprovider.ErrNotImplemented +} + +// NewNodeGroup builds a theoretical node group based on the node definition provided. The node group is not automatically +// created on the cloud provider side. The node group is not returned by NodeGroups() until it is created. +// Implementation optional. +func (e *externalGrpcCloudProvider) NewNodeGroup(machineType string, labels map[string]string, systemLabels map[string]string, + taints []apiv1.Taint, extraResources map[string]resource.Quantity) (cloudprovider.NodeGroup, error) { + return nil, cloudprovider.ErrNotImplemented +} + +// GetResourceLimiter returns struct containing limits (max, min) for resources (cores, memory etc.). +func (e *externalGrpcCloudProvider) GetResourceLimiter() (*cloudprovider.ResourceLimiter, error) { + return e.resourceLimiter, nil +} + +// GPULabel returns the label added to nodes with GPU resource. +func (e *externalGrpcCloudProvider) GPULabel() string { + e.mutex.Lock() + defer e.mutex.Unlock() + + if e.gpuLabelCache != nil { + klog.V(5).Info("Returning cached GPULabel") + return *e.gpuLabelCache + } + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Info("Performing gRPC call GPULabel") + res, err := e.client.GPULabel(ctx, &protos.GPULabelRequest{}) + if err != nil { + klog.V(1).Infof("Error on gRPC call GPULabel: %v", err) + return "" + } + gpuLabel := res.GetLabel() + e.gpuLabelCache = &gpuLabel + return gpuLabel +} + +// GetAvailableGPUTypes return all available GPU types cloud provider supports. +func (e *externalGrpcCloudProvider) GetAvailableGPUTypes() map[string]struct{} { + e.mutex.Lock() + defer e.mutex.Unlock() + + if e.gpuTypesCache != nil { + klog.V(5).Info("Returning cached GetAvailableGPUTypes") + return e.gpuTypesCache + } + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Info("Performing gRPC call GetAvailableGPUTypes") + res, err := e.client.GetAvailableGPUTypes(ctx, &protos.GetAvailableGPUTypesRequest{}) + if err != nil { + klog.V(1).Infof("Error on gRPC call GetAvailableGPUTypes: %v", err) + return nil + } + gpuTypes := make(map[string]struct{}) + var empty struct{} + for k := range res.GetGpuTypes() { + gpuTypes[k] = empty + } + e.gpuTypesCache = gpuTypes + return gpuTypes +} + +// Cleanup cleans up open resources before the cloud provider is destroyed, i.e. go routines etc. +func (e *externalGrpcCloudProvider) Cleanup() error { + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Info("Performing gRPC call Cleanup") + _, err := e.client.Cleanup(ctx, &protos.CleanupRequest{}) + if err != nil { + klog.V(1).Infof("Error on gRPC call Cleanup: %v", err) + return err + } + return nil +} + +// Refresh is called before every main loop and can be used to dynamically update cloud provider state. +// In particular the list of node groups returned by NodeGroups can change as a result of CloudProvider.Refresh(). +func (e *externalGrpcCloudProvider) Refresh() error { + // invalidate cache + e.mutex.Lock() + e.nodeGroupForNodeCache = make(map[string]cloudprovider.NodeGroup) + e.nodeGroupsCache = nil + e.mutex.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Info("Performing gRPC call Refresh") + _, err := e.client.Refresh(ctx, &protos.RefreshRequest{}) + if err != nil { + klog.V(1).Infof("Error on gRPC call Refresh: %v", err) + return err + } + return nil +} + +// BuildExternalGrpc builds the externalgrpc cloud provider. +func BuildExternalGrpc( + opts config.AutoscalingOptions, + do cloudprovider.NodeGroupDiscoveryOptions, + rl *cloudprovider.ResourceLimiter, +) cloudprovider.CloudProvider { + if opts.CloudConfig == "" { + klog.Fatal("No config file provided, please specify it via the --cloud-config flag") + } + config, err := ioutil.ReadFile(opts.CloudConfig) + if err != nil { + klog.Fatalf("Could not open cloud provider configuration file %q: %v", opts.CloudConfig, err) + } + client, err := newExternalGrpcCloudProviderClient(config) + if err != nil { + klog.Fatalf("Could not create gRPC client: %v", err) + } + return newExternalGrpcCloudProvider(client, rl) +} + +// cloudConfig is the struct hoding the configs to connect to the external cluster autoscaler provider service. +type cloudConfig struct { + Address string `yaml:"address"` // external cluster autoscaler provider address of the form "host:port", "host%zone:port", "[host]:port" or "[host%zone]:port" + Key string `yaml:"key"` // path to file containing the tls key + Cert string `yaml:"cert"` // path to file containing the tls certificate + Cacert string `yaml:"cacert"` // path to file containing the CA certificate +} + +func newExternalGrpcCloudProviderClient(config []byte) (protos.CloudProviderClient, error) { + var yamlConfig cloudConfig + err := yaml.Unmarshal([]byte(config), &yamlConfig) + if err != nil { + return nil, fmt.Errorf("can't parse YAML: %v", err) + } + host, _, err := net.SplitHostPort(yamlConfig.Address) + if err != nil { + return nil, fmt.Errorf("failed to parse address: %v", err) + } + var dialOpt grpc.DialOption + if len(yamlConfig.Cert) == 0 { + klog.V(5).Info("No certs specified in external gRPC provider config, using insecure mode") + dialOpt = grpc.WithInsecure() + } else { + certFile, err := ioutil.ReadFile(yamlConfig.Cert) + if err != nil { + return nil, fmt.Errorf("could not open Cert configuration file %q: %v", yamlConfig.Cert, err) + } + keyFile, err := ioutil.ReadFile(yamlConfig.Key) + if err != nil { + return nil, fmt.Errorf("could not open Key configuration file %q: %v", yamlConfig.Key, err) + } + cacertFile, err := ioutil.ReadFile(yamlConfig.Cacert) + if err != nil { + return nil, fmt.Errorf("could not open Cacert configuration file %q: %v", yamlConfig.Cacert, err) + } + cert, err := tls.X509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("failed to parse cert key pair: %v", err) + } + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM(cacertFile) + if !ok { + return nil, fmt.Errorf("failed to parse ca: %v", err) + } + transportCreds := credentials.NewTLS(&tls.Config{ + ServerName: host, + Certificates: []tls.Certificate{cert}, + RootCAs: certPool, + }) + dialOpt = grpc.WithTransportCredentials(transportCreds) + } + conn, err := grpc.Dial(yamlConfig.Address, dialOpt) + if err != nil { + return nil, fmt.Errorf("failed to dial server: %v", err) + } + return protos.NewCloudProviderClient(conn), nil +} + +func newExternalGrpcCloudProvider(client protos.CloudProviderClient, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider { + return &externalGrpcCloudProvider{ + resourceLimiter: rl, + client: client, + nodeGroupForNodeCache: make(map[string]cloudprovider.NodeGroup), + } +} + +// externalGrpcNode converts an apiv1.Node to a protos.ExternalGrpcNode. +func externalGrpcNode(apiv1Node *apiv1.Node) *protos.ExternalGrpcNode { + return &protos.ExternalGrpcNode{ + ProviderID: apiv1Node.Spec.ProviderID, + Name: apiv1Node.Name, + Labels: apiv1Node.Labels, + Annotations: apiv1Node.Annotations, + } +} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider_test.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider_test.go new file mode 100644 index 000000000000..a123214cb6c1 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider_test.go @@ -0,0 +1,507 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package externalgrpc + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "google.golang.org/protobuf/types/known/anypb" + apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/protos" +) + +func TestCloudProvider_NodeGroups(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + c := newExternalGrpcCloudProvider(client, nil) + + m.On("Refresh", mock.Anything, mock.Anything).Return(&protos.RefreshResponse{}, nil) + + // test answer with multiple node groups + m.On( + "NodeGroups", mock.Anything, mock.Anything, + ).Return( + &protos.NodeGroupsResponse{ + NodeGroups: []*protos.NodeGroup{ + {Id: "1", MinSize: 10, MaxSize: 20, Debug: "test1"}, + {Id: "2", MinSize: 30, MaxSize: 40, Debug: "test2"}, + }, + }, nil, + ).Times(2) + + ngs := c.NodeGroups() + assert.Equal(t, 2, len(ngs)) + for _, ng := range ngs { + if ng.Id() == "1" { + assert.Equal(t, 10, ng.MinSize()) + assert.Equal(t, 20, ng.MaxSize()) + assert.Equal(t, "test1", ng.Debug()) + } else if ng.Id() == "2" { + assert.Equal(t, 30, ng.MinSize()) + assert.Equal(t, 40, ng.MaxSize()) + assert.Equal(t, "test2", ng.Debug()) + } else { + assert.Fail(t, "node group id not recognized") + } + } + + // test cached answer + m.AssertNumberOfCalls(t, "NodeGroups", 1) + ngs = c.NodeGroups() + assert.Equal(t, 2, len(ngs)) + m.AssertNumberOfCalls(t, "NodeGroups", 1) + + // test answer after refresh to clear cached answer + err := c.Refresh() + assert.NoError(t, err) + ngs = c.NodeGroups() + assert.Equal(t, 2, len(ngs)) + m.AssertNumberOfCalls(t, "NodeGroups", 2) + + // test empty answer + err = c.Refresh() + assert.NoError(t, err) + + m.On( + "NodeGroups", mock.Anything, mock.Anything, + ).Return( + &protos.NodeGroupsResponse{ + NodeGroups: make([]*protos.NodeGroup, 0), + }, nil, + ).Once() + + ngs = c.NodeGroups() + assert.NotNil(t, ngs) + assert.Equal(t, 0, len(ngs)) + + // test grpc error + err = c.Refresh() + assert.NoError(t, err) + + m.On( + "NodeGroups", mock.Anything, mock.Anything, + ).Return( + &protos.NodeGroupsResponse{ + NodeGroups: make([]*protos.NodeGroup, 0), + }, + fmt.Errorf("mock error"), + ).Once() + + ngs = c.NodeGroups() + assert.NotNil(t, ngs) + assert.Equal(t, 0, len(ngs)) + +} + +func TestCloudProvider_NodeGroupForNode(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + c := newExternalGrpcCloudProvider(client, nil) + + m.On("Refresh", mock.Anything, mock.Anything).Return(&protos.RefreshResponse{}, nil) + + // test correct answer + m.On( + "NodeGroupForNode", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupForNodeRequest) bool { + return req.Node.Name == "node1" + }), + ).Return( + &protos.NodeGroupForNodeResponse{ + NodeGroup: &protos.NodeGroup{Id: "1", MinSize: 10, MaxSize: 20, Debug: "test1"}, + }, nil, + ) + m.On( + "NodeGroupForNode", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupForNodeRequest) bool { + return req.Node.Name == "node2" + }), + ).Return( + &protos.NodeGroupForNodeResponse{ + NodeGroup: &protos.NodeGroup{Id: "2", MinSize: 30, MaxSize: 40, Debug: "test2"}, + }, nil, + ) + + apiv1Node1 := &apiv1.Node{} + apiv1Node1.Name = "node1" + apiv1Node1.Spec.ProviderID = "providerId://node1" + + ng1, err := c.NodeGroupForNode(apiv1Node1) + assert.NoError(t, err) + assert.Equal(t, "1", ng1.Id()) + assert.Equal(t, 10, ng1.MinSize()) + assert.Equal(t, 20, ng1.MaxSize()) + assert.Equal(t, "test1", ng1.Debug()) + + apiv1Node2 := &apiv1.Node{} + apiv1Node2.Name = "node2" + apiv1Node2.Spec.ProviderID = "providerId://node2" + + ng2, err := c.NodeGroupForNode(apiv1Node2) + assert.NoError(t, err) + assert.Equal(t, "2", ng2.Id()) + assert.Equal(t, 30, ng2.MinSize()) + assert.Equal(t, 40, ng2.MaxSize()) + assert.Equal(t, "test2", ng2.Debug()) + + // test cached answer + ng1, err = c.NodeGroupForNode(apiv1Node1) + assert.NoError(t, err) + assert.Equal(t, "1", ng1.Id()) + m.AssertNumberOfCalls(t, "NodeGroupForNode", 2) + + // test clear cache + err = c.Refresh() + assert.NoError(t, err) + + ng1, err = c.NodeGroupForNode(apiv1Node1) + assert.NoError(t, err) + assert.Equal(t, "1", ng1.Id()) + m.AssertNumberOfCalls(t, "NodeGroupForNode", 3) + + //test no node group for node + err = c.Refresh() + assert.NoError(t, err) + + m.On( + "NodeGroupForNode", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupForNodeRequest) bool { + return req.Node.Name == "node3" + }), + ).Return( + &protos.NodeGroupForNodeResponse{ + NodeGroup: &protos.NodeGroup{Id: ""}, + }, nil, + ) + + apiv1Node3 := &apiv1.Node{} + apiv1Node3.Name = "node3" + apiv1Node3.Spec.ProviderID = "providerId://node3" + + ng3, err := c.NodeGroupForNode(apiv1Node3) + assert.NoError(t, err) + assert.Equal(t, nil, ng3) + + //test grpc error + err = c.Refresh() + assert.NoError(t, err) + + m.On( + "NodeGroupForNode", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupForNodeRequest) bool { + return req.Node.Name == "node4" + }), + ).Return( + &protos.NodeGroupForNodeResponse{ + NodeGroup: &protos.NodeGroup{Id: ""}, + }, + fmt.Errorf("mock error"), + ) + + apiv1Node4 := &apiv1.Node{} + apiv1Node4.Name = "node4" + apiv1Node4.Spec.ProviderID = "providerId://node4" + + _, err = c.NodeGroupForNode(apiv1Node4) + assert.Error(t, err) + + //test error is not cached + _, err = c.NodeGroupForNode(apiv1Node4) + assert.Error(t, err) + m.AssertNumberOfCalls(t, "NodeGroupForNode", 6) + + //test nil node param + _, err = c.NodeGroupForNode(nil) + assert.Error(t, err) +} + +func TestCloudProvider_Pricing(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + c := newExternalGrpcCloudProvider(client, nil) + + model, errPricing := c.Pricing() + assert.NoError(t, errPricing) + assert.NotNil(t, model) + + // test correct NodePrice call + m.On( + "PricingNodePrice", mock.Anything, mock.MatchedBy(func(req *protos.PricingNodePriceRequest) bool { + return req.Node.Name == "node1" + }), + ).Return( + &protos.PricingNodePriceResponse{Price: 100}, + nil, + ) + m.On( + "PricingNodePrice", mock.Anything, mock.MatchedBy(func(req *protos.PricingNodePriceRequest) bool { + return req.Node.Name == "node2" + }), + ).Return( + &protos.PricingNodePriceResponse{Price: 200}, + nil, + ) + + apiv1Node1 := &apiv1.Node{} + apiv1Node1.Name = "node1" + + price, err := model.NodePrice(apiv1Node1, time.Time{}, time.Time{}) + assert.NoError(t, err) + assert.Equal(t, float64(100), price) + + apiv1Node2 := &apiv1.Node{} + apiv1Node2.Name = "node2" + + price, err = model.NodePrice(apiv1Node2, time.Time{}, time.Time{}) + assert.NoError(t, err) + assert.Equal(t, float64(200), price) + + // test grpc error for NodePrice + m.On( + "PricingNodePrice", mock.Anything, mock.MatchedBy(func(req *protos.PricingNodePriceRequest) bool { + return req.Node.Name == "node3" + }), + ).Return( + &protos.PricingNodePriceResponse{}, + fmt.Errorf("mock error"), + ) + + apiv1Node3 := &apiv1.Node{} + apiv1Node3.Name = "node3" + + _, err = model.NodePrice(apiv1Node3, time.Time{}, time.Time{}) + assert.Error(t, err) + + // test correct PodPrice call + m.On( + "PricingPodPrice", mock.Anything, mock.MatchedBy(func(req *protos.PricingPodPriceRequest) bool { + return req.Pod.Name == "pod1" + }), + ).Return( + &protos.PricingPodPriceResponse{Price: 100}, + nil, + ) + m.On( + "PricingPodPrice", mock.Anything, mock.MatchedBy(func(req *protos.PricingPodPriceRequest) bool { + return req.Pod.Name == "pod2" + }), + ).Return( + &protos.PricingPodPriceResponse{Price: 200}, + nil, + ) + + apiv1Pod1 := &apiv1.Pod{} + apiv1Pod1.Name = "pod1" + + price, err = model.PodPrice(apiv1Pod1, time.Time{}, time.Time{}) + assert.NoError(t, err) + assert.Equal(t, float64(100), price) + + apiv1Pod2 := &apiv1.Pod{} + apiv1Pod2.Name = "pod2" + + price, err = model.PodPrice(apiv1Pod2, time.Time{}, time.Time{}) + assert.NoError(t, err) + assert.Equal(t, float64(200), price) + + // test grpc error for PodPrice + m.On( + "PricingPodPrice", mock.Anything, mock.MatchedBy(func(req *protos.PricingPodPriceRequest) bool { + return req.Pod.Name == "pod3" + }), + ).Return( + &protos.PricingPodPriceResponse{}, + fmt.Errorf("mock error"), + ) + + apiv1Pod3 := &apiv1.Pod{} + apiv1Pod3.Name = "pod3" + + _, err = model.PodPrice(apiv1Pod3, time.Time{}, time.Time{}) + assert.Error(t, err) +} + +func TestCloudProvider_GPULabel(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + c := newExternalGrpcCloudProvider(client, nil) + + m.On("Refresh", mock.Anything, mock.Anything).Return(&protos.RefreshResponse{}, nil) + + // test correct call + m.On( + "GPULabel", mock.Anything, mock.Anything, + ).Return( + &protos.GPULabelResponse{Label: "gpu_label"}, + nil, + ) + + label := c.GPULabel() + assert.Equal(t, "gpu_label", label) + + // test cache + label = c.GPULabel() + assert.Equal(t, "gpu_label", label) + m.AssertNumberOfCalls(t, "GPULabel", 1) + + // test grpc error + client2, m2, teardown2 := setupTest(t) + defer teardown2() + c2 := newExternalGrpcCloudProvider(client2, nil) + + m2.On("Refresh", mock.Anything, mock.Anything).Return(&protos.RefreshResponse{}, nil) + + m2.On( + "GPULabel", mock.Anything, mock.Anything, + ).Return( + &protos.GPULabelResponse{Label: "gpu_label"}, + fmt.Errorf("mock error"), + ) + label = c2.GPULabel() + assert.Equal(t, "", label) + + //test error is not cached + label = c2.GPULabel() + assert.Equal(t, "", label) + m2.AssertNumberOfCalls(t, "GPULabel", 2) + +} + +func TestCloudProvider_GetAvailableGPUTypes(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + c := newExternalGrpcCloudProvider(client, nil) + + m.On("Refresh", mock.Anything, mock.Anything).Return(&protos.RefreshResponse{}, nil) + + // test correct call + pbGpuTypes := make(map[string]*anypb.Any) + pbGpuTypes["type1"] = &anypb.Any{} + pbGpuTypes["type2"] = &anypb.Any{} + + m.On( + "GetAvailableGPUTypes", mock.Anything, mock.Anything, + ).Return( + &protos.GetAvailableGPUTypesResponse{GpuTypes: pbGpuTypes}, + nil, + ) + + gpuTypes := c.GetAvailableGPUTypes() + assert.NotNil(t, gpuTypes) + assert.NotNil(t, gpuTypes["type1"]) + assert.NotNil(t, gpuTypes["type2"]) + + // test cache + gpuTypes = c.GetAvailableGPUTypes() + assert.NotNil(t, gpuTypes) + assert.NotNil(t, gpuTypes["type1"]) + assert.NotNil(t, gpuTypes["type2"]) + m.AssertNumberOfCalls(t, "GetAvailableGPUTypes", 1) + + // test no gpu types + client2, m2, teardown2 := setupTest(t) + defer teardown2() + c2 := newExternalGrpcCloudProvider(client2, nil) + + m2.On( + "GetAvailableGPUTypes", mock.Anything, mock.Anything, + ).Return( + &protos.GetAvailableGPUTypesResponse{GpuTypes: nil}, + nil, + ) + + gpuTypes = c2.GetAvailableGPUTypes() + assert.NotNil(t, gpuTypes) + assert.Equal(t, 0, len(gpuTypes)) + + // test grpc error + client3, m3, teardown3 := setupTest(t) + defer teardown3() + c3 := newExternalGrpcCloudProvider(client3, nil) + + m3.On( + "GetAvailableGPUTypes", mock.Anything, mock.Anything, + ).Return( + &protos.GetAvailableGPUTypesResponse{GpuTypes: nil}, + fmt.Errorf("mock error"), + ) + + gpuTypes = c3.GetAvailableGPUTypes() + assert.Nil(t, gpuTypes) + + // test error is not cahced + gpuTypes = c3.GetAvailableGPUTypes() + assert.Nil(t, gpuTypes) + m3.AssertNumberOfCalls(t, "GetAvailableGPUTypes", 2) + +} + +func TestCloudProvider_Cleanup(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + c := newExternalGrpcCloudProvider(client, nil) + + // test correct call + m.On( + "Cleanup", mock.Anything, mock.Anything, + ).Return( + &protos.CleanupResponse{}, + nil, + ).Once() + + err := c.Cleanup() + assert.NoError(t, err) + + // test grpc error + m.On( + "Cleanup", mock.Anything, mock.Anything, + ).Return( + &protos.CleanupResponse{}, + fmt.Errorf("mock error"), + ).Once() + + err = c.Cleanup() + assert.Error(t, err) +} + +func TestCloudProvider_Refresh(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + c := newExternalGrpcCloudProvider(client, nil) + + // test correct call + m.On( + "Refresh", mock.Anything, mock.Anything, + ).Return( + &protos.RefreshResponse{}, + nil, + ).Once() + + err := c.Refresh() + assert.NoError(t, err) + + // test grpc error + m.On( + "Refresh", mock.Anything, mock.Anything, + ).Return( + &protos.RefreshResponse{}, + fmt.Errorf("mock error"), + ).Once() + + err = c.Refresh() + assert.Error(t, err) +} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go new file mode 100644 index 000000000000..e86e6b0b439d --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go @@ -0,0 +1,284 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package externalgrpc + +import ( + "context" + "sync" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/protos" + "k8s.io/autoscaler/cluster-autoscaler/config" + klog "k8s.io/klog/v2" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" +) + +// NodeGroup implements cloudprovider.NodeGroup interface. NodeGroup contains +// configuration info and functions to control a set of nodes that have the +// same capacity and set of labels. +// +type NodeGroup struct { + id string // this must be a stable identifier + minSize int // cached value + maxSize int // cached value + debug string // cached value + client protos.CloudProviderClient + + mutex sync.Mutex + nodeInfo **schedulerframework.NodeInfo // used to cache NodeGroupTemplateNodeInfo() grpc calls +} + +// MaxSize returns maximum size of the node group. +func (n *NodeGroup) MaxSize() int { + return n.maxSize +} + +// MinSize returns minimum size of the node group. +func (n *NodeGroup) MinSize() int { + return n.minSize +} + +// TargetSize returns the current target size of the node group. It is possible +// that the number of nodes in Kubernetes is different at the moment but should +// be equal to Size() once everything stabilizes (new nodes finish startup and +// registration or removed nodes are deleted completely). Implementation +// required. +func (n *NodeGroup) TargetSize() (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call NodeGroupTargetSize for node group %v", n.id) + res, err := n.client.NodeGroupTargetSize(ctx, &protos.NodeGroupTargetSizeRequest{ + Id: n.id, + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroupTargetSize: %v", err) + return 0, err + } + return int(res.TargetSize), nil +} + +// IncreaseSize increases the size of the node group. To delete a node you need +// to explicitly name it and use DeleteNode. This function should wait until +// node group size is updated. Implementation required. +func (n *NodeGroup) IncreaseSize(delta int) error { + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call NodeGroupIncreaseSize for node group %v", n.id) + _, err := n.client.NodeGroupIncreaseSize(ctx, &protos.NodeGroupIncreaseSizeRequest{ + Id: n.id, + Delta: int32(delta), + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroupIncreaseSize: %v", err) + return err + } + return nil +} + +// DeleteNodes deletes nodes from this node group (and also increasing the size +// of the node group with that). Error is returned either on failure or if the +// given node doesn't belong to this node group. This function should wait +// until node group size is updated. Implementation required. +func (n *NodeGroup) DeleteNodes(nodes []*apiv1.Node) error { + pbNodes := make([]*protos.ExternalGrpcNode, 0) + for _, n := range nodes { + pbNodes = append(pbNodes, externalGrpcNode(n)) + } + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call NodeGroupDeleteNodes for node group %v", n.id) + _, err := n.client.NodeGroupDeleteNodes(ctx, &protos.NodeGroupDeleteNodesRequest{ + Id: n.id, + Nodes: pbNodes, + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroupDeleteNodes: %v", err) + return err + } + return nil +} + +// DecreaseTargetSize decreases the target size of the node group. This function +// doesn't permit to delete any existing node and can be used only to reduce the +// request for new nodes that have not been yet fulfilled. Delta should be negative. +// It is assumed that cloud provider will not delete the existing nodes when there +// is an option to just decrease the target. Implementation required. +func (n *NodeGroup) DecreaseTargetSize(delta int) error { + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call NodeGroupDecreaseTargetSize for node group %v", n.id) + _, err := n.client.NodeGroupDecreaseTargetSize(ctx, &protos.NodeGroupDecreaseTargetSizeRequest{ + Id: n.id, + Delta: int32(delta), + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroupDecreaseTargetSize: %v", err) + return err + } + return nil +} + +// Id returns an unique identifier of the node group. +func (n *NodeGroup) Id() string { + return n.id +} + +// Debug returns a string containing all information regarding this node group. +func (n *NodeGroup) Debug() string { + return n.debug +} + +// Nodes returns a list of all nodes that belong to this node group. It is +// required that Instance objects returned by this method have Id field set. +// Other fields are optional. +func (n *NodeGroup) Nodes() ([]cloudprovider.Instance, error) { + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call NodeGroupNodes for node group %v", n.id) + res, err := n.client.NodeGroupNodes(ctx, &protos.NodeGroupNodesRequest{ + Id: n.id, + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroupNodes: %v", err) + return nil, err + } + instances := make([]cloudprovider.Instance, 0) + for _, pbInstance := range res.GetInstances() { + var instance cloudprovider.Instance + instance.Id = pbInstance.GetId() + pbStatus := pbInstance.GetStatus() + if pbStatus.GetInstanceState() != protos.InstanceStatus_unspecified { + instance.Status = new(cloudprovider.InstanceStatus) + instance.Status.State = cloudprovider.InstanceState(pbStatus.GetInstanceState()) + pbErrorInfo := pbStatus.GetErrorInfo() + if pbErrorInfo.GetErrorCode() != "" { + instance.Status.ErrorInfo = &cloudprovider.InstanceErrorInfo{ + ErrorClass: cloudprovider.InstanceErrorClass(pbErrorInfo.GetInstanceErrorClass()), + ErrorCode: pbErrorInfo.GetErrorCode(), + ErrorMessage: pbErrorInfo.GetErrorMessage(), + } + } + } + instances = append(instances, instance) + } + return instances, nil +} + +// TemplateNodeInfo returns a schedulerframework.NodeInfo structure of an empty +// (as if just started) node. This will be used in scale-up simulations to +// predict what would a new node look like if a node group was expanded. The +// returned NodeInfo is expected to have a fully populated Node object, with +// all of the labels, capacity and allocatable information as well as all pods +// that are started on the node by default, using manifest (most likely only +// kube-proxy). Implementation optional. +// +// The definition of a generic `NodeInfo` for each potential provider is a pretty +// complex approach and does not cover all the scenarios. For the sake of simplicity, +// the `nodeInfo` is defined as a Kubernetes `k8s.io.api.core.v1.Node` type +// where the system could still extract certain info about the node. +func (n *NodeGroup) TemplateNodeInfo() (*schedulerframework.NodeInfo, error) { + n.mutex.Lock() + defer n.mutex.Unlock() + + if n.nodeInfo != nil { + klog.V(5).Infof("Returning cached nodeInfo for node group %v", n.id) + return *n.nodeInfo, nil + } + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call NodeGroupTemplateNodeInfo for node group %v", n.id) + res, err := n.client.NodeGroupTemplateNodeInfo(ctx, &protos.NodeGroupTemplateNodeInfoRequest{ + Id: n.id, + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroupTemplateNodeInfo: %v", err) + return nil, err + } + pbNodeInfo := res.GetNodeInfo() + if pbNodeInfo == nil { + n.nodeInfo = new(*schedulerframework.NodeInfo) + return nil, nil + } + nodeInfo := schedulerframework.NewNodeInfo() + nodeInfo.SetNode(pbNodeInfo) + n.nodeInfo = &nodeInfo + return nodeInfo, nil +} + +// Exist checks if the node group really exists on the cloud provider side. +// Allows to tell the theoretical node group from the real one. Implementation +// required. +func (n *NodeGroup) Exist() bool { + return true +} + +// Create creates the node group on the cloud provider side. Implementation +// optional. +func (n *NodeGroup) Create() (cloudprovider.NodeGroup, error) { + return nil, cloudprovider.ErrNotImplemented +} + +// Delete deletes the node group on the cloud provider side. This will be +// executed only for autoprovisioned node groups, once their size drops to 0. +// Implementation optional. +func (n *NodeGroup) Delete() error { + return cloudprovider.ErrNotImplemented +} + +// Autoprovisioned returns true if the node group is autoprovisioned. An +// autoprovisioned group was created by CA and can be deleted when scaled to 0. +func (n *NodeGroup) Autoprovisioned() bool { + return false +} + +// GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular +// NodeGroup. Returning a nil will result in using default options. +func (n *NodeGroup) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*config.NodeGroupAutoscalingOptions, error) { + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() + klog.V(5).Infof("Performing gRPC call NodeGroupGetOptions for node group %v", n.id) + res, err := n.client.NodeGroupGetOptions(ctx, &protos.NodeGroupAutoscalingOptionsRequest{ + Id: n.id, + Defaults: &protos.NodeGroupAutoscalingOptions{ + ScaleDownUtilizationThreshold: defaults.ScaleDownUtilizationThreshold, + ScaleDownGpuUtilizationThreshold: defaults.ScaleDownGpuUtilizationThreshold, + ScaleDownUnneededTime: &metav1.Duration{ + Duration: defaults.ScaleDownUnneededTime, + }, + ScaleDownUnreadyTime: &metav1.Duration{ + Duration: defaults.ScaleDownUnreadyTime, + }, + }, + }) + if err != nil { + klog.V(1).Infof("Error on gRPC call NodeGroupGetOptions: %v", err) + return nil, err + } + pbOpts := res.GetNodeGroupAutoscalingOptions() + if pbOpts == nil { + return nil, nil + } + opts := &config.NodeGroupAutoscalingOptions{ + ScaleDownUtilizationThreshold: pbOpts.GetScaleDownUtilizationThreshold(), + ScaleDownGpuUtilizationThreshold: pbOpts.GetScaleDownGpuUtilizationThreshold(), + ScaleDownUnneededTime: pbOpts.GetScaleDownUnneededTime().Duration, + ScaleDownUnreadyTime: pbOpts.GetScaleDownUnreadyTime().Duration, + } + return opts, nil +} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go new file mode 100644 index 000000000000..51feebc9d803 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go @@ -0,0 +1,463 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package externalgrpc + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/protos" + "k8s.io/autoscaler/cluster-autoscaler/config" +) + +func TestCloudProvider_Nodes(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + + pbInstances := []*protos.Instance{ + {Id: "1", Status: &protos.InstanceStatus{ + InstanceState: protos.InstanceStatus_unspecified, + ErrorInfo: &protos.InstanceErrorInfo{}, + }, + }, + {Id: "2", Status: &protos.InstanceStatus{ + InstanceState: protos.InstanceStatus_instanceRunning, + ErrorInfo: &protos.InstanceErrorInfo{}, + }, + }, + {Id: "3", Status: &protos.InstanceStatus{ + InstanceState: protos.InstanceStatus_instanceRunning, + ErrorInfo: &protos.InstanceErrorInfo{ + ErrorCode: "error1", + ErrorMessage: "mock error", + InstanceErrorClass: 1, + }, + }, + }, + } + + // test correct call + m.On( + "NodeGroupNodes", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupNodesRequest) bool { + return req.Id == "nodeGroup1" + }), + ).Return( + &protos.NodeGroupNodesResponse{ + Instances: pbInstances, + }, nil, + ).Once() + + ng1 := NodeGroup{ + id: "nodeGroup1", + client: client, + } + + instances, err := ng1.Nodes() + assert.NoError(t, err) + assert.Equal(t, 3, len(instances)) + for _, i := range instances { + if i.Id == "1" { + assert.Nil(t, i.Status) + } else if i.Id == "2" { + assert.Equal(t, cloudprovider.InstanceRunning, i.Status.State) + assert.Nil(t, i.Status.ErrorInfo) + } else if i.Id == "3" { + assert.Equal(t, cloudprovider.InstanceRunning, i.Status.State) + assert.Equal(t, cloudprovider.OutOfResourcesErrorClass, i.Status.ErrorInfo.ErrorClass) + assert.Equal(t, "error1", i.Status.ErrorInfo.ErrorCode) + assert.Equal(t, "mock error", i.Status.ErrorInfo.ErrorMessage) + } else { + assert.Fail(t, "instance not recognized") + } + } + + // test grpc error + m.On( + "NodeGroupNodes", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupNodesRequest) bool { + return req.Id == "nodeGroup2" + }), + ).Return( + &protos.NodeGroupNodesResponse{ + Instances: pbInstances, + }, + fmt.Errorf("mock error"), + ).Once() + + ng2 := NodeGroup{ + id: "nodeGroup2", + client: client, + } + + _, err = ng2.Nodes() + assert.Error(t, err) + +} + +func TestCloudProvider_TemplateNodeInfo(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + + // test correct call + apiv1Node1 := &apiv1.Node{} + apiv1Node1.Name = "node1" + + apiv1Node2 := &apiv1.Node{} + apiv1Node2.Name = "node2" + + m.On( + "NodeGroupTemplateNodeInfo", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupTemplateNodeInfoRequest) bool { + return req.Id == "nodeGroup1" + }), + ).Return( + &protos.NodeGroupTemplateNodeInfoResponse{ + NodeInfo: apiv1Node1, + }, nil, + ).Once() + + m.On( + "NodeGroupTemplateNodeInfo", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupTemplateNodeInfoRequest) bool { + return req.Id == "nodeGroup2" + }), + ).Return( + &protos.NodeGroupTemplateNodeInfoResponse{ + NodeInfo: apiv1Node2, + }, nil, + ).Once() + + ng1 := NodeGroup{ + id: "nodeGroup1", + client: client, + } + + ng2 := NodeGroup{ + id: "nodeGroup2", + client: client, + } + + nodeInfo1, err := ng1.TemplateNodeInfo() + assert.NoError(t, err) + assert.Equal(t, apiv1Node1.Name, nodeInfo1.Node().Name) + + nodeInfo2, err := ng2.TemplateNodeInfo() + assert.NoError(t, err) + assert.Equal(t, apiv1Node2.Name, nodeInfo2.Node().Name) + + // test cached answer + nodeInfo1, err = ng1.TemplateNodeInfo() + assert.NoError(t, err) + assert.Equal(t, apiv1Node1.Name, nodeInfo1.Node().Name) + m.AssertNumberOfCalls(t, "NodeGroupTemplateNodeInfo", 2) + + // test nil TemplateNodeInfo + m.On( + "NodeGroupTemplateNodeInfo", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupTemplateNodeInfoRequest) bool { + return req.Id == "nodeGroup3" + }), + ).Return( + &protos.NodeGroupTemplateNodeInfoResponse{ + NodeInfo: nil, + }, nil, + ).Once() + + ng3 := NodeGroup{ + id: "nodeGroup3", + client: client, + } + + nodeInfo3, err := ng3.TemplateNodeInfo() + assert.NoError(t, err) + assert.Nil(t, nodeInfo3) + + // test grpc error + m.On( + "NodeGroupTemplateNodeInfo", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupTemplateNodeInfoRequest) bool { + return req.Id == "nodeGroup4" + }), + ).Return( + &protos.NodeGroupTemplateNodeInfoResponse{ + NodeInfo: nil, + }, + fmt.Errorf("mock error"), + ).Once() + + ng4 := NodeGroup{ + id: "nodeGroup4", + client: client, + } + + _, err = ng4.TemplateNodeInfo() + assert.Error(t, err) + +} + +func TestCloudProvider_GetOptions(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + + // test correct call + m.On( + "NodeGroupGetOptions", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupAutoscalingOptionsRequest) bool { + return req.Id == "nodeGroup1" + }), + ).Return( + &protos.NodeGroupAutoscalingOptionsResponse{ + NodeGroupAutoscalingOptions: &protos.NodeGroupAutoscalingOptions{ + ScaleDownUtilizationThreshold: 0.6, + ScaleDownGpuUtilizationThreshold: 0.7, + ScaleDownUnneededTime: &v1.Duration{Duration: time.Minute}, + ScaleDownUnreadyTime: &v1.Duration{Duration: time.Hour}, + }, + }, + nil, + ) + + ng1 := NodeGroup{ + id: "nodeGroup1", + client: client, + } + defaultsOpts := config.NodeGroupAutoscalingOptions{ + ScaleDownUtilizationThreshold: 0.6, + ScaleDownGpuUtilizationThreshold: 0.7, + ScaleDownUnneededTime: time.Minute, + ScaleDownUnreadyTime: time.Hour, + } + + opts, err := ng1.GetOptions(defaultsOpts) + assert.NoError(t, err) + assert.Equal(t, 0.6, opts.ScaleDownUtilizationThreshold) + assert.Equal(t, 0.7, opts.ScaleDownGpuUtilizationThreshold) + assert.Equal(t, time.Minute, opts.ScaleDownUnneededTime) + assert.Equal(t, time.Hour, opts.ScaleDownUnreadyTime) + + // test grpc error + m.On( + "NodeGroupGetOptions", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupAutoscalingOptionsRequest) bool { + return req.Id == "nodeGroup2" + }), + ).Return( + &protos.NodeGroupAutoscalingOptionsResponse{}, + fmt.Errorf("mock error"), + ) + + ng2 := NodeGroup{ + id: "nodeGroup2", + client: client, + } + + _, err = ng2.GetOptions(defaultsOpts) + assert.Error(t, err) + + // test no opts + m.On( + "NodeGroupGetOptions", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupAutoscalingOptionsRequest) bool { + return req.Id == "nodeGroup3" + }), + ).Return( + &protos.NodeGroupAutoscalingOptionsResponse{}, + nil, + ) + + ng3 := NodeGroup{ + id: "nodeGroup3", + client: client, + } + + opts, err = ng3.GetOptions(defaultsOpts) + assert.NoError(t, err) + assert.Nil(t, opts) +} + +func TestCloudProvider_TargetSize(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + + // test correct call + m.On( + "NodeGroupTargetSize", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupTargetSizeRequest) bool { + return req.Id == "nodeGroup1" + }), + ).Return( + &protos.NodeGroupTargetSizeResponse{ + TargetSize: 1, + }, nil, + ).Once() + + ng1 := NodeGroup{ + id: "nodeGroup1", + client: client, + } + + size, err := ng1.TargetSize() + assert.NoError(t, err) + assert.Equal(t, 1, size) + + // test grpc error + m.On( + "NodeGroupTargetSize", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupTargetSizeRequest) bool { + return req.Id == "nodeGroup2" + }), + ).Return( + &protos.NodeGroupTargetSizeResponse{}, + fmt.Errorf("mock error"), + ).Once() + + ng2 := NodeGroup{ + id: "nodeGroup2", + client: client, + } + + _, err = ng2.TargetSize() + assert.Error(t, err) + +} + +func TestCloudProvider_IncreaseSize(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + + // test correct call + m.On( + "NodeGroupIncreaseSize", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupIncreaseSizeRequest) bool { + return req.Id == "nodeGroup1" + }), + ).Return( + &protos.NodeGroupIncreaseSizeResponse{}, nil, + ).Once() + + ng1 := NodeGroup{ + id: "nodeGroup1", + client: client, + } + + err := ng1.IncreaseSize(1) + assert.NoError(t, err) + + // test grpc error + m.On( + "NodeGroupIncreaseSize", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupIncreaseSizeRequest) bool { + return req.Id == "nodeGroup2" + }), + ).Return( + &protos.NodeGroupIncreaseSizeResponse{}, + fmt.Errorf("mock error"), + ).Once() + + ng2 := NodeGroup{ + id: "nodeGroup2", + client: client, + } + + err = ng2.IncreaseSize(1) + assert.Error(t, err) + +} + +func TestCloudProvider_DecreaseSize(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + + // test correct call + m.On( + "NodeGroupDecreaseTargetSize", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupDecreaseTargetSizeRequest) bool { + return req.Id == "nodeGroup1" + }), + ).Return( + &protos.NodeGroupDecreaseTargetSizeResponse{}, nil, + ).Once() + + ng1 := NodeGroup{ + id: "nodeGroup1", + client: client, + } + + err := ng1.DecreaseTargetSize(1) + assert.NoError(t, err) + + // test grpc error + m.On( + "NodeGroupDecreaseTargetSize", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupDecreaseTargetSizeRequest) bool { + return req.Id == "nodeGroup2" + }), + ).Return( + &protos.NodeGroupDecreaseTargetSizeResponse{}, + fmt.Errorf("mock error"), + ).Once() + + ng2 := NodeGroup{ + id: "nodeGroup2", + client: client, + } + + err = ng2.DecreaseTargetSize(1) + assert.Error(t, err) + +} + +func TestCloudProvider_DeleteNodes(t *testing.T) { + client, m, teardown := setupTest(t) + defer teardown() + + apiv1Node1 := &apiv1.Node{} + apiv1Node1.Name = "node1" + + apiv1Node2 := &apiv1.Node{} + apiv1Node2.Name = "node2" + + nodes := []*apiv1.Node{apiv1Node1, apiv1Node2} + + // test correct call + m.On( + "NodeGroupDeleteNodes", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupDeleteNodesRequest) bool { + return req.Id == "nodeGroup1" + }), + ).Return( + &protos.NodeGroupDeleteNodesResponse{}, nil, + ).Once() + + ng1 := NodeGroup{ + id: "nodeGroup1", + client: client, + } + + err := ng1.DeleteNodes(nodes) + assert.NoError(t, err) + + // test grpc error + m.On( + "NodeGroupDeleteNodes", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupDeleteNodesRequest) bool { + return req.Id == "nodeGroup2" + }), + ).Return( + &protos.NodeGroupDeleteNodesResponse{}, + fmt.Errorf("mock error"), + ).Once() + + ng2 := NodeGroup{ + id: "nodeGroup2", + client: client, + } + + err = ng2.DeleteNodes(nodes) + assert.Error(t, err) + +} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_utils_test.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_utils_test.go new file mode 100644 index 000000000000..7241a39ee83c --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_utils_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package externalgrpc + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/externalgrpc/protos" +) + +type cloudProviderServerMock struct { + protos.UnimplementedCloudProviderServer + + mock.Mock +} + +func (c *cloudProviderServerMock) NodeGroups(ctx context.Context, req *protos.NodeGroupsRequest) (*protos.NodeGroupsResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupsResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) NodeGroupForNode(ctx context.Context, req *protos.NodeGroupForNodeRequest) (*protos.NodeGroupForNodeResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupForNodeResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) PricingNodePrice(ctx context.Context, req *protos.PricingNodePriceRequest) (*protos.PricingNodePriceResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.PricingNodePriceResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) PricingPodPrice(ctx context.Context, req *protos.PricingPodPriceRequest) (*protos.PricingPodPriceResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.PricingPodPriceResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) GPULabel(ctx context.Context, req *protos.GPULabelRequest) (*protos.GPULabelResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.GPULabelResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) GetAvailableGPUTypes(ctx context.Context, req *protos.GetAvailableGPUTypesRequest) (*protos.GetAvailableGPUTypesResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.GetAvailableGPUTypesResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) Cleanup(ctx context.Context, req *protos.CleanupRequest) (*protos.CleanupResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.CleanupResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) Refresh(ctx context.Context, req *protos.RefreshRequest) (*protos.RefreshResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.RefreshResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) NodeGroupTargetSize(ctx context.Context, req *protos.NodeGroupTargetSizeRequest) (*protos.NodeGroupTargetSizeResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupTargetSizeResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) NodeGroupIncreaseSize(ctx context.Context, req *protos.NodeGroupIncreaseSizeRequest) (*protos.NodeGroupIncreaseSizeResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupIncreaseSizeResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) NodeGroupDeleteNodes(ctx context.Context, req *protos.NodeGroupDeleteNodesRequest) (*protos.NodeGroupDeleteNodesResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupDeleteNodesResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) NodeGroupDecreaseTargetSize(ctx context.Context, req *protos.NodeGroupDecreaseTargetSizeRequest) (*protos.NodeGroupDecreaseTargetSizeResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupDecreaseTargetSizeResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) NodeGroupNodes(ctx context.Context, req *protos.NodeGroupNodesRequest) (*protos.NodeGroupNodesResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupNodesResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) NodeGroupTemplateNodeInfo(ctx context.Context, req *protos.NodeGroupTemplateNodeInfoRequest) (*protos.NodeGroupTemplateNodeInfoResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupTemplateNodeInfoResponse), args.Error(1) +} + +func (c *cloudProviderServerMock) NodeGroupGetOptions(ctx context.Context, req *protos.NodeGroupAutoscalingOptionsRequest) (*protos.NodeGroupAutoscalingOptionsResponse, error) { + args := c.Called(ctx, req) + return args.Get(0).(*protos.NodeGroupAutoscalingOptionsResponse), args.Error(1) +} + +func setupTest(t *testing.T) (protos.CloudProviderClient, *cloudProviderServerMock, func()) { + t.Helper() + lis, err := net.Listen("tcp", ":0") + require.NoError(t, err) + + conn, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure()) + require.NoError(t, err) + + server := grpc.NewServer() + m := &cloudProviderServerMock{} + protos.RegisterCloudProviderServer(server, m) + require.NoError(t, err) + + go server.Serve(lis) + + client := protos.NewCloudProviderClient(conn) + return client, m, func() { + server.Stop() + conn.Close() + lis.Close() + } +} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go new file mode 100644 index 000000000000..e815b88cc1ad --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go @@ -0,0 +1,2942 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.19.1 +// source: cloudprovider/externalgrpc/protos/externalgrpc.proto + +package protos + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + _ "google.golang.org/protobuf/types/descriptorpb" + anypb "google.golang.org/protobuf/types/known/anypb" + v11 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type InstanceStatus_InstanceState int32 + +const ( + // an Unspecified instanceState means the actual instance status is undefined (nil). + InstanceStatus_unspecified InstanceStatus_InstanceState = 0 + // InstanceRunning means instance is running. + InstanceStatus_instanceRunning InstanceStatus_InstanceState = 1 + // InstanceCreating means instance is being created. + InstanceStatus_instanceCreating InstanceStatus_InstanceState = 2 + // InstanceDeleting means instance is being deleted. + InstanceStatus_instanceDeleting InstanceStatus_InstanceState = 3 +) + +// Enum value maps for InstanceStatus_InstanceState. +var ( + InstanceStatus_InstanceState_name = map[int32]string{ + 0: "unspecified", + 1: "instanceRunning", + 2: "instanceCreating", + 3: "instanceDeleting", + } + InstanceStatus_InstanceState_value = map[string]int32{ + "unspecified": 0, + "instanceRunning": 1, + "instanceCreating": 2, + "instanceDeleting": 3, + } +) + +func (x InstanceStatus_InstanceState) Enum() *InstanceStatus_InstanceState { + p := new(InstanceStatus_InstanceState) + *p = x + return p +} + +func (x InstanceStatus_InstanceState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (InstanceStatus_InstanceState) Descriptor() protoreflect.EnumDescriptor { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_enumTypes[0].Descriptor() +} + +func (InstanceStatus_InstanceState) Type() protoreflect.EnumType { + return &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_enumTypes[0] +} + +func (x InstanceStatus_InstanceState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use InstanceStatus_InstanceState.Descriptor instead. +func (InstanceStatus_InstanceState) EnumDescriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{29, 0} +} + +type NodeGroup struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ID of the node group on the cloud provider. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // MinSize of the node group on the cloud provider. + MinSize int32 `protobuf:"varint,2,opt,name=minSize,proto3" json:"minSize,omitempty"` + // MaxSize of the node group on the cloud provider. + MaxSize int32 `protobuf:"varint,3,opt,name=maxSize,proto3" json:"maxSize,omitempty"` + // Debug returns a string containing all information regarding this node group. + Debug string `protobuf:"bytes,4,opt,name=debug,proto3" json:"debug,omitempty"` +} + +func (x *NodeGroup) Reset() { + *x = NodeGroup{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroup) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroup) ProtoMessage() {} + +func (x *NodeGroup) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroup.ProtoReflect.Descriptor instead. +func (*NodeGroup) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{0} +} + +func (x *NodeGroup) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *NodeGroup) GetMinSize() int32 { + if x != nil { + return x.MinSize + } + return 0 +} + +func (x *NodeGroup) GetMaxSize() int32 { + if x != nil { + return x.MaxSize + } + return 0 +} + +func (x *NodeGroup) GetDebug() string { + if x != nil { + return x.Debug + } + return "" +} + +type ExternalGrpcNode struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ID of the node assigned by the cloud provider in the format: ://. + ProviderID string `protobuf:"bytes,1,opt,name=providerID,proto3" json:"providerID,omitempty"` + // Name of the node assigned by the cloud provider. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + // labels is a map of {key,value} pairs with the node's labels. + Labels map[string]string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // If specified, the node's annotations. + Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *ExternalGrpcNode) Reset() { + *x = ExternalGrpcNode{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExternalGrpcNode) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExternalGrpcNode) ProtoMessage() {} + +func (x *ExternalGrpcNode) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExternalGrpcNode.ProtoReflect.Descriptor instead. +func (*ExternalGrpcNode) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{1} +} + +func (x *ExternalGrpcNode) GetProviderID() string { + if x != nil { + return x.ProviderID + } + return "" +} + +func (x *ExternalGrpcNode) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ExternalGrpcNode) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + +func (x *ExternalGrpcNode) GetAnnotations() map[string]string { + if x != nil { + return x.Annotations + } + return nil +} + +type NodeGroupsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *NodeGroupsRequest) Reset() { + *x = NodeGroupsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupsRequest) ProtoMessage() {} + +func (x *NodeGroupsRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupsRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupsRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{2} +} + +type NodeGroupsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // All the node groups that the cloud provider service supports. + NodeGroups []*NodeGroup `protobuf:"bytes,1,rep,name=nodeGroups,proto3" json:"nodeGroups,omitempty"` +} + +func (x *NodeGroupsResponse) Reset() { + *x = NodeGroupsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupsResponse) ProtoMessage() {} + +func (x *NodeGroupsResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupsResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupsResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{3} +} + +func (x *NodeGroupsResponse) GetNodeGroups() []*NodeGroup { + if x != nil { + return x.NodeGroups + } + return nil +} + +type NodeGroupForNodeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Node for which the request is performed. + Node *ExternalGrpcNode `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` +} + +func (x *NodeGroupForNodeRequest) Reset() { + *x = NodeGroupForNodeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupForNodeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupForNodeRequest) ProtoMessage() {} + +func (x *NodeGroupForNodeRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupForNodeRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupForNodeRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{4} +} + +func (x *NodeGroupForNodeRequest) GetNode() *ExternalGrpcNode { + if x != nil { + return x.Node + } + return nil +} + +type NodeGroupForNodeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Node group for the given node. nodeGroup with id = "" means no node group. + NodeGroup *NodeGroup `protobuf:"bytes,1,opt,name=nodeGroup,proto3" json:"nodeGroup,omitempty"` +} + +func (x *NodeGroupForNodeResponse) Reset() { + *x = NodeGroupForNodeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupForNodeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupForNodeResponse) ProtoMessage() {} + +func (x *NodeGroupForNodeResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupForNodeResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupForNodeResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{5} +} + +func (x *NodeGroupForNodeResponse) GetNodeGroup() *NodeGroup { + if x != nil { + return x.NodeGroup + } + return nil +} + +type PricingNodePriceRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Node for which the request is performed. + Node *ExternalGrpcNode `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` + // Start time for the request period. + StartTime *v1.Time `protobuf:"bytes,2,opt,name=startTime,proto3" json:"startTime,omitempty"` + // End time for the request period. + EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` +} + +func (x *PricingNodePriceRequest) Reset() { + *x = PricingNodePriceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PricingNodePriceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PricingNodePriceRequest) ProtoMessage() {} + +func (x *PricingNodePriceRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PricingNodePriceRequest.ProtoReflect.Descriptor instead. +func (*PricingNodePriceRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{6} +} + +func (x *PricingNodePriceRequest) GetNode() *ExternalGrpcNode { + if x != nil { + return x.Node + } + return nil +} + +func (x *PricingNodePriceRequest) GetStartTime() *v1.Time { + if x != nil { + return x.StartTime + } + return nil +} + +func (x *PricingNodePriceRequest) GetEndTime() *v1.Time { + if x != nil { + return x.EndTime + } + return nil +} + +type PricingNodePriceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Theoretical minimum price of running a node for a given period. + Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` +} + +func (x *PricingNodePriceResponse) Reset() { + *x = PricingNodePriceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PricingNodePriceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PricingNodePriceResponse) ProtoMessage() {} + +func (x *PricingNodePriceResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PricingNodePriceResponse.ProtoReflect.Descriptor instead. +func (*PricingNodePriceResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{7} +} + +func (x *PricingNodePriceResponse) GetPrice() float64 { + if x != nil { + return x.Price + } + return 0 +} + +type PricingPodPriceRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Pod for which the request is performed. + Pod *v11.Pod `protobuf:"bytes,1,opt,name=pod,proto3" json:"pod,omitempty"` + // Start time for the request period. + StartTime *v1.Time `protobuf:"bytes,2,opt,name=startTime,proto3" json:"startTime,omitempty"` + // End time for the request period. + EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` +} + +func (x *PricingPodPriceRequest) Reset() { + *x = PricingPodPriceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PricingPodPriceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PricingPodPriceRequest) ProtoMessage() {} + +func (x *PricingPodPriceRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PricingPodPriceRequest.ProtoReflect.Descriptor instead. +func (*PricingPodPriceRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{8} +} + +func (x *PricingPodPriceRequest) GetPod() *v11.Pod { + if x != nil { + return x.Pod + } + return nil +} + +func (x *PricingPodPriceRequest) GetStartTime() *v1.Time { + if x != nil { + return x.StartTime + } + return nil +} + +func (x *PricingPodPriceRequest) GetEndTime() *v1.Time { + if x != nil { + return x.EndTime + } + return nil +} + +type PricingPodPriceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Theoretical minimum price of running a pod for a given period. + Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` +} + +func (x *PricingPodPriceResponse) Reset() { + *x = PricingPodPriceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PricingPodPriceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PricingPodPriceResponse) ProtoMessage() {} + +func (x *PricingPodPriceResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PricingPodPriceResponse.ProtoReflect.Descriptor instead. +func (*PricingPodPriceResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{9} +} + +func (x *PricingPodPriceResponse) GetPrice() float64 { + if x != nil { + return x.Price + } + return 0 +} + +type GPULabelRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GPULabelRequest) Reset() { + *x = GPULabelRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GPULabelRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GPULabelRequest) ProtoMessage() {} + +func (x *GPULabelRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GPULabelRequest.ProtoReflect.Descriptor instead. +func (*GPULabelRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{10} +} + +type GPULabelResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Label added to nodes with a GPU resource. + Label string `protobuf:"bytes,1,opt,name=label,proto3" json:"label,omitempty"` +} + +func (x *GPULabelResponse) Reset() { + *x = GPULabelResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GPULabelResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GPULabelResponse) ProtoMessage() {} + +func (x *GPULabelResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GPULabelResponse.ProtoReflect.Descriptor instead. +func (*GPULabelResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{11} +} + +func (x *GPULabelResponse) GetLabel() string { + if x != nil { + return x.Label + } + return "" +} + +type GetAvailableGPUTypesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetAvailableGPUTypesRequest) Reset() { + *x = GetAvailableGPUTypesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetAvailableGPUTypesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetAvailableGPUTypesRequest) ProtoMessage() {} + +func (x *GetAvailableGPUTypesRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetAvailableGPUTypesRequest.ProtoReflect.Descriptor instead. +func (*GetAvailableGPUTypesRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{12} +} + +type GetAvailableGPUTypesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // GPU types passed in as opaque key-value pairs. + GpuTypes map[string]*anypb.Any `protobuf:"bytes,1,rep,name=gpuTypes,proto3" json:"gpuTypes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *GetAvailableGPUTypesResponse) Reset() { + *x = GetAvailableGPUTypesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetAvailableGPUTypesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetAvailableGPUTypesResponse) ProtoMessage() {} + +func (x *GetAvailableGPUTypesResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetAvailableGPUTypesResponse.ProtoReflect.Descriptor instead. +func (*GetAvailableGPUTypesResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{13} +} + +func (x *GetAvailableGPUTypesResponse) GetGpuTypes() map[string]*anypb.Any { + if x != nil { + return x.GpuTypes + } + return nil +} + +type CleanupRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *CleanupRequest) Reset() { + *x = CleanupRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CleanupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CleanupRequest) ProtoMessage() {} + +func (x *CleanupRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CleanupRequest.ProtoReflect.Descriptor instead. +func (*CleanupRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{14} +} + +type CleanupResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *CleanupResponse) Reset() { + *x = CleanupResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CleanupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CleanupResponse) ProtoMessage() {} + +func (x *CleanupResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CleanupResponse.ProtoReflect.Descriptor instead. +func (*CleanupResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{15} +} + +type RefreshRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RefreshRequest) Reset() { + *x = RefreshRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RefreshRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RefreshRequest) ProtoMessage() {} + +func (x *RefreshRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RefreshRequest.ProtoReflect.Descriptor instead. +func (*RefreshRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{16} +} + +type RefreshResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RefreshResponse) Reset() { + *x = RefreshResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RefreshResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RefreshResponse) ProtoMessage() {} + +func (x *RefreshResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RefreshResponse.ProtoReflect.Descriptor instead. +func (*RefreshResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{17} +} + +type NodeGroupTargetSizeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ID of the node group for the request. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *NodeGroupTargetSizeRequest) Reset() { + *x = NodeGroupTargetSizeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupTargetSizeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupTargetSizeRequest) ProtoMessage() {} + +func (x *NodeGroupTargetSizeRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupTargetSizeRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupTargetSizeRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{18} +} + +func (x *NodeGroupTargetSizeRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type NodeGroupTargetSizeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Current target size of the node group. + TargetSize int32 `protobuf:"varint,1,opt,name=targetSize,proto3" json:"targetSize,omitempty"` +} + +func (x *NodeGroupTargetSizeResponse) Reset() { + *x = NodeGroupTargetSizeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupTargetSizeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupTargetSizeResponse) ProtoMessage() {} + +func (x *NodeGroupTargetSizeResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupTargetSizeResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupTargetSizeResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{19} +} + +func (x *NodeGroupTargetSizeResponse) GetTargetSize() int32 { + if x != nil { + return x.TargetSize + } + return 0 +} + +type NodeGroupIncreaseSizeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Number of nodes to add. + Delta int32 `protobuf:"varint,1,opt,name=delta,proto3" json:"delta,omitempty"` + // ID of the node group for the request. + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *NodeGroupIncreaseSizeRequest) Reset() { + *x = NodeGroupIncreaseSizeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupIncreaseSizeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupIncreaseSizeRequest) ProtoMessage() {} + +func (x *NodeGroupIncreaseSizeRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupIncreaseSizeRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupIncreaseSizeRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{20} +} + +func (x *NodeGroupIncreaseSizeRequest) GetDelta() int32 { + if x != nil { + return x.Delta + } + return 0 +} + +func (x *NodeGroupIncreaseSizeRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type NodeGroupIncreaseSizeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *NodeGroupIncreaseSizeResponse) Reset() { + *x = NodeGroupIncreaseSizeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupIncreaseSizeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupIncreaseSizeResponse) ProtoMessage() {} + +func (x *NodeGroupIncreaseSizeResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupIncreaseSizeResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupIncreaseSizeResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{21} +} + +type NodeGroupDeleteNodesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // List of nodes to delete. + Nodes []*ExternalGrpcNode `protobuf:"bytes,1,rep,name=nodes,proto3" json:"nodes,omitempty"` + // ID of the node group for the request. + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *NodeGroupDeleteNodesRequest) Reset() { + *x = NodeGroupDeleteNodesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupDeleteNodesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupDeleteNodesRequest) ProtoMessage() {} + +func (x *NodeGroupDeleteNodesRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupDeleteNodesRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupDeleteNodesRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{22} +} + +func (x *NodeGroupDeleteNodesRequest) GetNodes() []*ExternalGrpcNode { + if x != nil { + return x.Nodes + } + return nil +} + +func (x *NodeGroupDeleteNodesRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type NodeGroupDeleteNodesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *NodeGroupDeleteNodesResponse) Reset() { + *x = NodeGroupDeleteNodesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupDeleteNodesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupDeleteNodesResponse) ProtoMessage() {} + +func (x *NodeGroupDeleteNodesResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupDeleteNodesResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupDeleteNodesResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{23} +} + +type NodeGroupDecreaseTargetSizeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Number of nodes to delete. + Delta int32 `protobuf:"varint,1,opt,name=delta,proto3" json:"delta,omitempty"` + // ID of the node group for the request. + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *NodeGroupDecreaseTargetSizeRequest) Reset() { + *x = NodeGroupDecreaseTargetSizeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupDecreaseTargetSizeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupDecreaseTargetSizeRequest) ProtoMessage() {} + +func (x *NodeGroupDecreaseTargetSizeRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupDecreaseTargetSizeRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupDecreaseTargetSizeRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{24} +} + +func (x *NodeGroupDecreaseTargetSizeRequest) GetDelta() int32 { + if x != nil { + return x.Delta + } + return 0 +} + +func (x *NodeGroupDecreaseTargetSizeRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type NodeGroupDecreaseTargetSizeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *NodeGroupDecreaseTargetSizeResponse) Reset() { + *x = NodeGroupDecreaseTargetSizeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupDecreaseTargetSizeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupDecreaseTargetSizeResponse) ProtoMessage() {} + +func (x *NodeGroupDecreaseTargetSizeResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupDecreaseTargetSizeResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupDecreaseTargetSizeResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{25} +} + +type NodeGroupNodesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ID of the node group for the request. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *NodeGroupNodesRequest) Reset() { + *x = NodeGroupNodesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupNodesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupNodesRequest) ProtoMessage() {} + +func (x *NodeGroupNodesRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupNodesRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupNodesRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{26} +} + +func (x *NodeGroupNodesRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type NodeGroupNodesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // list of cloud provider instances in a node group. + Instances []*Instance `protobuf:"bytes,1,rep,name=instances,proto3" json:"instances,omitempty"` +} + +func (x *NodeGroupNodesResponse) Reset() { + *x = NodeGroupNodesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupNodesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupNodesResponse) ProtoMessage() {} + +func (x *NodeGroupNodesResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupNodesResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupNodesResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{27} +} + +func (x *NodeGroupNodesResponse) GetInstances() []*Instance { + if x != nil { + return x.Instances + } + return nil +} + +type Instance struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Id of the instance. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // Status of the node. + Status *InstanceStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` +} + +func (x *Instance) Reset() { + *x = Instance{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Instance) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Instance) ProtoMessage() {} + +func (x *Instance) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Instance.ProtoReflect.Descriptor instead. +func (*Instance) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{28} +} + +func (x *Instance) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Instance) GetStatus() *InstanceStatus { + if x != nil { + return x.Status + } + return nil +} + +// InstanceStatus represents the instance status. +type InstanceStatus struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // InstanceState tells if the instance is running, being created or being deleted. + InstanceState InstanceStatus_InstanceState `protobuf:"varint,1,opt,name=instanceState,proto3,enum=clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus_InstanceState" json:"instanceState,omitempty"` + // ErrorInfo provides information about the error status. + // If there is no error condition related to instance, then errorInfo.errorCode should be an empty string. + ErrorInfo *InstanceErrorInfo `protobuf:"bytes,2,opt,name=errorInfo,proto3" json:"errorInfo,omitempty"` +} + +func (x *InstanceStatus) Reset() { + *x = InstanceStatus{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *InstanceStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InstanceStatus) ProtoMessage() {} + +func (x *InstanceStatus) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InstanceStatus.ProtoReflect.Descriptor instead. +func (*InstanceStatus) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{29} +} + +func (x *InstanceStatus) GetInstanceState() InstanceStatus_InstanceState { + if x != nil { + return x.InstanceState + } + return InstanceStatus_unspecified +} + +func (x *InstanceStatus) GetErrorInfo() *InstanceErrorInfo { + if x != nil { + return x.ErrorInfo + } + return nil +} + +// InstanceErrorInfo provides information about error condition on instance. +type InstanceErrorInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ErrorCode is cloud-provider specific error code for error condition. + // An empty string for errorCode means there is no errorInfo for the instance (nil). + ErrorCode string `protobuf:"bytes,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"` + // ErrorMessage is the human readable description of error condition. + ErrorMessage string `protobuf:"bytes,2,opt,name=errorMessage,proto3" json:"errorMessage,omitempty"` + // InstanceErrorClass defines the class of error condition. + InstanceErrorClass int32 `protobuf:"varint,3,opt,name=instanceErrorClass,proto3" json:"instanceErrorClass,omitempty"` +} + +func (x *InstanceErrorInfo) Reset() { + *x = InstanceErrorInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *InstanceErrorInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InstanceErrorInfo) ProtoMessage() {} + +func (x *InstanceErrorInfo) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InstanceErrorInfo.ProtoReflect.Descriptor instead. +func (*InstanceErrorInfo) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{30} +} + +func (x *InstanceErrorInfo) GetErrorCode() string { + if x != nil { + return x.ErrorCode + } + return "" +} + +func (x *InstanceErrorInfo) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +func (x *InstanceErrorInfo) GetInstanceErrorClass() int32 { + if x != nil { + return x.InstanceErrorClass + } + return 0 +} + +type NodeGroupTemplateNodeInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ID of the node group for the request. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *NodeGroupTemplateNodeInfoRequest) Reset() { + *x = NodeGroupTemplateNodeInfoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupTemplateNodeInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupTemplateNodeInfoRequest) ProtoMessage() {} + +func (x *NodeGroupTemplateNodeInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupTemplateNodeInfoRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupTemplateNodeInfoRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{31} +} + +func (x *NodeGroupTemplateNodeInfoRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type NodeGroupTemplateNodeInfoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // nodeInfo is the extracted data from the cloud provider, as a primitive Kubernetes Node type. + NodeInfo *v11.Node `protobuf:"bytes,1,opt,name=nodeInfo,proto3" json:"nodeInfo,omitempty"` +} + +func (x *NodeGroupTemplateNodeInfoResponse) Reset() { + *x = NodeGroupTemplateNodeInfoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupTemplateNodeInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupTemplateNodeInfoResponse) ProtoMessage() {} + +func (x *NodeGroupTemplateNodeInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupTemplateNodeInfoResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupTemplateNodeInfoResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{32} +} + +func (x *NodeGroupTemplateNodeInfoResponse) GetNodeInfo() *v11.Node { + if x != nil { + return x.NodeInfo + } + return nil +} + +type NodeGroupAutoscalingOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ScaleDownUtilizationThreshold sets threshold for nodes to be considered for scale down + // if cpu or memory utilization is over threshold. + ScaleDownUtilizationThreshold float64 `protobuf:"fixed64,1,opt,name=scaleDownUtilizationThreshold,proto3" json:"scaleDownUtilizationThreshold,omitempty"` + // ScaleDownGpuUtilizationThreshold sets threshold for gpu nodes to be + // considered for scale down if gpu utilization is over threshold. + ScaleDownGpuUtilizationThreshold float64 `protobuf:"fixed64,2,opt,name=scaleDownGpuUtilizationThreshold,proto3" json:"scaleDownGpuUtilizationThreshold,omitempty"` + // ScaleDownUnneededTime sets the duration CA expects a node to be + // unneeded/eligible for removal before scaling down the node. + ScaleDownUnneededTime *v1.Duration `protobuf:"bytes,3,opt,name=scaleDownUnneededTime,proto3" json:"scaleDownUnneededTime,omitempty"` + // ScaleDownUnreadyTime represents how long an unready node should be + // unneeded before it is eligible for scale down. + ScaleDownUnreadyTime *v1.Duration `protobuf:"bytes,4,opt,name=scaleDownUnreadyTime,proto3" json:"scaleDownUnreadyTime,omitempty"` +} + +func (x *NodeGroupAutoscalingOptions) Reset() { + *x = NodeGroupAutoscalingOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupAutoscalingOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupAutoscalingOptions) ProtoMessage() {} + +func (x *NodeGroupAutoscalingOptions) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupAutoscalingOptions.ProtoReflect.Descriptor instead. +func (*NodeGroupAutoscalingOptions) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{33} +} + +func (x *NodeGroupAutoscalingOptions) GetScaleDownUtilizationThreshold() float64 { + if x != nil { + return x.ScaleDownUtilizationThreshold + } + return 0 +} + +func (x *NodeGroupAutoscalingOptions) GetScaleDownGpuUtilizationThreshold() float64 { + if x != nil { + return x.ScaleDownGpuUtilizationThreshold + } + return 0 +} + +func (x *NodeGroupAutoscalingOptions) GetScaleDownUnneededTime() *v1.Duration { + if x != nil { + return x.ScaleDownUnneededTime + } + return nil +} + +func (x *NodeGroupAutoscalingOptions) GetScaleDownUnreadyTime() *v1.Duration { + if x != nil { + return x.ScaleDownUnreadyTime + } + return nil +} + +type NodeGroupAutoscalingOptionsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ID of the node group for the request. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // default node group autoscaling options. + Defaults *NodeGroupAutoscalingOptions `protobuf:"bytes,2,opt,name=defaults,proto3" json:"defaults,omitempty"` +} + +func (x *NodeGroupAutoscalingOptionsRequest) Reset() { + *x = NodeGroupAutoscalingOptionsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupAutoscalingOptionsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupAutoscalingOptionsRequest) ProtoMessage() {} + +func (x *NodeGroupAutoscalingOptionsRequest) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupAutoscalingOptionsRequest.ProtoReflect.Descriptor instead. +func (*NodeGroupAutoscalingOptionsRequest) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{34} +} + +func (x *NodeGroupAutoscalingOptionsRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *NodeGroupAutoscalingOptionsRequest) GetDefaults() *NodeGroupAutoscalingOptions { + if x != nil { + return x.Defaults + } + return nil +} + +type NodeGroupAutoscalingOptionsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // autoscaling options for the requested node. + NodeGroupAutoscalingOptions *NodeGroupAutoscalingOptions `protobuf:"bytes,1,opt,name=nodeGroupAutoscalingOptions,proto3" json:"nodeGroupAutoscalingOptions,omitempty"` +} + +func (x *NodeGroupAutoscalingOptionsResponse) Reset() { + *x = NodeGroupAutoscalingOptionsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeGroupAutoscalingOptionsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeGroupAutoscalingOptionsResponse) ProtoMessage() {} + +func (x *NodeGroupAutoscalingOptionsResponse) ProtoReflect() protoreflect.Message { + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeGroupAutoscalingOptionsResponse.ProtoReflect.Descriptor instead. +func (*NodeGroupAutoscalingOptionsResponse) Descriptor() ([]byte, []int) { + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP(), []int{35} +} + +func (x *NodeGroupAutoscalingOptionsResponse) GetNodeGroupAutoscalingOptions() *NodeGroupAutoscalingOptions { + if x != nil { + return x.NodeGroupAutoscalingOptions + } + return nil +} + +var File_cloudprovider_externalgrpc_protos_externalgrpc_proto protoreflect.FileDescriptor + +var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc = []byte{ + 0x0a, 0x34, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, + 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x73, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x2f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, + 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x34, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, + 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, + 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x22, 0x6b, 0x38, 0x73, 0x2e, + 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x67, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x65, + 0x0a, 0x09, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x6d, + 0x69, 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6d, 0x69, + 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x64, 0x65, 0x62, 0x75, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x64, 0x65, 0x62, 0x75, 0x67, 0x22, 0x9e, 0x03, 0x0a, 0x10, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x65, + 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4d, + 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, + 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, + 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x74, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x52, 0x2e, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, + 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, + 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x6e, + 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, + 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, + 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x13, 0x0a, 0x11, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, + 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x70, 0x0a, 0x12, 0x4e, + 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x5a, 0x0a, 0x0a, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, + 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x52, 0x0a, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22, 0x70, 0x0a, + 0x17, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x55, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x22, + 0x74, 0x0a, 0x18, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, + 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x6e, + 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, + 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, + 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x09, 0x6e, 0x6f, 0x64, 0x65, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x80, 0x02, 0x0a, 0x17, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, + 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x55, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, + 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, + 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x48, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, + 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, + 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, + 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, + 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, + 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, + 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x30, 0x0a, 0x18, 0x50, 0x72, 0x69, 0x63, + 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x72, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x05, 0x70, 0x72, 0x69, 0x63, 0x65, 0x22, 0xd3, 0x01, 0x0a, 0x16, 0x50, + 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x03, 0x70, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x52, 0x03, 0x70, 0x6f, 0x64, + 0x12, 0x48, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, + 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, + 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, + 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x65, 0x6e, + 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, + 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, + 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, + 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, + 0x22, 0x2f, 0x0a, 0x17, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, + 0x72, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x70, 0x72, 0x69, 0x63, + 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x22, 0x28, 0x0a, 0x10, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x22, 0x1d, + 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, + 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xea, 0x01, + 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, + 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x77, + 0x0a, 0x08, 0x67, 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x5b, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, + 0x50, 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x47, 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x67, + 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x1a, 0x51, 0x0a, 0x0d, 0x47, 0x70, 0x75, 0x54, 0x79, + 0x70, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x10, 0x0a, 0x0e, 0x43, 0x6c, + 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x11, 0x0a, 0x0f, + 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x10, 0x0a, 0x0e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x11, 0x0a, 0x0f, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2c, 0x0a, 0x1a, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x69, 0x64, 0x22, 0x3d, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, + 0x65, 0x22, 0x44, 0x0a, 0x1c, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, + 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1f, 0x0a, 0x1d, 0x4e, 0x6f, 0x64, 0x65, 0x47, + 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x86, 0x01, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, + 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x57, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, + 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, + 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x22, 0x1e, 0x0a, 0x1c, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x4a, 0x0a, 0x22, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, + 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x25, 0x0a, + 0x23, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, + 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x27, 0x0a, 0x15, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x71, 0x0a, + 0x16, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, + 0x6e, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, + 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, + 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x6e, 0x73, + 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, + 0x22, 0x73, 0x0a, 0x08, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x57, 0x0a, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, + 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, + 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x49, + 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0xca, 0x02, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, + 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x73, 0x0a, 0x0d, 0x69, 0x6e, 0x73, 0x74, + 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x4d, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, + 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0d, + 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x60, 0x0a, + 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x42, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x22, + 0x61, 0x0a, 0x0d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x0f, 0x0a, 0x0b, 0x75, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x65, 0x64, 0x10, + 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x75, 0x6e, + 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, + 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, + 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6e, 0x67, + 0x10, 0x03, 0x22, 0x85, 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x69, 0x6e, + 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6c, 0x61, 0x73, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x22, 0x32, 0x0a, 0x20, 0x4e, 0x6f, + 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, + 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x59, + 0x0a, 0x21, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, + 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xf9, 0x02, 0x0a, 0x1b, 0x4e, 0x6f, + 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, + 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x44, 0x0a, 0x1d, 0x73, 0x63, 0x61, + 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x1d, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x74, 0x69, 0x6c, 0x69, + 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, + 0x4a, 0x0a, 0x20, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x47, 0x70, 0x75, 0x55, + 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, + 0x6f, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x20, 0x73, 0x63, 0x61, 0x6c, 0x65, + 0x44, 0x6f, 0x77, 0x6e, 0x47, 0x70, 0x75, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x64, 0x0a, 0x15, 0x73, + 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x6e, 0x65, 0x65, 0x64, 0x65, 0x64, + 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6b, 0x38, 0x73, + 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, + 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, + 0x31, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x15, 0x73, 0x63, 0x61, 0x6c, + 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x6e, 0x65, 0x65, 0x64, 0x65, 0x64, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x62, 0x0a, 0x14, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, + 0x72, 0x65, 0x61, 0x64, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2e, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, + 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, + 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x14, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x72, 0x65, 0x61, 0x64, + 0x79, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x9e, 0x01, 0x0a, 0x22, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, + 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x68, 0x0a, 0x08, + 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4c, + 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, + 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, + 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x08, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x23, 0x4e, 0x6f, 0x64, 0x65, 0x47, + 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8e, + 0x01, 0x0a, 0x1b, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, + 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, + 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x1b, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, + 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x32, + 0xbf, 0x14, 0x0a, 0x0d, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x12, 0x97, 0x01, 0x0a, 0x0a, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, + 0x12, 0x42, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x43, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, + 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xa9, 0x01, 0x0a, 0x10, + 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, 0x65, + 0x12, 0x48, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, + 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x49, 0x2e, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, + 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, + 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, + 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xa9, 0x01, 0x0a, 0x10, 0x50, 0x72, 0x69, 0x63, + 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x12, 0x48, 0x2e, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, + 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, + 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, + 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x49, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, + 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0xa6, 0x01, 0x0a, 0x0f, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, + 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x12, 0x47, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, + 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, + 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x48, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x91, 0x01, 0x0a, + 0x08, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x40, 0x2e, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, + 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x50, 0x55, 0x4c, + 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x41, 0x2e, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, + 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x50, + 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0xb5, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, + 0x65, 0x47, 0x50, 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, + 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x41, + 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4d, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, + 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, + 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8e, 0x01, 0x0a, 0x07, 0x43, 0x6c, 0x65, + 0x61, 0x6e, 0x75, 0x70, 0x12, 0x3f, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, + 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x40, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, + 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8e, 0x01, 0x0a, 0x07, 0x52, 0x65, + 0x66, 0x72, 0x65, 0x73, 0x68, 0x12, 0x3f, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, + 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x40, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xb2, 0x01, 0x0a, 0x13, 0x4e, + 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, + 0x7a, 0x65, 0x12, 0x4b, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, + 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, + 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, + 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0xb8, 0x01, 0x0a, 0x15, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, + 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x4d, 0x2e, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, + 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4e, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, + 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xb5, 0x01, 0x0a, 0x14, 0x4e, + 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, + 0x64, 0x65, 0x73, 0x12, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, + 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x4d, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, + 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0xca, 0x01, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, + 0x7a, 0x65, 0x12, 0x53, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, + 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, + 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, + 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x54, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, + 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, + 0x6f, 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0xa3, 0x01, 0x0a, 0x0e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x6f, 0x64, + 0x65, 0x73, 0x12, 0x46, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, + 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, + 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x6f, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x47, 0x2e, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, + 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, + 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, + 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xc4, 0x01, 0x0a, 0x19, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, + 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x51, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, + 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x52, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, + 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xc2, 0x01, 0x0a, + 0x13, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, + 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x54, 0x2e, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, + 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x36, 0x5a, 0x34, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x75, 0x74, + 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, + 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescOnce sync.Once + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData = file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc +) + +func file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP() []byte { + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescOnce.Do(func() { + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData = protoimpl.X.CompressGZIP(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData) + }) + return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData +} + +var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes = []interface{}{ + (InstanceStatus_InstanceState)(0), // 0: clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus.InstanceState + (*NodeGroup)(nil), // 1: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroup + (*ExternalGrpcNode)(nil), // 2: clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode + (*NodeGroupsRequest)(nil), // 3: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsRequest + (*NodeGroupsResponse)(nil), // 4: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsResponse + (*NodeGroupForNodeRequest)(nil), // 5: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeRequest + (*NodeGroupForNodeResponse)(nil), // 6: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeResponse + (*PricingNodePriceRequest)(nil), // 7: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceRequest + (*PricingNodePriceResponse)(nil), // 8: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceResponse + (*PricingPodPriceRequest)(nil), // 9: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceRequest + (*PricingPodPriceResponse)(nil), // 10: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceResponse + (*GPULabelRequest)(nil), // 11: clusterautoscaler.cloudprovider.v1.externalgrpc.GPULabelRequest + (*GPULabelResponse)(nil), // 12: clusterautoscaler.cloudprovider.v1.externalgrpc.GPULabelResponse + (*GetAvailableGPUTypesRequest)(nil), // 13: clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesRequest + (*GetAvailableGPUTypesResponse)(nil), // 14: clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse + (*CleanupRequest)(nil), // 15: clusterautoscaler.cloudprovider.v1.externalgrpc.CleanupRequest + (*CleanupResponse)(nil), // 16: clusterautoscaler.cloudprovider.v1.externalgrpc.CleanupResponse + (*RefreshRequest)(nil), // 17: clusterautoscaler.cloudprovider.v1.externalgrpc.RefreshRequest + (*RefreshResponse)(nil), // 18: clusterautoscaler.cloudprovider.v1.externalgrpc.RefreshResponse + (*NodeGroupTargetSizeRequest)(nil), // 19: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTargetSizeRequest + (*NodeGroupTargetSizeResponse)(nil), // 20: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTargetSizeResponse + (*NodeGroupIncreaseSizeRequest)(nil), // 21: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupIncreaseSizeRequest + (*NodeGroupIncreaseSizeResponse)(nil), // 22: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupIncreaseSizeResponse + (*NodeGroupDeleteNodesRequest)(nil), // 23: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesRequest + (*NodeGroupDeleteNodesResponse)(nil), // 24: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesResponse + (*NodeGroupDecreaseTargetSizeRequest)(nil), // 25: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDecreaseTargetSizeRequest + (*NodeGroupDecreaseTargetSizeResponse)(nil), // 26: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDecreaseTargetSizeResponse + (*NodeGroupNodesRequest)(nil), // 27: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesRequest + (*NodeGroupNodesResponse)(nil), // 28: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesResponse + (*Instance)(nil), // 29: clusterautoscaler.cloudprovider.v1.externalgrpc.Instance + (*InstanceStatus)(nil), // 30: clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus + (*InstanceErrorInfo)(nil), // 31: clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceErrorInfo + (*NodeGroupTemplateNodeInfoRequest)(nil), // 32: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoRequest + (*NodeGroupTemplateNodeInfoResponse)(nil), // 33: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoResponse + (*NodeGroupAutoscalingOptions)(nil), // 34: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptions + (*NodeGroupAutoscalingOptionsRequest)(nil), // 35: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsRequest + (*NodeGroupAutoscalingOptionsResponse)(nil), // 36: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsResponse + nil, // 37: clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.LabelsEntry + nil, // 38: clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.AnnotationsEntry + nil, // 39: clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse.GpuTypesEntry + (*v1.Time)(nil), // 40: k8s.io.apimachinery.pkg.apis.meta.v1.Time + (*v11.Pod)(nil), // 41: k8s.io.api.core.v1.Pod + (*v11.Node)(nil), // 42: k8s.io.api.core.v1.Node + (*v1.Duration)(nil), // 43: k8s.io.apimachinery.pkg.apis.meta.v1.Duration + (*anypb.Any)(nil), // 44: google.protobuf.Any +} +var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_depIdxs = []int32{ + 37, // 0: clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.labels:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.LabelsEntry + 38, // 1: clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.annotations:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.AnnotationsEntry + 1, // 2: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsResponse.nodeGroups:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroup + 2, // 3: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeRequest.node:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode + 1, // 4: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeResponse.nodeGroup:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroup + 2, // 5: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceRequest.node:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode + 40, // 6: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceRequest.startTime:type_name -> k8s.io.apimachinery.pkg.apis.meta.v1.Time + 40, // 7: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceRequest.endTime:type_name -> k8s.io.apimachinery.pkg.apis.meta.v1.Time + 41, // 8: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceRequest.pod:type_name -> k8s.io.api.core.v1.Pod + 40, // 9: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceRequest.startTime:type_name -> k8s.io.apimachinery.pkg.apis.meta.v1.Time + 40, // 10: clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceRequest.endTime:type_name -> k8s.io.apimachinery.pkg.apis.meta.v1.Time + 39, // 11: clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse.gpuTypes:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse.GpuTypesEntry + 2, // 12: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesRequest.nodes:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode + 29, // 13: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesResponse.instances:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.Instance + 30, // 14: clusterautoscaler.cloudprovider.v1.externalgrpc.Instance.status:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus + 0, // 15: clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus.instanceState:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus.InstanceState + 31, // 16: clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus.errorInfo:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceErrorInfo + 42, // 17: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoResponse.nodeInfo:type_name -> k8s.io.api.core.v1.Node + 43, // 18: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptions.scaleDownUnneededTime:type_name -> k8s.io.apimachinery.pkg.apis.meta.v1.Duration + 43, // 19: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptions.scaleDownUnreadyTime:type_name -> k8s.io.apimachinery.pkg.apis.meta.v1.Duration + 34, // 20: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsRequest.defaults:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptions + 34, // 21: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsResponse.nodeGroupAutoscalingOptions:type_name -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptions + 44, // 22: clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse.GpuTypesEntry.value:type_name -> google.protobuf.Any + 3, // 23: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroups:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsRequest + 5, // 24: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupForNode:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeRequest + 7, // 25: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.PricingNodePrice:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceRequest + 9, // 26: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.PricingPodPrice:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceRequest + 11, // 27: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.GPULabel:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.GPULabelRequest + 13, // 28: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.GetAvailableGPUTypes:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesRequest + 15, // 29: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.Cleanup:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.CleanupRequest + 17, // 30: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.Refresh:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.RefreshRequest + 19, // 31: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupTargetSize:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTargetSizeRequest + 21, // 32: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupIncreaseSize:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupIncreaseSizeRequest + 23, // 33: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupDeleteNodes:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesRequest + 25, // 34: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupDecreaseTargetSize:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDecreaseTargetSizeRequest + 27, // 35: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupNodes:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesRequest + 32, // 36: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupTemplateNodeInfo:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoRequest + 35, // 37: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupGetOptions:input_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsRequest + 4, // 38: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroups:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsResponse + 6, // 39: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupForNode:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeResponse + 8, // 40: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.PricingNodePrice:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceResponse + 10, // 41: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.PricingPodPrice:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceResponse + 12, // 42: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.GPULabel:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.GPULabelResponse + 14, // 43: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.GetAvailableGPUTypes:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse + 16, // 44: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.Cleanup:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.CleanupResponse + 18, // 45: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.Refresh:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.RefreshResponse + 20, // 46: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupTargetSize:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTargetSizeResponse + 22, // 47: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupIncreaseSize:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupIncreaseSizeResponse + 24, // 48: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupDeleteNodes:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesResponse + 26, // 49: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupDecreaseTargetSize:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDecreaseTargetSizeResponse + 28, // 50: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupNodes:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesResponse + 33, // 51: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupTemplateNodeInfo:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoResponse + 36, // 52: clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider.NodeGroupGetOptions:output_type -> clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsResponse + 38, // [38:53] is the sub-list for method output_type + 23, // [23:38] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name +} + +func init() { file_cloudprovider_externalgrpc_protos_externalgrpc_proto_init() } +func file_cloudprovider_externalgrpc_protos_externalgrpc_proto_init() { + if File_cloudprovider_externalgrpc_protos_externalgrpc_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroup); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExternalGrpcNode); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupForNodeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupForNodeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PricingNodePriceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PricingNodePriceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PricingPodPriceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PricingPodPriceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GPULabelRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GPULabelResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetAvailableGPUTypesRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetAvailableGPUTypesResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CleanupRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CleanupResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RefreshRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RefreshResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupTargetSizeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupTargetSizeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupIncreaseSizeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupIncreaseSizeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupDeleteNodesRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupDeleteNodesResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupDecreaseTargetSizeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupDecreaseTargetSizeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupNodesRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupNodesResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Instance); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*InstanceStatus); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*InstanceErrorInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupTemplateNodeInfoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupTemplateNodeInfoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupAutoscalingOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupAutoscalingOptionsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeGroupAutoscalingOptionsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc, + NumEnums: 1, + NumMessages: 39, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes, + DependencyIndexes: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_depIdxs, + EnumInfos: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_enumTypes, + MessageInfos: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes, + }.Build() + File_cloudprovider_externalgrpc_protos_externalgrpc_proto = out.File + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc = nil + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes = nil + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_depIdxs = nil +} diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto new file mode 100644 index 000000000000..f83022f5b9a3 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto @@ -0,0 +1,371 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +syntax = "proto3"; + +package clusterautoscaler.cloudprovider.v1.externalgrpc; + +import "google/protobuf/descriptor.proto"; +import "google/protobuf/any.proto"; + +import "k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto"; +import "k8s.io/api/core/v1/generated.proto"; + +option go_package = "cluster-autoscaler/cloudprovider/externalgrpc/protos"; + +service CloudProvider { + // CloudProvider specific RPC functions + + // NodeGroups returns all node groups configured for this cloud provider. + rpc NodeGroups(NodeGroupsRequest) + returns (NodeGroupsResponse) {} + + // NodeGroupForNode returns the node group for the given node. + // The node group id is an empty string if the node should not + // be processed by cluster autoscaler. + rpc NodeGroupForNode(NodeGroupForNodeRequest) + returns (NodeGroupForNodeResponse) {} + + // PricingNodePrice returns a theoretical minimum price of running a node for + // a given period of time on a perfectly matching machine. + // Implementation optional. + rpc PricingNodePrice(PricingNodePriceRequest) + returns (PricingNodePriceResponse) {} + + // PricingPodPrice returns a theoretical minimum price of running a pod for a given + // period of time on a perfectly matching machine. + // Implementation optional. + rpc PricingPodPrice(PricingPodPriceRequest) + returns (PricingPodPriceResponse) {} + + // GPULabel returns the label added to nodes with GPU resource. + rpc GPULabel(GPULabelRequest) + returns (GPULabelResponse) {} + + // GetAvailableGPUTypes return all available GPU types cloud provider supports. + rpc GetAvailableGPUTypes(GetAvailableGPUTypesRequest) + returns (GetAvailableGPUTypesResponse) {} + + // Cleanup cleans up open resources before the cloud provider is destroyed, i.e. go routines etc. + rpc Cleanup(CleanupRequest) + returns (CleanupResponse) {} + + // Refresh is called before every main loop and can be used to dynamically update cloud provider state. + rpc Refresh(RefreshRequest) + returns (RefreshResponse) {} + + // NodeGroup specific RPC functions + + // NodeGroupTargetSize returns the current target size of the node group. It is possible + // that the number of nodes in Kubernetes is different at the moment but should be equal + // to the size of a node group once everything stabilizes (new nodes finish startup and + // registration or removed nodes are deleted completely). + rpc NodeGroupTargetSize(NodeGroupTargetSizeRequest) + returns (NodeGroupTargetSizeResponse) {} + + // NodeGroupIncreaseSize increases the size of the node group. To delete a node you need + // to explicitly name it and use NodeGroupDeleteNodes. This function should wait until + // node group size is updated. + rpc NodeGroupIncreaseSize(NodeGroupIncreaseSizeRequest) + returns (NodeGroupIncreaseSizeResponse) {} + + // NodeGroupDeleteNodes deletes nodes from this node group (and also decreasing the size + // of the node group with that). Error is returned either on failure or if the given node + // doesn't belong to this node group. This function should wait until node group size is updated. + rpc NodeGroupDeleteNodes(NodeGroupDeleteNodesRequest) + returns (NodeGroupDeleteNodesResponse) {} + + // NodeGroupDecreaseTargetSize decreases the target size of the node group. This function + // doesn't permit to delete any existing node and can be used only to reduce the request + // for new nodes that have not been yet fulfilled. Delta should be negative. It is assumed + // that cloud provider will not delete the existing nodes if the size when there is an option + // to just decrease the target. + rpc NodeGroupDecreaseTargetSize(NodeGroupDecreaseTargetSizeRequest) + returns (NodeGroupDecreaseTargetSizeResponse) {} + + // NodeGroupNodes returns a list of all nodes that belong to this node group. + rpc NodeGroupNodes(NodeGroupNodesRequest) + returns (NodeGroupNodesResponse) {} + + // NodeGroupTemplateNodeInfo returns a structure of an empty (as if just started) node, + // with all of the labels, capacity and allocatable information. This will be used in + // scale-up simulations to predict what would a new node look like if a node group was expanded. + // Implementation optional. + rpc NodeGroupTemplateNodeInfo(NodeGroupTemplateNodeInfoRequest) + returns (NodeGroupTemplateNodeInfoResponse) {} + + // GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular + // NodeGroup. Returning a grpc error will result in using default options. + // Implementation optional. + rpc NodeGroupGetOptions(NodeGroupAutoscalingOptionsRequest) + returns (NodeGroupAutoscalingOptionsResponse) {} +} + +message NodeGroup { + // ID of the node group on the cloud provider. + string id = 1; + + // MinSize of the node group on the cloud provider. + int32 minSize = 2; + + // MaxSize of the node group on the cloud provider. + int32 maxSize = 3; + + // Debug returns a string containing all information regarding this node group. + string debug = 4; +} + +message ExternalGrpcNode{ + // ID of the node assigned by the cloud provider in the format: ://. + string providerID = 1; + + // Name of the node assigned by the cloud provider. + string name = 2; + + // labels is a map of {key,value} pairs with the node's labels. + map labels = 3; + + // If specified, the node's annotations. + map annotations = 4; +} + +message NodeGroupsRequest { + // Intentionally empty. +} + +message NodeGroupsResponse { + // All the node groups that the cloud provider service supports. + repeated NodeGroup nodeGroups = 1; +} + +message NodeGroupForNodeRequest { + // Node for which the request is performed. + ExternalGrpcNode node = 1; +} + +message NodeGroupForNodeResponse { + // Node group for the given node. nodeGroup with id = "" means no node group. + NodeGroup nodeGroup = 1; +} + +message PricingNodePriceRequest { + // Node for which the request is performed. + ExternalGrpcNode node = 1; + + // Start time for the request period. + k8s.io.apimachinery.pkg.apis.meta.v1.Time startTime = 2; + + // End time for the request period. + k8s.io.apimachinery.pkg.apis.meta.v1.Time endTime = 3; +} + +message PricingNodePriceResponse { + // Theoretical minimum price of running a node for a given period. + double price = 1; +} + +message PricingPodPriceRequest { + // Pod for which the request is performed. + k8s.io.api.core.v1.Pod pod = 1; + + // Start time for the request period. + k8s.io.apimachinery.pkg.apis.meta.v1.Time startTime = 2; + + // End time for the request period. + k8s.io.apimachinery.pkg.apis.meta.v1.Time endTime = 3; +} + +message PricingPodPriceResponse { + // Theoretical minimum price of running a pod for a given period. + double price = 1; +} + +message GPULabelRequest { + // Intentionally empty. +} + +message GPULabelResponse { + // Label added to nodes with a GPU resource. + string label = 1; +} + +message GetAvailableGPUTypesRequest { + // Intentionally empty. +} + +message GetAvailableGPUTypesResponse { + // GPU types passed in as opaque key-value pairs. + map gpuTypes = 1; +} + +message CleanupRequest { + // Intentionally empty. +} + +message CleanupResponse { + // Intentionally empty. +} + +message RefreshRequest { + // Intentionally empty. +} + +message RefreshResponse { + // Intentionally empty. +} + +message NodeGroupTargetSizeRequest { + // ID of the node group for the request. + string id = 1; +} + +message NodeGroupTargetSizeResponse { + // Current target size of the node group. + int32 targetSize = 1; +} + +message NodeGroupIncreaseSizeRequest { + // Number of nodes to add. + int32 delta = 1; + + // ID of the node group for the request. + string id = 2; +} + +message NodeGroupIncreaseSizeResponse { + // Intentionally empty. +} + +message NodeGroupDeleteNodesRequest { + // List of nodes to delete. + repeated ExternalGrpcNode nodes = 1; + + // ID of the node group for the request. + string id = 2; +} + +message NodeGroupDeleteNodesResponse { + // Intentionally empty. + } + +message NodeGroupDecreaseTargetSizeRequest { + // Number of nodes to delete. + int32 delta = 1; + + // ID of the node group for the request. + string id = 2; +} + +message NodeGroupDecreaseTargetSizeResponse { + // Intentionally empty. +} + +message NodeGroupNodesRequest { + // ID of the node group for the request. + string id = 1; +} + +message NodeGroupNodesResponse { + // list of cloud provider instances in a node group. + repeated Instance instances = 1; +} + +message Instance { + // Id of the instance. + string id = 1; + + // Status of the node. + InstanceStatus status = 2; +} + +// InstanceStatus represents the instance status. +message InstanceStatus { + enum InstanceState { + // an Unspecified instanceState means the actual instance status is undefined (nil). + unspecified = 0; + + // InstanceRunning means instance is running. + instanceRunning = 1; + + // InstanceCreating means instance is being created. + instanceCreating = 2; + + // InstanceDeleting means instance is being deleted. + instanceDeleting = 3; + } + + // InstanceState tells if the instance is running, being created or being deleted. + InstanceState instanceState = 1; + + // ErrorInfo provides information about the error status. + // If there is no error condition related to instance, then errorInfo.errorCode should be an empty string. + InstanceErrorInfo errorInfo = 2; +} + +// InstanceErrorInfo provides information about error condition on instance. +message InstanceErrorInfo { + // ErrorCode is cloud-provider specific error code for error condition. + // An empty string for errorCode means there is no errorInfo for the instance (nil). + string errorCode = 1; + + // ErrorMessage is the human readable description of error condition. + string errorMessage = 2; + + // InstanceErrorClass defines the class of error condition. + int32 instanceErrorClass = 3; +} + +message NodeGroupTemplateNodeInfoRequest { + // ID of the node group for the request. + string id = 1; +} + +message NodeGroupTemplateNodeInfoResponse { + // nodeInfo is the extracted data from the cloud provider, as a primitive Kubernetes Node type. + k8s.io.api.core.v1.Node nodeInfo = 1; +} + +message NodeGroupAutoscalingOptions { + // ScaleDownUtilizationThreshold sets threshold for nodes to be considered for scale down + // if cpu or memory utilization is over threshold. + double scaleDownUtilizationThreshold = 1; + + // ScaleDownGpuUtilizationThreshold sets threshold for gpu nodes to be + // considered for scale down if gpu utilization is over threshold. + double scaleDownGpuUtilizationThreshold = 2; + + // ScaleDownUnneededTime sets the duration CA expects a node to be + // unneeded/eligible for removal before scaling down the node. + k8s.io.apimachinery.pkg.apis.meta.v1.Duration scaleDownUnneededTime = 3; + + // ScaleDownUnreadyTime represents how long an unready node should be + // unneeded before it is eligible for scale down. + k8s.io.apimachinery.pkg.apis.meta.v1.Duration scaleDownUnreadyTime = 4; +} + +message NodeGroupAutoscalingOptionsRequest { + // ID of the node group for the request. + string id = 1; + + // default node group autoscaling options. + NodeGroupAutoscalingOptions defaults = 2; +} + +message NodeGroupAutoscalingOptionsResponse { + // autoscaling options for the requested node. + NodeGroupAutoscalingOptions nodeGroupAutoscalingOptions = 1; +} + diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go new file mode 100644 index 000000000000..187820ef1d9c --- /dev/null +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go @@ -0,0 +1,695 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package protos + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// CloudProviderClient is the client API for CloudProvider service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type CloudProviderClient interface { + // NodeGroups returns all node groups configured for this cloud provider. + NodeGroups(ctx context.Context, in *NodeGroupsRequest, opts ...grpc.CallOption) (*NodeGroupsResponse, error) + // NodeGroupForNode returns the node group for the given node. + // The node group id is an empty string if the node should not + // be processed by cluster autoscaler. + NodeGroupForNode(ctx context.Context, in *NodeGroupForNodeRequest, opts ...grpc.CallOption) (*NodeGroupForNodeResponse, error) + // PricingNodePrice returns a theoretical minimum price of running a node for + // a given period of time on a perfectly matching machine. + // Implementation optional. + PricingNodePrice(ctx context.Context, in *PricingNodePriceRequest, opts ...grpc.CallOption) (*PricingNodePriceResponse, error) + // PricingPodPrice returns a theoretical minimum price of running a pod for a given + // period of time on a perfectly matching machine. + // Implementation optional. + PricingPodPrice(ctx context.Context, in *PricingPodPriceRequest, opts ...grpc.CallOption) (*PricingPodPriceResponse, error) + // GPULabel returns the label added to nodes with GPU resource. + GPULabel(ctx context.Context, in *GPULabelRequest, opts ...grpc.CallOption) (*GPULabelResponse, error) + // GetAvailableGPUTypes return all available GPU types cloud provider supports. + GetAvailableGPUTypes(ctx context.Context, in *GetAvailableGPUTypesRequest, opts ...grpc.CallOption) (*GetAvailableGPUTypesResponse, error) + // Cleanup cleans up open resources before the cloud provider is destroyed, i.e. go routines etc. + Cleanup(ctx context.Context, in *CleanupRequest, opts ...grpc.CallOption) (*CleanupResponse, error) + // Refresh is called before every main loop and can be used to dynamically update cloud provider state. + Refresh(ctx context.Context, in *RefreshRequest, opts ...grpc.CallOption) (*RefreshResponse, error) + // NodeGroupTargetSize returns the current target size of the node group. It is possible + // that the number of nodes in Kubernetes is different at the moment but should be equal + // to the size of a node group once everything stabilizes (new nodes finish startup and + // registration or removed nodes are deleted completely). + NodeGroupTargetSize(ctx context.Context, in *NodeGroupTargetSizeRequest, opts ...grpc.CallOption) (*NodeGroupTargetSizeResponse, error) + // NodeGroupIncreaseSize increases the size of the node group. To delete a node you need + // to explicitly name it and use NodeGroupDeleteNodes. This function should wait until + // node group size is updated. + NodeGroupIncreaseSize(ctx context.Context, in *NodeGroupIncreaseSizeRequest, opts ...grpc.CallOption) (*NodeGroupIncreaseSizeResponse, error) + // NodeGroupDeleteNodes deletes nodes from this node group (and also decreasing the size + // of the node group with that). Error is returned either on failure or if the given node + // doesn't belong to this node group. This function should wait until node group size is updated. + NodeGroupDeleteNodes(ctx context.Context, in *NodeGroupDeleteNodesRequest, opts ...grpc.CallOption) (*NodeGroupDeleteNodesResponse, error) + // NodeGroupDecreaseTargetSize decreases the target size of the node group. This function + // doesn't permit to delete any existing node and can be used only to reduce the request + // for new nodes that have not been yet fulfilled. Delta should be negative. It is assumed + // that cloud provider will not delete the existing nodes if the size when there is an option + // to just decrease the target. + NodeGroupDecreaseTargetSize(ctx context.Context, in *NodeGroupDecreaseTargetSizeRequest, opts ...grpc.CallOption) (*NodeGroupDecreaseTargetSizeResponse, error) + // NodeGroupNodes returns a list of all nodes that belong to this node group. + NodeGroupNodes(ctx context.Context, in *NodeGroupNodesRequest, opts ...grpc.CallOption) (*NodeGroupNodesResponse, error) + // NodeGroupTemplateNodeInfo returns a structure of an empty (as if just started) node, + // with all of the labels, capacity and allocatable information. This will be used in + // scale-up simulations to predict what would a new node look like if a node group was expanded. + // Implementation optional. + NodeGroupTemplateNodeInfo(ctx context.Context, in *NodeGroupTemplateNodeInfoRequest, opts ...grpc.CallOption) (*NodeGroupTemplateNodeInfoResponse, error) + // GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular + // NodeGroup. Returning a grpc error will result in using default options. + // Implementation optional. + NodeGroupGetOptions(ctx context.Context, in *NodeGroupAutoscalingOptionsRequest, opts ...grpc.CallOption) (*NodeGroupAutoscalingOptionsResponse, error) +} + +type cloudProviderClient struct { + cc grpc.ClientConnInterface +} + +func NewCloudProviderClient(cc grpc.ClientConnInterface) CloudProviderClient { + return &cloudProviderClient{cc} +} + +func (c *cloudProviderClient) NodeGroups(ctx context.Context, in *NodeGroupsRequest, opts ...grpc.CallOption) (*NodeGroupsResponse, error) { + out := new(NodeGroupsResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroups", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) NodeGroupForNode(ctx context.Context, in *NodeGroupForNodeRequest, opts ...grpc.CallOption) (*NodeGroupForNodeResponse, error) { + out := new(NodeGroupForNodeResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupForNode", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) PricingNodePrice(ctx context.Context, in *PricingNodePriceRequest, opts ...grpc.CallOption) (*PricingNodePriceResponse, error) { + out := new(PricingNodePriceResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/PricingNodePrice", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) PricingPodPrice(ctx context.Context, in *PricingPodPriceRequest, opts ...grpc.CallOption) (*PricingPodPriceResponse, error) { + out := new(PricingPodPriceResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/PricingPodPrice", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) GPULabel(ctx context.Context, in *GPULabelRequest, opts ...grpc.CallOption) (*GPULabelResponse, error) { + out := new(GPULabelResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/GPULabel", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) GetAvailableGPUTypes(ctx context.Context, in *GetAvailableGPUTypesRequest, opts ...grpc.CallOption) (*GetAvailableGPUTypesResponse, error) { + out := new(GetAvailableGPUTypesResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/GetAvailableGPUTypes", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) Cleanup(ctx context.Context, in *CleanupRequest, opts ...grpc.CallOption) (*CleanupResponse, error) { + out := new(CleanupResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/Cleanup", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) Refresh(ctx context.Context, in *RefreshRequest, opts ...grpc.CallOption) (*RefreshResponse, error) { + out := new(RefreshResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/Refresh", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) NodeGroupTargetSize(ctx context.Context, in *NodeGroupTargetSizeRequest, opts ...grpc.CallOption) (*NodeGroupTargetSizeResponse, error) { + out := new(NodeGroupTargetSizeResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupTargetSize", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) NodeGroupIncreaseSize(ctx context.Context, in *NodeGroupIncreaseSizeRequest, opts ...grpc.CallOption) (*NodeGroupIncreaseSizeResponse, error) { + out := new(NodeGroupIncreaseSizeResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupIncreaseSize", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) NodeGroupDeleteNodes(ctx context.Context, in *NodeGroupDeleteNodesRequest, opts ...grpc.CallOption) (*NodeGroupDeleteNodesResponse, error) { + out := new(NodeGroupDeleteNodesResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupDeleteNodes", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) NodeGroupDecreaseTargetSize(ctx context.Context, in *NodeGroupDecreaseTargetSizeRequest, opts ...grpc.CallOption) (*NodeGroupDecreaseTargetSizeResponse, error) { + out := new(NodeGroupDecreaseTargetSizeResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupDecreaseTargetSize", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) NodeGroupNodes(ctx context.Context, in *NodeGroupNodesRequest, opts ...grpc.CallOption) (*NodeGroupNodesResponse, error) { + out := new(NodeGroupNodesResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupNodes", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) NodeGroupTemplateNodeInfo(ctx context.Context, in *NodeGroupTemplateNodeInfoRequest, opts ...grpc.CallOption) (*NodeGroupTemplateNodeInfoResponse, error) { + out := new(NodeGroupTemplateNodeInfoResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupTemplateNodeInfo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cloudProviderClient) NodeGroupGetOptions(ctx context.Context, in *NodeGroupAutoscalingOptionsRequest, opts ...grpc.CallOption) (*NodeGroupAutoscalingOptionsResponse, error) { + out := new(NodeGroupAutoscalingOptionsResponse) + err := c.cc.Invoke(ctx, "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupGetOptions", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// CloudProviderServer is the server API for CloudProvider service. +// All implementations must embed UnimplementedCloudProviderServer +// for forward compatibility +type CloudProviderServer interface { + // NodeGroups returns all node groups configured for this cloud provider. + NodeGroups(context.Context, *NodeGroupsRequest) (*NodeGroupsResponse, error) + // NodeGroupForNode returns the node group for the given node. + // The node group id is an empty string if the node should not + // be processed by cluster autoscaler. + NodeGroupForNode(context.Context, *NodeGroupForNodeRequest) (*NodeGroupForNodeResponse, error) + // PricingNodePrice returns a theoretical minimum price of running a node for + // a given period of time on a perfectly matching machine. + // Implementation optional. + PricingNodePrice(context.Context, *PricingNodePriceRequest) (*PricingNodePriceResponse, error) + // PricingPodPrice returns a theoretical minimum price of running a pod for a given + // period of time on a perfectly matching machine. + // Implementation optional. + PricingPodPrice(context.Context, *PricingPodPriceRequest) (*PricingPodPriceResponse, error) + // GPULabel returns the label added to nodes with GPU resource. + GPULabel(context.Context, *GPULabelRequest) (*GPULabelResponse, error) + // GetAvailableGPUTypes return all available GPU types cloud provider supports. + GetAvailableGPUTypes(context.Context, *GetAvailableGPUTypesRequest) (*GetAvailableGPUTypesResponse, error) + // Cleanup cleans up open resources before the cloud provider is destroyed, i.e. go routines etc. + Cleanup(context.Context, *CleanupRequest) (*CleanupResponse, error) + // Refresh is called before every main loop and can be used to dynamically update cloud provider state. + Refresh(context.Context, *RefreshRequest) (*RefreshResponse, error) + // NodeGroupTargetSize returns the current target size of the node group. It is possible + // that the number of nodes in Kubernetes is different at the moment but should be equal + // to the size of a node group once everything stabilizes (new nodes finish startup and + // registration or removed nodes are deleted completely). + NodeGroupTargetSize(context.Context, *NodeGroupTargetSizeRequest) (*NodeGroupTargetSizeResponse, error) + // NodeGroupIncreaseSize increases the size of the node group. To delete a node you need + // to explicitly name it and use NodeGroupDeleteNodes. This function should wait until + // node group size is updated. + NodeGroupIncreaseSize(context.Context, *NodeGroupIncreaseSizeRequest) (*NodeGroupIncreaseSizeResponse, error) + // NodeGroupDeleteNodes deletes nodes from this node group (and also decreasing the size + // of the node group with that). Error is returned either on failure or if the given node + // doesn't belong to this node group. This function should wait until node group size is updated. + NodeGroupDeleteNodes(context.Context, *NodeGroupDeleteNodesRequest) (*NodeGroupDeleteNodesResponse, error) + // NodeGroupDecreaseTargetSize decreases the target size of the node group. This function + // doesn't permit to delete any existing node and can be used only to reduce the request + // for new nodes that have not been yet fulfilled. Delta should be negative. It is assumed + // that cloud provider will not delete the existing nodes if the size when there is an option + // to just decrease the target. + NodeGroupDecreaseTargetSize(context.Context, *NodeGroupDecreaseTargetSizeRequest) (*NodeGroupDecreaseTargetSizeResponse, error) + // NodeGroupNodes returns a list of all nodes that belong to this node group. + NodeGroupNodes(context.Context, *NodeGroupNodesRequest) (*NodeGroupNodesResponse, error) + // NodeGroupTemplateNodeInfo returns a structure of an empty (as if just started) node, + // with all of the labels, capacity and allocatable information. This will be used in + // scale-up simulations to predict what would a new node look like if a node group was expanded. + // Implementation optional. + NodeGroupTemplateNodeInfo(context.Context, *NodeGroupTemplateNodeInfoRequest) (*NodeGroupTemplateNodeInfoResponse, error) + // GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular + // NodeGroup. Returning a grpc error will result in using default options. + // Implementation optional. + NodeGroupGetOptions(context.Context, *NodeGroupAutoscalingOptionsRequest) (*NodeGroupAutoscalingOptionsResponse, error) + mustEmbedUnimplementedCloudProviderServer() +} + +// UnimplementedCloudProviderServer must be embedded to have forward compatible implementations. +type UnimplementedCloudProviderServer struct { +} + +func (UnimplementedCloudProviderServer) NodeGroups(context.Context, *NodeGroupsRequest) (*NodeGroupsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroups not implemented") +} +func (UnimplementedCloudProviderServer) NodeGroupForNode(context.Context, *NodeGroupForNodeRequest) (*NodeGroupForNodeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroupForNode not implemented") +} +func (UnimplementedCloudProviderServer) PricingNodePrice(context.Context, *PricingNodePriceRequest) (*PricingNodePriceResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PricingNodePrice not implemented") +} +func (UnimplementedCloudProviderServer) PricingPodPrice(context.Context, *PricingPodPriceRequest) (*PricingPodPriceResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PricingPodPrice not implemented") +} +func (UnimplementedCloudProviderServer) GPULabel(context.Context, *GPULabelRequest) (*GPULabelResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GPULabel not implemented") +} +func (UnimplementedCloudProviderServer) GetAvailableGPUTypes(context.Context, *GetAvailableGPUTypesRequest) (*GetAvailableGPUTypesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetAvailableGPUTypes not implemented") +} +func (UnimplementedCloudProviderServer) Cleanup(context.Context, *CleanupRequest) (*CleanupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Cleanup not implemented") +} +func (UnimplementedCloudProviderServer) Refresh(context.Context, *RefreshRequest) (*RefreshResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Refresh not implemented") +} +func (UnimplementedCloudProviderServer) NodeGroupTargetSize(context.Context, *NodeGroupTargetSizeRequest) (*NodeGroupTargetSizeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroupTargetSize not implemented") +} +func (UnimplementedCloudProviderServer) NodeGroupIncreaseSize(context.Context, *NodeGroupIncreaseSizeRequest) (*NodeGroupIncreaseSizeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroupIncreaseSize not implemented") +} +func (UnimplementedCloudProviderServer) NodeGroupDeleteNodes(context.Context, *NodeGroupDeleteNodesRequest) (*NodeGroupDeleteNodesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroupDeleteNodes not implemented") +} +func (UnimplementedCloudProviderServer) NodeGroupDecreaseTargetSize(context.Context, *NodeGroupDecreaseTargetSizeRequest) (*NodeGroupDecreaseTargetSizeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroupDecreaseTargetSize not implemented") +} +func (UnimplementedCloudProviderServer) NodeGroupNodes(context.Context, *NodeGroupNodesRequest) (*NodeGroupNodesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroupNodes not implemented") +} +func (UnimplementedCloudProviderServer) NodeGroupTemplateNodeInfo(context.Context, *NodeGroupTemplateNodeInfoRequest) (*NodeGroupTemplateNodeInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroupTemplateNodeInfo not implemented") +} +func (UnimplementedCloudProviderServer) NodeGroupGetOptions(context.Context, *NodeGroupAutoscalingOptionsRequest) (*NodeGroupAutoscalingOptionsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGroupGetOptions not implemented") +} +func (UnimplementedCloudProviderServer) mustEmbedUnimplementedCloudProviderServer() {} + +// UnsafeCloudProviderServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to CloudProviderServer will +// result in compilation errors. +type UnsafeCloudProviderServer interface { + mustEmbedUnimplementedCloudProviderServer() +} + +func RegisterCloudProviderServer(s grpc.ServiceRegistrar, srv CloudProviderServer) { + s.RegisterService(&CloudProvider_ServiceDesc, srv) +} + +func _CloudProvider_NodeGroups_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroups(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroups", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroups(ctx, req.(*NodeGroupsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_NodeGroupForNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupForNodeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroupForNode(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupForNode", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroupForNode(ctx, req.(*NodeGroupForNodeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_PricingNodePrice_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PricingNodePriceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).PricingNodePrice(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/PricingNodePrice", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).PricingNodePrice(ctx, req.(*PricingNodePriceRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_PricingPodPrice_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PricingPodPriceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).PricingPodPrice(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/PricingPodPrice", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).PricingPodPrice(ctx, req.(*PricingPodPriceRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_GPULabel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GPULabelRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).GPULabel(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/GPULabel", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).GPULabel(ctx, req.(*GPULabelRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_GetAvailableGPUTypes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetAvailableGPUTypesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).GetAvailableGPUTypes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/GetAvailableGPUTypes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).GetAvailableGPUTypes(ctx, req.(*GetAvailableGPUTypesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_Cleanup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CleanupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).Cleanup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/Cleanup", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).Cleanup(ctx, req.(*CleanupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_Refresh_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RefreshRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).Refresh(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/Refresh", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).Refresh(ctx, req.(*RefreshRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_NodeGroupTargetSize_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupTargetSizeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroupTargetSize(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupTargetSize", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroupTargetSize(ctx, req.(*NodeGroupTargetSizeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_NodeGroupIncreaseSize_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupIncreaseSizeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroupIncreaseSize(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupIncreaseSize", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroupIncreaseSize(ctx, req.(*NodeGroupIncreaseSizeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_NodeGroupDeleteNodes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupDeleteNodesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroupDeleteNodes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupDeleteNodes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroupDeleteNodes(ctx, req.(*NodeGroupDeleteNodesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_NodeGroupDecreaseTargetSize_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupDecreaseTargetSizeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroupDecreaseTargetSize(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupDecreaseTargetSize", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroupDecreaseTargetSize(ctx, req.(*NodeGroupDecreaseTargetSizeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_NodeGroupNodes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupNodesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroupNodes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupNodes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroupNodes(ctx, req.(*NodeGroupNodesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_NodeGroupTemplateNodeInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupTemplateNodeInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroupTemplateNodeInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupTemplateNodeInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroupTemplateNodeInfo(ctx, req.(*NodeGroupTemplateNodeInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CloudProvider_NodeGroupGetOptions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGroupAutoscalingOptionsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CloudProviderServer).NodeGroupGetOptions(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroupGetOptions", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CloudProviderServer).NodeGroupGetOptions(ctx, req.(*NodeGroupAutoscalingOptionsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// CloudProvider_ServiceDesc is the grpc.ServiceDesc for CloudProvider service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var CloudProvider_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider", + HandlerType: (*CloudProviderServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "NodeGroups", + Handler: _CloudProvider_NodeGroups_Handler, + }, + { + MethodName: "NodeGroupForNode", + Handler: _CloudProvider_NodeGroupForNode_Handler, + }, + { + MethodName: "PricingNodePrice", + Handler: _CloudProvider_PricingNodePrice_Handler, + }, + { + MethodName: "PricingPodPrice", + Handler: _CloudProvider_PricingPodPrice_Handler, + }, + { + MethodName: "GPULabel", + Handler: _CloudProvider_GPULabel_Handler, + }, + { + MethodName: "GetAvailableGPUTypes", + Handler: _CloudProvider_GetAvailableGPUTypes_Handler, + }, + { + MethodName: "Cleanup", + Handler: _CloudProvider_Cleanup_Handler, + }, + { + MethodName: "Refresh", + Handler: _CloudProvider_Refresh_Handler, + }, + { + MethodName: "NodeGroupTargetSize", + Handler: _CloudProvider_NodeGroupTargetSize_Handler, + }, + { + MethodName: "NodeGroupIncreaseSize", + Handler: _CloudProvider_NodeGroupIncreaseSize_Handler, + }, + { + MethodName: "NodeGroupDeleteNodes", + Handler: _CloudProvider_NodeGroupDeleteNodes_Handler, + }, + { + MethodName: "NodeGroupDecreaseTargetSize", + Handler: _CloudProvider_NodeGroupDecreaseTargetSize_Handler, + }, + { + MethodName: "NodeGroupNodes", + Handler: _CloudProvider_NodeGroupNodes_Handler, + }, + { + MethodName: "NodeGroupTemplateNodeInfo", + Handler: _CloudProvider_NodeGroupTemplateNodeInfo_Handler, + }, + { + MethodName: "NodeGroupGetOptions", + Handler: _CloudProvider_NodeGroupGetOptions_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "cloudprovider/externalgrpc/protos/externalgrpc.proto", +} diff --git a/hack/verify-gofmt.sh b/hack/verify-gofmt.sh index 593c9336fe3a..89854e22eed5 100755 --- a/hack/verify-gofmt.sh +++ b/hack/verify-gofmt.sh @@ -38,6 +38,7 @@ find_files() { -o -wholename './cluster-autoscaler/cloudprovider/magnum/gophercloud/*' \ -o -wholename './cluster-autoscaler/cloudprovider/digitalocean/godo/*' \ -o -wholename './cluster-autoscaler/cloudprovider/bizflycloud/gobizfly/*' \ + -o -wholename './cluster-autoscaler/cloudprovider/externalgrpc/protos/*' \ -o -wholename './cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud-sdk-go-v3/*' \ -o -wholename './cluster-autoscaler/cloudprovider/ionoscloud/ionos-cloud-sdk-go/*' \ -o -wholename './cluster-autoscaler/cloudprovider/hetzner/hcloud-go/*' \ diff --git a/hack/verify-golint.sh b/hack/verify-golint.sh index 7496893fcdae..79fc7b38107b 100755 --- a/hack/verify-golint.sh +++ b/hack/verify-golint.sh @@ -32,6 +32,7 @@ excluded_packages=( 'cluster-autoscaler/cloudprovider/brightbox/k8ssdk' 'cluster-autoscaler/cloudprovider/brightbox/linkheader' 'cluster-autoscaler/cloudprovider/brightbox/go-cache' + 'cluster-autoscaler/cloudprovider/externalgrpc/protos' 'cluster-autoscaler/cloudprovider/exoscale/internal' 'cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud-sdk-go-v3' 'cluster-autoscaler/cloudprovider/ionoscloud/ionos-cloud-sdk-go'