From 5d0f925d7f349ac95ae64314bb1543e4543598de Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 23 Jul 2021 16:39:32 -0400 Subject: [PATCH 1/2] Initial pass at some vtadmin docs Signed-off-by: Andrew Mason --- doc/vtadmin/api-server.md | 15 +++++++++ doc/vtadmin/cluster-config.md | 9 +++++ doc/vtadmin/clusters.yaml | 56 +++++++++++++++++++++++++++++++ doc/vtadmin/discovery.md | 6 ++++ go/vt/vtadmin/README.md | 63 +++++++++++++++++++++++++++++++++-- 5 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 doc/vtadmin/api-server.md create mode 100644 doc/vtadmin/cluster-config.md create mode 100644 doc/vtadmin/clusters.yaml create mode 100644 doc/vtadmin/discovery.md diff --git a/doc/vtadmin/api-server.md b/doc/vtadmin/api-server.md new file mode 100644 index 00000000000..c422d466209 --- /dev/null +++ b/doc/vtadmin/api-server.md @@ -0,0 +1,15 @@ +# API Server + +VTAdmin uses [`cmux`](https://github.com/soheilhy/cmux) to multiplex gRPC and +HTTP traffic over a single port. + +Internally, VTAdmin has an adaptation layer (see [`go/vt/vtadmin/http/api.go`][http/api.go]) that +converts query parameters into protobuf messages, so that all HTTP and gRPC requests +are powered by the same implementation. + +For a full listing of the available gRPC methods, see [vtadmin.proto][vtadmin.proto], +and for a full listing of available HTTP routes, see [`go/vt/vtadmin/api.go`][vtadmin/api.go]. + +[http/api.go]: https://github.com/vitessio/vitess/blob/a787cd1e80ef2392bfe1050dd0c5ddd4efde066f/go/vt/vtadmin/http/api.go#L64-L84 +[vtadmin.proto]: ../../../proto/vtadmin.proto +[vtadmin/api.go]: https://github.com/vitessio/vitess/blob/a787cd1e80ef2392bfe1050dd0c5ddd4efde066f/go/vt/vtadmin/api.go#L153-L191 diff --git a/doc/vtadmin/cluster-config.md b/doc/vtadmin/cluster-config.md new file mode 100644 index 00000000000..06d94cf74d9 --- /dev/null +++ b/doc/vtadmin/cluster-config.md @@ -0,0 +1,9 @@ +# Cluster Config + +VTAdmin operates on multiple clusters, each of which may have different +size/scaling constraints, naming conventions, and so on. + +TODO: +- what is a cluster +- merging +- hierarchy diff --git a/doc/vtadmin/clusters.yaml b/doc/vtadmin/clusters.yaml new file mode 100644 index 00000000000..d621de73c42 --- /dev/null +++ b/doc/vtadmin/clusters.yaml @@ -0,0 +1,56 @@ +defaults: + # Configuration options here provide default options for all clusters. + # Specifying a different value for an individual cluster will override the + # default setting for that cluster. + + # Discovery implementation to use by default. + discovery: "{consul|staticfile}" + + # Each discovery implementation has its own options, which are named according + # to the regex: + # ^discovery-(?P\w+)-(?P.+)$ + # The full set of options for each discovery implementation is defined in that + # implementations factory function, e.g. NewConsul in + # go/vt/vtadmin/cluster/discovery/discovery_consul.go and NewStaticFile in + # discovery_static_file.go in the same directory. + + # Service name to use when discovery vtctlds from consul. + discovery-consul-vtctld-service-name: "vtctld" + # Path to json file containing vtctld and vtgate hostnames when using staticfile. + discovery-staticfile-path: "/path/to/static/discovery.json" + + tablet-fqdn-tmpl: "" + + # Similar to how discovery flags are forwarded to the particular discovery + # implementation, the vtctld and vtgate proxy components for a cluster are + # configurable, by the prefixes "vtctld-" (for vtctld proxy) and "vtsql-" (for vtgate proxy). + # These flags are defined in code in the (*Config).Parse functions in packages + # go/vt/vtadmin/{vtctldclient,vtsql}. + + # Go template to produce a path to a json file containing Username and Password + # to make requests against vtctlds in this cluster. + vtctld-credentials-path-tmpl: "/path/to/vtctld/credentials/{{ .Cluster.Id }}.json" + + # Same as vtctld-credentials-path-tmpl, except used to make requests against + # vtgates in the cluster. + vtsql-credentials-path-tmpl: "/path/to/vtgate/credentials/{{ .Cluster.Id }}.json" + # Optional comma-separated list of tags to pass to a discovery implementation + # when discovering a vtgate in the cluster to connect to. The semantics of + # the tags depend on the discovery implementation used. + vtsql-discovery-tags: "tag1,tag2" + # Username to send queries on behalf of. See package callerid. + vtsql-effective-user: "my-effective-user" + + # VTAdmin also provides different RPC pools to gate the number of concurrent + # requests it will make against vtctlds/vtgates in a given cluster, to prevent + # overwhelming those components. + + # The backup read pool is used to constrain the number of GetBackups calls + # made to a cluster simultaneously. + backup-read-pool-size: 10 + backup-read-pool-timeout: 10ms + + # Other pools have the same size/timeout options, and include: + # - schema-read-pool => for GetSchema, GetSchemas, and FindSchema api methods + # - topo-read-pool => for generic topo methods (e.g. GetKeyspace, FindAllShardsInKeyspace) + # - workflow-read-pool => for GetWorkflow/GetWorkflows api methods. diff --git a/doc/vtadmin/discovery.md b/doc/vtadmin/discovery.md new file mode 100644 index 00000000000..62c8387026c --- /dev/null +++ b/doc/vtadmin/discovery.md @@ -0,0 +1,6 @@ +# Discovery + +TODO: +- pluggable discovery +- consul support +- staticfile support diff --git a/go/vt/vtadmin/README.md b/go/vt/vtadmin/README.md index 69affca3dba..a206ba1ac32 100644 --- a/go/vt/vtadmin/README.md +++ b/go/vt/vtadmin/README.md @@ -1,4 +1,63 @@ # VTAdmin -VTAdmin is in alpha, you may use it if you wish, but should not treat it as stable. -It does not meet the standard Vitess backwards compatibility guarantees, and is subject to change at any time. +VTAdmin is an experimental replacement for the vtctld2 web UI, that also allows +users to manage multiple Vitess clusters from a single API and web UI. + +## Setup + +The simplest VTAdmin deployment involves a single Vitess cluster. You can look +at the [local example](../../../examples/local/scripts/vtadmin-up.sh) for a +minimal invocation of the `vtadmin-api` and `vtadmin-web` binaries. + +### Important `vtadmin-api` flags + +Please refer to `vtadmin --help` for the full listing, but a few flags warrant +explanation here. + +* `--http-origin` — this flag sets up the allowed CORS origins that `vtadmin-api` + will serve HTTP requests for, and is required if you are (very likely) running + `vtadmin-api` and `vtadmin-web` on different domains. +* `--cluster`, `--cluster-defaults` — A [DSN-style][dsn] flag that allows cluster + configuration options to be specified on the command-line rather than needing + a config file. When both command-line cluster configs and a config file are + provided, any options for a given cluster on the command-line take precedence + over options for that cluster in the config file. + + For a description of the cluster configuration options, see [clusters.example.yaml](../../../doc/vtadmin/clusters.yaml). + +* `--http-tablet-url-tmpl` — Go template string to generate a reachable http(s) + address for a tablet, used to make passthrough requests to `/debug/vars` + endpoints. + +[dsn]: https://www.percona.com/doc/percona-toolkit/LATEST/dsn_data_source_name_specifications.html + +## Development + +### Building `vtadmin-api` + +If you are making changes to `.proto` files, make sure you run + +``` +source dev.env +make proto grpcvtctldclient vtadmin_web_proto_types +``` + +Then, you can run `make build`, and run `./bin/vtadmin` with any flags you need +(see the local example, and also the section on flags above). + +### Building and running `vtadmin-web` + +Make sure you are using node version 12.x. + +Then, you may run: + +``` +cd ./web/vtadmin +npm install + +# This should be the address you passed to `./vtadmin --addr`. For example, +# "http://127.0.0.1:14200". +export REACT_APP_VTADMIN_API_ADDRESS="${vtadmin_api_addr}" +export REACT_APP_ENABLE_EXPERIMENTAL_TABLET_DEBUG_VARS="true" +npm run start +``` From 407de8b63771471af8e71b0862aa44b9d4495bf1 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 23 Jul 2021 19:55:50 -0400 Subject: [PATCH 2/2] PR feedback: link to rfc, fix typos Signed-off-by: Andrew Mason --- doc/vtadmin/clusters.yaml | 4 ++-- go/vt/vtadmin/README.md | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/vtadmin/clusters.yaml b/doc/vtadmin/clusters.yaml index d621de73c42..55779df60b2 100644 --- a/doc/vtadmin/clusters.yaml +++ b/doc/vtadmin/clusters.yaml @@ -10,11 +10,11 @@ defaults: # to the regex: # ^discovery-(?P\w+)-(?P.+)$ # The full set of options for each discovery implementation is defined in that - # implementations factory function, e.g. NewConsul in + # implementation's factory function, e.g. NewConsul in # go/vt/vtadmin/cluster/discovery/discovery_consul.go and NewStaticFile in # discovery_static_file.go in the same directory. - # Service name to use when discovery vtctlds from consul. + # Service name to use when discovering vtctlds from consul. discovery-consul-vtctld-service-name: "vtctld" # Path to json file containing vtctld and vtgate hostnames when using staticfile. discovery-staticfile-path: "/path/to/static/discovery.json" diff --git a/go/vt/vtadmin/README.md b/go/vt/vtadmin/README.md index a206ba1ac32..d794df9069e 100644 --- a/go/vt/vtadmin/README.md +++ b/go/vt/vtadmin/README.md @@ -3,6 +3,8 @@ VTAdmin is an experimental replacement for the vtctld2 web UI, that also allows users to manage multiple Vitess clusters from a single API and web UI. +For a more detailed writeup, refer to the [original RFC](https://github.com/vitessio/vitess/issues/7117). + ## Setup The simplest VTAdmin deployment involves a single Vitess cluster. You can look