[v9] Backport initial Teleport Connect PR + fixes (#12205)

* teleterm (alpha)

* Add grpc-teleterm Makefile target

The grpc-tools package is needed to generate gRPC files for JavaScript.
However, at the moment it can't be installed on M1 MacBooks because of
missing prebuilt binaries for arm64. [1]

One of them, protoc, is already installed in our buildbox. We still need
to compile grpc_node_plugin from source though. This adds significant
overhead as we need to pull in cmake, build-essential and then about
300 MB of git repos from protocolbuffers/protobuf.

Initially, those Teleterm gRPC were generated within `make grpc` with other
files. M1 users who don't work on Teleterm would not be happy about incurring
that additional overhead, hence I extracted everything into separate target
and Dockerfile.

Teleterm proto files don't depend on any other proto files. Once grpc-tools
adds support for arm64, we'll be able to essentially almost revert this
commit and generate Teleterm gRPC files within `make grpc`.

[1] grpc/grpc-node#1405

* Use oneof for LoginRequest params

The login is either local or SSO but not both.

* Use db name for URI in Teleterm rather than db server host ID

The previous version of the code used GetHostId return value for the URI.
That caused problems as a single host can run multiple database servers.
This in turn resulted in stuff like Teleterm not listing all databases.

There's `Database.GetURI` function which I decided not to use, because it's
an URI on its own which might include stuff like port numbers and what not.
I wanted to avoid a situation in which the database URI creates some potential
conflicts with the Teleterm URIs.

I noticed that the Web UI code runs `DeduplicateDatabases` already and it
uses `Database.GetName` underneath, so I deemed it a good candidate to be
a part of a database URI in Teleterm.

Fixes gravitational/webapps.e#127

* Remove PAM build tag from tsh target in Makefile (#11666)

The PAM tag is not needed when building tsh. Moreover, it was causing
the push-build-windows-amd64 pipeline to fail since lib/teleterm imports
lib/srv/alpnproxy which in turn indirectly depends on lib/pam.

* Move WebConfig from lib/web/ui to api/client/webclient (#11690)

* Move WebConfig from lib/web/ui to api/client/webclient

Web config was shared with the Web UI through the dynamically generated
/web/config.js file available on the cluster. With the addition of
Teleport Terminal (RFD 63), the Electron app needs to get a hold of this
config as well.

However, unlike the Web UI which directly loads the file and injects
the config this way, any communication between the cluster and Teleport
Terminal is done through the tsh daemon (RFD 63). The tsh daemon needs
to essentially pipe this config from /web/config.js to the gRPC response
it gives to Teleport Terminal.

To achieve this, a GetWebConfig function was added to TeleportClient.
Unfortunately, this breaks the build on Windows as lib/web (where WebConfig
resides) includes code which is not meant to be compiled or run on Windows.

Since we need to share the web config with another frontend application,
it only makes sense to move it to the webclient package. We already have
types shared between the server and the client there, for example the
PingResponse struct.

Co-authored-by: Alexey Kontsevoy <[email protected]>

Makefile

@@ -198,15 +198,15 @@ all: version
 # If you are considering changing this behavior, please consult with dev team first
 .PHONY: $(BUILDDIR)/tctl
 $(BUILDDIR)/tctl: roletester
-	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG)" -o $(BUILDDIR)/tctl $(BUILDFLAGS) ./tool/tctl
+	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(FIPS_TAG) $(ROLETESTER_TAG)" -o $(BUILDDIR)/tctl $(BUILDFLAGS) ./tool/tctl
 
 .PHONY: $(BUILDDIR)/teleport
 $(BUILDDIR)/teleport: ensure-webassets bpf-bytecode rdpclient
 	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(WEBASSETS_TAG) $(RDPCLIENT_TAG)" -o $(BUILDDIR)/teleport $(BUILDFLAGS) ./tool/teleport
 
 .PHONY: $(BUILDDIR)/tsh
 $(BUILDDIR)/tsh:
-	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG_TSH) go build -tags "$(PAM_TAG) $(FIPS_TAG)" -o $(BUILDDIR)/tsh $(BUILDFLAGS) ./tool/tsh
+	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG_TSH) go build -tags "$(FIPS_TAG)" -o $(BUILDDIR)/tsh $(BUILDFLAGS) ./tool/tsh
 
 .PHONY: $(BUILDDIR)/tbot
 $(BUILDDIR)/tbot:
@@ -704,6 +704,7 @@ ADDLICENSE_ARGS := -c 'Gravitational, Inc' -l apache \
 		-ignore 'e/**' \
 		-ignore 'gitref.go' \
 		-ignore 'lib/web/build/**' \
+		-ignore 'lib/teleterm/api/protogen/**' \
 		-ignore 'version.go' \
 		-ignore 'webassets/**' \
 		-ignore 'ignoreme' \
@@ -883,6 +884,23 @@ buildbox-grpc:
 		--gogofast_out=plugins=grpc,$(GOGOPROTO_IMPORTMAP):. \
 		envelope.proto
 
+# grpc-teleterm generates Go, TypeScript and JavaScript gRPC stubs from definitions for Teleport
+# Terminal. This target runs in the buildbox-teleterm container.
+#
+# It exists as a separate target because on M1 MacBooks we must build grpc_node_plugin from source.
+# That involves apt-get install of cmake & build-essential as well pulling hundreds of megabytes of
+# git repos. It would significantly increase the time it takes to build buildbox for M1 users that
+# don't need to generate Teleterm gRPC files.
+# TODO(ravicious): incorporate grpc-teleterm into grpc once grpc-tools adds arm64 binary.
+# https://github.com/grpc/grpc-node/issues/1405
+.PHONY: grpc-teleterm
+grpc-teleterm:
+	$(MAKE) -C build.assets grpc-teleterm
+
+# buildbox-grpc generates GRPC stubs
+.PHONY: buildbox-grpc-teleterm
+buildbox-grpc-teleterm:
+	cd lib/teleterm && buf generate
 
 .PHONY: goinstall
 goinstall:

lib/web/ui/webconfig.go → api/client/webclient/webconfig.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2015 Gravitational, Inc.
+Copyright 2015-2022 Gravitational, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package ui
+package webclient
 
 import "github.com/gravitational/teleport/api/constants"
 
@@ -38,7 +38,7 @@ const (
 	WebConfigAuthProviderGitHubURL = "/v1/webapi/github/login/web?connector_id=:providerName&redirect_url=:redirect"
 )
 
-// WebConfig is web application configuration
+// WebConfig is web application configuration served by the backend to be used in frontend apps.
 type WebConfig struct {
 	// Auth contains Teleport auth. preferences
 	Auth WebConfigAuthSettings `json:"auth,omitempty"`

api/profile/profile.go

@@ -221,6 +221,16 @@ func SetCurrentProfileName(dir string, name string) error {
 	return nil
 }
 
+// RemoveProfile removes cluster profile file
+func RemoveProfile(dir, name string) error {
+	profilePath := filepath.Join(dir, name+".yaml")
+	if err := os.Remove(profilePath); err != nil {
+		return trace.ConvertSystemError(err)
+	}
+
+	return nil
+}
+
 // GetCurrentProfileName attempts to load the current profile name.
 func GetCurrentProfileName(dir string) (name string, err error) {
 	if dir == "" {

build.assets/Dockerfile-teleterm

@@ -0,0 +1,53 @@
+ARG BUILDBOX_VERSION
+# GRPC_NODE_PLUGIN_BINARY_TYPE can be "prebuilt" or "compiled"
+ARG GRPC_NODE_PLUGIN_BINARY_TYPE
+FROM quay.io/gravitational/teleport-buildbox:$BUILDBOX_VERSION as base
+
+ARG BUILDARCH
+
+# Install buf
+RUN BIN="/usr/local/bin" && \
+    VERSION="1.0.0-rc1" && \
+    BINARY_NAME="buf" && \
+    curl -sSL \
+     "https://github.com/bufbuild/buf/releases/download/v${VERSION}/${BINARY_NAME}-$(uname -s)-$(uname -m)" \
+    -o "${BIN}/${BINARY_NAME}" && \
+    chmod +x "${BIN}/${BINARY_NAME}"
+
+# Install node
+ARG NODE_VERSION=v15.14.0
+ENV NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${BUILDARCH}.tar.xz"
+ENV NODE_PATH="/usr/local/lib/node-${NODE_VERSION}-linux-${BUILDARCH}"
+ENV PATH="$PATH:${NODE_PATH}/bin"
+RUN (curl -o /tmp/nodejs.tar.xz -L ${NODE_URL} && tar -xJf /tmp/nodejs.tar.xz -C /usr/local/lib)
+
+# Install js proto tools
+RUN (npm install --global [email protected])
+RUN go install github.com/golang/protobuf/[email protected]
+
+FROM base as grpc_node_plugin_binary_prebuilt
+ONBUILD RUN (npm install --global [email protected])
+
+FROM base as grpc_node_plugin_binary_compiled
+ONBUILD RUN apt-get update -y && \
+    apt-get install -q -y --no-install-recommends build-essential cmake jq && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+ONBUILD RUN (npm install --global --ignore-scripts [email protected])
+ONBUILD COPY teleterm_linux_arm64.toolchain.cmake ./linux_arm64.toolchain.cmake
+ONBUILD RUN git clone --depth=1 [email protected] https://github.com/grpc/grpc-node.git && \
+    mv linux_arm64.toolchain.cmake grpc-node/packages/grpc-tools/. && \
+    cd grpc-node && \
+    git submodule update --init --recursive && \
+    cd packages/grpc-tools && \
+    cmake -DCMAKE_TOOLCHAIN_FILE=linux_arm64.toolchain.cmake . && \
+    cmake --build . --target clean && cmake --build . --target grpc_node_plugin -- -j 12 && \
+    cp grpc_node_plugin $(npm root -g)/grpc-tools/bin/. && \
+    # grpc-tools requires both protoc and grpc_node_plugin, but protoc is already installed by
+    # the buildbox image.
+    ln -s $(which protoc) $(npm root -g)/grpc-tools/bin/protoc && \
+    cd ../../.. && \
+    rm -rf grpc-node
+
+# Choose an appropriate image and run ONBUILD instructions from it.
+FROM grpc_node_plugin_binary_${GRPC_NODE_PLUGIN_BINARY_TYPE}

build.assets/Makefile

@@ -32,7 +32,7 @@ RUNTIME_ARCH_arm64 := arm64
 RUNTIME_ARCH_aarch64 := arm64
 RUNTIME_ARCH := $(RUNTIME_ARCH_$(HOST_ARCH))
 
-PROTOC_VER ?= 3.6.1
+PROTOC_VER ?= 3.13.0
 GOGO_PROTO_TAG ?= v1.3.2
 
 BUILDBOX=quay.io/gravitational/teleport-buildbox:$(BUILDBOX_VERSION)
@@ -41,6 +41,7 @@ BUILDBOX_CENTOS7=quay.io/gravitational/teleport-buildbox-centos7:$(BUILDBOX_VERS
 BUILDBOX_CENTOS7_FIPS=quay.io/gravitational/teleport-buildbox-centos7-fips:$(BUILDBOX_VERSION)
 BUILDBOX_ARM=quay.io/gravitational/teleport-buildbox-arm:$(BUILDBOX_VERSION)
 BUILDBOX_ARM_FIPS=quay.io/gravitational/teleport-buildbox-arm-fips:$(BUILDBOX_VERSION)
+BUILDBOX_TELETERM=quay.io/gravitational/teleport-buildbox-teleterm:$(BUILDBOX_VERSION)
 
 # These variables are used to dynamically change the name of the buildbox Docker image used by the 'release'
 # target. The other solution was to remove the 'buildbox' dependency from the 'release' target, but this would
@@ -66,6 +67,12 @@ endif
 # pass external gocache path through to docker containers
 DOCKERFLAGS := $(DOCKERFLAGS) -v $(GOCACHE):/go/cache -e GOCACHE=/go/cache
 endif
+
+ifeq ("$(RUNTIME_ARCH)","arm64")
+	GRPC_NODE_PLUGIN_BINARY_TYPE := compiled
+else
+	GRPC_NODE_PLUGIN_BINARY_TYPE := prebuilt
+endif
 export
 
 
@@ -197,13 +204,35 @@ buildbox-arm-fips: buildbox-fips
 		--cache-from $(BUILDBOX_ARM_FIPS) \
 		--tag $(BUILDBOX_ARM_FIPS) -f Dockerfile-arm-fips .
 
+#
+# Builds a Docker buildbox which has tools needed to install grpc-tools npm package on arm64.
+# See the `grpc-teleterm` target in the main Makefile for more details.
+#
+.PHONY:buildbox-teleterm
+buildbox-teleterm: buildbox
+	@if [[ $${DRONE} == "true" ]] && ! docker inspect --type=image $(BUILDBOX_TELETERM) 2>&1 >/dev/null; then docker pull $(BUILDBOX_TELETERM) || true; fi;
+	docker build \
+		--build-arg BUILDBOX_VERSION=$(BUILDBOX_VERSION) \
+		--build-arg BUILDARCH=$(RUNTIME_ARCH) \
+		--build-arg GRPC_NODE_PLUGIN_BINARY_TYPE=$(GRPC_NODE_PLUGIN_BINARY_TYPE) \
+		--cache-from $(BUILDBOX) \
+		--cache-from $(BUILDBOX_TELETERM) \
+		--tag $(BUILDBOX_TELETERM) -f Dockerfile-teleterm .
+
 # grpc generates GRPC stubs from inside the buildbox
 .PHONY: grpc
 grpc: buildbox
 	docker run \
 		$(DOCKERFLAGS) -e CLANG_FORMAT=/usr/bin/clang-format-10 -t $(BUILDBOX) \
 		make -C /go/src/github.com/gravitational/teleport buildbox-grpc
 
+# grpc-teleterm generates GRPC stubs for Teleterm from inside buildbox-teleterm
+.PHONY: grpc-teleterm
+grpc-teleterm: buildbox-teleterm
+	docker run \
+		$(DOCKERFLAGS) -e CLANG_FORMAT=/usr/bin/clang-format-10 -t $(BUILDBOX_TELETERM) \
+		make -C /go/src/github.com/gravitational/teleport buildbox-grpc-teleterm
+
 #
 # Removes the docker image
 #

build.assets/teleterm_linux_arm64.toolchain.cmake

@@ -0,0 +1,4 @@
+# CMake toolchain used to build grpc_node_plugin for arm64 which is needed by grpc-tools.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a" CACHE STRING "c++ flags")
+set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -march=armv8-a" CACHE STRING "c flags")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -march=armv8-a" CACHE STRING "ld flags")

go.mod

@@ -206,6 +206,7 @@ require (
 	github.com/smartystreets/goconvey v1.7.2 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/thales-e-security/pool v0.0.2 // indirect
+	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.0.2 // indirect

go.sum

@@ -987,6 +987,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tstranex/u2f v0.0.0-20160508205855-eb799ce68da4 h1:aR+lGR8m0zBjvDlHkHOCmdsk79ipIPeiP75GqUlywKM=
 github.com/tstranex/u2f v0.0.0-20160508205855-eb799ce68da4/go.mod h1:eahSLaqAS0zsIEv80+vXT7WanXs7MQQDg3j3wGBSayo=
+github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ=
+github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

lib/client/api.go

@@ -587,10 +587,10 @@ func RetryWithRelogin(ctx context.Context, tc *TeleportClient, fn func() error)
 	return fn()
 }
 
-// readProfile reads in the profile as well as the associated certificate
+// ReadProfileStatus reads in the profile as well as the associated certificate
 // and returns a *ProfileStatus which can be used to print the status of the
 // profile.
-func readProfile(profileDir string, profileName string) (*ProfileStatus, error) {
+func ReadProfileStatus(profileDir string, profileName string) (*ProfileStatus, error) {
 	var err error
 
 	if profileDir == "" {
@@ -799,7 +799,7 @@ func Status(profileDir, proxyHost string) (*ProfileStatus, []*ProfileStatus, err
 	// Read in the target profile first. If readProfile returns trace.NotFound,
 	// that means the profile may have been corrupted (for example keys were
 	// deleted but profile exists), treat this as the user not being logged in.
-	profileStatus, err = readProfile(profileDir, profileName)
+	profileStatus, err = ReadProfileStatus(profileDir, profileName)
 	if err != nil {
 		log.Debug(err)
 		if !trace.IsNotFound(err) {
@@ -820,7 +820,7 @@ func Status(profileDir, proxyHost string) (*ProfileStatus, []*ProfileStatus, err
 			// already loaded this one
 			continue
 		}
-		ps, err := readProfile(profileDir, name)
+		ps, err := ReadProfileStatus(profileDir, name)
 		if err != nil {
 			log.Debug(err)
 			// parts of profile are missing?
@@ -2508,6 +2508,15 @@ func (tc *TeleportClient) PingAndShowMOTD(ctx context.Context) (*webclient.PingR
 	return pr, nil
 }
 
+// GetWebConfig retreives Teleport proxy web config
+func (tc *TeleportClient) GetWebConfig(ctx context.Context) (*webclient.WebConfig, error) {
+	cfg, err := GetWebConfig(ctx, tc.WebProxyAddr, tc.InsecureSkipVerify)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	return cfg, nil
+}
+
 // Login logs the user into a Teleport cluster by talking to a Teleport proxy.
 //
 // Login may hijack stdin in some scenarios; it's strongly recommended for

lib/client/keyagent.go

@@ -175,13 +175,19 @@ func (a *LocalKeyAgent) UpdateProxyHost(proxyHost string) {
 	a.proxyHost = proxyHost
 }
 
+// UpdateUsername changes username that the local agent operates on.
+func (a *LocalKeyAgent) UpdateUsername(username string) {
+	a.username = username
+}
+
 // LoadKeyForCluster fetches a cluster-specific SSH key and loads it into the
 // SSH agent.
 func (a *LocalKeyAgent) LoadKeyForCluster(clusterName string) (*agent.AddedKey, error) {
 	key, err := a.GetKey(clusterName, WithSSHCerts{})
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
+
 	return a.LoadKey(*key)
 }
 

lib/client/weblogin.go

@@ -17,10 +17,12 @@ limitations under the License.
 package client
 
 import (
+	"bytes"
 	"context"
 	"crypto/x509"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net"
 	"net/url"
 	"os"
@@ -31,11 +33,13 @@ import (
 	"github.com/gravitational/roundtrip"
 	"github.com/gravitational/teleport"
 	"github.com/gravitational/teleport/api/client/proto"
+	"github.com/gravitational/teleport/api/client/webclient"
 	"github.com/gravitational/teleport/api/constants"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/lib/auth"
 	"github.com/gravitational/teleport/lib/auth/u2f"
 	"github.com/gravitational/teleport/lib/defaults"
+
 	"github.com/gravitational/trace"
 	"github.com/sirupsen/logrus"
 
@@ -482,3 +486,31 @@ func HostCredentials(ctx context.Context, proxyAddr string, insecure bool, req t
 	// DELETE IN 10.0.0 (zmb3)
 	return auth.UnmarshalLegacyCerts(resp.Bytes())
 }
+
+// GetWebConfig is used by teleterm to fetch webconfig.js from proxies
+func GetWebConfig(ctx context.Context, proxyAddr string, insecure bool) (*webclient.WebConfig, error) {
+	clt, _, err := initClient(proxyAddr, insecure, nil)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	response, err := clt.Get(ctx, clt.Endpoint("web", "config.js"), url.Values{})
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	body, err := io.ReadAll(response.Reader())
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	// WebConfig is served as JS file where GRV_CONFIG is a global object name
+	text := bytes.TrimSuffix(bytes.Replace(body, []byte("var GRV_CONFIG = "), []byte(""), 1), []byte(";"))
+
+	cfg := webclient.WebConfig{}
+	if err := json.Unmarshal(text, &cfg); err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	return &cfg, nil
+}

lib/teleterm/api/proto/buf.yaml

@@ -0,0 +1,9 @@
+version: v1
+lint:
+  use:
+    - DEFAULT
+  except:
+    - RPC_RESPONSE_STANDARD_NAME
+breaking:
+  use:
+    - FILE