Connectivity diagnostic with VC/ESXi from appliance

cmd/vic-init/toolbox.go

@@ -25,10 +25,11 @@ import (
 	"os/exec"
 	"strings"
 
-	log "github.com/Sirupsen/logrus"
-
 	"github.com/vmware/vic/pkg/trace"
+	"github.com/vmware/vic/pkg/vsphere/diag"
 	"github.com/vmware/vic/pkg/vsphere/toolbox"
+
+	log "github.com/Sirupsen/logrus"
 )
 
 // startCommand is the switch for the synthetic commands that are permitted within the appliance.
@@ -44,6 +45,8 @@ func startCommand(r *toolbox.VixMsgStartProgramRequest) (int, error) {
 		return -1, enableSSH(r.Arguments)
 	case "passwd":
 		return -1, passwd(r.Arguments)
+	case "test-vc-api":
+		return diag.CheckAPIAvailability(r.Arguments), nil
 	default:
 		return -1, fmt.Errorf("unknown command %q", r.ProgramPath)
 	}

cmd/vic-machine/create/create.go

@@ -15,7 +15,9 @@
 package create
 
 import (
+	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding"
 	"fmt"
 	"io/ioutil"
@@ -28,9 +30,7 @@ import (
 	"time"
 
 	log "github.com/Sirupsen/logrus"
-
 	"github.com/urfave/cli"
-
 	"github.com/vmware/vic/lib/install/data"
 	"github.com/vmware/vic/lib/install/management"
 	"github.com/vmware/vic/lib/install/validate"
@@ -39,10 +39,7 @@ import (
 	"github.com/vmware/vic/pkg/flags"
 	"github.com/vmware/vic/pkg/ip"
 	"github.com/vmware/vic/pkg/trace"
-
-	"crypto/x509"
-
-	"golang.org/x/net/context"
+	"github.com/vmware/vic/pkg/vsphere/diag"
 )
 
 const (
@@ -1195,9 +1192,42 @@ func (c *Create) Run(cliContext *cli.Context) (err error) {
 		return err
 	}
 
+	// vic-init will try to reach out to the vSphere target.
+	log.Info("Checking VCH connectivity with vSphere target")
+	vch, err := executor.NewVCHFromComputePath(c.Data.ComputeResourcePath, c.Data.DisplayName, validator)
+	if err != nil {
+		executor.CollectDiagnosticLogs()
+		log.Errorf("Failed to get Virtual Container Host %s", c.Data.DisplayName)
+		log.Error(err)
+		return errors.New("Running diagnostics failed.")
+	}
+
+	// Checking access to vSphere API
+	cd, err := executor.CheckAccessToVCAPI(ctx, vch, vchConfig.Target)
+	code := int(cd)
+	if err != nil {
+		log.Errorf("Failed to access target vSphere API %s: %v", vchConfig.Target, err)
+		executor.CollectDiagnosticLogs()
+		return fmt.Errorf("Could not run vSphere API diagnostic on VCH")
+	}
+
+	const apiTestTxt = "vSphere API Test:"
+	// In case of fatal error, log error and exist.
+	if code >= diag.StatusCodeFatalThreshold {
+		log.Errorf("%s %s %s", apiTestTxt, vchConfig.Target, diag.UserReadableVCAPITestDescription(code))
+		executor.CollectDiagnosticLogs()
+		return fmt.Errorf("Access to vSphere target from VCH failed")
+	}
+
+	// In case of non fatal error, log an error on warning level.
+	if code > 0 {
+		log.Warningf("%s %s %s", apiTestTxt, vchConfig.Target, diag.UserReadableVCAPITestDescription(code))
+	} else {
+		log.Infof("%s %s %s", apiTestTxt, vchConfig.Target, diag.UserReadableVCAPITestDescription(code))
+	}
+
 	// check the docker endpoint is responsive
 	if err = executor.CheckDockerAPI(vchConfig, c.clientCert); err != nil {
-
 		executor.CollectDiagnosticLogs()
 		return err
 	}

lib/install/management/dispatcher.go

@@ -15,6 +15,7 @@
 package management
 
 import (
+	"context"
 	"crypto/x509"
 	"errors"
 	"fmt"
@@ -23,9 +24,9 @@ import (
 	"os"
 	"strings"
 
-	log "github.com/Sirupsen/logrus"
-
+	"github.com/vmware/govmomi/guest"
 	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/types"
 	"github.com/vmware/vic/lib/config"
 	"github.com/vmware/vic/pkg/trace"
 	"github.com/vmware/vic/pkg/vsphere/compute"
@@ -34,7 +35,7 @@ import (
 	"github.com/vmware/vic/pkg/vsphere/session"
 	"github.com/vmware/vic/pkg/vsphere/vm"
 
-	"golang.org/x/net/context"
+	log "github.com/Sirupsen/logrus"
 )
 
 type Dispatcher struct {
@@ -218,6 +219,43 @@ func (d *Dispatcher) CollectDiagnosticLogs() {
 	}
 }
 
+func (d *Dispatcher) opManager(ctx context.Context, vch *vm.VirtualMachine) (*guest.ProcessManager, error) {
+	state, err := vch.PowerState(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get appliance power state, service might not be available at this moment.")
+	}
+	if state != types.VirtualMachinePowerStatePoweredOn {
+		return nil, fmt.Errorf("VCH appliance is not powered on, state %s", state)
+	}
+
+	running, err := vch.IsToolsRunning(ctx)
+	if err != nil || !running {
+		return nil, errors.New("Tools are not running in the appliance, unable to continue")
+	}
+
+	manager := guest.NewOperationsManager(d.session.Client.Client, vch.Reference())
+	processManager, err := manager.ProcessManager(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to manage processes in appliance VM: %s", err)
+	}
+	return processManager, nil
+}
+
+func (d *Dispatcher) CheckAccessToVCAPI(ctx context.Context, vch *vm.VirtualMachine, target string) (int64, error) {
+	pm, err := d.opManager(ctx, vch)
+	if err != nil {
+		return -1, err
+	}
+	auth := types.NamePasswordAuthentication{}
+	spec := types.GuestProgramSpec{
+		ProgramPath:      "test-vc-api",
+		Arguments:        target,
+		WorkingDirectory: "/",
+		EnvVariables:     []string{},
+	}
+	return pm.StartProgram(ctx, &auth, &spec)
+}
+
 // given a set of IP addresses this will determine what address, if any, can be used to
 // connect to the host certificate
 // if none can be found, will return empty string and an err

pkg/vsphere/diag/diag.go

@@ -0,0 +1,149 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package diag
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/vmware/vic/pkg/trace"
+)
+
+// StatusCodeFatalThreshold defines a threshold after which all codes can be treated as fatal.
+const StatusCodeFatalThreshold = 64
+
+const (
+	// VCStatusOK vSphere API is available.
+	VCStatusOK = 0
+	// VCStatusInvalidURL Provided vSphere API URL is wrong.
+	VCStatusInvalidURL = 64
+	// VCStatusErrorQuery Error happened trying to query vSphere API
+	VCStatusErrorQuery = 65
+	// VCStatusErrorResponse Received response doesn't contain expected data.
+	VCStatusErrorResponse = 66
+	// VCStatusIncorrectResponse Received in case if returned data from server is different from expected.
+	VCStatusIncorrectResponse = 67
+	// VCStatusNotXML Received response is not XML
+	VCStatusNotXML = 68
+	// VCStatusUnknownHost is returned in case if DNS failed to resolve name.
+	VCStatusUnknownHost = 69
+	// VCStatusHostIsNotReachable
+	VCStatusHostIsNotReachable = 70
+)
+
+// UserReadableVCAPITestDescription convert API test code into user readable text
+func UserReadableVCAPITestDescription(code int) string {
+	switch code {
+	case VCStatusOK:
+		return "vSphere API target responds as expected"
+	case VCStatusInvalidURL:
+		return "vSphere API target url is invalid"
+	case VCStatusErrorQuery:
+		return "vSphere API target failed to respond to the query"
+	case VCStatusIncorrectResponse:
+		return "vSphere API target returns unexpected response"
+	case VCStatusErrorResponse:
+		return "vSphere API target returns error"
+	case VCStatusNotXML:
+		return "vSphere API target returns non XML response"
+	case VCStatusUnknownHost:
+		return "vSphere API target can not be resolved from VCH"
+	case VCStatusHostIsNotReachable:
+		return "vSphere API target is out of reach. Wrong routing table?"
+	default:
+		return "vSphere API target test returned unknown code"
+	}
+}
+
+// CheckAPIAvailability accesses vSphere API to ensure it is a correct end point that is up and running.
+func CheckAPIAvailability(targetURL string) int {
+	op := trace.NewOperation(context.Background(), "api test")
+	errorCode := VCStatusErrorQuery
+
+	u, err := url.Parse(targetURL)
+	if err != nil {
+		return VCStatusInvalidURL
+	}
+
+	u.Path = "/sdk/vimService.wsdl"
+	apiURL := u.String()
+
+	op.Debugf("Checking access to: %s", apiURL)
+
+	for attempts := 5; errorCode != VCStatusOK && attempts > 0; attempts-- {
+
+		c := http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: true,
+				},
+			},
+			// Is 20 seconds enough to receive any response from vSphere target server?
+			Timeout: time.Second * 20,
+		}
+		errorCode = queryAPI(op, c.Get, apiURL)
+	}
+	return errorCode
+}
+
+func queryAPI(op trace.Operation, getter func(string) (*http.Response, error), apiURL string) int {
+	resp, err := getter(apiURL)
+	if err != nil {
+		errTxt := err.Error()
+		op.Errorf("Query error: %s", err)
+		if strings.Contains(errTxt, "no such host") {
+			return VCStatusUnknownHost
+		}
+		if strings.Contains(errTxt, "no route to host") {
+			return VCStatusHostIsNotReachable
+		}
+		if strings.Contains(errTxt, "host is down") {
+			return VCStatusHostIsNotReachable
+		}
+		return VCStatusErrorQuery
+	}
+
+	data := make([]byte, 65636)
+	n, err := io.ReadFull(resp.Body, data)
+	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+		op.Errorf("Query error: %s", err)
+		return VCStatusErrorResponse
+	}
+	if n >= len(data) {
+		io.Copy(ioutil.Discard, resp.Body)
+	}
+	resp.Body.Close()
+
+	contentType := strings.ToLower(resp.Header.Get("Content-Type"))
+	if !strings.Contains(contentType, "text/xml") {
+		op.Errorf("Unexpected content type %s, should be text/xml", contentType)
+		op.Errorf("Response from the server: %s", string(data))
+		return VCStatusNotXML
+	}
+	// we just want to make sure that response contains something familiar that we could
+	// use as vSphere API marker.
+	if !bytes.Contains(data, []byte("urn:vim25Service")) {
+		op.Errorf("Server response doesn't contain 'urn:vim25Service': %s", string(data))
+		return VCStatusIncorrectResponse
+	}
+	return VCStatusOK
+}