Merge pull request #6 from dell/release-1.4.0

goisilon changes for csi-isilon release 1.4.0

api/api.go

@@ -21,7 +21,10 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+	"fmt"
+	"golang.org/x/crypto/ssh"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -93,6 +96,13 @@ type Client interface {
 		params OrderedValues, headers map[string]string,
 		resp interface{}) error
 
+	// ExecuteSSHCommand sends a command string to execute on oneFS CLI using ssh
+	// Returns data from remote process's standard output and error as *bytes.Buffer, started through ssh
+	// Returns error in case of any error
+	ExecuteSSHCommand(
+		ctx context.Context,
+		command *string) (*bytes.Buffer, *bytes.Buffer, error)
+
 	// APIVersion returns the API version.
 	APIVersion() uint8
 
@@ -159,6 +169,15 @@ type ClientOptions struct {
 	Timeout time.Duration
 }
 
+type SSHReq struct {
+	command, isiIp, user *string
+}
+
+type SSHResp struct {
+	stdout, stderr *bytes.Buffer
+	err error
+}
+
 // New returns a new API client.
 func New(
 	ctx context.Context,
@@ -286,6 +305,57 @@ func (c *client) Delete(
 		ctx, http.MethodDelete, path, id, params, headers, nil, resp)
 }
 
+func (c *client) ExecuteSSHCommand(
+	ctx context.Context,
+	command *string) (*bytes.Buffer, *bytes.Buffer, error) {
+	var bufStdOut, bufStdErr bytes.Buffer
+
+	// Validate URL and fetch host IP
+	if ! strings.HasPrefix(c.hostname, "https://") {
+		return nil, nil, errors.New("invalid endpoint")
+	}
+
+	ipPort := strings.SplitAfter(c.hostname, "https://")
+	if len(ipPort) == 1 {
+		return nil, nil, errors.New("invalid endpoint")
+	}
+
+	isiIp, _, err := net.SplitHostPort(ipPort[1])
+	if err != nil {
+		return nil, nil, err
+	}
+
+	sshReq := &SSHReq{command, &isiIp, &c.username}
+	logSSHRequest(ctx, sshReq)
+
+	config := &ssh.ClientConfig{
+		User: c.username,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(c.password),
+		},
+		HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }),
+	}
+	sshClient, err := ssh.Dial("tcp", net.JoinHostPort(isiIp, "22"), config)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	session, err := sshClient.NewSession()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create session: %s", err)
+	}
+	defer session.Close()
+
+	// Capture command output
+	session.Stdout = &bufStdOut
+	session.Stderr = &bufStdErr
+
+	err = session.Run(*command)
+	resp := &SSHResp{&bufStdOut, &bufStdErr, err}
+	logSSHResponse(ctx, resp)
+	return &bufStdOut, &bufStdErr, err
+}
+
 func (c *client) Do(
 	ctx context.Context,
 	method, path, id string,

api/api_logging.go

@@ -1,4 +1,4 @@
-/* 
+/*
  Copyright (c) 2019 Dell Inc, or its subsidiaries.
 
  Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,19 +12,19 @@
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
- */
+*/
 package api
 
 import (
 	"bufio"
 	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"net/http"
 	"net/http/httputil"
 	"strings"
-	"encoding/base64"
 
 	log "github.com/akutz/gournal"
 )
@@ -38,7 +38,7 @@ func logRequest(ctx context.Context, w io.Writer, req *http.Request, verbose Ver
 	fmt.Fprint(w, "    -------------------------- ")
 	fmt.Fprint(w, "GOISILON HTTP REQUEST")
 	fmt.Fprintln(w, " -------------------------")
-	
+
 	switch verbose {
 	case Verbose_Low:
 		//minimal logging, i.e. print request line only
@@ -80,6 +80,42 @@ func logResponse(ctx context.Context, res *http.Response, verbose VerboseType) {
 	log.Debug(ctx, w.String())
 }
 
+func logSSHRequest(ctx context.Context, req *SSHReq) {
+	w := &bytes.Buffer{}
+	fmt.Fprintln(w, "")
+	fmt.Fprint(w, "    -------------------------- ")
+	fmt.Fprint(w, "GOISILON SSH REQUEST")
+	fmt.Fprintln(w, " -------------------------")
+
+	fmt.Fprintf(w, "    Command      : %s\n", *req.command)
+	fmt.Fprintf(w, "    Host         : %s\n", *req.isiIp)
+	fmt.Fprintf(w, "    Authorization: %s:******\n", *req.user)
+
+	log.Debug(ctx, w.String())
+}
+
+func logSSHResponse(ctx context.Context, resp *SSHResp) {
+	w := &bytes.Buffer{}
+
+	fmt.Fprintln(w)
+	fmt.Fprint(w, "    -------------------------- ")
+	fmt.Fprint(w, "GOISILON SSH RESPONSE")
+	fmt.Fprintln(w, " -------------------------")
+
+	fmt.Fprintf(w, "    Command Output: %s\n", resp.stdout)
+	fmt.Fprintf(w, "    Command Error : %s\n", resp.stderr)
+
+	var exitStatus interface{}
+	if resp.err != nil {
+		exitStatus = resp.err
+	} else {
+		exitStatus = 0
+	}
+
+	fmt.Fprintf(w, "    Exit Status   : %v\n", exitStatus)
+	log.Debug(ctx, w.String())
+}
+
 // WriteIndentedN indents all lines n spaces.
 func WriteIndentedN(w io.Writer, b []byte, n int) error {
 	s := bufio.NewScanner(bytes.NewReader(b))
@@ -112,11 +148,11 @@ func WriteIndented(w io.Writer, b []byte) error {
 	return WriteIndentedN(w, b, 4)
 }
 
-func decodeAuthorization(buf []byte) []byte{
+func decodeAuthorization(buf []byte) []byte {
 	sc := bufio.NewScanner(bytes.NewReader(buf))
 	ou := &bytes.Buffer{}
 	var l string
-	
+
 	for sc.Scan() {
 		l = sc.Text()
 		if strings.Contains(l, "Authorization") {
@@ -129,6 +165,6 @@ func decodeAuthorization(buf []byte) []byte{
 		}
 		fmt.Fprintln(ou, l)
 	}
-	
+
 	return ou.Bytes()
-}
+}

api/v1/api_v1_snapshots.go

@@ -16,10 +16,12 @@
 package v1
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"path"
+	"strings"
 
 	"github.com/dell/goisilon/api"
 )
@@ -124,18 +126,17 @@ func CopyIsiSnapshot(
 func CopyIsiSnapshotWithIsiPath(
 	ctx context.Context,
 	client api.Client,
-	isiPath, sourceSnapshotName, sourceVolume, destinationName string) (resp *IsiVolume, err error) {
-	// PAPI calls: PUT https://1.2.3.4:8080/namespace/path/to/volumes/destination_volume_name?merge=True
-	//             x-isi-ifs-copy-source: /path/to/snapshot/volumes/source_volume_name
-	headers := map[string]string{
-		"x-isi-ifs-copy-source": path.Join(
-			"/",
-			GetRealVolumeSnapshotPathWithIsiPath(isiPath, sourceSnapshotName),
-			sourceVolume),
-	}
-	// copy the volume
-	err = client.Put(ctx, GetRealNamespacePathWithIsiPath(isiPath), destinationName, mergeQS, headers, nil, &resp)
-	return resp, err
+	isiPath, sourceSnapshotName, sourceVolume, destinationName string) (*bytes.Buffer, *bytes.Buffer, error) {
+	// SSH call: ssh "cp <source_path> <destination path>
+	// copies contents from source path(i.e., snapshot directory) recursively to destination volume preserving file ownerships
+	snapshotPath := path.Join(GetRealVolumeSnapshotPathWithIsiPath(isiPath, sourceSnapshotName), sourceVolume)
+	parts := strings.SplitN(snapshotPath, namespacePath, 2)
+	srcPath := parts[1]
+	srcPath = srcPath + "/"
+	dstPath := path.Join(isiPath, destinationName)
+	cmd := strings.Join([]string{"cp -p -R -f", srcPath, dstPath}, " ")
+
+	return client.ExecuteSSHCommand(ctx, &cmd)
 }
 
 // RemoveIsiSnapshot deletes a snapshot from the cluster

api/v1/api_v1_volumes.go

@@ -16,8 +16,10 @@
 package v1
 
 import (
+	"bytes"
 	"context"
 	"path"
+	"strings"
 
 	"github.com/dell/goisilon/api"
 )
@@ -64,7 +66,6 @@ func CreateIsiVolumeWithIsiPath(
 	ctx context.Context,
 	client api.Client,
 	isiPath, name string) (resp *getIsiVolumesResp, err error) {
-
 	return CreateIsiVolumeWithACLAndIsiPath(ctx, client, isiPath, name, defaultACL)
 }
 
@@ -286,25 +287,15 @@ func CopyIsiVolume(
 func CopyIsiVolumeWithIsiPath(
 	ctx context.Context,
 	client api.Client,
-	isiPath, sourceName, destinationName string) (resp *getIsiVolumesResp, err error) {
-	// PAPI calls: PUT https://1.2.3.4:8080/namespace/path/to/volumes/destination_volume_name?merge=True
-	//             x-isi-ifs-copy-source: /path/to/volumes/source_volume_name
-
-	// copy the volume
-	err = client.Put(
-		ctx,
-		GetRealNamespacePathWithIsiPath(isiPath),
-		destinationName,
-		mergeQS,
-		map[string]string{
-			"x-isi-ifs-copy-source": path.Join(
-				"/",
-				GetRealNamespacePathWithIsiPath(isiPath),
-				sourceName),
-		},
-		nil,
-		&resp)
-	return resp, err
+	isiPath, sourceName, destinationName string) (*bytes.Buffer, *bytes.Buffer, error) {
+	// SSH call: ssh "cp <source_path> <destination path>
+	// copies contents from source volume path recursively to destination volume preserving file ownerships
+	srcPath := path.Join(isiPath, sourceName)
+	srcPath = srcPath + "/"
+	dstPath := path.Join(isiPath, destinationName)
+
+	cmd := strings.Join([]string{"cp -p -R -f", srcPath, dstPath}, " ")
+	return client.ExecuteSSHCommand(ctx, &cmd)
 }
 
 // GetIsiVolumeWithSize lists size of all the children files and subfolders in a directory

api/v5/api_v5_quotas.go

@@ -1,4 +1,4 @@
-/* 
+/*
  Copyright (c) 2019 Dell Inc, or its subsidiaries.
 
  Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,16 +12,16 @@
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
- */
+*/
 package v5
 
 import (
 	"context"
 	"errors"
 	"fmt"
 
-	log "github.com/sirupsen/logrus"
 	"github.com/dell/goisilon/api"
+	log "github.com/sirupsen/logrus"
 )
 
 const (

goisilon_test.go

@@ -1,4 +1,4 @@
-/* 
+/*
  Copyright (c) 2019 Dell Inc, or its subsidiaries.
 
  Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,15 +12,15 @@
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
- */
+*/
 package goisilon
 
 import (
 	"context"
 	"flag"
 	"os"
+	"strconv"
 	"testing"
-        "strconv"
 
 	log "github.com/akutz/gournal"
 	glogrus "github.com/akutz/gournal/logrus"
@@ -46,9 +46,9 @@ func init() {
 }
 
 func skipTest(t *testing.T) {
-  if os.Getenv("GOISILON_SKIP_TEST") != "" {
-    t.Skip("Skipping testing in CI environment")
-  }
+	if os.Getenv("GOISILON_SKIP_TEST") != "" {
+		t.Skip("Skipping testing in CI environment")
+	}
 }
 
 func TestMain(m *testing.M) {
@@ -60,10 +60,10 @@ func TestMain(m *testing.M) {
 			log.DebugLevel)
 	}
 
-        goInsecure, err := strconv.ParseBool(os.Getenv("GOISILON_INSECURE"))
-        if err != nil {
-            log.WithError(err).Panic(defaultCtx, "error fetching environment variable GOISILON_INSECURE")
-        }
+	goInsecure, err := strconv.ParseBool(os.Getenv("GOISILON_INSECURE"))
+	if err != nil {
+		log.WithError(err).Panic(defaultCtx, "error fetching environment variable GOISILON_INSECURE")
+	}
 
 	client, err = NewClientWithArgs(
 		defaultCtx,