Adds dev.odo.push.file attribute support for pushing only mentioned f…

…iles

pkg/devfile/adapters/common/types.go

@@ -52,6 +52,7 @@ type SyncParameters struct {
 	CompInfo        ComponentInfo
 	PodChanged      bool
 	ComponentExists bool
+	Files           map[string]string
 }
 
 // ComponentInfo is a struct that holds information about a component i.e.; pod name, container name, and source mount (if applicable)

pkg/devfile/adapters/common/utils.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"os"
+	"path/filepath"
 	"strings"
 
 	"k8s.io/klog"
@@ -241,3 +242,16 @@ func GetCommandsMap(commands []devfilev1.Command) map[string]devfilev1.Command {
 	}
 	return commandMap
 }
+
+func GetSyncFilesFromAttributes(commandsMap PushCommandsMap) map[string]string {
+	syncMap := make(map[string]string)
+	if value, ok := commandsMap[devfilev1.RunCommandGroupKind]; ok {
+		for key, value := range value.Attributes.Strings(nil) {
+			if strings.HasPrefix(key, "dev.odo.push.file:") {
+				localValue := strings.ReplaceAll(key, "dev.odo.push.file:", "")
+				syncMap[filepath.Clean(localValue)] = filepath.Clean(value)
+			}
+		}
+	}
+	return syncMap
+}

pkg/devfile/adapters/kubernetes/component/adapter.go

@@ -229,10 +229,20 @@ func (a Adapter) Push(parameters common.PushParameters) (err error) {
 		CompInfo:        compInfo,
 		ComponentExists: componentExists,
 		PodChanged:      podChanged,
+		Files:           common.GetSyncFilesFromAttributes(pushDevfileCommands),
 	}
-	execRequired, err := syncAdapter.SyncFiles(syncParams)
-	if err != nil {
-		return errors.Wrapf(err, "Failed to sync to component with name %s", a.ComponentName)
+
+	var execRequired bool
+	if len(syncParams.Files) > 0 {
+		execRequired, err = syncAdapter.SyncFileWithRemote(syncParams)
+		if err != nil {
+			return errors.Wrapf(err, "Failed to sync to component with name %s", a.ComponentName)
+		}
+	} else {
+		execRequired, err = syncAdapter.SyncFiles(syncParams)
+		if err != nil {
+			return errors.Wrapf(err, "Failed to sync to component with name %s", a.ComponentName)
+		}
 	}
 
 	// PostStart events from the devfile will only be executed when the component

pkg/sync/adapter.go

@@ -123,7 +123,7 @@ func (a Adapter) SyncFiles(syncParameters common.SyncParameters) (isPushRequired
 		}
 
 		// Run the indexer and find the modified/added/deleted/renamed files
-		ret, err = util.RunIndexer(pushParameters.Path, absIgnoreRules)
+		ret, err = util.RunIndexWithRemote(pushParameters.Path, absIgnoreRules, map[string]string{})
 		s.End(true)
 
 		if err != nil {
@@ -140,10 +140,11 @@ func (a Adapter) SyncFiles(syncParameters common.SyncParameters) (isPushRequired
 
 		// Remove the relative file directory from the list of deleted files
 		// in order to make the changes correctly within the Kubernetes pod
-		deletedFiles, err = util.RemoveRelativePathFromFiles(filesDeletedFiltered, pushParameters.Path)
-		if err != nil {
-			return false, errors.Wrap(err, "unable to remove relative path from list of changed/deleted files")
-		}
+		//deletedFiles, err = util.RemoveRelativePathFromFiles(filesDeletedFiltered, pushParameters.Path)
+		//if err != nil {
+		//	return false, errors.Wrap(err, "unable to remove relative path from list of changed/deleted files")
+		//}
+		deletedFiles = append(deletedFiles, ret.RemoteDeleted...)
 		klog.V(4).Infof("List of files to be deleted: +%v", deletedFiles)
 		changedFiles = filesChangedFiltered
 		klog.V(4).Infof("List of files changed: +%v", changedFiles)
@@ -173,6 +174,37 @@ func (a Adapter) SyncFiles(syncParameters common.SyncParameters) (isPushRequired
 	return true, nil
 }
 
+func (a Adapter) SyncFileWithRemote(syncParameters common.SyncParameters) (isPushRequired bool, err error) {
+	ret, err := util.RunIndexWithRemote(syncParameters.PushParams.Path, syncParameters.PushParams.IgnoredFiles, syncParameters.Files)
+	if err != nil {
+		return false, err
+	}
+
+	if len(ret.FilesChanged) == 0 && len(ret.FilesDeleted) == 0 {
+		return false, nil
+	}
+
+	ret.FilesDeleted = append(ret.FilesDeleted, ret.RemoteDeleted...)
+
+	err = a.pushLocalWithRemote(syncParameters.PushParams.Path,
+		ret.FilesChanged,
+		ret.FilesDeleted,
+		syncParameters.Files,
+		false,
+		util.GetAbsGlobExps(syncParameters.PushParams.Path, syncParameters.PushParams.IgnoredFiles),
+		syncParameters.CompInfo,
+		ret,
+	)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to sync to component with name %s", a.ComponentName)
+	}
+	err = util.WriteFile(ret.NewFileMap, ret.ResolvedPath)
+	if err != nil {
+		return false, errors.Wrapf(err, "Failed to write file")
+	}
+	return true, nil
+}
+
 // pushLocal syncs source code from the user's disk to the component
 func (a Adapter) pushLocal(path string, files []string, delFiles []string, isForcePush bool, globExps []string, compInfo common.ComponentInfo) error {
 	klog.V(4).Infof("Push: componentName: %s, path: %s, files: %s, delFiles: %s, isForcePush: %+v", a.ComponentName, path, files, delFiles, isForcePush)
@@ -232,6 +264,65 @@ func (a Adapter) pushLocal(path string, files []string, delFiles []string, isFor
 	return nil
 }
 
+// pushLocal syncs source code from the user's disk to the component
+func (a Adapter) pushLocalWithRemote(path string, files []string, delFiles []string, remoteLocations map[string]string, isForcePush bool, globExps []string, compInfo common.ComponentInfo, ret util.IndexerRet) error {
+	klog.V(4).Infof("Push: componentName: %s, path: %s, files: %s, delFiles: %s, isForcePush: %+v", a.ComponentName, path, files, delFiles, isForcePush)
+
+	// Edge case: check to see that the path is NOT empty.
+	emptyDir, err := util.IsEmpty(path)
+	if err != nil {
+		return errors.Wrapf(err, "unable to check directory: %s", path)
+	} else if emptyDir {
+		return errors.New(fmt.Sprintf("directory/file %s is empty", path))
+	}
+
+	// Sync the files to the pod
+	s := log.Spinner("Syncing files to the component")
+	defer s.End(false)
+
+	syncFolder := compInfo.SyncFolder
+
+	if syncFolder != generator.DevfileSourceVolumeMount {
+		// Need to make sure the folder already exists on the component or else sync will fail
+		klog.V(4).Infof("Creating %s on the remote container if it doesn't already exist", syncFolder)
+		cmdArr := getCmdToCreateSyncFolder(syncFolder)
+
+		err = common.ExecuteCommand(a.Client, compInfo, cmdArr, false, nil, nil)
+		if err != nil {
+			return err
+		}
+	}
+	// If there were any files deleted locally, delete them remotely too.
+	if len(delFiles) > 0 {
+		cmdArr := getCmdToDeleteFiles(delFiles, syncFolder)
+
+		err = common.ExecuteCommand(a.Client, compInfo, cmdArr, false, nil, nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !isForcePush {
+		if len(files) == 0 && len(delFiles) == 0 {
+			// nothing to push
+			s.End(true)
+			return nil
+		}
+	}
+
+	if isForcePush || len(files) > 0 {
+		klog.V(4).Infof("Copying files %s to pod", strings.Join(files, " "))
+		err = CopyFileWithRemote(a.Client, path, compInfo, syncFolder, files, globExps, remoteLocations, ret)
+		if err != nil {
+			s.End(false)
+			return errors.Wrap(err, "unable push files to pod")
+		}
+	}
+	s.End(true)
+
+	return nil
+}
+
 // updateIndexWithWatchChanges uses the pushParameters.WatchDeletedFiles and pushParamters.WatchFiles to update
 // the existing index file; the index file is required to exist when this function is called.
 func updateIndexWithWatchChanges(pushParameters common.PushParameters) error {

pkg/sync/sync.go

@@ -53,6 +53,40 @@ func CopyFile(client SyncClient, localPath string, compInfo common.ComponentInfo
 	return nil
 }
 
+// CopyFile copies localPath directory or list of files in copyFiles list to the directory in running Pod.
+// copyFiles is list of changed files captured during `odo watch` as well as binary file path
+// During copying binary components, localPath represent base directory path to binary and copyFiles contains path of binary
+// During copying local source components, localPath represent base directory path whereas copyFiles is empty
+// During `odo watch`, localPath represent base directory path whereas copyFiles contains list of changed Files
+func CopyFileWithRemote(client SyncClient, localPath string, compInfo common.ComponentInfo, targetPath string, copyFiles []string, globExps []string, remoteLocations map[string]string, ret util.IndexerRet) error {
+
+	// Destination is set to "ToSlash" as all containers being ran within OpenShift / S2I are all
+	// Linux based and thus: "\opt\app-root\src" would not work correctly.
+	dest := filepath.ToSlash(filepath.Join(targetPath, filepath.Base(localPath)))
+	targetPath = filepath.ToSlash(targetPath)
+
+	klog.V(4).Infof("CopyFile arguments: localPath %s, dest %s, targetPath %s, copyFiles %s, globalExps %s", localPath, dest, targetPath, copyFiles, globExps)
+	reader, writer := io.Pipe()
+	// inspired from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/cmd/cp.go#L235
+	go func() {
+		defer writer.Close()
+
+		err := makeTarWithRemote(localPath, dest, writer, copyFiles, globExps, remoteLocations, ret)
+		if err != nil {
+			log.Errorf("Error while creating tar: %#v", err)
+			os.Exit(1)
+		}
+
+	}()
+
+	err := client.ExtractProjectToComponent(compInfo, targetPath, reader)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // checkFileExist check if given file exists or not
 func checkFileExist(fileName string) bool {
 	_, err := os.Stat(fileName)
@@ -116,6 +150,73 @@ func makeTar(srcPath, destPath string, writer io.Writer, files []string, globExp
 	return nil
 }
 
+// makeTar function is copied from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/cmd/cp.go#L309
+// srcPath is ignored if files is set
+func makeTarWithRemote(srcPath, destPath string, writer io.Writer, files []string, globExps []string, remoteLocations map[string]string, ret util.IndexerRet) error {
+	// TODO: use compression here?
+	tarWriter := taro.NewWriter(writer)
+	defer tarWriter.Close()
+	srcPath = filepath.Clean(srcPath)
+
+	// "ToSlash" is used as all containers within OpenShift are Linux based
+	// and thus \opt\app-root\src would be an invalid path. Backward slashes
+	// are converted to forward.
+	destPath = filepath.ToSlash(filepath.Clean(destPath))
+	uniquePaths := make(map[string]bool)
+	klog.V(4).Infof("makeTar arguments: srcPath: %s, destPath: %s, files: %+v", srcPath, destPath, files)
+	if len(files) != 0 {
+		//watchTar
+		for _, fileName := range files {
+
+			if _, ok := uniquePaths[fileName]; ok {
+				continue
+			} else {
+				uniquePaths[fileName] = true
+			}
+
+			if checkFileExist(fileName) {
+				// Fetch path of source file relative to that of source base path so that it can be passed to recursiveTar
+				// which uses path relative to base path for taro header to correctly identify file location when untarred
+
+				// Yes, now that the file exists, now we need to get the absolute path.. if we don't, then when we pass in:
+				// 'odo push --context foobar' instead of 'odo push --context ~/foobar' it will NOT work..
+				fileAbsolutePath, err := util.GetAbsPath(fileName)
+				if err != nil {
+					return err
+				}
+				klog.V(4).Infof("Got abs path: %s", fileAbsolutePath)
+				klog.V(4).Infof("Making %s relative to %s", srcPath, fileAbsolutePath)
+
+				// We use "FromSlash" to make this OS-based (Windows uses \, Linux & macOS use /)
+				// we get the relative path by joining the two
+				destFile, err := filepath.Rel(filepath.FromSlash(srcPath), filepath.FromSlash(fileAbsolutePath))
+				if err != nil {
+					return err
+				}
+
+				// Now we get the source file and join it to the base directory.
+				srcFile := filepath.Join(filepath.Base(srcPath), destFile)
+
+				if value, ok := ret.NewFileMap[destFile]; ok {
+					destFile = value.RemoteAttribute
+				}
+
+				klog.V(4).Infof("makeTar srcFile: %s", srcFile)
+				klog.V(4).Infof("makeTar destFile: %s", destFile)
+
+				// The file could be a regular file or even a folder, so use recursiveTar which handles symlinks, regular files and folders
+				err = linearTar(srcPath, filepath.Dir(srcPath), srcFile, filepath.Dir(destPath), destFile, tarWriter, globExps, remoteLocations)
+				if err != nil {
+					return err
+				}
+
+			}
+		}
+	}
+
+	return nil
+}
+
 // recursiveTar function is copied from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/cmd/cp.go#L319
 func recursiveTar(srcBase, srcFile, destBase, destFile string, tw *taro.Writer, globExps []string) error {
 	klog.V(4).Infof("recursiveTar arguments: srcBase: %s, srcFile: %s, destBase: %s, destFile: %s", srcBase, srcFile, destBase, destFile)
@@ -211,3 +312,86 @@ func recursiveTar(srcBase, srcFile, destBase, destFile string, tw *taro.Writer,
 
 	return nil
 }
+
+// recursiveTar function is copied from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/cmd/cp.go#L319
+func linearTar(directory, srcBase, srcFile, destBase, destFile string, tw *taro.Writer, globExps []string, remoteDirectories map[string]string) error {
+	klog.V(4).Infof("recursiveTar arguments: srcBase: %s, srcFile: %s, destBase: %s, destFile: %s", srcBase, srcFile, destBase, destFile)
+
+	// The destination is a LINUX container and thus we *must* use ToSlash in order
+	// to get the copying over done correctly..
+	destBase = filepath.ToSlash(destBase)
+	destFile = filepath.ToSlash(destFile)
+	klog.V(4).Infof("Corrected destinations: base: %s file: %s", destBase, destFile)
+
+	joinedPath := filepath.Join(srcBase, srcFile)
+
+	stat, err := os.Lstat(joinedPath)
+	if err != nil {
+		return err
+	}
+
+	joinedRelPath, err := filepath.Rel(directory, joinedPath)
+	if err != nil {
+		return err
+	}
+
+	if stat.IsDir() {
+		files, err := ioutil.ReadDir(joinedPath)
+		if err != nil {
+			return err
+		}
+		if len(files) == 0 {
+			//case empty directory
+			hdr, _ := taro.FileInfoHeader(stat, joinedPath)
+			hdr.Name = destFile
+			if err := tw.WriteHeader(hdr); err != nil {
+				return err
+			}
+		}
+		return nil
+	} else if stat.Mode()&os.ModeSymlink != 0 {
+		//case soft link
+		hdr, _ := taro.FileInfoHeader(stat, joinedPath)
+		target, err := os.Readlink(joinedPath)
+		if err != nil {
+			return err
+		}
+
+		hdr.Linkname = target
+		if value, ok := remoteDirectories[joinedRelPath]; ok {
+			destFile = value
+		}
+		hdr.Name = destFile
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+	} else {
+		//case regular file or other file type like pipe
+		hdr, err := taro.FileInfoHeader(stat, joinedPath)
+		if err != nil {
+			return err
+		}
+		if value, ok := remoteDirectories[joinedRelPath]; ok {
+			destFile = value
+		}
+		hdr.Name = destFile
+
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+
+		f, err := os.Open(joinedPath)
+		if err != nil {
+			return err
+		}
+		defer f.Close() // #nosec G307
+
+		if _, err := io.Copy(tw, f); err != nil {
+			return err
+		}
+
+		return f.Close()
+	}
+
+	return nil
+}