Merge pull request #19 from lomorage/dev

add e2e test script and docker support

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@v4
       - run: make lint
 
-  install:
+  unit-test:
     runs-on: ubuntu-22.04
     container:
       image: lomorage/lomo-backup:1.0
@@ -28,20 +28,25 @@ jobs:
       GOFLAGS: "-buildvcs=false"
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: 'true'
-      - run: make install
+      - run: make unit-tests
 
-  test:
+  e2e-test:
     runs-on: ubuntu-22.04
     container:
       image: lomorage/lomo-backup:1.0
     env:
       PATH: /bin:/usr/bin:/usr/local/bin:/usr/local/go/bin
       GOBIN: /usr/local/bin
       GOFLAGS: "-buildvcs=false"
+      WORKDIR: /go/src/github.com/lomorage/lomo-backup
     steps:
+      - run: apt-get update && apt-get install -y git
       - uses: actions/checkout@v4
         with:
-          submodules: 'true'
-      - run: make test
+          submodules: true
+          fetch-depth: 0
+      - run: apt install -y tree rsync
+      - run: cd test/data; find . | while read -r file; do git log -1 --format="%ad" --date=iso "$file" | xargs -t -I{} touch -d {} "$file"; done
+      - run: mkdir -p $WORKDIR; rsync -avz * $WORKDIR/; cd $WORKDIR; tree --si -D test/data/
+      - run: cd $WORKDIR; make install
+      - run: cd $WORKDIR; make e2e-tests

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/data"]
+	path = test/data
+	url = https://github.com/lomorage/homepage

Makefile

@@ -2,6 +2,10 @@
 
 SHELL=/bin/bash # Use bash syntax
 GOPATH=/go
+USER_ID := $(shell id -u)
+USER_NAME := $(shell whoami)
+GROUP_ID := $(shell id -g)
+GROUP_NAME := $(shell id -gn)
 
 vendor:
 	go mod vendor
@@ -12,18 +16,32 @@ lint:
 install:
 	go install -v ./cmd/...
 
+localstack-tarball:
+	docker pull localstack/localstack:3.4.0
+	docker save -o localstack_3.4.0.tar localstack/localstack:3.4.0
+	gzip localstack_3.4.0.tar
+
 dev-container:
-	docker build --tag "lomorage/lomo-backup:1.0" -f dockerfiles/dev-image .
+	docker build --tag "lomorage/lomo-backup:build-stage1" -f dockerfiles/dev-image .
+	docker build --tag "lomorage/lomo-backup:build-stage2" -f dockerfiles/dev-image-load .
+	docker rm -f lomo-backup-build
+	docker run --name lomo-backup-build --privileged --net host -it "lomorage/lomo-backup:build-stage2" hostname
+	docker commit lomo-backup-build lomorage/lomo-backup:1.0
 
 dev:
-	docker build --tag "lomorage/lomo-backup" -f dockerfiles/dev-run .
+	docker build \
+	    --build-arg USER_ID=$(USER_ID) --build-arg USER_NAME=$(USER_NAME) \
+	    --build-arg GROUP_ID=$(GROUP_ID) --build-arg GROUP_NAME=$(GROUP_NAME) \
+		--tag "lomorage/lomo-backup" -f dockerfiles/dev-run .
 	docker rm -f lomo-backup
-	docker run \
-		--name lomo-backup --hostname lomo-backup \
-		--privileged --cap-add=ALL -v /dev:/dev -v /lib/modules:/lib/modules \
+	docker run --rm \
+		--name lomo-backup --hostname lomo-backup -v ./dockerfiles/hosts:/etc/hosts \
+		--privileged --net host --dns-search local \
 		-v "${PWD}:/go/src/github.com/lomorage/lomo-backup" \
-		--net host --dns-search local \
 		-it "lomorage/lomo-backup" -d bash
 
-test:
-	go test -v ./common/
+unit-tests:
+	go list ./common/... | xargs -I {} go test -v {}
+
+e2e-tests: install
+	cd test/scripts; ./e2e.sh

clients/upload.go

@@ -3,6 +3,7 @@ package clients
 import (
 	"context"
 	"io"
+	"os"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -70,6 +71,10 @@ func NewAWSClient(keyID, key, region string) (*AWSClient, error) {
 	if err != nil {
 		return nil, err
 	}
+	if os.Getenv("LOCALSTACK_ENDPOINT") != "" {
+		cfg.S3ForcePathStyle = aws.Bool(true)
+		cfg.Endpoint = aws.String(os.Getenv("LOCALSTACK_ENDPOINT"))
+	}
 	return &AWSClient{region: region, svc: s3.New(sess, cfg)}, nil
 }
 

cmd/lomob/iso.go

@@ -7,14 +7,14 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"text/tabwriter"
 	"time"
 
 	"github.com/diskfs/go-diskfs"
 	"github.com/diskfs/go-diskfs/filesystem"
-	"github.com/djherbis/times"
 	"github.com/lomorage/lomo-backup/common"
 	"github.com/lomorage/lomo-backup/common/datasize"
 	lomohash "github.com/lomorage/lomo-backup/common/hash"
@@ -28,12 +28,20 @@ import (
 var futuretime = time.Date(3000, time.December, 31, 0, 0, 0, 0, time.Now().UTC().Location())
 
 func mkISO(ctx *cli.Context) error {
-	isoSize, err := datasize.ParseString(ctx.String("iso-size"))
+	err := initLogLevel(ctx.GlobalInt("log-level"))
 	if err != nil {
 		return err
 	}
 
-	err = initLogLevel(ctx.GlobalInt("log-level"))
+	debug := ctx.Bool("debug")
+	if debug {
+		err = initLogLevel(int(logrus.DebugLevel))
+		if err != nil {
+			return err
+		}
+	}
+
+	isoSize, err := datasize.ParseString(ctx.String("iso-size"))
 	if err != nil {
 		return err
 	}
@@ -84,7 +92,7 @@ func mkISO(ctx *cli.Context) error {
 				datasize.ByteSize(iso.Size).HR())
 		}
 
-		size, filename, leftFiles, notExistFiles, err := createIso(isoSize.Bytes(), isoFilename, scanRootDirs, files)
+		size, filename, leftFiles, notExistFiles, err := createIso(isoSize.Bytes(), isoFilename, scanRootDirs, files, debug)
 		if err != nil {
 			return err
 		}
@@ -125,14 +133,6 @@ func mkISO(ctx *cli.Context) error {
 	}
 }
 
-func keepTime(src, dst string) error {
-	ts, err := times.Stat(src)
-	if err != nil {
-		return err
-	}
-	return os.Chtimes(dst, ts.AccessTime(), ts.ModTime())
-}
-
 func createFileInStaging(srcFile, dstFile string) error {
 	src, err := os.Open(srcFile)
 	if err != nil {
@@ -150,13 +150,17 @@ func createFileInStaging(srcFile, dstFile string) error {
 	return err
 }
 
-func createIso(maxSize uint64, isoFilename string, scanRootDirs map[int]string,
-	files []*types.FileInfo) (uint64, string, []*types.FileInfo, []*types.FileInfo, error) {
+func createIso(maxSize uint64, isoFilename string, scanRootDirs map[int]string, files []*types.FileInfo,
+	debug bool) (uint64, string, []*types.FileInfo, []*types.FileInfo, error) {
 	stagingDir, err := os.MkdirTemp("", "lomobackup-")
 	if err != nil {
 		return 0, "", nil, nil, err
 	}
-	defer os.RemoveAll(stagingDir)
+	if debug {
+		logrus.Infof("Staging directory '%s' is kept for debugging", stagingDir)
+	} else {
+		defer os.RemoveAll(stagingDir)
+	}
 
 	const seperater = ','
 	var (
@@ -207,7 +211,7 @@ func createIso(maxSize uint64, isoFilename string, scanRootDirs map[int]string,
 			end = f.ModTime
 		}
 
-		err = keepTime(srcFile, dstFile)
+		err = common.KeepTime(srcFile, dstFile, false)
 		if err != nil {
 			logrus.Warnf("Keep file original timestamp %s: %s", srcFile, err)
 		}
@@ -222,12 +226,7 @@ func createIso(maxSize uint64, isoFilename string, scanRootDirs map[int]string,
 		}
 
 		// change all destination directory's last modify time and access time
-		for dst, src := range dirsMap {
-			err = keepTime(src, dst)
-			if err != nil {
-				logrus.Warnf("Keep dir original timestamp %s: %s", src, err)
-			}
-		}
+		common.KeepDirsTime(stagingDir, dirsMap)
 
 		name := fmt.Sprintf("%d-%02d-%02d--%d-%02d-%02d", start.Year(), start.Month(), start.Day(),
 			end.Year(), end.Month(), end.Day())
@@ -334,6 +333,10 @@ func fileInfoFor(path string, fs filesystem.FileSystem, currNode treeprint.Tree)
 		return err
 	}
 
+	slices.SortStableFunc(files, func(a, b os.FileInfo) int {
+		return strings.Compare(a.Name(), b.Name())
+	})
+
 	for _, file := range files {
 		t := file.ModTime()
 		fullPath := filepath.Join(path, file.Name())

cmd/lomob/main.go

@@ -47,7 +47,7 @@ func main() {
 				cli.StringFlag{
 					Name:  "ignore-files, if",
 					Usage: "List of ignored files, separated by comma",
-					Value: ".DS_Store,._.DS_Store,Thumbs.db",
+					Value: ".DS_Store,._.DS_Store,Thumbs.db,.git",
 				},
 				cli.StringFlag{
 					Name:  "ignore-dirs, in",
@@ -78,7 +78,11 @@ func main() {
 						},
 						cli.StringFlag{
 							Name:  "store-dir,p",
-							Usage: "Directory to store the ISOs. It's urrent directory by default",
+							Usage: "Directory to store the ISOs. It's current directory by default",
+						},
+						cli.BoolFlag{
+							Name:  "debug",
+							Usage: "Keep temp directory for debugging purpose, and also dump more debug level log",
 						},
 					},
 				},

cmd/lomob/scan.go

@@ -68,6 +68,7 @@ func scanDir(ctx *cli.Context) (err error) {
 	dirs = make(map[string]scanDirInfo)
 	lock = &sync.Mutex{}
 
+	var wg sync.WaitGroup
 	ch := make(chan scan.FileCallback, nthreads)
 	go func() {
 		for {
@@ -76,10 +77,17 @@ func scanDir(ctx *cli.Context) (err error) {
 			if err != nil {
 				logrus.Warnf("Error handling file %s: %s", cb.Path, err)
 			}
+			wg.Done()
 		}
 	}()
 
-	return scan.Directory(scanRootDir, ignoreFiles, ignoreDirs, ch)
+	err = scan.Directory(scanRootDir, ignoreFiles, ignoreDirs, &wg, ch)
+	if err != nil {
+		return err
+	}
+
+	wg.Wait()
+	return nil
 }
 
 func selectOrInsertScanRootDir() error {

cmd/lomob/upload-files.go

@@ -224,6 +224,7 @@ func uploadFilesToS3(ctx *cli.Context) error {
 	}
 
 	for _, name := range ctx.Args() {
+		name = filepath.Clean(name)
 		fmt.Printf("Uploading file %s\n", name)
 
 		if masterKey == "" {

cmd/lomob/upload-iso.go

@@ -139,10 +139,10 @@ func prepareUploadRequest(cli *clients.AWSClient, region, bucket, storageClass s
 			return nil, errors.Errorf("%s exists in cloud and its size is %d, but provided file size is %d",
 				isoFilename, remoteInfo.Size, isoInfo.Size)
 		}
-		if isoInfo.HashRemote != "" {
+		if isoInfo.HashRemote != "" && remoteInfo.HashRemote != "" {
 			remoteHash := strings.Split(remoteInfo.HashRemote, "-")[0]
 			if remoteHash != isoInfo.HashRemote {
-				return nil, errors.Errorf("%s exists in cloud and its checksum is %s, but provided ccommonhecksum is %s",
+				return nil, errors.Errorf("%s exists in cloud and its checksum is %s, but provided checksum is %s",
 					isoFilename, remoteHash, isoInfo.HashRemote)
 			}
 		}
@@ -216,7 +216,7 @@ func uploadISOMetafile(cli *clients.AWSClient, bucket, storageClass, isoFilename
 		return err
 	}
 
-	treeBuf := []byte(tree)
+	treeBuf := []byte(tree + "\n")
 
 	metaFilename := mkIsoMetadataFilename(isoFilename)
 	err = validateISOMetafile(metaFilename, treeBuf)
@@ -567,7 +567,7 @@ func uploadISOs(ctx *cli.Context) error {
 
 	for _, isoFilename := range ctx.Args() {
 		err = uploadISO(accessKeyID, secretAccessKey, region, bucket, storageClass,
-			isoFilename, masterKey, partSize, saveParts, force)
+			filepath.Clean(isoFilename), masterKey, partSize, saveParts, force)
 		if err != nil {
 			return err
 		}

common/common_test.go

@@ -0,0 +1,51 @@
+package common
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/djherbis/times"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestKeepDirsTime(t *testing.T) {
+	logrus.SetLevel(logrus.DebugLevel)
+
+	tmpDir, err := os.MkdirTemp("", "lomotest")
+	require.Nil(t, err)
+
+	defer os.RemoveAll(tmpDir)
+
+	srcRootDir := "../"
+	dirs := []string{"cmd/lomob", "test", "test/scripts", "vendor/github.com/golang",
+		"vendor/github.com/aws", "vendor/github.com/google"}
+	testDirs := []string{"cmd", "cmd/lomob", "test", "test/scripts", "vendor", "vendor/github.com",
+		"vendor/github.com/golang", "vendor/github.com/google", "vendor/github.com/aws"}
+
+	dirsMap := map[string]string{}
+	for _, d := range dirs {
+		dst := filepath.Join(tmpDir, d)
+		require.Nil(t, os.MkdirAll(dst, 0755))
+
+		dirsMap[dst] = filepath.Join(srcRootDir, d)
+	}
+
+	KeepDirsTime(tmpDir, dirsMap)
+
+	for _, d := range testDirs {
+		dst := filepath.Join(tmpDir, d)
+		src := filepath.Join(srcRootDir, d)
+
+		dstStat, err := times.Stat(dst)
+		require.Nil(t, err)
+		srcStat, err := times.Stat(src)
+		require.Nil(t, err)
+
+		require.Equal(t, srcStat.AccessTime(), dstStat.AccessTime(), d)
+		require.Equal(t, srcStat.ModTime(), dstStat.ModTime(), d)
+	}
+
+	require.Nil(t, os.RemoveAll(tmpDir))
+}