From 6277b81118d3d7428c85432b20d8026b836b69ef Mon Sep 17 00:00:00 2001
From: Dweb Fan <dwebfan@gmail.com>
Date: Fri, 24 May 2024 09:12:30 -0700
Subject: [PATCH] add e2e test script and docker support

Signed-off-by: Dweb Fan <dwebfan@gmail.com>
---
 .github/workflows/main.yml                    |  18 ++-
 .gitmodules                                   |   3 +
 Makefile                                      |  34 +++-
 clients/upload.go                             |   5 +
 cmd/lomob/iso.go                              |  42 +++--
 cmd/lomob/main.go                             |   8 +-
 cmd/lomob/scan.go                             |  10 +-
 cmd/lomob/upload-files.go                     |   1 +
 cmd/lomob/upload-iso.go                       |   8 +-
 common/common_test.go                         |  51 ++++++
 common/scan/scan.go                           |  12 +-
 common/util.go                                |  76 +++++++++
 dockerfiles/dev-image                         |   5 +-
 dockerfiles/dev-image-load                    |   8 +
 dockerfiles/dev-run                           |  20 +++
 dockerfiles/entrypoint-aws.sh                 |  30 ++++
 dockerfiles/entrypoint-build.sh               |  34 ++++
 dockerfiles/entrypoint.sh                     |  30 +++-
 dockerfiles/hosts                             |   1 +
 dockerfiles/localstack.service                |  20 +++
 test/data                                     |   1 +
 ...uplicate_upload_2019-04-03--2024-04-17.txt |   3 +
 ...uplicate_upload_2021-04-26--2021-04-26.txt |   3 +
 ...uplicate_upload_2021-04-26--2021-07-31.txt |   3 +
 test/scripts/e2e.sh                           |  24 +++
 test/scripts/files_2019-04-03--2024-04-17.txt | 141 +++++++++++++++++
 test/scripts/files_2021-04-26--2021-04-26.txt |   8 +
 test/scripts/files_2021-04-26--2021-07-31.txt |  23 +++
 test/scripts/files_big.txt                    |  14 ++
 test/scripts/files_not_in_iso.txt             |  72 +++++++++
 test/scripts/test-iso-create.sh               |  24 +++
 test/scripts/test-iso-list.sh                 |  77 +++++++++
 test/scripts/test-list-bigfiles.sh            |  14 ++
 test/scripts/test-list-dirs.sh                | 149 ++++++++++++++++++
 test/scripts/test-scan.sh                     |  36 +++++
 test/scripts/test-upload-aws-encrypt.sh       |   3 +
 test/scripts/test-upload-aws-raw.sh           |  69 ++++++++
 37 files changed, 1019 insertions(+), 61 deletions(-)
 create mode 100644 .gitmodules
 create mode 100644 common/common_test.go
 create mode 100644 dockerfiles/dev-image-load
 create mode 100755 dockerfiles/entrypoint-aws.sh
 create mode 100755 dockerfiles/entrypoint-build.sh
 create mode 100644 dockerfiles/hosts
 create mode 100755 dockerfiles/localstack.service
 create mode 160000 test/data
 create mode 100644 test/scripts/duplicate_upload_2019-04-03--2024-04-17.txt
 create mode 100644 test/scripts/duplicate_upload_2021-04-26--2021-04-26.txt
 create mode 100644 test/scripts/duplicate_upload_2021-04-26--2021-07-31.txt
 create mode 100755 test/scripts/e2e.sh
 create mode 100755 test/scripts/files_2019-04-03--2024-04-17.txt
 create mode 100755 test/scripts/files_2021-04-26--2021-04-26.txt
 create mode 100755 test/scripts/files_2021-04-26--2021-07-31.txt
 create mode 100755 test/scripts/files_big.txt
 create mode 100755 test/scripts/files_not_in_iso.txt
 create mode 100755 test/scripts/test-iso-create.sh
 create mode 100755 test/scripts/test-iso-list.sh
 create mode 100755 test/scripts/test-list-bigfiles.sh
 create mode 100755 test/scripts/test-list-dirs.sh
 create mode 100755 test/scripts/test-scan.sh
 create mode 100755 test/scripts/test-upload-aws-encrypt.sh
 create mode 100755 test/scripts/test-upload-aws-raw.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 81ddabd..36bdee5 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@v4
       - run: make lint
 
-  install:
+  unit-test:
     runs-on: ubuntu-22.04
     container:
       image: lomorage/lomo-backup:1.0
@@ -28,11 +28,9 @@ jobs:
       GOFLAGS: "-buildvcs=false"
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: 'true'
-      - run: make install
+      - run: make unit-tests
 
-  test:
+  e2e-test:
     runs-on: ubuntu-22.04
     container:
       image: lomorage/lomo-backup:1.0
@@ -40,8 +38,12 @@ jobs:
       PATH: /bin:/usr/bin:/usr/local/bin:/usr/local/go/bin
       GOBIN: /usr/local/bin
       GOFLAGS: "-buildvcs=false"
+      WORKDIR: /go/src/github.com/lomorage/lomo-backup
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: 'true'
-      - run: make test
+      - run: apt install -y tree rsync
+      - run: git submodule update --init; sh -c -x 'cd test/data; find . | while read -r file; do git log -1 --format="%ad" --date=iso "$file"; done'
+      - run: cd test/data; find . | while read -r file; do git log -1 --format="%ad" --date=iso "$file" | xargs -t -I{} touch -d {} "$file"; done
+      - run: mkdir -p $WORKDIR; rsync -avz * $WORKDIR/; cd $WORKDIR; tree --si -D test/data/
+      - run: cd $WORKDIR; make install
+      - run: cd $WORKDIR; make e2e-tests
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..75bafd1
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "test/data"]
+	path = test/data
+	url = https://github.com/lomorage/homepage
diff --git a/Makefile b/Makefile
index 460ebfc..aae9e42 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,10 @@
 
 SHELL=/bin/bash # Use bash syntax
 GOPATH=/go
+USER_ID := $(shell id -u)
+USER_NAME := $(shell whoami)
+GROUP_ID := $(shell id -g)
+GROUP_NAME := $(shell id -gn)
 
 vendor:
 	go mod vendor
@@ -12,18 +16,32 @@ lint:
 install:
 	go install -v ./cmd/...
 
+localstack-tarball:
+	docker pull localstack/localstack:3.4.0
+	docker save -o localstack_3.4.0.tar localstack/localstack:3.4.0
+	gzip localstack_3.4.0.tar
+
 dev-container:
-	docker build --tag "lomorage/lomo-backup:1.0" -f dockerfiles/dev-image .
+	docker build --tag "lomorage/lomo-backup:build-stage1" -f dockerfiles/dev-image .
+	docker build --tag "lomorage/lomo-backup:build-stage2" -f dockerfiles/dev-image-load .
+	docker rm -f lomo-backup-build
+	docker run --name lomo-backup-build --privileged --net host -it "lomorage/lomo-backup:build-stage2" hostname
+	docker commit lomo-backup-build lomorage/lomo-backup:1.0
 
 dev:
-	docker build --tag "lomorage/lomo-backup" -f dockerfiles/dev-run .
+	docker build \
+	    --build-arg USER_ID=$(USER_ID) --build-arg USER_NAME=$(USER_NAME) \
+	    --build-arg GROUP_ID=$(GROUP_ID) --build-arg GROUP_NAME=$(GROUP_NAME) \
+		--tag "lomorage/lomo-backup" -f dockerfiles/dev-run .
 	docker rm -f lomo-backup
-	docker run \
-		--name lomo-backup --hostname lomo-backup \
-		--privileged --cap-add=ALL -v /dev:/dev -v /lib/modules:/lib/modules \
+	docker run --rm \
+		--name lomo-backup --hostname lomo-backup -v ./dockerfiles/hosts:/etc/hosts \
+		--privileged --net host --dns-search local \
 		-v "${PWD}:/go/src/github.com/lomorage/lomo-backup" \
-		--net host --dns-search local \
 		-it "lomorage/lomo-backup" -d bash
 
-test:
-	go test -v ./common/
\ No newline at end of file
+unit-tests:
+	go list ./common/... | xargs -I {} go test -v {}
+
+e2e-tests: install
+	cd test/scripts; ./e2e.sh
\ No newline at end of file
diff --git a/clients/upload.go b/clients/upload.go
index 7b7dc86..066315c 100644
--- a/clients/upload.go
+++ b/clients/upload.go
@@ -3,6 +3,7 @@ package clients
 import (
 	"context"
 	"io"
+	"os"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -70,6 +71,10 @@ func NewAWSClient(keyID, key, region string) (*AWSClient, error) {
 	if err != nil {
 		return nil, err
 	}
+	if os.Getenv("LOCALSTACK_ENDPOINT") != "" {
+		cfg.S3ForcePathStyle = aws.Bool(true)
+		cfg.Endpoint = aws.String(os.Getenv("LOCALSTACK_ENDPOINT"))
+	}
 	return &AWSClient{region: region, svc: s3.New(sess, cfg)}, nil
 }
 
diff --git a/cmd/lomob/iso.go b/cmd/lomob/iso.go
index b24363d..01a38f8 100644
--- a/cmd/lomob/iso.go
+++ b/cmd/lomob/iso.go
@@ -14,7 +14,6 @@ import (
 
 	"github.com/diskfs/go-diskfs"
 	"github.com/diskfs/go-diskfs/filesystem"
-	"github.com/djherbis/times"
 	"github.com/lomorage/lomo-backup/common"
 	"github.com/lomorage/lomo-backup/common/datasize"
 	lomohash "github.com/lomorage/lomo-backup/common/hash"
@@ -28,12 +27,20 @@ import (
 var futuretime = time.Date(3000, time.December, 31, 0, 0, 0, 0, time.Now().UTC().Location())
 
 func mkISO(ctx *cli.Context) error {
-	isoSize, err := datasize.ParseString(ctx.String("iso-size"))
+	err := initLogLevel(ctx.GlobalInt("log-level"))
 	if err != nil {
 		return err
 	}
 
-	err = initLogLevel(ctx.GlobalInt("log-level"))
+	debug := ctx.Bool("debug")
+	if debug {
+		err = initLogLevel(int(logrus.DebugLevel))
+		if err != nil {
+			return err
+		}
+	}
+
+	isoSize, err := datasize.ParseString(ctx.String("iso-size"))
 	if err != nil {
 		return err
 	}
@@ -84,7 +91,7 @@ func mkISO(ctx *cli.Context) error {
 				datasize.ByteSize(iso.Size).HR())
 		}
 
-		size, filename, leftFiles, notExistFiles, err := createIso(isoSize.Bytes(), isoFilename, scanRootDirs, files)
+		size, filename, leftFiles, notExistFiles, err := createIso(isoSize.Bytes(), isoFilename, scanRootDirs, files, debug)
 		if err != nil {
 			return err
 		}
@@ -125,14 +132,6 @@ func mkISO(ctx *cli.Context) error {
 	}
 }
 
-func keepTime(src, dst string) error {
-	ts, err := times.Stat(src)
-	if err != nil {
-		return err
-	}
-	return os.Chtimes(dst, ts.AccessTime(), ts.ModTime())
-}
-
 func createFileInStaging(srcFile, dstFile string) error {
 	src, err := os.Open(srcFile)
 	if err != nil {
@@ -150,13 +149,17 @@ func createFileInStaging(srcFile, dstFile string) error {
 	return err
 }
 
-func createIso(maxSize uint64, isoFilename string, scanRootDirs map[int]string,
-	files []*types.FileInfo) (uint64, string, []*types.FileInfo, []*types.FileInfo, error) {
+func createIso(maxSize uint64, isoFilename string, scanRootDirs map[int]string, files []*types.FileInfo,
+	debug bool) (uint64, string, []*types.FileInfo, []*types.FileInfo, error) {
 	stagingDir, err := os.MkdirTemp("", "lomobackup-")
 	if err != nil {
 		return 0, "", nil, nil, err
 	}
-	defer os.RemoveAll(stagingDir)
+	if debug {
+		logrus.Infof("Staging directory '%s' is kept for debugging", stagingDir)
+	} else {
+		defer os.RemoveAll(stagingDir)
+	}
 
 	const seperater = ','
 	var (
@@ -207,7 +210,7 @@ func createIso(maxSize uint64, isoFilename string, scanRootDirs map[int]string,
 			end = f.ModTime
 		}
 
-		err = keepTime(srcFile, dstFile)
+		err = common.KeepTime(srcFile, dstFile, false)
 		if err != nil {
 			logrus.Warnf("Keep file original timestamp %s: %s", srcFile, err)
 		}
@@ -222,12 +225,7 @@ func createIso(maxSize uint64, isoFilename string, scanRootDirs map[int]string,
 		}
 
 		// change all destination directory's last modify time and access time
-		for dst, src := range dirsMap {
-			err = keepTime(src, dst)
-			if err != nil {
-				logrus.Warnf("Keep dir original timestamp %s: %s", src, err)
-			}
-		}
+		common.KeepDirsTime(stagingDir, dirsMap)
 
 		name := fmt.Sprintf("%d-%02d-%02d--%d-%02d-%02d", start.Year(), start.Month(), start.Day(),
 			end.Year(), end.Month(), end.Day())
diff --git a/cmd/lomob/main.go b/cmd/lomob/main.go
index 7c23ecc..9ae7d41 100644
--- a/cmd/lomob/main.go
+++ b/cmd/lomob/main.go
@@ -47,7 +47,7 @@ func main() {
 				cli.StringFlag{
 					Name:  "ignore-files, if",
 					Usage: "List of ignored files, separated by comma",
-					Value: ".DS_Store,._.DS_Store,Thumbs.db",
+					Value: ".DS_Store,._.DS_Store,Thumbs.db,.git",
 				},
 				cli.StringFlag{
 					Name:  "ignore-dirs, in",
@@ -78,7 +78,11 @@ func main() {
 						},
 						cli.StringFlag{
 							Name:  "store-dir,p",
-							Usage: "Directory to store the ISOs. It's urrent directory by default",
+							Usage: "Directory to store the ISOs. It's current directory by default",
+						},
+						cli.BoolFlag{
+							Name:  "debug",
+							Usage: "Keep temp directory for debugging purpose, and also dump more debug level log",
 						},
 					},
 				},
diff --git a/cmd/lomob/scan.go b/cmd/lomob/scan.go
index 63d4406..fea5a04 100644
--- a/cmd/lomob/scan.go
+++ b/cmd/lomob/scan.go
@@ -68,6 +68,7 @@ func scanDir(ctx *cli.Context) (err error) {
 	dirs = make(map[string]scanDirInfo)
 	lock = &sync.Mutex{}
 
+	var wg sync.WaitGroup
 	ch := make(chan scan.FileCallback, nthreads)
 	go func() {
 		for {
@@ -76,10 +77,17 @@ func scanDir(ctx *cli.Context) (err error) {
 			if err != nil {
 				logrus.Warnf("Error handling file %s: %s", cb.Path, err)
 			}
+			wg.Done()
 		}
 	}()
 
-	return scan.Directory(scanRootDir, ignoreFiles, ignoreDirs, ch)
+	err = scan.Directory(scanRootDir, ignoreFiles, ignoreDirs, &wg, ch)
+	if err != nil {
+		return err
+	}
+
+	wg.Wait()
+	return nil
 }
 
 func selectOrInsertScanRootDir() error {
diff --git a/cmd/lomob/upload-files.go b/cmd/lomob/upload-files.go
index 931982e..7ca6d7d 100644
--- a/cmd/lomob/upload-files.go
+++ b/cmd/lomob/upload-files.go
@@ -224,6 +224,7 @@ func uploadFilesToS3(ctx *cli.Context) error {
 	}
 
 	for _, name := range ctx.Args() {
+		name = filepath.Clean(name)
 		fmt.Printf("Uploading file %s\n", name)
 
 		if masterKey == "" {
diff --git a/cmd/lomob/upload-iso.go b/cmd/lomob/upload-iso.go
index a2135c2..d5da697 100644
--- a/cmd/lomob/upload-iso.go
+++ b/cmd/lomob/upload-iso.go
@@ -139,10 +139,10 @@ func prepareUploadRequest(cli *clients.AWSClient, region, bucket, storageClass s
 			return nil, errors.Errorf("%s exists in cloud and its size is %d, but provided file size is %d",
 				isoFilename, remoteInfo.Size, isoInfo.Size)
 		}
-		if isoInfo.HashRemote != "" {
+		if isoInfo.HashRemote != "" && remoteInfo.HashRemote != "" {
 			remoteHash := strings.Split(remoteInfo.HashRemote, "-")[0]
 			if remoteHash != isoInfo.HashRemote {
-				return nil, errors.Errorf("%s exists in cloud and its checksum is %s, but provided ccommonhecksum is %s",
+				return nil, errors.Errorf("%s exists in cloud and its checksum is %s, but provided checksum is %s",
 					isoFilename, remoteHash, isoInfo.HashRemote)
 			}
 		}
@@ -216,7 +216,7 @@ func uploadISOMetafile(cli *clients.AWSClient, bucket, storageClass, isoFilename
 		return err
 	}
 
-	treeBuf := []byte(tree)
+	treeBuf := []byte(tree + "\n")
 
 	metaFilename := mkIsoMetadataFilename(isoFilename)
 	err = validateISOMetafile(metaFilename, treeBuf)
@@ -567,7 +567,7 @@ func uploadISOs(ctx *cli.Context) error {
 
 	for _, isoFilename := range ctx.Args() {
 		err = uploadISO(accessKeyID, secretAccessKey, region, bucket, storageClass,
-			isoFilename, masterKey, partSize, saveParts, force)
+			filepath.Clean(isoFilename), masterKey, partSize, saveParts, force)
 		if err != nil {
 			return err
 		}
diff --git a/common/common_test.go b/common/common_test.go
new file mode 100644
index 0000000..5e3a69b
--- /dev/null
+++ b/common/common_test.go
@@ -0,0 +1,51 @@
+package common
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/djherbis/times"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestKeepDirsTime(t *testing.T) {
+	logrus.SetLevel(logrus.DebugLevel)
+
+	tmpDir, err := os.MkdirTemp("", "lomotest")
+	require.Nil(t, err)
+
+	defer os.RemoveAll(tmpDir)
+
+	srcRootDir := "../"
+	dirs := []string{"cmd/lomob", "test", "test/scripts", "vendor/github.com/golang",
+		"vendor/github.com/aws", "vendor/github.com/google"}
+	testDirs := []string{"cmd", "cmd/lomob", "test", "test/scripts", "vendor", "vendor/github.com",
+		"vendor/github.com/golang", "vendor/github.com/google", "vendor/github.com/aws"}
+
+	dirsMap := map[string]string{}
+	for _, d := range dirs {
+		dst := filepath.Join(tmpDir, d)
+		require.Nil(t, os.MkdirAll(dst, 0755))
+
+		dirsMap[dst] = filepath.Join(srcRootDir, d)
+	}
+
+	KeepDirsTime(tmpDir, dirsMap)
+
+	for _, d := range testDirs {
+		dst := filepath.Join(tmpDir, d)
+		src := filepath.Join(srcRootDir, d)
+
+		dstStat, err := times.Stat(dst)
+		require.Nil(t, err)
+		srcStat, err := times.Stat(src)
+		require.Nil(t, err)
+
+		require.Equal(t, srcStat.AccessTime(), dstStat.AccessTime(), d)
+		require.Equal(t, srcStat.ModTime(), dstStat.ModTime(), d)
+	}
+
+	require.Nil(t, os.RemoveAll(tmpDir))
+}
diff --git a/common/scan/scan.go b/common/scan/scan.go
index 31eee64..5eeb723 100644
--- a/common/scan/scan.go
+++ b/common/scan/scan.go
@@ -18,9 +18,7 @@ type FileCallback struct {
 
 // Directory is to scan given root directory, and build DB tree
 func Directory(root string, ignoreFiles, ignoreDirs map[string]struct{},
-	ch chan FileCallback) error {
-	var wg sync.WaitGroup
-
+	wg *sync.WaitGroup, ch chan FileCallback) error {
 	processItem := func(path string, entry fs.DirEntry) error {
 		info, err := entry.Info()
 		if err != nil {
@@ -33,10 +31,7 @@ func Directory(root string, ignoreFiles, ignoreDirs map[string]struct{},
 		}
 
 		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			ch <- FileCallback{Path: path, Info: info}
-		}()
+		ch <- FileCallback{Path: path, Info: info}
 		return nil
 	}
 	// Launch a goroutine for each directory
@@ -71,8 +66,5 @@ func Directory(root string, ignoreFiles, ignoreDirs map[string]struct{},
 		return err
 	}
 
-	// Wait for all goroutines to finish
-	wg.Wait()
-
 	return nil
 }
diff --git a/common/util.go b/common/util.go
index b953449..f506a2d 100644
--- a/common/util.go
+++ b/common/util.go
@@ -1,9 +1,15 @@
 package common
 
 import (
+	"cmp"
 	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
 	"time"
 
+	"github.com/djherbis/times"
 	"github.com/sirupsen/logrus"
 )
 
@@ -19,3 +25,73 @@ func LogDebugObject(key string, obj any) {
 	content, _ := json.Marshal(obj)
 	logrus.Debugf("%s: %s", key, string(content))
 }
+
+func KeepDirsTime(dstRootDir string, dirs map[string]string) {
+	parentDirs := map[string]string{}
+	for dst, src := range dirs {
+		if dst == dstRootDir {
+			continue
+		}
+		for {
+			src = filepath.Dir(src)
+			dst = filepath.Dir(dst)
+			if dst == dstRootDir {
+				break
+			}
+
+			_, ok := dirs[dst]
+			if ok {
+				// already in the map, continue
+				continue
+			}
+
+			_, ok = parentDirs[dst]
+			if ok {
+				// already in the newly created map, continue
+				continue
+			}
+			parentDirs[dst] = src
+		}
+	}
+
+	logrus.Debugf("Add parent directories %v", parentDirs)
+
+	for dst, src := range parentDirs {
+		dirs[dst] = src
+	}
+
+	names := make([]string, len(dirs))
+	idx := 0
+	for dst := range dirs {
+		names[idx] = dst
+		idx++
+	}
+
+	// sort from the longest to shortest
+	slices.SortFunc(names, func(a, b string) int {
+		return cmp.Compare(strings.ToLower(b), strings.ToLower(a))
+	})
+
+	for _, dst := range names {
+		src := dirs[dst]
+		err := KeepTime(src, dst, true)
+		if err != nil {
+			logrus.Warnf("Keep dir original timestamp %s: %s", src, err)
+		}
+	}
+}
+
+func KeepTime(src, dst string, l bool) error {
+	ts, err := times.Stat(src)
+	if err != nil {
+		return err
+	}
+	if l {
+		logrus.Debugf("Keep %s access time %s, mod time %s with original %s", dst,
+			FormatTimeDateOnly(ts.AccessTime()),
+			FormatTimeDateOnly(ts.ModTime()),
+			src,
+		)
+	}
+	return os.Chtimes(dst, ts.AccessTime(), ts.ModTime())
+}
diff --git a/dockerfiles/dev-image b/dockerfiles/dev-image
index 72f32b1..a11c8f1 100644
--- a/dockerfiles/dev-image
+++ b/dockerfiles/dev-image
@@ -3,9 +3,8 @@ FROM ubuntu:22.04
 ENV DEBIAN_FRONTEND noninteractive
 ENV GOPATH=/go
 
-RUN apt-get -yqq update && apt-get install -yq --no-install-recommends man-db vim build-essential \
-  sqlite3 curl apt-transport-https ca-certificates software-properties-common \
-  && apt-get install -yq sqlite3 \
+RUN apt-get -yqq update && apt-get install -yq --no-install-recommends man-db vim build-essential rsync \
+  sqlite3 curl apt-transport-https ca-certificates software-properties-common sqlite3 sudo genisoimage tree \
   && apt-get clean autoclean && apt-get autoremove -y && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 # install docker
diff --git a/dockerfiles/dev-image-load b/dockerfiles/dev-image-load
new file mode 100644
index 0000000..588280c
--- /dev/null
+++ b/dockerfiles/dev-image-load
@@ -0,0 +1,8 @@
+FROM "lomorage/lomo-backup:build-stage1"
+
+COPY ./localstack_3.4.0.tar.gz /root/
+
+COPY ./dockerfiles/entrypoint-build.sh /entrypoint.sh
+RUN chmod 755 /entrypoint.sh
+
+ENTRYPOINT /entrypoint.sh "$@"
diff --git a/dockerfiles/dev-run b/dockerfiles/dev-run
index d64b995..82c8e74 100644
--- a/dockerfiles/dev-run
+++ b/dockerfiles/dev-run
@@ -4,6 +4,26 @@ ENV GOROOT=/usr/local/go
 ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
 WORKDIR /go/src/github.com/lomorage/lomo-backup
 
+ARG USER_ID
+ARG USER_NAME
+ARG GROUP_ID
+ARG GROUP_NAME
+
+RUN addgroup --gid $GROUP_ID $GROUP_NAME && \
+    adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID $USER_NAME && \
+    usermod -aG docker $USER_NAME
+
+RUN mkdir $GOPATH/bin && chown -R $USER_ID:$GROUP_ID $GOPATH/bin
+RUN mkdir $GOPATH/pkg && chown -R $USER_ID:$GROUP_ID $GOPATH/pkg
+
+# Add user to sudo group and configure sudo without password
+RUN echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+#COPY ./dockerfiles/localstack.service /etc/systemd/system/multi-user.target.wants
+
 COPY ./dockerfiles/entrypoint.sh /entrypoint.sh
 RUN chmod 755 /entrypoint.sh
+
+USER $USER_NAME
+
 ENTRYPOINT /entrypoint.sh "$@"
diff --git a/dockerfiles/entrypoint-aws.sh b/dockerfiles/entrypoint-aws.sh
new file mode 100755
index 0000000..4ccad3f
--- /dev/null
+++ b/dockerfiles/entrypoint-aws.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -x
+
+dockerd -s vfs &
+
+# Check if `docker ps` command works, otherwise retry
+while true; do
+    if docker ps; then
+        echo "Docker is running!"
+        break
+    else
+        echo "Docker is not yet ready. Retrying in 5 seconds..."
+        sleep 5
+    fi
+done
+
+#cd /root; gunzip localstack_3.4.0.tar.gz; docker load -i localstack_3.4.0.tar; rm localstack_3.4.0.tar
+
+#docker run --rm -d -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:3.4.0 &
+
+#while true; do
+#    if curl localhost:4566; then
+#        echo "Localstack is running!"
+#        break
+#    else
+#        echo "Localstack is not yet ready. Retrying in 5 seconds..."
+#        sleep 5
+#    fi
+#done
\ No newline at end of file
diff --git a/dockerfiles/entrypoint-build.sh b/dockerfiles/entrypoint-build.sh
new file mode 100755
index 0000000..22a4310
--- /dev/null
+++ b/dockerfiles/entrypoint-build.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -x
+
+dockerd -s vfs &
+
+# Check if `docker ps` command works, otherwise retry
+while true; do
+    if docker ps; then
+        echo "Docker is running!"
+        break
+    else
+        echo "Docker is not yet ready. Retrying in 5 seconds..."
+        sleep 5
+    fi
+done
+
+cd /root; gunzip localstack_3.4.0.tar.gz; docker load -i localstack_3.4.0.tar; rm localstack_3.4.0.tar
+
+pkill -9 dockerd
+
+while true; do
+    if docker ps; then
+        echo "Docker still is running. Retrying in 5 seconds..."
+        sleep 5
+    else
+        echo "Docker is dead"
+        break
+    fi
+done
+
+rm -rf /var/run/docker*
+
+$@
\ No newline at end of file
diff --git a/dockerfiles/entrypoint.sh b/dockerfiles/entrypoint.sh
index 1648ce8..c92ed78 100755
--- a/dockerfiles/entrypoint.sh
+++ b/dockerfiles/entrypoint.sh
@@ -2,7 +2,31 @@
 
 set -x
 
-echo user: $USER
-echo uid: $UID
+sudo rm -rf /var/run/docker*
+sudo service docker start
 
-$@
+# Check if `docker ps` command works, otherwise retry
+#while true; do
+#    if docker ps; then
+#        echo "Docker is running!"
+#        break
+#    else
+#        echo "Docker is not yet ready. Retrying in 5 seconds..."
+#        sudo service docker start
+#        sleep 5
+#    fi
+#done
+
+#docker run --rm -d -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:3.4.0 &
+
+#while true; do
+#    if curl localhost:4566; then
+#        echo "Localstack is running!"
+#        break
+#    else
+#        echo "Localstack is not yet ready. Retrying in 5 seconds..."
+#        sleep 5
+#    fi
+#done
+
+$@
\ No newline at end of file
diff --git a/dockerfiles/hosts b/dockerfiles/hosts
new file mode 100644
index 0000000..be73bf9
--- /dev/null
+++ b/dockerfiles/hosts
@@ -0,0 +1 @@
+127.0.0.1 lomo-backup
diff --git a/dockerfiles/localstack.service b/dockerfiles/localstack.service
new file mode 100755
index 0000000..4b0e813
--- /dev/null
+++ b/dockerfiles/localstack.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=Localstack service
+After=network-online.target docker.socket firewalld.service containerd.service time-set.target docker.service
+Wants=network-online.target containerd.service docker.service
+Requires=docker.service
+
+[Service]
+Type=simple
+# the default is not to use systemd for cgroups because the delegate issues still
+# exists and systemd currently does not support the cgroup feature set required
+# for containers run by docker
+ExecStart=docker run --name localstack -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:3.4.0
+ExecReload=docker rm -f localstack
+RestartSec=2
+Restart=always
+
+WillMode = process
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/test/data b/test/data
new file mode 160000
index 0000000..856dfb3
--- /dev/null
+++ b/test/data
@@ -0,0 +1 @@
+Subproject commit 856dfb3e58b17fa08f7ad4580db66cdb0f25e3ea
diff --git a/test/scripts/duplicate_upload_2019-04-03--2024-04-17.txt b/test/scripts/duplicate_upload_2019-04-03--2024-04-17.txt
new file mode 100644
index 0000000..c97c929
--- /dev/null
+++ b/test/scripts/duplicate_upload_2019-04-03--2024-04-17.txt
@@ -0,0 +1,3 @@
+Uploading un-encrypted metadata file 2019-04-03--2024-04-17.iso.meta.txt
+2019-04-03--2024-04-17.iso.meta.txt is already in bucket lomorage, no need upload again !
+2019-04-03--2024-04-17.iso is already in region us-east-1, bucket lomorage, no need upload again !
diff --git a/test/scripts/duplicate_upload_2021-04-26--2021-04-26.txt b/test/scripts/duplicate_upload_2021-04-26--2021-04-26.txt
new file mode 100644
index 0000000..5346d3b
--- /dev/null
+++ b/test/scripts/duplicate_upload_2021-04-26--2021-04-26.txt
@@ -0,0 +1,3 @@
+Uploading un-encrypted metadata file 2021-04-26--2021-04-26.iso.meta.txt
+2021-04-26--2021-04-26.iso.meta.txt is already in bucket lomorage, no need upload again !
+2021-04-26--2021-04-26.iso is already in region us-east-1, bucket lomorage, no need upload again !
diff --git a/test/scripts/duplicate_upload_2021-04-26--2021-07-31.txt b/test/scripts/duplicate_upload_2021-04-26--2021-07-31.txt
new file mode 100644
index 0000000..54164ee
--- /dev/null
+++ b/test/scripts/duplicate_upload_2021-04-26--2021-07-31.txt
@@ -0,0 +1,3 @@
+Uploading un-encrypted metadata file 2021-04-26--2021-07-31.iso.meta.txt
+2021-04-26--2021-07-31.iso.meta.txt is already in bucket lomorage, no need upload again !
+2021-04-26--2021-07-31.iso is already in region us-east-1, bucket lomorage, no need upload again !
diff --git a/test/scripts/e2e.sh b/test/scripts/e2e.sh
new file mode 100755
index 0000000..648feb6
--- /dev/null
+++ b/test/scripts/e2e.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+rm -f lomob.db; sqlite3 ./lomob.db < ../../common/dbx/schema/1.sql
+
+# test scan
+./test-scan.sh
+
+# test big file list
+./test-list-bigfiles.sh
+
+# test list scan dir
+./test-list-dirs.sh
+
+# test iso
+./test-iso-create.sh
+./test-iso-list.sh
+
+# test upload raw
+#./test-upload-aws-raw.sh
+
+# test upload encrypt file
+#./test-upload-aws-encrypt.sh
diff --git a/test/scripts/files_2019-04-03--2024-04-17.txt b/test/scripts/files_2019-04-03--2024-04-17.txt
new file mode 100755
index 0000000..98ea779
--- /dev/null
+++ b/test/scripts/files_2019-04-03--2024-04-17.txt
@@ -0,0 +1,141 @@
+/
+├── [	06/15/2021]  archetypes
+│   └── [	          84	06/15/2021]  default.md
+├── [	        3799	11/06/2023]  config.toml
+├── [	03/17/2024]  content
+│   ├── [	03/17/2024]  blog
+│   │   ├── [	        9089	08/24/2020]  covid19.md
+│   │   ├── [	        8308	06/12/2020]  covid19.zh.md
+│   │   ├── [	        2664	05/12/2020]  ddns.md.bak
+│   │   ├── [	        8252	10/14/2022]  flyio.md
+│   │   ├── [	        6206	05/18/2020]  import_my_cloud.zh.md
+│   │   ├── [	        5853	08/22/2020]  import_my_cloud.md
+│   │   ├── [	        3074	03/03/2023]  migrate_from_pi_win.zh.md
+│   │   ├── [	        2002	03/02/2023]  mypains.zh.md
+│   │   ├── [	        6747	03/17/2024]  photoprism.md
+│   │   ├── [	        6733	10/31/2022]  photoprism.zh.md
+│   │   ├── [	        4808	09/04/2021]  piq-size.md
+│   │   ├── [	        4715	11/06/2023]  raspberrypi-hd.zh.md
+│   │   ├── [	        5072	11/06/2023]  raspberrypi-hd.md
+│   │   ├── [	        1708	03/02/2023]  set_wd_as_backup.zh.md
+│   │   ├── [	        5550	03/02/2023]  set_wd_as_backup.md
+│   │   ├── [	        4130	08/22/2020]  transfer2pc.zh.md
+│   │   ├── [	        1899	11/17/2020]  windows-docker.zh.md
+│   │   ├── [	        2227	11/17/2020]  windows-docker.md
+│   │   ├── [	        1529	08/26/2021]  zero-cost-android-frame.md
+│   │   └── [	        1936	08/26/2021]  zero-cost-android-frame.zh.md
+│   ├── [	        7111	03/08/2021]  compare.md
+│   ├── [	        7571	03/08/2021]  compare.zh.md
+│   ├── [	         236	11/06/2023]  contact.md
+│   ├── [	         193	11/06/2023]  contact.zh.md
+│   ├── [	        7946	06/06/2022]  faq.md
+│   ├── [	        7612	06/06/2022]  faq.zh.md
+│   ├── [	         250	04/02/2021]  survey.md
+│   ├── [	         263	03/21/2022]  survey.zh.md
+│   ├── [	        3445	03/23/2022]  _index.md
+│   └── [	        3163	05/03/2022]  _index.zh.md
+├── [	         422	12/27/2019]  deploy.sh
+├── [	06/15/2021]  layouts
+│   └── [	         182	06/15/2021]  index.html
+├── [	        6014	01/29/2023]  README.md
+├── [	07/06/2021]  resources
+│   └── [	07/06/2021]  _gen
+│       └── [	07/06/2021]  assets
+│           ├── [	07/06/2021]  css
+│           │   └── [	07/06/2021]  css
+│           │       ├── [	       20971	06/15/2021]  style.css_48b060fe05b0a273d182ef83c0605941.content
+│           │       ├── [	         193	07/06/2021]  style.css_e753df1120945eee25a798cc3b6f156c.json
+│           │       ├── [	       21115	07/06/2021]  style.css_e753df1120945eee25a798cc3b6f156c.content
+│           │       └── [	         193	06/15/2021]  style.css_48b060fe05b0a273d182ef83c0605941.json
+│           └── [	06/15/2021]  js
+│               └── [	06/15/2021]  js
+│                   ├── [	       28236	06/15/2021]  scripts.js_d11fe7b62c27961c87ecd0f2490357b9.content
+│                   └── [	         207	06/15/2021]  scripts.js_d11fe7b62c27961c87ecd0f2490357b9.json
+├── [	04/17/2024]  static
+│   ├── [	          59	04/17/2024]  ads.txt
+│   ├── [	          59	03/17/2024]  app-ads.txt
+│   ├── [	          13	04/12/2019]  CNAME
+│   ├── [	03/03/2023]  img
+│   │   ├── [	        6340	04/03/2019]  apple-touch-icon.png
+│   │   ├── [	03/03/2023]  blog
+│   │   │   ├── [	08/26/2021]  android-frame
+│   │   │   │   ├── [	      620731	08/26/2021]  android-frame-banner.png
+│   │   │   │   ├── [	       66740	08/26/2021]  lomorage_wechat_qr.jpg
+│   │   │   │   └── [	    20867047	08/26/2021]  phone-share.gif
+│   │   │   ├── [	03/30/2020]  covid19
+│   │   │   │   ├── [	      112106	03/30/2020]  covid19.jpg
+│   │   │   │   ├── [	      207059	03/30/2020]  dashboard-en.png
+│   │   │   │   ├── [	      284169	03/30/2020]  dashboard-zh.png
+│   │   │   │   └── [	        6648	03/30/2020]  layout.png
+│   │   │   ├── [	10/14/2022]  flyio
+│   │   │   │   └── [	        9781	10/14/2022]  logo.svg
+│   │   │   ├── [	09/02/2021]  image-quality
+│   │   │   │   ├── [	      105323	09/02/2021]  320_perc.png
+│   │   │   │   ├── [	      106330	09/02/2021]  320_size.png
+│   │   │   │   ├── [	       79878	09/02/2021]  640_perc.png
+│   │   │   │   └── [	      101593	09/02/2021]  640_size.png
+│   │   │   ├── [	05/18/2020]  import_my_cloud
+│   │   │   │   ├── [	       25562	05/18/2020]  6822a363-9172-4e35-b77c-5318ea01c50a.png
+│   │   │   │   ├── [	       20850	05/18/2020]  ad38c19b-10b1-4b82-92ec-68b6f2481601-1.png
+│   │   │   │   ├── [	       36065	05/18/2020]  banner_mycloud_lomo.png
+│   │   │   │   ├── [	       66213	05/18/2020]  f14b9c18-f941-429a-a848-1426f547ab37.png
+│   │   │   │   └── [	        9906	05/18/2020]  f6a46087-7e46-47c6-8568-e551d062bd6e.png
+│   │   │   ├── [	       77900	03/02/2023]  lomorage_wechat_qr.jpg
+│   │   │   ├── [	03/03/2023]  migrate_from_pi_win
+│   │   │   │   ├── [	       52945	03/03/2023]  assets_db_zh.png
+│   │   │   │   ├── [	       21265	03/03/2023]  migrate_pi_win.jpg
+│   │   │   │   └── [	       13632	03/03/2023]  one_click_recover.png
+│   │   │   ├── [	07/20/2020]  mypains
+│   │   │   │   ├── [	       78378	07/20/2020]  banner.pdn
+│   │   │   │   └── [	       50489	07/20/2020]  banner.png
+│   │   │   ├── [	02/12/2022]  photoprism
+│   │   │   │   └── [	       25647	02/12/2022]  photoprism-banner.jpeg
+│   │   │   ├── [	06/15/2021]  raspberrypi-hd
+│   │   │   │   └── [	      238208	06/15/2021]  power-hub-adafruit.jpg
+│   │   │   ├── [	03/02/2023]  set_wd_as_backup
+│   │   │   │   ├── [	       74188	02/23/2023]  mount.png
+│   │   │   │   ├── [	       60212	03/02/2023]  mount_zh.png
+│   │   │   │   ├── [	       37990	03/02/2023]  set_redundancy_agent_zh.png
+│   │   │   │   ├── [	       34156	02/23/2023]  set_redundancy_agent.png
+│   │   │   │   └── [	       17092	02/23/2023]  set_wd_as_backup.jpg
+│   │   │   ├── [	03/31/2020]  transfer_pc
+│   │   │   │   ├── [	       59333	03/31/2020]  13c696c2-1c30-4906-be64-ed7ab3d115a7.png
+│   │   │   │   ├── [	      118026	03/31/2020]  3d2ba39a-e52a-47a0-9074-39c9bb43f58c.jpg
+│   │   │   │   ├── [	       51157	03/31/2020]  4db9e626-29d9-4586-963b-ff3e63cb93f9.png
+│   │   │   │   ├── [	       50054	03/31/2020]  5f783099-ce8c-4535-839e-444704ce9300.png
+│   │   │   │   ├── [	      128629	03/31/2020]  a155dcc4-37c3-44bf-aea1-8ec5306e2b78.jpg
+│   │   │   │   ├── [	       28646	03/31/2020]  a8f7743d-2770-4537-9b44-f79a698d7243.png
+│   │   │   │   ├── [	       44094	03/31/2020]  ad256b83-38a9-4a7c-a983-ad5a3c34fc58.png
+│   │   │   │   ├── [	       13926	03/31/2020]  cbd625dc-783d-430b-a51b-6204a28e488d.png
+│   │   │   │   ├── [	       26872	03/31/2020]  d7600bbe-28ac-4cfe-89c8-adcb8145dbb7.png
+│   │   │   │   └── [	       14442	03/31/2020]  transfer-photos.jpg
+│   │   │   └── [	11/17/2020]  win-docker
+│   │   │       ├── [	       53012	11/17/2020]  docker.png
+│   │   │       ├── [	       26175	11/17/2020]  web-client.png
+│   │   │       └── [	       61209	11/17/2020]  win-docker-banner.png
+│   │   ├── [	05/03/2021]  buy
+│   │   │   ├── [	     1431468	04/26/2021]  backup.png
+│   │   │   ├── [	     1132695	05/03/2021]  backup-en.png
+│   │   │   └── [	     1565661	05/03/2021]  local-en.png
+│   │   ├── [	        2667	06/15/2021]  data-storage-svgrepo-com.svg
+│   │   ├── [	       10205	06/15/2021]  delivery-truck-free-svgrepo-com.svg
+│   │   ├── [	        8333	06/15/2021]  family-svgrepo-com.svg
+│   │   ├── [	        7560	04/03/2019]  favicon.ico
+│   │   ├── [	        3688	06/15/2021]  file-storage-svgrepo-com.svg
+│   │   ├── [	        3880	06/15/2021]  hard-disk-svgrepo-com.svg
+│   │   ├── [	       20225	10/15/2021]  logo.png
+│   │   ├── [	        3984	06/15/2021]  platform-svgrepo-com.svg
+│   │   ├── [	        3540	06/15/2021]  raw-svgrepo-com.svg
+│   │   ├── [	        2806	06/15/2021]  security-svgrepo-com.svg
+│   │   └── [	        4678	06/15/2021]  undraw-devices.svg
+│   ├── [	      525451	01/06/2020]  lomorage.key
+│   ├── [	      105524	01/06/2020]  lomorage.zh.pdf
+│   ├── [	      497252	01/06/2020]  lomorage.zh.key
+│   ├── [	       60530	01/06/2020]  lomorage.pdf
+│   ├── [	      165694	11/13/2023]  privacy.html
+│   ├── [	         951	04/16/2024]  release.json
+│   └── [	      229986	11/15/2020]  term.pdf
+├── [	        5226	07/23/2021]  updatever.sh
+├── [	          16	04/03/2019]  gitignore
+└── [	         107	06/15/2021]  gitmodules
+
diff --git a/test/scripts/files_2021-04-26--2021-04-26.txt b/test/scripts/files_2021-04-26--2021-04-26.txt
new file mode 100755
index 0000000..d7df6be
--- /dev/null
+++ b/test/scripts/files_2021-04-26--2021-04-26.txt
@@ -0,0 +1,8 @@
+/
+└── [	04/17/2024]  static
+    └── [	03/03/2023]  img
+        └── [	05/03/2021]  buy
+            ├── [	     2494420	04/26/2021]  local.png
+            ├── [	    14885871	04/26/2021]  lomorage-setup.png
+            └── [	    14578386	04/26/2021]  lomorage-mini.png
+
diff --git a/test/scripts/files_2021-04-26--2021-07-31.txt b/test/scripts/files_2021-04-26--2021-07-31.txt
new file mode 100755
index 0000000..c6d4352
--- /dev/null
+++ b/test/scripts/files_2021-04-26--2021-07-31.txt
@@ -0,0 +1,23 @@
+/
+└── [	04/17/2024]  static
+    ├── [	03/03/2023]  img
+    │   ├── [	05/03/2021]  buy
+    │   │   ├── [	      476712	04/26/2021]  search.png
+    │   │   ├── [	      304081	05/03/2021]  search-en.png
+    │   │   ├── [	      351826	05/03/2021]  settings-en.png
+    │   │   ├── [	      382947	04/26/2021]  settings.png
+    │   │   ├── [	     1207368	04/26/2021]  web-gallery.png
+    │   │   ├── [	     1595170	05/03/2021]  web-gallery-en.png
+    │   │   ├── [	     1044760	04/26/2021]  web-upload.png
+    │   │   └── [	      669073	05/03/2021]  web-upload-en.png
+    │   └── [	07/31/2021]  links
+    │       ├── [	       72809	07/06/2021]  appinn.png
+    │       ├── [	       47005	07/06/2021]  canalapps.png
+    │       ├── [	       32607	07/06/2021]  futurezone.png
+    │       ├── [	      189043	07/06/2021]  iplaysoft.png
+    │       ├── [	       23906	07/06/2021]  medium.png
+    │       └── [	        8714	07/31/2021]  x1g.png
+    └── [	06/15/2021]  video
+        ├── [	    21164705	06/15/2021]  Lomorage-tutorial.zh.mp4
+        └── [	    13940681	06/15/2021]  Lomorage-tutorial.mp4
+
diff --git a/test/scripts/files_big.txt b/test/scripts/files_big.txt
new file mode 100755
index 0000000..8d1b1f6
--- /dev/null
+++ b/test/scripts/files_big.txt
@@ -0,0 +1,14 @@
+Name                                                                                               Size
+/go/src/github.com/lomorage/lomo-backup/test/data/static/video/Lomorage-tutorial.zh.mp4            21.2 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/android-frame/phone-share.gif    20.9 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/lomorage-setup.png                14.9 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/lomorage-mini.png                 14.6 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/video/Lomorage-tutorial.mp4               13.9 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/video/lomorage.mp4                        3.0 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/local.png                         2.5 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/web-gallery-en.png                1.6 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/local-en.png                      1.6 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/backup.png                        1.4 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/web-gallery.png                   1.2 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/backup-en.png                     1.1 MB
+/go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy/web-upload.png                    1.0 MB
diff --git a/test/scripts/files_not_in_iso.txt b/test/scripts/files_not_in_iso.txt
new file mode 100755
index 0000000..2651d59
--- /dev/null
+++ b/test/scripts/files_not_in_iso.txt
@@ -0,0 +1,72 @@
+In Cloud    Path
+            /go/src/github.com/lomorage/lomo-backup/test/data/static/video/lomorage.mp4
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/LICENSE.md
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/README.md
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/theme.toml
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/archetypes/default.md
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/js/app.js
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/js/libraries/flexibility.js
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/js/libraries/responsive-nav.js
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/custom.style.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/style.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/base/_containers.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/base/_globals.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/base/_variables.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_app.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_auth.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_cta.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_expanded.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_footer.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_hero.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_landing.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_navbar.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_page.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_steps.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components/_verticalMenu.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/elements/_buttons.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/elements/_forms.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/elements/_typography.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/utils/_helpers.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/utils/_mixins.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/utils/_normalize.scss
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/i18n/en.yaml
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/i18n/zh.yaml
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/images/screenshot.png
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/images/screenshot2.png
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/images/screenshot3.png
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/images/tn.png
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/_default/baseof.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/_default/list.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/_default/single.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/_default/terms.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/adsense.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/contact.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/footer.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/header.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/landing_cta.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/landing_hero.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/landing_single.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/landing_triple.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/nav.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials/scripts.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/shortcodes/table.html
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets/css/css/style.css_b95b077eb505d5c0aff8055eaced30ad.content
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets/css/css/style.css_b95b077eb505d5c0aff8055eaced30ad.json
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets/js/js/scripts.js_d11fe7b62c27961c87ecd0f2490357b9.content
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets/js/js/scripts.js_d11fe7b62c27961c87ecd0f2490357b9.json
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/evie_default_bg.jpeg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/hero_sm.png
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/tet.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/together.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_browser.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_creation.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_design.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_designer.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_elements.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_everywhere.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_fans.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_frameworks.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_hello_aeia.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_responsive.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_selfie.svg
+            /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images/undraw_tabs.svg
diff --git a/test/scripts/test-iso-create.sh b/test/scripts/test-iso-create.sh
new file mode 100755
index 0000000..d53b674
--- /dev/null
+++ b/test/scripts/test-iso-create.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+sqlite3 ./lomob.db "delete from isos"; sqlite3 lomob.db "update files set iso_id=0"; rm *.iso *.iso.meta.txt
+
+set -e
+
+lomob iso create -s 30M --debug
+
+#declare -A filesHash
+#filesHash["2019-04-03--2024-04-17.iso"]="616a786cbffd5f81069554e8517b1376e5a7f7f293d1babda4e3cc8b2e390f22"
+#filesHash["2021-04-26--2021-04-26.iso"]="8a0a2e091e5a5ea1ebfc260dcc1e058ee4ec0eb869837f323bf17ac6db6392c6"
+#filesHash["2021-04-26--2021-07-31.iso"]="988cdf7c20a1606990e66015e62bf82ff4b1d44f8946c06253f4b089181100ce"
+#
+## Iterate over the keys and values of the map
+#for key in "${!filesHash[@]}"; do
+#  expecValue="${filesHash[$key]}"
+#  value=$(sha256sum $key | awk -F' ' '{print $1}')
+#
+#  if [ "$value" = "$expecValue" ]; then
+#    echo "$key has same file sha as expected"
+#  else
+#    echo "FAIL: $key has different file sha as expected"
+#  fi
+#done
diff --git a/test/scripts/test-iso-list.sh b/test/scripts/test-iso-list.sh
new file mode 100755
index 0000000..7a8df2a
--- /dev/null
+++ b/test/scripts/test-iso-list.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+set -e
+
+#echo "Test list iso files in db"
+#
+#expectISOFiles="ID    Name                          Size       Status                   Region    Bucket    Files Count    Create Time            Local Hash
+#1     2019-04-03--2024-05-25.iso    30.7 MB    Created, not uploaded                        113            2024-05-25 23:21:21    54979da5dcdf0ff08c046dd97662677a7b186bb13ef700668e36899f2e0d1df5
+#2     2021-04-26--2021-04-26.iso    32.3 MB    Created, not uploaded                        3              2024-05-25 23:21:21    4a93ac387edc587bce1d5b1a51dd78358cd9b9275a02a90e5b7e433256f94444
+#3     2021-04-26--2021-07-31.iso    41.9 MB    Created, not uploaded                        16             2024-05-25 23:21:21    44434ef03763d476ed82602f7ee6991f40e2c698d37c7b7771fb39cd14fb3cb7"
+#
+#value=$(lomob iso list)
+#
+#if [ "$value" = "$expectISOFiles" ]; then
+#  echo "ISO file lists are same as expected"
+#else
+#  echo "ISO file lists are different"
+#  echo "Expect:"
+#  echo "$expectISOFiles"
+#  echo "Actual:"
+#  echo "$value"
+#fi
+
+echo "Test listing files not in iso yet"
+
+lomob list files > /tmp/tmp
+
+diff /tmp/tmp files_not_in_iso.txt
+
+if [ $? -eq 0 ]; then
+  echo "Files not in ISO are same as expected"
+else
+  echo "Files not in ISO are different"
+  exit 1
+fi
+
+echo "Test dump iso files"
+
+isoFile="2019-04-03--2024-04-17.iso"
+
+lomob iso dump $isoFile > /tmp/tmp
+
+diff /tmp/tmp files_2019-04-03--2024-04-17.txt
+
+
+if [ $? -eq 0 ]; then
+  echo "Dump iso $isoFile are same as expected"
+else
+  echo "Dump iso $isoFile are different"
+  exit 1
+fi
+
+isoFile="2021-04-26--2021-04-26.iso"
+
+lomob iso dump $isoFile > /tmp/tmp
+
+diff /tmp/tmp files_2021-04-26--2021-04-26.txt
+
+if [ $? -eq 0 ]; then
+  echo "Dump iso $isoFile are same as expected"
+else
+  echo "Dump iso $isoFile are different"
+  exit 1
+fi
+
+isoFile="2021-04-26--2021-07-31.iso"
+
+lomob iso dump $isoFile > /tmp/tmp
+
+diff /tmp/tmp files_2021-04-26--2021-07-31.txt
+
+if [ $? -eq 0 ]; then
+  echo "Dump iso $isoFile are same as expected"
+else
+  echo "Dump iso $isoFile are different"
+  exit 1
+fi
diff --git a/test/scripts/test-list-bigfiles.sh b/test/scripts/test-list-bigfiles.sh
new file mode 100755
index 0000000..f4bda67
--- /dev/null
+++ b/test/scripts/test-list-bigfiles.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+echo "Test list big files in scan table"
+
+lomob list bigfiles -s 1M > /tmp/tmp
+
+diff /tmp/tmp files_big.txt
+
+if [ $? -eq 0 ]; then
+  echo "Big file lists are same as expected"
+else
+  echo "Big file lists are different"
+  exit 1
+fi
diff --git a/test/scripts/test-list-dirs.sh b/test/scripts/test-list-dirs.sh
new file mode 100755
index 0000000..c5f4d8a
--- /dev/null
+++ b/test/scripts/test-list-dirs.sh
@@ -0,0 +1,149 @@
+#!/bin/bash
+
+set -e
+
+echo "Test list scan dirs"
+
+expectDirs="/go/src/github.com/lomorage/lomo-backup/test/data
+├── [     2021-06-15]  archetypes
+├── [     2024-03-17]  content
+│   └── [     2024-03-17]  blog
+├── [     2021-06-15]  layouts
+├── [     2024-03-31]  public
+├── [     2021-07-06]  resources
+│   └── [     2021-07-06]  _gen
+│       └── [     2021-07-06]  assets
+│           ├── [     2021-07-06]  css
+│           │   └── [     2021-07-06]  css
+│           └── [     2021-06-15]  js
+│               └── [     2021-06-15]  js
+├── [     2024-04-17]  static
+│   ├── [     2023-03-03]  img
+│   │   ├── [     2023-03-03]  blog
+│   │   │   ├── [     2021-08-26]  android-frame
+│   │   │   ├── [     2020-03-30]  covid19
+│   │   │   ├── [     2022-10-14]  flyio
+│   │   │   ├── [     2021-09-02]  image-quality
+│   │   │   ├── [     2020-05-18]  import_my_cloud
+│   │   │   ├── [     2023-03-03]  migrate_from_pi_win
+│   │   │   ├── [     2020-07-20]  mypains
+│   │   │   ├── [     2022-02-12]  photoprism
+│   │   │   ├── [     2021-06-15]  raspberrypi-hd
+│   │   │   ├── [     2023-03-02]  set_wd_as_backup
+│   │   │   ├── [     2020-03-31]  transfer_pc
+│   │   │   └── [     2020-11-17]  win-docker
+│   │   ├── [     2021-05-03]  buy
+│   │   └── [     2021-07-31]  links
+│   └── [     2021-06-15]  video
+└── [     2023-11-13]  themes
+    └── [     2023-11-13]  evie-hugo
+        ├── [     2021-06-15]  archetypes
+        ├── [     2021-07-06]  assets
+        │   ├── [     2021-06-15]  js
+        │   │   └── [     2021-06-15]  libraries
+        │   └── [     2021-07-06]  sass
+        │       ├── [     2021-06-15]  base
+        │       ├── [     2021-07-06]  components
+        │       ├── [     2021-06-16]  elements
+        │       └── [     2021-06-15]  utils
+        ├── [     2021-06-15]  i18n
+        ├── [     2021-06-15]  images
+        ├── [     2023-11-13]  layouts
+        │   ├── [     2023-11-13]  _default
+        │   ├── [     2023-11-13]  partials
+        │   └── [     2021-06-15]  shortcodes
+        ├── [     2021-06-15]  resources
+        │   └── [     2021-06-15]  _gen
+        │       └── [     2021-06-15]  assets
+        │           ├── [     2021-06-15]  css
+        │           │   └── [     2021-06-15]  css
+        │           └── [     2021-06-15]  js
+        │               └── [     2021-06-15]  js
+        └── [     2022-05-04]  static
+            └── [     2021-06-15]  images"
+
+value=$(lomob list dirs)
+
+if [ "$value" = "$expectDirs" ]; then
+  echo "Scan dir lists are same as expected"
+else
+  echo "Scan dir lists are different"
+  echo "Expect:"
+  echo "$expectDirs"
+  echo "Actual:"
+  echo "$value"
+  exit 1
+fi
+
+# table view
+expectDirs="File Counts    Total File Size    Mod Time               Path
+6              15.6 KB            2024-04-17 06:24:25    /go/src/github.com/lomorage/lomo-backup/test/data
+1              84 B               2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/archetypes
+10             37.8 KB            2024-03-17 18:06:22    /go/src/github.com/lomorage/lomo-backup/test/data/content
+20             92.5 KB            2024-03-17 18:06:22    /go/src/github.com/lomorage/lomo-backup/test/data/content/blog
+1              182 B              2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/layouts
+0              0 B                2024-03-31 04:13:14    /go/src/github.com/lomorage/lomo-backup/test/data/public
+0              0 B                2021-07-06 19:31:23    /go/src/github.com/lomorage/lomo-backup/test/data/resources
+0              0 B                2021-07-06 19:31:23    /go/src/github.com/lomorage/lomo-backup/test/data/resources/_gen
+0              0 B                2021-07-06 19:31:23    /go/src/github.com/lomorage/lomo-backup/test/data/resources/_gen/assets
+0              0 B                2021-07-06 19:31:23    /go/src/github.com/lomorage/lomo-backup/test/data/resources/_gen/assets/css
+4              42.5 KB            2021-07-06 19:31:23    /go/src/github.com/lomorage/lomo-backup/test/data/resources/_gen/assets/css/css
+0              0 B                2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/resources/_gen/assets/js
+2              28.4 KB            2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/resources/_gen/assets/js/js
+10             1.6 MB             2024-04-17 06:24:25    /go/src/github.com/lomorage/lomo-backup/test/data/static
+12             77.9 KB            2023-03-03 18:26:15    /go/src/github.com/lomorage/lomo-backup/test/data/static/img
+1              77.9 KB            2023-03-03 18:26:15    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog
+3              21.6 MB            2021-08-26 18:31:00    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/android-frame
+4              610.0 KB           2020-03-30 16:37:12    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/covid19
+1              9.8 KB             2022-10-14 17:50:33    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/flyio
+4              393.1 KB           2021-09-02 05:38:39    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/image-quality
+5              158.6 KB           2020-05-18 15:18:23    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/import_my_cloud
+3              87.8 KB            2023-03-03 18:26:15    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/migrate_from_pi_win
+2              128.9 KB           2020-07-20 04:22:51    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/mypains
+1              25.6 KB            2022-02-12 06:59:17    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/photoprism
+1              238.2 KB           2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/raspberrypi-hd
+5              223.6 KB           2023-03-02 17:18:16    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/set_wd_as_backup
+10             535.2 KB           2020-03-31 02:36:01    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/transfer_pc
+3              140.4 KB           2020-11-17 01:04:43    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/blog/win-docker
+14             42.1 MB            2021-05-03 20:55:52    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/buy
+6              374.1 KB           2021-07-31 07:06:41    /go/src/github.com/lomorage/lomo-backup/test/data/static/img/links
+3              38.1 MB            2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/static/video
+0              0 B                2023-11-13 01:55:39    /go/src/github.com/lomorage/lomo-backup/test/data/themes
+3              2.2 KB             2023-11-13 01:55:39    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo
+1              27 B               2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/archetypes
+0              0 B                2021-07-06 19:31:23    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets
+1              6.3 KB             2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/js
+2              39.1 KB            2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/js/libraries
+2              1.9 KB             2021-07-06 19:31:23    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass
+3              1.8 KB             2021-06-15 23:26:27    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/base
+11             13.1 KB            2021-07-06 19:31:23    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/components
+3              8.2 KB             2021-06-16 06:48:55    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/elements
+3              7.0 KB             2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/assets/sass/utils
+2              2.2 KB             2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/i18n
+4              822.2 KB           2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/images
+0              0 B                2023-11-13 01:55:39    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts
+4              4.8 KB             2023-11-13 01:55:39    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/_default
+10             12.5 KB            2023-11-13 01:55:39    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/partials
+1              215 B              2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/layouts/shortcodes
+0              0 B                2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources
+0              0 B                2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen
+0              0 B                2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets
+0              0 B                2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets/css
+2              21.0 KB            2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets/css/css
+0              0 B                2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets/js
+2              28.4 KB            2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/resources/_gen/assets/js/js
+0              0 B                2022-05-04 19:54:39    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static
+16             354.4 KB           2021-06-15 22:19:06    /go/src/github.com/lomorage/lomo-backup/test/data/themes/evie-hugo/static/images"
+
+value=$(lomob list dirs -t)
+
+if [ "$value" = "$expectDirs" ]; then
+  echo "Scan dir lists in tablew view are same as expected"
+else
+  echo "Scan dir lists in table view are different"
+  echo "Expect:"
+  echo "$expectDirs"
+  echo "Actual:"
+  echo "$value"
+  exit 1
+fi
diff --git a/test/scripts/test-scan.sh b/test/scripts/test-scan.sh
new file mode 100755
index 0000000..f1aa1b5
--- /dev/null
+++ b/test/scripts/test-scan.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+expectTotalDirs=58
+expectTotalFiles=202
+
+# make sure timestamp is correct
+cd ../data
+find . | while read -r file; do git log -1 --format="%ad" --date=iso "$file" | xargs -I{} touch -d {} "$file"; done
+cd ../scripts
+
+# try different parallel scan
+for i in 10 5 1; do
+  echo "scan directory with $i threads"
+  rm ./lomob.db
+  sqlite3 ./lomob.db < ../../common/dbx/schema/1.sql
+
+  lomob scan -t $i ../data
+
+  data=`sqlite3 ./lomob.db "select count(*) from dirs"`
+  if [ "$data" = "$expectTotalDirs" ]; then
+    echo "Number of scanned dirs pass"
+  else
+    echo "Number of scanned dirs is $data while expect $expectTotalDirs, failed!"
+    exit 1
+  fi
+
+  data=`sqlite3 ./lomob.db "select count(*) from files"`
+  if [ "$data" = "$expectTotalFiles" ]; then
+    echo "Number of scanned files pass"
+  else
+    echo "Number of scanned file is $data while expect $expectTotalFiles, failed!"
+    exit 1
+  fi
+done
\ No newline at end of file
diff --git a/test/scripts/test-upload-aws-encrypt.sh b/test/scripts/test-upload-aws-encrypt.sh
new file mode 100755
index 0000000..d9628ae
--- /dev/null
+++ b/test/scripts/test-upload-aws-encrypt.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+set -e
diff --git a/test/scripts/test-upload-aws-raw.sh b/test/scripts/test-upload-aws-raw.sh
new file mode 100755
index 0000000..fe033aa
--- /dev/null
+++ b/test/scripts/test-upload-aws-raw.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -e
+
+echo "Test upload ISO files into aws with raw file"
+
+sqlite3 lomob.db "update files set iso_id=0; update isos set upload_key='', upload_id='', hash_remote='', status=1; delete from parts"
+
+export LOCALSTACK_ENDPOINT="http://localhost:4566"
+export LOMOB_MASTER_KEY=1234
+export AWS_ACCESS_KEY_ID=dummy
+export AWS_SECRET_ACCESS_KEY=dummy
+export AWS_DEFAULT_REGION=us-east-1
+
+#LOCALSTACK_ENDPOINT="http://localhost:4566" LOMOB_MASTER_KEY=1234 AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy AWS_DEFAULT_REGION=us-east-1 lomob iso upload 2021-04-26--2021-07-31.iso
+#lomob iso upload --no-encrypt 2019-04-03--2024-04-17.iso 2021-04-26--2021-04-26.iso 2021-04-26--2021-07-31.iso
+
+for isoFile in 2019-04-03--2024-04-17.iso 2021-04-26--2021-04-26.iso 2021-04-26--2021-07-31.iso; do
+  # clean the left contents
+  curl -X DELETE http://localhost:4566/lomorage/$isoFile
+  curl -X DELETE http://localhost:4566/lomorage/$isoFile.meta.txt
+
+  lomob iso upload --no-encrypt $isoFile
+
+  # download and compare
+  curl -s -o /tmp/tmp.data http://localhost:4566/lomorage/$isoFile
+  expectSHA=$(sha256sum $isoFile | awk -F' ' '{print $1}')
+  actualSHA=$(sha256sum /tmp/tmp.data | awk -F' ' '{print $1}')
+
+  if [ "$expectSHA" = "$actualSHA" ]; then
+    echo "Upload $isoFile success"
+  else
+    echo "Upload $isoFile fail"
+    exit 1
+  fi
+
+  # compare meta with expect files
+  diff $isoFile.meta.txt files_`basename $isoFile .iso`.txt
+  if [ $? -eq 0 ]; then
+    echo "$isoFile metadata file is same as expected"
+  else
+    echo "$isoFile metadata file is different"
+    exit 1
+  fi
+
+  # compare meta with the one uploaded
+  curl -s -o /tmp/tmp.meta http://localhost:4566/lomorage/$isoFile.meta.txt
+  expectSHA=$(sha256sum $isoFile.meta.txt | awk -F' ' '{print $1}')
+  actualSHA=$(sha256sum /tmp/tmp.meta | awk -F' ' '{print $1}')
+
+  if [ "$expectSHA" = "$actualSHA" ]; then
+    echo "$isoFile metadata file is uploaded success as well"
+  else
+    echo "$isoFile metadata file upload fail"
+    exit 1
+  fi
+
+  # same file upload again will skip
+  lomob iso upload --no-encrypt $isoFile > /tmp/tmp
+
+  diff /tmp/tmp duplicate_upload_`basename $isoFile .iso`.txt
+
+  if [ $? -eq 0 ]; then
+    echo "Upload $isoFile again works as expected"
+  else
+    echo "Upload $isoFile again fail"
+    exit 1
+  fi
+done
\ No newline at end of file