Merge pull request #59 from xushiwei/cached

Cached
qiniu · Sep 3, 2023 · 4b0a53d · 4b0a53d
2 parents f66d960 + b1dbb07
commit 4b0a53d
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 16 deletions.
diff --git a/http/fs/cached/cached.go b/http/fs/cached/cached.go
@@ -1,13 +1,18 @@
 package cached
 
 import (
+	"errors"
 	"io"
 	"io/fs"
 	"net/http"
 	"os"
 	"path/filepath"
 )
 
+var (
+	ErrOffline = errors.New("remote filesystem is offline")
+)
+
 const (
 	ModeRemote = fs.ModeSymlink | fs.ModeIrregular
 )
@@ -19,29 +24,36 @@ func IsRemote(mode fs.FileMode) bool {
 // -----------------------------------------------------------------------------------------
 
 type Remote interface {
-	Init(local string)
+	Init(local string, offline bool)
 	Lstat(localFile string) (fs.FileInfo, error)
-	ReaddirAll(localDir string, dir *os.File) (fis []fs.FileInfo, err error)
+	ReaddirAll(localDir string, dir *os.File, offline bool) (fis []fs.FileInfo, err error)
 	SyncLstat(local string, name string) (fs.FileInfo, error)
 	SyncOpen(local string, name string) (http.File, error)
 }
 
 type fsCached struct {
-	local  string
-	remote Remote
+	local   string
+	remote  Remote
+	offline bool
 }
 
 func (p *fsCached) Open(name string) (file http.File, err error) {
 	remote, local := p.remote, p.local
 	localFile := filepath.Join(local, name)
 	fi, err := remote.Lstat(localFile)
 	if os.IsNotExist(err) {
+		if p.offline {
+			return nil, ErrOffline
+		}
 		fi, err = p.remote.SyncLstat(local, name)
 		if err != nil {
 			return
 		}
 	}
 	if IsRemote(fi.Mode()) {
+		if p.offline {
+			return nil, ErrOffline
+		}
 		return remote.SyncOpen(local, name)
 	}
 	f, err := os.Open(localFile)
@@ -50,7 +62,7 @@ func (p *fsCached) Open(name string) (file http.File, err error) {
 		return
 	}
 	if fi.IsDir() {
-		fis, e := remote.ReaddirAll(localFile, f)
+		fis, e := remote.ReaddirAll(localFile, f, p.offline)
 		if e != nil {
 			f.Close()
 			return nil, e
@@ -127,9 +139,13 @@ func (d dirEntry) Type() fs.FileMode {
 
 // -----------------------------------------------------------------------------------------
 
-func New(local string, remote Remote) http.FileSystem {
-	remote.Init(local)
-	return &fsCached{local, remote}
+func New(local string, remote Remote, offline ...bool) http.FileSystem {
+	var isOffline bool
+	if offline != nil {
+		isOffline = offline[0]
+	}
+	remote.Init(local, isOffline)
+	return &fsCached{local, remote, isOffline}
 }
 
 // -----------------------------------------------------------------------------------------
diff --git a/http/fs/lfs/dircache/readdir.go → http/fs/lfs/_dircache/readdir.go b/http/fs/lfs/dircache/readdir.go → http/fs/lfs/_dircache/readdir.go
@@ -13,10 +13,10 @@ const (
 	CacheFileName string = ".cache"
 )
 
-type FnRemoteStat = func(localFile string, fi fs.FileInfo) fs.FileInfo
-type FnIsRemote = func(fi fs.FileInfo, file string) bool
+func remoteStat(localFile string, fi fs.FileInfo) fs.FileInfo
+func isRemote(fi fs.FileInfo, file string) bool
 
-func ReaddirAll(localDir string, dir *os.File, remoteStat FnRemoteStat, isRemote FnIsRemote) (fis []fs.FileInfo, err error) {
+func ReaddirAll(localDir string, dir *os.File) (fis []fs.FileInfo, err error) {
 	cacheFile := filepath.Join(localDir, CacheFileName)
 	b, err := os.ReadFile(cacheFile)
 	if err == nil {

diff --git a/http/fs/lfs/lfs.go b/http/fs/lfs/lfs.go
@@ -79,19 +79,24 @@ func (p *remote) isRemote(fi fs.FileInfo, file string) bool {
 	return ok
 }
 
-func (p *remote) ReaddirAll(localDir string, dir *os.File) (fis []fs.FileInfo, err error) {
+func (p *remote) ReaddirAll(localDir string, dir *os.File, offline bool) (fis []fs.FileInfo, err error) {
 	if fis, err = dir.Readdir(-1); err != nil {
 		return
 	}
-	for i, fi := range fis {
+	n := 0
+	for _, fi := range fis {
 		name := fi.Name()
 		if p.isRemote(fi, name) {
+			if offline {
+				continue
+			}
 			localFile := filepath.Join(localDir, name)
 			fi = remoteStat(localFile, fi)
-			fis[i] = fi
 		}
+		fis[n] = fi
+		n++
 	}
-	return
+	return fis[:n], nil
 }
 
 func (p *remote) Lstat(localFile string) (fi fs.FileInfo, err error) {
@@ -151,7 +156,7 @@ func (p *remote) SyncOpen(local string, name string) (f http.File, err error) {
 	return closer.file, nil
 }
 
-func (p *remote) Init(local string) {
+func (p *remote) Init(local string, offline bool) {
 }
 
 func NewRemote(urlBase string, exts ...string) cached.Remote {
@@ -166,4 +171,8 @@ func NewCached(local string, urlBase string, exts ...string) http.FileSystem {
 	return cached.New(local, NewRemote(urlBase, exts...))
 }
 
+func NewOfflineCached(local string, urlBase string, exts ...string) http.FileSystem {
+	return cached.New(local, NewRemote(urlBase, exts...), true)
+}
+
 // -----------------------------------------------------------------------------------------