x/http/fs/cached; x/http/fs/lfs

.gitignore

@@ -6,6 +6,7 @@
 # Folders
 _obj
 _test
+_ttt
 
 # Architecture specific extensions/prefixes
 *.[568vq]

http/fs/cached/cached.go

@@ -0,0 +1,46 @@
+package cached
+
+import (
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type Remote interface {
+	Init(local string)
+	Lstat(localFile string) (fs.FileInfo, error)
+	SyncLstat(local string, name string) (fs.FileInfo, error)
+	SyncOpen(local string, name string) (http.File, error)
+}
+
+type fsCached struct {
+	local  string
+	remote Remote
+}
+
+func (p *fsCached) Open(name string) (f http.File, err error) {
+	if !strings.HasPrefix(name, "/") { // name should start with "/"
+		name = "/" + name
+	}
+	remote, local := p.remote, p.local
+	localFile := filepath.Join(local, name)
+	fi, err := remote.Lstat(localFile)
+	if os.IsNotExist(err) {
+		fi, err = p.remote.SyncLstat(local, name)
+		if err != nil {
+			return
+		}
+	}
+	mode := fi.Mode()
+	if (mode & fs.ModeSymlink) != 0 {
+		return remote.SyncOpen(local, name)
+	}
+	return os.Open(localFile)
+}
+
+func New(local string, remote Remote) http.FileSystem {
+	remote.Init(local)
+	return &fsCached{local, remote}
+}

http/fs/file.go

@@ -62,8 +62,12 @@ func (p *dataFile) Close() error {
 	return nil
 }
 
+func (p *dataFile) ReadDir(n int) ([]fs.DirEntry, error) {
+	return nil, os.ErrInvalid
+}
+
 func (p *dataFile) Readdir(count int) ([]fs.FileInfo, error) {
-	return nil, nil
+	return nil, os.ErrInvalid
 }
 
 func (p *dataFile) Stat() (fs.FileInfo, error) {
@@ -93,7 +97,7 @@ func (p *filesDataFS) Open(name string) (f http.File, err error) {
 	return nil, os.ErrNotExist
 }
 
-// FilesWithContent implenets a http.FileSystem by a list of file name and content.
+// FilesWithContent implements a http.FileSystem by a list of file name and content.
 func FilesWithContent(files ...string) http.FileSystem {
 	return &filesDataFS{files}
 }
@@ -116,7 +120,7 @@ func (p *filesFS) Open(name string) (f http.File, err error) {
 	return nil, os.ErrNotExist
 }
 
-// Files implenets a http.FileSystem by a list of file name and content file.
+// Files implements a http.FileSystem by a list of file name and content file.
 func Files(files ...string) http.FileSystem {
 	return &filesFS{files}
 }

http/fs/fs.go

@@ -5,6 +5,7 @@ import (
 	"io/fs"
 	"net/http"
 	"os"
+	"path"
 	"time"
 )
 
@@ -31,6 +32,35 @@ func Union(fs ...http.FileSystem) http.FileSystem {
 
 // -----------------------------------------------------------------------------------------
 
+type fsPlugins struct {
+	fs   http.FileSystem
+	exts map[string]Opener
+}
+
+func (p *fsPlugins) Open(name string) (http.File, error) {
+	ext := path.Ext(name)
+	if fn, ok := p.exts[ext]; ok {
+		return fn(p.fs, name)
+	}
+	return p.fs.Open(name)
+}
+
+type Opener = func(fs http.FileSystem, name string) (file http.File, err error)
+
+// Plugins implements a filesystem with plugins by specified (ext string, plugin Opener) pairs.
+func Plugins(fs http.FileSystem, plugins ...interface{}) http.FileSystem {
+	n := len(plugins)
+	exts := make(map[string]Opener, n/2)
+	for i := 0; i < n; i += 2 {
+		ext := plugins[i].(string)
+		fn := plugins[i+1].(Opener)
+		exts[ext] = fn
+	}
+	return &fsPlugins{fs, exts}
+}
+
+// -----------------------------------------------------------------------------------------
+
 type rootDir struct {
 }
 
@@ -70,6 +100,10 @@ func (p rootDir) Read(b []byte) (n int, err error) {
 	return 0, io.EOF
 }
 
+func (p rootDir) ReadDir(n int) ([]fs.DirEntry, error) {
+	return nil, io.EOF
+}
+
 func (p rootDir) Readdir(count int) ([]fs.FileInfo, error) {
 	return nil, io.EOF
 }
@@ -89,7 +123,7 @@ func (p rootDir) Open(name string) (f http.File, err error) {
 	return nil, os.ErrNotExist
 }
 
-// Root implents a http.FileSystem that only have a root directory.
+// Root implements a http.FileSystem that only have a root directory.
 func Root() http.FileSystem {
 	return rootDir{}
 }

http/fs/gzip/gzip.go

@@ -1,46 +1,21 @@
 package gzip
 
 import (
-	"bytes"
 	"compress/gzip"
-	"io"
 	"net/http"
-	"path"
 
-	"github.com/qiniu/x/http/fs"
+	xfs "github.com/qiniu/x/http/fs"
 )
 
-type fsGzip struct {
-	fs   http.FileSystem
-	exts map[string]struct{}
-}
-
-func (p *fsGzip) Open(name string) (file http.File, err error) {
-	file, err = p.fs.Open(name)
+func Open(fs http.FileSystem, name string) (file http.File, err error) {
+	file, err = fs.Open(name)
 	if err != nil {
 		return
 	}
-	ext := path.Ext(name)
-	if _, ok := p.exts[ext]; !ok {
-		return
-	}
 	defer file.Close()
 	gr, err := gzip.NewReader(file)
 	if err != nil {
 		return
 	}
-	defer gr.Close()
-	b, err := io.ReadAll(gr)
-	if err != nil {
-		return
-	}
-	return fs.File(name, bytes.NewReader(b)), nil
-}
-
-func New(fs http.FileSystem, exts ...string) http.FileSystem {
-	m := make(map[string]struct{}, len(exts))
-	for _, ext := range exts {
-		m[ext] = struct{}{}
-	}
-	return &fsGzip{fs, m}
+	return xfs.SequenceFile(name, gr), nil
 }

http/fs/lfs/lfs.go

@@ -0,0 +1,162 @@
+package lfs
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	xfs "github.com/qiniu/x/http/fs"
+	"github.com/qiniu/x/http/fs/cached"
+)
+
+// -----------------------------------------------------------------------------------------
+
+type cachedCloser struct {
+	io.ReadCloser
+	file      http.File
+	localFile string
+}
+
+func (p *cachedCloser) Close() error {
+	file := p.file
+	_, err := file.Seek(0, io.SeekStart)
+	if err == nil {
+		localFile := p.localFile
+		localFileDownloading := localFile + ".download" // TODO: use tempfile
+		err = download(localFileDownloading, file)
+		if err == nil {
+			err = os.Rename(localFileDownloading, localFile)
+			if err != nil {
+				log.Println("Cache failed:", err)
+			}
+		}
+	}
+	return p.ReadCloser.Close()
+}
+
+func download(destFile string, src http.File) (err error) {
+	f, err := os.Create(destFile)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	if tr, ok := src.(interface{ TryReader() *bytes.Reader }); ok {
+		if r := tr.TryReader(); r != nil {
+			_, err = r.WriteTo(f)
+			return
+		}
+	}
+	_, err = io.Copy(f, src)
+	return
+}
+
+// -----------------------------------------------------------------------------------------
+
+type fileInfo struct {
+	fs.FileInfo
+	size int64
+}
+
+func (p *fileInfo) Size() int64 {
+	return p.size
+}
+
+func (p *fileInfo) Mode() fs.FileMode {
+	return p.FileInfo.Mode() | fs.ModeSymlink
+}
+
+func (p *fileInfo) IsDir() bool {
+	return false
+}
+
+func (p *fileInfo) Sys() interface{} {
+	return nil
+}
+
+type remote struct {
+	exts map[string]struct{}
+	root string
+}
+
+const (
+	lfsSpec = "version https://git-lfs.github.com/spec/"
+	lfsSize = "size "
+)
+
+func (p *remote) Lstat(localFile string) (fi fs.FileInfo, err error) {
+	fi, err = os.Lstat(localFile)
+	if err != nil {
+		return
+	}
+	mode := fi.Mode()
+	if !mode.IsRegular() || fi.Size() > 255 {
+		return
+	}
+	ext := filepath.Ext(localFile)
+	if _, ok := p.exts[ext]; !ok {
+		return
+	}
+	b, e := os.ReadFile(localFile)
+	text := string(b)
+	if e != nil || !strings.HasPrefix(text, lfsSpec) {
+		return
+	}
+
+	lines := strings.SplitN(text, "\n", 4)
+	for _, line := range lines {
+		if strings.HasPrefix(line, lfsSize) {
+			if size, e := strconv.ParseInt(line[len(lfsSize):], 10, 64); e == nil {
+				return &fileInfo{fi, size}, nil
+			}
+			break
+		}
+	}
+	return
+}
+
+func (p *remote) SyncLstat(local string, name string) (fs.FileInfo, error) {
+	return nil, os.ErrNotExist
+}
+
+func (p *remote) SyncOpen(local string, name string) (f http.File, err error) {
+	resp, err := http.Get(p.root + name)
+	if err != nil {
+		return
+	}
+	if resp.StatusCode >= 400 {
+		url := "url"
+		if req := resp.Request; req != nil {
+			url = req.URL.String()
+		}
+		return nil, fmt.Errorf("http.Get %s error: status %d (%s)", url, resp.StatusCode, resp.Status)
+	}
+	localFile := filepath.Join(local, name)
+	closer := &cachedCloser{resp.Body, nil, localFile}
+	resp.Body = closer
+	closer.file = xfs.HttpFile(name, resp)
+	return closer.file, nil
+}
+
+func (p *remote) Init(local string) {
+}
+
+func NewRemote(root string, exts ...string) cached.Remote {
+	m := make(map[string]struct{}, len(exts))
+	for _, ext := range exts {
+		m[ext] = struct{}{}
+	}
+	return &remote{m, root}
+}
+
+func NewCached(local string, root string, exts ...string) http.FileSystem {
+	return cached.New(local, NewRemote(root, exts...))
+}
+
+// -----------------------------------------------------------------------------------------