wasi: replaces existing filesystem apis with fs.FS

This adds WASIConfig.WithFS, which allows end users to leverage standard
libraries that adhere to this, including OS, HTTP and embedded
filesystems.

For example, here's how you can allow WebAssembly modules to read
"/work/home/a.txt" as "a.txt":
```go
wasiConfig := wazero.NewWASIConfig().
	WithFS(os.DirFS("/work/home"))

wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1WithConfig(wasiConfig))
...
```

Signed-off-by: Adrian Cole <[email protected]>

README.md

@@ -41,12 +41,20 @@ on your machine unless you explicitly allow it.
 
 System access is defined by an emerging specification called WebAssembly
 System Interface ([WASI](https://github.com/WebAssembly/WASI)). WASI defines
-how WebAssembly programs interact with the host embedding them. For example,
-WASI defines functions for reading the time, or a random number.
+how WebAssembly programs interact with the host embedding them.
 
-This repository includes several [examples](examples) that expose system
-interfaces, via the module `wazero.WASISnapshotPreview1`. These examples are
-tested and a good way to learn what's possible with wazero.
+For example, here's how you can allow WebAssembly modules to read
+"/work/home/a.txt" as "a.txt":
+```go
+wasiConfig := wazero.NewWASIConfig().
+	WithFS(os.DirFS("/work/home"))
+
+wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1WithConfig(wasiConfig))
+...
+```
+
+The best way to learn this and other features you get with wazero is by trying
+[examples](examples).
 
 ## Runtime
 

examples/file_system_test.go

@@ -2,66 +2,49 @@ package examples
 
 import (
 	"bytes"
+	"embed"
 	_ "embed"
-	"io"
+	"io/fs"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/tetratelabs/wazero"
-	"github.com/tetratelabs/wazero/wasi"
 )
 
-// filesystemWasm was compiled from TinyGo testdata/file_system.go
-//go:embed testdata/file_system.wasm
-var filesystemWasm []byte
+// catFS is an embedded filesystem limited to cat.go
+//go:embed testdata/cat.go
+var catFS embed.FS
 
-func writeFile(fs wasi.FS, path string, data []byte) error {
-	f, err := fs.OpenWASI(0, path, wasi.O_CREATE|wasi.O_TRUNC, wasi.R_FD_WRITE, 0, 0)
-	if err != nil {
-		return err
-	}
+// catGo is the TinyGo source
+//go:embed testdata/cat.go
+var catGo string
 
-	if _, err := io.Copy(f, bytes.NewBuffer(data)); err != nil {
-		return err
-	}
+// catWasm was compiled from catGo
+//go:embed testdata/cat.wasm
+var catWasm []byte
 
-	return f.Close()
-}
-
-func readFile(fs wasi.FS, path string) ([]byte, error) {
-	f, err := fs.OpenWASI(0, path, 0, 0, 0, 0)
-	if err != nil {
-		return nil, err
-	}
-
-	buf := bytes.NewBuffer(nil)
-
-	if _, err := io.Copy(buf, f); err != nil {
-		return buf.Bytes(), nil
-	}
-
-	return buf.Bytes(), f.Close()
-}
-
-func Test_file_system(t *testing.T) {
+// Test_Cat writes the input file to stdout, just like `cat`.
+func Test_Cat(t *testing.T) {
 	r := wazero.NewRuntime()
+	stdoutBuf := bytes.NewBuffer(nil)
 
-	memFS := wazero.WASIMemFS()
-	err := writeFile(memFS, "input.txt", []byte("Hello, file system!"))
+	// Since wazero uses fs.FS we can use standard libraries to do things like trim the leading path.
+	rooted, err := fs.Sub(catFS, "testdata")
 	require.NoError(t, err)
 
-	wasiConfig := wazero.NewWASIConfig().WithPreopens(map[string]wasi.FS{".": memFS})
+	// Next, setup stdout so we can verify it. Then configure the filesystem to tell cat to print itself!
+	file := "cat.go"
+	wasiConfig := wazero.NewWASIConfig().WithStdout(stdoutBuf).WithFS(rooted).WithArgs(file)
 	wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1WithConfig(wasiConfig))
 	require.NoError(t, err)
 	defer wasi.Close()
 
-	// Note: TinyGo binaries must be treated as WASI Commands to initialize memory.
-	mod, err := wazero.StartWASICommandFromSource(r, filesystemWasm)
+	// Finally, start the program which executes the main function (compiled to Wasm as _start).
+	mod, err := wazero.StartWASICommandFromSource(r, catWasm)
 	require.NoError(t, err)
 	defer mod.Close()
 
-	out, err := readFile(memFS, "output.txt")
-	require.NoError(t, err)
-	require.Equal(t, "Hello, file system!", string(out))
+	// To ensure it worked, this verifies stdout from WebAssembly had what we expected.
+	require.Equal(t, catGo, stdoutBuf.String())
 }

examples/testdata/cat.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+// main is the same as cat.
+func main() {
+	for _, file := range os.Args {
+		bytes, err := os.ReadFile(file)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "couldn't read %s: %v", file, err)
+			os.Exit(1)
+		}
+
+		// Use write to avoid needing to worry about Windows newlines.
+		os.Stdout.Write(bytes)
+	}
+}

internal/wasi/wasi.go

@@ -10,6 +10,7 @@ import (
 	"io/fs"
 	"math"
 	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -1113,14 +1114,11 @@ func (a *wasiAPI) FdAllocate(ctx wasm.Module, fd uint32, offset, len uint64) was
 func (a *wasiAPI) FdClose(ctx wasm.Module, fd uint32) wasi.Errno {
 	cfg := a.config(ctx.Context())
 
-	f, ok := cfg.opened[fd]
-	if !ok {
+	if f := cfg.opened[fd]; f == nil {
 		return wasi.ErrnoBadf
-	}
-
-	if f.file != nil {
-		f.file.Close()
-	}
+	} else if f.file != nil {
+		f.file.Close() // TODO: maybe return a failure Errno if we can't close it
+	} // else it was a stdin stdout or stderr
 
 	delete(cfg.opened, fd)
 
@@ -1133,25 +1131,21 @@ func (a *wasiAPI) FdDatasync(ctx wasm.Module, fd uint32) wasi.Errno {
 }
 
 // FdFdstatGet implements SnapshotPreview1.FdFdstatGet
-// TODO: Currently FdFdstatget implements nothing except returning fake fs_right_inheriting
 func (a *wasiAPI) FdFdstatGet(ctx wasm.Module, fd uint32, resultStat uint32) wasi.Errno {
 	cfg := a.config(ctx.Context())
 
 	if _, ok := cfg.opened[fd]; !ok {
 		return wasi.ErrnoBadf
 	}
-	if !ctx.Memory().WriteUint64Le(resultStat+16, rightFDRead|rightFDWrite) {
-		return wasi.ErrnoFault
-	}
 	return wasi.ErrnoSuccess
 }
 
 // FdPrestatGet implements SnapshotPreview1.FdPrestatGet
 func (a *wasiAPI) FdPrestatGet(ctx wasm.Module, fd uint32, resultPrestat uint32) wasi.Errno {
 	cfg := a.config(ctx.Context())
 
-	entry, ok := cfg.opened[fd]
-	if !ok || entry.path == "" {
+	entry := cfg.opened[fd]
+	if entry == nil {
 		return wasi.ErrnoBadf
 	}
 
@@ -1173,6 +1167,7 @@ func (a *wasiAPI) FdFdstatSetFlags(ctx wasm.Module, fd uint32, flags uint32) was
 }
 
 // FdFdstatSetRights implements SnapshotPreview1.FdFdstatSetRights
+// Note: This will never be implemented per https://github.com/WebAssembly/WASI/issues/469#issuecomment-1045251844
 func (a *wasiAPI) FdFdstatSetRights(ctx wasm.Module, fd uint32, fsRightsBase, fsRightsInheriting uint64) wasi.Errno {
 	return wasi.ErrnoNosys // stubbed for GrainLang per #271
 }
@@ -1229,14 +1224,11 @@ func (a *wasiAPI) FdRead(ctx wasm.Module, fd, iovs, iovsCount, resultSize uint32
 
 	var reader io.Reader
 
-	switch fd {
-	case 0:
+	if fd == fdStdin {
 		reader = cfg.stdin
-	default:
-		f, ok := cfg.opened[fd]
-		if !ok || f.file == nil {
-			return wasi.ErrnoBadf
-		}
+	} else if f := cfg.opened[fd]; f == nil || f.file == nil {
+		return wasi.ErrnoBadf
+	} else {
 		reader = f.file
 	}
 
@@ -1283,24 +1275,24 @@ func (a *wasiAPI) FdRenumber(ctx wasm.Module, fd, to uint32) wasi.Errno {
 func (a *wasiAPI) FdSeek(ctx wasm.Module, fd uint32, offset uint64, whence uint32, resultNewoffset uint32) wasi.Errno {
 	cfg := a.config(ctx.Context())
 
-	f, ok := cfg.opened[fd]
-	if !ok || f.file == nil {
+	var seeker io.Seeker
+	// Check to see if the file descriptor is available
+	if f, ok := cfg.opened[fd]; !ok || f.file == nil {
 		return wasi.ErrnoBadf
-	}
-	seeker, ok := f.file.(io.Seeker)
-	if !ok {
+		// fs.FS doesn't declare io.Seeker, but implementations such as os.File implement it.
+	} else if seeker, ok = f.file.(io.Seeker); !ok {
 		return wasi.ErrnoBadf
 	}
 
 	if whence > io.SeekEnd /* exceeds the largest valid whence */ {
 		return wasi.ErrnoInval
 	}
-	newOffst, err := seeker.Seek(int64(offset), int(whence))
+	newOffset, err := seeker.Seek(int64(offset), int(whence))
 	if err != nil {
 		return wasi.ErrnoIo
 	}
 
-	if !ctx.Memory().WriteUint32Le(resultNewoffset, uint32(newOffst)) {
+	if !ctx.Memory().WriteUint32Le(resultNewoffset, uint32(newOffset)) {
 		return wasi.ErrnoFault
 	}
 
@@ -1324,16 +1316,18 @@ func (a *wasiAPI) FdWrite(ctx wasm.Module, fd, iovs, iovsCount, resultSize uint3
 	var writer io.Writer
 
 	switch fd {
-	case 1:
+	case fdStdout:
 		writer = cfg.stdout
-	case 2:
+	case fdStderr:
 		writer = cfg.stderr
 	default:
-		f, ok := cfg.opened[fd]
-		if !ok || f.file == nil {
+		// Check to see if the file descriptor is available
+		if f, ok := cfg.opened[fd]; !ok || f.file == nil {
+			return wasi.ErrnoBadf
+			// fs.FS doesn't declare io.Writer, but implementations such as os.File implement it.
+		} else if writer, ok = f.file.(io.Writer); !ok {
 			return wasi.ErrnoBadf
 		}
-		writer = f.file
 	}
 
 	var nwritten uint32
@@ -1383,87 +1377,69 @@ func (a *wasiAPI) PathLink(ctx wasm.Module, oldFd, oldFlags, oldPath, oldPathLen
 	return wasi.ErrnoNosys // stubbed for GrainLang per #271
 }
 
-const (
-	// WASI open flags
-	oflagCreate = 1 << iota
-	// TODO: oflagDir
-	oflagExclusive
-	oflagTrunc
-
-	// WASI FS rights
-	rightFDRead  = 1 << iota
-	rightFDWrite = 0x200
-)
-
-func posixOpenFlags(oFlags uint32, fsRights uint64) (pFlags int) {
-	// TODO: handle dirflags, which decides whether to follow symbolic links or not,
-	//       by O_NOFOLLOW. Note O_NOFOLLOW doesn't exist on Windows.
-	if fsRights&rightFDWrite != 0 {
-		if fsRights&rightFDRead != 0 {
-			pFlags |= os.O_RDWR
-		} else {
-			pFlags |= os.O_WRONLY
-		}
-	}
-	if oFlags&oflagCreate != 0 {
-		pFlags |= os.O_CREATE
-	}
-	if oFlags&oflagExclusive != 0 {
-		pFlags |= os.O_EXCL
-	}
-	if oFlags&oflagTrunc != 0 {
-		pFlags |= os.O_TRUNC
-	}
-	return
-}
-
 // PathOpen implements SnapshotPreview1.PathOpen
+// Note: Rights will never be implemented per https://github.com/WebAssembly/WASI/issues/469#issuecomment-1045251844
 func (a *wasiAPI) PathOpen(ctx wasm.Module, fd, dirflags, pathPtr, pathLen, oflags uint32, fsRightsBase,
 	fsRightsInheriting uint64, fdflags, resultOpenedFd uint32) (errno wasi.Errno) {
 	cfg := a.config(ctx.Context())
 
-	dir, ok := cfg.opened[fd]
-	if !ok || dir.fileSys == nil {
+	dir := cfg.opened[fd]
+	if dir == nil || dir.fs == nil {
 		return wasi.ErrnoBadf
 	}
 
 	b, ok := ctx.Memory().Read(pathPtr, pathLen)
 	if !ok {
 		return wasi.ErrnoFault
 	}
-	pathName := string(b)
-	f, err := dir.fileSys.OpenWASI(dirflags, pathName, oflags, fsRightsBase, fsRightsInheriting, fdflags)
-	if err != nil {
-		switch {
-		case errors.Is(err, fs.ErrNotExist):
-			return wasi.ErrnoNoent
-		case errors.Is(err, fs.ErrExist):
-			return wasi.ErrnoExist
-		default:
-			return wasi.ErrnoIo
-		}
-	}
-
-	// when ofagDir is set, the opened file must be a directory.
-	// TODO if oflags&oflagDir != 0 return wasi.ErrnoNotdir if stat != dir
 
 	newFD, err := a.randUnusedFD(cfg)
 	if err != nil {
 		return wasi.ErrnoIo
 	}
 
-	cfg.opened[newFD] = fileEntry{
-		path:    pathName,
-		fileSys: dir.fileSys,
-		file:    f,
+	// TODO: Consider dirflags and oflags. Also, allow non-read-only open based on config about the mount.
+	// Ex. allow os.O_RDONLY, os.O_WRONLY, or os.O_RDWR either by config flag or pattern on filename
+	// See #390
+	entry, errno := openFileEntry(dir.fs, path.Join(dir.path, string(b)))
+	if errno != wasi.ErrnoSuccess {
+		return errno
 	}
+	cfg.opened[newFD] = entry
 
 	if !ctx.Memory().WriteUint32Le(resultOpenedFd, newFD) {
 		return wasi.ErrnoFault
 	}
 	return wasi.ErrnoSuccess
 }
 
+func openFileEntry(rootFS fs.FS, pathName string) (*fileEntry, wasi.Errno) {
+	// Clean the path because fs.FS Open fails requires fs.ValidPath(path).
+	pathName = path.Clean(pathName)
+
+	// Root paths are relative, as there is no CWD in WASI
+	if path.IsAbs(pathName) {
+		pathName = pathName[1:]
+	}
+
+	f, err := rootFS.Open(pathName)
+	if err != nil {
+		switch {
+		case errors.Is(err, fs.ErrNotExist):
+			return nil, wasi.ErrnoNoent
+		case errors.Is(err, fs.ErrExist):
+			return nil, wasi.ErrnoExist
+		default:
+			return nil, wasi.ErrnoIo
+		}
+	}
+
+	// TODO: verify if oflags is a directory and fail with wasi.ErrnoNotdir if not
+	// See https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#-oflags-flagsu16
+
+	return &fileEntry{path: pathName, fs: rootFS, file: f}, wasi.ErrnoSuccess
+}
+
 // PathReadlink implements SnapshotPreview1.PathReadlink
 func (a *wasiAPI) PathReadlink(ctx wasm.Module, fd, path, pathLen, buf, bufLen, resultBufused uint32) wasi.Errno {
 	return wasi.ErrnoNosys // stubbed for GrainLang per #271
@@ -1544,10 +1520,17 @@ func (a *wasiAPI) SockShutdown(ctx wasm.Module, fd, how uint32) wasi.Errno {
 	return wasi.ErrnoNosys // stubbed for GrainLang per #271
 }
 
+const (
+	fdStdin  = 0
+	fdStdout = 1
+	fdStderr = 2
+)
+
 type fileEntry struct {
-	path    string
-	fileSys wasi.FS
-	file    wasi.File
+	path string
+	// fs is nil for fdStdin fdStdout and fdStderr
+	fs   fs.FS
+	file fs.File
 }
 
 // ConfigContextKey indicates a context.Context includes an overriding Config.
@@ -1561,7 +1544,7 @@ type Config struct {
 	stdin   io.Reader
 	stdout,
 	stderr io.Writer
-	opened map[uint32]fileEntry
+	opened map[uint32]*fileEntry
 	// timeNowUnixNano is mutable for testing
 	timeNowUnixNano func() uint64
 	randSource      func([]byte) error
@@ -1614,11 +1597,8 @@ func (c *Config) Environ(environ ...string) error {
 	return nil
 }
 
-func (c *Config) Preopen(dir string, fileSys wasi.FS) {
-	c.opened[uint32(len(c.opened))+3] = fileEntry{
-		path:    dir,
-		fileSys: fileSys,
-	}
+func (c *Config) FS(rootFS fs.FS) {
+	c.opened[uint32(len(c.opened))+3] = &fileEntry{path: "", fs: rootFS}
 }
 
 // NewConfig sets configuration defaults
@@ -1629,7 +1609,7 @@ func NewConfig() *Config {
 		stdin:   os.Stdin,
 		stdout:  os.Stdout,
 		stderr:  os.Stderr,
-		opened:  map[uint32]fileEntry{},
+		opened:  map[uint32]*fileEntry{},
 		timeNowUnixNano: func() uint64 {
 			return uint64(time.Now().UnixNano())
 		},

wasi.go

@@ -4,33 +4,21 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/fs"
 
 	internalwasi "github.com/tetratelabs/wazero/internal/wasi"
 	internalwasm "github.com/tetratelabs/wazero/internal/wasm"
 	"github.com/tetratelabs/wazero/wasi"
 	"github.com/tetratelabs/wazero/wasm"
 )
 
-// WASIDirFS returns a file system (a wasi.FS) for the tree of files rooted at
-// the directory dir. It's similar to os.DirFS, except that it implements
-// wasi.FS instead of the fs.FS interface.
-func WASIDirFS(dir string) wasi.FS {
-	return internalwasi.DirFS(dir)
-}
-
-func WASIMemFS() wasi.FS {
-	return &internalwasi.MemFS{
-		Files: map[string][]byte{},
-	}
-}
-
 type WASIConfig struct {
-	stdin    io.Reader
-	stdout   io.Writer
-	stderr   io.Writer
-	args     []string
-	environ  map[string]string
-	preopens map[string]wasi.FS
+	stdin   io.Reader
+	stdout  io.Writer
+	stderr  io.Writer
+	args    []string
+	environ map[string]string
+	rootFS  fs.FS
 }
 
 func NewWASIConfig() *WASIConfig {
@@ -62,8 +50,15 @@ func (c *WASIConfig) WithEnviron(environ map[string]string) *WASIConfig {
 	return c
 }
 
-func (c *WASIConfig) WithPreopens(preopens map[string]wasi.FS) *WASIConfig {
-	c.preopens = preopens
+// WithFS mounts the given fs.FS as a filesystem accessible to WASI.
+//
+// Files are available in WASI under the relative path. For example, if your FS has "example", the file should be looked
+// up the same way.
+//
+// Note: This doesn't currently support writing files.
+// See https://github.com/tetratelabs/wazero/issues/390
+func (c *WASIConfig) WithFS(rootFS fs.FS) *WASIConfig {
+	c.rootFS = rootFS
 	return c
 }
 
@@ -109,11 +104,7 @@ func newConfig(c *WASIConfig) *internalwasi.Config {
 			panic(err) // better to panic vs have bother users about unlikely size > uint32
 		}
 	}
-	if len(c.preopens) > 0 {
-		for k, v := range c.preopens {
-			cfg.Preopen(k, v)
-		}
-	}
+	cfg.FS(c.rootFS)
 	return cfg
 }
 

wasi/wasi.go

@@ -3,7 +3,6 @@ package wasi
 
 import (
 	"fmt"
-	"io"
 )
 
 const (
@@ -12,40 +11,6 @@ const (
 	ModuleSnapshotPreview1 = "wasi_snapshot_preview1"
 )
 
-// TODO: rename these according to other naming conventions
-const (
-	// WASI open flags
-
-	O_CREATE = 1 << iota
-	O_DIR
-	O_EXCL
-	O_TRUNC
-
-	// WASI fs rights
-
-	R_FD_READ = 1 << iota
-	R_FD_SEEK
-	R_FD_FDSTAT_SET_FLAGS
-	R_FD_SYNC
-	R_FD_TELL
-	R_FD_WRITE
-)
-
-// File combines file I/O interfaces supported by ModuleSnapshotPreview1.
-type File interface {
-	io.Reader
-	io.Writer
-	io.Seeker
-	io.Closer
-}
-
-// FS is an interface for a preopened directory.
-type FS interface {
-	// OpenWASI is a general method to open a file, similar to
-	// os.OpenFile, but with WASI flags and rights instead of POSIX.
-	OpenWASI(dirFlags uint32, path string, oFlags uint32, fsRights, fsRightsInheriting uint64, fdFlags uint32) (File, error)
-}
-
 // Errno are the error codes returned by WASI functions.
 //
 // Note: This is not always an error, as ErrnoSuccess is a valid code.