Skip to content

Commit

Permalink
internal/wasmtools, all: refinements by @ydnar
Browse files Browse the repository at this point in the history
.github/workflows: remove some (now) unneeded steps

.gitignore: condense internal/wasmtools/.gitignore

internal/wasmtools: use single quotes, add Rust edition

Makefile: update order of wasm-tools.wasm.\* targets)

internal/wasmtools: use gzipped wasm-tools.wasm.gz with sync.Once

internal/wasmtools: rebuild wasm-tools.wasm.gz

CHANGELOG: wordsmith the wasm-tools and Wazero update

internal/wasmtools, wit: remove optional name arg

Removing this did not seem to affect tests.

internal/wasmtools: leave timeout to the caller

Makefile: reorder gzip targets

internal/wasmtools: oops

Makefile: next try

wit: remove check for wasm-tools in PATH

internal/wasmtools, wit: swap key and value types for fsMap

Not all fs.FS are hashable, but all strings are, so use map[string]fs.FS instead.

wit/bindgen: use internal/wasmtools to run wasm-tools in WebAssembly

internal/wasmtools: use wazero.CompilationCache

This speeds up wit/bindgen tests 10x

internal/wasmtools: fix TinyGo impl

internal/wasmtools: remove Runner interface

This wasn’t used anywhere, so removing.

internal/wasmtools: change Run() to accept optional stdout and stderr

internal/wasmtools: move args to varadic trailing arg in Run

wit: (*testing.common).Errorf does not support error-wrapping directive %w

internal/wasmtools, wit/bindgen: additional cleanups for TinyGo and WASI

.github/workflows/test: TinyGo needs wasm-tools for -target=wasip2
  • Loading branch information
ydnar committed Dec 17, 2024
1 parent 557b289 commit 8a1a146
Show file tree
Hide file tree
Showing 18 changed files with 195 additions and 205 deletions.
3 changes: 0 additions & 3 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,6 @@ jobs:
with:
go-version-file: go.mod

- name: Set up wams-tools.wasm
run: make internal/wasmtools/wasm-tools.wasm

- name: Run Go tests
run: go test ${{ env.go-modules }}

Expand Down
24 changes: 2 additions & 22 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,8 @@ jobs:
with:
go-version-file: go.mod

- name: Set up wams-tools.wasm
run: make internal/wasmtools/wasm-tools.wasm

- name: Vet Go code
run: |
make build
go vet ${{ env.go-modules }} ./tests/...
run: go vet ${{ env.go-modules }} ./tests/...

# Test with Go
test-go:
Expand All @@ -61,14 +56,6 @@ jobs:
with:
go-version: ${{ matrix.go-version }}

- name: Set up wasm-tools
uses: bytecodealliance/actions/wasm-tools/setup@v1
with:
version: ${{ env.wasm-tools-version }}

- name: Set up wams-tools.wasm
run: make internal/wasmtools/wasm-tools.wasm

- name: Run Go tests
run: go test -v ${{ env.go-modules }}

Expand Down Expand Up @@ -111,11 +98,6 @@ jobs:
with:
tinygo-version: ${{ matrix.tinygo-version }}

- name: Set up wasm-tools
uses: bytecodealliance/actions/wasm-tools/setup@v1
with:
version: ${{ env.wasm-tools-version }}

- name: Test with TinyGo
run: tinygo test -v ${{ env.go-modules }}

Expand Down Expand Up @@ -150,14 +132,12 @@ jobs:
with:
tinygo-version: ${{ matrix.tinygo-version }}

# TinyGo needs wasm-tools for -target=wasip2
- name: Set up wasm-tools
uses: bytecodealliance/actions/wasm-tools/setup@v1
with:
version: ${{ env.wasm-tools-version }}

- name: Set up wams-tools.wasm
run: make internal/wasmtools/wasm-tools.wasm

- name: Set up Wasmtime
uses: bytecodealliance/actions/wasmtime/setup@v1
with:
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
.DS_Store
/generated
/internal/wasmtools/target
/internal/wasmtools/wasm-tools.wasm
go.work.sum
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

### Added

- `internal/wasmtools` now loads compiled `wasm-tools` Wasm module from the `wasm-tools` crate and executes it using wazero. This allows `wit-bindgen-go` to run on any platform without needing to install `wasm-tools` natively.
- [`wasm-tools`](https://crates.io/crates/wasm-tools) is now vendored as a WebAssembly module, executed using [Wazero](https://wazero.io/). This allows package `wit` and `wit-bindgen-go` to run on any supported platform without needing to separately install `wasm-tools`.

## [v0.5.0] — 2024-12-14

Expand Down
19 changes: 10 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,25 +21,26 @@ generated: clean json
.PHONY: clean
clean:
rm -rf ./generated/*
rm -f internal/wasmtools/wasm-tools.wasm
rm -f internal/wasmtools/wasm-tools.wasm.gz

# tests/generated writes generated Go code to the tests directory
.PHONY: tests/generated
tests/generated: json
go generate ./tests

# build builds the cmd/wit-bindgen-go binary
.PHONY: build
build: internal/wasmtools/wasm-tools.wasm
go build -o wit-bindgen-go ./cmd/wit-bindgen-go
# wasm-tools builds the internal/wasmtools/wasm-tools.wasm.gz artifact
.PHONY: wasm-tools
wasm-tools: internal/wasmtools/target/wasm32-wasip1/release/wasm-tools.wasm
gzip -c $< > internal/wasmtools/wasm-tools.wasm.gz

internal/wasmtools/wasm-tools.wasm: internal/wasmtools/wasm-tools.wasm.gz
gzip -dc $< > $@

internal/wasmtools/wasm-tools.wasm.gz:
internal/wasmtools/target/wasm32-wasip1/release/wasm-tools.wasm: internal/wasmtools/Cargo.*
cd internal/wasmtools && \
cargo build --target wasm32-wasip1 --release -p wasm-tools
gzip -c internal/wasmtools/target/wasm32-wasip1/release/wasm-tools.wasm > $@

# internal/wasmtools/wasm-tools.wasm decompresses wasm-tools.wasm.gz for other make targets
internal/wasmtools/wasm-tools.wasm: internal/wasmtools/wasm-tools.wasm.gz
gzip -dc internal/wasmtools/wasm-tools.wasm.gz > $@

# test runs Go and TinyGo tests
GOTESTARGS :=
Expand Down
2 changes: 0 additions & 2 deletions internal/wasmtools/.gitignore

This file was deleted.

11 changes: 6 additions & 5 deletions internal/wasmtools/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
[package]
name = "wasm-tools-go"
name = 'wasm-tools-go'
edition = '2021'

[dependencies]
wasm-tools = { version = "1.221.0", default-features = false, features = [
"component",
wasm-tools = { version = '1.221.0', default-features = false, features = [
'component',
] }

[profile.release]
lto = true # compile with link-time optimization
codegen-units = 1 # compile with a single codegen unit
opt-level = "z" # optimize for size
panic = "abort" # panic=abort will remove the panic code
opt-level = 'z' # optimize for size
panic = 'abort' # panic=abort will remove the panic code
strip = true # strip the debug information
14 changes: 14 additions & 0 deletions internal/wasmtools/instance.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package wasmtools

import (
"context"
"io"
"io/fs"
)

type instance interface {
Close(ctx context.Context) error
Run(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, fsMap map[string]fs.FS, args ...string) error
}

var _ instance = &Instance{}
102 changes: 102 additions & 0 deletions internal/wasmtools/instance_other.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
//go:build !tinygo

package wasmtools

import (
"bytes"
"compress/gzip"
"context"
"crypto/rand"
_ "embed"
"fmt"
"io"
"io/fs"
"sync"

"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
)

//go:embed wasm-tools.wasm.gz
var compressed []byte

var decompress = sync.OnceValues(func() ([]byte, error) {
r, err := gzip.NewReader(bytes.NewReader(compressed))
if err != nil {
return nil, err
}
defer r.Close()
var buf bytes.Buffer
_, err = buf.ReadFrom(r)
return buf.Bytes(), err
})

var compilationCache = wazero.NewCompilationCache()

// Instance is a compiled wazero instance.
type Instance struct {
runtime wazero.Runtime
module wazero.CompiledModule
}

// New creates a new wazero instance.
func New(ctx context.Context) (*Instance, error) {
c := wazero.NewRuntimeConfig().
WithCloseOnContextDone(true).
WithCompilationCache(compilationCache)

r := wazero.NewRuntimeWithConfig(ctx, c)
if _, err := wasi_snapshot_preview1.Instantiate(ctx, r); err != nil {
return nil, fmt.Errorf("error instantiating WASI: %w", err)
}

wasmTools, err := decompress()
if err != nil {
return nil, err
}

module, err := r.CompileModule(ctx, wasmTools)
if err != nil {
return nil, fmt.Errorf("error compiling wasm module: %w", err)
}
return &Instance{runtime: r, module: module}, nil
}

// Close closes the wazero runtime resource.
func (w *Instance) Close(ctx context.Context) error {
return w.runtime.Close(ctx)
}

// Run runs the wasm module with the context, arguments,
// and optional stdin, stdout, stderr, and filesystem map.
// Supply a context with a timeout or other cancellation mechanism to control execution time.
// Returns an error if instantiation fails.
func (w *Instance) Run(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, fsMap map[string]fs.FS, args ...string) error {
config := wazero.NewModuleConfig().
WithRandSource(rand.Reader).
WithSysNanosleep().
WithSysNanotime().
WithSysWalltime().
WithArgs(append([]string{"wasm-tools.wasm"}, args...)...)

if stdin != nil {
config = config.WithStdin(stdin)
}
if stdout != nil {
config = config.WithStdout(stdout)
}
if stderr != nil {
config = config.WithStderr(stderr)
}

if len(fsMap) != 0 {
fsConfig := wazero.NewFSConfig()
for guestPath, guestFS := range fsMap {
fsConfig = fsConfig.WithFSMount(guestFS, guestPath)
}
config = config.WithFSConfig(fsConfig)
}

_, err := w.runtime.InstantiateModule(ctx, w.module, config)
return err
}
26 changes: 26 additions & 0 deletions internal/wasmtools/instance_tinygo.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
//go:build tinygo

package wasmtools

import (
"context"
"errors"
"io"
"io/fs"
)

var errTinyGo = errors.New("wasm-tools disabled under TinyGo")

type Instance struct{}

func New(ctx context.Context) (*Instance, error) {
return &Instance{}, errTinyGo
}

func (w *Instance) Close(ctx context.Context) error {
return errTinyGo
}

func (w *Instance) Run(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, fsMap map[string]fs.FS, args ...string) error {
return errTinyGo
}
12 changes: 0 additions & 12 deletions internal/wasmtools/types.go

This file was deleted.

Binary file modified internal/wasmtools/wasm-tools.wasm.gz
Binary file not shown.
87 changes: 0 additions & 87 deletions internal/wasmtools/wasmtools.go

This file was deleted.

Loading

0 comments on commit 8a1a146

Please sign in to comment.