feat(gnovm): packages loader

.github/workflows/examples.yml

@@ -29,7 +29,7 @@ jobs:
         with:
           go-version: ${{ matrix.goversion }}
       - run: go install -v ./gnovm/cmd/gno
-      - run: go run ./gnovm/cmd/gno transpile -v --gobuild ./examples
+      - run: go run ./gnovm/cmd/gno transpile -v --gobuild ./examples/...
   test:
     strategy:
       fail-fast: false

contribs/gnodev/pkg/dev/node.go

@@ -16,7 +16,7 @@ import (
 	"github.com/gnolang/gno/gno.land/pkg/gnoland"
 	"github.com/gnolang/gno/gno.land/pkg/gnoland/ugnot"
 	"github.com/gnolang/gno/gno.land/pkg/integration"
-	"github.com/gnolang/gno/gnovm/pkg/gnomod"
+	"github.com/gnolang/gno/gnovm/pkg/packages"
 	"github.com/gnolang/gno/tm2/pkg/amino"
 	tmcfg "github.com/gnolang/gno/tm2/pkg/bft/config"
 	"github.com/gnolang/gno/tm2/pkg/bft/node"
@@ -44,6 +44,7 @@ type NodeConfig struct {
 	MaxGasPerBlock        int64
 	ChainID               string
 	ChainDomain           string
+	LoadConfig            *packages.LoadConfig
 }
 
 func DefaultNodeConfig(rootdir, domain string) *NodeConfig {
@@ -70,6 +71,7 @@ func DefaultNodeConfig(rootdir, domain string) *NodeConfig {
 		TMConfig:              tmc,
 		SkipFailingGenesisTxs: true,
 		MaxGasPerBlock:        10_000_000_000,
+		LoadConfig:            &packages.LoadConfig{Deps: true},
 	}
 }
 
@@ -98,7 +100,7 @@ type Node struct {
 var DefaultFee = std.NewFee(50000, std.MustParseCoin(ugnot.ValueString(1000000)))
 
 func NewDevNode(ctx context.Context, cfg *NodeConfig) (*Node, error) {
-	mpkgs, err := NewPackagesMap(cfg.PackagesPathList)
+	mpkgs, err := NewPackagesMap(cfg.LoadConfig, cfg.PackagesPathList)
 	if err != nil {
 		return nil, fmt.Errorf("unable map pkgs list: %w", err)
 	}
@@ -140,7 +142,7 @@ func (n *Node) Close() error {
 	return n.Node.Stop()
 }
 
-func (n *Node) ListPkgs() []gnomod.Pkg {
+func (n *Node) ListPkgs() []*packages.Package {
 	n.muNode.RLock()
 	defer n.muNode.RUnlock()
 
@@ -240,20 +242,20 @@ func (n *Node) updatePackages(paths ...string) error {
 		}
 
 		// List all packages from target path
-		pkgslist, err := gnomod.ListPkgs(abspath)
+		pkgslist, err := packages.Load(n.config.LoadConfig, filepath.Join(abspath, "..."))
 		if err != nil {
 			return fmt.Errorf("failed to list gno packages for %q: %w", path, err)
 		}
 
 		// Update or add package in the current known list.
 		for _, pkg := range pkgslist {
 			n.pkgs[pkg.Dir] = Package{
-				Pkg:     pkg,
+				Package: pkg,
 				Creator: deployer,
 				Deposit: deposit,
 			}
 
-			n.logger.Debug("pkgs update", "name", pkg.Name, "path", pkg.Dir)
+			n.logger.Debug("pkgs update", "name", pkg.ImportPath, "path", pkg.Dir)
 		}
 
 		pkgsUpdated += len(pkgslist)

contribs/gnodev/pkg/dev/node_state_test.go

@@ -158,7 +158,7 @@ func Render(_ string) string { return strconv.Itoa(value) }
 
 	// Call NewDevNode with no package should work
 	node, emitter := newTestingDevNode(t, counterPkg)
-	assert.Len(t, node.ListPkgs(), 1)
+	assert.Len(t, node.ListPkgs(), 23)
 
 	// Test rendering
 	render, err := testingRenderRealm(t, node, testCounterRealm)

contribs/gnodev/pkg/dev/node_test.go

@@ -499,7 +499,6 @@ func newTestingDevNodeWithConfig(t *testing.T, cfg *NodeConfig) (*Node, *mock.Se
 
 	node, err := NewDevNode(ctx, cfg)
 	require.NoError(t, err)
-	assert.Len(t, node.ListPkgs(), len(cfg.PackagesPathList))
 
 	t.Cleanup(func() {
 		node.Close()

contribs/gnodev/pkg/dev/packages.go

@@ -10,8 +10,8 @@ import (
 	"github.com/gnolang/gno/contribs/gnodev/pkg/address"
 	"github.com/gnolang/gno/gno.land/pkg/gnoland"
 	vmm "github.com/gnolang/gno/gno.land/pkg/sdk/vm"
-	gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
-	"github.com/gnolang/gno/gnovm/pkg/gnomod"
+	"github.com/gnolang/gno/gnovm/pkg/gnolang"
+	"github.com/gnolang/gno/gnovm/pkg/packages"
 	"github.com/gnolang/gno/tm2/pkg/crypto"
 	"github.com/gnolang/gno/tm2/pkg/std"
 )
@@ -63,7 +63,7 @@ func ResolvePackagePathQuery(bk *address.Book, path string) (PackagePath, error)
 }
 
 type Package struct {
-	gnomod.Pkg
+	*packages.Package
 	Creator crypto.Address
 	Deposit std.Coins
 }
@@ -75,7 +75,7 @@ var (
 	ErrEmptyDepositPackage = errors.New("no deposit specified for package")
 )
 
-func NewPackagesMap(ppaths []PackagePath) (PackagesMap, error) {
+func NewPackagesMap(cfg *packages.LoadConfig, ppaths []PackagePath) (PackagesMap, error) {
 	pkgs := make(map[string]Package)
 	for _, ppath := range ppaths {
 		if ppath.Creator.IsZero() {
@@ -88,7 +88,7 @@ func NewPackagesMap(ppaths []PackagePath) (PackagesMap, error) {
 		}
 
 		// list all packages from target path
-		pkgslist, err := gnomod.ListPkgs(abspath)
+		pkgslist, err := packages.Load(cfg, filepath.Join(abspath, "..."))
 		if err != nil {
 			return nil, fmt.Errorf("listing gno packages: %w", err)
 		}
@@ -102,7 +102,7 @@ func NewPackagesMap(ppaths []PackagePath) (PackagesMap, error) {
 				continue // skip
 			}
 			pkgs[pkg.Dir] = Package{
-				Pkg:     pkg,
+				Package: pkg,
 				Creator: ppath.Creator,
 				Deposit: ppath.Deposit,
 			}
@@ -112,10 +112,10 @@ func NewPackagesMap(ppaths []PackagePath) (PackagesMap, error) {
 	return pkgs, nil
 }
 
-func (pm PackagesMap) toList() gnomod.PkgList {
-	list := make([]gnomod.Pkg, 0, len(pm))
+func (pm PackagesMap) toList() packages.PkgList {
+	list := make([]*packages.Package, 0, len(pm))
 	for _, pkg := range pm {
-		list = append(list, pkg.Pkg)
+		list = append(list, pkg.Package)
 	}
 	return list
 }
@@ -132,13 +132,17 @@ func (pm PackagesMap) Load(fee std.Fee, start time.Time) ([]gnoland.TxWithMetada
 
 	metatxs := make([]gnoland.TxWithMetadata, 0, len(nonDraft))
 	for _, modPkg := range nonDraft {
+		if modPkg.ImportPath == "" || gnolang.IsStdlib(modPkg.ImportPath) {
+			continue
+		}
+
 		pkg := pm[modPkg.Dir]
 		if pkg.Creator.IsZero() {
 			return nil, fmt.Errorf("no creator set for %q", pkg.Dir)
 		}
 
 		// Open files in directory as MemPackage.
-		memPkg := gno.MustReadMemPackage(modPkg.Dir, modPkg.Name)
+		memPkg := gnolang.MustReadMemPackage(modPkg.Dir, modPkg.ImportPath)
 		if err := memPkg.Validate(); err != nil {
 			return nil, fmt.Errorf("invalid package: %w", err)
 		}

contribs/gnodev/pkg/watcher/watch.go

@@ -11,9 +11,9 @@ import (
 
 	emitter "github.com/gnolang/gno/contribs/gnodev/pkg/emitter"
 	events "github.com/gnolang/gno/contribs/gnodev/pkg/events"
+	"github.com/gnolang/gno/gnovm/pkg/packages"
 
 	"github.com/fsnotify/fsnotify"
-	"github.com/gnolang/gno/gnovm/pkg/gnomod"
 )
 
 type PackageWatcher struct {
@@ -118,7 +118,7 @@ func (p *PackageWatcher) Stop() {
 // Packages are sorted by their length in descending order to facilitate easier
 // and more efficient matching with corresponding paths. The longest paths are
 // compared first.
-func (p *PackageWatcher) AddPackages(pkgs ...gnomod.Pkg) error {
+func (p *PackageWatcher) AddPackages(pkgs ...*packages.Package) error {
 	for _, pkg := range pkgs {
 		dir := pkg.Dir
 

contribs/gnogenesis/internal/txs/txs_add_packages.go

@@ -7,12 +7,12 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/gnolang/gno/tm2/pkg/crypto/keys"
-
 	"github.com/gnolang/gno/gno.land/pkg/gnoland"
 	"github.com/gnolang/gno/gno.land/pkg/gnoland/ugnot"
+	"github.com/gnolang/gno/gnovm/pkg/packages"
 	"github.com/gnolang/gno/tm2/pkg/bft/types"
 	"github.com/gnolang/gno/tm2/pkg/commands"
+	"github.com/gnolang/gno/tm2/pkg/crypto/keys"
 	"github.com/gnolang/gno/tm2/pkg/std"
 )
 
@@ -125,7 +125,8 @@ func execTxsAddPackages(
 	parsedTxs := make([]gnoland.TxWithMetadata, 0)
 	for _, path := range args {
 		// Generate transactions from the packages (recursively)
-		txs, err := gnoland.LoadPackagesFromDir(path, creator, genesisDeployFee)
+		loadCfg := &packages.LoadConfig{IO: io, Deps: true}
+		txs, err := gnoland.LoadPackagesFromDir(loadCfg, path, creator, genesisDeployFee)
 		if err != nil {
 			return fmt.Errorf("unable to load txs from directory, %w", err)
 		}

examples/Makefile

@@ -23,29 +23,29 @@ GOIMPORTS_FLAGS ?= $(GOFMT_FLAGS)
 GOTEST_FLAGS ?= -v -p 1 -timeout=30m
 
 # Official packages (non-overridable): more reliable and tested modules, distinct from the experimentation area.
-OFFICIAL_PACKAGES = ./gno.land/p
-OFFICIAL_PACKAGES += ./gno.land/r/demo
-OFFICIAL_PACKAGES += ./gno.land/r/gnoland
-OFFICIAL_PACKAGES += ./gno.land/r/sys
-OFFICIAL_PACKAGES += ./gno.land/r/gov
+OFFICIAL_PACKAGES = ./gno.land/p/...
+OFFICIAL_PACKAGES += ./gno.land/r/demo/...
+OFFICIAL_PACKAGES += ./gno.land/r/gnoland/...
+OFFICIAL_PACKAGES += ./gno.land/r/sys/...
+OFFICIAL_PACKAGES += ./gno.land/r/gov/...
 
 ########################################
 # Dev tools
 .PHONY: transpile
 transpile:
-	go run ../gnovm/cmd/gno transpile -v .
+	go run ../gnovm/cmd/gno transpile -v ./...
 
 .PHONY: build
 build:
-	go run ../gnovm/cmd/gno transpile -v --gobuild .
+	go run ../gnovm/cmd/gno transpile -v --gobuild ./...
 
 .PHONY: test
 test:
 	go run ../gnovm/cmd/gno test -v ./...
 
 .PHONY: lint
 lint:
-	go run ../gnovm/cmd/gno lint -v $(OFFICIAL_PACKAGES)
+	go run ../gnovm/cmd/gno lint --root-examples -v $(OFFICIAL_PACKAGES)
 
 .PHONY: test.sync
 test.sync:

examples/gno.land/p/moul/md/md_test.gno

@@ -1,4 +1,4 @@
-package md
+package md_test
 
 import (
 	"testing"

examples/gno.land/r/gnoland/faucet/faucet_test.gno

@@ -1,4 +1,4 @@
-package faucet
+package faucet_test
 
 import (
 	"std"

examples/no_cycles_test.go

@@ -0,0 +1,117 @@
+package examples_test
+
+import (
+	"fmt"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/gnolang/gno/gnovm/pkg/gnoenv"
+	"github.com/gnolang/gno/gnovm/pkg/packages"
+	"github.com/stretchr/testify/require"
+)
+
+var injectedTestingLibs = []string{"encoding/json", "fmt", "os", "internal/os_test"}
+
+// TestNoCycles checks that there is no import cycles in stdlibs and non-draft examples
+func TestNoCycles(t *testing.T) {
+	// find examples and stdlibs
+	cfg := &packages.LoadConfig{SelfContained: true, Deps: true}
+	pkgs, err := packages.Load(cfg, filepath.Join(gnoenv.RootDir(), "examples", "..."))
+	require.NoError(t, err)
+
+	// detect cycles
+	visited := make(map[string]bool)
+	for _, p := range pkgs {
+		if p.Draft {
+			continue
+		}
+		require.NoError(t, detectCycles(p, pkgs, visited))
+	}
+}
+
+// detectCycles detects import cycles
+//
+// We need to check
+// 3 kinds of nodes
+//
+// - normal pkg: compiled source
+//
+// - xtest pkg: external test source (include xtests and filetests), can be treated as their own package
+//
+// - test pkg: embedded test sources,
+// these should not have their corresponding normal package in their dependencies tree
+//
+// The tricky thing is that we need to split test sources and normal source
+// while not considering them as distincitive packages.
+// Otherwise we will have false positive for example if we have these edges:
+//
+// - foo_pkg/foo_test.go imports bar_pkg
+//
+// - bar_pkg/bar_test.go import foo_pkg
+//
+// In go, the above example is allowed
+// but the following is not
+//
+// - foo_pkg/foo.go imports bar_pkg
+//
+// - bar_pkg/bar_test.go imports foo_pkg
+func detectCycles(root *packages.Package, pkgs []*packages.Package, visited map[string]bool) error {
+	// check cycles in package's sources
+	stack := []string{}
+	if err := visitPackage(root, pkgs, visited, stack); err != nil {
+		return fmt.Errorf("pkgsrc import: %w", err)
+	}
+	// check cycles in external tests' dependencies we might have missed
+	if err := visitImports([]packages.FileKind{packages.FileKindXTest, packages.FileKindFiletest}, root, pkgs, visited, stack); err != nil {
+		return fmt.Errorf("xtest import: %w", err)
+	}
+
+	// check cycles in tests' imports by marking the current package as visited while visiting the tests' imports
+	// we also consider PackageSource imports here because tests can call package code
+	visited = map[string]bool{root.ImportPath: true}
+	stack = []string{root.ImportPath}
+	if err := visitImports([]packages.FileKind{packages.FileKindPackageSource, packages.FileKindTest}, root, pkgs, visited, stack); err != nil {
+		return fmt.Errorf("test import: %w", err)
+	}
+
+	return nil
+}
+
+// visitImports resolves and visits imports by kinds
+func visitImports(kinds []packages.FileKind, root *packages.Package, pkgs []*packages.Package, visited map[string]bool, stack []string) error {
+	for _, imp := range root.Imports.Merge(kinds...) {
+		if slices.Contains(injectedTestingLibs, imp) {
+			continue
+		}
+		idx := slices.IndexFunc(pkgs, func(p *packages.Package) bool { return p.ImportPath == imp })
+		if idx == -1 {
+			return fmt.Errorf("import %q not found for %q tests", imp, root.ImportPath)
+		}
+		if err := visitPackage(pkgs[idx], pkgs, visited, stack); err != nil {
+			return fmt.Errorf("test import error: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// visitNode visits a package and its imports recursively. It only considers imports in PackageSource
+func visitPackage(pkg *packages.Package, pkgs []*packages.Package, visited map[string]bool, stack []string) error {
+	if slices.Contains(stack, pkg.ImportPath) {
+		return fmt.Errorf("cycle detected: %s -> %s", strings.Join(stack, " -> "), pkg.ImportPath)
+	}
+	if visited[pkg.ImportPath] {
+		return nil
+	}
+
+	visited[pkg.ImportPath] = true
+	stack = append(stack, pkg.ImportPath)
+
+	if err := visitImports([]packages.FileKind{packages.FileKindPackageSource}, pkg, pkgs, visited, stack); err != nil {
+		return err
+	}
+
+	return nil
+}