apiviewgo parses non-SDK modules as needed (#8699)

src/go/cmd/api_view_test.go

@@ -162,6 +162,27 @@ func TestDiagnostics(t *testing.T) {
 	}
 }
 
+func TestExternalModule(t *testing.T) {
+	review, err := createReview(filepath.Clean("testdata/test_external_module"))
+	require.NoError(t, err)
+	require.Equal(t, 1, len(review.Diagnostics))
+	require.Equal(t, aliasFor+"github.com/Azure/azure-sdk-for-go/sdk/azcore.Policy", review.Diagnostics[0].Text)
+	require.Equal(t, 1, len(review.Navigation))
+	require.Equal(t, 1, len(review.Navigation[0].ChildItems))
+	foundDo, foundPolicy := false, false
+	for _, token := range review.Tokens {
+		if token.DefinitionID != nil && *token.DefinitionID == "test_external_module.MyPolicy" {
+			require.Equal(t, "MyPolicy", token.Value)
+			foundPolicy = true
+		} else if token.Value == "Do" {
+			foundDo = true
+			require.Contains(t, *token.DefinitionID, "MyPolicy")
+		}
+	}
+	require.True(t, foundDo, "missing MyPolicy.Do()")
+	require.True(t, foundPolicy, "missing MyPolicy type")
+}
+
 func TestAliasDefinitions(t *testing.T) {
 	for _, test := range []struct {
 		name, path, sourceName string

src/go/cmd/download.go

@@ -0,0 +1,151 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"golang.org/x/mod/module"
+	"golang.org/x/mod/zip"
+)
+
+var errCachedModuleNotFound = errors.New("cached module not found")
+
+// GetExternalModule returns a Module representing mod. When GOMODCACHE is set,
+// it looks for mod's source in the mod cache. Otherwise, it downloads mod from
+// the module proxy.
+func GetExternalModule(mod module.Version) (*Module, error) {
+	m, err := cachedModule(mod)
+	if err != nil && !errors.Is(err, errCachedModuleNotFound) {
+		return nil, fmt.Errorf("failed to parse cached module %s: %w", mod.Path, err)
+	}
+	if m == nil {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+		defer cancel()
+		m, err = downloadModule(ctx, mod)
+	}
+	return m, err
+}
+
+// downloadModule downloads mod from the Go module proxy, storing it in a temporary directory.
+// That directory looks like:
+//
+//	~/apiviewgo{random suffix}
+//	├── [email protected]
+//	│   └── github.com/!azure/azure-sdk-for-go/sdk/[email protected]
+//	│       └── go.mod
+//	└── zip
+//	    └── github.com/!azure/azure-sdk-for-go/sdk/azcore
+//	        └── v1.0.0.zip
+//
+// This is perhaps odd but zip.Unzip() requires the target directory be entirely empty, so
+// obvious tidier schemes are impossible. Although downloadModule could in principle unzip
+// modules to the local Go module cache, it doesn't do so to avoid affecting other Go programs
+// or reimplementing whatever `go mod download` behavior is necessary to ensure correctness.
+func downloadModule(ctx context.Context, mod module.Version) (*Module, error) {
+	d, err := downloadDir()
+	if err != nil {
+		return nil, err
+	}
+	// We don't keep downloaded content because an apiviewgo instance doesn't need to download
+	// any mod twice (Review caches the Module this function returns) and we don't want to
+	// maintain a cache given the low performance impact of downloading modules (very few SDK
+	// modules export types defined elsewhere).
+	defer func() {
+		if err := os.RemoveAll(d); err != nil {
+			fmt.Fprintf(os.Stderr, "failed to remove download directory %s: %v", d, err)
+		}
+	}()
+	escaped, err := module.EscapePath(mod.Path)
+	if err != nil {
+		return nil, fmt.Errorf("unescapeable module path %q: %w", mod.Path, err)
+	}
+	u, err := url.Parse("https://" + path.Join("proxy.golang.org", escaped, "@v", mod.Version+".zip"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse module URL: %w", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to download module zip from %s: %w", u, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		msg := "module proxy responded %d"
+		if resp.Body != nil {
+			b, _ := io.ReadAll(resp.Body)
+			msg += ": " + string(b)
+		}
+		return nil, fmt.Errorf(msg, resp.StatusCode)
+	}
+	zp := filepath.Join(d, "zip", mustEscape(mod.Path), mod.Version+".zip")
+	err = os.MkdirAll(filepath.Dir(zp), 0700)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create directory for %s: %w", zp, err)
+	}
+	f, err := os.Create(zp)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create %s: %w", zp, err)
+	}
+	defer f.Close()
+	_, err = io.Copy(f, resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write %s: %w", zp, err)
+	}
+	modver := path.Base(mod.Path) + "@" + mod.Version
+	p := filepath.Join(d, modver, mustEscape(mod.Path)) + "@" + mod.Version
+	err = zip.Unzip(p, mod, zp)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unzip %s: %w", zp, err)
+	}
+	return NewModule(p)
+}
+
+// cachedModule returns a Module for mod if it's in either the local Go mod
+// cache or apiviewgo cache. It returns errCachedModuleNotFound when the
+// module isn't in either cache.
+func cachedModule(mod module.Version) (*Module, error) {
+	if modCache := os.Getenv("GOMODCACHE"); modCache != "" {
+		d := filepath.Join(modCache, mustEscape(mod.Path)) + "@" + mod.Version
+		if _, err := os.Stat(filepath.Join(d, "go.mod")); err == nil {
+			return NewModule(d)
+		}
+	}
+	return nil, errCachedModuleNotFound
+}
+
+// mustEscape escapes modPath. It panics when that fails.
+func mustEscape(modPath string) string {
+	escaped, err := module.EscapePath(modPath)
+	if err != nil {
+		panic(fmt.Errorf("failed to escape module path: %w", err))
+	}
+	return escaped
+}
+
+// downloadDir creates a directory to store downloaded module zips and source.
+// Callers are responsible for removing the directory when they're done with it.
+func downloadDir() (string, error) {
+	root, err := os.UserHomeDir()
+	if err != nil {
+		root = os.TempDir()
+	}
+	d, err := os.MkdirTemp(root, "apiviewgo")
+	if err != nil {
+		err = fmt.Errorf("failed to create download directory: %w", err)
+	}
+	return d, err
+}

src/go/cmd/module.go

@@ -59,9 +59,14 @@ func NewModule(dir string) (*Module, error) {
 	if err != nil {
 		return nil, err
 	}
+	name := filepath.Base(dir)
+	if before, _, found := strings.Cut(name, "@"); found {
+		// dir is in the module cache, something like "/home/me/go/pkg/mod/github.com/Foo/[email protected]"
+		name = before
+	}
 	m := Module{
 		ModFile:  mf,
-		Name:     filepath.Base(dir),
+		Name:     name,
 		Packages: map[string]*Pkg{},
 	}
 

src/go/cmd/pkg.go

@@ -63,7 +63,7 @@ func NewPkg(dir, modulePath string) (*Pkg, error) {
 	moduleName := filepath.Base(modulePathWithoutVersion)
 	if _, after, found := strings.Cut(dir, moduleName); found {
 		pk.relName = moduleName
-		if after != "" {
+		if after != "" && after[0] != '@' {
 			pk.relName += after
 		}
 		pk.relName = strings.ReplaceAll(pk.relName, "\\", "/")

src/go/cmd/review.go

@@ -178,23 +178,30 @@ func (r *Review) resolveAliases() error {
 		)
 		if m, ok = r.modules[ta.SourceMod.Path]; !ok {
 			m, err = r.findLocalModule(*ta)
+			if errors.Is(err, errExternalModule) {
+				m, err = GetExternalModule(ta.SourceMod)
+			}
 			if err == nil {
 				err = r.AddModule(m)
 			}
-			// TODO: handle errExternalModule by acquiring the module code from the external repository.
-			// For now, include the aliased type in the review without its definition and add a diagnostic.
-			if err != nil && !errors.Is(err, errExternalModule) {
+			if err != nil {
 				return err
 			}
 		}
 		def := typeDef{}
 		if m != nil {
-			impPath := ta.QualifiedName[:strings.LastIndex(ta.QualifiedName, ".")]
+			dot := strings.LastIndex(ta.QualifiedName, ".")
+			if len(ta.QualifiedName)-2 < dot || dot < 1 {
+				// there must be at least one rune before and after the dot
+				panic(fmt.Sprintf("alias %q refers to an invalid qualified name %q", ta.Name, ta.QualifiedName))
+			}
+			impPath := ta.QualifiedName[:dot]
 			p, ok := m.Packages[impPath]
 			if !ok {
 				return fmt.Errorf("couldn't find definition for " + ta.Name)
 			}
-			if d, ok := recursiveFindTypeDef(ta.Name, p, m.Packages); ok {
+			sourceName := ta.QualifiedName[dot+1:]
+			if d, ok := recursiveFindTypeDef(sourceName, p, m.Packages); ok {
 				def = d
 			}
 		}

src/go/cmd/testdata/test_external_module/go.mod

@@ -0,0 +1,5 @@
+module test_external_module
+
+go 1.18
+
+require github.com/Azure/azure-sdk-for-go/sdk/azcore v0.1.0

src/go/cmd/testdata/test_external_module/test.go

@@ -0,0 +1,5 @@
+package test_external_module
+
+import "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+
+type MyPolicy = azcore.Policy