diff --git a/cmd/app/factory.go b/cmd/app/factory.go index 79acb7d2..92f78349 100644 --- a/cmd/app/factory.go +++ b/cmd/app/factory.go @@ -14,6 +14,7 @@ import ( //"github.com/gardener/docforge/pkg/metrics" "github.com/gardener/docforge/pkg/processors" "github.com/gardener/docforge/pkg/reactor" + "github.com/gardener/docforge/pkg/resourcehandlers/fs" ghrs "github.com/gardener/docforge/pkg/resourcehandlers/github" "github.com/google/go-github/v32/github" @@ -112,7 +113,9 @@ func WithHugo(reactorOptions *reactor.Options, o *Options) { // initResourceHandlers initializes the resource handler // objects used by reactors func initResourceHandlers(ctx context.Context, o *Options) []resourcehandlers.ResourceHandler { - rhs := []resourcehandlers.ResourceHandler{} + rhs := []resourcehandlers.ResourceHandler{ + fs.NewFSResourceHandler(), + } if o.GitHubTokens != nil { if token, ok := o.GitHubTokens["github.com"]; ok { ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) diff --git a/pkg/api/types.go b/pkg/api/types.go index 687c1a02..bed2ddf3 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -232,7 +232,7 @@ type Template struct { // Mandatory Path string `yaml:"path"` // Sources maps variable names to ContentSelectors that will be - // used as specification for the content to fetch and assign ot that + // used as specification for the content to fetch and assign to // these variables Sources map[string]*ContentSelector `yaml:"sources,omitempty"` } diff --git a/pkg/reactor/content_processor.go b/pkg/reactor/content_processor.go index 277a98cf..158fe29a 100644 --- a/pkg/reactor/content_processor.go +++ b/pkg/reactor/content_processor.go @@ -218,21 +218,24 @@ func (c *nodeContentProcessor) resolveLink(ctx context.Context, node *api.Node, if err != nil { return nil, text, title, nil, err } - // can we handle this destination? - if u.IsAbs() && c.resourceHandlers.Get(destination) == nil { - // It's a valid absolute link that is not in our scope. Leave it be. - return &destination, text, title, nil, err - } - // force destination to absolute URL - handler := c.resourceHandlers.Get(contentSourcePath) - if handler == nil { - return &destination, text, title, nil, nil + if u.IsAbs() { + // can we handle changes to this destination? + if c.resourceHandlers.Get(destination) == nil { + // we don't have a handler for it. Leave it be. + return &destination, text, title, nil, err + } + absLink = destination } - absLink, err = handler.BuildAbsLink(contentSourcePath, destination) - if err != nil { - return nil, text, title, nil, err + if len(absLink) == 0 { + // build absolute path for the destination using contentSourcePath as base + handler := c.resourceHandlers.Get(contentSourcePath) + if handler == nil { + return &destination, text, title, nil, nil + } + if absLink, err = handler.BuildAbsLink(contentSourcePath, destination); err != nil { + return nil, text, title, nil, err + } } - // rewrite link if required if gLinks := c.globalLinksConfig; gLinks != nil { globalRewrites = gLinks.Rewrites diff --git a/pkg/reactor/document_worker.go b/pkg/reactor/document_worker.go index 7c7544b2..7301edbd 100644 --- a/pkg/reactor/document_worker.go +++ b/pkg/reactor/document_worker.go @@ -6,7 +6,7 @@ import ( "fmt" "io/ioutil" "path/filepath" - "strings" + "sync" "text/template" "github.com/gardener/docforge/pkg/api" @@ -31,6 +31,7 @@ type DocumentWorker struct { NodeContentProcessor NodeContentProcessor GitHubInfoController GitInfoController templates map[string]*template.Template + rwLock sync.RWMutex } // DocumentWorkTask implements jobs#Task @@ -53,6 +54,20 @@ func (g *GenericReader) Read(ctx context.Context, source string) ([]byte, error) return nil, fmt.Errorf("failed to get handler to read from %s", source) } +func (w *DocumentWorker) getTemplate(name string) *template.Template { + defer w.rwLock.Unlock() + w.rwLock.Lock() + if tmpl, ok := w.templates[name]; ok { + return tmpl + } + return nil +} +func (w *DocumentWorker) setTemplate(name string, tmpl *template.Template) { + defer w.rwLock.Unlock() + w.rwLock.Lock() + w.templates[name] = tmpl +} + // Work implements Worker#Work function func (w *DocumentWorker) Work(ctx context.Context, task interface{}, wq jobs.WorkQueue) *jobs.WorkerError { if task, ok := task.(*DocumentWorkTask); ok { @@ -96,21 +111,14 @@ func (w *DocumentWorker) Work(ctx context.Context, task interface{}, wq jobs.Wor templateBlob []byte tmpl *template.Template ) - if tmpl, ok = w.templates[task.Node.Template.Path]; !ok { - // TODO: temporary solution for local templates - if !strings.HasPrefix(task.Node.Template.Path, "https") { - if templateBlob, err = ioutil.ReadFile(task.Node.Template.Path); err != nil { - return jobs.NewWorkerError(err, 0) - } - } else { - if templateBlob, err = w.Reader.Read(ctx, task.Node.Template.Path); err != nil { - return jobs.NewWorkerError(err, 0) - } + if tmpl = w.getTemplate(task.Node.Template.Path); tmpl == nil { + if templateBlob, err = w.Reader.Read(ctx, task.Node.Template.Path); err != nil { + return jobs.NewWorkerError(err, 0) } - if tmpl, err = template.New("test").Parse(string(templateBlob)); err != nil { + if tmpl, err = template.New(task.Node.Template.Path).Parse(string(templateBlob)); err != nil { return jobs.NewWorkerError(err, 0) } - w.templates[task.Node.Template.Path] = tmpl + w.setTemplate(task.Node.Template.Path, tmpl) } if err := tmpl.Execute(&b, vars); err != nil { return jobs.NewWorkerError(err, 0) diff --git a/pkg/reactor/document_worker_test.go b/pkg/reactor/document_worker_test.go index 2daafd7b..2e64295e 100644 --- a/pkg/reactor/document_worker_test.go +++ b/pkg/reactor/document_worker_test.go @@ -44,18 +44,18 @@ func TestDocumentWorkerWork(t *testing.T) { testOutput := "#Heading 1\n" rhRegistry := resourcehandlers.NewRegistry(&FakeResourceHandler{}) testworker := &DocumentWorker{ - &TestWriter{ + Writer: &TestWriter{ make(map[string][]byte), }, - &TestReader{ + Reader: &TestReader{ make(map[string][]byte), }, - &TestProcessor{ + Processor: &TestProcessor{ func(documentBlob []byte, node *api.Node) ([]byte, error) { return documentBlob, nil }, }, - &nodeContentProcessor{ + NodeContentProcessor: &nodeContentProcessor{ downloadController: NewDownloadController(&TestReader{ make(map[string][]byte), }, &TestWriter{ @@ -63,8 +63,8 @@ func TestDocumentWorkerWork(t *testing.T) { }, 1, false, rhRegistry), resourceHandlers: rhRegistry, }, - nil, - nil, + GitHubInfoController: nil, + templates: nil, } testCases := []struct { diff --git a/pkg/resourcehandlers/fs/fs.go b/pkg/resourcehandlers/fs/fs.go new file mode 100644 index 00000000..470f0a9f --- /dev/null +++ b/pkg/resourcehandlers/fs/fs.go @@ -0,0 +1,240 @@ +package fs + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/gardener/docforge/pkg/api" + "github.com/gardener/docforge/pkg/git" + "github.com/gardener/docforge/pkg/resourcehandlers" + "github.com/google/go-github/v32/github" +) + +type fsHandler struct{} + +// NewFSResourceHandler create file system ResourceHandler +func NewFSResourceHandler() resourcehandlers.ResourceHandler { + return &fsHandler{} +} + +// Accept implements resourcehandlers.ResourceHandler#Accept +func (fs *fsHandler) Accept(uri string) bool { + _, err := os.Stat(uri) + return err == nil +} + +// ResolveNodeSelector implements resourcehandlers.ResourceHandler#ResolveNodeSelector +func (fs *fsHandler) ResolveNodeSelector(ctx context.Context, node *api.Node, excludePaths []string, frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) error { + var ( + fileInfo os.FileInfo + err error + ) + if node.NodeSelector == nil { + return nil + } + if fileInfo, err = os.Stat(node.NodeSelector.Path); err != nil { + return err + } + if !fileInfo.IsDir() { + return fmt.Errorf("nodeSelector path is not directory") + } + _node := &api.Node{ + Nodes: []*api.Node{}, + } + filepath.Walk(node.NodeSelector.Path, func(node *api.Node, parentPath string) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + if node.NodeSelector != nil { + return nil + } + if path != parentPath { + if len(strings.Split(path, "/"))-len(strings.Split(parentPath, "/")) != 1 { + node = node.Parent() + pathSegments := strings.Split(path, "/") + if len(pathSegments) > 0 { + pathSegments = pathSegments[:len(pathSegments)-1] + parentPath = filepath.Join(pathSegments...) + } + } + } + n := &api.Node{ + Name: info.Name(), + } + n.SetParent(node) + node.Nodes = append(node.Nodes, n) + if info.IsDir() { + node = n + node.Nodes = []*api.Node{} + parentPath = path + } else { + n.Source = path + } + return nil + } + }(_node, node.NodeSelector.Path)) + if len(_node.Nodes) > 0 && len(_node.Nodes[0].Nodes) > 0 { + node.Nodes = _node.Nodes[0].Nodes + for _, n := range node.Nodes { + n.SetParent(node) + } + } + return nil +} + +// Read implements resourcehandlers.ResourceHandler#Read +func (fs *fsHandler) Read(ctx context.Context, uri string) ([]byte, error) { + return ioutil.ReadFile(uri) +} + +// ReadGitInfo implements resourcehandlers.ResourceHandler#ReadGitInfo +func (fs *fsHandler) ReadGitInfo(ctx context.Context, uri string) ([]byte, error) { + var ( + log []*gitLogEntry + blob []byte + err error + ) + if !checkGitExists() { + return nil, fmt.Errorf("reading Git info for %s failed: git not found in PATH", uri) + } + + if log, err = gitLog(uri); err != nil { + return nil, err + } + + if len(log) == 0 { + return []byte(""), nil + } + + for _, logEntry := range log { + logEntry.Name = strings.Split(logEntry.Name, "<")[0] + logEntry.Name = strings.TrimSpace(logEntry.Name) + } + authorName := log[len(log)-1].Name + authorEmail := log[len(log)-1].Email + publishD := log[len(log)-1].Date + lastModD := log[0].Date + gitInfo := &git.GitInfo{ + PublishDate: &publishD, + LastModifiedDate: &lastModD, + Author: &github.User{ + Name: &authorName, + Email: &authorEmail, + }, + Contributors: []*github.User{}, + } + + for _, logEntry := range log { + if logEntry.Email != *gitInfo.Author.Email { + name := logEntry.Name + email := logEntry.Email + gitInfo.Contributors = append(gitInfo.Contributors, &github.User{ + Name: &name, + Email: &email, + }) + } + } + + if blob, err = json.MarshalIndent(gitInfo, "", " "); err != nil { + return nil, err + } + + return blob, nil +} + +// ResourceName implements resourcehandlers.ResourceHandler#ResourceName +func (fs *fsHandler) ResourceName(link string) (name string, extension string) { + _, name = filepath.Split(link) + if len(name) > 0 { + if e := filepath.Ext(name); len(e) > 0 { + extension = e[1:] + name = strings.TrimSuffix(name, e) + } + } + return +} + +// BuildAbsLink implements resourcehandlers.ResourceHandler#BuildAbsLink +func (fs *fsHandler) BuildAbsLink(source, link string) (string, error) { + if filepath.IsAbs(link) { + return link, nil + } + p := filepath.Join(source, link) + p = filepath.Clean(p) + if filepath.IsAbs(p) { + return p, nil + } + return filepath.Abs(p) +} + +// GetRawFormatLink implements resourcehandlers.ResourceHandler#GetRawFormatLink +func (fs *fsHandler) GetRawFormatLink(absLink string) (string, error) { + return absLink, nil +} + +// SetVersion implements resourcehandlers.ResourceHandler#SetVersion +func (fs *fsHandler) SetVersion(absLink, version string) (string, error) { + return absLink, nil +} + +type gitLogEntry struct { + Sha string + Author string + Date string + Message string + Email string + Name string +} + +type gitLogEntryAuthor struct { +} + +func gitLog(path string) ([]*gitLogEntry, error) { + var ( + log []byte + err error + errStr string + stdout, stderr bytes.Buffer + ) + + if _, err := os.Stat(path); err != nil { + return nil, err + } + + git := exec.Command("git", "log", "--date=short", `--pretty=format:'{%n "sha": "%H",%n "author": "%aN <%aE>",%n "date": "%ad",%n "message": "%s",%n "email": "%aE",%n "name": "%aN"%n },'`, "--follow", path) + git.Stdout = &stdout + git.Stderr = &stderr + if err = git.Run(); err != nil { + if _, ok := err.(*exec.ExitError); !ok { + return nil, err + } + } + log, errStr = stdout.Bytes(), string(stderr.Bytes()) + if len(errStr) > 0 { + return nil, fmt.Errorf("failed to execute git log for %s:\n%s", path, errStr) + } + + logS := string(log) + logS = strings.ReplaceAll(logS, "'{", "{") + logS = strings.ReplaceAll(logS, "},'", "},") + if strings.HasSuffix(logS, ",") { + logS = logS[:len(logS)-1] + } + logS = fmt.Sprintf("[%s]", logS) + + gitLog := []*gitLogEntry{} + if err := json.Unmarshal([]byte(logS), &gitLog); err != nil { + return nil, err + } + return gitLog, nil +} + +func checkGitExists() bool { + _, err := exec.LookPath("git") + return err == nil +} diff --git a/pkg/resourcehandlers/fs/fs_test.go b/pkg/resourcehandlers/fs/fs_test.go new file mode 100644 index 00000000..156c10ca --- /dev/null +++ b/pkg/resourcehandlers/fs/fs_test.go @@ -0,0 +1,78 @@ +package fs + +import ( + "path/filepath" + "testing" + + "github.com/gardener/docforge/pkg/api" + "github.com/stretchr/testify/assert" +) + +func TestGitLog(t *testing.T) { + var ( + log []*gitLogEntry + err error + ) + if log, err = gitLog(filepath.Join("testdata", "f00.md")); err != nil { + t.Fatalf("%s", err.Error()) + } + assert.NotNil(t, log) +} + +func TestReadGitInfo(t *testing.T) { + var ( + log []byte + err error + ) + fs := &fsHandler{} + if log, err = fs.ReadGitInfo(nil, filepath.Join("testdata", "f00.md")); err != nil { + t.Fatalf("%s", err.Error()) + } + assert.NotNil(t, log) +} + +func TestResolveNodeSelector(t *testing.T) { + var ( + err error + ) + fs := &fsHandler{} + node := &api.Node{ + NodeSelector: &api.NodeSelector{ + Path: "testdata", + }, + } + expected := &api.Node{ + NodeSelector: &api.NodeSelector{ + Path: "testdata", + }, + Nodes: []*api.Node{ + &api.Node{ + Name: "d00", + Nodes: []*api.Node{ + &api.Node{ + Name: "d02", + Nodes: []*api.Node{ + &api.Node{ + Name: "f020.md", + Source: "testdata/d00/d02/f020.md", + }, + }, + }, + &api.Node{ + Name: "f01.md", + Source: "testdata/d00/f01.md", + }, + }, + }, + &api.Node{ + Name: "f00.md", + Source: "testdata/f00.md", + }, + }, + } + expected.SetParentsDownwards() + if err = fs.ResolveNodeSelector(nil, node, nil, nil, nil, 0); err != nil { + t.Fatalf("%s", err.Error()) + } + assert.Equal(t, expected, node) +} diff --git a/pkg/resourcehandlers/fs/testdata/d00/d02/f020.md b/pkg/resourcehandlers/fs/testdata/d00/d02/f020.md new file mode 100644 index 00000000..e69de29b diff --git a/pkg/resourcehandlers/fs/testdata/d00/f01.md b/pkg/resourcehandlers/fs/testdata/d00/f01.md new file mode 100644 index 00000000..e69de29b diff --git a/pkg/resourcehandlers/fs/testdata/f00.md b/pkg/resourcehandlers/fs/testdata/f00.md new file mode 100644 index 00000000..e69de29b