change node api

cmd/app/cmd.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 
+	"github.com/gardener/docforge/pkg/api"
 	"github.com/gardener/docforge/pkg/hugo"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
@@ -37,13 +38,17 @@ func NewCommand(ctx context.Context, cancel context.CancelFunc) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "docforge",
 		Short: "Build documentation bundle",
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			options := NewOptions(flags)
 			doc := Manifest(flags.documentationManifestPath)
-			reactor := NewReactor(ctx, options)
+			if err := api.ValidateManifest(doc); err != nil {
+				return err
+			}
+			reactor := NewReactor(ctx, options, doc.Links)
 			if err := reactor.Run(ctx, doc, flags.dryRun); err != nil {
-				klog.Errorf(err.Error())
+				return err
 			}
+			return nil
 		},
 	}
 

cmd/app/factory.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"path/filepath"
 
+	"github.com/gardener/docforge/pkg/api"
 	"github.com/gardener/docforge/pkg/hugo"
 	"github.com/gardener/docforge/pkg/metrics"
 	"github.com/gardener/docforge/pkg/resourcehandlers"
@@ -44,7 +45,7 @@ type Metering struct {
 }
 
 // NewReactor creates a Reactor from Options
-func NewReactor(ctx context.Context, options *Options) *reactor.Reactor {
+func NewReactor(ctx context.Context, options *Options, globalLinksCfg *api.Links) *reactor.Reactor {
 	dryRunWriters := writers.NewDryRunWritersFactory(options.DryRunWriter)
 	o := &reactor.Options{
 		MaxWorkersCount:              options.MaxWorkersCount,
@@ -59,6 +60,7 @@ func NewReactor(ctx context.Context, options *Options) *reactor.Reactor {
 		ResourceHandlers:             initResourceHandlers(ctx, options),
 		DryRunWriter:                 dryRunWriters,
 		Resolve:                      options.Resolve,
+		GlobalLinksConfig:            globalLinksCfg,
 	}
 	if options.DryRunWriter != nil {
 		o.Writer = dryRunWriters.GetWriter(options.DestinationPath)

example/advanced/00.yaml

@@ -1,69 +1,62 @@
-root:
-    name: doc
+structure:
+  - name: doc
     nodes:
     - name: overview
-      contentSelectors:
-        - source: https://github.com/gardener/documentation/wiki/Architecture.md
-    - name: gardenlet
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md
+      source: https://github.com/gardener/documentation/wiki/Architecture.md
+    - source: https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md
       # linkSubstitutes define changes to links in documents.They apply to links and images
       # specified with markdown markup.
-      linksSubstitutes:
+      links:
         # The key in the mapping is an absolute form of a document link that will be 
         # subject to transformation
-        "https://github.com/gardener/gardener/blob/master/docs/usage/shooted_seed.md":
-          # destination is the link reference URL. If it is empty string, 
-          # the links markup is removed leaving only link text behind.
-          # For images, the entire markup is removed.
-          destination: ""
-        "https://kubernetes.io/docs/concepts/extend-kubernetes/operator/":
-          destination: "https://kubernetes.io/docs/concepts/extend-kubernetes/operator1111"
-          # text is a link text element (alt-text for images). Specifying text
-          # will change it to the new value. Empty string is valid only with 
-          # `destination=""`
-          text: smooth operator
-          # title is the title element of a link or image. Specifying text
-          # will change it to the new value.
-          title: a title
-      # localityDomains can be specified on node level too.
-      # Node's localityDomain definitions override and amend global ones.
-      localityDomain:
-        github.com/gardener/gardener:
-          version: v1.11.1
-          path: gardener/gardener
-          # exclude omits resources from path. You can 
-          # also use include with the reverse semantics
-          exclude:
-          - example
-        # downloadSubstitutes is a list of regular expressions matching
-        # links to resources on documents that will be downloaded, mapped
-        # to name expressions tha define how the downloaded resources will
-        # named.
-        # There is a set of variables that can be used to construct the
-        # expressions:
-        # - $name: the original name of the resource
-        # - $path: the original path of the resource
-        # - $uuid: a UUID generated for the resource
-        # - $ext: a original resource extension
-        # The default expression applying to all resources is: $uuid.$ext
-        # Besides regular-expression-to-expression mappings it is possible
-        # to map exact URLs (escaped) to concrete names.
-        downloadSubstitutes:
-          "\\.(jpg|gif|png)": "$name-hires-$uuid.$ext"
-    - name: deploying-gardenlets
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet.md
-    - name: automatic-deployment
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet_automatically.md
-    - name: deploy-gardenlet-manually
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/2a33b26458dddd7ad09c4c3b2311d3391db890e7/docs/deployment/deploy_gardenlet_manually.md
-    - name: shooted-seeds
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/master/docs/usage/shooted_seed.md
-localityDomain:
-  github.com/gardener/gardener:
-    version: v1.10.0
-    path: gardener/gardener
+        rewrites:
+          github.com/gardener/gardener:
+            version: v1.11.1
+          "https://github.com/gardener/gardener/blob/master/docs/usage/shooted_seed.md":
+            # destination is the link reference URL. If it is empty string, 
+            # the links markup is removed leaving only link text behind.
+            # For images, the entire markup is removed.
+            destination: ""
+          "https://kubernetes.io/docs/concepts/extend-kubernetes/operator/":
+            destination: "https://kubernetes.io/docs/concepts/extend-kubernetes/operator1111"
+            # text is a link text element (alt-text for images). Specifying text
+            # will change it to the new value. Empty string is valid only with 
+            # `destination=""`
+            text: smooth operator
+            # title is the title element of a link or image. Specifying text
+            # will change it to the new value.
+            title: a title
+        downloads:
+          # localityDomains can be specified on node level too.
+          # Node's localityDomain definitions override and amend global ones.
+          scope:
+            github.com/gardener/gardener:
+              version: v1.11.1
+          # downloadSubstitutes is a list of regular expressions matching
+          # links to resources on documents that will be downloaded, mapped
+          # to name expressions tha define how the downloaded resources will
+          # named.
+          # There is a set of variables that can be used to construct the
+          # expressions:
+          # - $name: the original name of the resource
+          # - $path: the original path of the resource
+          # - $uuid: a UUID generated for the resource
+          # - $ext: a original resource extension
+          # The default expression applying to all resources is: $uuid.$ext
+          # Besides regular-expression-to-expression mappings it is possible
+          # to map exact URLs (escaped) to concrete names.              
+          renames:
+            "\\.(jpg|gif|png)": "$name-hires-$uuid.$ext"
+    - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet.md
+    - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet_automatically.md
+    - source: https://github.com/gardener/gardener/blob/2a33b26458dddd7ad09c4c3b2311d3391db890e7/docs/deployment/deploy_gardenlet_manually.md
+    - source: https://github.com/gardener/gardener/blob/master/docs/usage/shooted_seed.md
+links:
+  # The key in the mapping is an absolute form of a document link that will be 
+  # subject to transformation
+  rewrites:
+    github.com/gardener/gardener:
+      version: v1.10.0
+  downloads:
+    scope:
+      github.com/gardener/gardener: ~

example/simple/00.yaml

@@ -1,11 +1,11 @@
 # The documentation structure root node. Mandatory.
-root:
-    name: doc
+structure:
+  - name: doc
     # nodeSelector is resolved to a node hierarchy, where nodes are selected
     # by criteria (criteria is not implemented yet, i.e no filters). 
     nodesSelector:
       # A node selector path defines the scope that will be used to
       # generate a hierarchy. For GitHub paths that is a folder in a GitHub repo
       # and the generated nodes hierarchy corresponds ot the file/folder structure 
       # available in the repository at that path.
-      path: https://github.com/gardener/gardener/tree/v1.10.0/docs
+      path: https://github.com/gardener/gardener/tree/v1.10.0/docs/concepts

example/simple/01.yaml

@@ -1,41 +1,58 @@
-# The documentation structure root node. Mandatory.
-root:
-    name: doc
-    # nodeSelector is resolved to a node hierarchy, where nodes are selected
-    # by criteria (criteria is not implemented yet, i.e no filters). 
+# The documentation structure root. Mandatory.
+structure:
+  # Top-level container node with descendent hierarchy defined both with rules
+  # by its nodeSelector and explicitly with its nodes list property
+  # name stands for the name of the node. It is mandatory property for container nodes
+  - name: doc
+    # nodeSelector is resolved to a node hierarchy using path and rules to select nodes
+    # to be part of the hierarchy. not specifying anything but path will end up in 
+    # selecting the whole file/folder structure at path.
     nodesSelector:
       # A node selector path defines the scope that will be used to
       # generate a hierarchy. For GitHub paths that is a folder in a GitHub repo
       # and the generated nodes hierarchy corresponds ot the file/folder structure 
       # available in the repository at that path.
-      path: https://github.com/gardener/gardener/tree/master/docs
-    # A list of child nodes to this structure node to form document structure hierarchy.
-    # Note that if a nodeSelector is specified on this node, will be merged with other
-    # existing nodes in `nodes`. Nodes with the same name will have their other properties
-    # merged, making it possible to add properties to the generated structure or additional
-    # nodes.
+      path: https://github.com/gardener/gardener/tree/v1.11.1/docs/concepts
+    # A list of child nodes to this structure node to explicitly define document structure hierarchy.
+    # Merging nodeSelector hierarchy with nodes:
+    # If both a nodeSelector and nodes are specified on this node, when the node selector is resolved, 
+    # its resulting top-level nodes will merge with other existing nodes in `nodes`. Nodes with the same
+    # name will have their other properties merged, making it possible to add properties to
+    # the generated structure or additional nodes.
     nodes:
+        # Name of this document node.
+        # Name is not mandatory for document nodes if source is provided. With source and no name,
+        # the name will be resolved to the resource name in source.
       - name: aws_provider
-        # contentSelectors is a list of source selection specifications.
-        # Normally, there will be one but it is possible to specify several and
-        # they will be appended in that order. 
-        contentSelectors:
-          # Source specifies location of document source. 
-          # The supported sources as of now are GitHub repository documents and wiki pages.
-        - source: https://github.com/gardener/gardener-extension-provider-aws/blob/v1.13.0/docs/usage-as-end-user.md
-# A localityDomain defines the scope of documentation structure "local" 
-# resources that are downloaded along with structure's documents.
-localityDomain:
-  # A locality domain. GFor GitHub it is in the form
-  # <host>/<organization>/<repository>
-  github.com/gardener/gardener:
-    # The version, if specified, is applied to all links inside this domain. 
-    # Document-local resources that will be downloaded (inside `path`), and
-    # links that will be absolute in this domain (github.com/gardener/gardener)
-    # will be rewritten with this version in their URLs. 
-    version: v1.11.1
-    # Path inside this domain that defines the scope of the "document-local"
-    # resources, which will be downloaded along with documents.
-    # If version is specified, the links used to download the resources are
-    # rewritten to match the version.
-    path: gardener/gardener/docs
+        # Source specifies location of document source. 
+        # The supported sources as of now are GitHub repository documents and wiki pages.
+        source: https://github.com/gardener/gardener-extension-provider-aws/blob/v1.13.0/docs/usage-as-end-user.md
+# Links define configuration for handling document resource references 
+# including both hyperlinks and images. 
+links:
+  # rewrites defines rewrite rules for document links. 
+  # rules are mapped to regular expressions to match links in documents
+  rewrites:
+    # A regex to rule mapping binding all links that features gardener/gardener/blob,
+    # gardener/gardener/tree or gardener/gardener/raw to this rule
+    gardener/gardener/(blob|tree|raw):
+      # The version, if specified, is applied to all links matched by this regex. 
+      # Both document-local resources that will be downloaded, and
+      # links that will be absolute in this domain (gardener/gardener)
+      # will be rewritten with this version in their URLs.
+      version: v1.11.1
+    # Mapping a regular expression to nil effectively removes matching links
+    # from their documents
+    gardener/gardener/(pulls|pull|issue|issues): ~
+  # downloads define the downloads domain and global renaming rules for
+  # downloaded resources
+  downloads:
+    # Scope defines the download scope for documents, mapping regular expressions
+    # to optional rename rules valid for links matched by them. Resources with URLs
+    # that match the regex will be downloaded.
+    # A mapping to nil will apply a default rename pattern ($uuid.$ext)
+    scope:
+      # Download all referenced blobs and raw type of files in gardener/gardener's docs folder
+      gardener/gardener/(blob|raw)/v1.11.1/docs: ~
+      # Download all referenced gardener/gardener wiki pages
+      gardener/gardener/wiki: ~

pkg/api/nodes.go

@@ -142,38 +142,53 @@ func (n *Node) AddStats(s ...*Stat) {
 	}
 }
 
-// FindNodeByContentSource traverses up and then all around the
+// FindNodeBySource traverses up and then all around the
 // tree paths in the node's documentation structure, looking for
-// a node that has contentSource path nodeContentSource
-func FindNodeByContentSource(nodeContentSource string, node *Node) *Node {
+// a node that has the source string either in source, contentSelector
+// or template
+func FindNodeBySource(source string, node *Node) *Node {
 	if node == nil {
 		return nil
 	}
-
-	for _, contentSelector := range node.ContentSelectors {
-		if contentSelector.Source == nodeContentSource {
-			return node
-		}
+	if n := matchAnySource(source, node); n != nil {
+		return n
 	}
 	root := node.GetRootNode()
 	if root == nil {
 		root = node
 	}
-	return withMatchinContentSelectorSource(nodeContentSource, root)
+	return withMatchinContentSelectorSource(source, root)
 }
 
-func withMatchinContentSelectorSource(nodeContentSource string, node *Node) *Node {
-	if node == nil {
-		return nil
+func matchAnySource(source string, node *Node) *Node {
+	if node.Source == source {
+		return node
 	}
 	for _, contentSelector := range node.ContentSelectors {
-		if contentSelector.Source == nodeContentSource {
+		if contentSelector.Source == source {
 			return node
 		}
 	}
+	if t := node.Template; t != nil {
+		for _, contentSelector := range t.Sources {
+			if contentSelector.Source == source {
+				return node
+			}
+		}
+	}
+	return nil
+}
+
+func withMatchinContentSelectorSource(source string, node *Node) *Node {
+	if node == nil {
+		return nil
+	}
+	if n := matchAnySource(source, node); n != nil {
+		return n
+	}
 
 	for i := range node.Nodes {
-		foundNode := withMatchinContentSelectorSource(nodeContentSource, node.Nodes[i])
+		foundNode := withMatchinContentSelectorSource(source, node.Nodes[i])
 		if foundNode != nil {
 			return foundNode
 		}