change node api

cmd/app/cmd.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 
+	"github.com/gardener/docforge/pkg/api"
 	"github.com/gardener/docforge/pkg/hugo"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
@@ -37,13 +38,17 @@ func NewCommand(ctx context.Context, cancel context.CancelFunc) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "docforge",
 		Short: "Build documentation bundle",
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			options := NewOptions(flags)
 			doc := Manifest(flags.documentationManifestPath)
-			reactor := NewReactor(ctx, options)
+			if err := api.ValidateManifest(doc); err != nil {
+				return err
+			}
+			reactor := NewReactor(ctx, options, doc.Links)
 			if err := reactor.Run(ctx, doc, flags.dryRun); err != nil {
-				klog.Errorf(err.Error())
+				return err
 			}
+			return nil
 		},
 	}
 

cmd/app/factory.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"path/filepath"
 
+	"github.com/gardener/docforge/pkg/api"
 	"github.com/gardener/docforge/pkg/hugo"
 	"github.com/gardener/docforge/pkg/metrics"
 	"github.com/gardener/docforge/pkg/resourcehandlers"
@@ -44,7 +45,7 @@ type Metering struct {
 }
 
 // NewReactor creates a Reactor from Options
-func NewReactor(ctx context.Context, options *Options) *reactor.Reactor {
+func NewReactor(ctx context.Context, options *Options, globalLinksCfg *api.Links) *reactor.Reactor {
 	dryRunWriters := writers.NewDryRunWritersFactory(options.DryRunWriter)
 	o := &reactor.Options{
 		MaxWorkersCount:              options.MaxWorkersCount,
@@ -59,6 +60,7 @@ func NewReactor(ctx context.Context, options *Options) *reactor.Reactor {
 		ResourceHandlers:             initResourceHandlers(ctx, options),
 		DryRunWriter:                 dryRunWriters,
 		Resolve:                      options.Resolve,
+		GlobalLinksConfig:            globalLinksCfg,
 	}
 	if options.DryRunWriter != nil {
 		o.Writer = dryRunWriters.GetWriter(options.DestinationPath)

example/advanced/00.yaml

@@ -1,69 +1,62 @@
-root:
-    name: doc
+structure:
+  - name: doc
     nodes:
     - name: overview
-      contentSelectors:
-        - source: https://github.com/gardener/documentation/wiki/Architecture.md
-    - name: gardenlet
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md
+      source: https://github.com/gardener/documentation/wiki/Architecture.md
+    - source: https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md
       # linkSubstitutes define changes to links in documents.They apply to links and images
       # specified with markdown markup.
-      linksSubstitutes:
+      links:
         # The key in the mapping is an absolute form of a document link that will be 
         # subject to transformation
-        "https://github.com/gardener/gardener/blob/master/docs/usage/shooted_seed.md":
-          # destination is the link reference URL. If it is empty string, 
-          # the links markup is removed leaving only link text behind.
-          # For images, the entire markup is removed.
-          destination: ""
-        "https://kubernetes.io/docs/concepts/extend-kubernetes/operator/":
-          destination: "https://kubernetes.io/docs/concepts/extend-kubernetes/operator1111"
-          # text is a link text element (alt-text for images). Specifying text
-          # will change it to the new value. Empty string is valid only with 
-          # `destination=""`
-          text: smooth operator
-          # title is the title element of a link or image. Specifying text
-          # will change it to the new value.
-          title: a title
-      # localityDomains can be specified on node level too.
-      # Node's localityDomain definitions override and amend global ones.
-      localityDomain:
-        github.com/gardener/gardener:
-          version: v1.11.1
-          path: gardener/gardener
-          # exclude omits resources from path. You can 
-          # also use include with the reverse semantics
-          exclude:
-          - example
-        # downloadSubstitutes is a list of regular expressions matching
-        # links to resources on documents that will be downloaded, mapped
-        # to name expressions tha define how the downloaded resources will
-        # named.
-        # There is a set of variables that can be used to construct the
-        # expressions:
-        # - $name: the original name of the resource
-        # - $path: the original path of the resource
-        # - $uuid: a UUID generated for the resource
-        # - $ext: a original resource extension
-        # The default expression applying to all resources is: $uuid.$ext
-        # Besides regular-expression-to-expression mappings it is possible
-        # to map exact URLs (escaped) to concrete names.
-        downloadSubstitutes:
-          "\\.(jpg|gif|png)": "$name-hires-$uuid.$ext"
-    - name: deploying-gardenlets
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet.md
-    - name: automatic-deployment
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet_automatically.md
-    - name: deploy-gardenlet-manually
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/2a33b26458dddd7ad09c4c3b2311d3391db890e7/docs/deployment/deploy_gardenlet_manually.md
-    - name: shooted-seeds
-      contentSelectors:
-        - source: https://github.com/gardener/gardener/blob/master/docs/usage/shooted_seed.md
-localityDomain:
-  github.com/gardener/gardener:
-    version: v1.10.0
-    path: gardener/gardener
+        rewrites:
+          github.com/gardener/gardener:
+            version: v1.11.1
+          "https://github.com/gardener/gardener/blob/master/docs/usage/shooted_seed.md":
+            # destination is the link reference URL. If it is empty string, 
+            # the links markup is removed leaving only link text behind.
+            # For images, the entire markup is removed.
+            destination: ""
+          "https://kubernetes.io/docs/concepts/extend-kubernetes/operator/":
+            destination: "https://kubernetes.io/docs/concepts/extend-kubernetes/operator1111"
+            # text is a link text element (alt-text for images). Specifying text
+            # will change it to the new value. Empty string is valid only with 
+            # `destination=""`
+            text: smooth operator
+            # title is the title element of a link or image. Specifying text
+            # will change it to the new value.
+            title: a title
+        downloads:
+          # localityDomains can be specified on node level too.
+          # Node's localityDomain definitions override and amend global ones.
+          scope:
+            github.com/gardener/gardener:
+              version: v1.11.1
+          # downloadSubstitutes is a list of regular expressions matching
+          # links to resources on documents that will be downloaded, mapped
+          # to name expressions tha define how the downloaded resources will
+          # named.
+          # There is a set of variables that can be used to construct the
+          # expressions:
+          # - $name: the original name of the resource
+          # - $path: the original path of the resource
+          # - $uuid: a UUID generated for the resource
+          # - $ext: a original resource extension
+          # The default expression applying to all resources is: $uuid.$ext
+          # Besides regular-expression-to-expression mappings it is possible
+          # to map exact URLs (escaped) to concrete names.              
+          renames:
+            "\\.(jpg|gif|png)": "$name-hires-$uuid.$ext"
+    - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet.md
+    - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet_automatically.md
+    - source: https://github.com/gardener/gardener/blob/2a33b26458dddd7ad09c4c3b2311d3391db890e7/docs/deployment/deploy_gardenlet_manually.md
+    - source: https://github.com/gardener/gardener/blob/master/docs/usage/shooted_seed.md
+links:
+  # The key in the mapping is an absolute form of a document link that will be 
+  # subject to transformation
+  rewrites:
+    github.com/gardener/gardener:
+      version: v1.10.0
+  downloads:
+    scope:
+      github.com/gardener/gardener: ~

example/simple/00.yaml

@@ -1,11 +1,46 @@
-# The documentation structure root node. Mandatory.
-root:
-    name: doc
-    # nodeSelector is resolved to a node hierarchy, where nodes are selected
-    # by criteria (criteria is not implemented yet, i.e no filters). 
-    nodesSelector:
-      # A node selector path defines the scope that will be used to
-      # generate a hierarchy. For GitHub paths that is a folder in a GitHub repo
-      # and the generated nodes hierarchy corresponds ot the file/folder structure 
-      # available in the repository at that path.
-      path: https://github.com/gardener/gardener/tree/v1.10.0/docs
+# Structuring Material
+# 
+# Container and Document Nodes
+#
+# A documentation structure definition
+# A structure consists of a list of node definitions.
+# These could be A) document nodes that reference markdown content sources(s),
+# or B) container nodes that structure nodes hierarchically and can contain
+# other nodes.
+# The structure below will be serialized as follows at destination <d>:
+# <d>
+#  |__ overview.md (file)
+#  |__ concepts (folder)
+#  |____ apiserver.md (file)
+#
+structure:
+  # Example of a document node that will be serialized as top-level file in the
+  # destination configured in docforge, with name overview.md
+  # A node name will be used to identify the serialized node content. When 
+  # writing to filesystem, the name translates ot a file name.
+  #
+  # A document node that has 'source' property provides several additional 
+  # options for node names:
+  # 1. Node name can be inferred from its source path.
+  # 2. Node name can be constructed from an expression using the variables 
+  # $name and $ext, which stand for the name of the resource at node's source 
+  # path, and $ext stands for its extension.
+  - name: overview.md
+    # A document node should always have content form source(s) assigned to it.
+    # There are multiple options to assign content form source to a node.
+    # The simplest one is to use source as in this example, which assigns the 
+    # content of the resources at the URL provided by source to this node.
+    # Other options with more sophisticated options to control and reorganize
+    # content from potentially multiple source are contentSelector and template.
+    source: https://github.com/gardener/gardener/blob/master/docs/README.md
+    # Example of a container node with no content source with name concepts.
+    # Writing this structure to a file system will serialize it as folder `concepts`.
+  - name: concepts
+    # The property `nodes` references the direct descendant nodes of this node. It
+    # is used to organize nodes into a tree.
+    # The property nodes is specific for container nodes. Document nodes do not
+    # specify nodes, because they are leafs in the tree nodes structure.
+    nodes:
+    # Example of a minimal document node specification. The node name is 
+    # inferred to be 'apiserver.md' from the path in its source property.
+    - source: https://github.com/gardener/gardener/blob/master/docs/concepts/apiserver.md

example/simple/01.yaml

@@ -1,41 +1,22 @@
-# The documentation structure root node. Mandatory.
-root:
-    name: doc
+# Structuring Material
+# 
+# Node selectors
+#
+# Node selectors provide options for pulling whole remote hierarchies
+# into node (sub)structures, and filtering them to a subset.
+structure:
+  - name: concepts
     # nodeSelector is resolved to a node hierarchy, where nodes are selected
-    # by criteria (criteria is not implemented yet, i.e no filters). 
+    # by optional filtering criteria, and attached to a container node as its 
+    # descendants hierarchy.
+    # In this example the specified node selector will be resolved to a node hierarchy 
+    # produced from the file-folder structure at the specified path. Only markdown 
+    # documents are included as nodes. Empty folders are not included.
+    # The resolved structure is attached at this node, so the contents of the 
+    # `concepts` folder below will become contents of this node (and folder when serialized)
     nodesSelector:
-      # A node selector path defines the scope that will be used to
+      # A node selector path defines the top-level scope that will be used to
       # generate a hierarchy. For GitHub paths that is a folder in a GitHub repo
       # and the generated nodes hierarchy corresponds ot the file/folder structure 
       # available in the repository at that path.
-      path: https://github.com/gardener/gardener/tree/master/docs
-    # A list of child nodes to this structure node to form document structure hierarchy.
-    # Note that if a nodeSelector is specified on this node, will be merged with other
-    # existing nodes in `nodes`. Nodes with the same name will have their other properties
-    # merged, making it possible to add properties to the generated structure or additional
-    # nodes.
-    nodes:
-      - name: aws_provider
-        # contentSelectors is a list of source selection specifications.
-        # Normally, there will be one but it is possible to specify several and
-        # they will be appended in that order. 
-        contentSelectors:
-          # Source specifies location of document source. 
-          # The supported sources as of now are GitHub repository documents and wiki pages.
-        - source: https://github.com/gardener/gardener-extension-provider-aws/blob/v1.13.0/docs/usage-as-end-user.md
-# A localityDomain defines the scope of documentation structure "local" 
-# resources that are downloaded along with structure's documents.
-localityDomain:
-  # A locality domain. GFor GitHub it is in the form
-  # <host>/<organization>/<repository>
-  github.com/gardener/gardener:
-    # The version, if specified, is applied to all links inside this domain. 
-    # Document-local resources that will be downloaded (inside `path`), and
-    # links that will be absolute in this domain (github.com/gardener/gardener)
-    # will be rewritten with this version in their URLs. 
-    version: v1.11.1
-    # Path inside this domain that defines the scope of the "document-local"
-    # resources, which will be downloaded along with documents.
-    # If version is specified, the links used to download the resources are
-    # rewritten to match the version.
-    path: gardener/gardener/docs
+      path: https://github.com/gardener/gardener/tree/master/docs/concepts

example/simple/02.yaml

@@ -0,0 +1,12 @@
+# Structuring Material
+# 
+# Node selectors
+#
+# Node selectors provide options for pulling whole remote hierarchies
+# into node (sub)structures, and filtering them to a subset.
+nodesSelector:
+  # A node selector path defines the top-level scope that will be used to
+  # generate a hierarchy. For GitHub paths that is a folder in a GitHub repo
+  # and the generated nodes hierarchy corresponds ot the file/folder structure 
+  # available in the repository at that path.
+  path: https://github.com/gardener/gardener/tree/master/docs/concepts

example/simple/03.yaml

@@ -0,0 +1,56 @@
+# The documentation structure.
+structure:
+  # Top-level container node with descendent hierarchy defined both with rules
+  # by its nodeSelector and explicitly with its nodes list property
+  # name stands for the name of the node. It is mandatory property for container nodes
+  - name: concepts
+    # nodeSelector is resolved to a node hierarchy using path and rules to select nodes
+    # to be part of the hierarchy. not specifying anything but path will end up in 
+    # selecting the whole file/folder structure at path.
+    nodesSelector:
+      # A node selector path defines the scope that will be used to
+      # generate a hierarchy. For GitHub paths that is a folder in a GitHub repo
+      # and the generated nodes hierarchy corresponds ot the file/folder structure 
+      # available in the repository at that path.
+      path: https://github.com/gardener/gardener/tree/v1.11.1/docs/concepts
+    # A list of child nodes to this structure node to explicitly define document structure hierarchy.
+    # Merging nodeSelector hierarchy with nodes:
+    # If both a nodeSelector and nodes are specified on this node, when the node selector is resolved, 
+    # its resulting top-level nodes will merge with other existing nodes in `nodes`. Nodes with the same
+    # name will have their other properties merged, making it possible to add properties to
+    # the generated structure or additional nodes.
+    nodes:
+        # Name of this document node.
+        # Name is not mandatory for document nodes if source is provided. With source and no name,
+        # the name will be resolved to the resource name in source.
+      - name: architecture
+        # Source specifies location of document source. 
+        # The supported sources as of now are GitHub repository documents and wiki pages.
+        source: https://github.com/gardener/documentation/wiki/Architecture
+# Links define configuration for handling document resource references 
+# including both hyperlinks and images. 
+links:
+  # rewrites defines rewrite rules for document links. 
+  # rules are mapped to regular expressions to match links in documents
+  rewrites:
+    # A regex to rule mapping binding all links that features gardener/gardener/blob,
+    # gardener/gardener/tree or gardener/gardener/raw to this rule
+    gardener/gardener/(blob|tree|raw):
+      # The version, if specified, is applied to all links matched by this regex. 
+      # Both document-local resources that will be downloaded, and
+      # links that will be absolute in this domain (gardener/gardener)
+      # will be rewritten with this version in their URLs.
+      version: v1.11.1
+    # Mapping a regular expression to nil effectively removes matching links
+    # from their documents
+    gardener/gardener/(pulls|pull|issue|issues): ~
+  # downloads define the downloads domain and global renaming rules for
+  # downloaded resources
+  downloads:
+    # Scope defines the download scope for documents, mapping regular expressions
+    # to optional rename rules valid for links matched by them. Resources with URLs
+    # that match the regex will be downloaded.
+    # A mapping to nil will apply a default rename pattern ($uuid.$ext)
+    scope:
+      # Download all referenced blobs and raw type of files in gardener/gardener's docs folder
+      gardener/gardener/(blob|raw)/v1.11.1/docs: ~