[mdatagen] Improve generated documentation (#16556)

- Split Metrics into two sections: Default Metrics and Optional Metrics
- Split Attributes section and move attributes to the metrics where they applicable to make it clear which attributes applied to a particular metric. 
- Add "Aggregation Temporality" and "Monotonic" fields to the metric table
- Add enum values to Resource attributes

.chloggen/mdatagen-improve-documentation.yaml

@@ -0,0 +1,11 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: cmd/mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Improve generated documentation
+
+# One or more tracking issues related to the change
+issues: [16556]

cmd/mdatagen/loader_test.go

@@ -74,7 +74,7 @@ func Test_loadMetadata(t *testing.T) {
 						Unit:                  "s",
 						Sum: &sum{
 							MetricValueType: MetricValueType{pmetric.NumberDataPointValueTypeDouble},
-							Aggregated:      Aggregated{Aggregation: "cumulative"},
+							Aggregated:      Aggregated{Aggregation: pmetric.AggregationTemporalityCumulative},
 							Mono:            Mono{Monotonic: true},
 						},
 						Attributes: []attributeName{"freeFormAttribute", "freeFormAttributeWithValue", "enumAttribute", "booleanValueType"},

cmd/mdatagen/main.go

@@ -58,19 +58,21 @@ func run(ymlPath string) error {
 	if err = os.MkdirAll(codeDir, 0700); err != nil {
 		return fmt.Errorf("unable to create output directory %q: %w", codeDir, err)
 	}
-	if err = generateCode(tmplDir, codeDir, "metrics.go", md); err != nil {
+	if err = generateFile(filepath.Join(tmplDir, "metrics.go.tmpl"),
+		filepath.Join(codeDir, "generated_metrics.go"), md); err != nil {
 		return err
 	}
-	if err = generateCode(tmplDir, codeDir, "metrics_test.go", md); err != nil {
+	if err = generateFile(filepath.Join(tmplDir, "metrics_test.go.tmpl"),
+		filepath.Join(codeDir, "generated_metrics_test.go"), md); err != nil {
 		return err
 	}
-	return generateDocumentation(tmplDir, ymlDir, md)
+	return generateFile(filepath.Join(tmplDir, "documentation.md.tmpl"), filepath.Join(ymlDir, "documentation.md"), md)
 }
 
-func generateCode(tmplDir string, outputDir string, tmplFile string, md metadata) error {
+func generateFile(tmplFile string, outputFile string, md metadata) error {
 	tmpl := template.Must(
 		template.
-			New(tmplFile + ".tmpl").
+			New(filepath.Base(tmplFile)).
 			Option("missingkey=error").
 			Funcs(map[string]interface{}{
 				"publicVar": func(s string) (string, error) {
@@ -85,6 +87,9 @@ func generateCode(tmplDir string, outputDir string, tmplFile string, md metadata
 					}
 					return string(an)
 				},
+				"metricInfo": func(mn metricName) metric {
+					return md.Metrics[mn]
+				},
 				"parseImportsRequired": func(metrics map[metricName]metric) bool {
 					for _, m := range metrics {
 						if m.Data().HasMetricInputType() {
@@ -93,56 +98,34 @@ func generateCode(tmplDir string, outputDir string, tmplFile string, md metadata
 					}
 					return false
 				},
-			}).ParseFiles(filepath.Join(tmplDir, tmplFile+".tmpl")))
+				"stringsJoin": strings.Join,
+			}).ParseFiles(tmplFile))
+
 	buf := bytes.Buffer{}
 
 	if err := tmpl.Execute(&buf, templateContext{metadata: md, Package: "metadata"}); err != nil {
 		return fmt.Errorf("failed executing template: %w", err)
 	}
 
-	formatted, err := format.Source(buf.Bytes())
-
-	if err != nil {
-		errstr := strings.Builder{}
-		_, _ = fmt.Fprintf(&errstr, "failed formatting source: %v", err)
-		errstr.WriteString("--- BEGIN SOURCE ---")
-		errstr.Write(buf.Bytes())
-		errstr.WriteString("--- END SOURCE ---")
-		return errors.New(errstr.String())
-	}
-
-	outputFilepath := filepath.Join(outputDir, "generated_"+tmplFile)
-	if err := os.Remove(outputFilepath); err != nil && !errors.Is(err, os.ErrNotExist) {
-		return fmt.Errorf("unable to remove genererated file %q: %w", outputFilepath, err)
-	}
-	if err := os.WriteFile(outputFilepath, formatted, 0600); err != nil {
-		return fmt.Errorf("failed writing %q: %w", outputFilepath, err)
+	result := buf.Bytes()
+
+	if strings.HasSuffix(outputFile, ".go") {
+		var err error
+		result, err = format.Source(buf.Bytes())
+		if err != nil {
+			errstr := strings.Builder{}
+			_, _ = fmt.Fprintf(&errstr, "failed formatting source: %v", err)
+			errstr.WriteString("--- BEGIN SOURCE ---")
+			errstr.Write(buf.Bytes())
+			errstr.WriteString("--- END SOURCE ---")
+			return errors.New(errstr.String())
+		}
 	}
 
-	return nil
-}
-
-func generateDocumentation(tmplDir string, outputDir string, md metadata) error {
-	tmpl := template.Must(
-		template.
-			New("documentation.md.tmpl").
-			Option("missingkey=error").
-			Funcs(map[string]interface{}{
-				"publicVar": func(s string) (string, error) {
-					return formatIdentifier(s, true)
-				},
-				"stringsJoin": strings.Join,
-			}).ParseFiles(filepath.Join(tmplDir, "documentation.md.tmpl")))
-
-	buf := bytes.Buffer{}
-
-	tmplCtx := templateContext{metadata: md, Package: "metadata"}
-	if err := tmpl.Execute(&buf, tmplCtx); err != nil {
-		return fmt.Errorf("failed executing template: %w", err)
+	if err := os.Remove(outputFile); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("unable to remove genererated file %q: %w", outputFile, err)
 	}
-
-	outputFile := filepath.Join(outputDir, "documentation.md")
-	if err := os.WriteFile(outputFile, buf.Bytes(), 0600); err != nil {
+	if err := os.WriteFile(outputFile, result, 0600); err != nil {
 		return fmt.Errorf("failed writing %q: %w", outputFile, err)
 	}
 

cmd/mdatagen/main_test.go

@@ -55,22 +55,17 @@ func Test_runContents(t *testing.T) {
 		yml string
 	}
 	tests := []struct {
-		name                  string
-		args                  args
-		expectedDocumentation string
-		want                  string
-		wantErr               bool
+		name    string
+		args    args
+		wantErr bool
 	}{
 		{
-			name:                  "valid metadata",
-			args:                  args{validMetadata},
-			expectedDocumentation: "testdata/documentation.md",
-			want:                  "",
+			name: "valid metadata",
+			args: args{validMetadata},
 		},
 		{
 			name:    "invalid yaml",
 			args:    args{"invalid"},
-			want:    "",
 			wantErr: true,
 		},
 	}
@@ -89,18 +84,7 @@ func Test_runContents(t *testing.T) {
 			require.NoError(t, err)
 
 			require.FileExists(t, filepath.Join(tmpdir, "internal/metadata/generated_metrics.go"))
-
-			actualDocumentation := filepath.Join(tmpdir, "documentation.md")
-			require.FileExists(t, actualDocumentation)
-			if tt.expectedDocumentation != "" {
-				expectedFileBytes, err := os.ReadFile(tt.expectedDocumentation)
-				require.NoError(t, err)
-
-				actualFileBytes, err := os.ReadFile(actualDocumentation)
-				require.NoError(t, err)
-
-				require.Equal(t, expectedFileBytes, actualFileBytes)
-			}
+			require.FileExists(t, filepath.Join(tmpdir, "documentation.md"))
 		})
 	}
 }

cmd/mdatagen/metricdata.go

@@ -34,22 +34,29 @@ type MetricData interface {
 }
 
 // Aggregated defines a metric aggregation type.
+// TODO: Rename to AggregationTemporality
 type Aggregated struct {
 	// Aggregation describes if the aggregator reports delta changes
 	// since last report time, or cumulative changes since a fixed start time.
-	Aggregation string `mapstructure:"aggregation" validate:"oneof=delta cumulative"`
+	Aggregation pmetric.AggregationTemporality `validate:"required"`
 }
 
-// Type gets the metric aggregation type.
-func (agg Aggregated) Type() string {
-	switch agg.Aggregation {
-	case "delta":
-		return "pmetric.AggregationTemporalityDelta"
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (agg *Aggregated) UnmarshalText(text []byte) error {
+	switch vtStr := string(text); vtStr {
 	case "cumulative":
-		return "pmetric.AggregationTemporalityCumulative"
+		agg.Aggregation = pmetric.AggregationTemporalityCumulative
+	case "delta":
+		agg.Aggregation = pmetric.AggregationTemporalityDelta
 	default:
-		return "pmetric.AggregationTemporalityUnknown"
+		return fmt.Errorf("invalid aggregation: %q", vtStr)
 	}
+	return nil
+}
+
+// String returns string representation of the aggregation temporality.
+func (agg *Aggregated) String() string {
+	return agg.Aggregation.String()
 }
 
 // Mono defines the metric monotonicity.
@@ -127,7 +134,7 @@ func (d gauge) HasMetricInputType() bool {
 }
 
 type sum struct {
-	Aggregated      `mapstructure:",squash"`
+	Aggregated      `mapstructure:"aggregation"`
 	Mono            `mapstructure:",squash"`
 	MetricValueType `mapstructure:"value_type"`
 	MetricInputType `mapstructure:",squash"`

cmd/mdatagen/metricdata_test.go

@@ -35,14 +35,3 @@ func TestMetricData(t *testing.T) {
 		assert.Equal(t, arg.hasMonotonic, arg.metricData.HasMonotonic())
 	}
 }
-
-func TestAggregation(t *testing.T) {
-	delta := Aggregated{Aggregation: "delta"}
-	assert.Equal(t, "pmetric.AggregationTemporalityDelta", delta.Type())
-
-	cumulative := Aggregated{Aggregation: "cumulative"}
-	assert.Equal(t, "pmetric.AggregationTemporalityCumulative", cumulative.Type())
-
-	unknown := Aggregated{Aggregation: ""}
-	assert.Equal(t, "pmetric.AggregationTemporalityUnknown", unknown.Type())
-}

cmd/mdatagen/templates/documentation.md.tmpl

@@ -1,43 +1,88 @@
+{{- define "metric-documenation" -}}
+{{- $metricName := . }}
+{{- $metric := $metricName | metricInfo -}}
+
+### {{ $metricName }}
+
+{{ $metric.Description }}
+
+{{- if $metric.ExtendedDocumentation }}
+
+{{ $metric.ExtendedDocumentation }}
+
+{{- end }}
+
+| Unit | Metric Type | Value Type |{{ if $metric.Data.HasAggregated }} Aggregation Temporality |{{ end }}{{ if $metric.Data.HasMonotonic }} Monotonic |{{ end }}
+| ---- | ----------- | ---------- |{{ if $metric.Data.HasAggregated }} ----------------------- |{{ end }}{{ if $metric.Data.HasMonotonic }} --------- |{{ end }}
+| {{ $metric.Unit }} | {{ $metric.Data.Type }} | {{ $metric.Data.MetricValueType }} |
+{{- if $metric.Data.HasAggregated }} {{ $metric.Data.Aggregated }} |{{ end }}
+{{- if $metric.Data.HasMonotonic }} {{ $metric.Data.Monotonic }} |{{ end }}
+
+{{- if $metric.Attributes }}
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+{{- range $metric.Attributes }}
+{{- $attribute := . | attributeInfo }}
+| {{ . | attributeKey }} | {{ $attribute.Description }} |
+{{- if $attribute.Enum }} {{ $attribute.Type }}: ``{{ stringsJoin $attribute.Enum "``, ``" }}``{{ else }} Any {{ $attribute.Type }}{{ end }} |
+{{- end }}
+
+{{- end }}
+
+{{- end -}}
+
 [comment]: <> (Code generated by mdatagen. DO NOT EDIT.)
 
 # {{ .Name }}
 
-## Metrics
+## Default Metrics
+
+The following metrics are emitted by default. Each of them can be disabled by applying the following configuration:
+
+```yaml
+metrics:
+  <metric_name>:
+    enabled: false
+```
+
+{{- range $metricName, $metric := .Metrics }}
+{{- if $metric.IsEnabled }}
 
-These are the metrics available for this scraper.
+{{ template "metric-documenation" $metricName }}
 
-| Name | Description | Unit | Type | Attributes |
-| ---- | ----------- | ---- | ---- | ---------- |
-{{- range $metricName, $metricInfo := .Metrics }}
-| {{ if $metricInfo.IsEnabled }}**{{ end }}{{ $metricName }}{{ if $metricInfo.IsEnabled }}**
-{{- end }} | {{ $metricInfo.Description }}{{ if $metricInfo.ExtendedDocumentation }} {{ $metricInfo.ExtendedDocumentation }}{{ end }} | {{ $metricInfo.Unit }} | {{ $metricInfo.Data.Type }}({{ $metricInfo.Data.MetricValueType }}) | <ul>
-{{- range $index, $attributeName := $metricInfo.Attributes }} <li>{{ $attributeName }}</li> {{- end }} </ul> |
 {{- end }}
+{{- end }}
+
+## Optional Metrics
 
-**Highlighted metrics** are emitted by default. Other metrics are optional and not emitted by default.
-Any metric can be enabled or disabled with the following scraper configuration:
+The following metrics are not emitted by default. Each of them can be enabled by applying the following configuration:
 
 ```yaml
 metrics:
   <metric_name>:
-    enabled: <true|false>
+    enabled: true
 ```
 
-{{- if .ResourceAttributes }}
+{{- range $metricName, $metric := .Metrics }}
+{{- if $metric.IsEnabled }}
 
-## Resource attributes
+{{ template "metric-documenation" $metricName }}
 
-| Name | Description | Type |
-| ---- | ----------- | ---- |
-{{- range $attributeName, $attributeInfo := .ResourceAttributes }}
-| {{ $attributeName }} | {{ $attributeInfo.Description }} | {{ $attributeInfo.Type }} |
 {{- end }}
 {{- end }}
 
-## Metric attributes
+{{- if .ResourceAttributes }}
+
+## Resource Attributes
 
 | Name | Description | Values |
 | ---- | ----------- | ------ |
-{{- range $attributeName, $attributeInfo := .Attributes }}
-| {{ $attributeName }}{{- if $attributeInfo.Value }} ({{ $attributeInfo.Value }}){{- end}} | {{ $attributeInfo.Description }} | {{ stringsJoin $attributeInfo.Enum ", " }} |
+{{- range $attributeName, $attribute := .ResourceAttributes }}
+| {{ $attributeName }} | {{ $attribute.Description }} |
+{{- if $attribute.Enum }} {{ $attribute.Type }}: ``{{ stringsJoin $attribute.Enum "``, ``" }}``{{ else }} Any {{ $attribute.Type }}{{ end }} |
+{{- end }}
+
 {{- end }}

cmd/mdatagen/templates/metrics.go.tmpl

@@ -109,7 +109,7 @@ func (m *metric{{ $name.Render }}) init() {
 	m.data.{{ $metric.Data.Type }}().SetIsMonotonic({{ $metric.Data.Monotonic }})
 	{{- end }}
 	{{- if $metric.Data.HasAggregated }}
-	m.data.{{ $metric.Data.Type }}().SetAggregationTemporality({{ $metric.Data.Aggregated.Type }})
+	m.data.{{ $metric.Data.Type }}().SetAggregationTemporality(pmetric.AggregationTemporality{{ $metric.Data.Aggregated }})
 	{{- end }}
 	{{- if $metric.Attributes }}
 	m.data.{{ $metric.Data.Type }}().DataPoints().EnsureCapacity(m.capacity)

cmd/mdatagen/templates/metrics_test.go.tmpl

@@ -100,7 +100,7 @@ func TestAllMetrics(t *testing.T) {
 			assert.Equal(t, {{ $metric.Data.Monotonic }}, ms.At(i).{{ $metric.Data.Type }}().IsMonotonic())
 			{{- end }}
 			{{- if $metric.Data.HasAggregated }}
-			assert.Equal(t, {{ $metric.Data.Aggregated.Type }}, ms.At(i).{{ $metric.Data.Type }}().AggregationTemporality())
+			assert.Equal(t, pmetric.AggregationTemporality{{ $metric.Data.Aggregated }}, ms.At(i).{{ $metric.Data.Type }}().AggregationTemporality())
 			{{- end }}
 			dp := ms.At(i).{{ $metric.Data.Type }}().DataPoints().At(0)
 			assert.Equal(t, start, dp.StartTimestamp())