From 9c7cdaf3aa297d1ba66edd5fe8ba488fa6cb5931 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Wed, 30 Nov 2022 15:28:57 +0100 Subject: [PATCH 01/17] restructure configextractor Signed-off-by: Christian Richter --- docs/helpers/configenvextractor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/helpers/configenvextractor.go b/docs/helpers/configenvextractor.go index 0e9f2e4b7db..664855da187 100644 --- a/docs/helpers/configenvextractor.go +++ b/docs/helpers/configenvextractor.go @@ -17,7 +17,7 @@ var targets = map[string]string{ "environment-variable-docs-generator.go.tmpl": "output/env/environment-variable-docs-generator.go", } -func main() { +func RenderTemplates() { fmt.Println("Getting relevant packages") paths, err := filepath.Glob("../../services/*/pkg/config/defaults/defaultconfig.go") if err != nil { From 3ccf79c6fec27561e3f8114f49b7e0a0b6e7b6ff Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Wed, 30 Nov 2022 15:29:18 +0100 Subject: [PATCH 02/17] create main.go Signed-off-by: Christian Richter --- docs/helpers/main.go | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/helpers/main.go diff --git a/docs/helpers/main.go b/docs/helpers/main.go new file mode 100644 index 00000000000..08177297b78 --- /dev/null +++ b/docs/helpers/main.go @@ -0,0 +1,6 @@ +package main + +func main() { + RenderTemplates() + GetRogueEnvs() +} From f4e3bb99a338f78d301369c41b3029cf5f0d0d2b Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Wed, 30 Nov 2022 15:29:30 +0100 Subject: [PATCH 03/17] add rogue env var collector Signed-off-by: Christian Richter --- docs/helpers/rogueEnv.go | 92 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/helpers/rogueEnv.go diff --git a/docs/helpers/rogueEnv.go b/docs/helpers/rogueEnv.go new file mode 100644 index 00000000000..c05625c431f --- /dev/null +++ b/docs/helpers/rogueEnv.go @@ -0,0 +1,92 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "gopkg.in/yaml.v2" +) + +const yamlSource = "global_vars.yaml" + +type ConfigVars struct { + Variables []Variable `yaml:"variables"` +} + +type Variable struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + Default string `yaml:"default"` + Description string `yaml:"description"` + DependendServices []Service `yaml:"dependend_services"` + DoIgnore bool `yaml:"do_ignore"` +} + +type Service struct { + Name string `yaml:"name"` + Path string `yaml:"path"` +} + +func GetRogueEnvs() { + curdir, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + fullYamlPath := filepath.Join(curdir, yamlSource) + re := regexp.MustCompile(`"[A-z0-9_]{1,}"`) + vars := &ConfigVars{} + fmt.Printf("Reading existing variable definitions from %s\n", fullYamlPath) + yfile, err := ioutil.ReadFile(fullYamlPath) + if err == nil { + err := yaml.Unmarshal(yfile, &vars) + if err != nil { + log.Fatalf("could not unmarshall %s", fullYamlPath) + } + } + + if err := os.Chdir("../../"); err != nil { + log.Fatal(err) + } + fmt.Println("Gathering variable definitions from source") + out, err := exec.Command("bash", "-c", "grep -R os.Getenv | grep -v rogue-env.go").Output() + if err != nil { + log.Fatal(err) + } + lines := strings.Split(string(out), "\n") + for _, l := range lines { + r := strings.SplitN(l, ":\t", 2) + if r[0] != "" && r[1] != "" { + res := re.FindAll([]byte(r[1]), -1) + for _, item := range res { + AddUniqueToStruct(vars, Variable{Name: strings.Trim(string(item), "\""), Type: ""}) + } + } + } + output, err := yaml.Marshal(vars) + if err != nil { + log.Fatal(err) + } + fmt.Printf("Writing new variable definitions to %s\n", fullYamlPath) + err = ioutil.WriteFile(fullYamlPath, output, 0666) + if err != nil { + log.Fatal("could not write %s", fullYamlPath) + } + if err := os.Chdir(curdir); err != nil { + log.Fatal(err) + } +} + +func AddUniqueToStruct(variables *ConfigVars, variable Variable) { + for _, itm := range variables.Variables { + if itm.Name == variable.Name { + return + } + } + variables.Variables = append(variables.Variables, variable) +} From 472ba967ca83df2d7d76da4ec04beb8a9ffab6c9 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Wed, 30 Nov 2022 15:29:47 +0100 Subject: [PATCH 04/17] add initial yaml for global env vars Signed-off-by: Christian Richter --- docs/helpers/global_vars.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docs/helpers/global_vars.yaml diff --git a/docs/helpers/global_vars.yaml b/docs/helpers/global_vars.yaml new file mode 100644 index 00000000000..97af595b8d4 --- /dev/null +++ b/docs/helpers/global_vars.yaml @@ -0,0 +1,31 @@ +variables: +- name: CS3_GATEWAY + type: "" + default: "" + description: "" + dependend_services: [] + do_ignore: false +- name: CS3_MACHINE_AUTH_API_KEY + type: "" + default: "" + description: "" + dependend_services: [] + do_ignore: false +- name: OCIS_BASE_DATA_PATH + type: "" + default: "" + description: "" + dependend_services: [] + do_ignore: false +- name: OCIS_CONFIG_DIR + type: "" + default: "" + description: "" + dependend_services: [] + do_ignore: false +- name: MICRO_LOG_LEVEL + type: "" + default: "" + description: "" + dependend_services: [] + do_ignore: false From 3d02f805e3437350d2dfe4cb1528a6f4f9bafbf8 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Wed, 30 Nov 2022 15:31:16 +0100 Subject: [PATCH 05/17] update docs-generate make target to reflect the changes Signed-off-by: Christian Richter --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 0de6a1b5378..93a18b7be29 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ help: .PHONY: docs-generate docs-generate: ## run docs-generate for all oCIS services - @pushd helpers && go run configenvextractor.go; popd + @pushd helpers && go run .; popd @$(MAKE) --no-print-directory -C ../ docs-generate .PHONY: docs-init From b88ea444fb58e0fcedff8b56076c35127b5d56b3 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Wed, 30 Nov 2022 16:43:27 +0100 Subject: [PATCH 06/17] add adoc template rendering Signed-off-by: Christian Richter --- docs/helpers/global_vars.yaml | 10 ++++---- docs/helpers/main.go | 1 + docs/helpers/rogueEnv.go | 42 +++++++++++++++++++++++++++++++-- docs/templates/ADOC_global.tmpl | 23 ++++++++++++++++++ 4 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 docs/templates/ADOC_global.tmpl diff --git a/docs/helpers/global_vars.yaml b/docs/helpers/global_vars.yaml index 97af595b8d4..71b9d518f27 100644 --- a/docs/helpers/global_vars.yaml +++ b/docs/helpers/global_vars.yaml @@ -1,31 +1,31 @@ variables: - name: CS3_GATEWAY type: "" - default: "" + default_value: "" description: "" dependend_services: [] do_ignore: false - name: CS3_MACHINE_AUTH_API_KEY type: "" - default: "" + default_value: "" description: "" dependend_services: [] do_ignore: false - name: OCIS_BASE_DATA_PATH type: "" - default: "" + default_value: "" description: "" dependend_services: [] do_ignore: false - name: OCIS_CONFIG_DIR type: "" - default: "" + default_value: "" description: "" dependend_services: [] do_ignore: false - name: MICRO_LOG_LEVEL type: "" - default: "" + default_value: "" description: "" dependend_services: [] do_ignore: false diff --git a/docs/helpers/main.go b/docs/helpers/main.go index 08177297b78..86076042c0f 100644 --- a/docs/helpers/main.go +++ b/docs/helpers/main.go @@ -3,4 +3,5 @@ package main func main() { RenderTemplates() GetRogueEnvs() + RenderGlobalVarsTemplate() } diff --git a/docs/helpers/rogueEnv.go b/docs/helpers/rogueEnv.go index c05625c431f..28eab2d493b 100644 --- a/docs/helpers/rogueEnv.go +++ b/docs/helpers/rogueEnv.go @@ -9,6 +9,7 @@ import ( "path/filepath" "regexp" "strings" + "text/template" "gopkg.in/yaml.v2" ) @@ -22,7 +23,7 @@ type ConfigVars struct { type Variable struct { Name string `yaml:"name"` Type string `yaml:"type"` - Default string `yaml:"default"` + DefaultValue string `yaml:"default_value"` Description string `yaml:"description"` DependendServices []Service `yaml:"dependend_services"` DoIgnore bool `yaml:"do_ignore"` @@ -46,7 +47,7 @@ func GetRogueEnvs() { if err == nil { err := yaml.Unmarshal(yfile, &vars) if err != nil { - log.Fatalf("could not unmarshall %s", fullYamlPath) + log.Fatal(err) } } @@ -90,3 +91,40 @@ func AddUniqueToStruct(variables *ConfigVars, variable Variable) { } variables.Variables = append(variables.Variables, variable) } + +func RenderGlobalVarsTemplate() { + curdir, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + fullYamlPath := filepath.Join(curdir, yamlSource) + + content, err := ioutil.ReadFile("../../docs/templates/ADOC_global.tmpl") + if err != nil { + log.Fatal(err) + } + + targetFolder := "../../docs/services/_includes/adoc/" + + vars := &ConfigVars{} + fmt.Printf("Reading existing variable definitions from %s\n", fullYamlPath) + yfile, err := ioutil.ReadFile(fullYamlPath) + if err != nil { + log.Fatal(err) + } + err = yaml.Unmarshal(yfile, &vars) + if err != nil { + log.Fatal(err) + } + + targetFile, err := os.Create(filepath.Join(targetFolder, "global_configvars.adoc")) + if err != nil { + log.Fatalf("Failed to create target file: %s", err) + } + defer targetFile.Close() + + tpl := template.Must(template.New("").Parse(string(content))) + if err = tpl.Execute(targetFile, *vars); err != nil { + log.Fatalf("Failed to execute template: %s", err) + } +} diff --git a/docs/templates/ADOC_global.tmpl b/docs/templates/ADOC_global.tmpl new file mode 100644 index 00000000000..dca1dd99881 --- /dev/null +++ b/docs/templates/ADOC_global.tmpl @@ -0,0 +1,23 @@ +// set the attribute to true or leave empty, true without any quotes. + +[caption=] +.Global environment Variables +[width="100%",cols="~,~,~,~",options="header"] +| === | +| Name | +| Type | +| Default Value | +| Description | + +{{- range .Variables}} + +a| `{{- .Name }}` + +a| [subs=-attributes] +++{{ .Type }} ++ +a| [subs=-attributes] +++{{.DefaultValue}} ++ +a| [subs=-attributes] +{{.Description}} + +{{- end }} +| === | From 8817d634bdf5595685b100b6e6768f4bd68d96b3 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Wed, 30 Nov 2022 16:46:45 +0100 Subject: [PATCH 07/17] add changelog Signed-off-by: Christian Richter --- changelog/unreleased/add-global-vars-extrator.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/add-global-vars-extrator.md diff --git a/changelog/unreleased/add-global-vars-extrator.md b/changelog/unreleased/add-global-vars-extrator.md new file mode 100644 index 00000000000..80d01b0425e --- /dev/null +++ b/changelog/unreleased/add-global-vars-extrator.md @@ -0,0 +1,7 @@ +Enhancement: add global env variable extractor + +We have added a little tool that will extract global env vars, that are loaded +only through os.Getenv for documentation purposes + +https://github.com/owncloud/ocis/issues/4916 +https://github.com/owncloud/ocis/pull/5164 From 879f7996e3f899d5afe01e0d79d975fa63114d42 Mon Sep 17 00:00:00 2001 From: Christian Richter <1058116+dragonchaser@users.noreply.github.com> Date: Thu, 1 Dec 2022 09:59:10 +0100 Subject: [PATCH 08/17] Bugfixing Co-authored-by: Martin Signed-off-by: Christian Richter --- docs/helpers/rogueEnv.go | 7 ++++--- docs/templates/ADOC_global.tmpl | 16 ++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/helpers/rogueEnv.go b/docs/helpers/rogueEnv.go index 28eab2d493b..d6e00de0529 100644 --- a/docs/helpers/rogueEnv.go +++ b/docs/helpers/rogueEnv.go @@ -55,14 +55,15 @@ func GetRogueEnvs() { log.Fatal(err) } fmt.Println("Gathering variable definitions from source") - out, err := exec.Command("bash", "-c", "grep -R os.Getenv | grep -v rogue-env.go").Output() + out, err := exec.Command("bash", "-c", "grep -R os.Getenv | grep -v rogue-env.go |grep \\.go").Output() if err != nil { log.Fatal(err) } lines := strings.Split(string(out), "\n") for _, l := range lines { r := strings.SplitN(l, ":\t", 2) - if r[0] != "" && r[1] != "" { + if len(r) == 2 && r[0] != "" && r[1] != "" { + fmt.Printf("Parsing %s\n", r[0]) res := re.FindAll([]byte(r[1]), -1) for _, item := range res { AddUniqueToStruct(vars, Variable{Name: strings.Trim(string(item), "\""), Type: ""}) @@ -76,7 +77,7 @@ func GetRogueEnvs() { fmt.Printf("Writing new variable definitions to %s\n", fullYamlPath) err = ioutil.WriteFile(fullYamlPath, output, 0666) if err != nil { - log.Fatal("could not write %s", fullYamlPath) + log.Fatalf("could not write %s", fullYamlPath) } if err := os.Chdir(curdir); err != nil { log.Fatal(err) diff --git a/docs/templates/ADOC_global.tmpl b/docs/templates/ADOC_global.tmpl index dca1dd99881..6e21fb482ef 100644 --- a/docs/templates/ADOC_global.tmpl +++ b/docs/templates/ADOC_global.tmpl @@ -1,13 +1,13 @@ -// set the attribute to true or leave empty, true without any quotes. +// collected through docs/helpers/rougeEnv.go [caption=] -.Global environment Variables +.Environment variables with global scope not included in a service [width="100%",cols="~,~,~,~",options="header"] -| === | -| Name | -| Type | -| Default Value | -| Description | +|=== +| Name +| Type +| Default Value +| Description {{- range .Variables}} @@ -20,4 +20,4 @@ a| [subs=-attributes] {{.Description}} {{- end }} -| === | +|=== From a82b776791a20719cee80d7b7fdb51f49e7bb638 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 7 Dec 2022 16:05:30 +0100 Subject: [PATCH 09/17] include all envs and add ignore parameter Signed-off-by: jkoberg --- docs/helpers/configenvextractor.go | 9 +-- docs/helpers/global_vars.yaml | 62 ++++++++++++++++---- docs/helpers/rogueEnv.go | 93 +++++++++++++++++++++--------- docs/templates/ADOC_global.tmpl | 3 + 4 files changed, 126 insertions(+), 41 deletions(-) diff --git a/docs/helpers/configenvextractor.go b/docs/helpers/configenvextractor.go index 664855da187..46178d9982a 100644 --- a/docs/helpers/configenvextractor.go +++ b/docs/helpers/configenvextractor.go @@ -17,6 +17,7 @@ var targets = map[string]string{ "environment-variable-docs-generator.go.tmpl": "output/env/environment-variable-docs-generator.go", } +// RenderTemplates does something with templates func RenderTemplates() { fmt.Println("Getting relevant packages") paths, err := filepath.Glob("../../services/*/pkg/config/defaults/defaultconfig.go") @@ -32,14 +33,14 @@ func RenderTemplates() { } for template, output := range targets { - GenerateIntermediateCode(template, output, paths) - RunIntermediateCode(output) + generateIntermediateCode(template, output, paths) + runIntermediateCode(output) } fmt.Println("Cleaning up") os.RemoveAll("output") } -func GenerateIntermediateCode(templatePath string, intermediateCodePath string, paths []string) { +func generateIntermediateCode(templatePath string, intermediateCodePath string, paths []string) { content, err := os.ReadFile(templatePath) if err != nil { log.Fatal(err) @@ -60,7 +61,7 @@ func GenerateIntermediateCode(templatePath string, intermediateCodePath string, } } -func RunIntermediateCode(intermediateCodePath string) { +func runIntermediateCode(intermediateCodePath string) { fmt.Println("Running intermediate go code for " + intermediateCodePath) defaultPath := "~/.ocis" os.Setenv("OCIS_BASE_DATA_PATH", defaultPath) diff --git a/docs/helpers/global_vars.yaml b/docs/helpers/global_vars.yaml index 71b9d518f27..290a90cecab 100644 --- a/docs/helpers/global_vars.yaml +++ b/docs/helpers/global_vars.yaml @@ -1,31 +1,73 @@ variables: -- name: CS3_GATEWAY +- rawname: CS3_GATEWAY + path: services/idp/pkg/backends/cs3/bootstrap/cs3.go:76 + foundincode: true + name: CS3_GATEWAY type: "" default_value: "" description: "" - dependend_services: [] do_ignore: false -- name: CS3_MACHINE_AUTH_API_KEY +- rawname: registryAddressEnv + path: ocis-pkg/registry/registry.go:42 + foundincode: true + name: registryAddressEnv type: "" default_value: "" description: "" - dependend_services: [] do_ignore: false -- name: OCIS_BASE_DATA_PATH +- rawname: MICRO_LOG_LEVEL + path: ocis-pkg/log/log.go:34 + foundincode: true + name: MICRO_LOG_LEVEL type: "" default_value: "" description: "" - dependend_services: [] do_ignore: false -- name: OCIS_CONFIG_DIR +- rawname: CS3_MACHINE_AUTH_API_KEY + path: services/idp/pkg/backends/cs3/bootstrap/cs3.go:77 + foundincode: true + name: CS3_MACHINE_AUTH_API_KEY type: "" default_value: "" description: "" - dependend_services: [] do_ignore: false -- name: MICRO_LOG_LEVEL +- rawname: registryEnv + path: ocis-pkg/registry/registry.go:44 + foundincode: true + name: registryEnv + type: "" + default_value: "" + description: "" + do_ignore: false +- rawname: OCIS_BASE_DATA_PATH + path: ocis-pkg/config/defaults/paths.go:23 + foundincode: true + name: OCIS_BASE_DATA_PATH + type: "" + default_value: "" + description: "" + do_ignore: false +- rawname: OCIS_CONFIG_DIR + path: ocis-pkg/config/defaults/paths.go:56 + foundincode: true + name: OCIS_CONFIG_DIR + type: "" + default_value: "" + description: "" + do_ignore: false +- rawname: parts[0] + path: ocis-pkg/config/envdecode/envdecode.go:382 + foundincode: true + name: parts[0] + type: "" + default_value: "" + description: "" + do_ignore: false +- rawname: MICRO_LOG_LEVEL + path: ocis-pkg/log/log.go:30 + foundincode: true + name: MICRO_LOG_LEVEL type: "" default_value: "" description: "" - dependend_services: [] do_ignore: false diff --git a/docs/helpers/rogueEnv.go b/docs/helpers/rogueEnv.go index d6e00de0529..f61958a7a49 100644 --- a/docs/helpers/rogueEnv.go +++ b/docs/helpers/rogueEnv.go @@ -16,31 +16,45 @@ import ( const yamlSource = "global_vars.yaml" +// ConfigVars is the main yaml source type ConfigVars struct { Variables []Variable `yaml:"variables"` } +// Variable contains all information about one rogue envvar type Variable struct { - Name string `yaml:"name"` - Type string `yaml:"type"` - DefaultValue string `yaml:"default_value"` - Description string `yaml:"description"` - DependendServices []Service `yaml:"dependend_services"` - DoIgnore bool `yaml:"do_ignore"` -} - -type Service struct { - Name string `yaml:"name"` + // These field structs are automatically filled: + // RawName can be the name of the envvar or the name of its var + RawName string `yaml:"rawname"` + // Path to the envvar with linenumber Path string `yaml:"path"` + // FoundInCode indicates if the variable is still found in the codebase. TODO: delete immediately? + FoundInCode bool `yaml:"foundincode"` + // Name is equal to RawName but will not be overwritten in consecutive runs + Name string `yaml:"name"` + + // These field structs need manual filling: + // Type of the envvar + Type string `yaml:"type"` + // DefaultValue of the envvar + DefaultValue string `yaml:"default_value"` + // Description of what this envvar does + Description string `yaml:"description"` + // Ignore this envvar when creating docs? + Ignore bool `yaml:"do_ignore"` + + // For simplicity ignored for now: + // DependendServices []Service `yaml:"dependend_services"` } +// GetRogueEnvs extracts the rogue envs from the code func GetRogueEnvs() { curdir, err := os.Getwd() if err != nil { log.Fatal(err) } fullYamlPath := filepath.Join(curdir, yamlSource) - re := regexp.MustCompile(`"[A-z0-9_]{1,}"`) + re := regexp.MustCompile(`os.Getenv\(([^\)]+)\)`) vars := &ConfigVars{} fmt.Printf("Reading existing variable definitions from %s\n", fullYamlPath) yfile, err := ioutil.ReadFile(fullYamlPath) @@ -55,21 +69,54 @@ func GetRogueEnvs() { log.Fatal(err) } fmt.Println("Gathering variable definitions from source") - out, err := exec.Command("bash", "-c", "grep -R os.Getenv | grep -v rogue-env.go |grep \\.go").Output() + out, err := exec.Command("bash", "-c", "grep -RHn os.Getenv | grep -v rogueEnv.go |grep \\.go").Output() if err != nil { log.Fatal(err) } lines := strings.Split(string(out), "\n") + + // find current vars + currentVars := make(map[string]Variable) for _, l := range lines { + fmt.Printf("Parsing %s\n", l) r := strings.SplitN(l, ":\t", 2) - if len(r) == 2 && r[0] != "" && r[1] != "" { - fmt.Printf("Parsing %s\n", r[0]) - res := re.FindAll([]byte(r[1]), -1) - for _, item := range res { - AddUniqueToStruct(vars, Variable{Name: strings.Trim(string(item), "\""), Type: ""}) - } + if len(r) != 2 || r[0] == "" || r[1] == "" { + continue + } + + res := re.FindAllSubmatch([]byte(r[1]), -1) + if len(res) != 1 || len(res[0]) < 2 { + fmt.Printf("Error envvar not matching pattern: %s", r[1]) + continue + } + + path := r[0] + name := strings.Trim(string(res[0][1]), "\"") + currentVars[path+name] = Variable{ + RawName: name, + Path: path, + FoundInCode: true, + Name: name, } } + + // adjust existing vars + for i, v := range vars.Variables { + _, ok := currentVars[v.Path+v.RawName] + if !ok { + vars.Variables[i].FoundInCode = false + continue + } + + vars.Variables[i].FoundInCode = true + delete(currentVars, v.Path+v.RawName) + } + + // add new envvars + for _, v := range currentVars { + vars.Variables = append(vars.Variables, v) + } + output, err := yaml.Marshal(vars) if err != nil { log.Fatal(err) @@ -84,15 +131,7 @@ func GetRogueEnvs() { } } -func AddUniqueToStruct(variables *ConfigVars, variable Variable) { - for _, itm := range variables.Variables { - if itm.Name == variable.Name { - return - } - } - variables.Variables = append(variables.Variables, variable) -} - +// RenderGlobalVarsTemplate renders the global vars template func RenderGlobalVarsTemplate() { curdir, err := os.Getwd() if err != nil { diff --git a/docs/templates/ADOC_global.tmpl b/docs/templates/ADOC_global.tmpl index 6e21fb482ef..ae388a5d5f6 100644 --- a/docs/templates/ADOC_global.tmpl +++ b/docs/templates/ADOC_global.tmpl @@ -11,6 +11,9 @@ {{- range .Variables}} +{{- if .Ignore }} + {{ continue }} +{{- end }} a| `{{- .Name }}` + a| [subs=-attributes] ++{{ .Type }} ++ From 0bd800ac7302ef80019c9da1473705336e168c30 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Mon, 12 Dec 2022 15:53:25 +0100 Subject: [PATCH 10/17] sort envvars by name Signed-off-by: jkoberg --- docs/helpers/global_vars.yaml | 34 ++++++++++++++++----------------- docs/helpers/rogueEnv.go | 7 +++++++ docs/templates/ADOC_global.tmpl | 4 +++- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/helpers/global_vars.yaml b/docs/helpers/global_vars.yaml index 290a90cecab..2f0de5386e2 100644 --- a/docs/helpers/global_vars.yaml +++ b/docs/helpers/global_vars.yaml @@ -7,10 +7,10 @@ variables: default_value: "" description: "" do_ignore: false -- rawname: registryAddressEnv - path: ocis-pkg/registry/registry.go:42 +- rawname: CS3_MACHINE_AUTH_API_KEY + path: services/idp/pkg/backends/cs3/bootstrap/cs3.go:77 foundincode: true - name: registryAddressEnv + name: CS3_MACHINE_AUTH_API_KEY type: "" default_value: "" description: "" @@ -23,18 +23,10 @@ variables: default_value: "" description: "" do_ignore: false -- rawname: CS3_MACHINE_AUTH_API_KEY - path: services/idp/pkg/backends/cs3/bootstrap/cs3.go:77 - foundincode: true - name: CS3_MACHINE_AUTH_API_KEY - type: "" - default_value: "" - description: "" - do_ignore: false -- rawname: registryEnv - path: ocis-pkg/registry/registry.go:44 +- rawname: MICRO_LOG_LEVEL + path: ocis-pkg/log/log.go:30 foundincode: true - name: registryEnv + name: MICRO_LOG_LEVEL type: "" default_value: "" description: "" @@ -63,10 +55,18 @@ variables: default_value: "" description: "" do_ignore: false -- rawname: MICRO_LOG_LEVEL - path: ocis-pkg/log/log.go:30 +- rawname: registryAddressEnv + path: ocis-pkg/registry/registry.go:42 foundincode: true - name: MICRO_LOG_LEVEL + name: registryAddressEnv + type: "" + default_value: "" + description: "" + do_ignore: false +- rawname: registryEnv + path: ocis-pkg/registry/registry.go:44 + foundincode: true + name: registryEnv type: "" default_value: "" description: "" diff --git a/docs/helpers/rogueEnv.go b/docs/helpers/rogueEnv.go index f61958a7a49..ec177bd4cbc 100644 --- a/docs/helpers/rogueEnv.go +++ b/docs/helpers/rogueEnv.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "regexp" + "sort" "strings" "text/template" @@ -117,6 +118,12 @@ func GetRogueEnvs() { vars.Variables = append(vars.Variables, v) } + less := func(i, j int) bool { + return vars.Variables[i].Name < vars.Variables[j].Name + } + + sort.Slice(vars.Variables, less) + output, err := yaml.Marshal(vars) if err != nil { log.Fatal(err) diff --git a/docs/templates/ADOC_global.tmpl b/docs/templates/ADOC_global.tmpl index ae388a5d5f6..45f6af9629b 100644 --- a/docs/templates/ADOC_global.tmpl +++ b/docs/templates/ADOC_global.tmpl @@ -14,13 +14,15 @@ {{- if .Ignore }} {{ continue }} {{- end }} + a| `{{- .Name }}` + a| [subs=-attributes] ++{{ .Type }} ++ a| [subs=-attributes] ++{{.DefaultValue}} ++ a| [subs=-attributes] -{{.Description}} +++{{.Description}} ++ +a| [subs=-attributes] {{- end }} |=== From 62acfacf343ff663e1ae8cd41701db573720252d Mon Sep 17 00:00:00 2001 From: jkoberg Date: Mon, 12 Dec 2022 16:26:36 +0100 Subject: [PATCH 11/17] manually edit envvars as reference Signed-off-by: jkoberg --- docs/helpers/global_vars.yaml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/helpers/global_vars.yaml b/docs/helpers/global_vars.yaml index 2f0de5386e2..bbf01387a18 100644 --- a/docs/helpers/global_vars.yaml +++ b/docs/helpers/global_vars.yaml @@ -31,43 +31,43 @@ variables: default_value: "" description: "" do_ignore: false -- rawname: OCIS_BASE_DATA_PATH - path: ocis-pkg/config/defaults/paths.go:23 +- rawname: registryEnv + path: ocis-pkg/registry/registry.go:44 foundincode: true - name: OCIS_BASE_DATA_PATH + name: MICRO_REGISTRY type: "" default_value: "" description: "" do_ignore: false -- rawname: OCIS_CONFIG_DIR - path: ocis-pkg/config/defaults/paths.go:56 +- rawname: registryAddressEnv + path: ocis-pkg/registry/registry.go:42 foundincode: true - name: OCIS_CONFIG_DIR + name: MICRO_REGISTRY_ADDRESS type: "" default_value: "" description: "" do_ignore: false -- rawname: parts[0] - path: ocis-pkg/config/envdecode/envdecode.go:382 +- rawname: OCIS_BASE_DATA_PATH + path: ocis-pkg/config/defaults/paths.go:23 foundincode: true - name: parts[0] + name: OCIS_BASE_DATA_PATH type: "" default_value: "" description: "" do_ignore: false -- rawname: registryAddressEnv - path: ocis-pkg/registry/registry.go:42 +- rawname: OCIS_CONFIG_DIR + path: ocis-pkg/config/defaults/paths.go:56 foundincode: true - name: registryAddressEnv + name: OCIS_CONFIG_DIR type: "" default_value: "" description: "" do_ignore: false -- rawname: registryEnv - path: ocis-pkg/registry/registry.go:44 +- rawname: parts[0] + path: ocis-pkg/config/envdecode/envdecode.go:382 foundincode: true - name: registryEnv + name: parts[0] type: "" default_value: "" - description: "" - do_ignore: false + description: false positive - code that extract envvars for config structs + do_ignore: true From 20b2472f75e8a4a52f63c149b29b23f6e093114e Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 13 Dec 2022 11:02:51 +0100 Subject: [PATCH 12/17] fix docs-generate Signed-off-by: jkoberg --- Makefile | 3 ++- docs/Makefile | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b4038cfa476..d71f052dbbd 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,8 @@ OCIS_MODULES = \ services/web \ services/webdav\ ocis \ - ocis-pkg + ocis-pkg \ + docs # bin file definitions PHP_CS_FIXER=php -d zend.enable_gc=0 vendor-bin/owncloud-codestyle/vendor/bin/php-cs-fixer diff --git a/docs/Makefile b/docs/Makefile index 93a18b7be29..d5d3f7c01ae 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -9,7 +9,6 @@ help: .PHONY: docs-generate docs-generate: ## run docs-generate for all oCIS services @pushd helpers && go run .; popd - @$(MAKE) --no-print-directory -C ../ docs-generate .PHONY: docs-init docs-init: From 58913b0034c75c10f5fee068b939ff449977b293 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 13 Dec 2022 11:59:29 +0100 Subject: [PATCH 13/17] add README Signed-off-by: jkoberg --- docs/helpers/README.md | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/helpers/README.md diff --git a/docs/helpers/README.md b/docs/helpers/README.md new file mode 100644 index 00000000000..e4473f292f5 --- /dev/null +++ b/docs/helpers/README.md @@ -0,0 +1,52 @@ +# Docs Helpers + +`docs/helpers` contains a small go program creating docs by extracting information from the code. It has three main responsibilities +- Generate docs for envvars in config structs +- Extract envvars that are not mentioned in config structs (aka "Rogue" envvars) +- Generate docs for rogue envvars + +## Generate Envvar docs for config structs + +Generates docs from a template file, mainly extracting `"env"` and `"desc"` tags from the config structs. + +Templates can be found in `docs/helpers` folder. (Same as this `README`.) Check `.tmpl` files + +## Extract Rogue Envvars + +It `grep`s over the code, looking for `os.Getenv` and parses these contents to a yaml file along with the following information: +```golang +// Variable contains all information about one rogue envvar +type Variable struct { + // These field structs are automatically filled: + // RawName can be the name of the envvar or the name of its var + RawName string `yaml:"rawname"` + // Path to the envvar with linenumber + Path string `yaml:"path"` + // FoundInCode indicates if the variable is still found in the codebase. + FoundInCode bool `yaml:"foundincode"` + // Name is equal to RawName but will not be overwritten in consecutive runs + Name string `yaml:"name"` + + // These field structs need manual filling: + // Type of the envvar + Type string `yaml:"type"` + // DefaultValue of the envvar + DefaultValue string `yaml:"default_value"` + // Description of what this envvar does + Description string `yaml:"description"` + // Ignore this envvar when creating docs? + Ignore bool `yaml:"do_ignore"` + + // For simplicity ignored for now: + // DependendServices []Service `yaml:"dependend_services"` +} +``` +This yaml file can later be manually edited to add descriptions, default values, etc. + +IMPORTANT: `RawName`, `Path` and `FoundInCode` are automatically filled by the program. DO NOT EDIT THESE VALUES MANUALLY + +## Generate Rogue Envvar docs + +It picks up the `yaml` file generated in `Extract Rogue Envvars` step and renders it to a adoc file using a go template. + +Template for this can be found at `docs/templates/ADOC_global.tmpl` From 826a8b5de5c0ed03abd09074b1e568a8d48486e7 Mon Sep 17 00:00:00 2001 From: mmattel Date: Tue, 13 Dec 2022 12:17:41 +0100 Subject: [PATCH 14/17] adding description values and fix adoc template --- docs/helpers/global_vars.yaml | 48 ++++++++++++++++----------------- docs/templates/ADOC_global.tmpl | 1 - 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/helpers/global_vars.yaml b/docs/helpers/global_vars.yaml index bbf01387a18..25f11a9157f 100644 --- a/docs/helpers/global_vars.yaml +++ b/docs/helpers/global_vars.yaml @@ -2,66 +2,66 @@ variables: - rawname: CS3_GATEWAY path: services/idp/pkg/backends/cs3/bootstrap/cs3.go:76 foundincode: true - name: CS3_GATEWAY + name: "CS3_GATEWAY" type: "" default_value: "" description: "" - do_ignore: false + do_ignore: true - rawname: CS3_MACHINE_AUTH_API_KEY path: services/idp/pkg/backends/cs3/bootstrap/cs3.go:77 foundincode: true - name: CS3_MACHINE_AUTH_API_KEY + name: "CS3_MACHINE_AUTH_API_KEY" type: "" default_value: "" description: "" - do_ignore: false + do_ignore: true - rawname: MICRO_LOG_LEVEL path: ocis-pkg/log/log.go:34 foundincode: true - name: MICRO_LOG_LEVEL - type: "" - default_value: "" - description: "" + name: "MICRO_LOG_LEVEL" + type: "string" + default_value: "Error" + description: "Set the log level for the internal go micro framework. Only change on supervision of ownCloud Support." do_ignore: false - rawname: MICRO_LOG_LEVEL path: ocis-pkg/log/log.go:30 foundincode: true - name: MICRO_LOG_LEVEL + name: "MICRO_LOG_LEVEL" type: "" default_value: "" description: "" - do_ignore: false + do_ignore: true - rawname: registryEnv path: ocis-pkg/registry/registry.go:44 foundincode: true - name: MICRO_REGISTRY - type: "" + name: "MICRO_REGISTRY" + type: "string" default_value: "" - description: "" + description: "Go micro registry type to use. Supported types are: 'nats', 'kubernetes', 'etcd', 'consul' and 'memory'. Will be selected automatically. Only change on supervision of ownCloud Support." do_ignore: false - rawname: registryAddressEnv path: ocis-pkg/registry/registry.go:42 foundincode: true - name: MICRO_REGISTRY_ADDRESS - type: "" + name: "MICRO_REGISTRY_ADDRESS" + type: "string" default_value: "" - description: "" + description: "The bind address of the internal go micro framework. Only change on supervision of ownCloud Support." do_ignore: false - rawname: OCIS_BASE_DATA_PATH path: ocis-pkg/config/defaults/paths.go:23 foundincode: true - name: OCIS_BASE_DATA_PATH - type: "" - default_value: "" - description: "" + name: "OCIS_BASE_DATA_PATH" + type: "string" + default_value: "'/var/lib/ocis' or '$HOME/.ocis/'" + description: "The base directory location used by several services and for user data. Predefined to '/var/lib/ocis' for container images (inside the container) or '$HOME/.ocis/' for binary releases. Can be adapted for services individually." do_ignore: false - rawname: OCIS_CONFIG_DIR path: ocis-pkg/config/defaults/paths.go:56 foundincode: true - name: OCIS_CONFIG_DIR - type: "" - default_value: "" - description: "" + name: "OCIS_CONFIG_DIR" + type: "string" + default_value: "'/etc/ocis' or '$HOME/.ocis/config'" + description: "The default directory location for config files. Predefined to '/etc/ocis' for container images (inside the container) or '$HOME/.ocis/config' for binary releases." do_ignore: false - rawname: parts[0] path: ocis-pkg/config/envdecode/envdecode.go:382 diff --git a/docs/templates/ADOC_global.tmpl b/docs/templates/ADOC_global.tmpl index 45f6af9629b..6b0f5478f84 100644 --- a/docs/templates/ADOC_global.tmpl +++ b/docs/templates/ADOC_global.tmpl @@ -22,7 +22,6 @@ a| [subs=-attributes] ++{{.DefaultValue}} ++ a| [subs=-attributes] ++{{.Description}} ++ -a| [subs=-attributes] {{- end }} |=== From db22bb7011794761b447dfec1316cab366c32571 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 13 Dec 2022 12:27:09 +0100 Subject: [PATCH 15/17] fix makefile Signed-off-by: jkoberg --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d71f052dbbd..cd7f7392f32 100644 --- a/Makefile +++ b/Makefile @@ -46,8 +46,7 @@ OCIS_MODULES = \ services/web \ services/webdav\ ocis \ - ocis-pkg \ - docs + ocis-pkg # bin file definitions PHP_CS_FIXER=php -d zend.enable_gc=0 vendor-bin/owncloud-codestyle/vendor/bin/php-cs-fixer @@ -147,6 +146,7 @@ docs-generate: @for mod in $(OCIS_MODULES); do \ $(MAKE) --no-print-directory -C $$mod docs-generate || exit 1; \ done + $(MAKE) --no-print-directory -C docs docs-generate || exit 1 .PHONY: ci-go-generate ci-go-generate: From 82348b0aded44eeb95fb6e7b8904b2b99af1092e Mon Sep 17 00:00:00 2001 From: mmattel Date: Tue, 13 Dec 2022 13:29:08 +0100 Subject: [PATCH 16/17] add more info to readme.md --- docs/helpers/README.md | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/docs/helpers/README.md b/docs/helpers/README.md index e4473f292f5..39f99fa6f61 100644 --- a/docs/helpers/README.md +++ b/docs/helpers/README.md @@ -1,10 +1,35 @@ # Docs Helpers -`docs/helpers` contains a small go program creating docs by extracting information from the code. It has three main responsibilities +`docs/helpers` contains a small go program creating docs by extracting information from the code. It is manually started with `make docs-generate` or via the CI and has three main responsibilities: + - Generate docs for envvars in config structs - Extract envvars that are not mentioned in config structs (aka "Rogue" envvars) - Generate docs for rogue envvars +Output: + +- The generated yaml files can be found at: `docs/services/_includes` when running locally respectively in the docs branch after the CI has finished. +- The generated adoc files can be found at: `docs/services/_includes/adoc` when running locally respectively in the docs branch after the CI has finished. +- The file name for rouge envvars is named: `global_configvars.adoc`. + +Doc process: + +Whenever a build from the admin documentation is triggered, the files are included into the build process and added in a proper manner defined by the admin documentation. + +Genreal info: + +"Rouge" envvars are variables that need to be present *before* the core or services are starting up as they depend on the info provided like path for config files etc. Therefore they are _not_ bound to services like other envvars do. + +It can happen that rouge envvars are found but do not need to be published as they are for internal use only. Those rouge envvars can be defined to be ignored for further processing. + +IMPORTANT: + +- Once a rouge envvar has been identified, it is added to the `global_vars.yaml` file but never changed or touched by the process. There is one exception with respect to single/double quote usage. While you manually can (and will) define a text like: `"'/var/lib/ocis'"`, quotes are transformed by the process in the .yaml file to: `'''/var/lib/ocis'''`. There is no need to change this back, as the final step transforms this correctly to the adoc table. + +- Because rouge envvars do not have the same structural setup as "normal" envvars like type, description or defaults, these infos need to be provided manually one time - even if found multiple times. Any change on this info will be used on the next CI run and published on the next admin docs build. + +- Do not change the sort order of rouge envvar blocks as they are automatically reordered alphabetically. + ## Generate Envvar docs for config structs Generates docs from a template file, mainly extracting `"env"` and `"desc"` tags from the config structs. @@ -34,7 +59,7 @@ type Variable struct { DefaultValue string `yaml:"default_value"` // Description of what this envvar does Description string `yaml:"description"` - // Ignore this envvar when creating docs? + // Do not export this envvar into the generated adoc table Ignore bool `yaml:"do_ignore"` // For simplicity ignored for now: @@ -43,10 +68,10 @@ type Variable struct { ``` This yaml file can later be manually edited to add descriptions, default values, etc. -IMPORTANT: `RawName`, `Path` and `FoundInCode` are automatically filled by the program. DO NOT EDIT THESE VALUES MANUALLY +IMPORTANT: `RawName`, `Path` and `FoundInCode` are automatically filled by the program. DO NOT EDIT THESE VALUES MANUALLY. ## Generate Rogue Envvar docs -It picks up the `yaml` file generated in `Extract Rogue Envvars` step and renders it to a adoc file using a go template. +It picks up the `yaml` file generated in `Extract Rogue Envvars` step and renders it to a adoc file (table) using a go template. -Template for this can be found at `docs/templates/ADOC_global.tmpl` +The adoc template file for this step can be found at `docs/templates/ADOC_global.tmpl`. From 04d0e54feb72797283b8ad7afd19816e7a684952 Mon Sep 17 00:00:00 2001 From: mmattel Date: Tue, 13 Dec 2022 13:46:04 +0100 Subject: [PATCH 17/17] small readme fix --- docs/helpers/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/helpers/README.md b/docs/helpers/README.md index 39f99fa6f61..c69f16931e3 100644 --- a/docs/helpers/README.md +++ b/docs/helpers/README.md @@ -12,9 +12,9 @@ Output: - The generated adoc files can be found at: `docs/services/_includes/adoc` when running locally respectively in the docs branch after the CI has finished. - The file name for rouge envvars is named: `global_configvars.adoc`. -Doc process: +Admin doc process: -Whenever a build from the admin documentation is triggered, the files are included into the build process and added in a proper manner defined by the admin documentation. +Whenever a build from the admin documentation is triggered, the files generated here are included into the build process and added in a proper manner defined by the admin documentation. Genreal info: