diff --git a/learning/tour-of-beam/backend/internal/fs_content/load.go b/learning/tour-of-beam/backend/internal/fs_content/load.go index af532a178c1f..b0e0a8bfab41 100644 --- a/learning/tour-of-beam/backend/internal/fs_content/load.go +++ b/learning/tour-of-beam/backend/internal/fs_content/load.go @@ -16,13 +16,14 @@ package fs_content import ( + "bytes" "fmt" "io/fs" - "io/ioutil" "log" "os" "path/filepath" "regexp" + "text/template" tob "beam.apache.org/learning/tour-of-beam/backend/internal" ) @@ -38,11 +39,12 @@ const ( ) type learningPathInfo struct { - Sdk string `yaml:"sdk"` + Sdk []string `yaml:"sdk"` Content []string `yaml:"content"` } type learningModuleInfo struct { + Sdk []string `yaml:"sdk"` Id string `yaml:"id"` Name string `yaml:"name"` Complexity string `yaml:"complexity"` @@ -50,20 +52,32 @@ type learningModuleInfo struct { } type learningGroupInfo struct { + Sdk []string `yaml:"sdk"` Id string `yaml:"id"` Name string `yaml:"name"` Content []string `yaml:"content"` } type learningUnitInfo struct { - Id string `yaml:"id"` - Name string `yaml:"name"` - TaskName string `yaml:"taskName"` - SolutionName string `yaml:"solutionName"` + Sdk []string `yaml:"sdk"` + Id string `yaml:"id"` + Name string `yaml:"name"` + TaskName string `yaml:"taskName"` + SolutionName string `yaml:"solutionName"` } func collectUnit(infopath string, ctx *sdkContext) (unit *tob.Unit, err error) { info := loadLearningUnitInfo(infopath) + + supported, err := isSupportedSdk(info.Sdk, ctx, infopath) + if err != nil { + return nil, err + } + if !supported { + log.Printf("Unit %v at %v not supported in %v\n", info.Id, infopath, ctx.sdk) + return nil, nil + } + log.Printf("Found Unit %v metadata at %v\n", info.Id, infopath) ctx.idsWatcher.CheckId(info.Id) builder := NewUnitBuilder(info, ctx.sdk) @@ -77,7 +91,11 @@ func collectUnit(infopath string, ctx *sdkContext) (unit *tob.Unit, err error) { return filepath.SkipDir case d.Name() == descriptionMd: - content, err := ioutil.ReadFile(path) + templateSource, err := os.ReadFile(path) + if err != nil { + return err + } + content, err := processTemplate(templateSource, ctx.sdk) if err != nil { return err } @@ -85,7 +103,7 @@ func collectUnit(infopath string, ctx *sdkContext) (unit *tob.Unit, err error) { // Here we rely on that WalkDir entries are lexically sorted case regexp.MustCompile(hintMdRegexp).MatchString(d.Name()): - content, err := ioutil.ReadFile(path) + content, err := os.ReadFile(path) if err != nil { return err } @@ -97,8 +115,34 @@ func collectUnit(infopath string, ctx *sdkContext) (unit *tob.Unit, err error) { return builder.Build(), err } +func processTemplate(source []byte, sdk tob.Sdk) ([]byte, error) { + t := template.New("") + t, err := t.Parse(string(source)) + if err != nil { + return nil, err + } + + var output bytes.Buffer + err = t.Execute(&output, struct{ Sdk tob.Sdk }{Sdk: sdk}) + if err != nil { + return nil, err + } + + return output.Bytes(), nil +} + func collectGroup(infopath string, ctx *sdkContext) (*tob.Group, error) { info := loadLearningGroupInfo(infopath) + + supported, err := isSupportedSdk(info.Sdk, ctx, infopath) + if err != nil { + return nil, err + } + if !supported { + log.Printf("Group %v at %v not supported in %v\n", info.Id, infopath, ctx.sdk) + return nil, nil + } + log.Printf("Found Group %v metadata at %v\n", info.Name, infopath) group := tob.Group{Id: info.Id, Title: info.Name} for _, item := range info.Content { @@ -106,18 +150,22 @@ func collectGroup(infopath string, ctx *sdkContext) (*tob.Group, error) { if err != nil { return &group, err } - group.Nodes = append(group.Nodes, node) + if node == nil { + continue + } + group.Nodes = append(group.Nodes, *node) } return &group, nil } // Collect node which is either a unit or a group. -func collectNode(rootpath string, ctx *sdkContext) (node tob.Node, err error) { +func collectNode(rootpath string, ctx *sdkContext) (*tob.Node, error) { files, err := os.ReadDir(rootpath) if err != nil { - return node, err + return nil, err } + node := &tob.Node{} for _, f := range files { switch f.Name() { case unitInfoYaml: @@ -127,46 +175,73 @@ func collectNode(rootpath string, ctx *sdkContext) (node tob.Node, err error) { node.Type = tob.NODE_GROUP node.Group, err = collectGroup(filepath.Join(rootpath, groupInfoYaml), ctx) } + if err != nil { + return nil, err + } } if node.Type == tob.NODE_UNDEFINED { return node, fmt.Errorf("node undefined at %v", rootpath) } + if node.Group == nil && node.Unit == nil { + return nil, err + } return node, err } -func collectModule(infopath string, ctx *sdkContext) (tob.Module, error) { +func collectModule(infopath string, ctx *sdkContext) (*tob.Module, error) { info := loadLearningModuleInfo(infopath) + + supported, err := isSupportedSdk(info.Sdk, ctx, infopath) + if err != nil { + return nil, err + } + if !supported { + log.Printf("Module %v at %v not supported in %v\n", info.Id, infopath, ctx.sdk) + return nil, nil + } + log.Printf("Found Module %v metadata at %v\n", info.Id, infopath) ctx.idsWatcher.CheckId(info.Id) module := tob.Module{Id: info.Id, Title: info.Name, Complexity: info.Complexity} for _, item := range info.Content { node, err := collectNode(filepath.Join(infopath, "..", item), ctx) if err != nil { - return tob.Module{}, err + return nil, err } - module.Nodes = append(module.Nodes, node) + if node == nil { + continue + } + module.Nodes = append(module.Nodes, *node) } - return module, nil + return &module, nil } -func collectSdk(infopath string) (tree tob.ContentTree, err error) { +func collectSdk(infopath string) (trees []tob.ContentTree, err error) { info := loadLearningPathInfo(infopath) - tree.Sdk = tob.ParseSdk(info.Sdk) - if tree.Sdk == tob.SDK_UNDEFINED { - return tree, fmt.Errorf("unknown SDK at %v", infopath) + + sdks, err := getSupportedSdk(info.Sdk, infopath) + if err != nil { + return trees, err } - log.Printf("Found Sdk %v metadata at %v\n", info.Sdk, infopath) - ctx := newSdkContext(tree.Sdk) - for _, item := range info.Content { - mod, err := collectModule(filepath.Join(infopath, "..", item, moduleInfoYaml), ctx) - if err != nil { - return tree, err + for sdk := range sdks { + tree := tob.ContentTree{} + tree.Sdk = sdk + log.Printf("Found Sdk %v metadata at %v\n", sdk, infopath) + ctx := newSdkContext(tree.Sdk) + for _, item := range info.Content { + mod, err := collectModule(filepath.Join(infopath, "..", item, moduleInfoYaml), ctx) + if err != nil { + return trees, err + } + if mod != nil { + tree.Modules = append(tree.Modules, *mod) + } } - tree.Modules = append(tree.Modules, mod) + trees = append(trees, tree) } - return tree, nil + return trees, nil } // Build a content tree for each SDK @@ -179,11 +254,12 @@ func CollectLearningTree(rootpath string) (trees []tob.ContentTree, err error) { return err } if d.Name() == contentInfoYaml { - tree, err := collectSdk(path) + + collected, err := collectSdk(path) if err != nil { return err } - trees = append(trees, tree) + trees = append(trees, collected...) // don't walk into SDK subtree (already done by collectSdk) return filepath.SkipDir } @@ -192,3 +268,25 @@ func CollectLearningTree(rootpath string) (trees []tob.ContentTree, err error) { return trees, err } + +func isSupportedSdk(sdks []string, ctx *sdkContext, infopath string) (ok bool, err error) { + sdk, err := getSupportedSdk(sdks, infopath) + if err != nil { + return false, err + } + _, ok = sdk[ctx.sdk] + return +} + +func getSupportedSdk(sdk []string, infopath string) (map[tob.Sdk]bool, error) { + sdks := make(map[tob.Sdk]bool) + for _, s := range sdk { + curSdk := tob.ParseSdk(s) + if curSdk == tob.SDK_UNDEFINED { + return sdks, fmt.Errorf("unknown SDK at %v", infopath) + } + sdks[curSdk] = true + } + + return sdks, nil +} diff --git a/learning/tour-of-beam/backend/internal/fs_content/load_test.go b/learning/tour-of-beam/backend/internal/fs_content/load_test.go index 8e50f6360b7a..1e3948a38fcd 100644 --- a/learning/tour-of-beam/backend/internal/fs_content/load_test.go +++ b/learning/tour-of-beam/backend/internal/fs_content/load_test.go @@ -90,3 +90,70 @@ func TestSample(t *testing.T) { }, }, trees[1]) } + +// TestTemplates test that templating engine is used correctly. +// The test itself is intended as an example of typical template usage. +func TestTemplateProcessing(t *testing.T) { + goSdkExpected := "Go SDK" + pythonSdkExpected := "Python SDK" + javaSdkExpected := "Java SDK" + scioSdkExpected := "SCIO SDK" + template := fmt.Sprintf( + "Using "+ + "{{if (eq .Sdk \"go\")}}%s{{end}}"+ + "{{if (eq .Sdk \"python\")}}%s{{end}}"+ + "{{if (eq .Sdk \"java\")}}%s{{end}}"+ + "{{if (eq .Sdk \"scio\")}}%s{{end}}", + goSdkExpected, pythonSdkExpected, javaSdkExpected, scioSdkExpected) + + goOrJavaExpected := "Text for Go or Java SDK" + templateAboutGoOrJava := fmt.Sprintf("{{if (eq .Sdk \"go\" \"java\")}}%s{{end}}", goOrJavaExpected) + + for _, s := range []struct { + sdk tob.Sdk + template string + expected string + }{ + { + sdk: tob.SDK_GO, + template: template, + expected: fmt.Sprintf("Using %s", goSdkExpected), + }, + { + sdk: tob.SDK_PYTHON, + template: template, + expected: fmt.Sprintf("Using %s", pythonSdkExpected), + }, + { + sdk: tob.SDK_JAVA, + template: template, + expected: fmt.Sprintf("Using %s", javaSdkExpected), + }, + { + sdk: tob.SDK_SCIO, + template: template, + expected: fmt.Sprintf("Using %s", scioSdkExpected), + }, + { + sdk: tob.SDK_GO, + template: templateAboutGoOrJava, + expected: goOrJavaExpected, + }, + { + sdk: tob.SDK_JAVA, + template: templateAboutGoOrJava, + expected: goOrJavaExpected, + }, + { + sdk: tob.SDK_SCIO, + template: templateAboutGoOrJava, + expected: "", + }, + } { + res, err := processTemplate([]byte(s.template), s.sdk) + if err != nil { + panic(err) + } + assert.Equal(t, s.expected, string(res)) + } +} diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/content-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/content-info.yaml similarity index 57% rename from learning/tour-of-beam/backend/samples/learning-content/java/content-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/content-info.yaml index 0d6a17105315..f26734290c8f 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/java/content-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/content-info.yaml @@ -1,4 +1,6 @@ -sdk: Java +sdk: + - Java + - Python content: - module 1 - module 2 \ No newline at end of file diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/group-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/group-info.yaml similarity index 84% rename from learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/group-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 1/group/group-info.yaml index 95c810a7b628..48d4297eddcb 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/group-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/group-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Python id: group1 name: The Group complexity: BASIC diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/description.md b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/description.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/description.md rename to learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/description.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/hint1.md b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/hint1.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/hint1.md rename to learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/hint1.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/hint2.md b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/hint2.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/hint2.md rename to learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/hint2.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/unit-info.yaml similarity index 87% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/unit-info.yaml index e20b873a3e05..32ab267ce716 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-challenge/unit-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Python id: challenge1 name: Challenge Name complexity: BASIC diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-example/unit-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-example/unit-info.yaml similarity index 78% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-example/unit-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-example/unit-info.yaml index ed0c81854cb0..4b544ebb6094 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/unit-example/unit-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 1/group/unit-example/unit-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Python id: example1 name: Example Unit Name taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/intro-unit/unit-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 1/intro-unit/unit-info.yaml similarity index 79% rename from learning/tour-of-beam/backend/samples/learning-content/python/module 1/intro-unit/unit-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 1/intro-unit/unit-info.yaml index ee96ce47e255..6f748fdc2ce1 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/intro-unit/unit-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 1/intro-unit/unit-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Python id: intro-unit name: Example Unit Name taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/module-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 1/module-info.yaml similarity index 65% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 1/module-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 1/module-info.yaml index 04384cb56b31..a5d902bb2a51 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/java/module 1/module-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 1/module-info.yaml @@ -1,7 +1,13 @@ +sdk: + - Java + - Python id: module1 name: Module One complexity: BASIC content: - unit-example - unit-challenge +- intro-unit +- group + diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/description.md b/learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/description.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/description.md rename to learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/description.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/hint1.md b/learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/hint1.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/hint1.md rename to learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/hint1.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/hint2.md b/learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/hint2.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/hint2.md rename to learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/hint2.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/unit-info.yaml similarity index 88% rename from learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/unit-info.yaml index e20b873a3e05..47a7be9c355f 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 1/unit-challenge/unit-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Java id: challenge1 name: Challenge Name complexity: BASIC diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 1/unit-example/unit-info.yaml similarity index 80% rename from learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 1/unit-example/unit-info.yaml index ed0c81854cb0..fec45e41007a 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 1/unit-example/unit-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Java id: example1 name: Example Unit Name taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/module-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 2/module-info.yaml similarity index 86% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 2/module-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 2/module-info.yaml index e152c45d22e1..9452b44d8168 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/module-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 2/module-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Java id: module2 name: Module Two complexity: MEDIUM diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/description.md b/learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/description.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/description.md rename to learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/description.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/hint1.md b/learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/hint1.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/hint1.md rename to learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/hint1.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/hint2.md b/learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/hint2.md similarity index 100% rename from learning/tour-of-beam/backend/samples/learning-content/python/module 1/group/unit-challenge/hint2.md rename to learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/hint2.md diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/unit-info.yaml similarity index 88% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/unit-info.yaml index ac65e8357d4c..4f5d7d04f6d2 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 2/unit-challenge/unit-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Java id: challenge21 name: Challenge Name complexity: BASIC diff --git a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-example/unit-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/module 2/unit-example/unit-info.yaml similarity index 80% rename from learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-example/unit-info.yaml rename to learning/tour-of-beam/backend/samples/learning-content/module 2/unit-example/unit-info.yaml index 674a6da71941..dfd62f9033fb 100644 --- a/learning/tour-of-beam/backend/samples/learning-content/java/module 2/unit-example/unit-info.yaml +++ b/learning/tour-of-beam/backend/samples/learning-content/module 2/unit-example/unit-info.yaml @@ -1,3 +1,5 @@ +sdk: + - Java id: example21 name: Example Unit Name taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/content-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/python/content-info.yaml deleted file mode 100644 index ee9d329b5030..000000000000 --- a/learning/tour-of-beam/backend/samples/learning-content/python/content-info.yaml +++ /dev/null @@ -1,3 +0,0 @@ -sdk: Python -content: - - module 1 diff --git a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/module-info.yaml b/learning/tour-of-beam/backend/samples/learning-content/python/module 1/module-info.yaml deleted file mode 100644 index 024d12fdc69c..000000000000 --- a/learning/tour-of-beam/backend/samples/learning-content/python/module 1/module-info.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: module1 -name: Module One -complexity: BASIC -content: -- intro-unit -- group - diff --git a/learning/tour-of-beam/learning-content/java/content-info.yaml b/learning/tour-of-beam/learning-content/content-info.yaml similarity index 93% rename from learning/tour-of-beam/learning-content/java/content-info.yaml rename to learning/tour-of-beam/learning-content/content-info.yaml index 7406bb513e32..1260e5ba1669 100644 --- a/learning/tour-of-beam/learning-content/java/content-info.yaml +++ b/learning/tour-of-beam/learning-content/content-info.yaml @@ -17,6 +17,10 @@ # under the License. # -sdk: Java +sdk: + - Java + - Python + - Go content: - - introduction \ No newline at end of file + - introduction + \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/content-info.yaml b/learning/tour-of-beam/learning-content/go/content-info.yaml deleted file mode 100644 index 8b75fa8771d2..000000000000 --- a/learning/tour-of-beam/learning-content/go/content-info.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -sdk: Go -content: - - introduction \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/description.md b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/description.md deleted file mode 100644 index 0a990668241e..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/description.md +++ /dev/null @@ -1,56 +0,0 @@ - -### Creating PCollection - -Now that you know how to create a Beam pipeline and pass parameters into it, it is time to learn how to create an initial `PCollection` and fill it with data. - -There are several options: - -→ You can create a PCollection of data stored in an in-memory collection class in your driver program. - -→ You can also read the data from a variety of external sources such as local or cloud-based files, databases, or other sources using Beam-provided I/O adapters - -Through the tour, most of the examples use either a `PCollection` created from in-memory data or data read from one of the cloud buckets "beam-examples" or "dataflow-samples". These buckets contain sample data sets specifically created for educational purposes. - -We encourage you to take a look, explore these data sets and use them while learning Apache Beam. - -### Creating a PCollection from in-memory data - -You can use the Beam-provided Create transform to create a `PCollection` from an in-memory Go Collection. You can apply Create transform directly to your Pipeline object itself. - -The following example code shows how to do this: - -``` -func main() { - ctx := context.Background() - - // First create pipeline - p, s := beam.NewPipelineWithRoot() - - //Now create the PCollection using list of strings - strings := beam.Create(s, "To", "be", "or", "not", "to", "be","that", "is", "the", "question") - - //Create a numerical PCollection - numbers := beam.Create(s, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - -} -``` - -### Playground exercise - -You can find the complete code of this example in the playground window you can run and experiment with. - -One of the differences you will notice is that it also contains the part to output `PCollection` elements to the console. Don’t worry if you don’t quite understand it, as the concept of `ParDo` transform will be explained later in the course. Feel free, however, to use it in exercises and challenges to explore results. - -Do you also notice in what order elements of PCollection appear in the console? Why is that? You can also run the example several times to see if the output stays the same or changes. \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/unit-info.yaml b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/unit-info.yaml deleted file mode 100644 index d26548d6cda3..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/unit-info.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -id: from-memory -name: Creating in-memory PCollections -taskName: ParDo \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/group-info.yaml b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/group-info.yaml deleted file mode 100644 index 418ed516f2b0..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/group-info.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -id: creating-collections -name: Creating Collections -content: -- from-memory -- reading-from-text -- reading-from-csv \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-csv/unit-info.yaml b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-csv/unit-info.yaml deleted file mode 100644 index 3d12a4dc6532..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-csv/unit-info.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -id: from-csv -name: Creating PCollections from csv files -taskName: CSV \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-text/description.md b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-text/description.md deleted file mode 100644 index 4968aa7d9528..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-text/description.md +++ /dev/null @@ -1,41 +0,0 @@ - -### Reading from text file - -You use one of the Beam-provided I/O adapters to read from an external source. The adapters vary in their exact usage, but all of them read from some external data source and return a `PCollection` whose elements represent the data records in that source. - -Each data source adapter has a Read transform; to read, you must apply that transform to the Pipeline object itself. - -`TextIO.Read` , for example, reads from an external text file and returns a `PCollection` whose elements are of type String. Each String represents one line from the text file. Here’s how you would apply `TextIO.Read` to your Pipeline to create a `PCollection`: - -``` -func main() { - ctx := context.Background() - - // First create pipline - p, s := beam.NewPipelineWithRoot() - - // Now create the PCollection by reading text files. Separate elements will be added for each line in the input file - lines := textio.Read(scope, 'gs://some/inputData.txt') - -} -``` - -### Playground exercise - -In the playground window, you can find an example that reads a king lear poem from the text file stored in the Google Storage bucket and fills PCollection with individual lines and then with individual words. Try it out and see what the output is. - -One of the differences you will see is that the output is much shorter than the input file itself. This is because the number of elements in the output `PCollection` is limited with the `top.Largest(s,lines,10,less)` transform. Use Sample.fixedSizeGlobally transform of is another technique you can use to troubleshoot and limit the output sent to the console for debugging purposes in case of large input datasets. - -Overview [file](https://storage.googleapis.com/apache-beam-samples/shakespeare/kinglear.txt) \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/group-info.yaml b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/group-info.yaml deleted file mode 100644 index 19ef8ae2fd5b..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/group-info.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -id: concepts -name: Beam Concepts -content: -- runner-concepts -- pipeline-concepts -- creating-collections \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/description.md b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/description.md deleted file mode 100644 index 3aa6c64978cd..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/description.md +++ /dev/null @@ -1,36 +0,0 @@ - -### Creating a pipeline - -The `Pipeline` abstraction encapsulates all the data and steps in your data processing task. Your Beam driver program typically starts by constructing a Pipeline object, and then using that object as the basis for creating the pipeline’s data sets as PCollections and its operations as `Transforms`. - -To use Beam, your driver program must first create an instance of the Beam SDK class Pipeline (typically in the main() function). When you create your `Pipeline`, you’ll also need to set some configuration options. You can set your pipeline’s configuration options programmatically, but it’s often easier to set the options ahead of time (or read them from the command line) and pass them to the Pipeline object when you create the object. - -``` -// beam.Init() is an initialization hook that must be called -// near the beginning of main(), before creating a pipeline. -beam.Init() - -// Create the Pipeline object and root scope. -pipeline, scope := beam.NewPipelineWithRoot() -``` - -### Playground exercise - -You can find the full code of the above example in the playground window, which you can run and experiment with. And you can create a `pipeline`, `scope` separately, it is an alternative to `beam.NewPipelineWithRoot()`. It is convenient if manipulations are needed before creating an element. - -``` -pipeline := beam.NewPipeline() -scope := p.Root() -``` \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/unit-info.yaml b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/unit-info.yaml deleted file mode 100644 index f3c47a5fe1ed..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/unit-info.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -id: creating-pipeline -name: Creating pipelines -taskName: creating-pipeline \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/group-info.yaml b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/group-info.yaml deleted file mode 100644 index dd46ad6e61ad..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/group-info.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -id: pipeline-concepts -name: Pipeline concepts -content: -- overview-pipeline -- creating-pipeline -- setting-pipeline \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/description.md b/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/description.md deleted file mode 100644 index 23781b1e5ea2..000000000000 --- a/learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/description.md +++ /dev/null @@ -1,71 +0,0 @@ - -### Configuring pipeline options - -Use the pipeline options to configure different aspects of your pipeline, such as the pipeline runner that will execute your pipeline and any runner-specific configuration required by the chosen runner. Your pipeline options will potentially include information such as your project ID or a location for storing files. - -### Setting PipelineOptions from command-line arguments - -Use Go flags to parse command line arguments to configure your pipeline. Flags must be parsed before `beam.Init()` is called. - -``` -// If beamx or Go flags are used, flags must be parsed first, -// before beam.Init() is called. -flag.Parse() -``` - -This interprets command-line arguments this follow the format: - -``` ---