Merge branch 'python_testing'

asciidoc/parse/current.go

@@ -30,10 +30,6 @@ func (c *current) previousRuneIsWhitespace() bool {
 	return unicode.IsSpace(r)
 }
 
-func (c *current) currentColumn() int {
-	return c.pos.col
-}
-
 func (c *current) currentPosition() (line, col, offset int) {
 	line = c.parser.offset.line + c.pos.line
 	col = c.pos.col

asciidoc/parse/current_column.go

@@ -0,0 +1,7 @@
+//go:build !windows
+
+package parse
+
+func (c *current) currentColumn() int {
+	return c.pos.col
+}

asciidoc/parse/current_column_windows.go

@@ -0,0 +1,11 @@
+//go:build windows
+
+package parse
+
+func (c *current) currentColumn() int {
+	if len(c.text) == 2 && c.text[0] == '\r' && c.text[1] == '\n' {
+		// Special case on Windows; NewLine matches \r\n, but the parser only sees \n as a col reset, so we fake it
+		return 0
+	}
+	return c.pos.col
+}

asciidoc/parse/grammar/comment.peg

@@ -8,7 +8,7 @@ SingleLineCommentDelimiter = "//" {
     return string(c.text), nil
 }
 
-SingleLineCommentContent = [^\n]* {
+SingleLineCommentContent = [^\r\n]* {
     debugPosition(c, "matched comment content\n")
     return string(c.text), nil
 }

asciidoc/parse/grammar/path.peg

@@ -4,7 +4,7 @@ Path = !"[" elements:( PathElement+) {
     return mergeStrings(elements.([]any)), nil
 } 
 
-PathElement = (([^\r\n{.<>!?,;[\] ])+ 
+PathElement = (([^\n{.<>!?,;[\] ])+ 
  / ([.?!;,] &(!_ !EndOfFile))
  / AttributeReference) 
 

asciidoc/parse/grammar/text.peg

@@ -38,7 +38,7 @@ EmptyLine = BeginningOfLine AnySpaces NewLine {
 
 SoftNewLine = !"\\" NewLine
 
-NewLine = ("\n" / "\r" / "\r\n"  ) {
+NewLine = ("\n" / "\r\n") {
     debugPosition(c, "newline\n")
     return compose(c, &asciidoc.NewLine{})
 }
@@ -89,7 +89,7 @@ UnbrokenText = CharacterClassInline+ {
     return string(c.text), nil
 }
 
-AnyText = [^\n]* 
+AnyText = [^\r\n]* 
 
 AnySpaces = [ \t]* 
 
@@ -98,7 +98,7 @@ CharacterClassAll = (. / NewLine) {
     return string(c.text), nil
 }
 
-CharacterClassInline = [^\n] 
+CharacterClassInline = [^\r\n] 
 
 CharacterGroupWord = [\p{M}\p{Nd}\p{Pc}\p{Join_Control}\p{L}] 
 

cmd/cli.go

@@ -10,6 +10,7 @@ import (
 	"github.com/project-chip/alchemy/cmd/format"
 	"github.com/project-chip/alchemy/cmd/testplan"
 	"github.com/project-chip/alchemy/cmd/validate"
+	"github.com/project-chip/alchemy/cmd/yaml2python"
 	"github.com/project-chip/alchemy/cmd/zap"
 )
 
@@ -28,4 +29,5 @@ func init() {
 	rootCmd.AddCommand(dm.Command)
 	rootCmd.AddCommand(testplan.Command)
 	rootCmd.AddCommand(validate.Command)
+	rootCmd.AddCommand(yaml2python.Command)
 }

cmd/testplan/cmd.go

@@ -26,6 +26,7 @@ var Command = &cobra.Command{
 func init() {
 	Command.Flags().String("specRoot", "connectedhomeip-spec", "the src root of your clone of CHIP-Specifications/connectedhomeip-spec")
 	Command.Flags().String("testRoot", "chip-test-plans", "the root of your clone of CHIP-Specifications/chip-test-plans")
+	Command.Flags().String("templateRoot", "", "the root of your local template files; if not specified, Alchemy will use an internal copy")
 	Command.Flags().Bool("overwrite", false, "overwrite existing test plans")
 }
 
@@ -36,10 +37,16 @@ func tp(cmd *cobra.Command, args []string) (err error) {
 	specRoot, _ := cmd.Flags().GetString("specRoot")
 	testRoot, _ := cmd.Flags().GetString("testRoot")
 	overwrite, _ := cmd.Flags().GetBool("overwrite")
+	templateRoot, _ := cmd.Flags().GetString("templateRoot")
 
 	asciiSettings := common.ASCIIDocAttributes(cmd)
 	fileOptions := files.Flags(cmd)
 	pipelineOptions := pipeline.Flags(cmd)
+	var testplanGeneratorOptions []testplan.GeneratorOption
+
+	if templateRoot != "" {
+		testplanGeneratorOptions = append(testplanGeneratorOptions, testplan.TemplateRoot(templateRoot))
+	}
 
 	errata.LoadErrataConfig(specRoot)
 
@@ -91,7 +98,7 @@ func tp(cmd *cobra.Command, args []string) (err error) {
 		}
 	}
 
-	generator := testplan.NewGenerator(testRoot, overwrite)
+	generator := testplan.NewGenerator(testRoot, overwrite, testplanGeneratorOptions...)
 	var testplans pipeline.Map[string, *pipeline.Data[string]]
 	testplans, err = pipeline.Process[*spec.Doc, string](cxt, pipelineOptions, generator, specDocs)
 	if err != nil {

cmd/yaml2python/cmd.go

@@ -0,0 +1,122 @@
+package yaml2python
+
+import (
+	"context"
+	"path/filepath"
+
+	"github.com/project-chip/alchemy/cmd/common"
+	"github.com/project-chip/alchemy/errata"
+	"github.com/project-chip/alchemy/internal/files"
+	"github.com/project-chip/alchemy/internal/pipeline"
+	"github.com/project-chip/alchemy/matter/spec"
+	"github.com/project-chip/alchemy/testing/generate"
+	"github.com/project-chip/alchemy/testing/parse"
+	"github.com/spf13/cobra"
+)
+
+var Command = &cobra.Command{
+	Use:   "yaml2python",
+	Short: "create a shell python script from a test YAML",
+	RunE:  tp,
+}
+
+func init() {
+	Command.Flags().String("specRoot", "connectedhomeip-spec", "the src root of your clone of CHIP-Specifications/connectedhomeip-spec")
+	Command.Flags().String("sdkRoot", "connectedhomeip", "the root of your clone of project-chip/connectedhomeip")
+	Command.Flags().String("templateRoot", "", "the root of your local template files; if not specified, Alchemy will use an internal copy")
+	Command.Flags().Bool("overwrite", true, "overwrite existing test scripts")
+}
+
+func tp(cmd *cobra.Command, args []string) (err error) {
+
+	cxt := context.Background()
+
+	specRoot, _ := cmd.Flags().GetString("specRoot")
+	sdkRoot, _ := cmd.Flags().GetString("sdkRoot")
+
+	asciiSettings := common.ASCIIDocAttributes(cmd)
+	fileOptions := files.Flags(cmd)
+	pipelineOptions := pipeline.Flags(cmd)
+
+	overwrite, _ := cmd.Flags().GetBool("overwrite")
+	templateRoot, _ := cmd.Flags().GetString("templateRoot")
+	generatorOptions := []generate.GeneratorOption{
+		generate.Overwrite(overwrite),
+		generate.TemplateRoot(templateRoot),
+	}
+
+	var inputs pipeline.Map[string, *pipeline.Data[struct{}]]
+	inputs, err = pipeline.Start[struct{}](cxt, files.PathsTargeter(args...))
+	if err != nil {
+		return err
+	}
+
+	inputs, err = pipeline.ProcessCollectiveFunc(cxt, inputs, "Filtering YAML tests", func(cxt context.Context, inputs []*pipeline.Data[struct{}]) (outputs []*pipeline.Data[struct{}], err error) {
+		for _, input := range inputs {
+			switch filepath.Base(input.Path) {
+			case "PICS.yaml":
+			default:
+				outputs = append(outputs, input)
+			}
+		}
+		return
+	})
+	if err != nil {
+		return err
+	}
+
+	errata.LoadErrataConfig(specRoot)
+
+	var parser parse.TestYamlParser
+	parser, err = parse.NewTestYamlParser(sdkRoot)
+	if err != nil {
+		return err
+	}
+
+	var tests pipeline.Map[string, *pipeline.Data[*parse.Test]]
+	tests, err = pipeline.Process[struct{}, *parse.Test](cxt, pipelineOptions, parser, inputs)
+	if err != nil {
+		return err
+	}
+
+	docParser, err := spec.NewParser(specRoot, asciiSettings)
+	if err != nil {
+		return err
+	}
+
+	specFiles, err := pipeline.Start[struct{}](cxt, spec.Targeter(specRoot))
+	if err != nil {
+		return err
+	}
+
+	specDocs, err := pipeline.Process[struct{}, *spec.Doc](cxt, pipelineOptions, docParser, specFiles)
+	if err != nil {
+		return err
+	}
+
+	specBuilder := spec.NewBuilder()
+	_, err = pipeline.Process[*spec.Doc, *spec.Doc](cxt, pipelineOptions, &specBuilder, specDocs)
+	if err != nil {
+		return err
+	}
+
+	picsLabels, err := parse.LoadPICSLabels(sdkRoot)
+	if err != nil {
+		return err
+	}
+
+	generator := generate.NewPythonTestGenerator(specBuilder.Spec, sdkRoot, picsLabels, generatorOptions...)
+	var scripts pipeline.Map[string, *pipeline.Data[string]]
+	scripts, err = pipeline.Process[*parse.Test, string](cxt, pipelineOptions, generator, tests)
+	if err != nil {
+		return err
+	}
+
+	writer := files.NewWriter[string]("Writing test scripts", fileOptions)
+	_, err = pipeline.Process[string, struct{}](cxt, pipelineOptions, writer, scripts)
+	if err != nil {
+		return err
+	}
+
+	return
+}

cmd/zap/cmd.go

@@ -128,7 +128,7 @@ func zapTemplates(cmd *cobra.Command, args []string) (err error) {
 
 	var patchedDeviceTypes pipeline.Map[string, *pipeline.Data[[]byte]]
 	if deviceTypes.Size() > 0 {
-		deviceTypePatcher := generate.NewDeviceTypesPatcher(sdkRoot, specBuilder.Spec, clusterAliases)
+		deviceTypePatcher := generate.NewDeviceTypesPatcher(sdkRoot, specBuilder.Spec, clusterAliases, generate.DeviceTypePatcherGenerateFeatureXML(featureXML))
 		patchedDeviceTypes, err = pipeline.Process[[]*matter.DeviceType, []byte](cxt, pipelineOptions, deviceTypePatcher, deviceTypes)
 		if err != nil {
 			return err
@@ -256,8 +256,7 @@ func patchSpec(spec *spec.Specification) {
 	}
 	for _, s := range label.Structs {
 		if s.Name == "LabelStruct" {
-			s.ParentEntity = fixedLabel
-			fixedLabel.Structs = append(fixedLabel.Structs, s)
+			fixedLabel.MoveStruct(s)
 			break
 		}
 	}

dm/conformance.go

@@ -53,25 +53,7 @@ func renderConformance(doc *spec.Doc, identifierStore conformance.IdentifierStor
 		parent.CreateElement("provisionalConform")
 	case *conformance.Optional:
 		oc := parent.CreateElement("optionalConform")
-		if con.Choice != nil {
-			oc.CreateAttr("choice", con.Choice.Set)
-			if con.Choice.Limit != nil {
-				switch l := con.Choice.Limit.(type) {
-				case *conformance.ChoiceExactLimit:
-					oc.CreateAttr("min", strconv.Itoa(l.Limit))
-					oc.CreateAttr("max", strconv.Itoa(l.Limit))
-				case *conformance.ChoiceMinLimit:
-					oc.CreateAttr("more", "true") // Existing data model does this for some reason
-					oc.CreateAttr("min", strconv.Itoa(l.Min))
-				case *conformance.ChoiceMaxLimit:
-					oc.CreateAttr("max", strconv.Itoa(l.Max))
-				case *conformance.ChoiceRangeLimit:
-					oc.CreateAttr("more", "true") // Existing data model does this for some reason
-					oc.CreateAttr("min", strconv.Itoa(l.Min))
-					oc.CreateAttr("max", strconv.Itoa(l.Max))
-				}
-			}
-		}
+		renderChoice(con.Choice, oc)
 		return renderConformanceExpression(doc, identifierStore, con.Expression, oc)
 	case *conformance.Disallowed:
 		parent.CreateElement("disallowConform")
@@ -92,6 +74,30 @@ func renderConformance(doc *spec.Doc, identifierStore conformance.IdentifierStor
 	return nil
 }
 
+func renderChoice(choice *conformance.Choice, parent *etree.Element) {
+	if choice == nil {
+		return
+	}
+	parent.CreateAttr("choice", choice.Set)
+	if choice.Limit == nil {
+		return
+	}
+	switch l := choice.Limit.(type) {
+	case *conformance.ChoiceExactLimit:
+		parent.CreateAttr("min", strconv.Itoa(l.Limit))
+		parent.CreateAttr("max", strconv.Itoa(l.Limit))
+	case *conformance.ChoiceMinLimit:
+		parent.CreateAttr("more", "true") // Existing data model does this for some reason
+		parent.CreateAttr("min", strconv.Itoa(l.Min))
+	case *conformance.ChoiceMaxLimit:
+		parent.CreateAttr("max", strconv.Itoa(l.Max))
+	case *conformance.ChoiceRangeLimit:
+		parent.CreateAttr("more", "true") // Existing data model does this for some reason
+		parent.CreateAttr("min", strconv.Itoa(l.Min))
+		parent.CreateAttr("max", strconv.Itoa(l.Max))
+	}
+}
+
 func renderConformanceExpression(doc *spec.Doc, identifierStore conformance.IdentifierStore, exp conformance.Expression, parent *etree.Element) error {
 	if exp == nil {
 		return nil

errata/config.go

@@ -1,6 +1,7 @@
 package errata
 
 import (
+	_ "embed"
 	"log/slog"
 	"os"
 	"path/filepath"
@@ -9,6 +10,9 @@ import (
 	"github.com/project-chip/alchemy/internal/files"
 )
 
+//go:embed default.yaml
+var defaultErrata []byte
+
 func LoadErrataConfig(specRoot string) {
 	if specRoot == "" {
 		return
@@ -19,31 +23,33 @@ func LoadErrataConfig(specRoot string) {
 		slog.Warn("error checking for errata path", slog.Any("error", err))
 		return
 	}
+	var b []byte
 	if !exists {
-		slog.Warn("errata file does not exist", slog.Any("path", errataPath))
-		for p := range Erratas {
-			path := filepath.Join(specRoot, p)
-			exists, _ := files.Exists(path)
-			if !exists {
-				slog.Warn("errata points to non-existent file", "path", p)
-			}
-		}
+		slog.Warn("errata file does not exist; using embedded errata", slog.Any("path", errataPath))
+		b = defaultErrata
 	} else {
-		b, err := os.ReadFile(errataPath)
+		var err error
+		b, err = os.ReadFile(errataPath)
+		slog.Debug("Using errata overlay", slog.Any("path", errataPath))
 		if err != nil {
 			slog.Warn("error reading errata file", slog.Any("error", err))
 			return
 		}
-		var errataOverlay errataOverlay
-		err = yaml.Unmarshal(b, &errataOverlay)
-		if err != nil {
-			slog.Warn("error parsing errata file", slog.Any("error", err))
-			return
+	}
+	var errataOverlay errataOverlay
+	err = yaml.Unmarshal(b, &errataOverlay)
+	if err != nil {
+		slog.Warn("error parsing errata file", slog.Any("error", err))
+		return
+	}
+	Erratas = errataOverlay.Errata
+	for p := range Erratas {
+		path := filepath.Join(specRoot, p)
+		exists, _ := files.Exists(path)
+		if !exists {
+			slog.Warn("errata points to non-existent file", "path", p)
 		}
-		slog.Debug("Using errata overlay", slog.Any("path", errataPath), slog.Any("count", len(errataOverlay.Errata)))
-		Erratas = errataOverlay.Errata
 	}
-
 }
 
 type errataOverlay struct {