Add clang variant to host builds (#16325)

* Make clang variant as a requirement for asan/tsan builds, add support for required variants for build systems

* Restyle

* Also update workflows

* Adjust test group ordering since we have placed clang as a last variant

* More moving of chip tool variants to make the options in yaml file make sense

* add missin $ for var specifier

* Add clang variant to tsan

* Asan/tsan not limited by clang, so updated as such

* Restyle

.github/workflows/fuzzing-build.yaml

@@ -124,7 +124,7 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                      "./scripts/build/build_examples.py \
-                        --target darwin-x64-all-clusters-no-ble-asan-libfuzzer \
+                        --target darwin-x64-all-clusters-no-ble-asan-libfuzzer-clang \
                         build \
                         --copy-artifacts-to objdir-clone \
                      "

.github/workflows/tests.yaml

@@ -32,7 +32,7 @@ jobs:
 
         strategy:
             matrix:
-                build_variant: [no-ble-tsan]
+                build_variant: [no-ble-tsan-clang]
                 chip_tool: ["", -same-event-loop]
         env:
             BUILD_VARIANT: ${{matrix.build_variant}}
@@ -79,8 +79,8 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                      "./scripts/build/build_examples.py \
-                        --target linux-x64-chip-tool-${BUILD_VARIANT}${CHIP_TOOL_VARIANT} \
-                        --target linux-x64-all-clusters-${BUILD_VARIANT}-test-group \
+                        --target linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \
+                        --target linux-x64-all-clusters-test-group-${BUILD_VARIANT} \
                         --target linux-x64-door-lock-${BUILD_VARIANT} \
                         --target linux-x64-tv-app-${BUILD_VARIANT} \
                         build \
@@ -91,10 +91,10 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                   "./scripts/tests/run_test_suite.py \
-                     --chip-tool ./out/linux-x64-chip-tool-${BUILD_VARIANT}${CHIP_TOOL_VARIANT}/chip-tool \
+                     --chip-tool ./out/linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \
                      run \
                      --iterations 1 \
-                     --all-clusters-app ./out/linux-x64-all-clusters-${BUILD_VARIANT}-test-group/chip-all-clusters-app \
+                     --all-clusters-app ./out/linux-x64-all-clusters-test-group-${BUILD_VARIANT}/chip-all-clusters-app \
                      --door-lock-app ./out/linux-x64-door-lock-${BUILD_VARIANT}/chip-door-lock-app \
                      --tv-app ./out/linux-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \
                   "
@@ -120,7 +120,7 @@ jobs:
 
         strategy:
             matrix:
-                build_variant: [no-ble-tsan, no-ble-asan]
+                build_variant: [no-ble-tsan-clang, no-ble-asan-clang]
                 chip_tool: ["", -same-event-loop]
         env:
             BUILD_VARIANT: ${{matrix.build_variant}}
@@ -174,7 +174,7 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                      "./scripts/build/build_examples.py \
-                        --target darwin-x64-chip-tool-${BUILD_VARIANT}${CHIP_TOOL_VARIANT} \
+                        --target darwin-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \
                         --target darwin-x64-all-clusters-${BUILD_VARIANT} \
                         --target darwin-x64-door-lock-${BUILD_VARIANT} \
                         --target darwin-x64-tv-app-${BUILD_VARIANT} \
@@ -186,7 +186,7 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                   "./scripts/tests/run_test_suite.py \
-                     --chip-tool ./out/darwin-x64-chip-tool-${BUILD_VARIANT}${CHIP_TOOL_VARIANT}/chip-tool \
+                     --chip-tool ./out/darwin-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \
                      --target-skip-glob '{TestGroupMessaging}' \
                      run \
                      --iterations 1 \
@@ -222,7 +222,7 @@ jobs:
 
         strategy:
             matrix:
-                build_variant: [no-ble-no-wifi-tsan]
+                build_variant: [no-ble-no-wifi-tsan-clang]
 
         env:
             BUILD_VARIANT: ${{matrix.build_variant}}
@@ -300,7 +300,7 @@ jobs:
 
         strategy:
             matrix:
-                build_variant: [no-ble-no-wifi-tsan]
+                build_variant: [no-ble-no-wifi-tsan-clang]
         env:
             BUILD_VARIANT: ${{matrix.build_variant}}
             TSAN_OPTIONS: "halt_on_error=1"

examples/all-clusters-app/linux/README.md

@@ -8,25 +8,25 @@ This example supports compilation with libfuzzer enabled.
 
 To compile with libfuzzer enabled on Mac, run:
 
-    $ ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target darwin-x64-all-clusters-no-ble-asan-libfuzzer build"
+    $ ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target darwin-x64-all-clusters-no-ble-asan-libfuzzer-clang build"
 
 at the top level of the Matter tree.
 
 Similarly, to compile on Linux run:
 
-    $ ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target linux-x64-all-clusters-no-ble-asan-libfuzzer build"
+    $ ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target linux-x64-all-clusters-no-ble-asan-libfuzzer-clang build"
 
 ### Running libfuzzer-enabled binaries
 
 #### Initial run
 
 To run the resulting binary with no particular inputs do:
 
-    $ ./out/darwin-x64-all-clusters-no-ble-asan-libfuzzer/chip-all-clusters-app-fuzzing
+    $ ./out/darwin-x64-all-clusters-no-ble-asan-libfuzzer-clang/chip-all-clusters-app-fuzzing
 
 or
 
-    $ ./out/linux-x64-all-clusters-no-ble-asan-libfuzzer/chip-all-clusters-app-fuzzing
+    $ ./out/linux-x64-all-clusters-no-ble-asan-libfuzzer-clang/chip-all-clusters-app-fuzzing
 
 If this crashes, it will output the input that caused the crash in a variety of
 formats, looking something like this:
@@ -44,11 +44,11 @@ To run the binary with a specific input, place the input bytes in a file (which
 a crashing run of the fuzzer does automatically). If `$(INPUT_FILE)` is the name
 of that file, then run:
 
-    $ ./out/darwin-x64-all-clusters-no-ble-asan-libfuzzer/chip-all-clusters-app-fuzzing $(INPUT_FILE)
+    $ ./out/darwin-x64-all-clusters-no-ble-asan-libfuzzer-clang/chip-all-clusters-app-fuzzing $(INPUT_FILE)
 
 or
 
-    $ ./out/linux-x64-all-clusters-no-ble-asan-libfuzzer/chip-all-clusters-app-fuzzing $(INPUT_FILE)
+    $ ./out/linux-x64-all-clusters-no-ble-asan-libfuzzer-clang/chip-all-clusters-app-fuzzing $(INPUT_FILE)
 
 #### Additional execution options.
 

scripts/build/build/targets.py

@@ -113,11 +113,12 @@ def Accept(self, name: str):
 
 
 class BuildVariant:
-    def __init__(self, name: str, validator=AcceptAnyName(), conflicts: List[str] = [], **buildargs):
+    def __init__(self, name: str, validator=AcceptAnyName(), conflicts: List[str] = [], requires: List[str] = [], **buildargs):
         self.name = name
         self.validator = validator
         self.conflicts = conflicts
         self.buildargs = buildargs
+        self.requires = requires
 
 
 def HasConflicts(items: List[BuildVariant]) -> bool:
@@ -127,12 +128,27 @@ def HasConflicts(items: List[BuildVariant]) -> bool:
     return False
 
 
+def AllRequirementsMet(items: List[BuildVariant]) -> bool:
+    """
+    Check that item.requires is satisfied for all items in the given list
+    """
+    available = set([item.name for item in items])
+
+    for item in items:
+        for requirement in item.requires:
+            if not requirement in available:
+                return False
+
+    return True
+
+
 class VariantBuilder:
     """Handles creating multiple build variants based on a starting target.
     """
 
-    def __init__(self, targets: List[Target]):
-        self.targets = targets
+    def __init__(self, targets: List[Target] = []):
+        # note the clone in case the default arg is used
+        self.targets = targets[:]
         self.variants = []
         self.glob_whitelist = []
 
@@ -176,6 +192,9 @@ def AllVariants(self):
                     if HasConflicts(subgroup):
                         continue
 
+                    if not AllRequirementsMet(subgroup):
+                        continue
+
                     # Target ready to be created - no conflicts
                     variant_target = target.Clone()
                     for option in subgroup:
@@ -223,21 +242,22 @@ def HostTargets():
         app_targets.append(target.Extend(
             'ota-requestor', app=HostApp.OTA_REQUESTOR, enable_ble=False))
 
-    builder = VariantBuilder([])
+    builder = VariantBuilder()
 
     # Possible build variants. Note that number of potential
     # builds is exponential here
+    builder.AppendVariant(name="test-group", validator=AcceptNameWithSubstrings(
+        ['-all-clusters', '-chip-tool']), test_group=True),
+    builder.AppendVariant(name="same-event-loop", validator=AcceptNameWithSubstrings(
+        ['-chip-tool']), separate_event_loop=False),
     builder.AppendVariant(name="ipv6only", enable_ipv4=False),
     builder.AppendVariant(name="no-ble", enable_ble=False),
     builder.AppendVariant(name="no-wifi", enable_wifi=False),
     builder.AppendVariant(name="tsan", conflicts=['asan'], use_tsan=True),
     builder.AppendVariant(name="asan", conflicts=['tsan'], use_asan=True),
-    builder.AppendVariant(
-        name="libfuzzer", use_libfuzzer=True, use_clang=True),
-    builder.AppendVariant(name="test-group", validator=AcceptNameWithSubstrings(
-        ['-all-clusters', '-chip-tool']), test_group=True),
-    builder.AppendVariant(name="same-event-loop", validator=AcceptNameWithSubstrings(
-        ['-chip-tool']), separate_event_loop=False),
+    builder.AppendVariant(name="libfuzzer", requires=[
+                          "clang"], use_libfuzzer=True),
+    builder.AppendVariant(name="clang", use_clang=True),
 
     builder.WhitelistVariantNameForGlob('ipv6only')
 
@@ -308,7 +328,7 @@ def Efr32Targets():
             'only user requested')
     ]
 
-    builder = VariantBuilder([])
+    builder = VariantBuilder()
 
     for board_target in board_targets:
         builder.targets.append(board_target.Extend(