From 6e36919889c4d24e8efb85ae05b02fed593cec79 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 18 Mar 2022 14:46:15 -0400 Subject: [PATCH] Add clang variant to host builds (#16325) * Make clang variant as a requirement for asan/tsan builds, add support for required variants for build systems * Restyle * Also update workflows * Adjust test group ordering since we have placed clang as a last variant * More moving of chip tool variants to make the options in yaml file make sense * add missin $ for var specifier * Add clang variant to tsan * Asan/tsan not limited by clang, so updated as such * Restyle --- .github/workflows/fuzzing-build.yaml | 2 +- .github/workflows/tests.yaml | 20 +++++------ examples/all-clusters-app/linux/README.md | 12 +++---- scripts/build/build/targets.py | 42 +++++++++++++++++------ 4 files changed, 48 insertions(+), 28 deletions(-) diff --git a/.github/workflows/fuzzing-build.yaml b/.github/workflows/fuzzing-build.yaml index e5297224742778..96a5dca9e2c73b 100644 --- a/.github/workflows/fuzzing-build.yaml +++ b/.github/workflows/fuzzing-build.yaml @@ -124,7 +124,7 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ - --target darwin-x64-all-clusters-no-ble-asan-libfuzzer \ + --target darwin-x64-all-clusters-no-ble-asan-libfuzzer-clang \ build \ --copy-artifacts-to objdir-clone \ " diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e6bbe5ee46600a..6dab37ab7a264e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -32,7 +32,7 @@ jobs: strategy: matrix: - build_variant: [no-ble-tsan] + build_variant: [no-ble-tsan-clang] chip_tool: ["", -same-event-loop] env: BUILD_VARIANT: ${{matrix.build_variant}} @@ -79,8 +79,8 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ - --target linux-x64-chip-tool-${BUILD_VARIANT}${CHIP_TOOL_VARIANT} \ - --target linux-x64-all-clusters-${BUILD_VARIANT}-test-group \ + --target linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \ + --target linux-x64-all-clusters-test-group-${BUILD_VARIANT} \ --target linux-x64-door-lock-${BUILD_VARIANT} \ --target linux-x64-tv-app-${BUILD_VARIANT} \ build \ @@ -91,10 +91,10 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ - --chip-tool ./out/linux-x64-chip-tool-${BUILD_VARIANT}${CHIP_TOOL_VARIANT}/chip-tool \ + --chip-tool ./out/linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \ run \ --iterations 1 \ - --all-clusters-app ./out/linux-x64-all-clusters-${BUILD_VARIANT}-test-group/chip-all-clusters-app \ + --all-clusters-app ./out/linux-x64-all-clusters-test-group-${BUILD_VARIANT}/chip-all-clusters-app \ --door-lock-app ./out/linux-x64-door-lock-${BUILD_VARIANT}/chip-door-lock-app \ --tv-app ./out/linux-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \ " @@ -120,7 +120,7 @@ jobs: strategy: matrix: - build_variant: [no-ble-tsan, no-ble-asan] + build_variant: [no-ble-tsan-clang, no-ble-asan-clang] chip_tool: ["", -same-event-loop] env: BUILD_VARIANT: ${{matrix.build_variant}} @@ -174,7 +174,7 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ - --target darwin-x64-chip-tool-${BUILD_VARIANT}${CHIP_TOOL_VARIANT} \ + --target darwin-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \ --target darwin-x64-all-clusters-${BUILD_VARIANT} \ --target darwin-x64-door-lock-${BUILD_VARIANT} \ --target darwin-x64-tv-app-${BUILD_VARIANT} \ @@ -186,7 +186,7 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ - --chip-tool ./out/darwin-x64-chip-tool-${BUILD_VARIANT}${CHIP_TOOL_VARIANT}/chip-tool \ + --chip-tool ./out/darwin-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \ --target-skip-glob '{TestGroupMessaging}' \ run \ --iterations 1 \ @@ -222,7 +222,7 @@ jobs: strategy: matrix: - build_variant: [no-ble-no-wifi-tsan] + build_variant: [no-ble-no-wifi-tsan-clang] env: BUILD_VARIANT: ${{matrix.build_variant}} @@ -300,7 +300,7 @@ jobs: strategy: matrix: - build_variant: [no-ble-no-wifi-tsan] + build_variant: [no-ble-no-wifi-tsan-clang] env: BUILD_VARIANT: ${{matrix.build_variant}} TSAN_OPTIONS: "halt_on_error=1" diff --git a/examples/all-clusters-app/linux/README.md b/examples/all-clusters-app/linux/README.md index c7e72959cbc11b..d029d6eff5f14d 100644 --- a/examples/all-clusters-app/linux/README.md +++ b/examples/all-clusters-app/linux/README.md @@ -8,13 +8,13 @@ This example supports compilation with libfuzzer enabled. To compile with libfuzzer enabled on Mac, run: - $ ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target darwin-x64-all-clusters-no-ble-asan-libfuzzer build" + $ ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target darwin-x64-all-clusters-no-ble-asan-libfuzzer-clang build" at the top level of the Matter tree. Similarly, to compile on Linux run: - $ ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target linux-x64-all-clusters-no-ble-asan-libfuzzer build" + $ ./scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target linux-x64-all-clusters-no-ble-asan-libfuzzer-clang build" ### Running libfuzzer-enabled binaries @@ -22,11 +22,11 @@ Similarly, to compile on Linux run: To run the resulting binary with no particular inputs do: - $ ./out/darwin-x64-all-clusters-no-ble-asan-libfuzzer/chip-all-clusters-app-fuzzing + $ ./out/darwin-x64-all-clusters-no-ble-asan-libfuzzer-clang/chip-all-clusters-app-fuzzing or - $ ./out/linux-x64-all-clusters-no-ble-asan-libfuzzer/chip-all-clusters-app-fuzzing + $ ./out/linux-x64-all-clusters-no-ble-asan-libfuzzer-clang/chip-all-clusters-app-fuzzing If this crashes, it will output the input that caused the crash in a variety of formats, looking something like this: @@ -44,11 +44,11 @@ To run the binary with a specific input, place the input bytes in a file (which a crashing run of the fuzzer does automatically). If `$(INPUT_FILE)` is the name of that file, then run: - $ ./out/darwin-x64-all-clusters-no-ble-asan-libfuzzer/chip-all-clusters-app-fuzzing $(INPUT_FILE) + $ ./out/darwin-x64-all-clusters-no-ble-asan-libfuzzer-clang/chip-all-clusters-app-fuzzing $(INPUT_FILE) or - $ ./out/linux-x64-all-clusters-no-ble-asan-libfuzzer/chip-all-clusters-app-fuzzing $(INPUT_FILE) + $ ./out/linux-x64-all-clusters-no-ble-asan-libfuzzer-clang/chip-all-clusters-app-fuzzing $(INPUT_FILE) #### Additional execution options. diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index e591dfaee481f3..fdfcc8a8a071b6 100644 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -113,11 +113,12 @@ def Accept(self, name: str): class BuildVariant: - def __init__(self, name: str, validator=AcceptAnyName(), conflicts: List[str] = [], **buildargs): + def __init__(self, name: str, validator=AcceptAnyName(), conflicts: List[str] = [], requires: List[str] = [], **buildargs): self.name = name self.validator = validator self.conflicts = conflicts self.buildargs = buildargs + self.requires = requires def HasConflicts(items: List[BuildVariant]) -> bool: @@ -127,12 +128,27 @@ def HasConflicts(items: List[BuildVariant]) -> bool: return False +def AllRequirementsMet(items: List[BuildVariant]) -> bool: + """ + Check that item.requires is satisfied for all items in the given list + """ + available = set([item.name for item in items]) + + for item in items: + for requirement in item.requires: + if not requirement in available: + return False + + return True + + class VariantBuilder: """Handles creating multiple build variants based on a starting target. """ - def __init__(self, targets: List[Target]): - self.targets = targets + def __init__(self, targets: List[Target] = []): + # note the clone in case the default arg is used + self.targets = targets[:] self.variants = [] self.glob_whitelist = [] @@ -176,6 +192,9 @@ def AllVariants(self): if HasConflicts(subgroup): continue + if not AllRequirementsMet(subgroup): + continue + # Target ready to be created - no conflicts variant_target = target.Clone() for option in subgroup: @@ -223,21 +242,22 @@ def HostTargets(): app_targets.append(target.Extend( 'ota-requestor', app=HostApp.OTA_REQUESTOR, enable_ble=False)) - builder = VariantBuilder([]) + builder = VariantBuilder() # Possible build variants. Note that number of potential # builds is exponential here + builder.AppendVariant(name="test-group", validator=AcceptNameWithSubstrings( + ['-all-clusters', '-chip-tool']), test_group=True), + builder.AppendVariant(name="same-event-loop", validator=AcceptNameWithSubstrings( + ['-chip-tool']), separate_event_loop=False), builder.AppendVariant(name="ipv6only", enable_ipv4=False), builder.AppendVariant(name="no-ble", enable_ble=False), builder.AppendVariant(name="no-wifi", enable_wifi=False), builder.AppendVariant(name="tsan", conflicts=['asan'], use_tsan=True), builder.AppendVariant(name="asan", conflicts=['tsan'], use_asan=True), - builder.AppendVariant( - name="libfuzzer", use_libfuzzer=True, use_clang=True), - builder.AppendVariant(name="test-group", validator=AcceptNameWithSubstrings( - ['-all-clusters', '-chip-tool']), test_group=True), - builder.AppendVariant(name="same-event-loop", validator=AcceptNameWithSubstrings( - ['-chip-tool']), separate_event_loop=False), + builder.AppendVariant(name="libfuzzer", requires=[ + "clang"], use_libfuzzer=True), + builder.AppendVariant(name="clang", use_clang=True), builder.WhitelistVariantNameForGlob('ipv6only') @@ -308,7 +328,7 @@ def Efr32Targets(): 'only user requested') ] - builder = VariantBuilder([]) + builder = VariantBuilder() for board_target in board_targets: builder.targets.append(board_target.Extend(