pw_ide: Support multiple comp DB search paths

Bug: 280363633 Change-Id: I3f5d46a3153e083c4640e3fc66d5c1a19b0cd34a Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/144210 Presubmit-Verified: CQ Bot Account <[email protected]> Reviewed-by: Anthony DiGirolamo <[email protected]> Commit-Queue: Chad Norvell <[email protected]>
google · Jul 18, 2023 · c6ca138 · c6ca138
1 parent b3c696e
commit c6ca138
Show file tree

Hide file tree

Showing 6 changed files with 150 additions and 78 deletions.
diff --git a/pw_ide/docs.rst b/pw_ide/docs.rst
@@ -26,8 +26,7 @@ source control, and applies only to that checkout of the project. All of these
 files have the same schema, in which these options can be configured:
 
 .. autoproperty:: pw_ide.settings.PigweedIdeSettings.working_dir
-.. autoproperty:: pw_ide.settings.PigweedIdeSettings.build_dir
-.. autoproperty:: pw_ide.settings.PigweedIdeSettings.compdb_paths
+.. autoproperty:: pw_ide.settings.PigweedIdeSettings.compdb_search_paths
 .. autoproperty:: pw_ide.settings.PigweedIdeSettings.targets
 .. autoproperty:: pw_ide.settings.PigweedIdeSettings.target_inference
 .. autoproperty:: pw_ide.settings.PigweedIdeSettings.default_target

diff --git a/pw_ide/py/cpp_test.py b/pw_ide/py/cpp_test.py
@@ -970,7 +970,7 @@ class TestCppCompilationDatabase(PwIdeTestCase):
     """Tests CppCompilationDatabase"""
 
     def setUp(self):
-        self.build_dir = Path('/pigweed/pigweed/out')
+        self.root_dir = Path('/pigweed/pigweed/out')
 
         self.fixture: List[CppCompileCommandDict] = [
             {
@@ -1046,10 +1046,10 @@ def setUp(self):
 
     def test_merge(self):
         compdb1 = CppCompilationDatabase.load(
-            self.fixture_merge_1, self.build_dir
+            self.fixture_merge_1, self.root_dir
         )
         compdb2 = CppCompilationDatabase.load(
-            self.fixture_merge_2, self.build_dir
+            self.fixture_merge_2, self.root_dir
         )
         compdb1.merge(compdb2)
         result = [compile_command.as_dict() for compile_command in compdb1]
@@ -1058,22 +1058,22 @@ def test_merge(self):
 
     def test_merge_no_dupes(self):
         compdb1 = CppCompilationDatabase.load(
-            self.fixture_merge_1, self.build_dir
+            self.fixture_merge_1, self.root_dir
         )
         fixture_combo = [*self.fixture_merge_1, *self.fixture_merge_2]
-        compdb2 = CppCompilationDatabase.load(fixture_combo, self.build_dir)
+        compdb2 = CppCompilationDatabase.load(fixture_combo, self.root_dir)
         compdb1.merge(compdb2)
         result = [compile_command.as_dict() for compile_command in compdb1]
         expected = [*self.fixture_merge_1, *self.fixture_merge_2]
         self.assertCountEqual(result, expected)
 
     def test_load_from_dicts(self):
-        compdb = CppCompilationDatabase.load(self.fixture, self.build_dir)
+        compdb = CppCompilationDatabase.load(self.fixture, self.root_dir)
         self.assertCountEqual(compdb.as_dicts(), self.expected)
 
     def test_load_from_json(self):
         compdb = CppCompilationDatabase.load(
-            json.dumps(self.fixture), self.build_dir
+            json.dumps(self.fixture), self.root_dir
         )
         self.assertCountEqual(compdb.as_dicts(), self.expected)
 
@@ -1084,15 +1084,15 @@ def test_load_from_path(self):
         ) as (_, file_path):
             path = file_path
 
-        compdb = CppCompilationDatabase.load(path, self.build_dir)
+        compdb = CppCompilationDatabase.load(path, self.root_dir)
         self.assertCountEqual(compdb.as_dicts(), self.expected)
 
     def test_load_from_file_handle(self):
         with self.make_temp_file(
             COMPDB_FILE_NAME,
             json.dumps(self.fixture),
         ) as (file, _):
-            compdb = CppCompilationDatabase.load(file, self.build_dir)
+            compdb = CppCompilationDatabase.load(file, self.root_dir)
 
         self.assertCountEqual(compdb.as_dicts(), self.expected)
 
@@ -1111,22 +1111,22 @@ def test_process(self):
         raw_db: List[CppCompileCommandDict] = [
             {
                 'command': 'arm-none-eabi-g++ -MMD -MF stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o.d -Wno-psabi -mabi=aapcs -mthumb --sysroot=../environment/cipd/packages/arm -specs=nano.specs -specs=nosys.specs -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -Wshadow -Wredundant-decls -u_printf_float -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -DPW_ARMV7M_ENABLE_FPU=1  -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/assert_compatibility_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o',
-                'directory': str(self.build_dir),
+                'directory': str(self.root_dir),
                 'file': '../pw_allocator/block.cc',
             },
             {
                 'command': '../environment/cipd/packages/pigweed/bin/isosceles-clang++ -MMD -MF isosceles_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o isosceles_debug/obj/pw_allocator/block.block.cc.o',
-                'directory': str(self.build_dir),
+                'directory': str(self.root_dir),
                 'file': '../pw_allocator/block.cc',
             },
             {
                 'command': 'ccache ../environment/cipd/packages/pigweed/bin/clang++ -MMD -MF pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o  pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o',
-                'directory': str(self.build_dir),
+                'directory': str(self.root_dir),
                 'file': '../pw_allocator/block.cc',
             },
             {
                 'command': "python ../pw_toolchain/py/pw_toolchain/clang_tidy.py --source-exclude 'third_party/.*' --source-exclude '.*packages/mbedtls.*' --source-exclude '.*packages/boringssl.*' --skip-include-path 'mbedtls/include' --skip-include-path 'mbedtls' --skip-include-path 'boringssl/src/include' --skip-include-path 'boringssl' --skip-include-path 'pw_tls_client/generate_test_data' --source-file ../pw_allocator/block.cc --source-root '../' --export-fixes  pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o.yaml -- ../environment/cipd/packages/pigweed/bin/clang++ END_OF_INVOKER -MMD -MF pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o && touch pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o",
-                'directory': str(self.build_dir),
+                'directory': str(self.root_dir),
                 'file': '../pw_allocator/block.cc',
             },
         ]
@@ -1137,7 +1137,7 @@ def test_process(self):
                     'command':
                     # Ensures path format matches OS (e.g. Windows)
                     f'{Path("../environment/cipd/packages/pigweed/bin/isosceles-clang++")} -MMD -MF isosceles_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o isosceles_debug/obj/pw_allocator/block.block.cc.o',
-                    'directory': str(self.build_dir),
+                    'directory': str(self.root_dir),
                     'file': '../pw_allocator/block.cc',
                     'output': 'isosceles_debug/obj/pw_allocator/block.block.cc.o',
                 },
@@ -1147,7 +1147,7 @@ def test_process(self):
                     'command':
                     # Ensures path format matches OS (e.g. Windows)
                     f'ccache {Path("../environment/cipd/packages/pigweed/bin/clang++")} -MMD -MF pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o',
-                    'directory': str(self.build_dir),
+                    'directory': str(self.root_dir),
                     'file': '../pw_allocator/block.cc',
                     'output': 'pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o',
                 },
@@ -1157,7 +1157,7 @@ def test_process(self):
                     'command':
                     # Ensures this test avoids the unpathed compiler search
                     f'{self.temp_dir_path / "arm-none-eabi-g++"} -MMD -MF stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o.d -Wno-psabi -mabi=aapcs -mthumb --sysroot=../environment/cipd/packages/arm -specs=nano.specs -specs=nosys.specs -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -Wshadow -Wredundant-decls -u_printf_float -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -DPW_ARMV7M_ENABLE_FPU=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/assert_compatibility_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o',
-                    'directory': str(self.build_dir),
+                    'directory': str(self.root_dir),
                     'file': '../pw_allocator/block.cc',
                     'output': 'stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o',
                 },
@@ -1166,7 +1166,7 @@ def test_process(self):
         # pylint: enable=line-too-long
 
         compdbs = CppCompilationDatabase.load(
-            raw_db, build_dir=self.build_dir
+            raw_db, root_dir=self.root_dir
         ).process(settings, default_path=self.temp_dir_path)
         compdbs_as_dicts = {
             target: compdb.as_dicts() for target, compdb in compdbs.items()

diff --git a/pw_ide/py/pw_ide/cli.py b/pw_ide/py/pw_ide/cli.py
@@ -16,7 +16,6 @@
 import argparse
 import enum
 from inspect import cleandoc
-from pathlib import Path
 import re
 from typing import Any, Callable, Dict, List, Optional, Protocol
 
@@ -313,11 +312,6 @@ def _build_argument_parser() -> argparse.ArgumentParser:
         help='process a file or several files matching '
         'the clang compilation database format',
     )
-    parser_cpp.add_argument(
-        '--build-dir',
-        type=Path,
-        help='override the build directory defined in pw_ide settings',
-    )
     parser_cpp.add_argument(
         '--clangd-command',
         action='store_true',

diff --git a/pw_ide/py/pw_ide/commands.py b/pw_ide/py/pw_ide/commands.py
@@ -18,7 +18,7 @@
 import shlex
 import subprocess
 import sys
-from typing import cast, Dict, List, Optional, Set
+from typing import cast, Dict, List, Optional, Set, Tuple
 
 from pw_cli.env import pigweed_environment
 
@@ -280,13 +280,37 @@ def _process_compdbs(  # pylint: disable=too-many-locals
     # Associate processed compilation databases with their original sources
     all_processed_compdbs: Dict[Path, CppCompilationDatabasesMap] = {}
 
-    build_dir: Path = pw_ide_settings.build_dir
-    compdb_file_paths = list(build_dir.rglob(str(COMPDB_FILE_NAME)))
-
-    for compdb_file_path in compdb_file_paths:
+    # Get a list of paths to search for compilation databases.
+    compdb_search_paths: List[
+        Tuple[Path, str]
+    ] = pw_ide_settings.compdb_search_paths
+    # Get the list of files for each search path, tupled with the search path.
+    compdb_file_path_groups = [
+        (search_path, list(search_path[0].rglob(str(COMPDB_FILE_NAME))))
+        for search_path in compdb_search_paths
+    ]
+    # Flatten that list.
+    compdb_file_paths: List[Tuple[Path, Path, str]] = [
+        (search_path, file_path, target_inference)
+        for (
+            (search_path, target_inference),
+            file_path_group,
+        ) in compdb_file_path_groups
+        for file_path in file_path_group
+    ]
+
+    for (
+        compdb_root_dir,
+        compdb_file_path,
+        target_inference,
+    ) in compdb_file_paths:
         # Load the compilation database
         try:
-            compdb = CppCompilationDatabase.load(compdb_file_path, build_dir)
+            compdb = CppCompilationDatabase.load(
+                compdb_to_load=compdb_file_path,
+                root_dir=compdb_root_dir,
+                target_inference=target_inference,
+            )
         except MissingCompDbException:
             reporter.err(f'File not found: {str(compdb_file_path)}')
             sys.exit(1)
@@ -330,14 +354,16 @@ def _process_compdbs(  # pylint: disable=too-many-locals
         if processed_compdbs is None:
             # Infer the name of the target from the path
             name = '_'.join(
-                compdb_file_path.relative_to(build_dir).parent.parts
+                compdb_file_path.relative_to(compdb_root_dir).parent.parts
             )
 
             target = CppIdeFeaturesTarget(
                 name=name,
                 compdb_file_path=compdb_file_path,
                 num_commands=len(
-                    CppCompilationDatabase.load(compdb_file_path, build_dir)
+                    CppCompilationDatabase.load(
+                        compdb_file_path, compdb_root_dir
+                    )
                 ),
             )
 
@@ -456,7 +482,6 @@ def cmd_cpp(  # pylint: disable=too-many-arguments, too-many-locals, too-many-br
     should_list_targets: bool,
     should_get_target: bool,
     target_to_set: Optional[str],
-    build_dir: Optional[Path],
     process: bool = True,
     use_default_target: bool = False,
     clangd_command: bool = False,
@@ -556,10 +581,6 @@ def cmd_cpp(  # pylint: disable=too-many-arguments, too-many-locals, too-many-br
     state = CppIdeFeaturesState(pw_ide_settings)
     should_update_ides = False
 
-    build_dir = (
-        build_dir if build_dir is not None else pw_ide_settings.build_dir
-    )
-
     if process:
         default = False
         should_update_ides = _process_compdbs(reporter, pw_ide_settings)