Update pigweed & enable arm64 host support (#3319)

* Update pigweed & enable arm64 host support This enabled building on the Raspberry Pi 4 with a 64-bit OS installed. Fork pigweed's requirements and drop its bundled python, as this is breaking installing various pip packages for us. This allows installing mobly in the usual way in our requirements.txt. * Still install bundled python on Mac It works OK on Mac unlike on Linux, and on Mac the system python does not work. * Fix initial bootstrap * Remove .chip-banner.txt * Keep PW_ROOT * Keep _PW_ACTUAL_ENVIRONMENT_ROOT * Update docker images to 0.4.11 * Fix botched replacement
project-chip · Nov 20, 2020 · 9014720 · 9014720
1 parent 4ee3e34
commit 9014720
Show file tree

Hide file tree

Showing 7 changed files with 166 additions and 64 deletions.
diff --git a/.chip-banner.txt b/.chip-banner.txt
diff --git a/scripts/activate.sh b/scripts/activate.sh
diff --git a/scripts/activate.sh b/scripts/activate.sh
@@ -0,0 +1 @@
+bootstrap.sh
diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh
@@ -14,28 +14,76 @@
 # limitations under the License.
 #
 
-if [[ -n ${BASH_SOURCE[0]} ]]; then
-    CHIP_ROOT=$(cd "${BASH_SOURCE[0]%/*}/.." && pwd)
-else
-    CHIP_ROOT=$(cd "${0%/*}/.." && pwd)
-fi
+_bootstrap_or_activate() {
+    if [ -n "$BASH" ]; then
+        local _BOOTSTRAP_PATH="${BASH_SOURCE[0]}"
+    else
+        local _BOOTSTRAP_PATH="$1"
+    fi
 
-export PW_BRANDING_BANNER="$CHIP_ROOT/.chip-banner.txt"
-export PW_BRANDING_BANNER_COLOR="bold_white"
-export PW_VIRTUALENV_REQUIREMENTS="$CHIP_ROOT/scripts/requirements.txt"
+    local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}"
+    local _CHIP_ROOT="$(cd "${_BOOTSTRAP_PATH%/*}/.." && pwd)"
 
-if test -f "$CHIP_ROOT/scripts/helpers/rename_submodules.sh"; then
-    "$CHIP_ROOT/scripts/helpers/rename_submodules.sh"
-fi
+    if [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" ] ||
+        [ ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then
+        git submodule update --init
+    fi
 
-git submodule update --init
+    local _CHIP_BANNER="$(
+        cat <<EOF
+  ▄███▒  ░▓█  ░▓█ ░▓█▓ ▒█████▄
+ ██▒ ▀█▒  ▒█   ▒█  ░█▒  ▒█░  █░
+ █▓░      ▒██████  ░█▒  ▒█▄▄▄█░
+ ▓█   █▒  ▒█   ▒█  ░█░  ▒█▀
+ ░▓███▀  ░▓███░▓█▒ ░█░  ▒█
+EOF
+    )"
 
-export PW_VIRTUALENV_SETUP_PY_ROOTS="$CHIP_ROOT/integrations/mobly"
+    PW_ROOT="$_CHIP_ROOT/third_party/pigweed/repo"
+    export PW_ROOT
 
-# shellcheck source=/dev/null
-source "$CHIP_ROOT/third_party/pigweed/repo/bootstrap.sh"
+    . "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh"
 
-#TODO - remove this once native python building is solved for
-#       psutil (one of mobly's dependencies which CHIP does
-#       not actually need, so --no-deps is OK)
-pip install --no-deps portpicker mobly
+    _chip_bootstrap_banner() {
+        if [ -z "$PW_ENVSETUP_QUIET" ] && [ -z "$PW_ENVSETUP_NO_BANNER" ]; then
+            pw_bold_white "$_CHIP_BANNER\n"
+        fi
+    }
+
+    local _PW_BANNER_FUNC="_chip_bootstrap_banner"
+
+    export _PW_ACTUAL_ENVIRONMENT_ROOT="$(pw_get_env_root)"
+    local _SETUP_SH="$_PW_ACTUAL_ENVIRONMENT_ROOT/activate.sh"
+
+    export PW_DOCTOR_SKIP_CIPD_CHECKS=1
+
+    if [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" ] ||
+        [ ! -f "$_SETUP_SH" ] ||
+        [ ! -s "$_SETUP_SH" ]; then
+        pw_bootstrap --shell-file "$_SETUP_SH" \
+            --install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT" \
+            --virtualenv-requirements "$_CHIP_ROOT/scripts/requirements.txt" \
+            --cipd-package-file "$_CHIP_ROOT/scripts/pigweed.json" \
+            --virtualenv-setup-py-root "$_CHIP_ROOT/third_party/pigweed" \
+            --virtualenv-setup-py-root "$_CHIP_ROOT/integrations/mobly"
+        pw_finalize bootstrap "$_SETUP_SH"
+    else
+        pw_activate
+        pw_finalize activate "$_SETUP_SH"
+    fi
+}
+
+_bootstrap_or_activate "$0"
+unset -f _bootstrap_or_activate
+
+pw_cleanup
+
+unset PW_CIPD_INSTALL_DIR
+unset CIPD_CACHE_DIR
+unset _PW_BANNER_FUNC
+unset _PW_TEXT
+unset PW_DOCTOR_SKIP_CIPD_CHECKS
+
+unset -f pw_cleanup
+unset -f _pw_hello
+unset -f _chip_bootstrap_banner
diff --git a/scripts/pigweed.json b/scripts/pigweed.json
@@ -0,0 +1,85 @@
+[
+    {
+        "path": "gn/gn/${os}-${arch=amd64,arm64}",
+        "tags": ["git_revision:132588e1f9c2bf07b0cc0516e190b445de1153a7"]
+    },
+    {
+        "path": "infra/ninja/${os}-${arch=amd64}",
+        "tags": ["version:1.9.0"]
+    },
+    {
+        "path": "infra/cmake/${os}-${arch=amd64}",
+        "tags": ["version:3.16.1"]
+    },
+    {
+        "_comment": "TODO(pwbug/93): Package Bazel for Windows.",
+        "path": "pigweed/third_party/bazel/${os=linux,mac}-${arch=amd64}",
+        "tags": ["version:3.2.0"]
+    },
+    {
+        "path": "pigweed/third_party/bloaty-embedded/${os=linux,mac}-${arch=amd64}",
+        "tags": ["git_revision:2d87d204057b419f5290f8d38b61b9c2c5b4fb52-2"]
+    },
+    {
+        "path": "pigweed/third_party/gcc-arm-none-eabi/${os}-${arch=amd64}",
+        "tags": ["version:9-2020-q2-update"]
+    },
+    {
+        "path": "infra/tools/protoc/${os}-${arch=amd64}",
+        "tags": ["protobuf_version:v3.8.0"]
+    },
+    {
+        "_comment": "TODO(pwbug/70) Put clang in cipd for Windows.",
+        "path": "fuchsia/third_party/clang/${os=linux,mac}-${arch}",
+        "tags": ["git_revision:8af160b0b8ca8102b9490a287244af75727872f5"]
+    },
+    {
+        "_comment": "When bumping the minor version (e.g., to 3.9.x) also update env_setup/virtualenv/init.py to check for the new version.",
+        "path": "infra/3pp/tools/cpython3/${os=mac}-${arch}",
+        "tags": ["version:3.8.2.chromium.10"]
+    },
+    {
+        "path": "infra/go/${os}-${arch=amd64}",
+        "tags": ["version:1.13.5"]
+    },
+    {
+        "path": "pigweed/third_party/protoc-gen-go/${os}-${arch=amd64}",
+        "tags": ["version:1.3.2"]
+    },
+    {
+        "_comment": "TODO(pwbug/66) Put openocd in cipd for Windows.",
+        "path": "pigweed/third_party/openocd/${os=linux,mac}-${arch=amd64}",
+        "tags": ["git_revision:e41c0f4906e46d1076ce62a0da5518aa1ca280b8"]
+    },
+    {
+        "_comment": "TODO(pwbug/69) Put rust in cipd for Windows.",
+        "path": "fuchsia/rust/${os=linux,mac}-${arch=amd64}",
+        "tags": ["git_revision:027149919e36ce5645ca5d02d55b97ef52eb55ba"]
+    },
+    {
+        "path": "pigweed/third_party/mingw64-x86_64-win32-seh/${os=windows}-${arch=amd64}",
+        "tags": ["version:8.1.0"]
+    },
+    {
+        "path": "pigweed/host_tools/${os}-${arch=amd64}",
+        "tags": ["git_revision:918276537a01f4f470fa61511347d4dfe1c9f297"]
+    },
+    {
+        "path": "infra/goma/client/${os}-${arch=amd64}",
+        "tags": ["git_revision:4e2693cc42d116902f0767b9b93643555f79e7c4"]
+    },
+    {
+        "path": "fuchsia/third_party/qemu/${os=linux,mac}-${arch}",
+        "tags": ["git_revision:841f14e74f5af7886cf49cfcd4fed264911ae58e"]
+    },
+    {
+        "path": "fuchsia/third_party/kythe",
+        "subdir": "kythe",
+        "tags": ["version:1.0.1"]
+    },
+    {
+        "path": "fuchsia/third_party/kythe-libs/${os=linux}-${arch=amd64}",
+        "subdir": "kythe",
+        "tags": ["version:2020-08-05"]
+    }
+]
diff --git a/scripts/requirements.in b/scripts/requirements.in
@@ -14,3 +14,7 @@ requests>=2.24.0
 
 # device controller wheel package
 wheel>=0.34.2
+
+# mobly tests
+portpicker
+mobly
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
@@ -9,17 +9,22 @@ cffi==1.14.3              # via cryptography
 chardet==3.0.4            # via requests
 click==7.1.2              # via -r requirements.in, pip-tools
 cryptography==3.1.1       # via -r requirements.in
-future==0.18.2            # via -r requirements.in
+future==0.18.2            # via -r requirements.in, mobly
 idna==2.10                # via requests
+mobly==1.10               # via -r requirements.in
 pip-tools==5.3.1          # via -r requirements.in
+portpicker==1.3.1         # via -r requirements.in, mobly
+psutil==5.7.2             # via mobly
 pycparser==2.20           # via cffi
 pyelftools==0.26          # via -r requirements.in
 pyparsing==2.3.1          # via -r requirements.in
-pyserial==3.4             # via -r requirements.in
+pyserial==3.4             # via -r requirements.in, mobly
+pyyaml==5.3.1             # via mobly
 requests==2.24.0          # via -r requirements.in
 six==1.15.0               # via cryptography, pip-tools
-wheel==0.34.2             # via -r requirements.in
+timeout-decorator==0.4.1  # via mobly
 urllib3==1.25.10          # via requests
+wheel==0.34.2             # via -r requirements.in
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip
diff --git a/third_party/pigweed/repo b/third_party/pigweed/repo