[Deps] Backport to v1.59.x: Fully vendored upb again (#34514)

Backport of #34508

.gitmodules

@@ -47,6 +47,3 @@
 	# generated file that makes Git consider the submodule dirty. This
 	# state can be ignored for day-to-day development on gRPC.
 	ignore = dirty
-[submodule "third_party/upb"]
-	path = third_party/upb
-	url = https://github.com/protocolbuffers/upb.git

test/distrib/cpp/run_distrib_test_cmake.sh

@@ -66,8 +66,7 @@ popd
 # Get list of submodules from the .gitmodules file since for running "git submodule foreach"
 # we'd need to be in a git workspace (and that's not the case when running
 # distribtests as a bazel action)
-# TODO(veblush): Remove upb exception
-grep 'path = ' .gitmodules | sed 's/^.*path = //' | grep -v 'third_party/upb' | xargs rm -rf
+grep 'path = ' .gitmodules | sed 's/^.*path = //' | xargs rm -rf
 
 # Install gRPC
 mkdir -p "cmake/build"

test/distrib/cpp/run_distrib_test_cmake_pkgconfig.sh

@@ -70,8 +70,7 @@ popd
 # Get list of submodules from the .gitmodules file since for "git submodule foreach"
 # we'd need to be in a git workspace (and that's not the case when running
 # distribtests as a bazel action)
-# TODO(veblush): Remove upb exception
-grep 'path = ' .gitmodules | sed 's/^.*path = //' | grep -v 'third_party/upb' | xargs rm -rf
+grep 'path = ' .gitmodules | sed 's/^.*path = //' | xargs rm -rf
 
 # Install gRPC
 # TODO(jtattermusch): avoid the need for setting utf8_range_DIR

third_party/README.md

@@ -120,9 +120,17 @@ In addition, please perform the following two steps to generate the Python `xds-
 
 ### Updating third_party/upb
 
-In addition, please perform the following an extra step to generate upb files:
-
-1. Run `tools/codegen/core/gen_upb_api.sh` to regenerate upb files.
+Since upb is vendored in the gRPC repo, you cannot use submodule to update it. Please follow the steps below;
+1. Update third_party/upb directory by running
+   - `export GRPC_ROOT=~/git/grpc`
+   - `wget https://github.com/protocolbuffers/upb/archive/refs/heads/main.zip`
+   - `rm -rf $GRPC_ROOT/third_party/upb`
+   - `unzip main.zip -d $GRPC_ROOT/third_party`
+   - `mv $GRPC_ROOT/third_party/upb-main $GRPC_ROOT/third_party/upb`
+2. Update the dependency in `grpc_deps.bzl` to the same commit
+3. Populate the bazel download mirror by running `bazel/update_mirror.sh`
+4. Run `tools/buildgen/generate_projects.sh` to regenerate the generated files
+5. Run `tools/codegen/core/gen_upb_api.sh` to regenerate upb files.
 
 ### Updating third_party/utf8_range
 

third_party/upb/.bazelci/presubmit.yml

@@ -0,0 +1,16 @@
+---
+tasks:
+  ubuntu:
+    platform: ubuntu2004
+    shell_commands:
+    - "sudo apt -y update && sudo apt -y install libreadline-dev cmake rsync"
+    build_flags:
+    - "--incompatible_disallow_empty_glob"
+    test_targets:
+    - //...
+  macos:
+    platform: macos
+    build_flags:
+    - "--incompatible_disallow_empty_glob"
+    test_targets:
+    - //...

third_party/upb/.bazelignore

@@ -0,0 +1 @@
+_build

third_party/upb/.bazelrc

@@ -0,0 +1,60 @@
+# temporary fix for https://github.com/bazelbuild/bazel/issues/12905 on macOS
+build --features=-debug_prefix_map_pwd_is_dot
+
+# Pin to C++17
+build --cxxopt=-std=c++17 --host_cxxopt=-std=c++17
+build:cpp17_msvc --cxxopt=/std:c++17 --host_cxxopt=/std:c++17
+
+# Use our custom-configured c++ toolchain.
+
+build:m32 --copt=-m32 --linkopt=-m32
+build:asan --copt=-fsanitize=address --linkopt=-fsanitize=address
+build:msan --copt=-fsanitize=memory --linkopt=-fsanitize=memory
+
+# For Valgrind, we have to disable checks of "possible" leaks because the Python
+# interpreter does the sorts of things that flag Valgrind "possible" leak checks.
+# Ideally we could enforce a stricter check for the non-Python tests, but I don't
+# know of an easy way to do that.
+#
+# We also have to disable pymalloc to avoid triggering Valgrind.
+build:valgrind --run_under='valgrind --leak-check=full --track-origins=yes --trace-children=yes --show-leak-kinds=all --error-exitcode=1 --num-callers=500 ' --action_env=PYTHONMALLOC=malloc
+
+build:ubsan --copt=-fsanitize=undefined --linkopt=-fsanitize=undefined --action_env=UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1
+# Workaround for the fact that Bazel links with $CC, not $CXX
+# https://github.com/bazelbuild/bazel/issues/11122#issuecomment-613746748
+build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr
+# Workaround for https://bugs.llvm.org/show_bug.cgi?id=16404
+build:ubsan --linkopt=--rtlib=compiler-rt --linkopt=-lunwind
+
+build:Werror --copt=-Werror
+build:Werror --per_file_copt=json/parser@-Wno-error
+build:Werror --per_file_copt=com_google_protobuf@-Wno-error
+
+# GCC's -fanalyzer, a deeper static analysis than normal warnings.
+build:analyzer --copt=-fanalyzer --copt=-Werror
+build:analyzer --per_file_copt=json/parser@-fno-analyzer
+build:analyzer --per_file_copt=com_google_protobuf@-fno-analyzer
+build:analyzer --per_file_copt=com_github_google_benchmark@-fno-analyzer
+
+# --config=asan-libfuzzer
+build:asan-libfuzzer --action_env=CC=clang
+build:asan-libfuzzer --action_env=CXX=clang++
+build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
+build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
+build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=asan
+
+# --config=msan-libfuzzer
+build:msan-libfuzzer --action_env=CC=clang
+build:msan-libfuzzer --action_env=CXX=clang++
+build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
+build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
+build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=msan
+
+# --config=ubsan-libfuzzer
+build:ubsan-libfuzzer --action_env=CC=clang
+build:ubsan-libfuzzer --action_env=CXX=clang++
+build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
+build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
+build:ubsan-libfuzzer --copt=-fsanitize=undefined
+build:ubsan-libfuzzer --linkopt=-fsanitize=undefined
+build:ubsan-libfuzzer --linkopt=-fsanitize-link-c++-runtime

third_party/upb/.clang-format

@@ -0,0 +1,3 @@
+BasedOnStyle: Google
+DerivePointerAlignment: false
+PointerAlignment: Left

third_party/upb/.github/actions/setup-bazel-cache/action.yml

@@ -0,0 +1,21 @@
+name: "Setup Bazel Caching"
+description: "Sets up Bazel caching"
+inputs:
+  cache_url:
+    description: "URL of the Bazel cache to read/write"
+    required: false
+    default: https://storage.googleapis.com/protobuf-bazel-cache/upb
+  read_only:
+    description: "If true, we can read from the cache but not write it."
+    required: false
+    default: ${{ github.event.pull_request.head.repo.full_name != 'protocolbuffers/upb' }}
+outputs:
+  cache_args:
+    description: "Caching related arguments to pass to 'bazel build"
+    value: --remote_cache=${{ inputs.cache_url }} ${{ steps.set_auth_arg.outputs.auth_arg }}
+runs:
+  using: "composite"
+  steps:
+    - id: set_auth_arg
+      run: echo auth_arg=${{ inputs.read_only == 'true' && '--remote_upload_local_results=false' || '--google_default_credentials' }} >> $GITHUB_OUTPUT
+      shell: bash

third_party/upb/.github/workflows/bazel_tests.yml

@@ -0,0 +1,101 @@
+name: Bazel Tests
+
+on:
+  push:
+    branches:
+      - main
+      - '[0-9]+.x'
+  pull_request:
+    branches:
+      - main
+      - '[0-9]+.x'
+  workflow_dispatch:
+
+jobs:
+
+  ubuntu:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        include:
+          - { NAME: "Fastbuild", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "" }
+          - { NAME: "Optimized", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "-c opt" }  # Some warnings only fire with -c opt
+          - { NAME: "GCC Optimized", BAZEL: bazel, CC: gcc-12, os: ubuntu-22.04, flags: "-c opt" }
+          - { NAME: "FastTable", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "--//:fasttable_enabled=true -- -cmake:test_generated_files" }
+          - { NAME: "ASAN", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "--config=asan -c dbg -- -benchmarks:benchmark -python/..." }
+          - { NAME: "UBSAN", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "--config=ubsan -c dbg -- -benchmarks:benchmark -python/... -lua/...", install: "libunwind-dev" }
+          - { NAME: "32-bit", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "--copt=-m32 --linkopt=-m32 -- -... benchmarks:benchmark ", install: "g++-multilib" }
+          - { NAME: "Windows", BAZEL: bazel, os: windows-2019, startup-flags: "--output_user_root=C:/tmp", flags: "--config=cpp17_msvc", targets: "upb/... upbc/... python/... protos/... protos_generator/..." }
+          - { NAME: "macOS", BAZEL: bazel, CC: clang, os: macos-11 }
+          # Current github runners are all Intel based, so just build/compile for Apple Silicon to detect issues there.
+          - { NAME: "macOS ARM (build only)", BAZEL: bazel, BAZEL_CMD: build, CC: clang, os: macos-11, flags: "--cpu=darwin_arm64"}
+          # We support two Bazel versions back per https://opensource.google/documentation/policies/cplusplus-support
+          - { NAME: "Bazel 5.3.0", BAZEL: bazel-5.3.0-linux-x86_64, CC: clang, os: ubuntu-20-large }
+          - { NAME: "Bazel 6.1.0", BAZEL: bazel-6.1.0-linux-x86_64, CC: clang, os: ubuntu-20-large }
+
+    name: ${{ matrix.NAME }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Cloud SDK
+        uses: google-github-actions/auth@v0
+        with:
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+          export_environment_variables: true
+        if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }}
+      - name: Download historical Bazel version
+        run: |
+             FILENAME=$HOME/bin/${{ matrix.BAZEL }}
+             VERSION=$(echo ${{ matrix.BAZEL }} | cut -d- -f 2 )
+             mkdir -p $HOME/bin
+             echo $HOME/bin >> $GITHUB_PATH
+             wget -O $FILENAME https://github.com/bazelbuild/bazel/releases/download/$VERSION/${{ matrix.BAZEL }}
+             chmod a+x $FILENAME
+        if: ${{ matrix.BAZEL != 'bazel' }}
+      - name: Check compiler versions
+        if: matrix.CC
+        run: ${{ matrix.CC }} --version
+      - name: Check Bazel versions
+        run: ${{ matrix.BAZEL }} --version
+      - id: bazel-cache
+        name: Set up Bazel caching
+        uses: ./.github/actions/setup-bazel-cache
+      - name: Setup Python venv
+        if: ${{ runner.os != 'Windows' }}
+        run: rm -rf /tmp/venv && python3 -m venv /tmp/venv && source /tmp/venv/bin/activate && python3 --version
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y ${{ matrix.install }}
+        if: matrix.install != ''
+      - name: Install numpy
+        run: pip3 install numpy
+      - name: Setup environment variables
+        if: matrix.CC
+        run: echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV
+      - name: Run tests
+        run: cd ${{ github.workspace }} && ${{ matrix.BAZEL }} ${{ matrix.startup-flags }} ${{ matrix.BAZEL_CMD || 'test' }} --test_output=errors ${{ steps.bazel-cache.outputs.cache_args }} ${{ matrix.targets || '...' }} ${{ matrix.flags }}
+
+  no-python:
+    runs-on: ubuntu-20-large
+
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+
+    name: "No System Python"
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Cloud SDK
+        uses: google-github-actions/auth@v0
+        with:
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+          export_environment_variables: true
+        if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }}
+      - id: bazel-cache
+        name: Set up Bazel caching
+        uses: ./.github/actions/setup-bazel-cache
+      - name: Uninstall python
+        run: which python3 && sudo mv `which python3` /tmp && ! which python3
+      - name: Run tests
+        run: cd ${{ github.workspace }} && bazel test --test_output=errors ${{ steps.bazel-cache.outputs.cache_args }} //python/... -- -//python/dist:source_wheel

third_party/upb/.github/workflows/clang_format.yml

@@ -0,0 +1,22 @@
+name: Check ClangFormat
+
+on:
+  push:
+    branches:
+      - main
+      - '[0-9]+.x'
+  pull_request:
+    branches:
+      - main
+      - '[0-9]+.x'
+  workflow_dispatch:
+
+jobs:
+  check_clang_format:
+    runs-on: ubuntu-20-large
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run ClangFormat
+        run: find . | grep -E '\.(c|h|cc)$' | grep -E -v '^./(third_party|cmake)' | xargs clang-format -i
+      - name: Check for differences
+        run: git diff --exit-code

third_party/upb/.github/workflows/generate_files.yml

@@ -0,0 +1,24 @@
+name: Generate Files
+
+# After any push to the main branch, re-generate pregenerated files.
+on:
+  push:
+    branches:
+      - main
+      - '[0-9]+.x'
+
+jobs:
+  generate:
+    if: github.repository == 'protocolbuffers/upb'
+    runs-on: ubuntu-22-large
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Note: this token has an expiration date, so if the workflow starts
+          # failing then you may need to generate a fresh token.
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+      - name: Configure name and email address in Git
+        run: cd ${{ github.workspace }} && git config user.name "Protobuf Team Bot" && git config user.email "[email protected]"
+      - name: Commit and push update
+        run: cd ${{ github.workspace }} && ./cmake/push_auto_update.sh

third_party/upb/.github/workflows/mergeable.yml

@@ -0,0 +1,15 @@
+mergeable:
+  pull_requests:
+    label:
+      and:
+        - must_exclude:
+            regex: '^disposition/DO NOT MERGE'
+            message: 'Pull request marked not mergeable'
+        - must_include:
+            regex: 'mergeable:force-allow'
+            message: 'Pull requests should not be merged directly and should instead
+              be handled by Copybara.
+
+              To enable Github merges, add the `mergeable:force-allow` label and get a second 
+              approval. This should almost never be used except for releases or as a break glass 
+              measure after discussing with the team.'