Merge #79808 #79810

79808: bazel: use pre-built archives of the c-deps r=mari-crl a=rickystewart Up until this point we've required building all of the `c-deps` via `rules_foreign_cc`. This has a couple downsides: the build is not parallelized (#77207), and while in principle developers should be able to build it once and then reuse the cached version forever, in practice this is not really the case (#76851). To this point we pre-build these libraries and add the requisite glue so that Bazel can download and link them in. 1. Rename all the `c-deps:libX` targets to `c-deps:libX_foreign`. 2. `libproj`, `libjemalloc`, and `libkrb5` are updated to compile w/ `-fPIC` unless `force_build_cdeps` is set. 3. Add a script `c-deps/buildcdeps.sh` that builds all the `libX_foreign` libraries in the various cross-configurations. 4. Add a script `build-and-publish-cdeps.sh` that can be invoked from TeamCity to build all the `c-deps` and push them to GCS. 5. Add `c-deps/archived.bzl` to wrap all the pre-compiled libraries from above. 6. Now `libgeos` is no longer (necessarily) found in `bazel-bin/c-deps/libgeos`, so we need to update a few places so it can continue to be found: `bazci`, `publish-artifacts`, and `publish-provisional-artifacts`, and `pkg/geos`. You can still force building the `c-deps` from scratch with `--force_build_cdeps`. Doing so will compile the `c-deps` WITHOUT `-fPIC`; the release pipeline is updated to use this to avoid any performance regression. You still must compile the `c-deps` from scratch on unsupported operating systems (OS's other than Linux, macOS, and Windows). Closes #77135. 79810: sql: enforce admin role for resetting sql stats and index usage stats r=maryliag a=Azhng Resolves #79688 Previously, SQL Stats and Index Usage Stats can be reset through SQL CLI using crdb_internal.reset_sql_stats() and crdb_internal.reset_index_usage_stats() builtins. However, these two builtins were not checking for users admin role. Hence, any user can reset SQL Stats and Index Usage Stats. This commit enforces the permission check. Release note (security update): crdb_internal.reset_sql_stats() and crdb_internal.reset_index_usage_stats() builtins now check if user has admin role. Co-authored-by: Ricky Stewart <[email protected]> Co-authored-by: Azhng <[email protected]>
cockroachdb · Apr 20, 2022 · 0dd5073 · 0dd5073
3 parents c50ffb4 + e0e302f + aeab727
commit 0dd5073
Show file tree

Hide file tree

Showing 23 changed files with 635 additions and 184 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -4,13 +4,15 @@ build --flag_alias=crdb_test=//build/toolchains:crdb_test_flag
 build --flag_alias=crdb_test_off=//build/toolchains:crdb_test_off_flag
 build --flag_alias=cross=//build/toolchains:cross_flag
 build --flag_alias=dev=//build/toolchains:dev_flag
+build --flag_alias=force_build_cdeps=//build/toolchains:force_build_cdeps_flag
 build --flag_alias=lintonbuild=//build/toolchains:nogo_flag
 build --flag_alias=nolintonbuild=//build/toolchains:nonogo_explicit_flag
 build --flag_alias=with_ui=//pkg/ui:with_ui_flag
 
 build:crdb_test_off --crdb_test_off
 build:cross --cross
 build:dev --dev
+build:force_build_cdeps --force_build_cdeps
 build:lintonbuild --lintonbuild
 build:nolintonbuild --nolintonbuild
 # Note: nonogo is classically the name of the nolintonbuild configuration.

diff --git a/build/teamcity/internal/release/build-and-publish-cdeps.sh b/build/teamcity/internal/release/build-and-publish-cdeps.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+google_credentials="$GOOGLE_EPHEMERAL_CREDENTIALS"
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-support.sh"  # for root
+source "$dir/teamcity-bazel-support.sh"  # for run_bazel
+log_into_gcloud
+
+set -x
+
+tc_start_block "Build c-deps"
+run_bazel c-deps/buildcdeps.sh
+tc_end_block "Build c-deps"
+
+tc_start_block "Publish artifacts"
+loc=$(date +%Y%m%d-%H%M%S)
+gsutil cp -r $root/artifacts gs://public-bazel-artifacts/c-deps/$loc
+tc_end_block "Publish artifacts"
diff --git a/build/toolchains/BUILD.bazel b/build/toolchains/BUILD.bazel
@@ -432,3 +432,39 @@ config_setting(
     },
     visibility = ["//build/bazelutil:__pkg__"],
 )
+
+bool_flag(
+    name = "force_build_cdeps_flag",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "force_build_cdeps",
+    flag_values = {
+        ":force_build_cdeps_flag": "true",
+    },
+    visibility = ["//c-deps:__pkg__"],
+)
+
+config_setting(
+    name = "is_macos_force_build_cdeps",
+    constraint_values = [
+        "@io_bazel_rules_go//go/toolchain:darwin",
+    ],
+    flag_values = {
+        ":force_build_cdeps_flag": "true",
+    },
+    visibility = ["//c-deps:__pkg__"],
+)
+
+config_setting(
+    name = "is_windows_force_build_cdeps",
+    constraint_values = [
+        "@io_bazel_rules_go//go/toolchain:windows",
+    ],
+    flag_values = {
+        ":force_build_cdeps_flag": "true",
+    },
+    visibility = ["//c-deps:__pkg__"],
+)
diff --git a/c-deps/BUILD.bazel b/c-deps/BUILD.bazel
@@ -1,10 +1,12 @@
 # Load the components that lets us use cmake/make in third party deps.
 load("@rules_foreign_cc//foreign_cc:cmake.bzl", "cmake")
 load("@rules_foreign_cc//foreign_cc:configure.bzl", "configure_make")
+load("archived.bzl", "LIBKRB5_LIBS", "archived_cdeps", "cdep_alias")
+
+archived_cdeps()
 
-# Define the build target for libjemalloc.
 configure_make(
-    name = "libjemalloc",
+    name = "libjemalloc_foreign",
     autoconf = True,
     configure_in_place = True,
     configure_options = [
@@ -28,6 +30,10 @@ configure_make(
         ],
         "//conditions:default": [],
     }),
+    copts = select({
+        "//build/toolchains:force_build_cdeps": [],
+        "//conditions:default": ["-fPIC"],
+    }),
     env = select({
         "//build/toolchains:dev": {"AR": ""},
         "//conditions:default": {},
@@ -45,26 +51,47 @@ configure_make(
     visibility = ["//visibility:public"],
 )
 
-# Define the build target for libproj.
+cdep_alias("libjemalloc")
+
 cmake(
-    name = "libproj",
+    name = "libproj_foreign",
     cache_entries = select({
         # TODO(ricky): The repetition here is dumb, but I don't know a cleaner
         # way to do it?
         # https://github.com/bazelbuild/bazel/issues/12457 would help.
+        "//build/toolchains:is_windows_force_build_cdeps": {
+            "BUILD_LIBPROJ_SHARED": "OFF",
+            "CMAKE_BUILD_TYPE": "Release",
+            "CMAKE_SYSTEM_NAME": "Generic",
+        },
         "@io_bazel_rules_go//go/platform:windows": {
+            "BUILD_LIBPROJ_SHARED": "OFF",
+            "CMAKE_BUILD_TYPE": "Release",
+            "CMAKE_C_FLAGS": "-fPIC",
+            "CMAKE_CXX_FLAGS": "-fPIC",
+            "CMAKE_SYSTEM_NAME": "Generic",
+        },
+        "//build/toolchains:is_macos_force_build_cdeps": {
             "BUILD_LIBPROJ_SHARED": "OFF",
             "CMAKE_BUILD_TYPE": "Release",
             "CMAKE_SYSTEM_NAME": "Generic",
         },
         "@io_bazel_rules_go//go/platform:darwin": {
             "BUILD_LIBPROJ_SHARED": "OFF",
             "CMAKE_BUILD_TYPE": "Release",
+            "CMAKE_C_FLAGS": "-fPIC",
+            "CMAKE_CXX_FLAGS": "-fPIC",
             "CMAKE_SYSTEM_NAME": "Generic",
         },
+        "//build/toolchains:force_build_cdeps": {
+            "BUILD_LIBPROJ_SHARED": "OFF",
+            "CMAKE_BUILD_TYPE": "Release",
+        },
         "//conditions:default": {
             "BUILD_LIBPROJ_SHARED": "OFF",
             "CMAKE_BUILD_TYPE": "Release",
+            "CMAKE_C_FLAGS": "-fPIC",
+            "CMAKE_CXX_FLAGS": "-fPIC",
         },
     }),
     generate_args = ["-GUnix Makefiles"],
@@ -73,9 +100,10 @@ cmake(
     visibility = ["//visibility:public"],
 )
 
-# Define the targets for libgeos.
+cdep_alias("libproj")
+
 cmake(
-    name = "libgeos",
+    name = "libgeos_foreign",
     cache_entries = select({
         "@io_bazel_rules_go//go/platform:windows": {
             "CMAKE_BUILD_TYPE": "Release",
@@ -96,13 +124,15 @@ cmake(
         },
     }),
     data = select({
+        # TODO: This select needs 4 cases to handle the differing host platforms,
+        # not just these 2 based on the target.
         "//build/toolchains:is_cross_macos_x86_64": [
             "@cross_x86_64_macos_toolchain//:bin/x86_64-apple-darwin21.2-install_name_tool",
             "@cross_x86_64_macos_toolchain//:bin/x86_64-apple-darwin21.2-otool",
         ],
         "//build/toolchains:is_cross_macos_arm64": [
-            "@cross_arm64_macos_toolchain//:bin/arm64-apple-darwin21.2-install_name_tool",
-            "@cross_arm64_macos_toolchain//:bin/arm64-apple-darwin21.2-otool",
+            "@cross_x86_64_macos_arm_toolchain//:bin/arm64-apple-darwin21.2-install_name_tool",
+            "@cross_x86_64_macos_arm_toolchain//:bin/arm64-apple-darwin21.2-otool",
         ],
         "//conditions:default": [],
     }),
@@ -112,8 +142,8 @@ cmake(
             "OTOOL": "$(execpath @cross_x86_64_macos_toolchain//:bin/x86_64-apple-darwin21.2-otool)",
         },
         "//build/toolchains:is_cross_macos_arm64": {
-            "CMAKE_INSTALL_NAME_TOOL": "$(execpath @cross_arm64_macos_toolchain//:bin/arm64-apple-darwin21.2-install_name_tool)",
-            "OTOOL": "$(execpath @cross_arm64_macos_toolchain//:bin/arm64-apple-darwin21.2-otool)",
+            "CMAKE_INSTALL_NAME_TOOL": "$(execpath @cross_x86_64_macos_arm_toolchain//:bin/arm64-apple-darwin21.2-install_name_tool)",
+            "OTOOL": "$(execpath @cross_x86_64_macos_arm_toolchain//:bin/arm64-apple-darwin21.2-otool)",
         },
         "//conditions:default": {},
     }),
@@ -159,9 +189,10 @@ cmake(
     visibility = ["//visibility:public"],
 )
 
-# Define the build target for kerberos.
+cdep_alias("libgeos")
+
 configure_make(
-    name = "libkrb5",
+    name = "libkrb5_foreign",
     autoreconf = True,
     autoreconf_directory = "src",
     autoreconf_options = [
@@ -177,8 +208,14 @@ configure_make(
         "@io_bazel_rules_go//go/platform:linux_arm64": ["--host=aarch64-unknown-linux-gnu"],
         "//conditions:default": [],
     }),
-    # We specify -fcommon to get around duplicate definition errors in recent gcc.
-    copts = ["-fcommon"],
+    copts = select({
+        # NB: We specify -fcommon to get around duplicate definition errors in recent gcc.
+        "//build/toolchains:force_build_cdeps": ["-fcommon"],
+        "//conditions:default": [
+            "-fcommon",
+            "-fPIC",
+        ],
+    }),
     data = [":autom4te"],
     env = select({
         "//build/toolchains:cross": {
@@ -192,13 +229,7 @@ configure_make(
         },
     }),
     lib_source = "@krb5//:all",
-    out_static_libs = [
-        "libgssapi_krb5.a",
-        "libkrb5.a",
-        "libkrb5support.a",
-        "libk5crypto.a",
-        "libcom_err.a",
-    ],
+    out_static_libs = LIBKRB5_LIBS,
     postfix_script = ("""mkdir -p libkrb5/lib
 cp lib/libcom_err.a libkrb5/lib
 cp lib/libgssapi_krb5.a libkrb5/lib
@@ -210,25 +241,4 @@ cp include/gssapi/gssapi.h libkrb5/include/gssapi"""),
     visibility = ["//visibility:public"],
 )
 
-# This is extremely stupid and unnecessary, but in certain cases to depend on
-# the output of a `cmake` target, we need to launder through a filegroup:
-# https://github.com/bazelbuild/rules_foreign_cc/issues/619#issuecomment-844473637
-# This is apparently a bug. In the meantime, people can depend on the :*_files
-# targets rather than :libgeos where it matters.
-filegroup(
-    name = "libgeos_files",
-    srcs = [":libgeos"],
-    visibility = ["//visibility:public"],
-)
-
-filegroup(
-    name = "libproj_files",
-    srcs = [":libproj"],
-    visibility = ["//visibility:public"],
-)
-
-filegroup(
-    name = "libedit_files",
-    srcs = ["@com_github_knz_go_libedit//unix:edit"],
-    visibility = ["//visibility:public"],
-)
+cdep_alias("libkrb5")
diff --git a/c-deps/REPOSITORIES.bzl b/c-deps/REPOSITORIES.bzl
@@ -1,3 +1,5 @@
+load(":archived.bzl", "archived_cdep_repository")
+
 # c-deps repository definitions.
 
 # Define the repositories for each of our c-dependencies. BUILD_ALL_CONTENT is
@@ -34,3 +36,88 @@ def c_deps():
         path = "c-deps/proj",
         build_file_content = BUILD_ALL_CONTENT,
     )
+    archived_cdep_repository(
+        lib = "libgeos",
+        config = "linux",
+        sha256 = "a9ce2803197243bf3d21557667b4a3b76f08befbb161e32ad2bbabd74029fd60",
+    )
+    archived_cdep_repository(
+        lib = "libjemalloc",
+        config = "linux",
+        sha256 = "e849bd4a7abb1d00270eb1105655be833a6cd9ae8c8b389de32f9b75098bc6e2",
+    )
+    archived_cdep_repository(
+        lib = "libkrb5",
+        config = "linux",
+        sha256 = "db0facc2c38f732cf965fb57c6c236a7316aca576dbeec92504d20d95a88302a",
+    )
+    archived_cdep_repository(
+        lib = "libproj",
+        config = "linux",
+        sha256 = "86bca5abd7c8cc35bb442e86441dacdcf53a13a3d637df6c5182425afff84130",
+    )
+    archived_cdep_repository(
+        lib = "libgeos",
+        config = "linuxarm",
+        sha256 = "eb13317c26d323a366be0be54235364e36fe7fb83628b3b254f53cefecfe1f7c",
+    )
+    archived_cdep_repository(
+        lib = "libjemalloc",
+        config = "linuxarm",
+        sha256 = "00f7c6e7481a6240a284566824b01881caa555a9761fa9d0e0c4123af634fd3d",
+    )
+    archived_cdep_repository(
+        lib = "libkrb5",
+        config = "linuxarm",
+        sha256 = "544ee020c050646b4015248e1baaf815d333e394534edf89ac41c0857be9fac1",
+    )
+    archived_cdep_repository(
+        lib = "libproj",
+        config = "linuxarm",
+        sha256 = "4d8c682aadad5115fa31221c25581fcc0ada51ef20f64e85d1eb0c48a7116b22",
+    )
+    archived_cdep_repository(
+        lib = "libgeos",
+        config = "macos",
+        sha256 = "40119c33bb1bf0bc0c685c1468b9cf720ea71517eb99932907626012bbedfc7d",
+    )
+    archived_cdep_repository(
+        lib = "libjemalloc",
+        config = "macos",
+        sha256 = "16e7ca370de9b7244914259e2810734664b4466167c31c7c5107d16eaaa17d2c",
+    )
+    archived_cdep_repository(
+        lib = "libproj",
+        config = "macos",
+        sha256 = "997e998fb8d19379b7642305a4604bf826217f0381032bb079fcdb1ff8c00708",
+    )
+    archived_cdep_repository(
+        lib = "libgeos",
+        config = "macosarm",
+        sha256 = "e35abe052c0cc15db6328dd7dfad1882a24415d226bd6fe3f26955eee9add11f",
+    )
+    archived_cdep_repository(
+        lib = "libjemalloc",
+        config = "macosarm",
+        sha256 = "6f546e8f2d154f561735aa527c9ee4f12740f8afbd5d5ef87812230eb6bae2ce",
+    )
+    archived_cdep_repository(
+        lib = "libproj",
+        config = "macosarm",
+        sha256 = "9efe655a8751212c76e34e573fdc139e1aa919c3acb06759b089af3472a43571",
+    )
+    archived_cdep_repository(
+        lib = "libgeos",
+        config = "windows",
+        sha256 = "43ea718397688416db884c99880a7785b50156c3d9cbd0176198d219a7e2a25e",
+    )
+    archived_cdep_repository(
+        lib = "libjemalloc",
+        config = "windows",
+        sha256 = "24620658fd5e8b8b767433166af9b6da0ed6afd1ba6e774db5138c1a327c535a",
+    )
+    archived_cdep_repository(
+        lib = "libproj",
+        config = "windows",
+        sha256 = "026c2d888224d54c1a6e13ccaa30f809787eabe3a8b298b51aeb61d712fcca80",
+    )