Merge cockroachdb#83604 cockroachdb#85386 cockroachdb#85624

83604: sql: Support privileges on virtual tables r=rafiss a=RichardJCai

Caching is added in this PR. `cacheutil.cache` is added to generalize the caching work done for role membership cache and userAuth cache. 

There is more testing work to be done on the cache + migration work to add an entry for `public` having `SELECT` on every virtual table.

Release note (sql change): Support privileges on virtual tables.
Previously users were unable to GRANT on virtual tables, ie
tables in crdb_internal / pg_catalog / information_schema.

Now users can GRANT/REVOKE `SELECT` privilege on virtual tables.
`SELECT` is needed to query a virtual table.

85386: logictest: add TestLogic_tmp for testing untracked logic test files r=mgartner a=mgartner

This commit improves the developer experience when working with logic
tests. The logic test generator no longer generates a dedicated test for
logic tests files prefixed with "_". The hard-coded `TestLogic_tmp` test
has been added which runs all such logic test files. This allows
developers to create and run temporary test files that are not checked
into the repository, without repeatedly regenerating and reverting
changes to the `generated_test.go` files.

For example, if you add a git-ignored file logic test file,
`pkg/sql/logictest/testdata/logic_test/__test`, you can run it with the
`local` config with the command:

    ./dev test pkg/sql/logictest/tests/local -f TestLogic_tmp

Previously, to facilitate running these types of tests, `./dev test`
would automatically regenerate logic tests every time tests were run.
This was primarily motivated by the desire to make testing temporary
logic test files easier - a developer would not have to run
`./dev gen logictest` every time they wanted to test a temporary file.

However, this was a poor experience because it made running tests
significantly slower, and logic tests generated for temporary logic test
files would have to be reverted repeatedly. `TestLogic_tmp` eliminates
the need for this automatic regeneration, so it has been removed. The
`--no-gen` flag, which disabled automatic regeneration, has also been
removed. When committing a new logic test file to the repository,
developers must now run `./dev gen testlogic` to generate logic tests
for it.

Release note: None

85624: roachtest: extend bounds for tpch_concurrency with high refresh spans bytes r=yuzefovich a=yuzefovich

We recently fixed several omissions around the way we track the refresh
spans memory usage, so this roachtest now hits the upper bound
consistently. This commit removes the lowered bounds for one config, and
we'll follow the behavior for a week or so - fingers crossed, we can
remove lowering of the setting.

Release note: None

Co-authored-by: richardjcai <[email protected]>
Co-authored-by: Marcus Gartner <[email protected]>
Co-authored-by: Yahor Yuzefovich <[email protected]>

pkg/BUILD.bazel

@@ -255,6 +255,7 @@ ALL_TESTS = [
     "//pkg/spanconfig/spanconfigtestutils:spanconfigtestutils_test",
     "//pkg/spanconfig:spanconfig_test",
     "//pkg/sql/backfill:backfill_test",
+    "//pkg/sql/cacheutil:cacheutil_test",
     "//pkg/sql/catalog/catalogkeys:catalogkeys_test",
     "//pkg/sql/catalog/catformat:catformat_test",
     "//pkg/sql/catalog/catpb:catpb_disallowed_imports_test",
@@ -1269,6 +1270,8 @@ GO_TARGETS = [
     "//pkg/spanconfig:spanconfig_test",
     "//pkg/sql/backfill:backfill",
     "//pkg/sql/backfill:backfill_test",
+    "//pkg/sql/cacheutil:cacheutil",
+    "//pkg/sql/cacheutil:cacheutil_test",
     "//pkg/sql/catalog/bootstrap:bootstrap",
     "//pkg/sql/catalog/catalogkeys:catalogkeys",
     "//pkg/sql/catalog/catalogkeys:catalogkeys_test",
@@ -2435,6 +2438,7 @@ GET_X_DATA_TARGETS = [
     "//pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster:get_x_data",
     "//pkg/sql:get_x_data",
     "//pkg/sql/backfill:get_x_data",
+    "//pkg/sql/cacheutil:get_x_data",
     "//pkg/sql/catalog:get_x_data",
     "//pkg/sql/catalog/bootstrap:get_x_data",
     "//pkg/sql/catalog/catalogkeys:get_x_data",

pkg/ccl/backupccl/restore_old_versions_test.go

@@ -246,6 +246,7 @@ ORDER BY object_type, object_name`, [][]string{
 	})
 
 	t.Run("zoneconfig_privilege_restore", func(t *testing.T) {
+		skip.WithIssue(t, 84359)
 		dirs, err := ioutil.ReadDir(privilegeDirs)
 		require.NoError(t, err)
 		for _, dir := range dirs {

pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant

@@ -8,11 +8,9 @@ SELECT count(distinct(node_id)), count(*)  FROM crdb_internal.node_runtime_info
 query IT
 SELECT node_id, name FROM crdb_internal.leases ORDER BY name
 ----
-0  defaultdb
 0  eventlog
 0  jobs
 0  locations
-0  privileges
 0  protected_ts_meta
 0  protected_ts_records
 0  role_members
@@ -36,76 +34,76 @@ DROP DATABASE crdb_internal
 query TTTTIT
 SHOW TABLES FROM crdb_internal
 ----
-crdb_internal  active_range_feeds               table  NULL  NULL  NULL
-crdb_internal  backward_dependencies            table  NULL  NULL  NULL
-crdb_internal  builtin_functions                table  NULL  NULL  NULL
-crdb_internal  cluster_contended_indexes        view   NULL  NULL  NULL
-crdb_internal  cluster_contended_keys           view   NULL  NULL  NULL
-crdb_internal  cluster_contended_tables         view   NULL  NULL  NULL
-crdb_internal  cluster_contention_events        table  NULL  NULL  NULL
-crdb_internal  cluster_database_privileges      table  NULL  NULL  NULL
-crdb_internal  cluster_distsql_flows            table  NULL  NULL  NULL
-crdb_internal  cluster_inflight_traces          table  NULL  NULL  NULL
-crdb_internal  cluster_locks                    table  NULL  NULL  NULL
-crdb_internal  cluster_queries                  table  NULL  NULL  NULL
-crdb_internal  cluster_sessions                 table  NULL  NULL  NULL
-crdb_internal  cluster_settings                 table  NULL  NULL  NULL
-crdb_internal  cluster_statement_statistics     table  NULL  NULL  NULL
-crdb_internal  cluster_transaction_statistics   table  NULL  NULL  NULL
-crdb_internal  cluster_transactions             table  NULL  NULL  NULL
-crdb_internal  create_schema_statements         table  NULL  NULL  NULL
-crdb_internal  create_statements                table  NULL  NULL  NULL
-crdb_internal  create_type_statements           table  NULL  NULL  NULL
-crdb_internal  cross_db_references              table  NULL  NULL  NULL
-crdb_internal  databases                        table  NULL  NULL  NULL
-crdb_internal  default_privileges               table  NULL  NULL  NULL
-crdb_internal  feature_usage                    table  NULL  NULL  NULL
-crdb_internal  forward_dependencies             table  NULL  NULL  NULL
-crdb_internal  gossip_alerts                    table  NULL  NULL  NULL
-crdb_internal  gossip_liveness                  table  NULL  NULL  NULL
-crdb_internal  gossip_network                   table  NULL  NULL  NULL
-crdb_internal  gossip_nodes                     table  NULL  NULL  NULL
-crdb_internal  index_columns                    table  NULL  NULL  NULL
-crdb_internal  index_usage_statistics           table  NULL  NULL  NULL
-crdb_internal  invalid_objects                  table  NULL  NULL  NULL
-crdb_internal  jobs                             table  NULL  NULL  NULL
-crdb_internal  kv_node_liveness                 table  NULL  NULL  NULL
-crdb_internal  kv_node_status                   table  NULL  NULL  NULL
-crdb_internal  kv_store_status                  table  NULL  NULL  NULL
-crdb_internal  leases                           table  NULL  NULL  NULL
-crdb_internal  lost_descriptors_with_data       table  NULL  NULL  NULL
-crdb_internal  node_build_info                  table  NULL  NULL  NULL
-crdb_internal  node_contention_events           table  NULL  NULL  NULL
-crdb_internal  node_distsql_flows               table  NULL  NULL  NULL
-crdb_internal  node_execution_insights          table  NULL  NULL  NULL
-crdb_internal  node_inflight_trace_spans        table  NULL  NULL  NULL
-crdb_internal  node_metrics                     table  NULL  NULL  NULL
-crdb_internal  node_queries                     table  NULL  NULL  NULL
-crdb_internal  node_runtime_info                table  NULL  NULL  NULL
-crdb_internal  node_sessions                    table  NULL  NULL  NULL
-crdb_internal  node_statement_statistics        table  NULL  NULL  NULL
-crdb_internal  node_transaction_statistics      table  NULL  NULL  NULL
-crdb_internal  node_transactions                table  NULL  NULL  NULL
-crdb_internal  node_txn_stats                   table  NULL  NULL  NULL
-crdb_internal  partitions                       table  NULL  NULL  NULL
-crdb_internal  pg_catalog_table_is_implemented  table  NULL  NULL  NULL
-crdb_internal  predefined_comments              table  NULL  NULL  NULL
-crdb_internal  ranges                           view   NULL  NULL  NULL
-crdb_internal  ranges_no_leases                 table  NULL  NULL  NULL
-crdb_internal  regions                          table  NULL  NULL  NULL
-crdb_internal  schema_changes                   table  NULL  NULL  NULL
-crdb_internal  session_trace                    table  NULL  NULL  NULL
-crdb_internal  session_variables                table  NULL  NULL  NULL
-crdb_internal  statement_statistics             view   NULL  NULL  NULL
-crdb_internal  super_regions                    table  NULL  NULL  NULL
-crdb_internal  table_columns                    table  NULL  NULL  NULL
-crdb_internal  table_indexes                    table  NULL  NULL  NULL
-crdb_internal  table_row_statistics             table  NULL  NULL  NULL
-crdb_internal  tables                           table  NULL  NULL  NULL
-crdb_internal  tenant_usage_details             view   NULL  NULL  NULL
-crdb_internal  transaction_contention_events    table  NULL  NULL  NULL
-crdb_internal  transaction_statistics           view   NULL  NULL  NULL
-crdb_internal  zones                            table  NULL  NULL  NULL
+crdb_internal  active_range_feeds               table  admin  NULL  NULL
+crdb_internal  backward_dependencies            table  admin  NULL  NULL
+crdb_internal  builtin_functions                table  admin  NULL  NULL
+crdb_internal  cluster_contended_indexes        view   admin  NULL  NULL
+crdb_internal  cluster_contended_keys           view   admin  NULL  NULL
+crdb_internal  cluster_contended_tables         view   admin  NULL  NULL
+crdb_internal  cluster_contention_events        table  admin  NULL  NULL
+crdb_internal  cluster_database_privileges      table  admin  NULL  NULL
+crdb_internal  cluster_distsql_flows            table  admin  NULL  NULL
+crdb_internal  cluster_inflight_traces          table  admin  NULL  NULL
+crdb_internal  cluster_locks                    table  admin  NULL  NULL
+crdb_internal  cluster_queries                  table  admin  NULL  NULL
+crdb_internal  cluster_sessions                 table  admin  NULL  NULL
+crdb_internal  cluster_settings                 table  admin  NULL  NULL
+crdb_internal  cluster_statement_statistics     table  admin  NULL  NULL
+crdb_internal  cluster_transaction_statistics   table  admin  NULL  NULL
+crdb_internal  cluster_transactions             table  admin  NULL  NULL
+crdb_internal  create_schema_statements         table  admin  NULL  NULL
+crdb_internal  create_statements                table  admin  NULL  NULL
+crdb_internal  create_type_statements           table  admin  NULL  NULL
+crdb_internal  cross_db_references              table  admin  NULL  NULL
+crdb_internal  databases                        table  admin  NULL  NULL
+crdb_internal  default_privileges               table  admin  NULL  NULL
+crdb_internal  feature_usage                    table  admin  NULL  NULL
+crdb_internal  forward_dependencies             table  admin  NULL  NULL
+crdb_internal  gossip_alerts                    table  admin  NULL  NULL
+crdb_internal  gossip_liveness                  table  admin  NULL  NULL
+crdb_internal  gossip_network                   table  admin  NULL  NULL
+crdb_internal  gossip_nodes                     table  admin  NULL  NULL
+crdb_internal  index_columns                    table  admin  NULL  NULL
+crdb_internal  index_usage_statistics           table  admin  NULL  NULL
+crdb_internal  invalid_objects                  table  admin  NULL  NULL
+crdb_internal  jobs                             table  admin  NULL  NULL
+crdb_internal  kv_node_liveness                 table  admin  NULL  NULL
+crdb_internal  kv_node_status                   table  admin  NULL  NULL
+crdb_internal  kv_store_status                  table  admin  NULL  NULL
+crdb_internal  leases                           table  admin  NULL  NULL
+crdb_internal  lost_descriptors_with_data       table  admin  NULL  NULL
+crdb_internal  node_build_info                  table  admin  NULL  NULL
+crdb_internal  node_contention_events           table  admin  NULL  NULL
+crdb_internal  node_distsql_flows               table  admin  NULL  NULL
+crdb_internal  node_execution_insights          table  admin  NULL  NULL
+crdb_internal  node_inflight_trace_spans        table  admin  NULL  NULL
+crdb_internal  node_metrics                     table  admin  NULL  NULL
+crdb_internal  node_queries                     table  admin  NULL  NULL
+crdb_internal  node_runtime_info                table  admin  NULL  NULL
+crdb_internal  node_sessions                    table  admin  NULL  NULL
+crdb_internal  node_statement_statistics        table  admin  NULL  NULL
+crdb_internal  node_transaction_statistics      table  admin  NULL  NULL
+crdb_internal  node_transactions                table  admin  NULL  NULL
+crdb_internal  node_txn_stats                   table  admin  NULL  NULL
+crdb_internal  partitions                       table  admin  NULL  NULL
+crdb_internal  pg_catalog_table_is_implemented  table  admin  NULL  NULL
+crdb_internal  predefined_comments              table  admin  NULL  NULL
+crdb_internal  ranges                           view   admin  NULL  NULL
+crdb_internal  ranges_no_leases                 table  admin  NULL  NULL
+crdb_internal  regions                          table  admin  NULL  NULL
+crdb_internal  schema_changes                   table  admin  NULL  NULL
+crdb_internal  session_trace                    table  admin  NULL  NULL
+crdb_internal  session_variables                table  admin  NULL  NULL
+crdb_internal  statement_statistics             view   admin  NULL  NULL
+crdb_internal  super_regions                    table  admin  NULL  NULL
+crdb_internal  table_columns                    table  admin  NULL  NULL
+crdb_internal  table_indexes                    table  admin  NULL  NULL
+crdb_internal  table_row_statistics             table  admin  NULL  NULL
+crdb_internal  tables                           table  admin  NULL  NULL
+crdb_internal  tenant_usage_details             view   admin  NULL  NULL
+crdb_internal  transaction_contention_events    table  admin  NULL  NULL
+crdb_internal  transaction_statistics           view   admin  NULL  NULL
+crdb_internal  zones                            table  admin  NULL  NULL
 
 statement ok
 CREATE DATABASE testdb; CREATE TABLE testdb.foo(x INT)