From a745bef47b315bb11fecab4eeefa2bcc41be5951 Mon Sep 17 00:00:00 2001 From: ZetaSQL Team Date: Fri, 6 Oct 2023 16:33:17 -0700 Subject: [PATCH] Export of internal ZetaSQL changes. -- Change by ZetaSQL Team : No public description -- Change by ZetaSQL Team : Removed the `depth_limit_detector_test_cases_test` from zetasql. -- Change by John Fremlin : Internal change GitOrigin-RevId: e6d88e9515744687cd24e0c05460c1ee362c7f69 Change-Id: I056cb0c1a190714ff4e94715ec8ffe8dbb716013 --- bazel/BUILD | 1 + bazel/textmapper.bzl | 75 ++++ bazel/zetasql_deps_step_1.bzl | 22 ++ bazel/zetasql_deps_step_2.bzl | 6 + zetasql/compliance/BUILD | 36 -- .../depth_limit_detector_test_cases_test.cc | 345 ------------------ zetasql/parser/BUILD | 1 + 7 files changed, 105 insertions(+), 381 deletions(-) create mode 100644 bazel/textmapper.bzl delete mode 100644 zetasql/compliance/depth_limit_detector_test_cases_test.cc diff --git a/bazel/BUILD b/bazel/BUILD index 232922731..7519a279a 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -29,6 +29,7 @@ exports_files([ "bison.bzl", "flex.bzl", "maven_install.json", + "textmapper.bzl", "zetasql_deps_step_1.bzl", "zetasql_deps_step_2.bzl", "zetasql_deps_step_3.bzl", diff --git a/bazel/textmapper.bzl b/bazel/textmapper.bzl new file mode 100644 index 000000000..23c26070b --- /dev/null +++ b/bazel/textmapper.bzl @@ -0,0 +1,75 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Definitions to generate parsers through Textmapper""" + +def tm_syntax( + name, + src, + outs, + visibility = None, + compatible_with = None, + templates = None, + textmapper = "@com_github_inspirer_textmapper//cmd/textmapper:textmapper"): + """Generates the requested outs from Textmapper sources. + + Example usage: + + tm_syntax( + name = "syntax", + src = "java.tm", + outs = [ + "lexer.go", + "lexer_tables.go", + "token.go", + ], + ) + + Outs should enumerate all expected outputs for Textmapper. + + Args: + name: Unique identifier for the build rule. + src: Textmapper file for the language. + outs: Expected outputs from Textmapper. + visibility: Visibility of the rule. + compatible_with: Standard Blaze compatible_with attribute. + templates: Location of the Textmapper templates to use. + textmapper: Build target for Textmapper tool. + """ + + if not templates: + tmpl_arg = "" + srcs = [src] + else: + tmpl_arg = "-i $$(dirname $$(echo $(locations {templates}) | cut --delimiter ' ' --fields 1))".format( + templates = templates, + ) + srcs = [src, templates] + tm_cmd = """ + $(location {textmapper}) generate -o $(@D) {tmpl_arg} $(location {src}) + """.format( + src = src, + textmapper = textmapper, + tmpl_arg = tmpl_arg, + ) + native.genrule( + name = name, + srcs = srcs, + outs = outs, + cmd = tm_cmd, + visibility = visibility, + compatible_with = compatible_with, + tools = [textmapper], + exec_compatible_with = ["@platforms//os:linux", "@platforms//os:macos"], + ) diff --git a/bazel/zetasql_deps_step_1.bzl b/bazel/zetasql_deps_step_1.bzl index 5b23f5210..66727b694 100644 --- a/bazel/zetasql_deps_step_1.bzl +++ b/bazel/zetasql_deps_step_1.bzl @@ -26,6 +26,28 @@ load("@com_google_zetasql//bazel:zetasql_bazel_version.bzl", "zetasql_bazel_vers def zetasql_deps_step_1(add_bazel_version = True): if add_bazel_version: zetasql_bazel_version() + + # Install newer versions of io_bazel_rules_go and gazelle to import Textmapper from github + # Make sure this is installed first. Otherwise, grpc_deps.bzl would attempt to use older + # versions, which have some bugs when downloading the go toolchains. + http_archive( + name = "io_bazel_rules_go", + sha256 = "278b7ff5a826f3dc10f04feaf0b70d48b68748ccd512d7f98bf442077f043fe3", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", + ], + ) + + http_archive( + name = "bazel_gazelle", + sha256 = "d3fa66a39028e97d76f9e2db8f1b0c11c099e8e01bf363a923074784e451f809", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz", + ], + ) + http_archive( name = "platforms", sha256 = "5308fc1d8865406a49427ba24a9ab53087f17f5266a7aabbfc28823f3916e1ca", diff --git a/bazel/zetasql_deps_step_2.bzl b/bazel/zetasql_deps_step_2.bzl index 52819381b..e9bcff30e 100644 --- a/bazel/zetasql_deps_step_2.bzl +++ b/bazel/zetasql_deps_step_2.bzl @@ -16,6 +16,7 @@ """ Step 2 to load ZetaSQL dependencies. """ +load("@bazel_gazelle//:deps.bzl", "go_repository") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Followup from zetasql_deps_step_1.bzl @@ -407,6 +408,11 @@ alias( m4_register_toolchains(version = "1.4.18") flex_register_toolchains(version = "2.6.4") bison_register_toolchains(version = "3.3.2") + go_repository( + name = "com_github_inspirer_textmapper", + commit = "e0aa14dc6db169c7afdf6908e810b5d12bbae2db", + importpath = "github.com/inspirer/textmapper", + ) ########################################################################## # Rules which depend on rules_foreign_cc diff --git a/zetasql/compliance/BUILD b/zetasql/compliance/BUILD index 5f042318b..bf958b411 100644 --- a/zetasql/compliance/BUILD +++ b/zetasql/compliance/BUILD @@ -328,42 +328,6 @@ cc_library( ], ) -cc_test( - name = "depth_limit_detector_test_cases_test", - size = "large", - srcs = ["depth_limit_detector_test_cases_test.cc"], - data = [ - "//zetasql/public/functions:array_find_mode_proto", - "//zetasql/public/functions:rounding_mode_proto", - "//zetasql/testdata:test_proto3_proto", - "//zetasql/testdata:test_schema_proto", - "@com_google_googleapis//google/type:date_proto", - "@com_google_googleapis//google/type:latlng_proto", - "@com_google_googleapis//google/type:timeofday_proto", - "@com_google_protobuf//:timestamp_proto", - "@com_google_protobuf//:wrappers_proto", - ], - shard_count = 30, - deps = [ - ":depth_limit_detector_test_cases", - ":test_driver", - "//zetasql/base/testing:status_matchers", - "//zetasql/base/testing:zetasql_gtest_main", - "//zetasql/common:thread_stack", - "//zetasql/parser", - "//zetasql/public:language_options", - "//zetasql/public:options_cc_proto", - "//zetasql/public:parse_helpers", - "//zetasql/public/types", - "//zetasql/reference_impl:reference_driver", - "//zetasql/testing:type_util", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/time", - ], -) - cc_library( name = "compliance_test_cases", testonly = 1, diff --git a/zetasql/compliance/depth_limit_detector_test_cases_test.cc b/zetasql/compliance/depth_limit_detector_test_cases_test.cc deleted file mode 100644 index 5cfaca0bb..000000000 --- a/zetasql/compliance/depth_limit_detector_test_cases_test.cc +++ /dev/null @@ -1,345 +0,0 @@ -// -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include "zetasql/compliance/depth_limit_detector_test_cases.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "zetasql/base/testing/status_matchers.h" -#include "zetasql/common/thread_stack.h" -#include "zetasql/compliance/depth_limit_detector_internal.h" -#include "zetasql/compliance/test_driver.h" -#include "zetasql/parser/parser.h" -#include "zetasql/public/language_options.h" -#include "zetasql/public/options.pb.h" -#include "zetasql/public/parse_tokens.h" -#include "zetasql/public/types/type_factory.h" -#include "zetasql/reference_impl/reference_driver.h" -#include "zetasql/testing/type_util.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "absl/flags/flag.h" -#include "absl/status/status.h" -#include "absl/strings/numbers.h" -#include "absl/strings/str_cat.h" -#include "absl/time/clock.h" -#include "absl/time/time.h" - -ABSL_FLAG( - absl::Duration, depth_limit_detector_reference_max_probing_duration, - absl::Seconds(30), - "Amount of time to search for failing reference implementation cases"); - -namespace zetasql { -namespace depth_limit_detector_internal { -namespace { - -class ReferenceDriverStatementRunner { - ReferenceDriver driver_; - TestDatabase test_db_; - - public: - explicit ReferenceDriverStatementRunner(const LanguageOptions& options) - : driver_(options) { - for (const std::string& proto_file : - testing::ZetaSqlTestProtoFilepaths()) { - test_db_.proto_files.insert(proto_file); - } - for (const std::string& proto_name : testing::ZetaSqlTestProtoNames()) { - test_db_.proto_names.insert(proto_name); - } - for (const std::string& enum_name : testing::ZetaSqlTestEnumNames()) { - test_db_.enum_names.insert(enum_name); - } - ZETASQL_EXPECT_OK(driver_.CreateDatabase(test_db_)); - } - - absl::Status operator()(std::string_view sql) { - zetasql::TypeFactory type_factory; - ReferenceDriver::ExecuteStatementOptions options{ - .primary_key_mode = PrimaryKeyMode::DEFAULT}; - TestDatabase query_db; - - ReferenceDriver::ExecuteStatementAuxOutput aux_output_ignored; - return driver_ - .ExecuteStatementForReferenceDriver(sql, {}, options, &type_factory, - aux_output_ignored, &query_db) - .status(); - } -}; - -TEST(DepthLimitDetectorTest, DepthLimitDetectorSqrtTest) { - // As the testcase just generates a number, 8 bytes corresponds - // to 8 digits. - DepthLimitDetectorRuntimeControl runtime_control; - runtime_control.max_sql_bytes = 8; - DepthLimitDetectorTestCase test_case = DepthLimitDetectorTestCase{ - .depth_limit_test_case_name = "sqrt", - .depth_limit_template = - DepthLimitDetectorTemplate({DepthLimitDetectorDepthNumber()}), - }; - auto results = RunDepthLimitDetectorTestCase( - test_case, - [](std::string_view number) { - int64_t i; - ABSL_CHECK(absl::SimpleAtoi(number, &i)); - return absl::ResourceExhaustedError( - absl::StrCat(static_cast(sqrt(i)))); - }, - runtime_control); - - // Prove that there are not two return conditions next to each other with - // the same return status as they should be merged. - ASSERT_GT(results.depth_limit_detector_return_conditions.size(), 2); - absl::Status last_status = absl::UnknownError("DepthLimitDetectorTest"); - int64_t last_ending_depth = 0; - for (const auto& condition : results.depth_limit_detector_return_conditions) { - EXPECT_NE(last_status, condition.return_status) - << "Condition at starting depth " << condition.starting_depth - << " is the same status as the previous one"; - EXPECT_EQ(last_ending_depth, condition.starting_depth - 1) - << condition.return_status; - last_status = condition.return_status; - last_ending_depth = condition.ending_depth; - } - EXPECT_GT(last_ending_depth, 9999999); - - // Prove the return conditions are on the correct boundaries. - int64_t s = 1; - for (const auto& condition : results.depth_limit_detector_return_conditions) { - EXPECT_EQ(condition.starting_depth, s * s) << condition.return_status; - EXPECT_THAT( - condition.return_status, - ::zetasql_base::testing::StatusIs(absl::StatusCode::kResourceExhausted)) - << condition.return_status; - s++; - } -} - -TEST(DepthLimitDetectorTest, DepthLimitDetectorSleepTest) { - DepthLimitDetectorRuntimeControl control; - control.max_sql_bytes = 6; - DepthLimitDetectorTestCase test_case = DepthLimitDetectorTestCase{ - .depth_limit_test_case_name = "just_number", - .depth_limit_template = - DepthLimitDetectorTemplate({DepthLimitDetectorDepthNumber()}), - }; - - // Prove that we run the test many times if we don't have a time limit. - auto results = RunDepthLimitDetectorTestCase( - test_case, [](std::string_view number) { return absl::OkStatus(); }, - control); - EXPECT_EQ(results.depth_limit_detector_return_conditions.size(), 1); - EXPECT_GT(results.depth_limit_detector_return_conditions[0].ending_depth, - 100000); - EXPECT_LT(results.depth_limit_detector_return_conditions[0].ending_depth, - 1000000); - - // Prove we stop the test if we have a time limit. - control.max_probing_duration = absl::Seconds(3); - auto sleep_results = RunDepthLimitDetectorTestCase( - test_case, - [](std::string_view number) { - int64_t i; - ABSL_CHECK(absl::SimpleAtoi(number, &i)); - absl::SleepFor(absl::Seconds(i)); - return absl::OkStatus(); - }, - control); - EXPECT_EQ(sleep_results.depth_limit_detector_return_conditions.size(), 1); - EXPECT_LT( - sleep_results.depth_limit_detector_return_conditions[0].ending_depth, 10); -} - -TEST(DepthLimitDetectorTest, DepthLimitDetectorPayloadTest) { - // As the testcase just generates a number, 3 bytes corresponds - // to 3 digits. - DepthLimitDetectorRuntimeControl runtime_control; - runtime_control.max_sql_bytes = 3; - int64_t call_count = 0; - DepthLimitDetectorTestCase test_case = DepthLimitDetectorTestCase{ - .depth_limit_test_case_name = "payload", - .depth_limit_template = - DepthLimitDetectorTemplate({DepthLimitDetectorDepthNumber()}), - }; - auto results = RunDepthLimitDetectorTestCase( - test_case, - [&](std::string_view number) { - absl::Status ret = absl::ResourceExhaustedError("payload_test"); - ret.SetPayload("test_payload_url", - absl::Cord(absl::StrCat("payload", call_count++))); - return ret; - }, - runtime_control); - // Prove we were called multiple times - ASSERT_GT(call_count, 1); - // Prove that all the statuses are merged correctly - ASSERT_EQ(results.depth_limit_detector_return_conditions.size(), 1); - // Prove that only the first payload is remembered - EXPECT_EQ(*results.depth_limit_detector_return_conditions[0] - .return_status.GetPayload("test_payload_url"), - absl::Cord("payload0")); -} - -class DepthLimitDetectorTemplateTest - : public ::testing::TestWithParam< - std::reference_wrapper> {}; -} // namespace - -// Return a different error for each depth and check that is in the output. -TEST_P(DepthLimitDetectorTemplateTest, DepthLimitDetectorDisectTest) { - const DepthLimitDetectorTestCase& test_case = GetParam(); - auto results = - RunDepthLimitDetectorTestCase(test_case, [](std::string_view str) { - return absl::InvalidArgumentError(absl::StrCat(str.size())); - }); - ASSERT_GE(results.depth_limit_detector_return_conditions.size(), 1); - EXPECT_EQ(results.depth_limit_detector_return_conditions[0].starting_depth, - 1); - EXPECT_EQ(results.depth_limit_test_case_name, - test_case.depth_limit_test_case_name); - - int last_ending_depth = 0; - for (const auto& condition : results.depth_limit_detector_return_conditions) { - EXPECT_EQ(condition.starting_depth, last_ending_depth + 1); - EXPECT_EQ(condition.ending_depth, condition.starting_depth); - EXPECT_THAT( - condition.return_status, - ::zetasql_base::testing::StatusIs(absl::StatusCode::kInvalidArgument)); - last_ending_depth = condition.ending_depth; - } -} - -TEST_P(DepthLimitDetectorTemplateTest, ParserAcceptsInstantiation) { - const DepthLimitDetectorTestCase& test_case = GetParam(); - std::unique_ptr output; - zetasql::ParserOptions parser_options; - parser_options.set_language_options( - DepthLimitDetectorTestCaseLanguageOptions(test_case)); - - std::string level1 = DepthLimitDetectorTemplateToString(test_case, 1); - ZETASQL_EXPECT_OK(zetasql::ParseStatement(level1, parser_options, &output)) - << level1; - std::string level2 = DepthLimitDetectorTemplateToString(test_case, 2); - ZETASQL_EXPECT_OK(zetasql::ParseStatement(level2, parser_options, &output)) - << level2; - std::string level10 = DepthLimitDetectorTemplateToString(test_case, 10); - ZETASQL_EXPECT_OK(zetasql::ParseStatement(level10, parser_options, &output)) - << level10; - - // Very big SQL query that is too big to print out still should have no parser - // issues, but we shouldn't try to print it out. - std::string level1000 = DepthLimitDetectorTemplateToString(test_case, 1000); - ZETASQL_EXPECT_OK(zetasql::ParseStatement(level1000, parser_options, &output)); -} - -TEST_P(DepthLimitDetectorTemplateTest, ReferenceImplAcceptsInstantiation) { - const DepthLimitDetectorTestCase& test_case = GetParam(); - ReferenceDriverStatementRunner driver( - DepthLimitDetectorTestCaseLanguageOptions(test_case)); - - for (int i = 1; i <= 5; ++i) { - std::string sql = DepthLimitDetectorTemplateToString(test_case, i); - ZETASQL_ASSERT_OK(driver(sql)) << "Reference driver failed at depth " << i << "\n" - << sql; - } - - DepthLimitDetectorTestResult results; - - ABSL_LOG(INFO) << "Disecting reference implementation " << test_case; - - DepthLimitDetectorRuntimeControl runtime_control; - runtime_control.max_probing_duration = - absl::GetFlag(FLAGS_depth_limit_detector_reference_max_probing_duration); - - results = RunDepthLimitDetectorTestCase( - test_case, [&](std::string_view sql) { return driver(sql); }, - runtime_control); - - ABSL_LOG(INFO) << "Disection finished " << results; - EXPECT_THAT(results.depth_limit_detector_return_conditions[0].return_status, - ::zetasql_base::testing::IsOk()) - << results; - - std::string last_successful_sql = DepthLimitDetectorTemplateToString( - test_case, - results.depth_limit_detector_return_conditions[0].ending_depth); - ParseResumeLocation resume_location = - ParseResumeLocation::FromString(last_successful_sql); - ParseTokenOptions options; - options.language_options = - DepthLimitDetectorTestCaseLanguageOptions(test_case); - std::vector parse_tokens; - ZETASQL_ASSERT_OK(GetParseTokens(options, &resume_location, &parse_tokens)); - - RecordProperty("max_successful_tokens", parse_tokens.size()); - RecordProperty("max_successful_bytes", last_successful_sql.size()); - - for (int64_t i = 1; i < results.depth_limit_detector_return_conditions.size(); - ++i) { - EXPECT_TRUE(absl::IsResourceExhausted( - results.depth_limit_detector_return_conditions[i].return_status)) - << results.depth_limit_detector_return_conditions[i]; - } -} - -TEST_P(DepthLimitDetectorTemplateTest, LanguageOptionsAllNeeded) { - const DepthLimitDetectorTestCase& test_case = GetParam(); - const LanguageOptions options = - DepthLimitDetectorTestCaseLanguageOptions(test_case); - std::string sql = DepthLimitDetectorTemplateToString(test_case, 5); - for (LanguageFeature f : options.GetEnabledLanguageFeatures()) { - LanguageOptions without = options; - without.DisableLanguageFeature(f); - ReferenceDriverStatementRunner driver(without); - absl::Status status = driver(sql); - EXPECT_FALSE(status.ok()) - << "Language option " << LanguageFeature_Name(f) - << " is not needed to run " << test_case << " example " << sql; - } -} - -INSTANTIATE_TEST_CASE_P(DepthLimitDetectorTest, DepthLimitDetectorTemplateTest, - ::testing::ValuesIn(AllDepthLimitDetectorTestCases()), - ::testing::PrintToStringParamName()); - -// Validate repeated template output. -TEST(DepthLimitDetectorTest, SimpleTestCaseToString) { - std::string output = DepthLimitDetectorTemplateToString( - DepthLimitDetectorTemplate( - {"SELECT ", DepthLimitDetectorRepeatedTemplate({"+1"}), ";"}), - 2); - EXPECT_EQ("SELECT +1+1;", output); -} - -// Validate numbered template output. -TEST(DepthLimitDetectorTest, NumberedTestCaseToString) { - std::string output = DepthLimitDetectorTemplateToString( - DepthLimitDetectorTemplate({"SELECT ", - DepthLimitDetectorRepeatedTemplate( - {"+", DepthLimitDetectorDepthNumber()}), - ";"}), - 4); - EXPECT_EQ("SELECT +1+2+3+4;", output); -} -} // namespace depth_limit_detector_internal -} // namespace zetasql diff --git a/zetasql/parser/BUILD b/zetasql/parser/BUILD index ff69564c2..6ac145a12 100644 --- a/zetasql/parser/BUILD +++ b/zetasql/parser/BUILD @@ -20,6 +20,7 @@ # Placeholder: load py_test load("//bazel:bison.bzl", "genyacc") load("//bazel:flex.bzl", "genlex") +load("//bazel:textmapper.bzl", "tm_syntax") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load(":builddefs.bzl", "gen_parser_test")