tp: create PerfettoSqlPreprocessor

PerfettoSqlPreprocessor is currently responsible for tokenizing SQL
statements and returning them to the parser. In a followup, it will also
be reponsible for expanding macros which is why we bother making it a
standalone component.

Change-Id: Ib9aede77b8e410c87f085813a4a4bd56237d5fca

Android.bp

@@ -10617,6 +10617,7 @@ filegroup {
         "src/trace_processor/perfetto_sql/engine/function_util.cc",
         "src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc",
         "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.cc",
+        "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.cc",
         "src/trace_processor/perfetto_sql/engine/runtime_table_function.cc",
     ],
 }
@@ -10627,6 +10628,7 @@ filegroup {
     srcs: [
         "src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc",
         "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser_unittest.cc",
+        "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor_unittest.cc",
     ],
 }
 

BUILD

@@ -2263,6 +2263,8 @@ perfetto_filegroup(
         "src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h",
         "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.cc",
         "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h",
+        "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.cc",
+        "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.h",
         "src/trace_processor/perfetto_sql/engine/runtime_table_function.cc",
         "src/trace_processor/perfetto_sql/engine/runtime_table_function.h",
     ],

src/trace_processor/perfetto_sql/engine/BUILD.gn

@@ -26,6 +26,8 @@ source_set("engine") {
     "perfetto_sql_engine.h",
     "perfetto_sql_parser.cc",
     "perfetto_sql_parser.h",
+    "perfetto_sql_preprocessor.cc",
+    "perfetto_sql_preprocessor.h",
     "runtime_table_function.cc",
     "runtime_table_function.h",
   ]
@@ -50,6 +52,8 @@ perfetto_unittest_source_set("unittests") {
   sources = [
     "perfetto_sql_engine_unittest.cc",
     "perfetto_sql_parser_unittest.cc",
+    "perfetto_sql_preprocessor_unittest.cc",
+    "perfetto_sql_test_utils.h",
   ]
   deps = [
     ":engine",

src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc

@@ -28,6 +28,7 @@
 #include "src/trace_processor/perfetto_sql/engine/created_function.h"
 #include "src/trace_processor/perfetto_sql/engine/function_util.h"
 #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h"
+#include "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.h"
 #include "src/trace_processor/perfetto_sql/engine/runtime_table_function.h"
 #include "src/trace_processor/sqlite/db_sqlite_table.h"
 #include "src/trace_processor/sqlite/scoped_db.h"
@@ -36,6 +37,31 @@
 #include "src/trace_processor/tp_metatrace.h"
 #include "src/trace_processor/util/status_macros.h"
 
+// Implementation details
+// ----------------------
+//
+// The execution of PerfettoSQL statements is the joint responsibility of
+// several classes which all are linked together in the following way:
+//
+//  PerfettoSqlEngine -> PerfettoSqlParser -> PerfettoSqlPreprocessor
+//
+// The responsibility of each of these classes is as follows:
+//
+// * PerfettoSqlEngine: this class is responsible for the end-to-end processing
+//   of statements. It calls into PerfettoSqlParser to incrementally receive
+//   parsed SQL statements and then executes them. If the statement is a
+//   PerfettoSQL-only statement, the execution happens entirely in this class.
+//   Otherwise, if the statement is a valid SQLite statement, SQLite is called
+//   into to perform the execution.
+// * PerfettoSqlParser: this class is responsible for taking a chunk of SQL and
+//   incrementally converting them into parsed SQL statement. The parser calls
+//   into the PerfettoSqlPreprocessor to split the SQL chunk into a statement
+//   and perform any macro expansion. It then tries to parse any
+//   PerfettoSQL-only statements into their component parts and leaves SQLite
+//   statements as-is for execution by SQLite.
+// * PerfettoSqlPreprocessor: this class is responsible for taking a chunk of
+//   SQL and breaking them into statements, while also expanding any macros
+//   which might be present inside.
 namespace perfetto {
 namespace trace_processor {
 namespace {
@@ -91,6 +117,8 @@ base::Status AddTracebackIfNeeded(base::Status status,
   return status;
 }
 
+// This function is used when the the PerfettoSQL has been fully executed by the
+// PerfettoSqlEngine and a SqlSoruce is needed for SQLite to execute.
 SqlSource RewriteToDummySql(const SqlSource& source) {
   return source.RewriteAllIgnoreExisting(
       SqlSource::FromTraceProcessorImplementation("SELECT 0 WHERE 0"));
@@ -205,14 +233,10 @@ PerfettoSqlEngine::ExecuteUntilLastStatement(SqlSource sql_source) {
                    &parser.statement())) {
       RETURN_IF_ERROR(AddTracebackIfNeeded(
           RegisterRuntimeTable(cst->name, cst->sql), parser.statement_sql()));
-      // Since the rest of the code requires a statement, just use a no-value
-      // dummy statement.
       source = RewriteToDummySql(parser.statement_sql());
     } else if (auto* include = std::get_if<PerfettoSqlParser::Include>(
                    &parser.statement())) {
       RETURN_IF_ERROR(ExecuteInclude(*include, parser));
-      // Since the rest of the code requires a statement, just use a no-value
-      // dummy statement.
       source = RewriteToDummySql(parser.statement_sql());
     } else {
       // If none of the above matched, this must just be an SQL statement
@@ -450,9 +474,6 @@ base::StatusOr<SqlSource> PerfettoSqlEngine::ExecuteCreateFunction(
   if (!cf.is_table) {
     RETURN_IF_ERROR(
         RegisterSqlFunction(cf.replace, cf.prototype, cf.returns, cf.sql));
-
-    // Since the rest of the code requires a statement, just use a no-value
-    // dummy statement.
     return RewriteToDummySql(parser.statement_sql());
   }
 

src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h

@@ -25,6 +25,7 @@
 #include "perfetto/ext/base/status_or.h"
 #include "src/trace_processor/db/runtime_table.h"
 #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h"
+#include "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.h"
 #include "src/trace_processor/perfetto_sql/engine/runtime_table_function.h"
 #include "src/trace_processor/perfetto_sql/intrinsics/functions/sql_function.h"
 #include "src/trace_processor/perfetto_sql/intrinsics/table_functions/static_table_function.h"

src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.cc

@@ -24,6 +24,7 @@
 #include "perfetto/base/status.h"
 #include "perfetto/ext/base/status_or.h"
 #include "perfetto/ext/base/string_utils.h"
+#include "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.h"
 #include "src/trace_processor/sqlite/sql_source.h"
 #include "src/trace_processor/sqlite/sqlite_tokenizer.h"
 #include "src/trace_processor/util/status_macros.h"
@@ -80,11 +81,18 @@ bool ValidateModuleName(const std::string& name) {
 
 }  // namespace
 
-PerfettoSqlParser::PerfettoSqlParser(SqlSource sql)
-    : tokenizer_(std::move(sql)) {}
+PerfettoSqlParser::PerfettoSqlParser(SqlSource source)
+    : preprocessor_(std::move(source)),
+      tokenizer_(SqlSource::FromTraceProcessorImplementation("")) {}
 
 bool PerfettoSqlParser::Next() {
-  PERFETTO_DCHECK(status_.ok());
+  PERFETTO_CHECK(status_.ok());
+
+  if (!preprocessor_.NextStatement()) {
+    status_ = preprocessor_.status();
+    return false;
+  }
+  tokenizer_.Reset(preprocessor_.statement());
 
   State state = State::kStmtStart;
   std::optional<Token> first_non_space_token;
@@ -120,7 +128,9 @@ bool PerfettoSqlParser::Next() {
 
     switch (state) {
       case State::kPassthrough:
-        break;
+        statement_ = SqliteSql{};
+        statement_sql_ = preprocessor_.statement();
+        return true;
       case State::kStmtStart:
         if (TokenIsSqliteKeyword("create", token)) {
           state = State::kCreate;
@@ -149,9 +159,8 @@ bool PerfettoSqlParser::Next() {
         if (TokenIsSqliteKeyword("trigger", token)) {
           // TODO(lalitm): add this to the "errors" documentation page
           // explaining why this is the case.
-          base::StackString<1024> err(
-              "Creating triggers are not supported by trace processor.");
-          return ErrorAtToken(token, err.c_str());
+          return ErrorAtToken(
+              token, "Creating triggers is not supported in PerfettoSQL.");
         }
         if (TokenIsCustomKeyword("perfetto", token)) {
           state = State::kCreatePerfetto;

src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h

@@ -22,6 +22,7 @@
 #include <variant>
 
 #include "perfetto/ext/base/status_or.h"
+#include "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.h"
 #include "src/trace_processor/sqlite/sql_source.h"
 #include "src/trace_processor/sqlite/sqlite_tokenizer.h"
 
@@ -94,11 +95,12 @@ class PerfettoSqlParser {
   }
 
   // Returns the error status for the parser. This will be |base::OkStatus()|
-  // until
+  // until an unrecoverable error is encountered.
   const base::Status& status() const { return status_; }
 
  private:
-  // This cannot be moved because we keep pointers into |sql_| in |tokenizer_|.
+  // This cannot be moved because we keep pointers into |sql_| in
+  // |preprocessor_|.
   PerfettoSqlParser(PerfettoSqlParser&&) = delete;
   PerfettoSqlParser& operator=(PerfettoSqlParser&&) = delete;
 
@@ -114,7 +116,9 @@ class PerfettoSqlParser {
 
   bool ErrorAtToken(const SqliteTokenizer::Token&, const char* error);
 
+  PerfettoSqlPreprocessor preprocessor_;
   SqliteTokenizer tokenizer_;
+
   base::Status status_;
   std::optional<SqlSource> statement_sql_;
   std::optional<Statement> statement_;

src/trace_processor/perfetto_sql/engine/perfetto_sql_parser_unittest.cc

@@ -22,6 +22,7 @@
 
 #include "perfetto/base/logging.h"
 #include "perfetto/ext/base/status_or.h"
+#include "src/trace_processor/perfetto_sql/engine/perfetto_sql_test_utils.h"
 #include "src/trace_processor/sqlite/sql_source.h"
 #include "test/gtest_and_gmock.h"
 
@@ -35,36 +36,8 @@ using CreateFn = PerfettoSqlParser::CreateFunction;
 using CreateTable = PerfettoSqlParser::CreateTable;
 using Include = PerfettoSqlParser::Include;
 
-inline bool operator==(const SqlSource& a, const SqlSource& b) {
-  return a.sql() == b.sql();
-}
-
-inline bool operator==(const SqliteSql&, const SqliteSql&) {
-  return true;
-}
-
-inline bool operator==(const CreateFn& a, const CreateFn& b) {
-  return std::tie(a.returns, a.is_table, a.prototype, a.replace, a.sql) ==
-         std::tie(b.returns, b.is_table, b.prototype, b.replace, b.sql);
-}
-
-inline bool operator==(const CreateTable& a, const CreateTable& b) {
-  return std::tie(a.name, a.sql) == std::tie(b.name, b.sql);
-}
-
-inline bool operator==(const Include& a, const Include& b) {
-  return std::tie(a.key) == std::tie(b.key);
-}
-
 namespace {
 
-SqlSource FindSubstr(const SqlSource& source, const std::string& needle) {
-  size_t off = source.sql().find(needle);
-  PERFETTO_CHECK(off != std::string::npos);
-  return source.Substr(static_cast<uint32_t>(off),
-                       static_cast<uint32_t>(needle.size()));
-}
-
 class PerfettoSqlParserTest : public ::testing::Test {
  protected:
   base::StatusOr<std::vector<PerfettoSqlParser::Statement>> Parse(

src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.cc

@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.h"
+#include <unordered_set>
+
+#include "perfetto/base/status.h"
+#include "perfetto/ext/base/string_utils.h"
+#include "src/trace_processor/sqlite/sql_source.h"
+#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
+#include "src/trace_processor/util/status_macros.h"
+
+namespace perfetto {
+namespace trace_processor {
+
+PerfettoSqlPreprocessor::PerfettoSqlPreprocessor(SqlSource source)
+    : global_tokenizer_(std::move(source)) {}
+
+bool PerfettoSqlPreprocessor::NextStatement() {
+  PERFETTO_CHECK(status_.ok());
+
+  // Skip through any number of semi-colons (representing empty statements).
+  SqliteTokenizer::Token tok = global_tokenizer_.NextNonWhitespace();
+  while (tok.token_type == SqliteTokenType::TK_SEMI) {
+    tok = global_tokenizer_.NextNonWhitespace();
+  }
+
+  // If we still see a terminal token at this point, we must have hit EOF.
+  if (tok.IsTerminal()) {
+    PERFETTO_DCHECK(tok.token_type != SqliteTokenType::TK_SEMI);
+    return false;
+  }
+
+  SqlSource stmt =
+      global_tokenizer_.Substr(tok, global_tokenizer_.NextTerminal());
+  statement_ = std::move(stmt);
+  return true;
+}
+
+base::Status PerfettoSqlPreprocessor::ErrorAtToken(
+    const SqliteTokenizer& tokenizer,
+    const SqliteTokenizer::Token& token,
+    const char* error) {
+  std::string traceback = tokenizer.AsTraceback(token);
+  return base::ErrStatus("%s%s", traceback.c_str(), error);
+}
+
+}  // namespace trace_processor
+}  // namespace perfetto