From ed80e4c5f369ed7b09a72c190a88b5b1c3ad0811 Mon Sep 17 00:00:00 2001 From: Anna Mayzner Date: Tue, 29 Aug 2023 16:19:17 +0000 Subject: [PATCH] tp: Implement INCLUDE PERFETTO MODULE Bug:296388363 Change-Id: I6fc43ceb4fd8d4e80b59c34409574e3f3af95824 --- .../perfetto_sql/engine/BUILD.gn | 1 + .../engine/perfetto_sql_engine.cc | 57 ++++++++- .../perfetto_sql/engine/perfetto_sql_engine.h | 19 ++- .../engine/perfetto_sql_parser.cc | 89 +++++++++++-- .../perfetto_sql/engine/perfetto_sql_parser.h | 30 ++++- .../engine/perfetto_sql_parser_unittest.cc | 73 ++++++++--- .../intrinsics/functions/import.cc | 32 +---- .../intrinsics/functions/import.h | 1 - src/trace_processor/sqlite/sql_source.cc | 6 +- src/trace_processor/sqlite/sql_source.h | 5 +- .../trace_database_integrationtest.cc | 6 +- src/trace_processor/trace_processor_impl.cc | 17 ++- src/trace_processor/trace_processor_impl.h | 2 - src/trace_processor/trace_processor_shell.cc | 4 +- src/trace_processor/util/sql_modules.h | 8 +- .../diff_tests/perfetto_sql/tests.py | 121 ++++++++++++++++++ 16 files changed, 374 insertions(+), 97 deletions(-) diff --git a/src/trace_processor/perfetto_sql/engine/BUILD.gn b/src/trace_processor/perfetto_sql/engine/BUILD.gn index 0ed90d67e3..9374bf6a0b 100644 --- a/src/trace_processor/perfetto_sql/engine/BUILD.gn +++ b/src/trace_processor/perfetto_sql/engine/BUILD.gn @@ -41,6 +41,7 @@ source_set("engine") { "../../types", "../../util", "../../util:sql_argument", + "../../util:stdlib", ] } diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc index 8f0bfd933f..b34f5564c9 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc @@ -192,16 +192,24 @@ PerfettoSqlEngine::ExecuteUntilLastStatement(SqlSource sql_source) { std::optional source; if (auto* cf = std::get_if( &parser.statement())) { - auto source_or = ExecuteCreateFunction(*cf); - RETURN_IF_ERROR(AddTracebackIfNeeded(source_or.status(), cf->sql)); + auto source_or = ExecuteCreateFunction(*cf, parser); + RETURN_IF_ERROR( + AddTracebackIfNeeded(source_or.status(), parser.statement_sql())); source = std::move(source_or.value()); } else if (auto* cst = std::get_if( &parser.statement())) { RETURN_IF_ERROR(AddTracebackIfNeeded( - RegisterRuntimeTable(cst->name, cst->sql), cst->sql)); + RegisterRuntimeTable(cst->name, cst->sql), parser.statement_sql())); // Since the rest of the code requires a statement, just use a no-value // dummy statement. - source = cst->sql.FullRewrite( + source = parser.statement_sql().FullRewrite( + SqlSource::FromTraceProcessorImplementation("SELECT 0 WHERE 0")); + } else if (auto* include = std::get_if( + &parser.statement())) { + RETURN_IF_ERROR(ExecuteInclude(*include, parser)); + // Since the rest of the code requires a statement, just use a no-value + // dummy statement. + source = parser.statement_sql().FullRewrite( SqlSource::FromTraceProcessorImplementation("SELECT 0 WHERE 0")); } else { // If none of the above matched, this must just be an SQL statement @@ -209,7 +217,7 @@ PerfettoSqlEngine::ExecuteUntilLastStatement(SqlSource sql_source) { auto* sql = std::get_if(&parser.statement()); PERFETTO_CHECK(sql); - source = std::move(sql->sql); + source = parser.statement_sql(); } // Try to get SQLite to prepare the statement. @@ -399,15 +407,50 @@ base::Status PerfettoSqlEngine::EnableSqlFunctionMemoization( return CreatedFunction::EnableMemoization(ctx); } +base::Status PerfettoSqlEngine::ExecuteInclude( + const PerfettoSqlParser::Include& include, + const PerfettoSqlParser& parser) { + std::string key = include.key; + PERFETTO_TP_TRACE(metatrace::Category::TOPLEVEL, "Import", + [key](metatrace::Record* r) { r->AddArg("Import", key); }); + std::string module_name = sql_modules::GetModuleName(key); + auto module = FindModule(module_name); + if (!module) + return base::ErrStatus("INCLUDE: Unknown module name provided - %s", + key.c_str()); + + auto module_file = module->include_key_to_file.Find(key); + if (!module_file) { + return base::ErrStatus("INCLUDE: Unknown filename provided - %s", + key.c_str()); + } + // INCLUDE is noop for already included files. + if (module_file->included) { + return base::OkStatus(); + } + + auto it = Execute(SqlSource::FromModuleInclude(module_file->sql, key)); + if (!it.status().ok()) { + return base::ErrStatus("%s%s", + parser.statement_sql().AsTraceback(0).c_str(), + it.status().c_message()); + } + if (it->statement_count_with_output > 0) + return base::ErrStatus("INCLUDE: Included module returning values."); + module_file->included = true; + return base::OkStatus(); +} + base::StatusOr PerfettoSqlEngine::ExecuteCreateFunction( - const PerfettoSqlParser::CreateFunction& cf) { + const PerfettoSqlParser::CreateFunction& cf, + const PerfettoSqlParser& parser) { if (!cf.is_table) { RETURN_IF_ERROR( RegisterSqlFunction(cf.replace, cf.prototype, cf.returns, cf.sql)); // Since the rest of the code requires a statement, just use a no-value // dummy statement. - return cf.sql.FullRewrite( + return parser.statement_sql().FullRewrite( SqlSource::FromTraceProcessorImplementation("SELECT 0 WHERE 0")); } diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h index d3f1fa6f00..f4a0abdb58 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h @@ -32,6 +32,7 @@ #include "src/trace_processor/sqlite/sql_source.h" #include "src/trace_processor/sqlite/sqlite_engine.h" #include "src/trace_processor/sqlite/sqlite_utils.h" +#include "src/trace_processor/util/sql_modules.h" namespace perfetto { namespace trace_processor { @@ -128,9 +129,24 @@ class PerfettoSqlEngine { SqliteEngine* sqlite_engine() { return engine_.get(); } + // Makes new SQL module available to import. + void RegisterModule(const std::string& name, + sql_modules::RegisteredModule module) { + modules_.Insert(name, std::move(module)); + } + + // Fetches registered SQL module. + sql_modules::RegisteredModule* FindModule(const std::string& name) { + return modules_.Find(name); + } + private: base::StatusOr ExecuteCreateFunction( - const PerfettoSqlParser::CreateFunction&); + const PerfettoSqlParser::CreateFunction&, + const PerfettoSqlParser& parser); + + base::Status ExecuteInclude(const PerfettoSqlParser::Include&, + const PerfettoSqlParser& parser); // Registers a SQL-defined trace processor C++ table with SQLite. base::Status RegisterRuntimeTable(std::string name, SqlSource sql); @@ -140,6 +156,7 @@ class PerfettoSqlEngine { base::FlatHashMap> runtime_table_fn_states_; base::FlatHashMap> runtime_tables_; + base::FlatHashMap modules_; std::unique_ptr engine_; }; diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.cc index 459c6e0250..a627ebd3d2 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.cc +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.cc @@ -17,6 +17,7 @@ #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h" #include +#include #include #include "perfetto/base/logging.h" @@ -37,6 +38,8 @@ using Statement = PerfettoSqlParser::Statement; enum class State { kStmtStart, kCreate, + kInclude, + kIncludePerfetto, kCreateOr, kCreateOrReplace, kCreateOrReplacePerfetto, @@ -60,6 +63,21 @@ bool TokenIsCustomKeyword(std::string_view keyword, SqliteTokenizer::Token t) { return t.token_type == SqliteTokenType::TK_ID && KeywordEqual(keyword, t.str); } +bool IsValidModuleWord(const std::string& word) { + for (const char& c : word) { + if (!std::isalnum(c) & (c != '_') & !std::islower(c)) { + return false; + } + } + return true; +} + +bool ValidateModuleName(const std::string& name) { + std::vector packages = base::SplitString(name, "."); + return std::find_if(packages.begin(), packages.end(), + std::not_fn(IsValidModuleWord)) == packages.end(); +} + } // namespace PerfettoSqlParser::PerfettoSqlParser(SqlSource sql) @@ -81,8 +99,8 @@ bool PerfettoSqlParser::Next() { // If we have a non-space character we've seen, just return all the stuff // after that point. if (first_non_space_token) { - statement_ = - SqliteSql{tokenizer_.Substr(*first_non_space_token, token)}; + statement_ = SqliteSql{}; + statement_sql_ = tokenizer_.Substr(*first_non_space_token, token); return true; } // This means we've seen a semi-colon without any non-space content. Just @@ -104,9 +122,29 @@ bool PerfettoSqlParser::Next() { case State::kPassthrough: break; case State::kStmtStart: - state = TokenIsSqliteKeyword("create", token) ? State::kCreate - : State::kPassthrough; + if (TokenIsSqliteKeyword("create", token)) { + state = State::kCreate; + } else if (TokenIsCustomKeyword("include", token)) { + state = State::kInclude; + } else { + state = State::kPassthrough; + } + break; + case State::kInclude: + if (TokenIsCustomKeyword("perfetto", token)) { + state = State::kIncludePerfetto; + } else { + return ErrorAtToken(token, + "Use 'INCLUDE PERFETTO MODULE {include_key}'."); + } break; + case State::kIncludePerfetto: + if (TokenIsCustomKeyword("module", token)) { + return ParseIncludePerfettoModule(*first_non_space_token); + } else { + return ErrorAtToken(token, + "Use 'INCLUDE PERFETTO MODULE {include_key}'."); + } case State::kCreate: if (TokenIsSqliteKeyword("trigger", token)) { // TODO(lalitm): add this to the "errors" documentation page @@ -135,11 +173,11 @@ bool PerfettoSqlParser::Next() { case State::kCreateOrReplacePerfetto: case State::kCreatePerfetto: if (TokenIsCustomKeyword("function", token)) { - return ParseCreatePerfettoFunction(state == - State::kCreateOrReplacePerfetto); + return ParseCreatePerfettoFunction( + state == State::kCreateOrReplacePerfetto, *first_non_space_token); } if (TokenIsSqliteKeyword("table", token)) { - return ParseCreatePerfettoTable(); + return ParseCreatePerfettoTable(*first_non_space_token); } base::StackString<1024> err( "Expected 'FUNCTION' or 'TABLE' after 'CREATE PERFETTO', received " @@ -150,7 +188,26 @@ bool PerfettoSqlParser::Next() { } } -bool PerfettoSqlParser::ParseCreatePerfettoTable() { +bool PerfettoSqlParser::ParseIncludePerfettoModule( + Token first_non_space_token) { + auto tok = tokenizer_.NextNonWhitespace(); + auto terminal = tokenizer_.NextTerminal(); + std::string key = tokenizer_.Substr(tok, terminal).sql(); + + if (!ValidateModuleName(key)) { + base::StackString<1024> err( + "Only alphanumeric characters, dots and underscores allowed in include " + "keys: '%s'", + key.c_str()); + return ErrorAtToken(tok, err.c_str()); + } + + statement_ = Include{key}; + statement_sql_ = tokenizer_.Substr(first_non_space_token, terminal); + return true; +} + +bool PerfettoSqlParser::ParseCreatePerfettoTable(Token first_non_space_token) { Token table_name = tokenizer_.NextNonWhitespace(); if (table_name.token_type != SqliteTokenType::TK_ID) { base::StackString<1024> err("Invalid table name %.*s", @@ -170,12 +227,15 @@ bool PerfettoSqlParser::ParseCreatePerfettoTable() { } Token first = tokenizer_.NextNonWhitespace(); - statement_ = CreateTable{std::move(name), - tokenizer_.Substr(first, tokenizer_.NextTerminal())}; + Token terminal = tokenizer_.NextTerminal(); + statement_ = CreateTable{std::move(name), tokenizer_.Substr(first, terminal)}; + statement_sql_ = tokenizer_.Substr(first_non_space_token, terminal); return true; } -bool PerfettoSqlParser::ParseCreatePerfettoFunction(bool replace) { +bool PerfettoSqlParser::ParseCreatePerfettoFunction( + bool replace, + Token first_non_space_token) { std::string prototype; Token function_name = tokenizer_.NextNonWhitespace(); if (function_name.token_type != SqliteTokenType::TK_ID) { @@ -234,9 +294,10 @@ bool PerfettoSqlParser::ParseCreatePerfettoFunction(bool replace) { } Token first = tokenizer_.NextNonWhitespace(); - statement_ = CreateFunction{ - replace, std::move(prototype), std::move(ret), - tokenizer_.Substr(first, tokenizer_.NextTerminal()), table_return}; + Token terminal = tokenizer_.NextTerminal(); + statement_ = CreateFunction{replace, std::move(prototype), std::move(ret), + tokenizer_.Substr(first, terminal), table_return}; + statement_sql_ = tokenizer_.Substr(first_non_space_token, terminal); return true; } diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h b/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h index 52bf68f069..b7f67d56ab 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h @@ -17,6 +17,7 @@ #ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_PARSER_H_ #define SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_PARSER_H_ +#include #include #include @@ -41,9 +42,7 @@ class PerfettoSqlParser { public: // Indicates that the specified SQLite SQL was extracted directly from a // PerfettoSQL statement and should be directly executed with SQLite. - struct SqliteSql { - SqlSource sql; - }; + struct SqliteSql {}; // Indicates that the specified SQL was a CREATE PERFETTO FUNCTION statement // with the following parameters. struct CreateFunction { @@ -59,7 +58,13 @@ class PerfettoSqlParser { std::string name; SqlSource sql; }; - using Statement = std::variant; + // Indicates that the specified SQL was a INCLUDE PERFETTO MODULE statement + // with the following parameter. + struct Include { + std::string key; + }; + using Statement = + std::variant; // Creates a new SQL parser with the a block of PerfettoSQL statements. // Concretely, the passed string can contain >1 statement. @@ -80,6 +85,14 @@ class PerfettoSqlParser { return statement_.value(); } + // Returns the full statement which was parsed. This should return + // |statement()| and Perfetto SQL code that's in front. This function *must + // not* be called unless |Next()| returned true. + const SqlSource& statement_sql() const { + PERFETTO_CHECK(statement_sql_); + return *statement_sql_; + } + // Returns the error status for the parser. This will be |base::OkStatus()| // until const base::Status& status() const { return status_; } @@ -89,9 +102,13 @@ class PerfettoSqlParser { PerfettoSqlParser(PerfettoSqlParser&&) = delete; PerfettoSqlParser& operator=(PerfettoSqlParser&&) = delete; - bool ParseCreatePerfettoFunction(bool replace); + bool ParseCreatePerfettoFunction( + bool replace, + SqliteTokenizer::Token first_non_space_token); + + bool ParseCreatePerfettoTable(SqliteTokenizer::Token first_non_space_token); - bool ParseCreatePerfettoTable(); + bool ParseIncludePerfettoModule(SqliteTokenizer::Token first_non_space_token); bool ParseArgumentDefinitions(std::string*); @@ -99,6 +116,7 @@ class PerfettoSqlParser { SqliteTokenizer tokenizer_; base::Status status_; + std::optional statement_sql_; std::optional statement_; }; diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser_unittest.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser_unittest.cc index e5eb2f675f..b436225462 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser_unittest.cc +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_parser_unittest.cc @@ -29,16 +29,18 @@ namespace perfetto { namespace trace_processor { using Result = PerfettoSqlParser::Statement; +using Statement = PerfettoSqlParser::Statement; using SqliteSql = PerfettoSqlParser::SqliteSql; using CreateFn = PerfettoSqlParser::CreateFunction; using CreateTable = PerfettoSqlParser::CreateTable; +using Include = PerfettoSqlParser::Include; inline bool operator==(const SqlSource& a, const SqlSource& b) { return a.sql() == b.sql(); } -inline bool operator==(const SqliteSql& a, const SqliteSql& b) { - return a.sql == b.sql; +inline bool operator==(const SqliteSql&, const SqliteSql&) { + return true; } inline bool operator==(const CreateFn& a, const CreateFn& b) { @@ -50,6 +52,10 @@ inline bool operator==(const CreateTable& a, const CreateTable& b) { return std::tie(a.name, a.sql) == std::tie(b.name, b.sql); } +inline bool operator==(const Include& a, const Include& b) { + return std::tie(a.key) == std::tie(b.key); +} + namespace { SqlSource FindSubstr(const SqlSource& source, const std::string& needle) { @@ -80,25 +86,34 @@ TEST_F(PerfettoSqlParserTest, Empty) { } TEST_F(PerfettoSqlParserTest, SemiColonTerminatedStatement) { - auto res = SqlSource::FromExecuteQuery("SELECT * FROM slice;"); - ASSERT_THAT( - *Parse(res), - testing::ElementsAre(SqliteSql{FindSubstr(res, "SELECT * FROM slice")})); + SqlSource res = SqlSource::FromExecuteQuery("SELECT * FROM slice;"); + PerfettoSqlParser parser(res); + ASSERT_TRUE(parser.Next()); + ASSERT_EQ(parser.statement(), Statement{SqliteSql{}}); + ASSERT_EQ(parser.statement_sql(), FindSubstr(res, "SELECT * FROM slice")); } TEST_F(PerfettoSqlParserTest, MultipleStmts) { auto res = SqlSource::FromExecuteQuery("SELECT * FROM slice; SELECT * FROM s"); - ASSERT_THAT( - *Parse(res), - testing::ElementsAre(SqliteSql{FindSubstr(res, "SELECT * FROM slice")}, - SqliteSql{FindSubstr(res, "SELECT * FROM s")})); + PerfettoSqlParser parser(res); + ASSERT_TRUE(parser.Next()); + ASSERT_EQ(parser.statement(), Statement{SqliteSql{}}); + ASSERT_EQ(parser.statement_sql().sql(), + FindSubstr(res, "SELECT * FROM slice").sql()); + ASSERT_TRUE(parser.Next()); + ASSERT_EQ(parser.statement(), Statement{SqliteSql{}}); + ASSERT_EQ(parser.statement_sql().sql(), + FindSubstr(res, "SELECT * FROM s").sql()); } TEST_F(PerfettoSqlParserTest, IgnoreOnlySpace) { auto res = SqlSource::FromExecuteQuery(" ; SELECT * FROM s; ; ;"); - ASSERT_THAT(*Parse(res), testing::ElementsAre( - SqliteSql{FindSubstr(res, "SELECT * FROM s")})); + PerfettoSqlParser parser(res); + ASSERT_TRUE(parser.Next()); + ASSERT_EQ(parser.statement(), Statement{SqliteSql{}}); + ASSERT_EQ(parser.statement_sql().sql(), + FindSubstr(res, "SELECT * FROM s").sql()); } TEST_F(PerfettoSqlParserTest, CreatePerfettoFunctionScalar) { @@ -140,10 +155,36 @@ TEST_F(PerfettoSqlParserTest, CreatePerfettoFunctionScalarError) { TEST_F(PerfettoSqlParserTest, CreatePerfettoFunctionAndOther) { auto res = SqlSource::FromExecuteQuery( "create perfetto function foo() returns INT as select 1; select foo()"); - ASSERT_THAT(*Parse(res), - testing::ElementsAre(CreateFn{false, "foo()", "INT", - FindSubstr(res, "select 1"), false}, - SqliteSql{FindSubstr(res, "select foo()")})); + PerfettoSqlParser parser(res); + ASSERT_TRUE(parser.Next()); + CreateFn fn{false, "foo()", "INT", FindSubstr(res, "select 1"), false}; + ASSERT_EQ(parser.statement(), Statement{fn}); + ASSERT_EQ( + parser.statement_sql().sql(), + FindSubstr(res, "create perfetto function foo() returns INT as select 1") + .sql()); + ASSERT_TRUE(parser.Next()); + ASSERT_EQ(parser.statement(), Statement{SqliteSql{}}); + ASSERT_EQ(parser.statement_sql().sql(), + FindSubstr(res, "select foo()").sql()); +} + +TEST_F(PerfettoSqlParserTest, IncludePerfettoTrivial) { + auto res = + SqlSource::FromExecuteQuery("include perfetto module cheese.bre_ad;"); + ASSERT_THAT(*Parse(res), testing::ElementsAre(Include{"cheese.bre_ad"})); +} + +TEST_F(PerfettoSqlParserTest, IncludePerfettoErrorAdditionalChars) { + auto res = SqlSource::FromExecuteQuery( + "include perfetto module cheese.bre_ad blabla;"); + ASSERT_FALSE(Parse(res).status().ok()); +} + +TEST_F(PerfettoSqlParserTest, IncludePerfettoErrorWrongModuleName) { + auto res = + SqlSource::FromExecuteQuery("include perfetto module chees*e.bre_ad;"); + ASSERT_FALSE(Parse(res).status().ok()); } } // namespace diff --git a/src/trace_processor/perfetto_sql/intrinsics/functions/import.cc b/src/trace_processor/perfetto_sql/intrinsics/functions/import.cc index 68f2d8890e..30d363e1f2 100644 --- a/src/trace_processor/perfetto_sql/intrinsics/functions/import.cc +++ b/src/trace_processor/perfetto_sql/intrinsics/functions/import.cc @@ -58,33 +58,11 @@ base::Status Import::Run(Import::Context* ctx, const char* import_key = reinterpret_cast(sqlite3_value_text(import_val)); - PERFETTO_TP_TRACE( - metatrace::Category::TOPLEVEL, "Import", - [import_key](metatrace::Record* r) { r->AddArg("Import", import_key); }); - - std::string module_name = sql_modules::GetModuleName(import_key); - auto module = ctx->modules->Find(module_name); - if (!module) - return base::ErrStatus("IMPORT: Unknown module name provided - %s", - import_key); - - auto module_file = module->import_key_to_file.Find(import_key); - if (!module_file) { - return base::ErrStatus("IMPORT: Unknown filename provided - %s", - import_key); - } - // IMPORT is noop for already imported files. - if (module_file->imported) { - return base::OkStatus(); - } - - auto it = ctx->engine->Execute( - SqlSource::FromModuleImport(module_file->sql, import_key)); - RETURN_IF_ERROR(it.status()); - if (it->statement_count_with_output > 0) - return base::ErrStatus("IMPORT: Imported file returning values."); - module_file->imported = true; - return base::OkStatus(); + base::StackString<1024> create("INCLUDE PERFETTO MODULE %s;", import_key); + return ctx->engine + ->Execute( + SqlSource::FromTraceProcessorImplementation(create.ToStdString())) + .status(); } } // namespace trace_processor diff --git a/src/trace_processor/perfetto_sql/intrinsics/functions/import.h b/src/trace_processor/perfetto_sql/intrinsics/functions/import.h index 2e6784d77c..0f9ca650b1 100644 --- a/src/trace_processor/perfetto_sql/intrinsics/functions/import.h +++ b/src/trace_processor/perfetto_sql/intrinsics/functions/import.h @@ -33,7 +33,6 @@ namespace trace_processor { struct Import : public SqlFunction { struct Context { PerfettoSqlEngine* engine; - base::FlatHashMap* modules; }; static constexpr bool kVoidReturn = true; diff --git a/src/trace_processor/sqlite/sql_source.cc b/src/trace_processor/sqlite/sql_source.cc index f23079a56e..622a0c8286 100644 --- a/src/trace_processor/sqlite/sql_source.cc +++ b/src/trace_processor/sqlite/sql_source.cc @@ -80,9 +80,9 @@ SqlSource SqlSource::FromMetricFile(std::string sql, const std::string& name) { return SqlSource(std::move(sql), "Metric file \"" + name + "\"", false); } -SqlSource SqlSource::FromModuleImport(std::string sql, - const std::string& module) { - return SqlSource(std::move(sql), "Module import \"" + module + "\"", false); +SqlSource SqlSource::FromModuleInclude(std::string sql, + const std::string& module) { + return SqlSource(std::move(sql), "Module include \"" + module + "\"", false); } SqlSource SqlSource::FromTraceProcessorImplementation(std::string sql) { diff --git a/src/trace_processor/sqlite/sql_source.h b/src/trace_processor/sqlite/sql_source.h index eff1bf4ea8..805105e825 100644 --- a/src/trace_processor/sqlite/sql_source.h +++ b/src/trace_processor/sqlite/sql_source.h @@ -44,8 +44,9 @@ class SqlSource { static SqlSource FromMetricFile(std::string sql, const std::string& metric_file); - // Creates a SqlSource instance wrapping SQL executed when importing a module. - static SqlSource FromModuleImport(std::string sql, const std::string& module); + // Creates a SqlSource instance wrapping SQL executed when including a module. + static SqlSource FromModuleInclude(std::string sql, + const std::string& module); // Creates a SqlSource instance wrapping SQL which is an internal // implementation detail of trace processor. diff --git a/src/trace_processor/trace_database_integrationtest.cc b/src/trace_processor/trace_database_integrationtest.cc index 2fd2cddceb..35a4e89ad9 100644 --- a/src/trace_processor/trace_database_integrationtest.cc +++ b/src/trace_processor/trace_database_integrationtest.cc @@ -527,16 +527,16 @@ TEST_F(TraceProcessorIntegrationTest, ErrorMessageModule) { ASSERT_TRUE(Processor()->RegisterSqlModule(module).ok()); - auto it = Query("select IMPORT('foo.bar');"); + auto it = Query("include perfetto module foo.bar;"); ASSERT_FALSE(it.Next()); ASSERT_FALSE(it.Status().ok()); ASSERT_EQ(it.Status().message(), R"(Traceback (most recent call last): File "stdin" line 1 col 1 - select IMPORT('foo.bar') + include perfetto module foo.bar ^ - Module import "foo.bar" line 1 col 8 + Module include "foo.bar" line 1 col 8 select t from slice ^ no such column: t)"); diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc index 39f25f3692..56fb013c4e 100644 --- a/src/trace_processor/trace_processor_impl.cc +++ b/src/trace_processor/trace_processor_impl.cc @@ -53,7 +53,6 @@ #include "src/trace_processor/perfetto_sql/intrinsics/functions/clock_functions.h" #include "src/trace_processor/perfetto_sql/intrinsics/functions/create_function.h" #include "src/trace_processor/perfetto_sql/intrinsics/functions/create_view_function.h" -#include "src/trace_processor/perfetto_sql/intrinsics/functions/import.h" #include "src/trace_processor/perfetto_sql/intrinsics/functions/layout_functions.h" #include "src/trace_processor/perfetto_sql/intrinsics/functions/math.h" #include "src/trace_processor/perfetto_sql/intrinsics/functions/pprof_functions.h" @@ -337,7 +336,7 @@ void RegisterDevFunctions(PerfettoSqlEngine* engine) { sql_modules::NameToModule GetStdlibModules() { sql_modules::NameToModule modules; for (const auto& file_to_sql : stdlib::kFileToSql) { - std::string import_key = sql_modules::GetImportKey(file_to_sql.path); + std::string import_key = sql_modules::GetIncludeKey(file_to_sql.path); std::string module = sql_modules::GetModuleName(import_key); modules.Insert(module, {}).first->push_back({import_key, file_to_sql.sql}); } @@ -429,9 +428,9 @@ TraceProcessorImpl::TraceProcessorImpl(const Config& cfg) &engine_); RegisterFunction(&engine_, "EXPERIMENTAL_MEMOIZE", 1, &engine_); - RegisterFunction(&engine_, "IMPORT", 1, - std::unique_ptr( - new Import::Context{&engine_, &sql_modules_})); + RegisterFunction( + &engine_, "IMPORT", 1, + std::unique_ptr(new Import::Context{&engine_})); RegisterFunction( &engine_, "TO_FTRACE", 1, std::unique_ptr(new ToFtrace::Context{ @@ -760,7 +759,7 @@ bool TraceProcessorImpl::IsRootMetricField(const std::string& metric_name) { base::Status TraceProcessorImpl::RegisterSqlModule(SqlModule sql_module) { sql_modules::RegisteredModule new_module; std::string name = sql_module.name; - if (sql_modules_.Find(name) && !sql_module.allow_module_override) { + if (engine_.FindModule(name) && !sql_module.allow_module_override) { return base::ErrStatus( "Module '%s' is already registered. Choose a different name.\n" "If you want to replace the existing module using trace processor " @@ -775,10 +774,10 @@ base::Status TraceProcessorImpl::RegisterSqlModule(SqlModule sql_module) { "key should be module name. Import key: %s, module name: %s.", name_and_sql.first.c_str(), name.c_str()); } - new_module.import_key_to_file.Insert(name_and_sql.first, - {name_and_sql.second, false}); + new_module.include_key_to_file.Insert(name_and_sql.first, + {name_and_sql.second, false}); } - sql_modules_.Insert(name, std::move(new_module)); + engine_.RegisterModule(name, std::move(new_module)); return base::OkStatus(); } diff --git a/src/trace_processor/trace_processor_impl.h b/src/trace_processor/trace_processor_impl.h index b869a63371..8329bda89b 100644 --- a/src/trace_processor/trace_processor_impl.h +++ b/src/trace_processor/trace_processor_impl.h @@ -125,8 +125,6 @@ class TraceProcessorImpl : public TraceProcessor, DescriptorPool pool_; - // Map from module name to module contents. Used for IMPORT function. - base::FlatHashMap sql_modules_; std::vector sql_metrics_; std::unordered_map proto_field_to_sql_metric_path_; diff --git a/src/trace_processor/trace_processor_shell.cc b/src/trace_processor/trace_processor_shell.cc index 7c0380ff9e..f4b4e0c449 100644 --- a/src/trace_processor/trace_processor_shell.cc +++ b/src/trace_processor/trace_processor_shell.cc @@ -1183,7 +1183,7 @@ base::Status IncludeSqlModule(std::string root, bool allow_override) { return base::ErrStatus("Cannot read file %s", filename.c_str()); std::string import_key = - module_name + "." + sql_modules::GetImportKey(path); + module_name + "." + sql_modules::GetIncludeKey(path); modules.Insert(module_name, {}) .first->push_back({import_key, file_contents}); } @@ -1219,7 +1219,7 @@ base::Status LoadOverridenStdlib(std::string root) { if (!base::ReadFile(filename, &file_contents)) { return base::ErrStatus("Cannot read file %s", filename.c_str()); } - std::string import_key = sql_modules::GetImportKey(path); + std::string import_key = sql_modules::GetIncludeKey(path); std::string module = sql_modules::GetModuleName(import_key); modules.Insert(module, {}).first->push_back({import_key, file_contents}); } diff --git a/src/trace_processor/util/sql_modules.h b/src/trace_processor/util/sql_modules.h index a2d5933747..4a52e7daf5 100644 --- a/src/trace_processor/util/sql_modules.h +++ b/src/trace_processor/util/sql_modules.h @@ -31,14 +31,14 @@ using NameToModule = base::FlatHashMap>>; -// Map from import key to sql file. Import key is the string used in IMPORT +// Map from include key to sql file. Include key is the string used in INCLUDE // function. struct RegisteredModule { struct ModuleFile { std::string sql; - bool imported; + bool included; }; - base::FlatHashMap import_key_to_file; + base::FlatHashMap include_key_to_file; }; inline std::string ReplaceSlashWithDot(std::string str) { @@ -50,7 +50,7 @@ inline std::string ReplaceSlashWithDot(std::string str) { return str; } -inline std::string GetImportKey(std::string path) { +inline std::string GetIncludeKey(std::string path) { base::StringView path_view(path); auto path_no_extension = path_view.substr(0, path_view.rfind('.')); return ReplaceSlashWithDot(path_no_extension.ToStdString()); diff --git a/test/trace_processor/diff_tests/perfetto_sql/tests.py b/test/trace_processor/diff_tests/perfetto_sql/tests.py index cbfdd1a8a3..f011d3e2c3 100644 --- a/test/trace_processor/diff_tests/perfetto_sql/tests.py +++ b/test/trace_processor/diff_tests/perfetto_sql/tests.py @@ -36,3 +36,124 @@ def test_create_perfetto_table_double_metric_run(self): 1,"big" 2,"bigger" """)) + + def test_import(self): + return DiffTestBlueprint( + trace=TextProto(r""" + packet { + ftrace_events { + cpu: 1 + event { + timestamp: 1000 + pid: 1 + print { + buf: "C|1000|battery_stats.data_conn|13\n" + } + } + event { + timestamp: 4000 + pid: 1 + print { + buf: "C|1000|battery_stats.data_conn|20\n" + } + } + event { + timestamp: 1000 + pid: 1 + print { + buf: "C|1000|battery_stats.audio|1\n" + } + } + } + } + """), + query=""" + SELECT IMPORT('common.timestamps'); + + SELECT TRACE_START(); + """, + out=Csv(""" + "TRACE_START()" + 1000 + """)) + + def test_include_perfetto_module(self): + return DiffTestBlueprint( + trace=TextProto(r""" + packet { + ftrace_events { + cpu: 1 + event { + timestamp: 1000 + pid: 1 + print { + buf: "C|1000|battery_stats.data_conn|13\n" + } + } + event { + timestamp: 4000 + pid: 1 + print { + buf: "C|1000|battery_stats.data_conn|20\n" + } + } + event { + timestamp: 1000 + pid: 1 + print { + buf: "C|1000|battery_stats.audio|1\n" + } + } + } + } + """), + query=""" + INCLUDE PERFETTO MODULE common.timestamps; + + SELECT TRACE_START(); + """, + out=Csv(""" + "TRACE_START()" + 1000 + """)) + + def test_include_and_import(self): + return DiffTestBlueprint( + trace=TextProto(r""" + packet { + ftrace_events { + cpu: 1 + event { + timestamp: 1000 + pid: 1 + print { + buf: "C|1000|battery_stats.data_conn|13\n" + } + } + event { + timestamp: 4000 + pid: 1 + print { + buf: "C|1000|battery_stats.data_conn|20\n" + } + } + event { + timestamp: 1000 + pid: 1 + print { + buf: "C|1000|battery_stats.audio|1\n" + } + } + } + } + """), + query=""" + SELECT IMPORT('common.timestamps'); + INCLUDE PERFETTO MODULE common.timestamps; + + SELECT TRACE_START(); + """, + out=Csv(""" + "TRACE_START()" + 1000 + """))