tp: formalize different parsing modes of trace processor

Add an enum to the TP config which allows configuring how much of trace
processor's parsing should be executed. The following modes are
supported:
1) Default (tokenize, sort, parse)
2) Tokenise only
3) Tokenize and sort only

This change also migrates the existing environment variable for stopping
the parsing stage after sorting to a dev flag and updates the measure
script to use that flag

Change-Id: Iaf0b3565ea7d5d6f1a3d94ae3d9ea2d9ec08bc15

include/perfetto/trace_processor/basic_types.h

@@ -36,9 +36,35 @@ namespace perfetto::trace_processor {
 // protos are mounted onto a virtual path inside this directory.
 constexpr char kMetricProtoRoot[] = "protos/perfetto/metrics/";
 
+// Enum which encodes how trace processor should parse the ingested data.
+enum class ParsingMode {
+  // This option causes trace processor to tokenize the raw trace bytes, sort
+  // the events into timestamp order and parse the events into tables.
+  //
+  // This is the default mode.
+  kDefault = 0,
+
+  // This option causes trace processor to skip the sorting and parsing
+  // steps of ingesting a trace, only retaining any information which could be
+  // gathered during tokenization of the trace files.
+  //
+  // Note the exact information available with this option is left intentionally
+  // undefined as it relies heavily on implementation details of trace
+  // processor. It is mainly intended for use by the Perfetto UI which
+  // integrates very closely with trace processor. General users should use
+  // `kDefault` unless they know what they are doing.
+  kTokenizeOnly = 1,
+
+  // This option causes trace processor to skip the parsing step of ingesting
+  // a trace.
+  //
+  // Note this option does not offer any visible benefits over `kTokenizeOnly`
+  // but has the downside of being slower. It mainly exists for use by
+  // developers debugging performance of trace processor.
+  kTokenizeAndSort = 2,
+};
+
 // Enum which encodes how trace processor should try to sort the ingested data.
-// Note that these options are only applicable to proto traces; other trace
-// types (e.g. JSON, Fuchsia) use full sorts.
 enum class SortingMode {
   // This option allows trace processor to use built-in heuristics about how to
   // sort the data. Generally, this option is correct for most embedders as
@@ -50,8 +76,8 @@ enum class SortingMode {
   // This is the default mode.
   kDefaultHeuristics = 0,
 
-  // This option forces trace processor to wait for all trace packets to be
-  // passed to it before doing a full sort of all the packets. This causes any
+  // This option forces trace processor to wait for all events to be passed to
+  // it before doing a full sort of all the events. This causes any
   // heuristics trace processor would normally use to ingest partially sorted
   // data to be skipped.
   kForceFullSort = 1,
@@ -112,8 +138,13 @@ enum class DropTrackEventDataBefore {
 
 // Struct for configuring a TraceProcessor instance (see trace_processor.h).
 struct PERFETTO_EXPORT_COMPONENT Config {
-  // Indicates the sortinng mode that trace processor should use on the passed
-  // trace packets. See the enum documentation for more details.
+  // Indicates the parsing mode trace processor should use to extract
+  // information from the raw trace bytes. See the enum documentation for more
+  // details.
+  ParsingMode parsing_mode = ParsingMode::kDefault;
+
+  // Indicates the sortinng mode that trace processor should use on the
+  // passed trace packets. See the enum documentation for more details.
   SortingMode sorting_mode = SortingMode::kDefaultHeuristics;
 
   // When set to false, this option makes the trace processor not include ftrace

protos/perfetto/trace_processor/trace_processor.proto

@@ -60,7 +60,8 @@ enum TraceProcessorApiVersion {
   // 12. Changed UI to be more aggresive about version matching.
   //     Added version_code.
   // 13. Added TPM_REGISTER_SQL_MODULE method.
-  TRACE_PROCESSOR_CURRENT_API_VERSION = 13;
+  // 14. Added parsing mode option to RESET method.
+  TRACE_PROCESSOR_CURRENT_API_VERSION = 14;
 }
 
 // At lowest level, the wire-format of the RPC protocol is a linear sequence of
@@ -330,11 +331,17 @@ message ResetTraceProcessorArgs {
     NO_DROP = 0;
     TRACK_EVENT_RANGE_OF_INTEREST = 1;
   }
+  enum ParsingMode {
+    DEFAULT = 0;
+    TOKENIZE_ONLY = 1;
+    TOKENIZE_AND_SORT = 2;
+  }
   // Mirror of the corresponding perfetto::trace_processor::Config fields.
   optional DropTrackEventDataBefore drop_track_event_data_before = 1;
   optional bool ingest_ftrace_in_raw_table = 2;
   optional bool analyze_trace_proto_content = 3;
   optional bool ftrace_drop_until_all_cpus_valid = 4;
+  optional ParsingMode parsing_mode = 5;
 }
 
 message RegisterSqlPackageArgs {

src/trace_processor/forwarding_trace_parser.cc

@@ -138,7 +138,26 @@ void ForwardingTraceParser::UpdateSorterForTraceType(TraceType trace_type) {
   }
 
   if (!context_->sorter) {
-    context_->sorter.reset(new TraceSorter(context_, *minimum_sorting_mode));
+    TraceSorter::EventHandling event_handling;
+    switch (context_->config.parsing_mode) {
+      case ParsingMode::kDefault:
+        event_handling = TraceSorter::EventHandling::kSortAndPush;
+        break;
+      case ParsingMode::kTokenizeOnly:
+        event_handling = TraceSorter::EventHandling::kDrop;
+        break;
+      case ParsingMode::kTokenizeAndSort:
+        event_handling = TraceSorter::EventHandling::kSortAndDrop;
+        break;
+    }
+    if (context_->config.enable_dev_features) {
+      auto it = context_->config.dev_flags.find("drop-after-sort");
+      if (it != context_->config.dev_flags.end() && it->second == "true") {
+        event_handling = TraceSorter::EventHandling::kSortAndDrop;
+      }
+    }
+    context_->sorter = std::make_shared<TraceSorter>(
+        context_, *minimum_sorting_mode, event_handling);
   }
 
   switch (context_->sorter->sorting_mode()) {

src/trace_processor/rpc/rpc.cc

@@ -407,6 +407,18 @@ void Rpc::ResetTraceProcessor(const uint8_t* args, size_t len) {
             ? SoftDropFtraceDataBefore::kAllPerCpuBuffersValid
             : SoftDropFtraceDataBefore::kNoDrop;
   }
+  using Args = protos::pbzero::ResetTraceProcessorArgs;
+  switch (reset_trace_processor_args.parsing_mode()) {
+    case Args::ParsingMode::DEFAULT:
+      config.parsing_mode = ParsingMode::kDefault;
+      break;
+    case Args::ParsingMode::TOKENIZE_ONLY:
+      config.parsing_mode = ParsingMode::kTokenizeOnly;
+      break;
+    case Args::ParsingMode::TOKENIZE_AND_SORT:
+      config.parsing_mode = ParsingMode::kTokenizeAndSort;
+      break;
+  }
   ResetTraceProcessorInternal(config);
 }
 

src/trace_processor/sorter/trace_sorter.cc

@@ -45,13 +45,12 @@
 namespace perfetto::trace_processor {
 
 TraceSorter::TraceSorter(TraceProcessorContext* context,
-                         SortingMode sorting_mode)
-    : sorting_mode_(sorting_mode), storage_(context->storage) {
+                         SortingMode sorting_mode,
+                         EventHandling event_handling)
+    : sorting_mode_(sorting_mode),
+      storage_(context->storage),
+      event_handling_(event_handling) {
   AddMachineContext(context);
-  const char* env = getenv("TRACE_PROCESSOR_SORT_ONLY");
-  bypass_next_stage_for_testing_ = env && !strcmp(env, "1");
-  if (bypass_next_stage_for_testing_)
-    PERFETTO_ELOG("TEST MODE: bypassing protobuf parsing stage");
 }
 
 TraceSorter::~TraceSorter() {
@@ -427,12 +426,13 @@ void TraceSorter::MaybeExtractEvent(size_t min_machine_idx,
 
   latest_pushed_event_ts_ = std::max(latest_pushed_event_ts_, timestamp);
 
-  if (PERFETTO_UNLIKELY(bypass_next_stage_for_testing_)) {
+  if (PERFETTO_UNLIKELY(event_handling_ == EventHandling::kSortAndDrop)) {
     // Parse* would extract this event and push it to the next stage. Since we
     // are skipping that, just extract and discard it.
     ExtractAndDiscardTokenizedObject(event);
     return;
   }
+  PERFETTO_DCHECK(event_handling_ == EventHandling::kSortAndPush);
 
   if (queue_idx == 0) {
     ParseTracePacket(*machine_context, event);