feat: refactor row format with slice formant and single/multi slice row format

hybridse/include/codec/fe_row_codec.h

@@ -200,10 +200,12 @@ struct StringColInfo : public ColInfo {
           str_start_offset(str_start_offset) {}
 };
 
-class RowFormat {
+
+
+class SliceFormat {
  public:
-    explicit RowFormat(const hybridse::codec::Schema* schema);
-    virtual ~RowFormat() {}
+    explicit SliceFormat(const hybridse::codec::Schema* schema);
+    virtual ~SliceFormat() {}
 
     bool GetStringColumnInfo(size_t idx, StringColInfo* res) const;
 
@@ -217,6 +219,80 @@ class RowFormat {
     uint32_t str_field_start_offset_;
 };
 
+
+class RowFormat {
+ public:
+    virtual bool GetStringColumnInfo(size_t schema_idx, size_t idx, StringColInfo* res) const = 0;
+    virtual const ColInfo* GetColumnInfo(size_t schema_idx, size_t idx) const = 0;
+    virtual size_t GetSliceId(size_t schema_idx) const = 0;
+};
+
+class MultiSlicesRowFormat : public RowFormat {
+ public:
+    MultiSlicesRowFormat(const Schema* schema) {
+        slice_formats_.push_back(SliceFormat(schema));
+    }
+
+    MultiSlicesRowFormat(std::vector<const Schema*> schemas) {
+        for (auto schema: schemas) {
+            slice_formats_.push_back(SliceFormat(schema));
+        }
+
+    }
+    bool GetStringColumnInfo(size_t schema_idx, size_t idx, StringColInfo* res) const override {
+        return slice_formats_[schema_idx].GetStringColumnInfo(idx, res);
+    }
+
+    const ColInfo* GetColumnInfo(size_t schema_idx, size_t idx) const override {
+        return slice_formats_[schema_idx].GetColumnInfo(idx);
+    }
+
+    size_t GetSliceId(size_t schema_idx) const override {
+        return schema_idx;
+    }
+
+ private:
+    std::vector<SliceFormat> slice_formats_;
+};
+
+class SingleSliceRowFormat : public RowFormat {
+ public:
+    // TODO: Add deconstructor
+    SingleSliceRowFormat(const Schema* schema) {
+        slice_format_ = new SliceFormat(schema);
+    }
+
+    SingleSliceRowFormat(std::vector<const Schema*> schemas) {
+        Schema merge_schema;
+        int offset = 0;
+        for (auto schema: schemas) {
+            offsets_.push_back(offset);
+            offset += schema->size();
+            // TODO: Merge schema
+            //merge_schema.MergeFrom(schema);
+        }
+
+        slice_format_ = new SliceFormat(&merge_schema);
+    }
+
+    bool GetStringColumnInfo(size_t schema_idx, size_t idx, StringColInfo* res) const override {
+        return slice_format_->GetStringColumnInfo(offsets_[schema_idx] + idx, res);
+    }
+
+    const ColInfo* GetColumnInfo(size_t schema_idx, size_t idx) const override {
+        return slice_format_->GetColumnInfo(offsets_[schema_idx] + idx);
+    }
+
+    size_t GetSliceId(size_t schema_idx) const override {
+        return 0;
+    }
+
+ private:
+    std::vector<size_t> offsets_;
+    SliceFormat* slice_format_;
+};
+
+
 }  // namespace codec
 }  // namespace hybridse
 #endif  // HYBRIDSE_INCLUDE_CODEC_FE_ROW_CODEC_H_

hybridse/include/vm/schemas_context.h

@@ -172,9 +172,9 @@ class SchemasContext {
     const PhysicalOpNode* GetRoot() const;
 
     /**
-     * Get detailed format for `idx`th schema source.
+     * Get detailed format.
      */
-    const codec::RowFormat* GetRowFormat(size_t idx) const;
+    const codec::RowFormat* GetRowFormat() const;
 
     /**
      * Get `idx`th schema source.
@@ -267,7 +267,7 @@ class SchemasContext {
     std::vector<SchemaSource*> schema_sources_;
 
     // detailed schema format info
-    std::vector<codec::RowFormat> row_formats_;
+    const codec::RowFormat* row_format_ = nullptr;
 
     // owned schema object
     codec::Schema owned_concat_output_schema_;

hybridse/src/codec/fe_row_codec.cc

@@ -908,7 +908,7 @@ int32_t RowView::GetString(uint32_t idx, const char** val, uint32_t* length) {
                                  length);
 }
 
-RowFormat::RowFormat(const hybridse::codec::Schema* schema)
+SliceFormat::SliceFormat(const hybridse::codec::Schema* schema)
     : schema_(schema), infos_(), next_str_pos_(), str_field_start_offset_(0) {
     if (nullptr == schema) {
         return;
@@ -959,11 +959,11 @@ RowFormat::RowFormat(const hybridse::codec::Schema* schema)
     str_field_start_offset_ = offset;
 }
 
-const ColInfo* RowFormat::GetColumnInfo(size_t idx) const {
+const ColInfo* SliceFormat::GetColumnInfo(size_t idx) const {
     return idx < infos_.size() ? &infos_[idx] : nullptr;
 }
 
-bool RowFormat::GetStringColumnInfo(size_t idx, StringColInfo* res) const {
+bool SliceFormat::GetStringColumnInfo(size_t idx, StringColInfo* res) const {
     if (nullptr == res) {
         LOG(WARNING) << "input args have null";
         return false;
@@ -982,7 +982,7 @@ bool RowFormat::GetStringColumnInfo(size_t idx, StringColInfo* res) const {
         next_offset = nit->second;
     } else {
         if (FLAGS_enable_spark_unsaferow_format) {
-            // Do not need to get next offset for UnsafeRowOpt
+            // No need to get next offset for UnsafeRowOpt and ignore the warning
         } else {
             LOG(WARNING) << "fail to get string field next offset";
             return false;

hybridse/src/codegen/aggregate_ir_builder.cc

@@ -94,8 +94,8 @@ bool AggregateIRBuilder::CollectAggColumn(const hybridse::node::ExprNode* expr,
                 return false;
             }
             const codec::ColInfo& col_info =
-                *schema_context_->GetRowFormat(schema_idx)
-                     ->GetColumnInfo(col_idx);
+                *schema_context_->GetRowFormat()
+                     ->GetColumnInfo(schema_idx, col_idx);
             auto col_type = col_info.type;
             uint32_t offset = col_info.offset;
 
@@ -700,7 +700,7 @@ base::Status AggregateIRBuilder::BuildMulti(const std::string& base_funcname,
 
             ScopeVar dummy_scope_var;
             BufNativeIRBuilder buf_builder(
-                schema_idx, schema_context_->GetRowFormat(schema_idx),
+                schema_idx, schema_context_->GetRowFormat(),
                 body_block, &dummy_scope_var);
             NativeValue field_value;
             CHECK_TRUE(buf_builder.BuildGetField(info.col_idx, slice_info.first, slice_info.second, &field_value),

hybridse/src/codegen/buf_ir_builder.cc

@@ -44,7 +44,7 @@ bool BufNativeIRBuilder::BuildGetField(size_t col_idx, ::llvm::Value* row_ptr, :
     }
 
     node::TypeNode data_type;
-    const codec::ColInfo* col_info = format_->GetColumnInfo(col_idx);
+    const codec::ColInfo* col_info = format_->GetColumnInfo(schema_idx_, col_idx);
     if (col_info == nullptr) {
         LOG(WARNING) << "fail to resolve field info at " << col_idx;
         return false;
@@ -113,7 +113,7 @@ bool BufNativeIRBuilder::BuildGetField(size_t col_idx, ::llvm::Value* row_ptr, :
 
         case ::hybridse::node::kVarchar: {
             codec::StringColInfo str_info;
-            if (!format_->GetStringColumnInfo(col_idx, &str_info)) {
+            if (!format_->GetStringColumnInfo(schema_idx_, col_idx, &str_info)) {
                 LOG(WARNING) << "fail to get string filed offset and next offset" << col_info->name;
             }
             DLOG(INFO) << "get string with offset " << offset << " next offset " << str_info.str_next_offset

hybridse/src/codegen/context.cc

@@ -162,7 +162,7 @@ CodeGenContext::CodeGenContext(::llvm::Module* module,
       llvm_ir_builder_(*llvm_ctx_),
       schemas_context_(schemas_context),
       parameter_types_(parameter_types),
-      parameter_row_format_(parameter_types),
+      parameter_row_format_(new codec::MultiSlicesRowFormat(parameter_types)),
       node_manager_(node_manager) {}
 
 ::llvm::Function* CodeGenContext::GetCurrentFunction() const {
@@ -193,7 +193,7 @@ const vm::SchemasContext* CodeGenContext::schemas_context() const {
 const codec::Schema* CodeGenContext::parameter_types() const {
     return parameter_types_;
 }
-const codec::RowFormat& CodeGenContext::parameter_row_format() const {
+const codec::RowFormat* CodeGenContext::parameter_row_format() const {
     return parameter_row_format_;
 }
 node::NodeManager* CodeGenContext::node_manager() const {

hybridse/src/codegen/context.h

@@ -154,7 +154,7 @@ class CodeGenContext {
 
     const vm::SchemasContext* schemas_context() const;
     const codec::Schema* parameter_types() const;
-    const codec::RowFormat& parameter_row_format() const;
+    const codec::RowFormat* parameter_row_format() const;
     node::NodeManager* node_manager() const;
 
  private:
@@ -172,7 +172,7 @@ class CodeGenContext {
 
     const vm::SchemasContext* schemas_context_;
     const codec::Schema* parameter_types_;
-    const codec::RowFormat parameter_row_format_;
+    codec::RowFormat* parameter_row_format_ = nullptr;
 
     std::unordered_map<std::string, CodeScope> function_scopes_;
 

hybridse/src/codegen/expr_ir_builder.cc

@@ -500,7 +500,7 @@ Status ExprIRBuilder::BuildParameterExpr(const ::hybridse::node::ParameterExpr*
     size_t schema_idx = 0;
     CHECK_STATUS(ExtractSliceFromRow(parameter_row, schema_idx, &slice_ptr, &slice_size))
     BufNativeIRBuilder buf_builder(
-        schema_idx, &ctx_->parameter_row_format(),
+        schema_idx, ctx_->parameter_row_format(),
         ctx_->GetCurrentBlock(), ctx_->GetCurrentScope()->sv());
     CHECK_TRUE(
         buf_builder.BuildGetField(parameter->position()-1, slice_ptr, slice_size, output),
@@ -924,7 +924,7 @@ Status ExprIRBuilder::BuildGetFieldExpr(
         ::llvm::Value* slice_size = nullptr;
         CHECK_STATUS(ExtractSliceFromRow(input_value, schema_idx, &slice_ptr, &slice_size))
         BufNativeIRBuilder buf_builder(
-            schema_idx, schemas_context->GetRowFormat(schema_idx),
+            schema_idx, schemas_context->GetRowFormat(),
             ctx_->GetCurrentBlock(), ctx_->GetCurrentScope()->sv());
         CHECK_TRUE(
             buf_builder.BuildGetField(col_idx, slice_ptr, slice_size, output),

hybridse/src/codegen/window_ir_builder.cc

@@ -116,12 +116,12 @@ bool MemoryWindowDecodeIRBuilder::BuildGetCol(size_t schema_idx, size_t col_idx,
         return false;
     }
     ::hybridse::node::TypeNode data_type;
-    auto row_format = schemas_context_->GetRowFormat(schema_idx);
+    auto row_format = schemas_context_->GetRowFormat();
     if (row_format == nullptr) {
         LOG(WARNING) << "fail to get row format at " << schema_idx;
         return false;
     }
-    const codec::ColInfo* col_info = row_format->GetColumnInfo(col_idx);
+    const codec::ColInfo* col_info = row_format->GetColumnInfo(schema_idx, col_idx);
     if (col_info == nullptr) {
         LOG(WARNING) << "fail to get column info at " << schema_idx << ":"
                      << col_idx;
@@ -148,8 +148,8 @@ bool MemoryWindowDecodeIRBuilder::BuildGetCol(size_t schema_idx, size_t col_idx,
         }
         case ::hybridse::node::kVarchar: {
             codec::StringColInfo str_col_info;
-            if (!schemas_context_->GetRowFormat(schema_idx)
-                     ->GetStringColumnInfo(col_idx, &str_col_info)) {
+            if (!schemas_context_->GetRowFormat()
+                     ->GetStringColumnInfo(schema_idx, col_idx, &str_col_info)) {
                 LOG(WARNING)
                     << "fail to get string filed offset and next offset"
                     << " at " << col_idx;

hybridse/src/vm/schemas_context.cc

@@ -141,7 +141,9 @@ void SchemasContext::Clear() {
         delete ptr;
     }
     schema_sources_.clear();
-    row_formats_.clear();
+    // TODO(tobe): release row_format
+    //row_format_.clear();
+    delete row_format_;
     owned_concat_output_schema_.Clear();
 }
 
@@ -475,8 +477,8 @@ bool SchemasContext::IsColumnAmbiguous(const std::string& column_name) const {
     return column_id_set.size() != 1;
 }
 
-const codec::RowFormat* SchemasContext::GetRowFormat(size_t idx) const {
-    return idx < row_formats_.size() ? &row_formats_[idx] : nullptr;
+const codec::RowFormat* SchemasContext::GetRowFormat() const {
+    return row_format_;
 }
 
 const std::string& SchemasContext::GetName() const {
@@ -510,19 +512,23 @@ const codec::Schema* SchemasContext::GetOutputSchema() const {
 }
 
 bool SchemasContext::CheckBuild() const {
-    return row_formats_.size() == schema_sources_.size();
+    //return row_formats_.size() == schema_sources_.size();
+    return row_format_ == nullptr;
 }
 
 void SchemasContext::Build() {
     // initialize detailed formats
-    row_formats_.clear();
+    //row_formats_.clear();
+    std::vector<const hybridse::codec::Schema*> schemas;
     for (const auto& source : schema_sources_) {
         if (source->GetSchema() == nullptr) {
             LOG(WARNING) << "Source schema is null";
             return;
         }
-        row_formats_.emplace_back(codec::RowFormat(source->GetSchema()));
+        schemas.push_back(source->GetSchema());
     }
+    row_format_ = new codec::MultiSlicesRowFormat(schemas);
+
     // initialize mappings
     column_id_map_.clear();
     column_name_map_.clear();