From 7ee45be801d582d9c5d70c36b798f57077f6355b Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Mon, 17 Feb 2025 17:53:39 -0800 Subject: [PATCH 01/11] Add ValueStore parameter to GetKeyDescriptor and GetValueDescriptor --- .../doltcore/doltdb/durable/artifact_index.go | 4 +- go/libraries/doltcore/doltdb/durable/index.go | 2 +- go/libraries/doltcore/doltdb/ignore.go | 3 +- .../doltcore/merge/merge_prolly_indexes.go | 2 +- .../doltcore/merge/merge_prolly_rows.go | 38 ++++++++++--------- go/libraries/doltcore/merge/merge_test.go | 15 ++++++-- .../doltcore/merge/mutable_secondary_index.go | 7 ++-- go/libraries/doltcore/merge/row_merge_test.go | 4 +- .../doltcore/merge/schema_merge_test.go | 10 +++-- go/libraries/doltcore/merge/violations_fk.go | 2 +- go/libraries/doltcore/migrate/tuples.go | 4 +- go/libraries/doltcore/schema/schema.go | 8 ++-- .../sqle/binlogreplication/binlog_producer.go | 10 +++-- .../binlog_row_serialization.go | 8 ++-- .../sqle/dtables/conflicts_tables_prolly.go | 10 ++--- .../dtables/constraint_violations_prolly.go | 2 +- .../doltcore/sqle/dtables/prolly_row_conv.go | 2 +- .../sqle/dtables/query_catalog_table.go | 6 ++- .../doltcore/sqle/index/key_builder.go | 4 +- go/libraries/doltcore/sqle/kvexec/builder.go | 4 +- .../doltcore/sqle/kvexec/lookup_join.go | 4 +- .../doltcore/sqle/statsnoms/database.go | 5 ++- go/libraries/doltcore/sqle/statsnoms/write.go | 4 +- .../doltcore/sqle/writer/schema_cache.go | 2 +- .../editor/creation/external_build_index.go | 2 +- .../doltcore/table/editor/creation/index.go | 2 +- go/store/prolly/shim/shim.go | 8 ++-- go/store/prolly/tree/node.go | 4 +- 28 files changed, 97 insertions(+), 79 deletions(-) diff --git a/go/libraries/doltcore/doltdb/durable/artifact_index.go b/go/libraries/doltcore/doltdb/durable/artifact_index.go index 962602e681b..83a460cae05 100644 --- a/go/libraries/doltcore/doltdb/durable/artifact_index.go +++ b/go/libraries/doltcore/doltdb/durable/artifact_index.go @@ -64,7 +64,7 @@ func NewEmptyArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, ns tr panic("TODO") case types.Format_DOLT: - kd := tableSch.GetKeyDescriptor() + kd := tableSch.GetKeyDescriptor(ns) m, err := prolly.NewArtifactMapFromTuples(ctx, ns, kd) if err != nil { return nil, err @@ -108,7 +108,7 @@ func artifactIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tr if fileId != serial.MergeArtifactsFileID { return nil, fmt.Errorf("unexpected file ID for artifact node, expected %s, found %s", serial.MergeArtifactsFileID, fileId) } - kd := tableSch.GetKeyDescriptor() + kd := tableSch.GetKeyDescriptor(ns) m := prolly.NewArtifactMap(root, ns, kd) return ArtifactIndexFromProllyMap(m), nil diff --git a/go/libraries/doltcore/doltdb/durable/index.go b/go/libraries/doltcore/doltdb/durable/index.go index 726e2038154..d6f2ecef74f 100644 --- a/go/libraries/doltcore/doltdb/durable/index.go +++ b/go/libraries/doltcore/doltdb/durable/index.go @@ -153,7 +153,7 @@ func newEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS return IndexFromNomsMap(m, vrw, ns), nil case types.Format_DOLT: - kd, vd := sch.GetMapDescriptors() + kd, vd := sch.GetMapDescriptors(ns) if isKeylessSecondary { kd = prolly.AddHashToSchema(kd) } diff --git a/go/libraries/doltcore/doltdb/ignore.go b/go/libraries/doltcore/doltdb/ignore.go index c6dc85a9b2a..95189d66c9a 100644 --- a/go/libraries/doltcore/doltdb/ignore.go +++ b/go/libraries/doltcore/doltdb/ignore.go @@ -111,7 +111,8 @@ func GetIgnoredTablePatterns(ctx context.Context, roots Roots, schemas []string) if err != nil { return nil, err } - keyDesc, valueDesc := ignoreTableSchema.GetMapDescriptors() + // The ignore table doesn't have any addressable columns, so it's okay to not pass in a ValueStore here. + keyDesc, valueDesc := ignoreTableSchema.GetMapDescriptors(nil) ignoreTableMap, err := durable.ProllyMapFromIndex(index).IterAll(ctx) if err != nil { diff --git a/go/libraries/doltcore/merge/merge_prolly_indexes.go b/go/libraries/doltcore/merge/merge_prolly_indexes.go index 2b9e488776d..416e14d25db 100644 --- a/go/libraries/doltcore/merge/merge_prolly_indexes.go +++ b/go/libraries/doltcore/merge/merge_prolly_indexes.go @@ -122,7 +122,7 @@ func buildIndex( if err != nil { return nil, err } - kd := postMergeSchema.GetKeyDescriptor() + kd := postMergeSchema.GetKeyDescriptor(ns) kb := val.NewTupleBuilder(kd) p := m.Pool() diff --git a/go/libraries/doltcore/merge/merge_prolly_rows.go b/go/libraries/doltcore/merge/merge_prolly_rows.go index 82b69b70295..0f02d504737 100644 --- a/go/libraries/doltcore/merge/merge_prolly_rows.go +++ b/go/libraries/doltcore/merge/merge_prolly_rows.go @@ -119,6 +119,8 @@ func mergeProllyTable( // conflicts are detected, this function attempts to resolve them automatically if possible, and // if not, they are recorded as conflicts in the table's artifacts. func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Schema, mergeTbl *doltdb.Table, valueMerger *valueMerger, mergeInfo MergeInfo, diffInfo tree.ThreeWayDiffInfo) (*doltdb.Table, *MergeStats, error) { + ns := tm.ns + iter, err := threeWayDiffer(ctx, tm, valueMerger, diffInfo) if err != nil { return nil, nil, err @@ -128,7 +130,7 @@ func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Sch if err != nil { return nil, nil, err } - leftEditor := durable.ProllyMapFromIndex(lr).Rewriter(finalSch.GetKeyDescriptor(), finalSch.GetValueDescriptor()) + leftEditor := durable.ProllyMapFromIndex(lr).Rewriter(finalSch.GetKeyDescriptor(ns), finalSch.GetValueDescriptor(ns)) ai, err := mergeTbl.GetArtifacts(ctx) if err != nil { @@ -419,17 +421,17 @@ func (cv checkValidator) validateDiff(ctx *sql.Context, diff tree.ThreeWayDiff) return 0, nil case tree.DiffOpLeftAdd, tree.DiffOpLeftModify: valueTuple = diff.Left - valueDesc = cv.tableMerger.leftSch.GetValueDescriptor() + valueDesc = cv.tableMerger.leftSch.GetValueDescriptor(cv.tableMerger.ns) case tree.DiffOpRightAdd, tree.DiffOpRightModify: valueTuple = diff.Right - valueDesc = cv.tableMerger.rightSch.GetValueDescriptor() + valueDesc = cv.tableMerger.rightSch.GetValueDescriptor(cv.tableMerger.ns) case tree.DiffOpConvergentAdd, tree.DiffOpConvergentModify: // both sides made the same change, just take the left valueTuple = diff.Left - valueDesc = cv.tableMerger.leftSch.GetValueDescriptor() + valueDesc = cv.tableMerger.leftSch.GetValueDescriptor(cv.tableMerger.ns) case tree.DiffOpDivergentModifyResolved: valueTuple = diff.Merged - valueDesc = cv.tableMerger.leftSch.GetValueDescriptor() + valueDesc = cv.tableMerger.leftSch.GetValueDescriptor(cv.tableMerger.ns) } for checkName, checkExpression := range cv.checkExpressions { @@ -571,14 +573,14 @@ func (uv uniqValidator) validateDiff(ctx *sql.Context, diff tree.ThreeWayDiff) ( value = diff.Right // Don't remap the value to the merged schema if the table is keyless or if the mapping is an identity mapping. if !uv.valueMerger.keyless && !uv.valueMerger.rightMapping.IsIdentityMapping() { - modifiedValue := remapTuple(value, uv.tm.rightSch.GetValueDescriptor(), uv.valueMerger.rightMapping) + modifiedValue := remapTuple(value, uv.tm.rightSch.GetValueDescriptor(uv.valueMerger.ns), uv.valueMerger.rightMapping) value = val.NewTuple(uv.valueMerger.syncPool, modifiedValue...) } case tree.DiffOpLeftAdd, tree.DiffOpLeftModify: value = diff.Left // Don't remap the value to the merged schema if the table is keyless or if the mapping is an identity mapping. if !uv.valueMerger.keyless && !uv.valueMerger.leftMapping.IsIdentityMapping() { - modifiedValue := remapTuple(value, uv.tm.leftSch.GetValueDescriptor(), uv.valueMerger.leftMapping) + modifiedValue := remapTuple(value, uv.tm.leftSch.GetValueDescriptor(uv.valueMerger.ns), uv.valueMerger.leftMapping) value = val.NewTuple(uv.valueMerger.syncPool, modifiedValue...) } case tree.DiffOpRightDelete: @@ -1101,7 +1103,7 @@ func (m *primaryMerger) merge(ctx *sql.Context, diff tree.ThreeWayDiff, sourceSc ctx, diff.Key, diff.Right, - sourceSch.GetValueDescriptor(), + sourceSch.GetValueDescriptor(m.valueMerger.ns), m.valueMerger.rightMapping, m.tableMerger, m.tableMerger.rightSch, @@ -1139,7 +1141,7 @@ func (m *primaryMerger) merge(ctx *sql.Context, diff tree.ThreeWayDiff, sourceSc ctx, diff.Key, merged, - m.finalSch.GetValueDescriptor(), + m.finalSch.GetValueDescriptor(m.valueMerger.ns), m.valueMerger.rightMapping, m.tableMerger, m.tableMerger.rightSch, @@ -1170,7 +1172,7 @@ func (m *primaryMerger) merge(ctx *sql.Context, diff tree.ThreeWayDiff, sourceSc return fmt.Errorf("cannot merge keyless tables with reordered columns") } } else { - tempTupleValue, err := remapTupleWithColumnDefaults(ctx, diff.Key, newTupleValue, sourceSch.GetValueDescriptor(), + tempTupleValue, err := remapTupleWithColumnDefaults(ctx, diff.Key, newTupleValue, sourceSch.GetValueDescriptor(m.valueMerger.ns), m.valueMerger.leftMapping, m.tableMerger, m.tableMerger.leftSch, m.finalSch, m.defaults, m.valueMerger.syncPool, false) if err != nil { return err @@ -1284,7 +1286,7 @@ func newSecondaryMerger(ctx *sql.Context, tm *TableMerger, valueMerger *valueMer } // Use the mergedSchema to work with the secondary indexes, to pull out row data using the right // pri_index -> sec_index mapping. - lm, err := GetMutableSecondaryIdxsWithPending(ctx, leftSchema, mergedSchema, tm.name.Name, ls, secondaryMergerPendingSize) + lm, err := GetMutableSecondaryIdxsWithPending(ctx, tm.ns, leftSchema, mergedSchema, tm.name.Name, ls, secondaryMergerPendingSize) if err != nil { return nil, err } @@ -1339,7 +1341,7 @@ func (m *secondaryMerger) merge(ctx *sql.Context, diff tree.ThreeWayDiff, leftSc ctx, diff.Key, diff.Right, - m.valueMerger.rightSchema.GetValueDescriptor(), + m.valueMerger.rightSchema.GetValueDescriptor(m.valueMerger.ns), m.valueMerger.rightMapping, m.tableMerger, m.tableMerger.rightSch, @@ -1364,7 +1366,7 @@ func (m *secondaryMerger) merge(ctx *sql.Context, diff tree.ThreeWayDiff, leftSc diff.Key, diff.Base, // Only the right side was modified, so the base schema must be the same as the left schema - leftSchema.GetValueDescriptor(), + leftSchema.GetValueDescriptor(m.valueMerger.ns), m.valueMerger.baseMapping, tm, m.tableMerger.ancSch, @@ -1453,7 +1455,7 @@ func remapTupleWithColumnDefaults( pool pool.BuffPool, rightSide bool, ) (val.Tuple, error) { - tb := val.NewTupleBuilder(mergedSch.GetValueDescriptor()) + tb := val.NewTupleBuilder(mergedSch.GetValueDescriptor(tm.ns)) var secondPass []int for to, from := range mapping { @@ -1631,10 +1633,10 @@ func newValueMerger(merged, leftSch, rightSch, baseSch schema.Schema, syncPool p return &valueMerger{ numCols: merged.GetNonPKCols().StoredSize(), - baseVD: baseSch.GetValueDescriptor(), - rightVD: rightSch.GetValueDescriptor(), - resultVD: merged.GetValueDescriptor(), - leftVD: leftSch.GetValueDescriptor(), + baseVD: baseSch.GetValueDescriptor(ns), + rightVD: rightSch.GetValueDescriptor(ns), + resultVD: merged.GetValueDescriptor(ns), + leftVD: leftSch.GetValueDescriptor(ns), resultSchema: merged, leftMapping: leftMapping, rightMapping: rightMapping, diff --git a/go/libraries/doltcore/merge/merge_test.go b/go/libraries/doltcore/merge/merge_test.go index 330ffa697f4..347bf27223a 100644 --- a/go/libraries/doltcore/merge/merge_test.go +++ b/go/libraries/doltcore/merge/merge_test.go @@ -63,8 +63,8 @@ type rowV struct { col1, col2 int } -var vD = sch.GetValueDescriptor() -var vB = val.NewTupleBuilder(vD) +var vD val.TupleDesc +var vB *val.TupleBuilder var syncPool = pool.NewBuffPool() func (v rowV) value() val.Tuple { @@ -422,6 +422,13 @@ func setupMergeTest(t *testing.T) (*doltdb.DoltDB, types.ValueReadWriter, tree.N ddb := mustMakeEmptyRepo(t) vrw := ddb.ValueReadWriter() ns := ddb.NodeStore() + + vD = sch.GetValueDescriptor(ns) + vB = val.NewTupleBuilder(vD) + + kD = sch.GetKeyDescriptor(ns) + kB = val.NewTupleBuilder(kD) + sortTests(testRows) var initialKVs []val.Tuple @@ -773,8 +780,8 @@ func buildLeftRightAncCommitsAndBranches(t *testing.T, ddb *doltdb.DoltDB, rootT return mergeCommit, ancCm, root, mergeRoot, ancRoot } -var kD = sch.GetKeyDescriptor() -var kB = val.NewTupleBuilder(kD) +var kD val.TupleDesc +var kB *val.TupleBuilder func key(i int) val.Tuple { kB.PutInt64(0, int64(i)) diff --git a/go/libraries/doltcore/merge/mutable_secondary_index.go b/go/libraries/doltcore/merge/mutable_secondary_index.go index f91e1ee16b3..4d4271cd551 100644 --- a/go/libraries/doltcore/merge/mutable_secondary_index.go +++ b/go/libraries/doltcore/merge/mutable_secondary_index.go @@ -16,6 +16,7 @@ package merge import ( "context" + "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/go-mysql-server/sql" @@ -46,7 +47,7 @@ func GetMutableSecondaryIdxs(ctx *sql.Context, ourSch, sch schema.Schema, tableN // GetMutableSecondaryIdxsWithPending returns a MutableSecondaryIdx for each secondary index in |indexes|. If an index // is listed in the given |sch|, but does not exist in the given |indexes|, then it is skipped. This is useful when // merging a schema that has a new index, but the index does not exist on the index set being modified. -func GetMutableSecondaryIdxsWithPending(ctx *sql.Context, ourSch, sch schema.Schema, tableName string, indexes durable.IndexSet, pendingSize int) ([]MutableSecondaryIdx, error) { +func GetMutableSecondaryIdxsWithPending(ctx *sql.Context, ns tree.NodeStore, ourSch, sch schema.Schema, tableName string, indexes durable.IndexSet, pendingSize int) ([]MutableSecondaryIdx, error) { mods := make([]MutableSecondaryIdx, 0, sch.Indexes().Count()) for _, index := range sch.Indexes().AllIndexes() { @@ -78,11 +79,11 @@ func GetMutableSecondaryIdxsWithPending(ctx *sql.Context, ourSch, sch schema.Sch idxKeyDesc = idxKeyDesc.PrefixDesc(idxKeyDesc.Count() - 1) } - if !idxKeyDesc.Equals(index.Schema().GetKeyDescriptorWithNoConversion()) { + if !idxKeyDesc.Equals(index.Schema().GetKeyDescriptorWithNoConversion(ns)) { continue } - if !m.ValDesc().Equals(index.Schema().GetValueDescriptor()) { + if !m.ValDesc().Equals(index.Schema().GetValueDescriptor(ns)) { continue } diff --git a/go/libraries/doltcore/merge/row_merge_test.go b/go/libraries/doltcore/merge/row_merge_test.go index c8946f7c5e7..f40e34354de 100644 --- a/go/libraries/doltcore/merge/row_merge_test.go +++ b/go/libraries/doltcore/merge/row_merge_test.go @@ -216,7 +216,7 @@ func TestRowMerge(t *testing.T) { merged, ok, err := v.tryMerge(ctx, test.row, test.mergeRow, test.ancRow) assert.NoError(t, err) assert.Equal(t, test.expectConflict, !ok) - vD := test.mergedSch.GetValueDescriptor() + vD := test.mergedSch.GetValueDescriptor(ns) assert.Equal(t, vD.Format(test.expectedResult), vD.Format(merged)) }) } @@ -335,7 +335,7 @@ func buildTup(sch schema.Schema, r []*int) val.Tuple { return nil } - vD := sch.GetValueDescriptor() + vD := sch.GetValueDescriptor(ns) vB := val.NewTupleBuilder(vD) for i, v := range r { if v != nil { diff --git a/go/libraries/doltcore/merge/schema_merge_test.go b/go/libraries/doltcore/merge/schema_merge_test.go index 3f9820bacb9..d4e971f616a 100644 --- a/go/libraries/doltcore/merge/schema_merge_test.go +++ b/go/libraries/doltcore/merge/schema_merge_test.go @@ -1633,6 +1633,7 @@ func testSchemaMergeHelper(t *testing.T, tests []schemaMergeTest, flipSides bool runTest := func(t *testing.T, test schemaMergeTest, expectDataConflict bool, expConstraintViolations []constraintViolation) { ctx := context.Background() a, l, r, m := setupSchemaMergeTest(ctx, t, test) + ns := m.NodeStore() var mo merge.MergeOpts var eo editor.Options @@ -1669,7 +1670,7 @@ func testSchemaMergeHelper(t *testing.T, tests []schemaMergeTest, flipSides bool table, _, err := result.Root.GetTable(ctx, name) require.NoError(t, err) t.Logf("table %s:", name) - t.Log(table.DebugString(ctx, m.NodeStore())) + t.Log(table.DebugString(ctx, ns)) } } @@ -1690,7 +1691,8 @@ func testSchemaMergeHelper(t *testing.T, tests []schemaMergeTest, flipSides bool sch, err := actTbl.GetSchema(ctx) require.NoError(t, err) - kd, vd := sch.GetMapDescriptors() + + kd, vd := sch.GetMapDescriptors(m.NodeStore()) if len(expConstraintViolations) > 0 { artifacts, err := actTbl.GetArtifacts(ctx) @@ -1720,10 +1722,10 @@ func testSchemaMergeHelper(t *testing.T, tests []schemaMergeTest, flipSides bool require.NoError(t, err) actRowDataHash, err := actTbl.GetRowDataHash(ctx) require.NoError(t, err) - if !expSchema.GetKeyDescriptor().Equals(kd) { + if !expSchema.GetKeyDescriptor(ns).Equals(kd) { t.Fatal("Primary key descriptors unequal") } - if !expSchema.GetValueDescriptor().Equals(vd) { + if !expSchema.GetValueDescriptor(ns).Equals(vd) { t.Fatal("Value descriptors unequal") } if expRowDataHash != actRowDataHash { diff --git a/go/libraries/doltcore/merge/violations_fk.go b/go/libraries/doltcore/merge/violations_fk.go index 0fbd8e58222..d0ce2585067 100644 --- a/go/libraries/doltcore/merge/violations_fk.go +++ b/go/libraries/doltcore/merge/violations_fk.go @@ -267,7 +267,7 @@ func (f *foreignKeyViolationWriter) StartFK(ctx context.Context, fk doltdb.Forei artMap := durable.ProllyMapFromArtifactIndex(arts) f.artEditor = artMap.Editor() f.cInfoJsonData = jsonData - f.kd = sch.GetKeyDescriptor() + f.kd = sch.GetKeyDescriptor(tbl.NodeStore()) } else { violMap, err := tbl.GetConstraintViolations(ctx) if err != nil { diff --git a/go/libraries/doltcore/migrate/tuples.go b/go/libraries/doltcore/migrate/tuples.go index 8345e0378f8..a1489da0e09 100644 --- a/go/libraries/doltcore/migrate/tuples.go +++ b/go/libraries/doltcore/migrate/tuples.go @@ -51,8 +51,8 @@ type translator struct { } func tupleTranslatorsFromSchema(sch schema.Schema, ns tree.NodeStore) (kt, vt translator) { - kd := sch.GetKeyDescriptor() - vd := sch.GetValueDescriptor() + kd := sch.GetKeyDescriptor(ns) + vd := sch.GetValueDescriptor(ns) keyMap := sch.GetPKCols().TagToIdx valMap := sch.GetNonPKCols().TagToIdx diff --git a/go/libraries/doltcore/schema/schema.go b/go/libraries/doltcore/schema/schema.go index bbdbdfd6e14..f5844993779 100644 --- a/go/libraries/doltcore/schema/schema.go +++ b/go/libraries/doltcore/schema/schema.go @@ -74,20 +74,20 @@ type Schema interface { AddColumn(column Column, order *ColumnOrder) (Schema, error) // GetMapDescriptors returns the key and value tuple descriptors for this schema. - GetMapDescriptors() (keyDesc, valueDesc val.TupleDesc) + GetMapDescriptors(vs val.ValueStore) (keyDesc, valueDesc val.TupleDesc) // GetKeyDescriptor returns the key tuple descriptor for this schema. // If a column has a type that can't appear in a key (such as "address" columns), // that column will get converted to equivalent types that can. (Example: text -> varchar) - GetKeyDescriptor() val.TupleDesc + GetKeyDescriptor(vs val.ValueStore) val.TupleDesc // GetKeyDescriptorWithNoConversion returns the a descriptor for the columns used in the key. // Unlike `GetKeyDescriptor`, it doesn't attempt to convert columns if they can't appear in a key, // and returns them as they are. - GetKeyDescriptorWithNoConversion() val.TupleDesc + GetKeyDescriptorWithNoConversion(vs val.ValueStore) val.TupleDesc // GetValueDescriptor returns the value tuple descriptor for this schema. - GetValueDescriptor() val.TupleDesc + GetValueDescriptor(vs val.ValueStore) val.TupleDesc // GetCollation returns the table's collation. GetCollation() Collation diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_producer.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_producer.go index 4fcdec78d20..a17a9fedbbc 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_producer.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_producer.go @@ -575,9 +575,11 @@ func extractRowCountAndDiffType(sch schema.Schema, diff tree.Diff) (rowCount uin return 1, diff.Type, nil } + // This descriptor doesn't need a nodestore because it's never used to access an address field. + vd := sch.GetValueDescriptor(nil) switch diff.Type { case tree.AddedDiff: - toRowCount, notNull := sch.GetValueDescriptor().GetUint64(0, val.Tuple(diff.To)) + toRowCount, notNull := vd.GetUint64(0, val.Tuple(diff.To)) if !notNull { return 0, 0, fmt.Errorf( "row count for a keyless table row cannot be null") @@ -585,7 +587,7 @@ func extractRowCountAndDiffType(sch schema.Schema, diff tree.Diff) (rowCount uin return toRowCount, diff.Type, nil case tree.RemovedDiff: - fromRowCount, notNull := sch.GetValueDescriptor().GetUint64(0, val.Tuple(diff.From)) + fromRowCount, notNull := vd.GetUint64(0, val.Tuple(diff.From)) if !notNull { return 0, 0, fmt.Errorf( "row count for a keyless table row cannot be null") @@ -593,13 +595,13 @@ func extractRowCountAndDiffType(sch schema.Schema, diff tree.Diff) (rowCount uin return fromRowCount, diff.Type, nil case tree.ModifiedDiff: - toRowCount, notNull := sch.GetValueDescriptor().GetUint64(0, val.Tuple(diff.To)) + toRowCount, notNull := vd.GetUint64(0, val.Tuple(diff.To)) if !notNull { return 0, 0, fmt.Errorf( "row count for a keyless table row cannot be null") } - fromRowCount, notNull := sch.GetValueDescriptor().GetUint64(0, val.Tuple(diff.From)) + fromRowCount, notNull := vd.GetUint64(0, val.Tuple(diff.From)) if !notNull { return 0, 0, fmt.Errorf( "row count for a keyless table row cannot be null") diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_row_serialization.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_row_serialization.go index fb41d739022..04daa0d9b97 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_row_serialization.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_row_serialization.go @@ -43,13 +43,13 @@ type rowSerializationIter struct { // newRowSerializationIter creates a new rowSerializationIter for the specified |schema| and row data from the // |key| and |value| tuples. -func newRowSerializationIter(sch schema.Schema, key, value tree.Item) *rowSerializationIter { +func newRowSerializationIter(sch schema.Schema, key, value tree.Item, ns tree.NodeStore) *rowSerializationIter { return &rowSerializationIter{ sch: sch, key: val.Tuple(key), - keyDesc: sch.GetKeyDescriptor(), + keyDesc: sch.GetKeyDescriptor(ns), value: val.Tuple(value), - valueDesc: sch.GetValueDescriptor(), + valueDesc: sch.GetValueDescriptor(ns), keyIdx: -1, valueIdx: -1, colIdx: 0, @@ -95,7 +95,7 @@ func serializeRowToBinlogBytes(ctx *sql.Context, sch schema.Schema, key, value t columns := sch.GetAllCols().GetColumns() nullBitmap = mysql.NewServerBitmap(len(columns)) - iter := newRowSerializationIter(sch, key, value) + iter := newRowSerializationIter(sch, key, value, ns) rowIdx := -1 for iter.hasNext() { rowIdx++ diff --git a/go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go b/go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go index a9b323ac747..9c1bb1bdcea 100644 --- a/go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go +++ b/go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go @@ -163,10 +163,10 @@ func newProllyConflictRowIter(ctx *sql.Context, ct ProllyConflictsTable) (*proll keyless := schema.IsKeyless(ct.ourSch) - kd := ct.baseSch.GetKeyDescriptor() - baseVD := ct.baseSch.GetValueDescriptor() - oursVD := ct.ourSch.GetValueDescriptor() - theirsVD := ct.theirSch.GetValueDescriptor() + kd := ct.baseSch.GetKeyDescriptor(ct.root.NodeStore()) + baseVD := ct.baseSch.GetValueDescriptor(ct.root.NodeStore()) + oursVD := ct.ourSch.GetValueDescriptor(ct.root.NodeStore()) + theirsVD := ct.theirSch.GetValueDescriptor(ct.root.NodeStore()) b := 1 var o, t, n int @@ -531,7 +531,7 @@ func newProllyConflictDeleter(ct ProllyConflictsTable) *prollyConflictDeleter { ed := ct.artM.Editor() kB := val.NewTupleBuilder(kd) - vd := ct.ourSch.GetValueDescriptor() + vd := ct.ourSch.GetValueDescriptor(ct.root.NodeStore()) vB := val.NewTupleBuilder(vd) p := ct.artM.Pool() diff --git a/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go b/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go index 0ab8ccdf0ae..554a7850d7d 100644 --- a/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go +++ b/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go @@ -174,7 +174,7 @@ func (cvt *prollyConstraintViolationsTable) PartitionRows(ctx *sql.Context, part if err != nil { return nil, err } - kd, vd := sch.GetMapDescriptors() + kd, vd := sch.GetMapDescriptors(cvt.root.NodeStore()) // value tuples encoded in ConstraintViolationMeta may // violate the not null constraints assumed by fixed access diff --git a/go/libraries/doltcore/sqle/dtables/prolly_row_conv.go b/go/libraries/doltcore/sqle/dtables/prolly_row_conv.go index 687fcc21b2c..66b598fc80a 100644 --- a/go/libraries/doltcore/sqle/dtables/prolly_row_conv.go +++ b/go/libraries/doltcore/sqle/dtables/prolly_row_conv.go @@ -85,7 +85,7 @@ func NewProllyRowConverter(inSch, outSch schema.Schema, warnFn rowconv.WarnFunct nonPkTargetTypes = append([]sql.Type{nil}, nonPkTargetTypes...) } - kd, vd := inSch.GetMapDescriptors() + kd, vd := inSch.GetMapDescriptors(ns) return ProllyRowConverter{ inSchema: inSch, outSchema: outSch, diff --git a/go/libraries/doltcore/sqle/dtables/query_catalog_table.go b/go/libraries/doltcore/sqle/dtables/query_catalog_table.go index f3fa3e412ce..a3d3da20220 100644 --- a/go/libraries/doltcore/sqle/dtables/query_catalog_table.go +++ b/go/libraries/doltcore/sqle/dtables/query_catalog_table.go @@ -114,8 +114,10 @@ func (sq SavedQuery) asRow(nbf *types.NomsBinFormat) (row.Row, error) { } var DoltQueryCatalogSchema = schema.MustSchemaFromCols(queryCatalogCols) -var catalogKd = DoltQueryCatalogSchema.GetKeyDescriptor() -var catalogVd = DoltQueryCatalogSchema.GetValueDescriptor() + +// system tables do not contain addressable columns, and do not require nodestore access. +var catalogKd = DoltQueryCatalogSchema.GetKeyDescriptor(nil) +var catalogVd = DoltQueryCatalogSchema.GetValueDescriptor(nil) // Creates the query catalog table if it doesn't exist. func createQueryCatalogIfNotExists(ctx context.Context, root doltdb.RootValue) (doltdb.RootValue, error) { diff --git a/go/libraries/doltcore/sqle/index/key_builder.go b/go/libraries/doltcore/sqle/index/key_builder.go index e42045b7c8c..cd42c7617a7 100644 --- a/go/libraries/doltcore/sqle/index/key_builder.go +++ b/go/libraries/doltcore/sqle/index/key_builder.go @@ -144,7 +144,7 @@ func (b SecondaryKeyBuilder) SecondaryKeyFromRow(ctx context.Context, k, v val.T if b.canCopyRawBytes(to) { b.builder.PutRaw(to, v.GetField(from)) } else { - value, err := tree.GetField(ctx, b.sch.GetValueDescriptor(), from, v, b.nodeStore) + value, err := tree.GetField(ctx, b.sch.GetValueDescriptor(b.nodeStore), from, v, b.nodeStore) if err != nil { return nil, err } @@ -166,7 +166,7 @@ func (b SecondaryKeyBuilder) SecondaryKeyFromRow(ctx context.Context, k, v val.T // BuildRow returns a sql.Row for the given key/value tuple pair func BuildRow(ctx *sql.Context, key, value val.Tuple, sch schema.Schema, ns tree.NodeStore) (sql.Row, error) { prollyIter := prolly.NewPointLookup(key, value) - rowIter := NewProllyRowIterForSchema(sch, prollyIter, sch.GetKeyDescriptor(), sch.GetValueDescriptor(), sch.GetAllCols().Tags, ns) + rowIter := NewProllyRowIterForSchema(sch, prollyIter, sch.GetKeyDescriptor(ns), sch.GetValueDescriptor(ns), sch.GetAllCols().Tags, ns) return rowIter.Next(ctx) } diff --git a/go/libraries/doltcore/sqle/kvexec/builder.go b/go/libraries/doltcore/sqle/kvexec/builder.go index e982f34f8e0..825977cca33 100644 --- a/go/libraries/doltcore/sqle/kvexec/builder.go +++ b/go/libraries/doltcore/sqle/kvexec/builder.go @@ -190,8 +190,8 @@ func newRowJoiner(schemas []schema.Schema, splits []int, projections []uint64, n mappingStartIdx = splits[splitIdx-1] - virtualCnt } tupleDesc = append(tupleDesc, kvDesc{ - keyDesc: sch.GetKeyDescriptor(), - valDesc: sch.GetValueDescriptor(), + keyDesc: sch.GetKeyDescriptor(ns), + valDesc: sch.GetValueDescriptor(ns), keyMappings: allMap[mappingStartIdx:nextKeyIdx], // prev kv partition -> last key of this partition valMappings: allMap[nextKeyIdx : splits[splitIdx]-virtualCnt], // first val of partition -> next kv partition }) diff --git a/go/libraries/doltcore/sqle/kvexec/lookup_join.go b/go/libraries/doltcore/sqle/kvexec/lookup_join.go index 56e977f5a41..7ce6d0f140d 100644 --- a/go/libraries/doltcore/sqle/kvexec/lookup_join.go +++ b/go/libraries/doltcore/sqle/kvexec/lookup_join.go @@ -262,8 +262,8 @@ func newLookupKeyMapping(ctx context.Context, sourceSch schema.Schema, tgtKeyDes srcMapping: srcMapping, litTuple: litTuple, litKd: litDesc, - srcKd: sourceSch.GetKeyDescriptor(), - srcVd: sourceSch.GetValueDescriptor(), + srcKd: sourceSch.GetKeyDescriptor(ns), + srcVd: sourceSch.GetValueDescriptor(ns), targetKb: val.NewTupleBuilder(tgtKeyDesc), ns: ns, pool: ns.Pool(), diff --git a/go/libraries/doltcore/sqle/statsnoms/database.go b/go/libraries/doltcore/sqle/statsnoms/database.go index 6a972a3b103..527842b1d48 100644 --- a/go/libraries/doltcore/sqle/statsnoms/database.go +++ b/go/libraries/doltcore/sqle/statsnoms/database.go @@ -298,8 +298,9 @@ func (n *NomsStatsDatabase) trackBranch(ctx context.Context, branch string) erro n.tableHashes = append(n.tableHashes, make(map[string]hash.Hash)) n.schemaHashes = append(n.schemaHashes, make(map[string]hash.Hash)) - kd, vd := schema.StatsTableDoltSchema.GetMapDescriptors() - newMap, err := prolly.NewMapFromTuples(ctx, n.destDb.DbData().Ddb.NodeStore(), kd, vd) + ns := n.destDb.DbData().Ddb.NodeStore() + kd, vd := schema.StatsTableDoltSchema.GetMapDescriptors(ns) + newMap, err := prolly.NewMapFromTuples(ctx, ns, kd, vd) if err != nil { return err } diff --git a/go/libraries/doltcore/sqle/statsnoms/write.go b/go/libraries/doltcore/sqle/statsnoms/write.go index c23e1d93dc8..b97f87f673d 100644 --- a/go/libraries/doltcore/sqle/statsnoms/write.go +++ b/go/libraries/doltcore/sqle/statsnoms/write.go @@ -50,7 +50,7 @@ func deleteIndexRows(ctx context.Context, statsMap *prolly.MutableMap, dStats *s return ctx.Err() } sch := schema.StatsTableDoltSchema - kd, _ := sch.GetMapDescriptors() + kd, _ := sch.GetMapDescriptors(statsMap.NodeStore()) keyBuilder := val.NewTupleBuilder(kd) @@ -96,7 +96,7 @@ func putIndexRows(ctx context.Context, statsMap *prolly.MutableMap, dStats *stat return ctx.Err() } sch := schema.StatsTableDoltSchema - kd, vd := sch.GetMapDescriptors() + kd, vd := sch.GetMapDescriptors(statsMap.NodeStore()) keyBuilder := val.NewTupleBuilder(kd) valueBuilder := val.NewTupleBuilder(vd) diff --git a/go/libraries/doltcore/sqle/writer/schema_cache.go b/go/libraries/doltcore/sqle/writer/schema_cache.go index 6c27ccc421f..fe9c98e14c8 100644 --- a/go/libraries/doltcore/sqle/writer/schema_cache.go +++ b/go/libraries/doltcore/sqle/writer/schema_cache.go @@ -69,7 +69,7 @@ func newWriterSchema(ctx *sql.Context, t *doltdb.Table, tableName string, dbName if err != nil { return nil, err } - schState.PkKeyDesc, schState.PkValDesc = schState.DoltSchema.GetMapDescriptors() + schState.PkKeyDesc, schState.PkValDesc = schState.DoltSchema.GetMapDescriptors(t.NodeStore()) schState.PkSchema, err = sqlutil.FromDoltSchema(dbName, tableName, schState.DoltSchema) if err != nil { return nil, err diff --git a/go/libraries/doltcore/table/editor/creation/external_build_index.go b/go/libraries/doltcore/table/editor/creation/external_build_index.go index 285965ae827..9da64fc0901 100644 --- a/go/libraries/doltcore/table/editor/creation/external_build_index.go +++ b/go/libraries/doltcore/table/editor/creation/external_build_index.go @@ -47,7 +47,7 @@ func BuildProllyIndexExternal(ctx *sql.Context, vrw types.ValueReadWriter, ns tr } p := primary.Pool() - keyDesc, _ := idx.Schema().GetMapDescriptors() + keyDesc, _ := idx.Schema().GetMapDescriptors(ns) if schema.IsKeyless(sch) { keyDesc = prolly.AddHashToSchema(keyDesc) } diff --git a/go/libraries/doltcore/table/editor/creation/index.go b/go/libraries/doltcore/table/editor/creation/index.go index b52a56f0342..eb8bd507e75 100644 --- a/go/libraries/doltcore/table/editor/creation/index.go +++ b/go/libraries/doltcore/table/editor/creation/index.go @@ -171,7 +171,7 @@ func BuildSecondaryProllyIndex( ) (durable.Index, error) { var uniqCb DupEntryCb if idx.IsUnique() { - kd := idx.Schema().GetKeyDescriptor() + kd := idx.Schema().GetKeyDescriptor(ns) uniqCb = func(ctx context.Context, existingKey, newKey val.Tuple) error { msg := FormatKeyForUniqKeyErr(newKey, kd) return sql.NewUniqueKeyErr(msg, false, nil) diff --git a/go/store/prolly/shim/shim.go b/go/store/prolly/shim/shim.go index f51b6f48db5..978fd83955c 100644 --- a/go/store/prolly/shim/shim.go +++ b/go/store/prolly/shim/shim.go @@ -44,11 +44,11 @@ func MapFromValue(v types.Value, sch schema.Schema, ns tree.NodeStore, isKeyless if err != nil { return prolly.Map{}, err } - kd := sch.GetKeyDescriptor() + kd := sch.GetKeyDescriptor(ns) if isKeylessSecondary { kd = prolly.AddHashToSchema(kd) } - vd := sch.GetValueDescriptor() + vd := sch.GetValueDescriptor(ns) return prolly.NewMap(root, ns, kd, vd), nil } @@ -57,11 +57,11 @@ func MapInterfaceFromValue(ctx context.Context, v types.Value, sch schema.Schema if err != nil { return nil, err } - kd := sch.GetKeyDescriptor() + kd := sch.GetKeyDescriptor(ns) if isKeylessSecondary { kd = prolly.AddHashToSchema(kd) } - vd := sch.GetValueDescriptor() + vd := sch.GetValueDescriptor(ns) switch fileId { case serial.VectorIndexNodeFileID: // TODO: We should read the distance function and chunk size from the message. diff --git a/go/store/prolly/tree/node.go b/go/store/prolly/tree/node.go index 9dd48941c47..abd2888995f 100644 --- a/go/store/prolly/tree/node.go +++ b/go/store/prolly/tree/node.go @@ -211,8 +211,8 @@ func getLastKey(nd Node) Item { // to the type specified by the provided schema. All nodes have keys displayed in this manner. Interior nodes have // their child hash references spelled out, leaf nodes have value tuples delineated like the keys func OutputProllyNode(ctx context.Context, w io.Writer, node Node, ns NodeStore, schema schema.Schema) error { - kd := schema.GetKeyDescriptor() - vd := schema.GetValueDescriptor() + kd := schema.GetKeyDescriptor(ns) + vd := schema.GetValueDescriptor(ns) for i := 0; i < int(node.count); i++ { k := node.GetKey(i) kt := val.Tuple(k) From 8da227894251d68453f33614d0fbac42094938bf Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Mon, 17 Feb 2025 20:34:09 -0800 Subject: [PATCH 02/11] Add ValueStore interface. --- go/store/prolly/tree/node_store.go | 15 +++++++++++++++ go/store/prolly/tree/testutils.go | 8 ++++++++ go/store/val/value_store.go | 25 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 go/store/val/value_store.go diff --git a/go/store/prolly/tree/node_store.go b/go/store/prolly/tree/node_store.go index 8733393e9a0..f82002db66a 100644 --- a/go/store/prolly/tree/node_store.go +++ b/go/store/prolly/tree/node_store.go @@ -15,7 +15,9 @@ package tree import ( + "bytes" "context" + "github.com/dolthub/dolt/go/store/val" "sync" "github.com/dolthub/dolt/go/store/prolly/message" @@ -32,6 +34,8 @@ const ( // NodeStore reads and writes prolly tree Nodes. type NodeStore interface { + val.ValueStore + // Read reads a prolly tree Node from the store. Read(ctx context.Context, ref hash.Hash) (Node, error) @@ -204,3 +208,14 @@ func (ns nodeStore) Format() *types.NomsBinFormat { func (ns nodeStore) PurgeCaches() { ns.cache.purge() } + +func (ns nodeStore) ReadBytes(ctx context.Context, h hash.Hash) ([]byte, error) { + return NewByteArray(h, ns).ToBytes(ctx) +} + +func (ns nodeStore) WriteBytes(ctx context.Context, b []byte) (hash.Hash, error) { + _, h, err := SerializeBytesToAddr(ctx, ns, bytes.NewReader(b), len(b)) + return h, err +} + +var _ val.ValueStore = nodeStore{} diff --git a/go/store/prolly/tree/testutils.go b/go/store/prolly/tree/testutils.go index 7c98dd7d428..db20dbd9d74 100644 --- a/go/store/prolly/tree/testutils.go +++ b/go/store/prolly/tree/testutils.go @@ -272,6 +272,14 @@ type nodeStoreValidator struct { bbp *sync.Pool } +func (v nodeStoreValidator) ReadBytes(ctx context.Context, h hash.Hash) ([]byte, error) { + panic("not implemented") +} + +func (v nodeStoreValidator) WriteBytes(ctx context.Context, val []byte) (hash.Hash, error) { + panic("not implemented") +} + func (v nodeStoreValidator) Read(ctx context.Context, ref hash.Hash) (Node, error) { nd, err := v.ns.Read(ctx, ref) if err != nil { diff --git a/go/store/val/value_store.go b/go/store/val/value_store.go new file mode 100644 index 00000000000..4ffc57f7e84 --- /dev/null +++ b/go/store/val/value_store.go @@ -0,0 +1,25 @@ +// Copyright 2025 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package val + +import ( + "context" + "github.com/dolthub/dolt/go/store/hash" +) + +type ValueStore interface { + ReadBytes(ctx context.Context, h hash.Hash) ([]byte, error) + WriteBytes(ctx context.Context, val []byte) (hash.Hash, error) +} From 22fa1c48a9ef547b44ff2e64c600aa010c5745ab Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Mon, 17 Feb 2025 20:44:37 -0800 Subject: [PATCH 03/11] Add context parameter to TupleTypeHandler type. --- go/libraries/doltcore/schema/schema_impl.go | 35 ++++++++++++++------- go/store/prolly/tree/prolly_fields.go | 18 +++-------- go/store/val/tuple_descriptor.go | 6 ++-- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/go/libraries/doltcore/schema/schema_impl.go b/go/libraries/doltcore/schema/schema_impl.go index f23c46659fb..84ed53690ef 100644 --- a/go/libraries/doltcore/schema/schema_impl.go +++ b/go/libraries/doltcore/schema/schema_impl.go @@ -462,23 +462,23 @@ func (si schemaImpl) AddColumn(newCol Column, order *ColumnOrder) (Schema, error } // GetMapDescriptors implements the Schema interface. -func (si *schemaImpl) GetMapDescriptors() (keyDesc, valueDesc val.TupleDesc) { - keyDesc = si.GetKeyDescriptor() - valueDesc = si.GetValueDescriptor() +func (si *schemaImpl) GetMapDescriptors(vs val.ValueStore) (keyDesc, valueDesc val.TupleDesc) { + keyDesc = si.GetKeyDescriptor(vs) + valueDesc = si.GetValueDescriptor(vs) return } // GetKeyDescriptor implements the Schema interface. -func (si *schemaImpl) GetKeyDescriptor() val.TupleDesc { - return si.getKeyColumnsDescriptor(true) +func (si *schemaImpl) GetKeyDescriptor(vs val.ValueStore) val.TupleDesc { + return si.getKeyColumnsDescriptor(vs, true) } // GetKeyDescriptorWithNoConversion implements the Schema interface. -func (si *schemaImpl) GetKeyDescriptorWithNoConversion() val.TupleDesc { - return si.getKeyColumnsDescriptor(false) +func (si *schemaImpl) GetKeyDescriptorWithNoConversion(vs val.ValueStore) val.TupleDesc { + return si.getKeyColumnsDescriptor(vs, false) } -func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.TupleDesc { +func (si *schemaImpl) getKeyColumnsDescriptor(vs val.ValueStore, convertAddressColumns bool) val.TupleDesc { if IsKeyless(si) { return val.KeylessTupleDesc } @@ -496,15 +496,22 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu sqlType := col.TypeInfo.ToSqlType() queryType := sqlType.Type() var t val.Type + var handler val.TupleTypeHandler _, contentHashedField := contentHashedFields[tag] extendedType, isExtendedType := sqlType.(gmstypes.ExtendedType) if isExtendedType { + encoding := EncodingFromSqlType(sqlType) t = val.Type{ - Enc: val.Encoding(EncodingFromSqlType(sqlType)), + Enc: val.Encoding(encoding), Nullable: columnMissingNotNullConstraint(col), } + if encoding == serial.EncodingExtended { + handler = extendedType + } else { + handler = val.NewExtendedAddressTypeHandler(vs, extendedType) + } } else { if convertAddressColumns && !contentHashedField && queryType == query.Type_BLOB { t = val.Type{ @@ -538,7 +545,7 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu collations = append(collations, sql.Collation_Unspecified) } - handlers = append(handlers, extendedType) + handlers = append(handlers, handler) return }) @@ -555,7 +562,7 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu } // GetValueDescriptor implements the Schema interface. -func (si *schemaImpl) GetValueDescriptor() val.TupleDesc { +func (si *schemaImpl) GetValueDescriptor(vs val.ValueStore) val.TupleDesc { var tt []val.Type var handlers []val.TupleTypeHandler var collations []sql.CollationID @@ -572,9 +579,10 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc { } sqlType := col.TypeInfo.ToSqlType() + encoding := EncodingFromSqlType(sqlType) queryType := sqlType.Type() tt = append(tt, val.Type{ - Enc: val.Encoding(EncodingFromSqlType(sqlType)), + Enc: val.Encoding(encoding), Nullable: col.IsNullable(), }) if queryType == query.Type_CHAR || queryType == query.Type_VARCHAR { @@ -585,6 +593,9 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc { } if extendedType, ok := sqlType.(gmstypes.ExtendedType); ok { + if encoding == serial.EncodingExtendedAddr { + handlers = append(handlers, val.NewExtendedAddressTypeHandler(vs, extendedType)) + } handlers = append(handlers, extendedType) } else { handlers = append(handlers, nil) diff --git a/go/store/prolly/tree/prolly_fields.go b/go/store/prolly/tree/prolly_fields.go index 9ad180cbf51..4655d817ea4 100644 --- a/go/store/prolly/tree/prolly_fields.go +++ b/go/store/prolly/tree/prolly_fields.go @@ -144,17 +144,13 @@ func GetField(ctx context.Context, td val.TupleDesc, i int, tup val.Tuple, ns No var b []byte b, ok = td.GetExtended(i, tup) if ok { - v, err = td.Handlers[i].DeserializeValue(b) + v, err = td.Handlers[i].DeserializeValue(ctx, b) } case val.ExtendedAddrEnc: var h hash.Hash h, ok = td.GetExtendedAddr(i, tup) if ok { - var b []byte - b, err = NewByteArray(h, ns).ToBytes(ctx) - if err == nil { - v, err = td.Handlers[i].DeserializeValue(b) - } + v, err = td.Handlers[i].DeserializeValue(ctx, h[:]) } default: panic("unknown val.encoding") @@ -279,7 +275,7 @@ func PutField(ctx context.Context, ns NodeStore, tb *val.TupleBuilder, i int, v } tb.PutCell(i, ZCell(v.(types.GeometryValue))) case val.ExtendedEnc: - b, err := tb.Desc.Handlers[i].SerializeValue(v) + b, err := tb.Desc.Handlers[i].SerializeValue(ctx, v) if err != nil { return err } @@ -288,15 +284,11 @@ func PutField(ctx context.Context, ns NodeStore, tb *val.TupleBuilder, i int, v } tb.PutExtended(i, b) case val.ExtendedAddrEnc: - b, err := tb.Desc.Handlers[i].SerializeValue(v) - if err != nil { - return err - } - _, h, err := SerializeBytesToAddr(ctx, ns, bytes.NewReader(b), len(b)) + b, err := tb.Desc.Handlers[i].SerializeValue(ctx, v) if err != nil { return err } - tb.PutExtendedAddr(i, h) + tb.PutExtendedAddr(i, b) default: panic(fmt.Sprintf("unknown encoding %v %v", enc, v)) } diff --git a/go/store/val/tuple_descriptor.go b/go/store/val/tuple_descriptor.go index bd55519ab35..4385ebaaea8 100644 --- a/go/store/val/tuple_descriptor.go +++ b/go/store/val/tuple_descriptor.go @@ -53,11 +53,11 @@ type TupleDesc struct { type TupleTypeHandler interface { // SerializedCompare compares two byte slices that each represent a serialized value, without first deserializing // the value. - SerializedCompare(v1 []byte, v2 []byte) (int, error) + SerializedCompare(ctx context.Context, v1 []byte, v2 []byte) (int, error) // SerializeValue converts the given value into a binary representation. - SerializeValue(val any) ([]byte, error) + SerializeValue(ctx context.Context, val any) ([]byte, error) // DeserializeValue converts a binary representation of a value into its canonical type. - DeserializeValue(val []byte) (any, error) + DeserializeValue(ctx context.Context, val []byte) (any, error) // FormatValue returns a string version of the value. Primarily intended for display. FormatValue(val any) (string, error) } From 8b1cff9facd54953bc27ae9fdc8a9fe108b76152 Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Mon, 17 Feb 2025 20:49:11 -0800 Subject: [PATCH 04/11] Add AddressTypeHandler class. --- go/store/val/tuple_descriptor.go | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/go/store/val/tuple_descriptor.go b/go/store/val/tuple_descriptor.go index 4385ebaaea8..92fe02ee1cd 100644 --- a/go/store/val/tuple_descriptor.go +++ b/go/store/val/tuple_descriptor.go @@ -15,6 +15,7 @@ package val import ( + "context" "encoding/hex" "fmt" "os" @@ -669,3 +670,53 @@ func (td TupleDesc) Equals(other TupleDesc) bool { } return true } + +type AddressTypeHandler struct { + vs ValueStore + childHandler TupleTypeHandler +} + +func NewExtendedAddressTypeHandler(vs ValueStore, childHandler TupleTypeHandler) AddressTypeHandler { + return AddressTypeHandler{ + vs: vs, + childHandler: childHandler, + } +} + +func (handler AddressTypeHandler) SerializedCompare(ctx context.Context, v1 []byte, v2 []byte) (int, error) { + v1Bytes, err := handler.vs.ReadBytes(ctx, hash.New(v1)) + if err != nil { + return 0, err + } + v2Bytes, err := handler.vs.ReadBytes(ctx, hash.New(v2)) + if err != nil { + return 0, err + } + return handler.childHandler.SerializedCompare(ctx, v1Bytes, v2Bytes) +} + +func (handler AddressTypeHandler) SerializeValue(ctx context.Context, val any) ([]byte, error) { + b, err := handler.childHandler.SerializeValue(ctx, val) + if err != nil { + return nil, err + } + h, err := handler.vs.WriteBytes(context.Background(), b) + if err != nil { + return nil, err + } + return h[:], err +} + +func (handler AddressTypeHandler) DeserializeValue(ctx context.Context, val []byte) (any, error) { + b, err := handler.vs.ReadBytes(ctx, hash.New(val)) + if err != nil { + return nil, err + } + return handler.childHandler.DeserializeValue(ctx, b) +} + +func (handler AddressTypeHandler) FormatValue(val any) (string, error) { + return handler.childHandler.FormatValue(val) +} + +var _ TupleTypeHandler = AddressTypeHandler{} From 0f2c3f7b79876baec27736f9a49d6ec2df62723a Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Tue, 18 Feb 2025 13:17:24 -0800 Subject: [PATCH 05/11] Add context parameter to key comparison logic. --- go/libraries/doltcore/diff/diff_stat.go | 6 +- .../merge/keyless_integration_test.go | 8 +-- .../doltcore/merge/merge_prolly_rows.go | 20 +++---- go/libraries/doltcore/merge/row_merge_test.go | 2 +- .../doltcore/merge/violations_fk_prolly.go | 2 +- .../doltcore/schema/collation_comparator.go | 15 ++--- .../doltcore/sqle/enginetest/validation.go | 6 +- .../doltcore/sqle/index/dolt_index.go | 2 +- .../doltcore/sqle/index/doltgres_iter.go | 4 +- .../doltcore/sqle/index/index_reader.go | 2 +- .../doltcore/sqle/index/secondary_iter.go | 12 ++-- go/libraries/doltcore/sqle/kvexec/builder.go | 2 +- .../doltcore/sqle/kvexec/merge_join.go | 14 +++-- go/libraries/doltcore/sqle/statspro/update.go | 12 ++-- .../doltcore/sqle/statspro/update_test.go | 2 +- .../sqle/writer/prolly_index_writer.go | 12 ++-- .../writer/prolly_index_writer_keyless.go | 4 +- .../editor/creation/external_build_index.go | 4 +- .../doltcore/table/editor/creation/index.go | 14 ++--- .../doltcore/table/table_iterator_test.go | 2 +- go/store/datas/commit_closure.go | 2 +- go/store/prolly/address_map.go | 2 +- go/store/prolly/artifact_map.go | 12 ++-- .../benchmark/benchmark_batch_writes_test.go | 10 ++-- go/store/prolly/commit_closure.go | 6 +- go/store/prolly/commit_closure_test.go | 6 +- go/store/prolly/map_diff_key_range_test.go | 10 ++-- go/store/prolly/map_diff_test.go | 10 ++-- go/store/prolly/map_merge_test.go | 4 +- go/store/prolly/map_range_diff_test.go | 20 +++---- go/store/prolly/map_test.go | 30 +++++----- go/store/prolly/mutable_map_read_test.go | 8 +-- go/store/prolly/mutable_map_write_test.go | 2 +- go/store/prolly/proximity_map.go | 2 +- go/store/prolly/proximity_mutable_map.go | 13 +++-- go/store/prolly/sort/external_test.go | 8 +-- go/store/prolly/tree/diff.go | 2 +- go/store/prolly/tree/json_location.go | 3 +- go/store/prolly/tree/map.go | 12 ++-- go/store/prolly/tree/merge.go | 2 +- go/store/prolly/tree/mutable_map.go | 20 +++---- go/store/prolly/tree/mutator.go | 4 +- go/store/prolly/tree/node_cursor.go | 22 +++---- go/store/prolly/tree/node_cursor_test.go | 2 +- go/store/prolly/tree/prolly_fields.go | 2 +- go/store/prolly/tree/proximity_map.go | 4 +- go/store/prolly/tree/stats.go | 4 +- go/store/prolly/tree/testutils.go | 22 +++---- go/store/prolly/tree/three_way_differ.go | 2 +- go/store/prolly/tuple_map.go | 12 ++-- go/store/prolly/tuple_mutable_map.go | 16 +++--- go/store/prolly/tuple_range.go | 57 ++++++++++--------- go/store/prolly/tuple_range_iter.go | 24 ++++---- go/store/prolly/tuple_range_iter_test.go | 50 ++++++++-------- go/store/skip/list.go | 35 ++++++------ go/store/skip/list_bench_test.go | 8 +-- go/store/skip/list_test.go | 52 ++++++++--------- go/store/val/codec.go | 5 +- go/store/val/extended_comparator.go | 12 ++-- go/store/val/keyless_tuple.go | 5 +- go/store/val/tuple_builder_test.go | 5 +- go/store/val/tuple_compare.go | 13 +++-- go/store/val/tuple_descriptor.go | 20 +++---- go/store/valuefile/value_file_test.go | 2 +- 64 files changed, 361 insertions(+), 346 deletions(-) diff --git a/go/libraries/doltcore/diff/diff_stat.go b/go/libraries/doltcore/diff/diff_stat.go index b346c35632b..dc358ca6bb3 100644 --- a/go/libraries/doltcore/diff/diff_stat.go +++ b/go/libraries/doltcore/diff/diff_stat.go @@ -225,7 +225,7 @@ func reportPkChanges(ctx context.Context, vMapping val.OrdinalMapping, fromD, to case tree.RemovedDiff: stat.Removes++ case tree.ModifiedDiff: - stat.CellChanges = prollyCountCellDiff(vMapping, fromD, toD, val.Tuple(change.From), val.Tuple(change.To)) + stat.CellChanges = prollyCountCellDiff(ctx, vMapping, fromD, toD, val.Tuple(change.From), val.Tuple(change.To)) stat.Changes++ default: return errors.New("unknown change type") @@ -269,7 +269,7 @@ func reportKeylessChanges(ctx context.Context, vMapping val.OrdinalMapping, from // prollyCountCellDiff counts the number of changes columns between two tuples // |from| and |to|. |mapping| should map columns from |from| to |to|. -func prollyCountCellDiff(mapping val.OrdinalMapping, fromD, toD val.TupleDesc, from val.Tuple, to val.Tuple) uint64 { +func prollyCountCellDiff(ctx context.Context, mapping val.OrdinalMapping, fromD, toD val.TupleDesc, from, to val.Tuple) uint64 { newCols := uint64(toD.Count()) changed := uint64(0) for i, j := range mapping { @@ -286,7 +286,7 @@ func prollyCountCellDiff(mapping val.OrdinalMapping, fromD, toD val.TupleDesc, f continue } - if fromD.CompareField(toD.GetField(j, to), i, from) != 0 { + if fromD.CompareField(ctx, toD.GetField(j, to), i, from) != 0 { // column was modified changed++ continue diff --git a/go/libraries/doltcore/merge/keyless_integration_test.go b/go/libraries/doltcore/merge/keyless_integration_test.go index 658ad163d39..f4766b79f1d 100644 --- a/go/libraries/doltcore/merge/keyless_integration_test.go +++ b/go/libraries/doltcore/merge/keyless_integration_test.go @@ -358,15 +358,15 @@ func assertProllyConflicts(t *testing.T, ctx context.Context, tbl *doltdb.Table, if expectedConf.base != nil { _, value := expectedConf.base.HashAndValue() - require.Equal(t, valDesc.Format(value), valDesc.Format(base)) + require.Equal(t, valDesc.Format(ctx, value), valDesc.Format(ctx, base)) } if expectedConf.ours != nil { _, value := expectedConf.ours.HashAndValue() - require.Equal(t, valDesc.Format(value), valDesc.Format(ours)) + require.Equal(t, valDesc.Format(ctx, value), valDesc.Format(ctx, ours)) } if expectedConf.theirs != nil { _, value := expectedConf.theirs.HashAndValue() - require.Equal(t, valDesc.Format(value), valDesc.Format(theirs)) + require.Equal(t, valDesc.Format(ctx, value), valDesc.Format(ctx, theirs)) } } @@ -457,7 +457,7 @@ func assertKeylessProllyRows(t *testing.T, ctx context.Context, tbl *doltdb.Tabl copy(h[:], hashId.GetField(0)) expectedVal, ok := expectedSet[h] assert.True(t, ok) - assert.Equal(t, valDesc.Format(expectedVal), valDesc.Format(value)) + assert.Equal(t, valDesc.Format(ctx, expectedVal), valDesc.Format(ctx, value)) } require.Equal(t, len(expected), c) diff --git a/go/libraries/doltcore/merge/merge_prolly_rows.go b/go/libraries/doltcore/merge/merge_prolly_rows.go index 0f02d504737..9a8371deffe 100644 --- a/go/libraries/doltcore/merge/merge_prolly_rows.go +++ b/go/libraries/doltcore/merge/merge_prolly_rows.go @@ -804,7 +804,7 @@ func (idx uniqIndex) findCollisions(ctx context.Context, key, value val.Tuple, c // collided with row(|key|, |value|) and pass both to |cb| err = idx.clustered.Get(ctx, clusteredKey, func(k val.Tuple, v val.Tuple) error { if k == nil { - s := idx.clusteredKeyDesc.Format(clusteredKey) + s := idx.clusteredKeyDesc.Format(ctx, clusteredKey) return errors.New("failed to find key: " + s) } collisionDetected = true @@ -1765,7 +1765,7 @@ func (m *valueMerger) processBaseColumn(ctx context.Context, i int, left, right, if err != nil { return false, err } - if isEqual(m.baseVD.Comparator(), i, baseCol, rightCol, m.rightVD.Types[rightColIdx]) { + if isEqual(ctx, m.baseVD.Comparator(), i, baseCol, rightCol, m.rightVD.Types[rightColIdx]) { // right column did not change, so there is no conflict. return false, nil } @@ -1788,7 +1788,7 @@ func (m *valueMerger) processBaseColumn(ctx context.Context, i int, left, right, if err != nil { return false, err } - if isEqual(m.baseVD.Comparator(), i, baseCol, leftCol, m.leftVD.Types[leftColIdx]) { + if isEqual(ctx, m.baseVD.Comparator(), i, baseCol, leftCol, m.leftVD.Types[leftColIdx]) { // left column did not change, so there is no conflict. return false, nil } @@ -1828,7 +1828,7 @@ func (m *valueMerger) processBaseColumn(ctx context.Context, i int, left, right, if err != nil { return false, err } - if modifiedVD.Comparator().CompareValues(i, baseCol, modifiedCol, modifiedVD.Types[modifiedColIdx]) == 0 { + if modifiedVD.Comparator().CompareValues(ctx, i, baseCol, modifiedCol, modifiedVD.Types[modifiedColIdx]) == 0 { return false, nil } return true, nil @@ -1879,7 +1879,7 @@ func (m *valueMerger) processColumn(ctx *sql.Context, i int, left, right, base v return nil, false, err } - if isEqual(m.leftVD.Comparator(), i, leftCol, rightCol, resultType) { + if isEqual(ctx, m.leftVD.Comparator(), i, leftCol, rightCol, resultType) { // Columns are equal, returning either would be correct. // However, for certain types the two columns may have different bytes. // We need to ensure that merges are deterministic regardless of the merge direction. @@ -1933,14 +1933,14 @@ func (m *valueMerger) processColumn(ctx *sql.Context, i int, left, right, base v if err != nil { return nil, true, nil } - rightModified = !isEqual(m.resultVD.Comparator(), i, rightCol, baseCol, resultType) + rightModified = !isEqual(ctx, m.resultVD.Comparator(), i, rightCol, baseCol, resultType) } leftCol, err = convert(ctx, m.leftVD, m.resultVD, m.resultSchema, leftColIdx, i, left, leftCol, m.ns) if err != nil { return nil, true, nil } - if isEqual(m.resultVD.Comparator(), i, leftCol, rightCol, resultType) { + if isEqual(ctx, m.resultVD.Comparator(), i, leftCol, rightCol, resultType) { // Columns are equal, returning either would be correct. // However, for certain types the two columns may have different bytes. // We need to ensure that merges are deterministic regardless of the merge direction. @@ -1951,7 +1951,7 @@ func (m *valueMerger) processColumn(ctx *sql.Context, i int, left, right, base v return rightCol, false, nil } - leftModified = !isEqual(m.resultVD.Comparator(), i, leftCol, baseCol, resultType) + leftModified = !isEqual(ctx, m.resultVD.Comparator(), i, leftCol, baseCol, resultType) switch { case leftModified && rightModified: @@ -2134,8 +2134,8 @@ func mergeJSON(ctx context.Context, ns tree.NodeStore, base, left, right sql.JSO } } -func isEqual(cmp val.TupleComparator, i int, left []byte, right []byte, resultType val.Type) bool { - return cmp.CompareValues(i, left, right, resultType) == 0 +func isEqual(ctx context.Context, cmp val.TupleComparator, i int, left []byte, right []byte, resultType val.Type) bool { + return cmp.CompareValues(ctx, i, left, right, resultType) == 0 } func getColumn(tuple *val.Tuple, mapping *val.OrdinalMapping, idx int) (col []byte, colIndex int, exists bool) { diff --git a/go/libraries/doltcore/merge/row_merge_test.go b/go/libraries/doltcore/merge/row_merge_test.go index f40e34354de..43ad35b7198 100644 --- a/go/libraries/doltcore/merge/row_merge_test.go +++ b/go/libraries/doltcore/merge/row_merge_test.go @@ -217,7 +217,7 @@ func TestRowMerge(t *testing.T) { assert.NoError(t, err) assert.Equal(t, test.expectConflict, !ok) vD := test.mergedSch.GetValueDescriptor(ns) - assert.Equal(t, vD.Format(test.expectedResult), vD.Format(merged)) + assert.Equal(t, vD.Format(ctx, test.expectedResult), vD.Format(ctx, merged)) }) } } diff --git a/go/libraries/doltcore/merge/violations_fk_prolly.go b/go/libraries/doltcore/merge/violations_fk_prolly.go index e501ef720f6..60d99e83f6b 100644 --- a/go/libraries/doltcore/merge/violations_fk_prolly.go +++ b/go/libraries/doltcore/merge/violations_fk_prolly.go @@ -117,7 +117,7 @@ func prollyParentPriDiffFkConstraintViolations( return nil } - partialKeyRange := prolly.PrefixRange(partialKey, partialDesc) + partialKeyRange := prolly.PrefixRange(ctx, partialKey, partialDesc) itr, err := postParentIndexData.IterRange(ctx, partialKeyRange) if err != nil { return err diff --git a/go/libraries/doltcore/schema/collation_comparator.go b/go/libraries/doltcore/schema/collation_comparator.go index 205aba0c2c9..a88fe743180 100644 --- a/go/libraries/doltcore/schema/collation_comparator.go +++ b/go/libraries/doltcore/schema/collation_comparator.go @@ -16,6 +16,7 @@ package schema import ( "bytes" + "context" "unicode/utf8" "github.com/dolthub/dolt/go/store/val" @@ -30,11 +31,11 @@ type CollationTupleComparator struct { var _ val.TupleComparator = CollationTupleComparator{} // Compare implements TupleComparator -func (c CollationTupleComparator) Compare(left, right val.Tuple, desc val.TupleDesc) (cmp int) { +func (c CollationTupleComparator) Compare(ctx context.Context, left, right val.Tuple, desc val.TupleDesc) (cmp int) { fast := desc.GetFixedAccess() for i := range fast { start, stop := fast[i][0], fast[i][1] - cmp = collationCompare(desc.Types[i], c.Collations[i], left[start:stop], right[start:stop]) + cmp = collationCompare(ctx, desc.Types[i], c.Collations[i], left[start:stop], right[start:stop]) if cmp != 0 { return cmp } @@ -43,7 +44,7 @@ func (c CollationTupleComparator) Compare(left, right val.Tuple, desc val.TupleD off := len(fast) for i, typ := range desc.Types[off:] { j := i + off - cmp = collationCompare(typ, c.Collations[j], left.GetField(j), right.GetField(j)) + cmp = collationCompare(ctx, typ, c.Collations[j], left.GetField(j), right.GetField(j)) if cmp != 0 { return cmp } @@ -52,8 +53,8 @@ func (c CollationTupleComparator) Compare(left, right val.Tuple, desc val.TupleD } // CompareValues implements TupleComparator -func (c CollationTupleComparator) CompareValues(index int, left, right []byte, typ val.Type) int { - return collationCompare(typ, c.Collations[index], left, right) +func (c CollationTupleComparator) CompareValues(ctx context.Context, index int, left, right []byte, typ val.Type) int { + return collationCompare(ctx, typ, c.Collations[index], left, right) } // Prefix implements TupleComparator @@ -95,7 +96,7 @@ func (c CollationTupleComparator) Validated(types []val.Type) val.TupleComparato return CollationTupleComparator{Collations: newCollations} } -func collationCompare(typ val.Type, collation sql.CollationID, left, right []byte) int { +func collationCompare(ctx context.Context, typ val.Type, collation sql.CollationID, left, right []byte) int { // order NULLs first if left == nil || right == nil { if bytes.Equal(left, right) { @@ -110,7 +111,7 @@ func collationCompare(typ val.Type, collation sql.CollationID, left, right []byt if typ.Enc == val.StringEnc { return compareCollatedStrings(collation, left[:len(left)-1], right[:len(right)-1]) } else { - return val.DefaultTupleComparator{}.CompareValues(0, left, right, typ) + return val.DefaultTupleComparator{}.CompareValues(ctx, 0, left, right, typ) } } diff --git a/go/libraries/doltcore/sqle/enginetest/validation.go b/go/libraries/doltcore/sqle/enginetest/validation.go index 28bed245cd1..b45ebeebe00 100644 --- a/go/libraries/doltcore/sqle/enginetest/validation.go +++ b/go/libraries/doltcore/sqle/enginetest/validation.go @@ -160,7 +160,7 @@ func printIndexContents(ctx context.Context, prollyMap prolly.MapInterface) { if err == io.EOF { break } - fmt.Printf(" - k: %v \n", kd.Format(k)) + fmt.Printf(" - k: %v \n", kd.Format(ctx, k)) } } @@ -234,7 +234,7 @@ func validateKeylessIndex(ctx context.Context, sch schema.Schema, def schema.Ind } if !ok { printIndexContents(ctx, secondary) - return fmt.Errorf("index key %s not found in index %s", builder.Desc.Format(k), def.Name()) + return fmt.Errorf("index key %s not found in index %s", builder.Desc.Format(ctx, k), def.Name()) } } } @@ -329,7 +329,7 @@ func validatePkIndex(ctx context.Context, sch schema.Schema, def schema.Index, p } if !ok { printIndexContents(ctx, secondary) - return fmt.Errorf("index key %v not found in index %s", builder.Desc.Format(k), def.Name()) + return fmt.Errorf("index key %v not found in index %s", builder.Desc.Format(ctx, k), def.Name()) } } } diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index 11f2c8b1d86..0e0f9f707b8 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -1309,7 +1309,7 @@ func (di *doltIndex) prollyRangesFromSqlRanges(ctx context.Context, ns tree.Node for i, field := range fields { // lookups on non-unique indexes can't be point lookups typ := di.keyBld.Desc.Types[i] - cmp := order.CompareValues(i, field.Hi.Value, field.Lo.Value, typ) + cmp := order.CompareValues(ctx, i, field.Hi.Value, field.Lo.Value, typ) fields[i].BoundsAreEqual = cmp == 0 if !di.unique { diff --git a/go/libraries/doltcore/sqle/index/doltgres_iter.go b/go/libraries/doltcore/sqle/index/doltgres_iter.go index 222af449eba..8cdb781ece2 100644 --- a/go/libraries/doltcore/sqle/index/doltgres_iter.go +++ b/go/libraries/doltcore/sqle/index/doltgres_iter.go @@ -215,7 +215,7 @@ func NewDoltgresPartitionIter(ctx *sql.Context, lookup sql.IndexLookup) (sql.Par func doltgresProllyMapIterator(ctx *sql.Context, keyDesc val.TupleDesc, ns tree.NodeStore, root tree.Node, rang DoltgresRange) (prolly.MapIter, error) { searchRow := make(sql.Row, len(keyDesc.Types)) var findStartErr error - findStart := func(nd tree.Node) int { + findStart := func(_ context.Context, nd tree.Node) int { return sort.Search(nd.Count(), func(i int) bool { key := val.Tuple(nd.GetKey(i)) if err := doltgresMapSearchKeyToRow(ctx, key, keyDesc, ns, searchRow); err != nil { @@ -235,7 +235,7 @@ func doltgresProllyMapIterator(ctx *sql.Context, keyDesc val.TupleDesc, ns tree. }) } var findStopErr error - findStop := func(nd tree.Node) (idx int) { + findStop := func(_ context.Context, nd tree.Node) (idx int) { return sort.Search(nd.Count(), func(i int) bool { key := val.Tuple(nd.GetKey(i)) if err := doltgresMapSearchKeyToRow(ctx, key, keyDesc, ns, searchRow); err != nil { diff --git a/go/libraries/doltcore/sqle/index/index_reader.go b/go/libraries/doltcore/sqle/index/index_reader.go index 1ebfd09595e..e048a548b9c 100644 --- a/go/libraries/doltcore/sqle/index/index_reader.go +++ b/go/libraries/doltcore/sqle/index/index_reader.go @@ -465,7 +465,7 @@ func (ib *baseIndexImplBuilder) newPointLookup(ctx *sql.Context, rang prolly.Ran return nil, fmt.Errorf("can't perform point lookup with a proximity index") } err = ib.sec.GetPrefix(ctx, rang.Tup, ib.prefDesc, func(key val.Tuple, value val.Tuple) (err error) { - if key != nil && rang.Matches(key) { + if key != nil && rang.Matches(ctx, key) { iter = prolly.NewPointLookup(key, value) } else { iter = prolly.EmptyPointLookup diff --git a/go/libraries/doltcore/sqle/index/secondary_iter.go b/go/libraries/doltcore/sqle/index/secondary_iter.go index 9b751ae975b..1119c8053a9 100644 --- a/go/libraries/doltcore/sqle/index/secondary_iter.go +++ b/go/libraries/doltcore/sqle/index/secondary_iter.go @@ -184,12 +184,12 @@ func (c *covLaxSecondaryLookupGen) New(ctx context.Context, k val.Tuple) (prolly if c.prefixDesc.Count() >= c.m.KeyDesc().Count()-1 { // key range optimization only works for full length key start := k - stop, ok := prolly.IncrementTuple(start, c.prefixDesc.Count()-1, c.prefixDesc, c.m.Pool()) + stop, ok := prolly.IncrementTuple(ctx, start, c.prefixDesc.Count()-1, c.prefixDesc, c.m.Pool()) if ok { return c.m.IterKeyRange(ctx, start, stop) } } - rng := prolly.PrefixRange(k, c.prefixDesc) + rng := prolly.PrefixRange(ctx, k, c.prefixDesc) iter, err := c.m.IterRange(ctx, rng) if err != nil { @@ -244,7 +244,7 @@ func (c *nonCovLaxSecondaryLookupGen) New(ctx context.Context, k val.Tuple) (pro // TODO: widen this restriction for multiple PKs. need to count the number // of PK cols in the index colset vs outside start := k - stop, ok := prolly.IncrementTuple(start, c.prefixDesc.Count()-1, c.prefixDesc, c.sec.Pool()) + stop, ok := prolly.IncrementTuple(ctx, start, c.prefixDesc.Count()-1, c.prefixDesc, c.sec.Pool()) if ok { secIter, err := c.sec.IterKeyRange(ctx, start, stop) if err != nil { @@ -253,7 +253,7 @@ func (c *nonCovLaxSecondaryLookupGen) New(ctx context.Context, k val.Tuple) (pro return &nonCoveringMapIter{indexIter: secIter, primary: c.pri, pkMap: c.pkMap, pkBld: c.pkBld}, nil } } - rng := prolly.PrefixRange(k, c.prefixDesc) + rng := prolly.PrefixRange(ctx, k, c.prefixDesc) secIter, err := c.sec.IterRange(ctx, rng) if err != nil { return nil, err @@ -297,7 +297,7 @@ func (c *keylessSecondaryLookupGen) New(ctx context.Context, k val.Tuple) (proll // key range optimization only works if full key // keyless indexes should include all rows start := k - stop, ok := prolly.IncrementTuple(start, c.prefixDesc.Count()-1, c.prefixDesc, c.sec.Pool()) + stop, ok := prolly.IncrementTuple(ctx, start, c.prefixDesc.Count()-1, c.prefixDesc, c.sec.Pool()) if ok { secIter, err := c.sec.IterKeyRange(ctx, start, stop) if err != nil { @@ -306,7 +306,7 @@ func (c *keylessSecondaryLookupGen) New(ctx context.Context, k val.Tuple) (proll return &keylessLookupIter{pri: c.pri, secIter: secIter, pkMap: c.pkMap, pkBld: c.pkBld, prefixDesc: c.prefixDesc}, nil } } - rng := prolly.PrefixRange(k, c.prefixDesc) + rng := prolly.PrefixRange(ctx, k, c.prefixDesc) secIter, err := c.sec.IterRange(ctx, rng) if err != nil { return nil, err diff --git a/go/libraries/doltcore/sqle/kvexec/builder.go b/go/libraries/doltcore/sqle/kvexec/builder.go index 825977cca33..aeec12679c2 100644 --- a/go/libraries/doltcore/sqle/kvexec/builder.go +++ b/go/libraries/doltcore/sqle/kvexec/builder.go @@ -80,7 +80,7 @@ func (b Builder) Build(ctx *sql.Context, n sql.Node, r sql.Row) (sql.RowIter, er // - usually key tuple, but for keyless tables it's val tuple. // - use primary table projections as reference for comparison // filter indexes. - if lrCmp, llCmp, ok := mergeComparer(filters[0], leftState, rightState, projections); ok { + if lrCmp, llCmp, ok := mergeComparer(ctx, filters[0], leftState, rightState, projections); ok { split := len(leftState.tags) var rowJoiner *prollyToSqlJoiner rowJoiner = newRowJoiner([]schema.Schema{leftState.priSch, rightState.priSch}, []int{split}, projections, leftState.idxMap.NodeStore()) diff --git a/go/libraries/doltcore/sqle/kvexec/merge_join.go b/go/libraries/doltcore/sqle/kvexec/merge_join.go index 1cdfaa67408..e8538a6c403 100644 --- a/go/libraries/doltcore/sqle/kvexec/merge_join.go +++ b/go/libraries/doltcore/sqle/kvexec/merge_join.go @@ -15,6 +15,7 @@ package kvexec import ( + "context" "errors" "io" @@ -394,6 +395,7 @@ func (l *mergeJoinKvIter) initialize(ctx *sql.Context) error { var defCmp = val.DefaultTupleComparator{} func mergeComparer( + ctx context.Context, filter sql.Expression, lState, rState mergeState, projections []uint64, @@ -450,17 +452,17 @@ func mergeComparer( if lKeyOk { lTyp = lState.idxMap.KeyDesc().Types[lKeyIdx] llCmp = func(leftKey, _, rightKey, _ val.Tuple) int { - return defCmp.CompareValues(0, leftKey.GetField(lKeyIdx), rightKey.GetField(lKeyIdx), lTyp) + return defCmp.CompareValues(ctx, 0, leftKey.GetField(lKeyIdx), rightKey.GetField(lKeyIdx), lTyp) } if rKeyOk { rTyp = rState.idxMap.KeyDesc().Types[rKeyIdx] lrCmp = func(leftKey, _, rightKey, _ val.Tuple) int { - return defCmp.CompareValues(0, leftKey.GetField(lKeyIdx), rightKey.GetField(rKeyIdx), lTyp) + return defCmp.CompareValues(ctx, 0, leftKey.GetField(lKeyIdx), rightKey.GetField(rKeyIdx), lTyp) } } else if rValOk { rTyp = rState.idxMap.ValDesc().Types[rValIdx] lrCmp = func(leftKey, _, _, rightVal val.Tuple) int { - return defCmp.CompareValues(0, leftKey.GetField(lKeyIdx), rightVal.GetField(rValIdx), lTyp) + return defCmp.CompareValues(ctx, 0, leftKey.GetField(lKeyIdx), rightVal.GetField(rValIdx), lTyp) } } else { return nil, nil, false @@ -468,17 +470,17 @@ func mergeComparer( } else if lValOk { lTyp = lState.idxMap.ValDesc().Types[lValIdx] llCmp = func(_, leftVal, _, rightVal val.Tuple) int { - return defCmp.CompareValues(0, leftVal.GetField(lValIdx), rightVal.GetField(lValIdx), lTyp) + return defCmp.CompareValues(ctx, 0, leftVal.GetField(lValIdx), rightVal.GetField(lValIdx), lTyp) } if rKeyOk { rTyp = rState.idxMap.KeyDesc().Types[rKeyIdx] lrCmp = func(_, leftVal, rightKey, _ val.Tuple) int { - return defCmp.CompareValues(0, leftVal.GetField(lValIdx), rightKey.GetField(rKeyIdx), lTyp) + return defCmp.CompareValues(ctx, 0, leftVal.GetField(lValIdx), rightKey.GetField(rKeyIdx), lTyp) } } else if rValOk { rTyp = rState.idxMap.ValDesc().Types[rValIdx] lrCmp = func(_, leftVal, _, rightVal val.Tuple) int { - return defCmp.CompareValues(0, leftVal.GetField(lValIdx), rightVal.GetField(rValIdx), lTyp) + return defCmp.CompareValues(ctx, 0, leftVal.GetField(lValIdx), rightVal.GetField(rValIdx), lTyp) } } else { return nil, nil, false diff --git a/go/libraries/doltcore/sqle/statspro/update.go b/go/libraries/doltcore/sqle/statspro/update.go index 562e82c5679..36efbad5830 100644 --- a/go/libraries/doltcore/sqle/statspro/update.go +++ b/go/libraries/doltcore/sqle/statspro/update.go @@ -134,7 +134,7 @@ func createNewStatsBuckets(ctx *sql.Context, sqlTable sql.Table, dTab *doltdb.Ta keyBuilder.PutRaw(i, keyBytes.GetField(i)) } - updater.add(keyBuilder.BuildPrefixNoRecycle(prollyMap.Pool(), updater.prefixLen)) + updater.add(ctx, keyBuilder.BuildPrefixNoRecycle(prollyMap.Pool(), updater.prefixLen)) keyBuilder.Recycle() } @@ -302,10 +302,10 @@ func (u *bucketBuilder) finalize(ctx context.Context, ns tree.NodeStore) (DoltBu // add inputs a new row for a histogram bucket aggregation. We assume // the key has already been truncated to the appropriate prefix length. -func (u *bucketBuilder) add(key val.Tuple) { - newKey := u.currentKey == nil || u.tupleDesc.Compare(u.currentKey, key) != 0 +func (u *bucketBuilder) add(ctx context.Context, key val.Tuple) { + newKey := u.currentKey == nil || u.tupleDesc.Compare(ctx, u.currentKey, key) != 0 if newKey { - u.newKey(key) + u.newKey(ctx, key) } else { u.currentCnt++ } @@ -321,10 +321,10 @@ func (u *bucketBuilder) add(key val.Tuple) { } // newKey updates state for a new key in the rolling stream. -func (u *bucketBuilder) newKey(key val.Tuple) { +func (u *bucketBuilder) newKey(ctx context.Context, key val.Tuple) { u.updateMcv() if u.prevBound != nil { - if u.tupleDesc.Compare(u.prevBound, key) != 0 { + if u.tupleDesc.Compare(ctx, u.prevBound, key) != 0 { u.globalDistinct++ u.prevBound = nil } else { diff --git a/go/libraries/doltcore/sqle/statspro/update_test.go b/go/libraries/doltcore/sqle/statspro/update_test.go index ef670e19c8b..b599dfb390b 100644 --- a/go/libraries/doltcore/sqle/statspro/update_test.go +++ b/go/libraries/doltcore/sqle/statspro/update_test.go @@ -198,7 +198,7 @@ func TestBucketBuilder(t *testing.T) { err := tree.PutField(ctx, nil, kb, i, v) assert.NoError(t, err) } - b.add(kb.Build(pool)) + b.add(ctx, kb.Build(pool)) } // |ns| only needed for out of band tuples bucket, err := b.finalize(ctx, nil) diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go index 196a18bca91..6e2ede1011b 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go @@ -131,7 +131,7 @@ func (m prollyIndexWriter) ValidateKeyViolations(ctx context.Context, sqlRow sql from := m.keyMap.MapOrdinal(to) remappedSqlRow[to] = sqlRow[from] } - keyStr := FormatKeyForUniqKeyErr(k, m.keyBld.Desc, remappedSqlRow) + keyStr := FormatKeyForUniqKeyErr(ctx, k, m.keyBld.Desc, remappedSqlRow) return m.uniqueKeyError(ctx, keyStr, k, true) } return nil @@ -193,7 +193,7 @@ func (m prollyIndexWriter) Update(ctx context.Context, oldRow sql.Row, newRow sq from := m.keyMap.MapOrdinal(to) remappedSqlRow[to] = newRow[from] } - keyStr := FormatKeyForUniqKeyErr(newKey, m.keyBld.Desc, remappedSqlRow) + keyStr := FormatKeyForUniqKeyErr(ctx, newKey, m.keyBld.Desc, remappedSqlRow) return m.uniqueKeyError(ctx, keyStr, newKey, true) } @@ -344,7 +344,7 @@ func (m prollySecondaryIndexWriter) checkForUniqueKeyErr(ctx context.Context, sq // build a val.Tuple containing only fields for the unique column prefix key := m.keyBld.BuildPrefix(ns.Pool(), m.idxCols) desc := m.keyBld.Desc.PrefixDesc(m.idxCols) - rng := prolly.PrefixRange(key, desc) + rng := prolly.PrefixRange(ctx, key, desc) iter, err := m.mut.IterRange(ctx, rng) if err != nil { return err @@ -371,7 +371,7 @@ func (m prollySecondaryIndexWriter) checkForUniqueKeyErr(ctx context.Context, sq remappedSqlRow[to] = m.trimKeyPart(to, sqlRow[from]) } return secondaryUniqueKeyError{ - keyStr: FormatKeyForUniqKeyErr(key, desc, remappedSqlRow), + keyStr: FormatKeyForUniqKeyErr(ctx, key, desc, remappedSqlRow), existingKey: existingPK, } } @@ -429,7 +429,7 @@ func (m prollySecondaryIndexWriter) IterRange(ctx context.Context, rng prolly.Ra // FormatKeyForUniqKeyErr formats the given tuple |key| using |d|. The resulting // string is suitable for use in a sql.UniqueKeyError -func FormatKeyForUniqKeyErr(key val.Tuple, d val.TupleDesc, sqlRow sql.Row) string { +func FormatKeyForUniqKeyErr(ctx context.Context, key val.Tuple, d val.TupleDesc, sqlRow sql.Row) string { var sb strings.Builder sb.WriteString("[") seenOne := false @@ -443,7 +443,7 @@ func FormatKeyForUniqKeyErr(key val.Tuple, d val.TupleDesc, sqlRow sql.Row) stri case val.BytesAddrEnc, val.StringAddrEnc: sb.WriteString(fmt.Sprintf("%s", sqlRow[i])) default: - sb.WriteString(d.FormatValue(i, key.GetField(i))) + sb.WriteString(d.FormatValue(ctx, i, key.GetField(i))) } } sb.WriteString("]") diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go index f637d7c9d30..971f1bda2e6 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go @@ -276,7 +276,7 @@ func (writer prollyKeylessSecondaryWriter) checkForUniqueKeyError(ctx context.Co } } - rng := prolly.PrefixRange(prefixKey, writer.prefixBld.Desc) + rng := prolly.PrefixRange(ctx, prefixKey, writer.prefixBld.Desc) itr, err := writer.mut.IterRange(ctx, rng) if err != nil { return err @@ -291,7 +291,7 @@ func (writer prollyKeylessSecondaryWriter) checkForUniqueKeyError(ctx context.Co from := writer.keyMap.MapOrdinal(to) remappedSqlRow[to] = writer.trimKeyPart(to, sqlRow[from]) } - keyStr := FormatKeyForUniqKeyErr(prefixKey, writer.prefixBld.Desc, remappedSqlRow) + keyStr := FormatKeyForUniqKeyErr(ctx, prefixKey, writer.prefixBld.Desc, remappedSqlRow) writer.hashBld.PutRaw(0, k.GetField(k.Count()-1)) existingKey := writer.hashBld.Build(sharePool) return secondaryUniqueKeyError{keyStr: keyStr, existingKey: existingKey} diff --git a/go/libraries/doltcore/table/editor/creation/external_build_index.go b/go/libraries/doltcore/table/editor/creation/external_build_index.go index 9da64fc0901..07faf56c101 100644 --- a/go/libraries/doltcore/table/editor/creation/external_build_index.go +++ b/go/libraries/doltcore/table/editor/creation/external_build_index.go @@ -63,7 +63,7 @@ func BuildProllyIndexExternal(ctx *sql.Context, vrw types.ValueReadWriter, ns tr } sorter := sort.NewTupleSorter(batchSize, fileMax, func(t1, t2 val.Tuple) bool { - return keyDesc.Compare(t1, t2) < 0 + return keyDesc.Compare(ctx, t1, t2) < 0 }, tempfiles.MovableTempFileProvider) defer sorter.Close() @@ -178,7 +178,7 @@ func (t *tupleIterWithCb) Next(ctx context.Context) (val.Tuple, val.Tuple) { } return nil, nil } - if t.lastKey != nil && t.prefixDesc.Compare(t.lastKey, curKey) == 0 && t.uniqCb != nil { + if t.lastKey != nil && t.prefixDesc.Compare(ctx, t.lastKey, curKey) == 0 && t.uniqCb != nil { // register a constraint violation if |key| collides with |lastKey| if err := t.uniqCb(ctx, t.lastKey, curKey); err != nil { t.err = err diff --git a/go/libraries/doltcore/table/editor/creation/index.go b/go/libraries/doltcore/table/editor/creation/index.go index eb8bd507e75..489aee993a5 100644 --- a/go/libraries/doltcore/table/editor/creation/index.go +++ b/go/libraries/doltcore/table/editor/creation/index.go @@ -173,7 +173,7 @@ func BuildSecondaryProllyIndex( if idx.IsUnique() { kd := idx.Schema().GetKeyDescriptor(ns) uniqCb = func(ctx context.Context, existingKey, newKey val.Tuple) error { - msg := FormatKeyForUniqKeyErr(newKey, kd) + msg := FormatKeyForUniqKeyErr(ctx, newKey, kd) return sql.NewUniqueKeyErr(msg, false, nil) } } @@ -183,7 +183,7 @@ func BuildSecondaryProllyIndex( // FormatKeyForUniqKeyErr formats the given tuple |key| using |d|. The resulting // string is suitable for use in a sql.UniqueKeyError // This is copied from the writer package to avoid pulling in that dependency and prevent cycles -func FormatKeyForUniqKeyErr(key val.Tuple, d val.TupleDesc) string { +func FormatKeyForUniqKeyErr(ctx context.Context, key val.Tuple, d val.TupleDesc) string { var sb strings.Builder sb.WriteString("[") seenOne := false @@ -192,7 +192,7 @@ func FormatKeyForUniqKeyErr(key val.Tuple, d val.TupleDesc) string { sb.WriteString(",") } seenOne = true - sb.WriteString(d.FormatValue(i, key.GetField(i))) + sb.WriteString(d.FormatValue(ctx, i, key.GetField(i))) } sb.WriteString("]") return sb.String() @@ -283,7 +283,7 @@ type PrefixItr struct { } func NewPrefixItr(ctx context.Context, p val.Tuple, d val.TupleDesc, m rangeIterator) (PrefixItr, error) { - rng := prolly.PrefixRange(p, d) + rng := prolly.PrefixRange(ctx, p, d) itr, err := m.IterRange(ctx, rng) if err != nil { return PrefixItr{}, err @@ -330,16 +330,16 @@ type prollyUniqueKeyErr struct { // Error implements the error interface. func (u *prollyUniqueKeyErr) Error() string { - keyStr, _ := formatKey(u.k, u.kd) + keyStr, _ := formatKey(context.Background(), u.k, u.kd) return fmt.Sprintf("duplicate unique key given: %s", keyStr) } // formatKey returns a comma-separated string representation of the key given // that matches the output of the old format. -func formatKey(key val.Tuple, td val.TupleDesc) (string, error) { +func formatKey(ctx context.Context, key val.Tuple, td val.TupleDesc) (string, error) { vals := make([]string, td.Count()) for i := 0; i < td.Count(); i++ { - vals[i] = td.FormatValue(i, key.GetField(i)) + vals[i] = td.FormatValue(ctx, i, key.GetField(i)) } return fmt.Sprintf("[%s]", strings.Join(vals, ",")), nil diff --git a/go/libraries/doltcore/table/table_iterator_test.go b/go/libraries/doltcore/table/table_iterator_test.go index 7bb5af0f4fb..036897cc618 100644 --- a/go/libraries/doltcore/table/table_iterator_test.go +++ b/go/libraries/doltcore/table/table_iterator_test.go @@ -71,7 +71,7 @@ func mustMakeProllyMap(t *testing.T, count int) (prolly.Map, [][2]val.Tuple) { ns := tree.NewTestNodeStore() - tuples := tree.RandomTuplePairs(count, kd, vd, ns) + tuples := tree.RandomTuplePairs(ctx, count, kd, vd, ns) om := mustProllyMapFromTuples(t, kd, vd, tuples) return om, tuples diff --git a/go/store/datas/commit_closure.go b/go/store/datas/commit_closure.go index 1b20a1ac28a..b348655edcc 100644 --- a/go/store/datas/commit_closure.go +++ b/go/store/datas/commit_closure.go @@ -251,7 +251,7 @@ func (i *fbParentsClosureIterator) Next(ctx context.Context) bool { func (i *fbParentsClosureIterator) Less(ctx context.Context, nbf *types.NomsBinFormat, otherI parentsClosureIter) bool { other := otherI.(*fbParentsClosureIterator) - return i.curr.Less(other.curr) + return i.curr.Less(ctx, other.curr) } func writeTypesCommitParentClosure(ctx context.Context, vrw types.ValueReadWriter, parentRefsL types.List) (types.Ref, bool, error) { diff --git a/go/store/prolly/address_map.go b/go/store/prolly/address_map.go index 4b1987f0463..32339b314e0 100644 --- a/go/store/prolly/address_map.go +++ b/go/store/prolly/address_map.go @@ -57,7 +57,7 @@ type lexicographic struct{} var _ tree.Ordering[stringSlice] = lexicographic{} -func (l lexicographic) Compare(left, right stringSlice) int { +func (l lexicographic) Compare(ctx context.Context, left, right stringSlice) int { return bytes.Compare(left, right) } diff --git a/go/store/prolly/artifact_map.go b/go/store/prolly/artifact_map.go index 7601b014d67..f148ad41743 100644 --- a/go/store/prolly/artifact_map.go +++ b/go/store/prolly/artifact_map.go @@ -370,7 +370,7 @@ func (wr *ArtifactsEditor) Add(ctx context.Context, srcKey val.Tuple, srcRootish // existing violation exists but has a different |meta.VInfo| value then // ErrMergeArtifactCollision is a returned. func (wr *ArtifactsEditor) ReplaceConstraintViolation(ctx context.Context, srcKey val.Tuple, srcRootish hash.Hash, artType ArtifactType, meta ConstraintViolationMeta) error { - itr, err := wr.mut.IterRange(ctx, PrefixRange(srcKey, wr.srcKeyDesc)) + itr, err := wr.mut.IterRange(ctx, PrefixRange(ctx, srcKey, wr.srcKeyDesc)) if err != nil { return err } @@ -401,7 +401,7 @@ func (wr *ArtifactsEditor) ReplaceConstraintViolation(ctx context.Context, srcKe if bytes.Compare(currMeta.Value, meta.Value) == 0 { if bytes.Compare(currMeta.VInfo, meta.VInfo) != 0 { - return artifactCollisionErr(srcKey, wr.srcKeyDesc, currMeta.VInfo, meta.VInfo) + return artifactCollisionErr(ctx, srcKey, wr.srcKeyDesc, currMeta.VInfo, meta.VInfo) } // Key and Value is the same, so delete this err = wr.Delete(ctx, art.ArtKey) @@ -426,9 +426,9 @@ func (wr *ArtifactsEditor) ReplaceConstraintViolation(ctx context.Context, srcKe return nil } -func artifactCollisionErr(key val.Tuple, desc val.TupleDesc, old, new []byte) error { +func artifactCollisionErr(ctx context.Context, key val.Tuple, desc val.TupleDesc, old, new []byte) error { return fmt.Errorf("error storing constraint violation for primary key (%s): another violation already exists\n"+ - "new violation: %s old violation: (%s)", desc.Format(key), string(old), string(new)) + "new violation: %s old violation: (%s)", desc.Format(ctx, key), string(old), string(new)) } func (wr *ArtifactsEditor) Delete(ctx context.Context, key val.Tuple) error { @@ -650,9 +650,9 @@ func ArtifactDebugFormat(ctx context.Context, m ArtifactMap) (string, error) { } sb.WriteString("\t") - sb.WriteString(kd.Format(k)) + sb.WriteString(kd.Format(ctx, k)) sb.WriteString(": ") - sb.WriteString(vd.Format(v)) + sb.WriteString(vd.Format(ctx, v)) sb.WriteString(",\n") } sb.WriteString("}") diff --git a/go/store/prolly/benchmark/benchmark_batch_writes_test.go b/go/store/prolly/benchmark/benchmark_batch_writes_test.go index ac04577928c..24f8b3a2c55 100644 --- a/go/store/prolly/benchmark/benchmark_batch_writes_test.go +++ b/go/store/prolly/benchmark/benchmark_batch_writes_test.go @@ -102,8 +102,8 @@ type bboltWriter struct { db *bbolt.DB } -func (wr *bboltWriter) Put(key, value []byte) error { - wr.edits.Put(key, value) +func (wr *bboltWriter) Put(ctx context.Context, key, value []byte) error { + wr.edits.Put(ctx, key, value) return nil } @@ -130,7 +130,7 @@ type doltWriter struct { cs *nbs.NomsBlockStore } -func (wr *doltWriter) Put(key, value []byte) error { +func (wr *doltWriter) Put(ctx context.Context, key, value []byte) error { return wr.mut.Put(context.Background(), key, value) } @@ -153,7 +153,7 @@ func benchmarkBatchWrite(b *testing.B, wr writer) { dp := newDataProvider(batch) for i := 0; i < b.N; i++ { k, v := dp.next() - require.NoError(b, wr.Put(k, v)) + require.NoError(b, wr.Put(ctx, k, v)) if dp.empty() { require.NoError(b, wr.Flush()) dp = newDataProvider(batch) @@ -162,7 +162,7 @@ func benchmarkBatchWrite(b *testing.B, wr writer) { } type writer interface { - Put(key, value []byte) error + Put(ctx context.Context, key, value []byte) error Flush() error } diff --git a/go/store/prolly/commit_closure.go b/go/store/prolly/commit_closure.go index 0f79b9103b8..75d4ad04136 100644 --- a/go/store/prolly/commit_closure.go +++ b/go/store/prolly/commit_closure.go @@ -44,7 +44,7 @@ type commitClosureKeyOrdering struct{} var _ tree.Ordering[CommitClosureKey] = commitClosureKeyOrdering{} -func (o commitClosureKeyOrdering) Compare(left, right CommitClosureKey) int { +func (o commitClosureKeyOrdering) Compare(ctx context.Context, left, right CommitClosureKey) int { lh, rh := left.Height(), right.Height() if lh == rh { return bytes.Compare(left[prefixWidth:], right[prefixWidth:]) @@ -176,8 +176,8 @@ func (k CommitClosureKey) Addr() hash.Hash { return hash.New(k[prefixWidth:]) } -func (k CommitClosureKey) Less(other CommitClosureKey) bool { - return commitClosureKeyOrdering{}.Compare(k, other) < 0 +func (k CommitClosureKey) Less(ctx context.Context, other CommitClosureKey) bool { + return commitClosureKeyOrdering{}.Compare(ctx, k, other) < 0 } var emptyCommitClosureValue CommitClosureValue = CommitClosureValue(make([]byte, 1)) diff --git a/go/store/prolly/commit_closure_test.go b/go/store/prolly/commit_closure_test.go index 15b525b75cf..cf8aa8a1ecf 100644 --- a/go/store/prolly/commit_closure_test.go +++ b/go/store/prolly/commit_closure_test.go @@ -40,13 +40,13 @@ func TestCommitClosure(t *testing.T) { assert.True(t, k0.Addr().Equal(hash.Hash{})) k0_ := NewCommitClosureKey(ns.Pool(), 0, hash.Parse("00000000000000000000000000000000")) - assert.False(t, k0.Less(k0_)) + assert.False(t, k0.Less(ctx, k0_)) h := hash.Parse("00000000000000000000000000000001") k0_1 := NewCommitClosureKey(ns.Pool(), 0, h) assert.True(t, k0_1.Addr().Equal(h)) - assert.True(t, k0.Less(k0_1)) - assert.False(t, k0_1.Less(k0_)) + assert.True(t, k0.Less(ctx, k0_1)) + assert.False(t, k0_1.Less(ctx, k0_)) }) diff --git a/go/store/prolly/map_diff_key_range_test.go b/go/store/prolly/map_diff_key_range_test.go index d81507ec305..f8bd491753e 100644 --- a/go/store/prolly/map_diff_key_range_test.go +++ b/go/store/prolly/map_diff_key_range_test.go @@ -183,7 +183,7 @@ func testKeyRngDeleteDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numDelet deletes := tups[:numDeletes] sort.Slice(deletes, func(i, j int) bool { - return from.keyDesc.Compare(deletes[i][0], deletes[j][0]) < 0 + return from.keyDesc.Compare(ctx, deletes[i][0], deletes[j][0]) < 0 }) inRange := getPairsInKeyRange(deletes, rngTest.keyRange) to := makeMapWithDeletes(t, from, deletes...) @@ -229,7 +229,7 @@ func testKeyRngUpdateDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numUpdat sub := tups[:numUpdates] sort.Slice(sub, func(i, j int) bool { - return from.keyDesc.Compare(sub[i][0], sub[j][0]) < 0 + return from.keyDesc.Compare(ctx, sub[i][0], sub[j][0]) < 0 }) kd, vd := from.Descriptors() @@ -291,7 +291,7 @@ func makeRandomUnboundedUpperKeyRange(kd val.TupleDesc, tuples [][2]val.Tuple) k func makeBoundedKeyRangeWithMissingKeys(t *testing.T, m Map, kd val.TupleDesc, vd val.TupleDesc, tuples [][2]val.Tuple) keyRangeDiffTest { inserts := generateInserts(t, m, kd, vd, 2) low, hi := inserts[0][0], inserts[1][0] - if kd.Compare(low, hi) > 0 { + if kd.Compare(ctx, low, hi) > 0 { hi, low = low, hi } @@ -316,10 +316,10 @@ type keyRange struct { } func (kR keyRange) includes(k val.Tuple) bool { - if len(kR.start) != 0 && kR.kd.Compare(k, kR.start) < 0 { + if len(kR.start) != 0 && kR.kd.Compare(ctx, k, kR.start) < 0 { return false } - if len(kR.stop) != 0 && kR.kd.Compare(k, kR.stop) >= 0 { + if len(kR.stop) != 0 && kR.kd.Compare(ctx, k, kR.stop) >= 0 { return false } return true diff --git a/go/store/prolly/map_diff_test.go b/go/store/prolly/map_diff_test.go index c3de2b4df9d..3ce4c913d6a 100644 --- a/go/store/prolly/map_diff_test.go +++ b/go/store/prolly/map_diff_test.go @@ -165,7 +165,7 @@ func testDeleteDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numDeletes int deletes := tups[:numDeletes] sort.Slice(deletes, func(i, j int) bool { - return from.keyDesc.Compare(deletes[i][0], deletes[j][0]) < 0 + return from.keyDesc.Compare(ctx, deletes[i][0], deletes[j][0]) < 0 }) to := makeMapWithDeletes(t, from, deletes...) @@ -207,7 +207,7 @@ func testUpdateDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numUpdates int sub := tups[:numUpdates] sort.Slice(sub, func(i, j int) bool { - return from.keyDesc.Compare(sub[i][0], sub[j][0]) < 0 + return from.keyDesc.Compare(ctx, sub[i][0], sub[j][0]) < 0 }) kd, vd := from.Descriptors() @@ -268,7 +268,7 @@ func makeMapWithInserts(t *testing.T, m Map, numInserts int) (Map, [][2]val.Tupl func generateInserts(t *testing.T, m testMap, kd, vd val.TupleDesc, numInserts int) [][2]val.Tuple { ctx := context.Background() ns := tree.NewTestNodeStore() - tups := tree.RandomTuplePairs(numInserts*2, kd, vd, ns) + tups := tree.RandomTuplePairs(ctx, numInserts*2, kd, vd, ns) inserts, extra := tups[:numInserts], tups[numInserts:] j := 0 @@ -293,7 +293,7 @@ func generateInserts(t *testing.T, m testMap, kd, vd val.TupleDesc, numInserts i require.True(t, j < len(extra)) } } - tree.SortTuplePairs(inserts, kd) + tree.SortTuplePairs(ctx, inserts, kd) return inserts } @@ -323,7 +323,7 @@ func makeUpdatesToTuples(kd, vd val.TupleDesc, tuples ...[2]val.Tuple) (updates } sort.Slice(updates, func(i, j int) bool { - return kd.Compare(updates[i][0], updates[j][0]) < 0 + return kd.Compare(ctx, updates[i][0], updates[j][0]) < 0 }) return diff --git a/go/store/prolly/map_merge_test.go b/go/store/prolly/map_merge_test.go index 988ecb5b226..94f6605ad1f 100644 --- a/go/store/prolly/map_merge_test.go +++ b/go/store/prolly/map_merge_test.go @@ -151,7 +151,7 @@ func testThreeWayMapMerge(t *testing.T, kd, vd val.TupleDesc, sz int, ns tree.No func testTupleMergeFn(t *testing.T, kd, vd val.TupleDesc, sz int, ns tree.NodeStore) { ctx := context.Background() - tuples := tree.RandomTuplePairs(sz, kd, vd, ns) + tuples := tree.RandomTuplePairs(ctx, sz, kd, vd, ns) base := mustProllyMapFromTuples(t, kd, vd, tuples) mutSz := sz / 10 @@ -227,7 +227,7 @@ type mutationSet struct { func makeTuplesAndMutations(kd, vd val.TupleDesc, sz int, ns tree.NodeStore) (base [][2]val.Tuple, left, right mutationSet) { mutSz := sz / 10 totalSz := sz + (mutSz * 2) - tuples := tree.RandomTuplePairs(totalSz, kd, vd, ns) + tuples := tree.RandomTuplePairs(ctx, totalSz, kd, vd, ns) base = tuples[:sz] diff --git a/go/store/prolly/map_range_diff_test.go b/go/store/prolly/map_range_diff_test.go index 8ad7b3005d0..cb486698e32 100644 --- a/go/store/prolly/map_range_diff_test.go +++ b/go/store/prolly/map_range_diff_test.go @@ -150,7 +150,7 @@ func testRngMapDiffAgainstEmpty(t *testing.T, scale int, rngTest rangeDiffTest) assert.Equal(t, inRange[cnt][0], val.Tuple(diff.Key)) assert.Equal(t, inRange[cnt][1], val.Tuple(diff.From)) assert.Nil(t, val.Tuple(diff.To)) - assert.True(t, rngTest.rng.Matches(val.Tuple(diff.Key))) + assert.True(t, rngTest.rng.Matches(ctx, val.Tuple(diff.Key))) cnt++ return nil }) @@ -162,7 +162,7 @@ func testRngMapDiffAgainstEmpty(t *testing.T, scale int, rngTest rangeDiffTest) assert.Equal(t, inRange[cnt][0], val.Tuple(diff.Key)) assert.Equal(t, inRange[cnt][1], val.Tuple(diff.To)) assert.Nil(t, val.Tuple(diff.From)) - assert.True(t, rngTest.rng.Matches(val.Tuple(diff.Key))) + assert.True(t, rngTest.rng.Matches(ctx, val.Tuple(diff.Key))) cnt++ return nil }) @@ -178,7 +178,7 @@ func testRngDeleteDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numDeletes deletes := tups[:numDeletes] sort.Slice(deletes, func(i, j int) bool { - return from.keyDesc.Compare(deletes[i][0], deletes[j][0]) < 0 + return from.keyDesc.Compare(ctx, deletes[i][0], deletes[j][0]) < 0 }) inRange := getPairsInRange(deletes, rngTest.rng) to := makeMapWithDeletes(t, from, deletes...) @@ -187,7 +187,7 @@ func testRngDeleteDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numDeletes err := RangeDiffMaps(ctx, from, to, rngTest.rng, func(ctx context.Context, diff tree.Diff) error { assert.Equal(t, tree.RemovedDiff, diff.Type) assert.Equal(t, inRange[cnt][0], val.Tuple(diff.Key)) - assert.True(t, rngTest.rng.Matches(val.Tuple(diff.Key))) + assert.True(t, rngTest.rng.Matches(ctx, val.Tuple(diff.Key))) cnt++ return nil }) @@ -207,7 +207,7 @@ func testRngInsertDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numInserts } assert.Equal(t, inRange[cnt][0], val.Tuple(diff.Key)) assert.Equal(t, inRange[cnt][1], val.Tuple(diff.To)) - assert.True(t, rngTest.rng.Matches(val.Tuple(diff.Key))) + assert.True(t, rngTest.rng.Matches(ctx, val.Tuple(diff.Key))) cnt++ return nil }) @@ -224,7 +224,7 @@ func testRngUpdateDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numUpdates sub := tups[:numUpdates] sort.Slice(sub, func(i, j int) bool { - return from.keyDesc.Compare(sub[i][0], sub[j][0]) < 0 + return from.keyDesc.Compare(ctx, sub[i][0], sub[j][0]) < 0 }) kd, vd := from.Descriptors() @@ -232,7 +232,7 @@ func testRngUpdateDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numUpdates to := makeMapWithUpdates(t, from, updates...) var inRange [][3]val.Tuple for _, pair := range updates { - if rngTest.rng.Matches(pair[0]) { + if rngTest.rng.Matches(ctx, pair[0]) { inRange = append(inRange, pair) } } @@ -243,7 +243,7 @@ func testRngUpdateDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numUpdates assert.Equal(t, inRange[cnt][0], val.Tuple(diff.Key)) assert.Equal(t, inRange[cnt][1], val.Tuple(diff.From)) assert.Equal(t, inRange[cnt][2], val.Tuple(diff.To)) - assert.True(t, rngTest.rng.Matches(val.Tuple(diff.Key))) + assert.True(t, rngTest.rng.Matches(ctx, val.Tuple(diff.Key))) cnt++ return nil }) @@ -265,7 +265,7 @@ func makeRandomOpenStopRangeTest(kd val.TupleDesc, tuples [][2]val.Tuple) rangeD start := tuples[i][0] stop := tuples[j][0] - return rangeDiffTest{tuples: tuples, rng: OpenStopRange(start, stop, kd)} + return rangeDiffTest{tuples: tuples, rng: OpenStopRange(ctx, start, stop, kd)} } func makeRandomGreaterOrEqualRangeTest(kd val.TupleDesc, tuples [][2]val.Tuple) rangeDiffTest { @@ -282,7 +282,7 @@ func makeRandomLesserRangeTest(kd val.TupleDesc, tuples [][2]val.Tuple) rangeDif func getPairsInRange(tuples [][2]val.Tuple, rng Range) (keys [][2]val.Tuple) { for _, pair := range tuples { - if rng.Matches(pair[0]) { + if rng.Matches(ctx, pair[0]) { keys = append(keys, pair) } } diff --git a/go/store/prolly/map_test.go b/go/store/prolly/map_test.go index 73bac6657be..7ccb93be20c 100644 --- a/go/store/prolly/map_test.go +++ b/go/store/prolly/map_test.go @@ -112,7 +112,7 @@ func TestMutateMapWithTupleIter(t *testing.T) { for _, s := range scales { t.Run("scale "+strconv.Itoa(s), func(t *testing.T) { - all := tree.RandomTuplePairs(s, kd, vd, ns) + all := tree.RandomTuplePairs(ctx, s, kd, vd, ns) // randomize |all| and partition rand.Shuffle(s, func(i, j int) { @@ -123,7 +123,7 @@ func TestMutateMapWithTupleIter(t *testing.T) { // unchanged tuples statics := make([][2]val.Tuple, s/4) copy(statics, all[:q1]) - tree.SortTuplePairs(statics, kd) + tree.SortTuplePairs(ctx, statics, kd) // tuples to be updated updates := make([][2]val.Tuple, s/4) @@ -132,7 +132,7 @@ func TestMutateMapWithTupleIter(t *testing.T) { // shuffle values relative to keys updates[i][1], updates[j][1] = updates[j][1], updates[i][1] }) - tree.SortTuplePairs(updates, kd) + tree.SortTuplePairs(ctx, updates, kd) // tuples to be deleted deletes := make([][2]val.Tuple, s/4) @@ -140,22 +140,22 @@ func TestMutateMapWithTupleIter(t *testing.T) { for i := range deletes { deletes[i][1] = nil } - tree.SortTuplePairs(deletes, kd) + tree.SortTuplePairs(ctx, deletes, kd) // tuples to be inserted inserts := make([][2]val.Tuple, s/4) copy(inserts, all[q3:]) - tree.SortTuplePairs(inserts, kd) + tree.SortTuplePairs(ctx, inserts, kd) var mutations [][2]val.Tuple mutations = append(mutations, inserts...) mutations = append(mutations, updates...) mutations = append(mutations, deletes...) - tree.SortTuplePairs(mutations, kd) + tree.SortTuplePairs(ctx, mutations, kd) // original tuples, before modification base := all[:q3] - tree.SortTuplePairs(base, kd) + tree.SortTuplePairs(ctx, base, kd) before := mustProllyMapFromTuples(t, kd, vd, base) ctx := context.Background() @@ -307,7 +307,7 @@ func makeProllyMap(t *testing.T, count int) (testMap, [][2]val.Tuple) { ) ns := tree.NewTestNodeStore() - tuples := tree.RandomTuplePairs(count, kd, vd, ns) + tuples := tree.RandomTuplePairs(ctx, count, kd, vd, ns) om := mustProllyMapFromTuples(t, kd, vd, tuples) return om, tuples @@ -320,7 +320,7 @@ func makeProllySecondaryIndex(t *testing.T, count int) (testMap, [][2]val.Tuple) ) vd := val.NewTupleDescriptor() ns := tree.NewTestNodeStore() - tuples := tree.RandomCompositeTuplePairs(count, kd, vd, ns) + tuples := tree.RandomCompositeTuplePairs(ctx, count, kd, vd, ns) om := mustProllyMapFromTuples(t, kd, vd, tuples) return om, tuples @@ -352,8 +352,8 @@ func testGet(t *testing.T, om testMap, tuples [][2]val.Tuple) { inserts := generateInserts(t, om, kd, vd, len(tuples)/2) for _, kv := range inserts { err := om.Get(ctx, kv[0], func(key, val val.Tuple) (err error) { - assert.Equal(t, 0, len(key), "Got %s", kd.Format(key)) - assert.Equal(t, 0, len(val), "Got %s", vd.Format(val)) + assert.Equal(t, 0, len(key), "Got %s", kd.Format(ctx, key)) + assert.Equal(t, 0, len(val), "Got %s", vd.Format(ctx, val)) return nil }) require.NoError(t, err) @@ -362,7 +362,7 @@ func testGet(t *testing.T, om testMap, tuples [][2]val.Tuple) { // find the expected ordinal return value for this non-existent key exp := len(tuples) for i := 0; i < len(tuples); i++ { - if kd.Compare(tuples[i][0], kv[0]) >= 0 { + if kd.Compare(ctx, tuples[i][0], kv[0]) >= 0 { exp = i break } @@ -432,7 +432,7 @@ func testIterAll(t *testing.T, om testMap, tuples [][2]val.Tuple) { } func pointRangeFromTuple(tup val.Tuple, desc val.TupleDesc) Range { - return closedRange(tup, tup, desc) + return closedRange(ctx, tup, tup, desc) } func formatTuples(tuples [][2]val.Tuple, kd, vd val.TupleDesc) string { @@ -442,9 +442,9 @@ func formatTuples(tuples [][2]val.Tuple, kd, vd val.TupleDesc) string { sb.WriteString(") {\n") for _, kv := range tuples { sb.WriteString("\t") - sb.WriteString(kd.Format(kv[0])) + sb.WriteString(kd.Format(ctx, kv[0])) sb.WriteString(", ") - sb.WriteString(vd.Format(kv[1])) + sb.WriteString(vd.Format(ctx, kv[1])) sb.WriteString("\n") } sb.WriteString("}\n") diff --git a/go/store/prolly/mutable_map_read_test.go b/go/store/prolly/mutable_map_read_test.go index e6b125ee07d..8e787f46997 100644 --- a/go/store/prolly/mutable_map_read_test.go +++ b/go/store/prolly/mutable_map_read_test.go @@ -125,7 +125,7 @@ func makeMutableMap(t *testing.T, count int) (testMap, [][2]val.Tuple) { val.Type{Enc: val.Uint32Enc, Nullable: true}, ) - tuples := tree.RandomTuplePairs(count, kd, vd, ns) + tuples := tree.RandomTuplePairs(ctx, count, kd, vd, ns) // 2/3 of tuples in Map // 1/3 of tuples in memoryMap clone := tree.CloneRandomTuples(tuples) @@ -134,8 +134,8 @@ func makeMutableMap(t *testing.T, count int) (testMap, [][2]val.Tuple) { mapTuples := clone[:split] memTuples := clone[split:] - tree.SortTuplePairs(mapTuples, kd) - tree.SortTuplePairs(memTuples, kd) + tree.SortTuplePairs(ctx, mapTuples, kd) + tree.SortTuplePairs(ctx, memTuples, kd) serializer := message.NewProllyMapSerializer(vd, ns.Pool()) chunker, err := tree.NewEmptyChunker(ctx, ns, serializer) @@ -174,7 +174,7 @@ func deleteFromMutableMap(mut *MutableMap, tt [][2]val.Tuple) (*MutableMap, [][2 // re-sort the remaining tuples remaining := tt[count/4:] desc := keyDescFromMap(mut) - tree.SortTuplePairs(remaining, desc) + tree.SortTuplePairs(ctx, remaining, desc) ctx := context.Background() for _, kv := range deletes { diff --git a/go/store/prolly/mutable_map_write_test.go b/go/store/prolly/mutable_map_write_test.go index a4b4b1c5bfe..f719d478689 100644 --- a/go/store/prolly/mutable_map_write_test.go +++ b/go/store/prolly/mutable_map_write_test.go @@ -560,7 +560,7 @@ func materializeMap(t *testing.T, mut *MutableMap) Map { if next == nil { break } - cmp := mut.keyDesc.Compare(val.Tuple(prev), val.Tuple(next)) + cmp := mut.keyDesc.Compare(ctx, val.Tuple(prev), val.Tuple(next)) assert.True(t, cmp < 0) prev = next } diff --git a/go/store/prolly/proximity_map.go b/go/store/prolly/proximity_map.go index 467196fbf16..309e5df214c 100644 --- a/go/store/prolly/proximity_map.go +++ b/go/store/prolly/proximity_map.go @@ -487,7 +487,7 @@ func (b *ProximityMapBuilder) getNextPathSegmentCandidates(ctx context.Context, prefixTupleBuilder.PutByteString(0, currentPath) prefixTuple := prefixTupleBuilder.Build(b.ns.Pool()) - prefixRange := PrefixRange(prefixTuple, prefixTupleBuilder.Desc) + prefixRange := PrefixRange(ctx, prefixTuple, prefixTupleBuilder.Desc) return pathMap.IterRange(ctx, prefixRange) } diff --git a/go/store/prolly/proximity_mutable_map.go b/go/store/prolly/proximity_mutable_map.go index 961f53699d6..b63c8a1103e 100644 --- a/go/store/prolly/proximity_mutable_map.go +++ b/go/store/prolly/proximity_mutable_map.go @@ -171,7 +171,7 @@ func (f ProximityFlusher) visitNode( var nodeSubtrees []uint64 if node.IsLeaf() { - keys, values, nodeSubtrees = f.rebuildLeafNodeWithEdits(node, edits, keyDesc) + keys, values, nodeSubtrees = f.rebuildLeafNodeWithEdits(ctx, node, edits, keyDesc) } else { // sort the list of edits based on which child node contains them. childEdits := make(map[int]childEditList) @@ -274,6 +274,7 @@ func serializeVectorIndexNode( // rebuildLeafNodeWithEdits creates a new leaf node by applying a list of edits to an existing node. func (f ProximityFlusher) rebuildLeafNodeWithEdits( + ctx context.Context, originalNode tree.Node, edits []VectorIndexKV, keyDesc val.TupleDesc, @@ -300,7 +301,7 @@ func (f ProximityFlusher) rebuildLeafNodeWithEdits( } editKey := val.Tuple(edits[editIdx].key) nodeKey := val.Tuple(originalNode.GetKey(nodeIdx)) - cmp := keyDesc.Compare(editKey, nodeKey) + cmp := keyDesc.Compare(ctx, editKey, nodeKey) if cmp < 0 { //edit comes first // Edit doesn't match an existing key: it must be an insert. @@ -341,17 +342,17 @@ func (f ProximityFlusher) rebuildNode(ctx context.Context, ns tree.NodeStore, no if err != nil { return tree.Node{}, 0, err } - editSkipList := skip.NewSkipList(func(left, right []byte) int { - return keyDesc.Compare(left, right) + editSkipList := skip.NewSkipList(func(ctx context.Context, left, right []byte) int { + return keyDesc.Compare(ctx, left, right) }) for _, edit := range edits { - editSkipList.Put(edit.key, edit.value) + editSkipList.Put(ctx, edit.key, edit.value) } insertFromNode := func(nd tree.Node, i int) error { key := nd.GetKey(i) value := nd.GetValue(i) - _, hasNewVal := editSkipList.Get(key) + _, hasNewVal := editSkipList.Get(ctx, key) if !hasNewVal { // TODO: Is it faster if we fetch the level from the current tree? keyLevel := tree.DeterministicHashLevel(f.logChunkSize, key) diff --git a/go/store/prolly/sort/external_test.go b/go/store/prolly/sort/external_test.go index ea4bad0a977..4c5867d08b8 100644 --- a/go/store/prolly/sort/external_test.go +++ b/go/store/prolly/sort/external_test.go @@ -91,7 +91,7 @@ func TestFlush(t *testing.T) { } keyCmp := func(l, r val.Tuple) bool { - return tt.td.Compare(l, r) <= 0 + return tt.td.Compare(ctx, l, r) <= 0 } t.Run("sorting", func(t *testing.T) { @@ -175,7 +175,7 @@ func TestMerge(t *testing.T) { for _, tt := range tests { t.Run(name(tt.td, tt.counts), func(t *testing.T) { keyCmp := func(l, r val.Tuple) bool { - return tt.td.Compare(l, r) <= 0 + return tt.td.Compare(ctx, l, r) <= 0 } var keyMems []keyIterable @@ -281,7 +281,7 @@ func TestCompact(t *testing.T) { for _, tt := range tests { t.Run(name(tt.td, tt.fileCnt), func(t *testing.T) { keyCmp := func(l, r val.Tuple) bool { - return tt.td.Compare(l, r) <= 0 + return tt.td.Compare(ctx, l, r) <= 0 } var keyFiles []keyIterable @@ -416,7 +416,7 @@ func TestFileE2E(t *testing.T) { for _, tt := range tests { t.Run(fmt.Sprintf("%s %d-rows %d-batch %d-files", tt.name, tt.rows, tt.batchSize, tt.fileMax), func(t *testing.T) { keyCmp := func(l, r val.Tuple) bool { - return tt.td.Compare(l, r) <= 0 + return tt.td.Compare(ctx, l, r) <= 0 } ctx := context.Background() diff --git a/go/store/prolly/tree/diff.go b/go/store/prolly/tree/diff.go index 9fcd841e701..773c14c4ae6 100644 --- a/go/store/prolly/tree/diff.go +++ b/go/store/prolly/tree/diff.go @@ -139,7 +139,7 @@ func (td Differ[K, O]) next(ctx context.Context, advanceCursors bool) (diff Diff f := td.from.CurrentKey() t := td.to.CurrentKey() - cmp := td.order.Compare(K(f), K(t)) + cmp := td.order.Compare(ctx, K(f), K(t)) switch { case cmp < 0: diff --git a/go/store/prolly/tree/json_location.go b/go/store/prolly/tree/json_location.go index 2d270efad2c..25156d3caca 100644 --- a/go/store/prolly/tree/json_location.go +++ b/go/store/prolly/tree/json_location.go @@ -16,6 +16,7 @@ package tree import ( "bytes" + "context" "fmt" "slices" "strconv" @@ -464,7 +465,7 @@ type jsonLocationOrdering struct { var _ Ordering[[]byte] = &jsonLocationOrdering{} -func (o *jsonLocationOrdering) Compare(left, right []byte) int { +func (o *jsonLocationOrdering) Compare(ctx context.Context, left, right []byte) int { // A JSON document that fits entirely in a single chunk has no keys, if len(left) == 0 && len(right) == 0 { return 0 diff --git a/go/store/prolly/tree/map.go b/go/store/prolly/tree/map.go index 555d235d39e..24118b0d506 100644 --- a/go/store/prolly/tree/map.go +++ b/go/store/prolly/tree/map.go @@ -225,8 +225,8 @@ func (t StaticMap[K, V, O]) HashOf() hash.Hash { func (t StaticMap[K, V, O]) Mutate() MutableMap[K, V, O, StaticMap[K, V, O]] { return MutableMap[K, V, O, StaticMap[K, V, O]]{ - Edits: skip.NewSkipList(func(left, right []byte) int { - return t.Order.Compare(left, right) + Edits: skip.NewSkipList(func(ctx context.Context, left, right []byte) int { + return t.Order.Compare(ctx, left, right) }), Static: t, } @@ -251,7 +251,7 @@ func (t StaticMap[K, V, O]) Get(ctx context.Context, query K, cb KeyValueFn[K, V if cur.Valid() { key = K(cur.CurrentKey()) - if t.Order.Compare(query, key) == 0 { + if t.Order.Compare(ctx, query, key) == 0 { value = V(cur.currentValue()) } else { key = nil @@ -271,7 +271,7 @@ func (t StaticMap[K, V, O]) GetPrefix(ctx context.Context, query K, prefixOrder if cur.Valid() { key = K(cur.CurrentKey()) - if prefixOrder.Compare(query, key) == 0 { + if prefixOrder.Compare(ctx, query, key) == 0 { value = V(cur.currentValue()) } else { key = nil @@ -285,7 +285,7 @@ func (t StaticMap[K, V, O]) Has(ctx context.Context, query K) (ok bool, err erro if err != nil { return false, err } else if cur.Valid() { - ok = t.Order.Compare(query, K(cur.CurrentKey())) == 0 + ok = t.Order.Compare(ctx, query, K(cur.CurrentKey())) == 0 } return } @@ -296,7 +296,7 @@ func (t StaticMap[K, V, O]) HasPrefix(ctx context.Context, query K, prefixOrder return false, err } else if cur.Valid() { // true if |query| is a prefix of |cur.currentKey()| - ok = prefixOrder.Compare(query, K(cur.CurrentKey())) == 0 + ok = prefixOrder.Compare(ctx, query, K(cur.CurrentKey())) == 0 } return } diff --git a/go/store/prolly/tree/merge.go b/go/store/prolly/tree/merge.go index 40d456c4bd5..792bdbd69d2 100644 --- a/go/store/prolly/tree/merge.go +++ b/go/store/prolly/tree/merge.go @@ -157,7 +157,7 @@ func sendPatches[K ~[]byte, O Ordering[K]]( } for lok && rok { - cmp := l.order.Compare(K(left.Key), K(right.Key)) + cmp := l.order.Compare(ctx, K(left.Key), K(right.Key)) switch { case cmp < 0: diff --git a/go/store/prolly/tree/mutable_map.go b/go/store/prolly/tree/mutable_map.go index 2130c91fc66..b3dec5cdffd 100644 --- a/go/store/prolly/tree/mutable_map.go +++ b/go/store/prolly/tree/mutable_map.go @@ -26,18 +26,18 @@ type MutableMap[K, V ~[]byte, O Ordering[K], M MapInterface[K, V, O]] struct { Static M } -func (m MutableMap[K, V, O, M]) Put(_ context.Context, key K, value V) error { - m.Edits.Put(key, value) +func (m MutableMap[K, V, O, M]) Put(ctx context.Context, key K, value V) error { + m.Edits.Put(ctx, key, value) return nil } -func (m MutableMap[K, V, O, M]) Delete(_ context.Context, key K) error { - m.Edits.Put(key, nil) +func (m MutableMap[K, V, O, M]) Delete(ctx context.Context, key K) error { + m.Edits.Put(ctx, key, nil) return nil } func (m MutableMap[K, V, O, M]) Get(ctx context.Context, key K, cb KeyValueFn[K, V]) (err error) { - value, ok := m.Edits.Get(key) + value, ok := m.Edits.Get(ctx, key) if ok { if value == nil { key = nil // there is a pending delete of |key| in |m.Edits|. @@ -51,12 +51,12 @@ func (m MutableMap[K, V, O, M]) Get(ctx context.Context, key K, cb KeyValueFn[K, func (m MutableMap[K, V, O, M]) GetPrefix(ctx context.Context, key K, prefixOrder O, cb KeyValueFn[K, V]) (err error) { iter := m.Edits.GetIterFromSeekFn(func(k []byte) (advance bool) { if k != nil { // seek until |k| >= |key| - advance = prefixOrder.Compare(k, key) < 0 + advance = prefixOrder.Compare(ctx, k, key) < 0 } return }) k, v := iter.Current() - if k != nil && prefixOrder.Compare(k, key) == 0 { + if k != nil && prefixOrder.Compare(ctx, k, key) == 0 { if v == nil { k = nil // there is a pending delete of |key| in |m.Edits|. } @@ -66,7 +66,7 @@ func (m MutableMap[K, V, O, M]) GetPrefix(ctx context.Context, key K, prefixOrde } func (m MutableMap[K, V, O, M]) Has(ctx context.Context, key K) (present bool, err error) { - value, ok := m.Edits.Get(key) + value, ok := m.Edits.Get(ctx, key) if ok { present = value != nil return @@ -77,12 +77,12 @@ func (m MutableMap[K, V, O, M]) Has(ctx context.Context, key K) (present bool, e func (m MutableMap[K, V, O, M]) HasPrefix(ctx context.Context, key K, prefixOrder O) (present bool, err error) { iter := m.Edits.GetIterFromSeekFn(func(k []byte) (advance bool) { if k != nil { // seek until |k| >= |key| - advance = prefixOrder.Compare(k, key) < 0 + advance = prefixOrder.Compare(ctx, k, key) < 0 } return }) k, v := iter.Current() - if k != nil && prefixOrder.Compare(k, key) == 0 { + if k != nil && prefixOrder.Compare(ctx, k, key) == 0 { present = v != nil return } diff --git a/go/store/prolly/tree/mutator.go b/go/store/prolly/tree/mutator.go index e6474e16cbf..824f3013744 100644 --- a/go/store/prolly/tree/mutator.go +++ b/go/store/prolly/tree/mutator.go @@ -91,7 +91,7 @@ func ApplyMutations[K ~[]byte, O Ordering[K], S message.Serializer]( if cur.Valid() { // Compare mutations |newKey| and |newValue| // to the existing pair from the cursor - if order.Compare(K(newKey), K(cur.CurrentKey())) == 0 { + if order.Compare(ctx, K(newKey), K(cur.CurrentKey())) == 0 { oldValue = cur.currentValue() } @@ -132,7 +132,7 @@ func ApplyMutations[K ~[]byte, O Ordering[K], S message.Serializer]( prev := newKey newKey, newValue = edits.NextMutation(ctx) if newKey != nil { - assertTrue(order.Compare(K(newKey), K(prev)) > 0, "expected sorted edits") + assertTrue(order.Compare(ctx, K(newKey), K(prev)) > 0, "expected sorted edits") } } diff --git a/go/store/prolly/tree/node_cursor.go b/go/store/prolly/tree/node_cursor.go index 2bcecdef57b..f1dfbe2c128 100644 --- a/go/store/prolly/tree/node_cursor.go +++ b/go/store/prolly/tree/node_cursor.go @@ -37,10 +37,10 @@ type cursor struct { nrw NodeStore } -type SearchFn func(nd Node) (idx int) +type SearchFn func(ctx context.Context, nd Node) (idx int) type Ordering[K ~[]byte] interface { - Compare(left, right K) int + Compare(ctx context.Context, left, right K) int } func newCursorAtStart(ctx context.Context, ns NodeStore, nd Node) (cur *cursor, err error) { @@ -102,7 +102,7 @@ func newCursorAtOrdinal(ctx context.Context, ns NodeStore, nd Node, ord uint64) } distance := int64(ord) - return newCursorFromSearchFn(ctx, ns, nd, func(nd Node) (idx int) { + return newCursorFromSearchFn(ctx, ns, nd, func(ctx context.Context, nd Node) (idx int) { if nd.IsLeaf() { return int(distance) } @@ -163,7 +163,7 @@ func newCursorAtKey[K ~[]byte, O Ordering[K]](ctx context.Context, ns NodeStore, func newCursorFromSearchFn(ctx context.Context, ns NodeStore, nd Node, search SearchFn) (cur *cursor, err error) { cur = &cursor{nd: nd, nrw: ns} - cur.idx = search(cur.nd) + cur.idx = search(ctx, cur.nd) for !cur.isLeaf() { // stay in bounds for internal nodes cur.keepInBounds() @@ -176,7 +176,7 @@ func newCursorFromSearchFn(ctx context.Context, ns NodeStore, nd Node, search Se parent := cur cur = &cursor{nd: nd, parent: parent, nrw: ns} - cur.idx = search(cur.nd) + cur.idx = search(ctx, cur.nd) } return } @@ -189,7 +189,7 @@ func newLeafCursorAtKey[K ~[]byte, O Ordering[K]](ctx context.Context, ns NodeSt i, j := 0, cur.nd.Count() for i < j { h := int(uint(i+j) >> 1) - cmp := order.Compare(key, K(cur.nd.GetKey(h))) + cmp := order.Compare(ctx, key, K(cur.nd.GetKey(h))) if cmp > 0 { i = h + 1 } else { @@ -216,7 +216,7 @@ func newLeafCursorAtKey[K ~[]byte, O Ordering[K]](ctx context.Context, ns NodeSt // searchForKey returns a SearchFn for |key|. func searchForKey[K ~[]byte, O Ordering[K]](key K, order O) SearchFn { - return func(nd Node) (idx int) { + return func(ctx context.Context, nd Node) (idx int) { // A flattened leaf node contains 1 value and 0 keys. We check for this and return the index of the only value, // in order to prevent a comparison against the nonexistent key. if nd.keys.IsEmpty() { @@ -228,7 +228,7 @@ func searchForKey[K ~[]byte, O Ordering[K]](key K, order O) SearchFn { i, j := 0, n for i < j { h := int(uint(i+j) >> 1) // avoid overflow when computing h - less := order.Compare(key, K(nd.GetKey(h))) <= 0 + less := order.Compare(ctx, key, K(nd.GetKey(h))) <= 0 // i ≤ h < j if !less { i = h + 1 // preserves f(i-1) == false @@ -335,8 +335,8 @@ func currentCursorItems(cur *cursor) (key, value Item) { func Seek[K ~[]byte, O Ordering[K]](ctx context.Context, cur *cursor, key K, order O) (err error) { inBounds := true if cur.parent != nil { - inBounds = inBounds && order.Compare(key, K(cur.firstKey())) >= 0 - inBounds = inBounds && order.Compare(key, K(cur.lastKey())) <= 0 + inBounds = inBounds && order.Compare(ctx, key, K(cur.firstKey())) >= 0 + inBounds = inBounds && order.Compare(ctx, key, K(cur.lastKey())) <= 0 } if !inBounds { @@ -354,7 +354,7 @@ func Seek[K ~[]byte, O Ordering[K]](ctx context.Context, cur *cursor, key K, ord } } - cur.idx = searchForKey(key, order)(cur.nd) + cur.idx = searchForKey(key, order)(ctx, cur.nd) return } diff --git a/go/store/prolly/tree/node_cursor_test.go b/go/store/prolly/tree/node_cursor_test.go index 1d46b5cfa6f..7e8836b9e4b 100644 --- a/go/store/prolly/tree/node_cursor_test.go +++ b/go/store/prolly/tree/node_cursor_test.go @@ -164,7 +164,7 @@ var valDesc = val.NewTupleDescriptor( ) func randomTupleItemPairs(count int, ns NodeStore) (items [][2]Item) { - tups := RandomTuplePairs(count, keyDesc, valDesc, ns) + tups := RandomTuplePairs(ctx, count, keyDesc, valDesc, ns) items = make([][2]Item, count) if len(tups) != len(items) { panic("mismatch") diff --git a/go/store/prolly/tree/prolly_fields.go b/go/store/prolly/tree/prolly_fields.go index 4655d817ea4..55605c2bf3c 100644 --- a/go/store/prolly/tree/prolly_fields.go +++ b/go/store/prolly/tree/prolly_fields.go @@ -288,7 +288,7 @@ func PutField(ctx context.Context, ns NodeStore, tb *val.TupleBuilder, i int, v if err != nil { return err } - tb.PutExtendedAddr(i, b) + tb.PutExtendedAddr(i, hash.New(b)) default: panic(fmt.Sprintf("unknown encoding %v %v", enc, v)) } diff --git a/go/store/prolly/tree/proximity_map.go b/go/store/prolly/tree/proximity_map.go index db0906ccdc6..f526ea7af97 100644 --- a/go/store/prolly/tree/proximity_map.go +++ b/go/store/prolly/tree/proximity_map.go @@ -59,8 +59,8 @@ func (t ProximityMap[K, V, O]) HasPrefix(ctx context.Context, query K, prefixOrd func (t ProximityMap[K, V, O]) Mutate() MutableMap[K, V, O, ProximityMap[K, V, O]] { return MutableMap[K, V, O, ProximityMap[K, V, O]]{ - Edits: skip.NewSkipList(func(left, right []byte) int { - return t.Order.Compare(left, right) + Edits: skip.NewSkipList(func(ctx context.Context, left, right []byte) int { + return t.Order.Compare(ctx, left, right) }), Static: t, } diff --git a/go/store/prolly/tree/stats.go b/go/store/prolly/tree/stats.go index 1573d01893d..9bc488af4e9 100644 --- a/go/store/prolly/tree/stats.go +++ b/go/store/prolly/tree/stats.go @@ -65,7 +65,7 @@ func DiffChunksAtLevel[K, V ~[]byte, O Ordering[K]](ctx context.Context, level u f := fromNode.GetKey(i) t := toNode.GetKey(j) - cmp := from.Order.Compare(K(f), K(t)) + cmp := from.Order.Compare(ctx, K(f), K(t)) if cmp == 0 { // replace from->to diffs = append(diffs, chunkDiff{from: []hash.Hash{fromAddr}, to: []hash.Hash{toAddr}}) @@ -85,7 +85,7 @@ func DiffChunksAtLevel[K, V ~[]byte, O Ordering[K]](ctx context.Context, level u } f = fromNode.GetKey(i) t = toNode.GetKey(j) - cmp = from.Order.Compare(K(f), K(t)) + cmp = from.Order.Compare(ctx, K(f), K(t)) } // either addrs equal, or keys synced var newChunkDiff chunkDiff diff --git a/go/store/prolly/tree/testutils.go b/go/store/prolly/tree/testutils.go index db20dbd9d74..e55b56e2c68 100644 --- a/go/store/prolly/tree/testutils.go +++ b/go/store/prolly/tree/testutils.go @@ -45,7 +45,7 @@ func NewTupleLeafNode(keys, values []val.Tuple) Node { return newLeafNode(ks, vs) } -func RandomTuplePairs(count int, keyDesc, valDesc val.TupleDesc, ns NodeStore) (items [][2]val.Tuple) { +func RandomTuplePairs(ctx context.Context, count int, keyDesc, valDesc val.TupleDesc, ns NodeStore) (items [][2]val.Tuple) { keyBuilder := val.NewTupleBuilder(keyDesc) valBuilder := val.NewTupleBuilder(valDesc) @@ -57,12 +57,12 @@ func RandomTuplePairs(count int, keyDesc, valDesc val.TupleDesc, ns NodeStore) ( dupes := make([]int, 0, count) for { - SortTuplePairs(items, keyDesc) + SortTuplePairs(ctx, items, keyDesc) for i := range items { if i == 0 { continue } - if keyDesc.Compare(items[i][0], items[i-1][0]) == 0 { + if keyDesc.Compare(ctx, items[i][0], items[i-1][0]) == 0 { dupes = append(dupes, i) } } @@ -79,7 +79,7 @@ func RandomTuplePairs(count int, keyDesc, valDesc val.TupleDesc, ns NodeStore) ( return items } -func RandomCompositeTuplePairs(count int, keyDesc, valDesc val.TupleDesc, ns NodeStore) (items [][2]val.Tuple) { +func RandomCompositeTuplePairs(ctx context.Context, count int, keyDesc, valDesc val.TupleDesc, ns NodeStore) (items [][2]val.Tuple) { // preconditions if count%5 != 0 { panic("expected empty divisible by 5") @@ -88,7 +88,7 @@ func RandomCompositeTuplePairs(count int, keyDesc, valDesc val.TupleDesc, ns Nod panic("expected composite key") } - tt := RandomTuplePairs(count, keyDesc, valDesc, ns) + tt := RandomTuplePairs(ctx, count, keyDesc, valDesc, ns) tuples := make([][2]val.Tuple, len(tt)*3) for i := range tuples { @@ -106,9 +106,9 @@ func RandomCompositeTuplePairs(count int, keyDesc, valDesc val.TupleDesc, ns Nod copy(f2, swap) }) - SortTuplePairs(tuples, keyDesc) + SortTuplePairs(ctx, tuples, keyDesc) - tuples = deduplicateTuples(keyDesc, tuples) + tuples = deduplicateTuples(ctx, keyDesc, tuples) return tuples[:count] } @@ -142,9 +142,9 @@ func CloneRandomTuples(items [][2]val.Tuple) (clone [][2]val.Tuple) { return } -func SortTuplePairs(items [][2]val.Tuple, keyDesc val.TupleDesc) { +func SortTuplePairs(ctx context.Context, items [][2]val.Tuple, keyDesc val.TupleDesc) { sort.Slice(items, func(i, j int) bool { - return keyDesc.Compare(items[i][0], items[j][0]) < 0 + return keyDesc.Compare(ctx, items[i][0], items[j][0]) < 0 }) } @@ -178,12 +178,12 @@ func newLeafNode(keys, values []Item) Node { } // assumes a sorted list -func deduplicateTuples(desc val.TupleDesc, tups [][2]val.Tuple) (uniq [][2]val.Tuple) { +func deduplicateTuples(ctx context.Context, desc val.TupleDesc, tups [][2]val.Tuple) (uniq [][2]val.Tuple) { uniq = make([][2]val.Tuple, 1, len(tups)) uniq[0] = tups[0] for i := 1; i < len(tups); i++ { - cmp := desc.Compare(tups[i-1][0], tups[i][0]) + cmp := desc.Compare(ctx, tups[i-1][0], tups[i][0]) if cmp < 0 { uniq = append(uniq, tups[i]) } diff --git a/go/store/prolly/tree/three_way_differ.go b/go/store/prolly/tree/three_way_differ.go index ba767d29704..e1bc3d7ded2 100644 --- a/go/store/prolly/tree/three_way_differ.go +++ b/go/store/prolly/tree/three_way_differ.go @@ -152,7 +152,7 @@ func (d *ThreeWayDiffer[K, O]) Next(ctx *sql.Context) (ThreeWayDiff, error) { nextState = dsCompare } case dsCompare: - cmp := d.lIter.order.Compare(K(d.lDiff.Key), K(d.rDiff.Key)) + cmp := d.lIter.order.Compare(ctx, K(d.lDiff.Key), K(d.rDiff.Key)) switch { case cmp < 0: nextState = dsNewLeft diff --git a/go/store/prolly/tuple_map.go b/go/store/prolly/tuple_map.go index 1f2243544af..b2443a44e80 100644 --- a/go/store/prolly/tuple_map.go +++ b/go/store/prolly/tuple_map.go @@ -159,7 +159,7 @@ func makeDiffCallBack(from, to Map, innerCb tree.DiffFn) tree.DiffFn { // Skip diffs produced by non-canonical tuples. A canonical-tuple is a // tuple where any null suffixes have been trimmed. if diff.Type == tree.ModifiedDiff && - from.valDesc.Compare(val.Tuple(diff.From), val.Tuple(diff.To)) == 0 { + from.valDesc.Compare(ctx, val.Tuple(diff.From), val.Tuple(diff.To)) == 0 { return nil } return innerCb(ctx, diff) @@ -313,7 +313,7 @@ func (m Map) HasPrefix(ctx context.Context, preKey val.Tuple, preDesc val.TupleD // IterRange returns a mutableMapIter that iterates over a Range. func (m Map) IterRange(ctx context.Context, rng Range) (iter MapIter, err error) { - stop, ok := rng.KeyRangeLookup(m.Pool()) + stop, ok := rng.KeyRangeLookup(ctx, m.Pool()) if ok { iter, err = m.IterKeyRange(ctx, rng.Tup, stop) } else { @@ -367,8 +367,8 @@ func (m Map) Pool() pool.BuffPool { return m.tuples.NodeStore.Pool() } -func (m Map) CompareItems(left, right tree.Item) int { - return m.keyDesc.Compare(val.Tuple(left), val.Tuple(right)) +func (m Map) CompareItems(ctx context.Context, left, right tree.Item) int { + return m.keyDesc.Compare(ctx, val.Tuple(left), val.Tuple(right)) } func treeIterFromRange( @@ -437,9 +437,9 @@ func DebugFormat(ctx context.Context, m Map) (string, error) { } sb.WriteString("\t") - sb.WriteString(kd.Format(k)) + sb.WriteString(kd.Format(ctx, k)) sb.WriteString(": ") - sb.WriteString(vd.Format(v)) + sb.WriteString(vd.Format(ctx, v)) sb.WriteString(",\n") } sb.WriteString("}") diff --git a/go/store/prolly/tuple_mutable_map.go b/go/store/prolly/tuple_mutable_map.go index 47bacfbbc3d..be91daf5824 100644 --- a/go/store/prolly/tuple_mutable_map.go +++ b/go/store/prolly/tuple_mutable_map.go @@ -149,7 +149,7 @@ func (mut *GenericMutableMap[M, T]) Checkpoint(context.Context) error { } // Revert discards writes made since the last checkpoint. -func (mut *GenericMutableMap[M, T]) Revert(context.Context) { +func (mut *GenericMutableMap[M, T]) Revert(ctx context.Context) { // if we've accumulated a large number of writes // since we check-pointed, our last checkpoint // may be stashed in a separate tree.MutableMap @@ -157,7 +157,7 @@ func (mut *GenericMutableMap[M, T]) Revert(context.Context) { mut.tuples = *mut.stash return } - mut.tuples.Edits.Revert() + mut.tuples.Edits.Revert(ctx) } func (mut *GenericMutableMap[M, T]) flushPending(ctx context.Context) error { @@ -166,7 +166,7 @@ func (mut *GenericMutableMap[M, T]) flushPending(ctx context.Context) error { // must stash a copy of |mut.tuples| we can revert to. if mut.tuples.Edits.HasCheckpoint() { cp := mut.tuples.Copy() - cp.Edits.Revert() + cp.Edits.Revert(ctx) stash = &cp } serializer := mut.flusher.GetDefaultSerializer(ctx, mut) @@ -199,7 +199,7 @@ func (mut *GenericMutableMap[M, T]) IterRange(ctx context.Context, rng Range) (M if err != nil { return nil, err } - memIter := memIterFromRange(mut.tuples.Edits, rng) + memIter := memIterFromRange(ctx, mut.tuples.Edits, rng) iter := &mutableMapIter[val.Tuple, val.Tuple, val.TupleDesc]{ memory: memIter, @@ -272,9 +272,9 @@ func debugFormat(ctx context.Context, m *MutableMap) (string, error) { break } sb.WriteString("\t\t") - sb.WriteString(kd.Format(k)) + sb.WriteString(kd.Format(ctx, k)) sb.WriteString(": ") - sb.WriteString(vd.Format(v)) + sb.WriteString(vd.Format(ctx, v)) sb.WriteString(",\n") editIter.Advance() } @@ -296,9 +296,9 @@ func debugFormat(ctx context.Context, m *MutableMap) (string, error) { return "", err } sb.WriteString("\t\t") - sb.WriteString(kd.Format(k)) + sb.WriteString(kd.Format(ctx, k)) sb.WriteString(": ") - sb.WriteString(vd.Format(v)) + sb.WriteString(vd.Format(ctx, v)) sb.WriteString(",\n") } sb.WriteString("\t}\n}\n") diff --git a/go/store/prolly/tuple_range.go b/go/store/prolly/tuple_range.go index e9fde518e2b..4219ebc522b 100644 --- a/go/store/prolly/tuple_range.go +++ b/go/store/prolly/tuple_range.go @@ -15,6 +15,7 @@ package prolly import ( + "context" "math" "sort" @@ -26,8 +27,8 @@ import ( ) // OpenStopRange defines a half-open Range of Tuples [start, stop). -func OpenStopRange(start, stop val.Tuple, desc val.TupleDesc) Range { - return openStopRange(start, stop, desc) +func OpenStopRange(ctx context.Context, start, stop val.Tuple, desc val.TupleDesc) Range { + return openStopRange(ctx, start, stop, desc) } // GreaterOrEqualRange defines a Range of Tuples greater than or equal to |start|. @@ -41,8 +42,8 @@ func LesserRange(stop val.Tuple, desc val.TupleDesc) Range { } // PrefixRange constructs a Range for Tuples with a prefix of |prefix|. -func PrefixRange(prefix val.Tuple, desc val.TupleDesc) Range { - return closedRange(prefix, prefix, desc) +func PrefixRange(ctx context.Context, prefix val.Tuple, desc val.TupleDesc) Range { + return closedRange(ctx, prefix, prefix, desc) } // Range defines a subset of a prolly Tree Tuple index. @@ -86,7 +87,7 @@ type Bound struct { // aboveStart is used to find the start of the // physical partition defined by a Range. -func (r Range) aboveStart(t val.Tuple) bool { +func (r Range) aboveStart(ctx context.Context, t val.Tuple) bool { order := r.Desc.Comparator() for i := range r.Fields { bound := r.Fields[i].Lo @@ -97,7 +98,7 @@ func (r Range) aboveStart(t val.Tuple) bool { field := r.Desc.GetField(i, t) typ := r.Desc.Types[i] - cmp := order.CompareValues(i, field, bound.Value, typ) + cmp := order.CompareValues(ctx, i, field, bound.Value, typ) if cmp < 0 { // |field| is outside Range return false @@ -118,7 +119,7 @@ func (r Range) aboveStart(t val.Tuple) bool { // belowStop is used to find the end of the // physical partition defined by a Range. -func (r Range) belowStop(t val.Tuple) bool { +func (r Range) belowStop(ctx context.Context, t val.Tuple) bool { order := r.Desc.Comparator() for i := range r.Fields { bound := r.Fields[i].Hi @@ -129,7 +130,7 @@ func (r Range) belowStop(t val.Tuple) bool { field := r.Desc.GetField(i, t) typ := r.Desc.Types[i] - cmp := order.CompareValues(i, field, bound.Value, typ) + cmp := order.CompareValues(ctx, i, field, bound.Value, typ) if cmp > 0 { // |field| is outside Range return false @@ -150,7 +151,7 @@ func (r Range) belowStop(t val.Tuple) bool { // Matches returns true if all the filter predicates // for Range |r| are true for Tuple |t|. -func (r Range) Matches(t val.Tuple) bool { +func (r Range) Matches(ctx context.Context, t val.Tuple) bool { order := r.Desc.Comparator() for i := range r.Fields { field := r.Desc.GetField(i, t) @@ -158,7 +159,7 @@ func (r Range) Matches(t val.Tuple) bool { if r.Fields[i].BoundsAreEqual { v := r.Fields[i].Lo.Value - if order.CompareValues(i, field, v, typ) == 0 { + if order.CompareValues(ctx, i, field, v, typ) == 0 { continue } return false @@ -166,7 +167,7 @@ func (r Range) Matches(t val.Tuple) bool { lo := r.Fields[i].Lo if lo.Binding { - cmp := order.CompareValues(i, field, lo.Value, typ) + cmp := order.CompareValues(ctx, i, field, lo.Value, typ) if cmp < 0 || (cmp == 0 && !lo.Inclusive) { return false } @@ -174,7 +175,7 @@ func (r Range) Matches(t val.Tuple) bool { hi := r.Fields[i].Hi if hi.Binding { - cmp := order.CompareValues(i, field, hi.Value, typ) + cmp := order.CompareValues(ctx, i, field, hi.Value, typ) if cmp > 0 || (cmp == 0 && !hi.Inclusive) { return false } @@ -202,7 +203,7 @@ func (r Range) IsStrictKeyLookup(desc val.TupleDesc) bool { // numeric or string. The stop key adds +1 to a numeric final field, and appends // '0' to a string final field. // TODO: support non-exact final field, and use range upper bound? -func (r Range) KeyRangeLookup(pool pool.BuffPool) (val.Tuple, bool) { +func (r Range) KeyRangeLookup(ctx context.Context, pool pool.BuffPool) (val.Tuple, bool) { if r.Tup == nil { return nil, false } @@ -246,10 +247,10 @@ func (r Range) KeyRangeLookup(pool pool.BuffPool) (val.Tuple, bool) { } - return IncrementTuple(r.Tup, n, r.Desc, pool) + return IncrementTuple(ctx, r.Tup, n, r.Desc, pool) } -func IncrementTuple(start val.Tuple, n int, desc val.TupleDesc, pool pool.BuffPool) (val.Tuple, bool) { +func IncrementTuple(ctx context.Context, start val.Tuple, n int, desc val.TupleDesc, pool pool.BuffPool) (val.Tuple, bool) { tb := val.NewTupleBuilder(desc) for i := 0; i < n; i++ { if i != n { @@ -339,7 +340,7 @@ func IncrementTuple(start val.Tuple, n int, desc val.TupleDesc, pool pool.BuffPo return nil, false } stop := tb.Build(pool) - if desc.Compare(start, stop) >= 0 { + if desc.Compare(ctx, start, stop) >= 0 { // If cmp == 0, we lost precision serializing. // If cmp > 0, we overflowed and |stop| < |start|. // |stop| has to be strictly greater than |start| @@ -350,31 +351,31 @@ func IncrementTuple(start val.Tuple, n int, desc val.TupleDesc, pool pool.BuffPo } func rangeStartSearchFn(rng Range) tree.SearchFn { - return func(nd tree.Node) int { + return func(ctx context.Context, nd tree.Node) int { // todo(andy): inline sort.Search() return sort.Search(nd.Count(), func(i int) (in bool) { // if |tup| ∈ |rng|, set |in| to true tup := val.Tuple(nd.GetKey(i)) - in = rng.aboveStart(tup) + in = rng.aboveStart(ctx, tup) return }) } } func rangeStopSearchFn(rng Range) tree.SearchFn { - return func(nd tree.Node) (idx int) { + return func(ctx context.Context, nd tree.Node) (idx int) { // todo(andy): inline sort.Search() return sort.Search(nd.Count(), func(i int) (out bool) { // if |tup| ∈ |rng|, set |out| to false tup := val.Tuple(nd.GetKey(i)) - out = !rng.belowStop(tup) + out = !rng.belowStop(ctx, tup) return }) } } // closedRange defines an inclusive Range of Tuples from [start, stop]. -func closedRange(start, stop val.Tuple, desc val.TupleDesc) (rng Range) { +func closedRange(ctx context.Context, start, stop val.Tuple, desc val.TupleDesc) (rng Range) { rng = Range{ Fields: make([]RangeField, len(desc.Types)), Desc: desc, @@ -383,7 +384,7 @@ func closedRange(start, stop val.Tuple, desc val.TupleDesc) (rng Range) { for i := range rng.Fields { lo := desc.GetField(i, start) hi := desc.GetField(i, stop) - isEq := order.CompareValues(i, lo, hi, desc.Types[i]) == 0 + isEq := order.CompareValues(ctx, i, lo, hi, desc.Types[i]) == 0 rng.Fields[i] = RangeField{ Lo: Bound{Binding: true, Inclusive: true, Value: lo}, Hi: Bound{Binding: true, Inclusive: true, Value: hi}, @@ -394,8 +395,8 @@ func closedRange(start, stop val.Tuple, desc val.TupleDesc) (rng Range) { } // OpenStartRange defines a half-open Range of Tuples (start, stop]. -func openStartRange(start, stop val.Tuple, desc val.TupleDesc) (rng Range) { - rng = closedRange(start, stop, desc) +func openStartRange(ctx context.Context, start, stop val.Tuple, desc val.TupleDesc) (rng Range) { + rng = closedRange(ctx, start, stop, desc) last := len(rng.Fields) - 1 rng.Fields[last].Lo.Inclusive = false rng.Fields[last].BoundsAreEqual = false @@ -403,8 +404,8 @@ func openStartRange(start, stop val.Tuple, desc val.TupleDesc) (rng Range) { } // OpenStopRange defines a half-open Range of Tuples [start, stop). -func openStopRange(start, stop val.Tuple, desc val.TupleDesc) (rng Range) { - rng = closedRange(start, stop, desc) +func openStopRange(ctx context.Context, start, stop val.Tuple, desc val.TupleDesc) (rng Range) { + rng = closedRange(ctx, start, stop, desc) last := len(rng.Fields) - 1 rng.Fields[last].Hi.Inclusive = false rng.Fields[last].BoundsAreEqual = false @@ -412,8 +413,8 @@ func openStopRange(start, stop val.Tuple, desc val.TupleDesc) (rng Range) { } // OpenRange defines a non-inclusive Range of Tuples from (start, stop). -func openRange(start, stop val.Tuple, desc val.TupleDesc) (rng Range) { - rng = closedRange(start, stop, desc) +func openRange(ctx context.Context, start, stop val.Tuple, desc val.TupleDesc) (rng Range) { + rng = closedRange(ctx, start, stop, desc) last := len(rng.Fields) - 1 rng.Fields[last].Lo.Inclusive = false rng.Fields[last].Hi.Inclusive = false diff --git a/go/store/prolly/tuple_range_iter.go b/go/store/prolly/tuple_range_iter.go index 8a3154b30ae..a0e6a510255 100644 --- a/go/store/prolly/tuple_range_iter.go +++ b/go/store/prolly/tuple_range_iter.go @@ -55,7 +55,7 @@ func (it mutableMapIter[K, V, O]) Next(ctx context.Context) (key K, value V, err return nil, nil, io.EOF } - cmp := it.compareKeys(pk, mk) + cmp := it.compareKeys(ctx, pk, mk) switch { case cmp < 0: key, value = pk, pv @@ -98,32 +98,32 @@ func (it mutableMapIter[K, V, O]) currentKeys() (memKey, proKey K) { return } -func (it mutableMapIter[K, V, O]) compareKeys(memKey, proKey K) int { +func (it mutableMapIter[K, V, O]) compareKeys(ctx context.Context, memKey, proKey K) int { if memKey == nil { return 1 } if proKey == nil { return -1 } - return it.order.Compare(memKey, proKey) + return it.order.Compare(ctx, memKey, proKey) } -func memIterFromRange(list *skip.List, rng Range) *memRangeIter { +func memIterFromRange(ctx context.Context, list *skip.List, rng Range) *memRangeIter { // use the lower bound of |rng| to construct a skip.ListIter - iter := list.GetIterFromSeekFn(skipSearchFromRange(rng)) + iter := list.GetIterFromSeekFn(skipSearchFromRange(ctx, rng)) // enforce range start var key val.Tuple for { key, _ = iter.Current() - if key == nil || rng.aboveStart(key) { + if key == nil || rng.aboveStart(ctx, key) { break // |i| inside |rng| } iter.Advance() } // enforce range end - if key == nil || !rng.belowStop(key) { + if key == nil || !rng.belowStop(ctx, key) { iter = nil } @@ -137,12 +137,12 @@ func memIterFromRange(list *skip.List, rng Range) *memRangeIter { // a skip.List iterator for a given Range. The skip.SearchFn // returns true if the iter being initialized is not yet // within the bounds of Range |rng|. -func skipSearchFromRange(rng Range) skip.SeekFn { +func skipSearchFromRange(ctx context.Context, rng Range) skip.SeekFn { return func(nodeKey []byte) bool { if nodeKey == nil { return false } - return !rng.aboveStart(nodeKey) + return !rng.aboveStart(ctx, nodeKey) } } @@ -166,12 +166,12 @@ func (it *memRangeIter) Current() (key, value val.Tuple) { } // Iterate progresses the iter inside its range. -func (it *memRangeIter) Iterate(context.Context) (err error) { +func (it *memRangeIter) Iterate(ctx context.Context) (err error) { for { it.iter.Advance() k, _ := it.Current() - if k == nil || !it.rng.belowStop(k) { + if k == nil || !it.rng.belowStop(ctx, k) { it.iter = nil // range exhausted } @@ -202,7 +202,7 @@ func (f filteredIter) Next(ctx context.Context) (k, v val.Tuple, err error) { if err != nil { return nil, nil, err } - if !f.rng.Matches(k) { + if !f.rng.Matches(ctx, k) { continue } return diff --git a/go/store/prolly/tuple_range_iter_test.go b/go/store/prolly/tuple_range_iter_test.go index 70467009dce..928b2b8f60f 100644 --- a/go/store/prolly/tuple_range_iter_test.go +++ b/go/store/prolly/tuple_range_iter_test.go @@ -52,22 +52,22 @@ func testIterRange(t *testing.T, om testMap, tuples [][2]val.Tuple) { // two-sided ranges { name: "OpenRange", - testRange: openRange(start, stop, desc), + testRange: openRange(ctx, start, stop, desc), expCount: nonNegative((z - a) - 1), }, { name: "OpenStartRange", - testRange: openStartRange(start, stop, desc), + testRange: openStartRange(ctx, start, stop, desc), expCount: z - a, }, { name: "OpenStopRange", - testRange: openStopRange(start, stop, desc), + testRange: openStopRange(ctx, start, stop, desc), expCount: z - a, }, { name: "closedRange", - testRange: closedRange(start, stop, desc), + testRange: closedRange(ctx, start, stop, desc), expCount: (z - a) + 1, }, @@ -109,7 +109,7 @@ func testIterRange(t *testing.T, om testMap, tuples [][2]val.Tuple) { key, _, err = iter.Next(ctx) if key != nil { - assert.True(t, desc.Compare(prev, key) < 0) + assert.True(t, desc.Compare(ctx, prev, key) < 0) } } assert.Equal(t, io.EOF, err) @@ -151,19 +151,19 @@ func testIterPrefixRange(t *testing.T, om testMap, tuples [][2]val.Tuple) { // two-sided ranges { name: "OpenRange", - testRange: openRange(start, stop, prefixDesc), + testRange: openRange(ctx, start, stop, prefixDesc), }, { name: "OpenStartRange", - testRange: openStartRange(start, stop, prefixDesc), + testRange: openStartRange(ctx, start, stop, prefixDesc), }, { name: "OpenStopRange", - testRange: openStopRange(start, stop, prefixDesc), + testRange: openStopRange(ctx, start, stop, prefixDesc), }, { name: "closedRange", - testRange: closedRange(start, stop, prefixDesc), + testRange: closedRange(ctx, start, stop, prefixDesc), }, // one-sided ranges @@ -197,7 +197,7 @@ func testIterPrefixRange(t *testing.T, om testMap, tuples [][2]val.Tuple) { key, _, err = iter.Next(ctx) if key != nil { - assert.True(t, prefixDesc.Compare(prev, key) < 0) + assert.True(t, prefixDesc.Compare(ctx, prev, key) < 0) } } assert.Equal(t, io.EOF, err) @@ -224,7 +224,7 @@ func getKeyPrefix(key val.Tuple, desc val.TupleDesc) (partial val.Tuple) { func getExpectedRangeSize(rng Range, tuples [][2]val.Tuple) (sz int) { for i := range tuples { k := tuples[i][0] - if rng.aboveStart(k) && rng.belowStop(k) { + if rng.aboveStart(ctx, k) && rng.belowStop(ctx, k) { sz++ } } @@ -277,25 +277,25 @@ func TestMapIterRange(t *testing.T) { // partial-key range scan { name: "range [1:4]", - rng: closedRange(intTuple(1), intTuple(4), partialDesc), + rng: closedRange(ctx, intTuple(1), intTuple(4), partialDesc), physical: [2]int{0, 24}, logical: []int{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22}, }, { name: "range (1:4]", - rng: openStartRange(intTuple(1), intTuple(4), partialDesc), + rng: openStartRange(ctx, intTuple(1), intTuple(4), partialDesc), physical: [2]int{6, 24}, logical: []int{6, 8, 10, 12, 14, 16, 18, 20, 22}, }, { name: "range [1:4)", - rng: openStopRange(intTuple(1), intTuple(4), partialDesc), + rng: openStopRange(ctx, intTuple(1), intTuple(4), partialDesc), physical: [2]int{0, 18}, logical: []int{0, 2, 4, 6, 8, 10, 12, 14, 16}, }, { name: "range (1:4)", - rng: openRange(intTuple(1), intTuple(4), partialDesc), + rng: openRange(ctx, intTuple(1), intTuple(4), partialDesc), physical: [2]int{6, 18}, logical: []int{6, 8, 10, 12, 14, 16}, }, @@ -303,55 +303,55 @@ func TestMapIterRange(t *testing.T) { // full-key range scan { name: "range (1,1:4,3)", - rng: openRange(intTuple(1, 1), intTuple(4, 3), fullDesc), + rng: openRange(ctx, intTuple(1, 1), intTuple(4, 3), fullDesc), physical: [2]int{0, 24}, logical: []int{2, 8, 14, 20}, }, { name: "range (1,1:4,3]", - rng: openStartRange(intTuple(1, 1), intTuple(4, 3), fullDesc), + rng: openStartRange(ctx, intTuple(1, 1), intTuple(4, 3), fullDesc), physical: [2]int{0, 24}, logical: []int{2, 4, 8, 10, 14, 16, 20, 22}, }, { name: "range [1,1:4,3)", - rng: openStopRange(intTuple(1, 1), intTuple(4, 3), fullDesc), + rng: openStopRange(ctx, intTuple(1, 1), intTuple(4, 3), fullDesc), physical: [2]int{0, 24}, logical: []int{0, 2, 6, 8, 12, 14, 18, 20}, }, { name: "range [1,1:4,3]", - rng: closedRange(intTuple(1, 1), intTuple(4, 3), fullDesc), + rng: closedRange(ctx, intTuple(1, 1), intTuple(4, 3), fullDesc), physical: [2]int{0, 24}, logical: []int{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22}, }, { name: "range [1,2:4,2]", - rng: closedRange(intTuple(1, 2), intTuple(4, 2), fullDesc), + rng: closedRange(ctx, intTuple(1, 2), intTuple(4, 2), fullDesc), physical: [2]int{0, 24}, logical: []int{2, 8, 14, 20}, }, { name: "range (1,2:4,2]", - rng: openStartRange(intTuple(1, 2), intTuple(4, 2), fullDesc), + rng: openStartRange(ctx, intTuple(1, 2), intTuple(4, 2), fullDesc), physical: [2]int{0, 24}, logical: []int{}, }, { name: "range [2,2:3,2]", - rng: closedRange(intTuple(2, 2), intTuple(3, 2), fullDesc), + rng: closedRange(ctx, intTuple(2, 2), intTuple(3, 2), fullDesc), physical: [2]int{6, 18}, logical: []int{8, 14}, }, { name: "range [2,2:2,3]", - rng: closedRange(intTuple(2, 2), intTuple(2, 3), fullDesc), + rng: closedRange(ctx, intTuple(2, 2), intTuple(2, 3), fullDesc), physical: [2]int{8, 12}, logical: []int{8, 10}, }, { name: "range [2,2:2,2]", - rng: closedRange(intTuple(2, 2), intTuple(2, 2), fullDesc), + rng: closedRange(ctx, intTuple(2, 2), intTuple(2, 2), fullDesc), physical: [2]int{8, 10}, logical: []int{8}, }, @@ -499,7 +499,7 @@ func testIterKeyRange(t *testing.T, m Map, tuples [][2]val.Tuple) { t.Run("RandomKeyRange", func(t *testing.T) { bounds := generateInserts(t, m, m.keyDesc, m.valDesc, 2) start, stop := bounds[0][0], bounds[1][0] - if m.keyDesc.Compare(start, stop) > 0 { + if m.keyDesc.Compare(ctx, start, stop) > 0 { start, stop = stop, start } kR := keyRange{kd: m.keyDesc, start: start, stop: stop} diff --git a/go/store/skip/list.go b/go/store/skip/list.go index d9303a9443e..6f4e1991a7d 100644 --- a/go/store/skip/list.go +++ b/go/store/skip/list.go @@ -15,6 +15,7 @@ package skip import ( + "context" "hash/maphash" "math" ) @@ -27,7 +28,7 @@ const ( ) // A KeyOrder determines the ordering of two keys |l| and |r|. -type KeyOrder func(l, r []byte) (cmp int) +type KeyOrder func(ctx context.Context, l, r []byte) (cmp int) // A SeekFn facilitates seeking into a List. It returns true // if the seek operation should advance past |key|. @@ -99,12 +100,12 @@ func (l *List) HasCheckpoint() bool { } // Revert reverts to the last recorded checkpoint. -func (l *List) Revert() { +func (l *List) Revert(ctx context.Context) { cp := l.checkpoint keepers := l.nodes[1:cp] l.Truncate() for _, nd := range keepers { - l.Put(nd.key, nd.val) + l.Put(ctx, nd.key, nd.val) } l.checkpoint = cp } @@ -126,21 +127,21 @@ func (l *List) Count() int { } // Has returns true if |key| is a member of the list. -func (l *List) Has(key []byte) (ok bool) { - _, ok = l.Get(key) +func (l *List) Has(ctx context.Context, key []byte) (ok bool) { + _, ok = l.Get(ctx, key) return } // Get returns the value associated with |key| and true // if |key| is a member of the list, otherwise it returns // nil and false. -func (l *List) Get(key []byte) (val []byte, ok bool) { +func (l *List) Get(ctx context.Context, key []byte) (val []byte, ok bool) { var id nodeId next, prev := l.headTower(), sentinelId for lvl := maxHeight; lvl >= 0; { nd := l.nodePtr(next[lvl]) // descend if we can't advance at |lvl| - if l.compareKeys(key, nd.key) < 0 { + if l.compareKeys(ctx, key, nd.key) < 0 { id = prev lvl-- continue @@ -150,14 +151,14 @@ func (l *List) Get(key []byte) (val []byte, ok bool) { prev = nd.id } node := l.nodePtr(id) - if l.compareKeys(key, node.key) == 0 { + if l.compareKeys(ctx, key, node.key) == 0 { val, ok = node.val, true } return } // Put adds |key| and |values| to the list. -func (l *List) Put(key, val []byte) { +func (l *List) Put(ctx context.Context, key, val []byte) { if key == nil { panic("key must be non-nil") } else if len(l.nodes) >= maxCount { @@ -171,7 +172,7 @@ func (l *List) Put(key, val []byte) { for h := maxHeight; h >= 0; { curr := l.nodePtr(next[h]) // descend if we can't advance at |lvl| - if l.compareKeys(key, curr.key) <= 0 { + if l.compareKeys(ctx, key, curr.key) <= 0 { path[h] = prev h-- continue @@ -185,7 +186,7 @@ func (l *List) Put(key, val []byte) { node := l.nodePtr(path[0]) node = l.nodePtr(node.next[0]) - if l.compareKeys(key, node.key) == 0 { + if l.compareKeys(ctx, key, node.key) == 0 { l.overwrite(key, val, &path, node) } else { l.insert(key, val, &path) @@ -270,9 +271,9 @@ func (it *ListIter) Retreat() { // GetIterAt creates an iterator starting at the first item // of the list whose key is greater than or equal to |key|. -func (l *List) GetIterAt(key []byte) (it *ListIter) { +func (l *List) GetIterAt(ctx context.Context, key []byte) (it *ListIter) { return l.GetIterFromSeekFn(func(nodeKey []byte) bool { - return l.compareKeys(key, nodeKey) > 0 + return l.compareKeys(ctx, key, nodeKey) > 0 }) } @@ -307,9 +308,9 @@ func (l *List) IterAtEnd() *ListIter { } // seek returns the skipNode with the smallest key >= |key|. -func (l *List) seek(key []byte) *skipNode { +func (l *List) seek(ctx context.Context, key []byte) *skipNode { return l.seekWithFn(func(curr []byte) (advance bool) { - return l.compareKeys(key, curr) > 0 + return l.compareKeys(ctx, key, curr) > 0 }) } @@ -346,11 +347,11 @@ func (l *List) nextNodeId() nodeId { return nodeId(len(l.nodes)) } -func (l *List) compareKeys(left, right []byte) int { +func (l *List) compareKeys(ctx context.Context, left, right []byte) int { if right == nil { return -1 // |right| is sentinel key } - return l.keyOrder(left, right) + return l.keyOrder(ctx, left, right) } var ( diff --git a/go/store/skip/list_bench_test.go b/go/store/skip/list_bench_test.go index 18517dd4ab8..5bc99614320 100644 --- a/go/store/skip/list_bench_test.go +++ b/go/store/skip/list_bench_test.go @@ -131,11 +131,11 @@ func BenchmarkIterAll(b *testing.B) { func benchmarkGet(b *testing.B, vals [][]byte) { l := NewSkipList(bytes.Compare) for i := range vals { - l.Put(vals[i], vals[i]) + l.Put(ctx, vals[i], vals[i]) } b.ResetTimer() for i := 0; i < b.N; i++ { - _, ok := l.Get(vals[i%len(vals)]) + _, ok := l.Get(ctx, vals[i%len(vals)]) if !ok { b.Fail() } @@ -150,7 +150,7 @@ func benchmarkPut(b *testing.B, vals [][]byte) { if j == 0 { l.Truncate() } - l.Put(vals[j], vals[j]) + l.Put(ctx, vals[j], vals[j]) } b.ReportAllocs() } @@ -158,7 +158,7 @@ func benchmarkPut(b *testing.B, vals [][]byte) { func benchmarkIterAll(b *testing.B, vals [][]byte) { l := NewSkipList(bytes.Compare) for i := range vals { - l.Put(vals[i], vals[i]) + l.Put(ctx, vals[i], vals[i]) } b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/go/store/skip/list_test.go b/go/store/skip/list_test.go index 6c9e2e510d9..654da2a9b6f 100644 --- a/go/store/skip/list_test.go +++ b/go/store/skip/list_test.go @@ -96,7 +96,7 @@ func testSkipList(t *testing.T, compare KeyOrder, vals ...[]byte) { t.Run("test puts", func(t *testing.T) { // |list| is populated for _, v := range vals { - list.Put(v, v) + list.Put(ctx, v, v) } testSkipListPuts(t, list, vals...) }) @@ -130,13 +130,13 @@ func testSkipListGets(t *testing.T, list *List, vals ...[]byte) { }) for _, exp := range vals { - act, ok := list.Get(exp) + act, ok := list.Get(ctx, exp) assert.True(t, ok) assert.Equal(t, exp, act) } // test absent key - act, ok := list.Get(b("12345678")) + act, ok := list.Get(ctx, b("12345678")) assert.False(t, ok) assert.Nil(t, act) } @@ -144,7 +144,7 @@ func testSkipListGets(t *testing.T, list *List, vals ...[]byte) { func testSkipListUpdates(t *testing.T, list *List, vals ...[]byte) { v2 := []byte("789") for _, v := range vals { - list.Put(v, v2) + list.Put(ctx, v, v2) } assert.Equal(t, len(vals), list.Count()) @@ -152,7 +152,7 @@ func testSkipListUpdates(t *testing.T, list *List, vals ...[]byte) { vals[i], vals[j] = vals[j], vals[i] }) for _, exp := range vals { - act, ok := list.Get(exp) + act, ok := list.Get(ctx, exp) assert.True(t, ok) assert.Equal(t, v2, act) } @@ -164,7 +164,7 @@ func testSkipListUpdates(t *testing.T, list *List, vals ...[]byte) { func testSkipListIterForward(t *testing.T, list *List, vals ...[]byte) { // put |vals| back in keyOrder sort.Slice(vals, func(i, j int) bool { - return list.compareKeys(vals[i], vals[j]) < 0 + return list.compareKeys(ctx, vals[i], vals[j]) < 0 }) idx := 0 @@ -193,7 +193,7 @@ func testSkipListIterForward(t *testing.T, list *List, vals ...[]byte) { func testSkipListIterBackward(t *testing.T, list *List, vals ...[]byte) { // put |vals| back in keyOrder sort.Slice(vals, func(i, j int) bool { - return list.compareKeys(vals[i], vals[j]) < 0 + return list.compareKeys(ctx, vals[i], vals[j]) < 0 }) // test iter at @@ -217,10 +217,10 @@ func testSkipListTruncate(t *testing.T, list *List, vals ...[]byte) { assert.Equal(t, list.Count(), 0) for i := range vals { - assert.False(t, list.Has(vals[i])) + assert.False(t, list.Has(ctx, vals[i])) } for i := range vals { - v, ok := list.Get(vals[i]) + v, ok := list.Get(ctx, vals[i]) assert.False(t, ok) assert.Nil(t, v) } @@ -240,24 +240,24 @@ func testSkipListTruncate(t *testing.T, list *List, vals ...[]byte) { validateIter(list.IterAtStart()) validateIter(list.IterAtEnd()) - validateIter(list.GetIterAt(vals[0])) + validateIter(list.GetIterAt(ctx, vals[0])) } func validateIterForwardFrom(t *testing.T, l *List, key []byte) (count int) { - iter := l.GetIterAt(key) + iter := l.GetIterAt(ctx, key) k, _ := iter.Current() for k != nil { count++ iter.Advance() prev := k k, _ = iter.Current() - assert.True(t, l.compareKeys(prev, k) < 0) + assert.True(t, l.compareKeys(ctx, prev, k) < 0) } return } func validateIterBackwardFrom(t *testing.T, l *List, key []byte) (count int) { - iter := l.GetIterAt(key) + iter := l.GetIterAt(ctx, key) k, _ := iter.Current() for k != nil { count++ @@ -266,7 +266,7 @@ func validateIterBackwardFrom(t *testing.T, l *List, key []byte) (count int) { k, _ = iter.Current() if k != nil { - assert.True(t, l.compareKeys(prev, k) > 0) + assert.True(t, l.compareKeys(ctx, prev, k) > 0) } } return @@ -331,55 +331,55 @@ func testSkipListCheckpoints(t *testing.T, compare KeyOrder, data ...[]byte) { list := NewSkipList(compare) // test empty revert - list.Revert() + list.Revert(ctx) for _, v := range init { - list.Put(v, v) + list.Put(ctx, v, v) } for _, v := range init { - act, ok := list.Get(v) + act, ok := list.Get(ctx, v) assert.True(t, ok) assert.Equal(t, v, act) } for _, v := range inserts { - assert.False(t, list.Has(v)) + assert.False(t, list.Has(ctx, v)) } list.Checkpoint() up := []byte("update") for _, v := range updates { - list.Put(v, up) + list.Put(ctx, v, up) } for _, v := range inserts { - list.Put(v, v) + list.Put(ctx, v, v) } for _, v := range static { - act, ok := list.Get(v) + act, ok := list.Get(ctx, v) assert.True(t, ok) assert.Equal(t, v, act) } for _, v := range inserts { - act, ok := list.Get(v) + act, ok := list.Get(ctx, v) assert.True(t, ok) assert.Equal(t, v, act) } for _, v := range updates { - act, ok := list.Get(v) + act, ok := list.Get(ctx, v) assert.True(t, ok) assert.Equal(t, up, act) } - list.Revert() + list.Revert(ctx) for _, v := range init { - act, ok := list.Get(v) + act, ok := list.Get(ctx, v) assert.True(t, ok) assert.Equal(t, v, act) } for _, v := range inserts { - assert.False(t, list.Has(v)) + assert.False(t, list.Has(ctx, v)) } } diff --git a/go/store/val/codec.go b/go/store/val/codec.go index 36b23366bce..5bd3679bd0b 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -16,6 +16,7 @@ package val import ( "bytes" + "context" "encoding/binary" "math" "math/big" @@ -595,8 +596,8 @@ func compareByteString(l, r []byte) int { return bytes.Compare(l, r) } -func readExtended(handler TupleTypeHandler, val []byte) any { - v, err := handler.DeserializeValue(val) +func readExtended(ctx context.Context, handler TupleTypeHandler, val []byte) any { + v, err := handler.DeserializeValue(ctx, val) if err != nil { panic(err) } diff --git a/go/store/val/extended_comparator.go b/go/store/val/extended_comparator.go index fae9c37514e..c8006fd76d5 100644 --- a/go/store/val/extended_comparator.go +++ b/go/store/val/extended_comparator.go @@ -14,6 +14,8 @@ package val +import "context" + // ExtendedTupleComparator is a comparator that properly handles extended types. type ExtendedTupleComparator struct { innerCmp TupleComparator @@ -24,11 +26,11 @@ type ExtendedTupleComparator struct { var _ TupleComparator = ExtendedTupleComparator{} // Compare implements the TupleComparator interface. -func (c ExtendedTupleComparator) Compare(left, right Tuple, desc TupleDesc) (cmp int) { +func (c ExtendedTupleComparator) Compare(ctx context.Context, left, right Tuple, desc TupleDesc) (cmp int) { fast := desc.GetFixedAccess() for i := range fast { start, stop := fast[i][0], fast[i][1] - cmp = c.CompareValues(i, left[start:stop], right[start:stop], desc.Types[i]) + cmp = c.CompareValues(ctx, i, left[start:stop], right[start:stop], desc.Types[i]) if cmp != 0 { return cmp } @@ -37,7 +39,7 @@ func (c ExtendedTupleComparator) Compare(left, right Tuple, desc TupleDesc) (cmp off := len(fast) for i, typ := range desc.Types[off:] { j := i + off - cmp = c.CompareValues(j, left.GetField(j), right.GetField(j), typ) + cmp = c.CompareValues(ctx, j, left.GetField(j), right.GetField(j), typ) if cmp != 0 { return cmp } @@ -46,10 +48,10 @@ func (c ExtendedTupleComparator) Compare(left, right Tuple, desc TupleDesc) (cmp } // CompareValues implements the TupleComparator interface. -func (c ExtendedTupleComparator) CompareValues(index int, left, right []byte, typ Type) int { +func (c ExtendedTupleComparator) CompareValues(ctx context.Context, index int, left, right []byte, typ Type) int { switch typ.Enc { case ExtendedEnc, ExtendedAddrEnc: - cmp, err := c.handlers[index].SerializedCompare(left, right) + cmp, err := c.handlers[index].SerializedCompare(ctx, left, right) if err != nil { panic(err) } diff --git a/go/store/val/keyless_tuple.go b/go/store/val/keyless_tuple.go index 13cff70562a..35514301c02 100644 --- a/go/store/val/keyless_tuple.go +++ b/go/store/val/keyless_tuple.go @@ -16,6 +16,7 @@ package val import ( "bytes" + "context" "encoding/binary" "github.com/zeebo/xxh3" @@ -79,12 +80,12 @@ type keylessCompare struct{} var _ TupleComparator = keylessCompare{} // Compare implements TupleComparator -func (k keylessCompare) Compare(left, right Tuple, _ TupleDesc) int { +func (k keylessCompare) Compare(ctx context.Context, left, right Tuple, desc TupleDesc) int { return bytes.Compare(left, right) } // CompareValues implements TupleComparator -func (k keylessCompare) CompareValues(_ int, left, right []byte, typ Type) int { +func (k keylessCompare) CompareValues(ctx context.Context, index int, left, right []byte, typ Type) int { return compare(typ, left, right) } diff --git a/go/store/val/tuple_builder_test.go b/go/store/val/tuple_builder_test.go index bc8d147b371..c69d03894f2 100644 --- a/go/store/val/tuple_builder_test.go +++ b/go/store/val/tuple_builder_test.go @@ -15,6 +15,7 @@ package val import ( + "context" "math" "math/rand" "testing" @@ -205,7 +206,7 @@ type testCompare struct{} var _ TupleComparator = testCompare{} -func (tc testCompare) Compare(left, right Tuple, desc TupleDesc) (cmp int) { +func (tc testCompare) Compare(ctx context.Context, left, right Tuple, desc TupleDesc) (cmp int) { for i, typ := range desc.Types { cmp = compare(typ, left.GetField(i), right.GetField(i)) if cmp != 0 { @@ -215,7 +216,7 @@ func (tc testCompare) Compare(left, right Tuple, desc TupleDesc) (cmp int) { return } -func (tc testCompare) CompareValues(_ int, left, right []byte, typ Type) int { +func (tc testCompare) CompareValues(ctx context.Context, index int, left, right []byte, typ Type) int { return compare(typ, left, right) } diff --git a/go/store/val/tuple_compare.go b/go/store/val/tuple_compare.go index 82f1d5bdefb..ef195c4459e 100644 --- a/go/store/val/tuple_compare.go +++ b/go/store/val/tuple_compare.go @@ -14,15 +14,18 @@ package val -import "bytes" +import ( + "bytes" + "context" +) // TupleComparator compares Tuples. type TupleComparator interface { // Compare compares pairs of Tuples. - Compare(left, right Tuple, desc TupleDesc) int + Compare(ctx context.Context, left, right Tuple, desc TupleDesc) int // CompareValues compares pairs of values. The index should match the index used to retrieve the type. - CompareValues(index int, left, right []byte, typ Type) int + CompareValues(ctx context.Context, index int, left, right []byte, typ Type) int // Prefix returns a TupleComparator for the first n types. Prefix(n int) TupleComparator @@ -40,7 +43,7 @@ type DefaultTupleComparator struct{} var _ TupleComparator = DefaultTupleComparator{} // Compare implements TupleComparator -func (d DefaultTupleComparator) Compare(left, right Tuple, desc TupleDesc) (cmp int) { +func (d DefaultTupleComparator) Compare(ctx context.Context, left, right Tuple, desc TupleDesc) (cmp int) { for i := range desc.fast { start, stop := desc.fast[i][0], desc.fast[i][1] cmp = compare(desc.Types[i], left[start:stop], right[start:stop]) @@ -61,7 +64,7 @@ func (d DefaultTupleComparator) Compare(left, right Tuple, desc TupleDesc) (cmp } // CompareValues implements TupleComparator -func (d DefaultTupleComparator) CompareValues(_ int, left, right []byte, typ Type) (cmp int) { +func (d DefaultTupleComparator) CompareValues(ctx context.Context, index int, left, right []byte, typ Type) (cmp int) { return compare(typ, left, right) } diff --git a/go/store/val/tuple_descriptor.go b/go/store/val/tuple_descriptor.go index 92fe02ee1cd..8a92983f328 100644 --- a/go/store/val/tuple_descriptor.go +++ b/go/store/val/tuple_descriptor.go @@ -157,12 +157,12 @@ func (td TupleDesc) GetField(i int, tup Tuple) []byte { } // Compare compares |left| and |right|. -func (td TupleDesc) Compare(left, right Tuple) (cmp int) { - return td.cmp.Compare(left, right, td) +func (td TupleDesc) Compare(ctx context.Context, left, right Tuple) (cmp int) { + return td.cmp.Compare(ctx, left, right, td) } // CompareField compares |value| with the ith field of |tup|. -func (td TupleDesc) CompareField(value []byte, i int, tup Tuple) (cmp int) { +func (td TupleDesc) CompareField(ctx context.Context, value []byte, i int, tup Tuple) (cmp int) { var v []byte if i < len(td.fast) { start, stop := td.fast[i][0], td.fast[i][1] @@ -170,7 +170,7 @@ func (td TupleDesc) CompareField(value []byte, i int, tup Tuple) (cmp int) { } else { v = tup.GetField(i) } - return td.cmp.CompareValues(i, value, v, td.Types[i]) + return td.cmp.CompareValues(ctx, i, value, v, td.Types[i]) } // Comparator returns the TupleDescriptor's TupleComparator. @@ -546,7 +546,7 @@ func (td TupleDesc) GetCell(i int, tup Tuple) (v Cell, ok bool) { } // Format prints a Tuple as a string. -func (td TupleDesc) Format(tup Tuple) string { +func (td TupleDesc) Format(ctx context.Context, tup Tuple) string { if tup == nil || tup.Count() == 0 { return "( )" } @@ -560,20 +560,20 @@ func (td TupleDesc) Format(tup Tuple) string { sb.WriteString(", ") } seenOne = true - sb.WriteString(td.FormatValue(i, tup.GetField(i))) + sb.WriteString(td.FormatValue(ctx, i, tup.GetField(i))) } sb.WriteString(" )") return sb.String() } -func (td TupleDesc) FormatValue(i int, value []byte) string { +func (td TupleDesc) FormatValue(ctx context.Context, i int, value []byte) string { if value == nil { return "NULL" } - return td.formatValue(td.Types[i].Enc, i, value) + return td.formatValue(ctx, td.Types[i].Enc, i, value) } -func (td TupleDesc) formatValue(enc Encoding, i int, value []byte) string { +func (td TupleDesc) formatValue(ctx context.Context, enc Encoding, i int, value []byte) string { switch enc { case Int8Enc: v := readInt8(value) @@ -645,7 +645,7 @@ func (td TupleDesc) formatValue(enc Encoding, i int, value []byte) string { return hex.EncodeToString(value) case ExtendedEnc: handler := td.Handlers[i] - v := readExtended(handler, value) + v := readExtended(ctx, handler, value) str, err := handler.FormatValue(v) if err != nil { panic(err) diff --git a/go/store/valuefile/value_file_test.go b/go/store/valuefile/value_file_test.go index 2d410ad0800..b3911b98226 100644 --- a/go/store/valuefile/value_file_test.go +++ b/go/store/valuefile/value_file_test.go @@ -132,7 +132,7 @@ var vd = val.NewTupleDescriptor( ) func makeProllyMap(t *testing.T, ns tree.NodeStore, count int) (prolly.Map, [][2]val.Tuple) { - tuples := tree.RandomTuplePairs(count, kd, vd, ns) + tuples := tree.RandomTuplePairs(ctx, count, kd, vd, ns) om := mustProllyMapFromTuples(t, kd, vd, ns, tuples) for i := 0; i < len(tuples); i++ { From 4a61174d60d72a010f7bda9c16f7be70200528a5 Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Tue, 18 Feb 2025 16:02:21 -0800 Subject: [PATCH 06/11] Add context parameters to test functions. --- go/libraries/doltcore/doltdb/ignore.go | 8 +++--- go/libraries/doltcore/schema/schema_impl.go | 3 ++- .../doltcore/table/table_iterator_test.go | 1 + .../benchmark/benchmark_batch_writes_test.go | 7 ++++- go/store/prolly/map_diff_key_range_test.go | 2 ++ go/store/prolly/map_diff_test.go | 1 + go/store/prolly/map_merge_test.go | 1 + go/store/prolly/map_range_diff_test.go | 2 ++ go/store/prolly/map_test.go | 8 ++++-- go/store/prolly/mutable_map_read_test.go | 2 +- go/store/prolly/sort/external_test.go | 5 +++- go/store/prolly/tree/node_cursor_test.go | 1 + go/store/prolly/tuple_range_iter_test.go | 1 + go/store/prolly/tuple_range_test.go | 4 +-- go/store/skip/list_bench_test.go | 10 ++++--- go/store/skip/list_test.go | 26 ++++++++++++++----- go/store/val/codec_test.go | 5 ++-- go/store/valuefile/value_file_test.go | 1 + 18 files changed, 65 insertions(+), 23 deletions(-) diff --git a/go/libraries/doltcore/doltdb/ignore.go b/go/libraries/doltcore/doltdb/ignore.go index 95189d66c9a..a51478bf151 100644 --- a/go/libraries/doltcore/doltdb/ignore.go +++ b/go/libraries/doltcore/doltdb/ignore.go @@ -61,7 +61,7 @@ var ConvertTupleToIgnoreBoolean = convertTupleToIgnoreBoolean // GetIgnoreTablePatternKey is a function that converts a Tuple to a string for the pattern field. This is used to handle the Doltgres extended string type. var GetIgnoreTablePatternKey = getIgnoreTablePatternKey -func convertTupleToIgnoreBoolean(valueDesc val.TupleDesc, valueTuple val.Tuple) (bool, error) { +func convertTupleToIgnoreBoolean(ctx context.Context, valueDesc val.TupleDesc, valueTuple val.Tuple) (bool, error) { if !valueDesc.Equals(val.NewTupleDescriptor(val.Type{Enc: val.Int8Enc, Nullable: false})) { return false, fmt.Errorf("dolt_ignore had unexpected value type, this should never happen") } @@ -72,7 +72,7 @@ func convertTupleToIgnoreBoolean(valueDesc val.TupleDesc, valueTuple val.Tuple) return ignore, nil } -func getIgnoreTablePatternKey(keyDesc val.TupleDesc, keyTuple val.Tuple, _ tree.NodeStore) (string, error) { +func getIgnoreTablePatternKey(ctx context.Context, keyDesc val.TupleDesc, keyTuple val.Tuple, _ tree.NodeStore) (string, error) { if !keyDesc.Equals(val.NewTupleDescriptor(val.Type{Enc: val.StringEnc, Nullable: false})) { return "", fmt.Errorf("dolt_ignore had unexpected key type, this should never happen") } @@ -129,12 +129,12 @@ func GetIgnoredTablePatterns(ctx context.Context, roots Roots, schemas []string) m := durable.MapFromIndex(index) - pattern, err := GetIgnoreTablePatternKey(keyDesc, keyTuple, m.NodeStore()) + pattern, err := GetIgnoreTablePatternKey(ctx, keyDesc, keyTuple, m.NodeStore()) if err != nil { return nil, err } - ignore, err := ConvertTupleToIgnoreBoolean(valueDesc, valueTuple) + ignore, err := ConvertTupleToIgnoreBoolean(ctx, valueDesc, valueTuple) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/schema/schema_impl.go b/go/libraries/doltcore/schema/schema_impl.go index 84ed53690ef..8fce2f9e185 100644 --- a/go/libraries/doltcore/schema/schema_impl.go +++ b/go/libraries/doltcore/schema/schema_impl.go @@ -595,8 +595,9 @@ func (si *schemaImpl) GetValueDescriptor(vs val.ValueStore) val.TupleDesc { if extendedType, ok := sqlType.(gmstypes.ExtendedType); ok { if encoding == serial.EncodingExtendedAddr { handlers = append(handlers, val.NewExtendedAddressTypeHandler(vs, extendedType)) + } else { + handlers = append(handlers, extendedType) } - handlers = append(handlers, extendedType) } else { handlers = append(handlers, nil) } diff --git a/go/libraries/doltcore/table/table_iterator_test.go b/go/libraries/doltcore/table/table_iterator_test.go index 036897cc618..47ec688aaf8 100644 --- a/go/libraries/doltcore/table/table_iterator_test.go +++ b/go/libraries/doltcore/table/table_iterator_test.go @@ -68,6 +68,7 @@ var vd = val.NewTupleDescriptor( ) func mustMakeProllyMap(t *testing.T, count int) (prolly.Map, [][2]val.Tuple) { + ctx := context.Background() ns := tree.NewTestNodeStore() diff --git a/go/store/prolly/benchmark/benchmark_batch_writes_test.go b/go/store/prolly/benchmark/benchmark_batch_writes_test.go index 24f8b3a2c55..090ba42421d 100644 --- a/go/store/prolly/benchmark/benchmark_batch_writes_test.go +++ b/go/store/prolly/benchmark/benchmark_batch_writes_test.go @@ -43,6 +43,10 @@ var ( bucket = []byte("bolt") ) +func compareBytes(ctx context.Context, a, b []byte) int { + return bytes.Compare(a, b) +} + func BenchmarkImportBBolt(b *testing.B) { makeWriter := func() writer { path, err := os.MkdirTemp("", "*") @@ -63,7 +67,7 @@ func BenchmarkImportBBolt(b *testing.B) { }) require.NoError(b, err) return &bboltWriter{ - edits: skip.NewSkipList(bytes.Compare), + edits: skip.NewSkipList(compareBytes), db: db, } } @@ -150,6 +154,7 @@ func (wr *doltWriter) Flush() error { } func benchmarkBatchWrite(b *testing.B, wr writer) { + ctx := context.Background() dp := newDataProvider(batch) for i := 0; i < b.N; i++ { k, v := dp.next() diff --git a/go/store/prolly/map_diff_key_range_test.go b/go/store/prolly/map_diff_key_range_test.go index f8bd491753e..25349a7372a 100644 --- a/go/store/prolly/map_diff_key_range_test.go +++ b/go/store/prolly/map_diff_key_range_test.go @@ -289,6 +289,7 @@ func makeRandomUnboundedUpperKeyRange(kd val.TupleDesc, tuples [][2]val.Tuple) k } func makeBoundedKeyRangeWithMissingKeys(t *testing.T, m Map, kd val.TupleDesc, vd val.TupleDesc, tuples [][2]val.Tuple) keyRangeDiffTest { + ctx := context.Background() inserts := generateInserts(t, m, kd, vd, 2) low, hi := inserts[0][0], inserts[1][0] if kd.Compare(ctx, low, hi) > 0 { @@ -316,6 +317,7 @@ type keyRange struct { } func (kR keyRange) includes(k val.Tuple) bool { + ctx := context.Background() if len(kR.start) != 0 && kR.kd.Compare(ctx, k, kR.start) < 0 { return false } diff --git a/go/store/prolly/map_diff_test.go b/go/store/prolly/map_diff_test.go index 3ce4c913d6a..f95f9a87df9 100644 --- a/go/store/prolly/map_diff_test.go +++ b/go/store/prolly/map_diff_test.go @@ -311,6 +311,7 @@ func makeMapWithUpdates(t *testing.T, m Map, updates ...[3]val.Tuple) Map { } func makeUpdatesToTuples(kd, vd val.TupleDesc, tuples ...[2]val.Tuple) (updates [][3]val.Tuple) { + ctx := context.Background() updates = make([][3]val.Tuple, len(tuples)) valBuilder := val.NewTupleBuilder(vd) diff --git a/go/store/prolly/map_merge_test.go b/go/store/prolly/map_merge_test.go index 94f6605ad1f..a8f152af90a 100644 --- a/go/store/prolly/map_merge_test.go +++ b/go/store/prolly/map_merge_test.go @@ -225,6 +225,7 @@ type mutationSet struct { } func makeTuplesAndMutations(kd, vd val.TupleDesc, sz int, ns tree.NodeStore) (base [][2]val.Tuple, left, right mutationSet) { + ctx := context.Background() mutSz := sz / 10 totalSz := sz + (mutSz * 2) tuples := tree.RandomTuplePairs(ctx, totalSz, kd, vd, ns) diff --git a/go/store/prolly/map_range_diff_test.go b/go/store/prolly/map_range_diff_test.go index cb486698e32..76e6bbac0b6 100644 --- a/go/store/prolly/map_range_diff_test.go +++ b/go/store/prolly/map_range_diff_test.go @@ -257,6 +257,7 @@ type rangeDiffTest struct { } func makeRandomOpenStopRangeTest(kd val.TupleDesc, tuples [][2]val.Tuple) rangeDiffTest { + ctx := context.Background() i := rand.Intn(len(tuples)) j := rand.Intn(len(tuples)) if j < i { @@ -281,6 +282,7 @@ func makeRandomLesserRangeTest(kd val.TupleDesc, tuples [][2]val.Tuple) rangeDif } func getPairsInRange(tuples [][2]val.Tuple, rng Range) (keys [][2]val.Tuple) { + ctx := context.Background() for _, pair := range tuples { if rng.Matches(ctx, pair[0]) { keys = append(keys, pair) diff --git a/go/store/prolly/map_test.go b/go/store/prolly/map_test.go index 7ccb93be20c..4df6fcf212c 100644 --- a/go/store/prolly/map_test.go +++ b/go/store/prolly/map_test.go @@ -38,6 +38,7 @@ var testRand = rand.New(rand.NewSource(1)) var sharedPool = pool.NewBuffPool() func TestMap(t *testing.T) { + ctx := context.Background() scales := []int{ 10, 100, @@ -73,7 +74,6 @@ func TestMap(t *testing.T) { testHas(t, pm, tuples) }) - ctx := context.Background() t.Run("walk addresses smoke test", func(t *testing.T) { err := pm.WalkAddresses(ctx, func(_ context.Context, addr hash.Hash) error { assert.True(t, addr != hash.Hash{}) @@ -93,6 +93,7 @@ func TestMap(t *testing.T) { } func TestMutateMapWithTupleIter(t *testing.T) { + ctx := context.Background() kd := val.NewTupleDescriptor( val.Type{Enc: val.Uint32Enc, Nullable: false}, ) @@ -158,7 +159,6 @@ func TestMutateMapWithTupleIter(t *testing.T) { tree.SortTuplePairs(ctx, base, kd) before := mustProllyMapFromTuples(t, kd, vd, base) - ctx := context.Background() ds, err := DebugFormat(ctx, before) assert.NoError(t, err) assert.NotNil(t, ds) @@ -297,6 +297,7 @@ func TestMapGetAllocs(t *testing.T) { } func makeProllyMap(t *testing.T, count int) (testMap, [][2]val.Tuple) { + ctx := context.Background() kd := val.NewTupleDescriptor( val.Type{Enc: val.Uint32Enc, Nullable: false}, ) @@ -314,6 +315,7 @@ func makeProllyMap(t *testing.T, count int) (testMap, [][2]val.Tuple) { } func makeProllySecondaryIndex(t *testing.T, count int) (testMap, [][2]val.Tuple) { + ctx := context.Background() kd := val.NewTupleDescriptor( val.Type{Enc: val.Uint32Enc, Nullable: true}, val.Type{Enc: val.Uint32Enc, Nullable: false}, @@ -432,10 +434,12 @@ func testIterAll(t *testing.T, om testMap, tuples [][2]val.Tuple) { } func pointRangeFromTuple(tup val.Tuple, desc val.TupleDesc) Range { + ctx := context.Background() return closedRange(ctx, tup, tup, desc) } func formatTuples(tuples [][2]val.Tuple, kd, vd val.TupleDesc) string { + ctx := context.Background() var sb strings.Builder sb.WriteString("Tuples (") sb.WriteString(strconv.Itoa(len(tuples))) diff --git a/go/store/prolly/mutable_map_read_test.go b/go/store/prolly/mutable_map_read_test.go index 8e787f46997..5cf648b3ee5 100644 --- a/go/store/prolly/mutable_map_read_test.go +++ b/go/store/prolly/mutable_map_read_test.go @@ -163,6 +163,7 @@ func makeMutableSecondaryIndex(t *testing.T, count int) (testMap, [][2]val.Tuple } func deleteFromMutableMap(mut *MutableMap, tt [][2]val.Tuple) (*MutableMap, [][2]val.Tuple, [][2]val.Tuple) { + ctx := context.Background() count := len(tt) testRand.Shuffle(count, func(i, j int) { tt[i], tt[j] = tt[j], tt[i] @@ -176,7 +177,6 @@ func deleteFromMutableMap(mut *MutableMap, tt [][2]val.Tuple) (*MutableMap, [][2 desc := keyDescFromMap(mut) tree.SortTuplePairs(ctx, remaining, desc) - ctx := context.Background() for _, kv := range deletes { if err := mut.Delete(ctx, kv[0]); err != nil { panic(err) diff --git a/go/store/prolly/sort/external_test.go b/go/store/prolly/sort/external_test.go index 4c5867d08b8..40707fc5719 100644 --- a/go/store/prolly/sort/external_test.go +++ b/go/store/prolly/sort/external_test.go @@ -29,6 +29,7 @@ import ( ) func TestFlush(t *testing.T) { + ctx := context.Background() tests := []struct { td val.TupleDesc cnt int @@ -118,6 +119,7 @@ func TestFlush(t *testing.T) { } func TestMerge(t *testing.T) { + ctx := context.Background() tests := []struct { td val.TupleDesc counts []int @@ -211,7 +213,6 @@ func TestMerge(t *testing.T) { t.Run("file merge", func(t *testing.T) { target := newKeyFile(mustNewFile(t, tmpProv), batchSize) - ctx := context.Background() m, _ := newFileMerger(ctx, keyCmp, target, keyFiles...) m.run(ctx) @@ -224,6 +225,7 @@ func TestMerge(t *testing.T) { } func TestCompact(t *testing.T) { + ctx := context.Background() // run compact until there's only 1 file // check at each iteration that we halved the file count, cnt and size is still the same tests := []struct { @@ -323,6 +325,7 @@ func TestCompact(t *testing.T) { } func TestFileE2E(t *testing.T) { + ctx := context.Background() // simulate full lifecycle // vary batch size and file count so multiple compacts/merges // make the batch size and file size small enough that diff --git a/go/store/prolly/tree/node_cursor_test.go b/go/store/prolly/tree/node_cursor_test.go index 7e8836b9e4b..270dde8a58e 100644 --- a/go/store/prolly/tree/node_cursor_test.go +++ b/go/store/prolly/tree/node_cursor_test.go @@ -164,6 +164,7 @@ var valDesc = val.NewTupleDescriptor( ) func randomTupleItemPairs(count int, ns NodeStore) (items [][2]Item) { + ctx := context.Background() tups := RandomTuplePairs(ctx, count, keyDesc, valDesc, ns) items = make([][2]Item, count) if len(tups) != len(items) { diff --git a/go/store/prolly/tuple_range_iter_test.go b/go/store/prolly/tuple_range_iter_test.go index 928b2b8f60f..3d4edc6eead 100644 --- a/go/store/prolly/tuple_range_iter_test.go +++ b/go/store/prolly/tuple_range_iter_test.go @@ -222,6 +222,7 @@ func getKeyPrefix(key val.Tuple, desc val.TupleDesc) (partial val.Tuple) { // computes expected range on full tuples set func getExpectedRangeSize(rng Range, tuples [][2]val.Tuple) (sz int) { + ctx := context.Background() for i := range tuples { k := tuples[i][0] if rng.aboveStart(ctx, k) && rng.belowStop(ctx, k) { diff --git a/go/store/prolly/tuple_range_test.go b/go/store/prolly/tuple_range_test.go index 070e700c1ba..df95b0d1688 100644 --- a/go/store/prolly/tuple_range_test.go +++ b/go/store/prolly/tuple_range_test.go @@ -172,11 +172,11 @@ func TestRangeSearch(t *testing.T) { lo, hi := test.physical[0], test.physical[1] startSearch := rangeStartSearchFn(rng) - idx := startSearch(testNode) + idx := startSearch(ctx, testNode) assert.Equal(t, lo, idx, "range should start at index %d", lo) stopSearch := rangeStopSearchFn(rng) - idx = stopSearch(testNode) + idx = stopSearch(ctx, testNode) assert.Equal(t, hi, idx, "range should stop before index %d", hi) // validate physical range (unfiltered iter) diff --git a/go/store/skip/list_bench_test.go b/go/store/skip/list_bench_test.go index 5bc99614320..bb0f575f8db 100644 --- a/go/store/skip/list_bench_test.go +++ b/go/store/skip/list_bench_test.go @@ -16,6 +16,7 @@ package skip import ( "bytes" + "context" "sort" "testing" ) @@ -129,7 +130,8 @@ func BenchmarkIterAll(b *testing.B) { } func benchmarkGet(b *testing.B, vals [][]byte) { - l := NewSkipList(bytes.Compare) + ctx := context.Background() + l := NewSkipList(compareBytes) for i := range vals { l.Put(ctx, vals[i], vals[i]) } @@ -144,7 +146,8 @@ func benchmarkGet(b *testing.B, vals [][]byte) { } func benchmarkPut(b *testing.B, vals [][]byte) { - l := NewSkipList(bytes.Compare) + ctx := context.Background() + l := NewSkipList(compareBytes) for i := 0; i < b.N; i++ { j := i % len(vals) if j == 0 { @@ -156,7 +159,8 @@ func benchmarkPut(b *testing.B, vals [][]byte) { } func benchmarkIterAll(b *testing.B, vals [][]byte) { - l := NewSkipList(bytes.Compare) + ctx := context.Background() + l := NewSkipList(compareBytes) for i := range vals { l.Put(ctx, vals[i], vals[i]) } diff --git a/go/store/skip/list_test.go b/go/store/skip/list_test.go index 654da2a9b6f..f06a92fe479 100644 --- a/go/store/skip/list_test.go +++ b/go/store/skip/list_test.go @@ -16,6 +16,7 @@ package skip import ( "bytes" + "context" "encoding/binary" "math/rand" "sort" @@ -27,6 +28,10 @@ import ( var randSrc = rand.New(rand.NewSource(0)) +func compareBytes(ctx context.Context, a, b []byte) int { + return bytes.Compare(a, b) +} + func TestSkipList(t *testing.T) { t.Run("test skip list", func(t *testing.T) { vals := [][]byte{ @@ -34,15 +39,15 @@ func TestSkipList(t *testing.T) { b("f"), b("g"), b("h"), b("i"), b("j"), b("k"), b("l"), b("m"), b("n"), b("o"), } - testSkipList(t, bytes.Compare, vals...) + testSkipList(t, compareBytes, vals...) }) t.Run("test skip list of random bytes", func(t *testing.T) { vals := randomVals((randSrc.Int63() % 10_000) + 100) - testSkipList(t, bytes.Compare, vals...) + testSkipList(t, compareBytes, vals...) }) t.Run("test with custom compare function", func(t *testing.T) { - compare := func(left, right []byte) int { + compare := func(ctx context.Context, left, right []byte) int { l := int64(binary.LittleEndian.Uint64(left)) r := int64(binary.LittleEndian.Uint64(right)) return int(l - r) @@ -59,15 +64,15 @@ func TestSkipListCheckpoints(t *testing.T) { b("f"), b("g"), b("h"), b("i"), b("j"), b("k"), b("l"), b("m"), b("n"), b("o"), } - testSkipListCheckpoints(t, bytes.Compare, vals...) + testSkipListCheckpoints(t, compareBytes, vals...) }) t.Run("test skip list of random bytes", func(t *testing.T) { vals := randomVals((randSrc.Int63() % 10_000) + 100) - testSkipListCheckpoints(t, bytes.Compare, vals...) + testSkipListCheckpoints(t, compareBytes, vals...) }) t.Run("test with custom compare function", func(t *testing.T) { - compare := func(left, right []byte) int { + compare := func(ctx context.Context, left, right []byte) int { l := int64(binary.LittleEndian.Uint64(left)) r := int64(binary.LittleEndian.Uint64(right)) return int(l - r) @@ -86,6 +91,7 @@ func TestMemoryFootprint(t *testing.T) { } func testSkipList(t *testing.T, compare KeyOrder, vals ...[]byte) { + ctx := context.Background() randSrc.Shuffle(len(vals), func(i, j int) { vals[i], vals[j] = vals[j], vals[i] }) @@ -124,6 +130,7 @@ func testSkipListPuts(t *testing.T, list *List, vals ...[]byte) { } func testSkipListGets(t *testing.T, list *List, vals ...[]byte) { + ctx := context.Background() // get in different keyOrder randSrc.Shuffle(len(vals), func(i, j int) { vals[i], vals[j] = vals[j], vals[i] @@ -142,6 +149,7 @@ func testSkipListGets(t *testing.T, list *List, vals ...[]byte) { } func testSkipListUpdates(t *testing.T, list *List, vals ...[]byte) { + ctx := context.Background() v2 := []byte("789") for _, v := range vals { list.Put(ctx, v, v2) @@ -162,6 +170,7 @@ func testSkipListUpdates(t *testing.T, list *List, vals ...[]byte) { } func testSkipListIterForward(t *testing.T, list *List, vals ...[]byte) { + ctx := context.Background() // put |vals| back in keyOrder sort.Slice(vals, func(i, j int) bool { return list.compareKeys(ctx, vals[i], vals[j]) < 0 @@ -191,6 +200,7 @@ func testSkipListIterForward(t *testing.T, list *List, vals ...[]byte) { } func testSkipListIterBackward(t *testing.T, list *List, vals ...[]byte) { + ctx := context.Background() // put |vals| back in keyOrder sort.Slice(vals, func(i, j int) bool { return list.compareKeys(ctx, vals[i], vals[j]) < 0 @@ -211,6 +221,7 @@ func testSkipListIterBackward(t *testing.T, list *List, vals ...[]byte) { } func testSkipListTruncate(t *testing.T, list *List, vals ...[]byte) { + ctx := context.Background() assert.Equal(t, list.Count(), len(vals)) list.Truncate() @@ -244,6 +255,7 @@ func testSkipListTruncate(t *testing.T, list *List, vals ...[]byte) { } func validateIterForwardFrom(t *testing.T, l *List, key []byte) (count int) { + ctx := context.Background() iter := l.GetIterAt(ctx, key) k, _ := iter.Current() for k != nil { @@ -257,6 +269,7 @@ func validateIterForwardFrom(t *testing.T, l *List, key []byte) (count int) { } func validateIterBackwardFrom(t *testing.T, l *List, key []byte) (count int) { + ctx := context.Background() iter := l.GetIterAt(ctx, key) k, _ := iter.Current() for k != nil { @@ -317,6 +330,7 @@ func iterAllBackwards(l *List, cb func([]byte, []byte)) { } func testSkipListCheckpoints(t *testing.T, compare KeyOrder, data ...[]byte) { + ctx := context.Background() randSrc.Shuffle(len(data), func(i, j int) { data[i], data[j] = data[j], data[i] }) diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index 52df79cde48..b1c966b57d4 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -250,11 +250,12 @@ func TestCompare(t *testing.T) { } for _, test := range tests { + ctx := context.Background() act := compare(test.typ, test.l, test.r) assert.Equal(t, test.cmp, act, "expected %s %s %s ", - TupleDesc{}.formatValue(test.typ.Enc, 0, test.l), + TupleDesc{}.formatValue(ctx, test.typ.Enc, 0, test.l), fmtComparator(test.cmp), - TupleDesc{}.formatValue(test.typ.Enc, 0, test.r)) + TupleDesc{}.formatValue(ctx, test.typ.Enc, 0, test.r)) } } diff --git a/go/store/valuefile/value_file_test.go b/go/store/valuefile/value_file_test.go index b3911b98226..3d3caff4887 100644 --- a/go/store/valuefile/value_file_test.go +++ b/go/store/valuefile/value_file_test.go @@ -132,6 +132,7 @@ var vd = val.NewTupleDescriptor( ) func makeProllyMap(t *testing.T, ns tree.NodeStore, count int) (prolly.Map, [][2]val.Tuple) { + ctx := context.Background() tuples := tree.RandomTuplePairs(ctx, count, kd, vd, ns) om := mustProllyMapFromTuples(t, kd, vd, ns, tuples) From 39ac8d61d5cf69f5144ef98514a83fe83acdd640 Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Tue, 18 Feb 2025 16:04:27 -0800 Subject: [PATCH 07/11] Bump GMS dependency. --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index 742e0c0b8b7..dea447e75fb 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,7 +56,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.19.1-0.20250214230802-dc03db8b1d93 + github.com/dolthub/go-mysql-server v0.19.1-0.20250218220938-e4840a923dc0 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/dolthub/swiss v0.1.0 github.com/esote/minmaxheap v1.0.0 diff --git a/go/go.sum b/go/go.sum index 9e2e257f44f..f528ba4ea67 100644 --- a/go/go.sum +++ b/go/go.sum @@ -181,6 +181,8 @@ github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 h1:Sni8jrP0sy github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA= github.com/dolthub/go-mysql-server v0.19.1-0.20250214230802-dc03db8b1d93 h1:RYb18gXSJFgch5pjwQTeacNm4S+PNKXutENF5wK5gqw= github.com/dolthub/go-mysql-server v0.19.1-0.20250214230802-dc03db8b1d93/go.mod h1:JTlrabhq5TJqvlL+J3NKlm0EzTHQQugUAH6yAxWi4Ww= +github.com/dolthub/go-mysql-server v0.19.1-0.20250218220938-e4840a923dc0 h1:seiyp5wL3jiPHknx716R990AKwHpE16Nx87v8snisQQ= +github.com/dolthub/go-mysql-server v0.19.1-0.20250218220938-e4840a923dc0/go.mod h1:JTlrabhq5TJqvlL+J3NKlm0EzTHQQugUAH6yAxWi4Ww= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= From 069b1e5593dcc5f308dfaf60a448539a9984b798 Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Wed, 19 Feb 2025 12:22:47 -0800 Subject: [PATCH 08/11] Remove nodestore parameter from getIgnoreTablePatternKey --- go/libraries/doltcore/doltdb/ignore.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/go/libraries/doltcore/doltdb/ignore.go b/go/libraries/doltcore/doltdb/ignore.go index a51478bf151..df63597ef92 100644 --- a/go/libraries/doltcore/doltdb/ignore.go +++ b/go/libraries/doltcore/doltdb/ignore.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" - "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" "github.com/dolthub/dolt/go/store/val" ) @@ -72,7 +71,7 @@ func convertTupleToIgnoreBoolean(ctx context.Context, valueDesc val.TupleDesc, v return ignore, nil } -func getIgnoreTablePatternKey(ctx context.Context, keyDesc val.TupleDesc, keyTuple val.Tuple, _ tree.NodeStore) (string, error) { +func getIgnoreTablePatternKey(ctx context.Context, keyDesc val.TupleDesc, keyTuple val.Tuple) (string, error) { if !keyDesc.Equals(val.NewTupleDescriptor(val.Type{Enc: val.StringEnc, Nullable: false})) { return "", fmt.Errorf("dolt_ignore had unexpected key type, this should never happen") } @@ -111,10 +110,10 @@ func GetIgnoredTablePatterns(ctx context.Context, roots Roots, schemas []string) if err != nil { return nil, err } - // The ignore table doesn't have any addressable columns, so it's okay to not pass in a ValueStore here. - keyDesc, valueDesc := ignoreTableSchema.GetMapDescriptors(nil) + m := durable.MapFromIndex(index) + keyDesc, valueDesc := ignoreTableSchema.GetMapDescriptors(m.NodeStore()) - ignoreTableMap, err := durable.ProllyMapFromIndex(index).IterAll(ctx) + ignoreTableMap, err := m.IterAll(ctx) if err != nil { return nil, err } @@ -127,9 +126,7 @@ func GetIgnoredTablePatterns(ctx context.Context, roots Roots, schemas []string) return nil, err } - m := durable.MapFromIndex(index) - - pattern, err := GetIgnoreTablePatternKey(ctx, keyDesc, keyTuple, m.NodeStore()) + pattern, err := GetIgnoreTablePatternKey(ctx, keyDesc, keyTuple) if err != nil { return nil, err } From 26cc33d550a0f558f98f5a983aec8eb4465c1189 Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Mon, 24 Feb 2025 14:05:14 -0800 Subject: [PATCH 09/11] Add comments and fix tests. --- go/libraries/doltcore/merge/row_merge_test.go | 4 ++-- go/libraries/doltcore/merge/schema_merge_test.go | 2 +- go/store/val/codec_test.go | 1 + go/store/val/tuple_descriptor.go | 4 ++++ go/store/val/value_store.go | 3 +++ 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/merge/row_merge_test.go b/go/libraries/doltcore/merge/row_merge_test.go index 43ad35b7198..bb318419c96 100644 --- a/go/libraries/doltcore/merge/row_merge_test.go +++ b/go/libraries/doltcore/merge/row_merge_test.go @@ -216,7 +216,7 @@ func TestRowMerge(t *testing.T) { merged, ok, err := v.tryMerge(ctx, test.row, test.mergeRow, test.ancRow) assert.NoError(t, err) assert.Equal(t, test.expectConflict, !ok) - vD := test.mergedSch.GetValueDescriptor(ns) + vD := test.mergedSch.GetValueDescriptor(v.ns) assert.Equal(t, vD.Format(ctx, test.expectedResult), vD.Format(ctx, merged)) }) } @@ -335,7 +335,7 @@ func buildTup(sch schema.Schema, r []*int) val.Tuple { return nil } - vD := sch.GetValueDescriptor(ns) + vD := sch.GetValueDescriptor(nil) vB := val.NewTupleBuilder(vD) for i, v := range r { if v != nil { diff --git a/go/libraries/doltcore/merge/schema_merge_test.go b/go/libraries/doltcore/merge/schema_merge_test.go index d4e971f616a..38ae03d30b5 100644 --- a/go/libraries/doltcore/merge/schema_merge_test.go +++ b/go/libraries/doltcore/merge/schema_merge_test.go @@ -1633,7 +1633,7 @@ func testSchemaMergeHelper(t *testing.T, tests []schemaMergeTest, flipSides bool runTest := func(t *testing.T, test schemaMergeTest, expectDataConflict bool, expConstraintViolations []constraintViolation) { ctx := context.Background() a, l, r, m := setupSchemaMergeTest(ctx, t, test) - ns := m.NodeStore() + ns := a.NodeStore() var mo merge.MergeOpts var eo editor.Options diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index b1c966b57d4..bbdfe146485 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -15,6 +15,7 @@ package val import ( + "context" "math" "testing" "time" diff --git a/go/store/val/tuple_descriptor.go b/go/store/val/tuple_descriptor.go index 8a92983f328..c43ea8c8a23 100644 --- a/go/store/val/tuple_descriptor.go +++ b/go/store/val/tuple_descriptor.go @@ -671,6 +671,9 @@ func (td TupleDesc) Equals(other TupleDesc) bool { return true } +// AddressTypeHandler is an implementation of TupleTypeHandler for columns that contain a content-address to some value +// stored in a ValueStore. This TypeHandler converts between the address and the underlying value as needed, allowing +// these columns to be used in contexts that need access to the underlying value, such as in primary indexes. type AddressTypeHandler struct { vs ValueStore childHandler TupleTypeHandler @@ -684,6 +687,7 @@ func NewExtendedAddressTypeHandler(vs ValueStore, childHandler TupleTypeHandler) } func (handler AddressTypeHandler) SerializedCompare(ctx context.Context, v1 []byte, v2 []byte) (int, error) { + // TODO: If the child handler allows, compare the values one chunk at a time instead of always fully deserializing them. v1Bytes, err := handler.vs.ReadBytes(ctx, hash.New(v1)) if err != nil { return 0, err diff --git a/go/store/val/value_store.go b/go/store/val/value_store.go index 4ffc57f7e84..de80b519861 100644 --- a/go/store/val/value_store.go +++ b/go/store/val/value_store.go @@ -19,6 +19,9 @@ import ( "github.com/dolthub/dolt/go/store/hash" ) +// ValueStore is an interface for a key-value store that can store byte sequences, keyed by a content hash. +// The only implementation is tree.NodeStore, but ValueStore can be used without depending on the tree package. +// This is useful for type handlers. type ValueStore interface { ReadBytes(ctx context.Context, h hash.Hash) ([]byte, error) WriteBytes(ctx context.Context, val []byte) (hash.Hash, error) From c086e9539081475f66226ff1c15aeeedccf544af Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Mon, 24 Feb 2025 14:05:21 -0800 Subject: [PATCH 10/11] Bump GMS version. --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index dea447e75fb..00c04f37b32 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,7 +56,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.19.1-0.20250218220938-e4840a923dc0 + github.com/dolthub/go-mysql-server v0.19.1-0.20250224212452-a3d336db732b github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/dolthub/swiss v0.1.0 github.com/esote/minmaxheap v1.0.0 diff --git a/go/go.sum b/go/go.sum index ae17a1163ec..fa87ba0bf82 100644 --- a/go/go.sum +++ b/go/go.sum @@ -181,6 +181,8 @@ github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 h1:Sni8jrP0sy github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA= github.com/dolthub/go-mysql-server v0.19.1-0.20250218220938-e4840a923dc0 h1:seiyp5wL3jiPHknx716R990AKwHpE16Nx87v8snisQQ= github.com/dolthub/go-mysql-server v0.19.1-0.20250218220938-e4840a923dc0/go.mod h1:JTlrabhq5TJqvlL+J3NKlm0EzTHQQugUAH6yAxWi4Ww= +github.com/dolthub/go-mysql-server v0.19.1-0.20250224212452-a3d336db732b h1:5/NOX/GNuXLYLJY5NPSgQaPW5M7aOr1wEmL4ataRI1o= +github.com/dolthub/go-mysql-server v0.19.1-0.20250224212452-a3d336db732b/go.mod h1:JTlrabhq5TJqvlL+J3NKlm0EzTHQQugUAH6yAxWi4Ww= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= From 53f404cc4cfb365b6dd053a07ba3e2f091607e19 Mon Sep 17 00:00:00 2001 From: nicktobey Date: Mon, 24 Feb 2025 22:24:59 +0000 Subject: [PATCH 11/11] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/merge/mutable_secondary_index.go | 2 +- go/store/prolly/tree/node_store.go | 5 ++--- go/store/val/value_store.go | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/libraries/doltcore/merge/mutable_secondary_index.go b/go/libraries/doltcore/merge/mutable_secondary_index.go index 4d4271cd551..cf72af8ddeb 100644 --- a/go/libraries/doltcore/merge/mutable_secondary_index.go +++ b/go/libraries/doltcore/merge/mutable_secondary_index.go @@ -16,7 +16,6 @@ package merge import ( "context" - "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/go-mysql-server/sql" @@ -24,6 +23,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/val" ) diff --git a/go/store/prolly/tree/node_store.go b/go/store/prolly/tree/node_store.go index f82002db66a..4f933444857 100644 --- a/go/store/prolly/tree/node_store.go +++ b/go/store/prolly/tree/node_store.go @@ -17,15 +17,14 @@ package tree import ( "bytes" "context" - "github.com/dolthub/dolt/go/store/val" "sync" - "github.com/dolthub/dolt/go/store/prolly/message" - "github.com/dolthub/dolt/go/store/chunks" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/pool" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/types" + "github.com/dolthub/dolt/go/store/val" ) const ( diff --git a/go/store/val/value_store.go b/go/store/val/value_store.go index de80b519861..815c2f5dc1e 100644 --- a/go/store/val/value_store.go +++ b/go/store/val/value_store.go @@ -16,6 +16,7 @@ package val import ( "context" + "github.com/dolthub/dolt/go/store/hash" )