Skip to content

Commit

Permalink
sql: run check constraint mutations and validation by schema changer
Browse files Browse the repository at this point in the history 
This change moves check constraint adds and drops through the schema
changer when the transaction commits, moving the constraint through the
same intermediate states as when index/columns are added or dropped. The
only small differences are:
 - Constraint adds can start in write-only and immediately start being
   enforced if all used columns are in write-only or public.
 - Constraint drops can move immediately from public to absent if they
   have not yet been validated.

Therefore, the check constraint is no longer immediately public when it
is added, but allows data validation of the constraint to be performed
when the constraint is added (and is now the default behaviour).
Constraints can now also be added on columns in the process of being
added.

This change also ensures that there are no data anomalies in either
versions of the schema when dropping a validated check constraint, as
previously the transition moved the constraint from public -> absent.
Writes on the new version were not being checked even though nodes on an
older version expect all rows to conform to the constraint.

Release note (sql change): Check constraint adds by default will
validate table data with the added constraint asynchronously after the
transaction commits.
  • Loading branch information
Erik Trinh committed Dec 19, 2018
1 parent 66c50ae commit cb6ba5c
Show file tree
Hide file tree
Showing 20 changed files with 1,331 additions and 581 deletions.
58 changes: 38 additions & 20 deletions pkg/sql/alter_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package sql

import (
"bytes"
"context"
gojson "encoding/json"
"fmt"
Expand Down Expand Up @@ -179,6 +178,11 @@ func (n *alterTableNode) startExec(params runParams) error {
for k := range info {
inuseNames[k] = struct{}{}
}
for _, m := range n.tableDesc.Mutations {
if ck := m.GetCheck(); ck != nil {
inuseNames[ck.Name] = struct{}{}
}
}
switch d := t.ConstraintDef.(type) {
case *tree.UniqueConstraintTableDef:
if d.PrimaryKey {
Expand Down Expand Up @@ -212,13 +216,21 @@ func (n *alterTableNode) startExec(params runParams) error {
}

case *tree.CheckConstraintTableDef:
ck, err := MakeCheckConstraint(params.ctx,
// A previous command could have added a column which the new constraint uses,
// allocate IDs now.
if err != n.tableDesc.AllocateIDs() {
return err
}
ck, canSkipDelOnly, err := MakeCheckConstraint(params.ctx,
n.tableDesc, d, inuseNames, &params.p.semaCtx, params.EvalContext(), n.n.Table)
if err != nil {
return err
}
ck.Validity = sqlbase.ConstraintValidity_Unvalidated
n.tableDesc.Checks = append(n.tableDesc.Checks, ck)
if ck.Validity == sqlbase.ConstraintValidity_Validating {
n.tableDesc.AddCheckAddMutation(*ck, canSkipDelOnly)
} else {
n.tableDesc.AddCheck(*ck)
}
descriptorChanged = true

case *tree.ForeignKeyConstraintTableDef:
Expand Down Expand Up @@ -403,6 +415,8 @@ func (n *alterTableNode) startExec(params runParams) error {
return err
} else if !used {
validChecks = append(validChecks, check)
} else {
n.tableDesc.MaybeAddCheckDropMutation(*check)
}
}

Expand All @@ -411,6 +425,17 @@ func (n *alterTableNode) startExec(params runParams) error {
descriptorChanged = true
}

// Check that no dependent check constraints are queued to be added.
for _, m := range n.tableDesc.Mutations {
if ck := m.GetCheck(); ck != nil && m.Direction == sqlbase.DescriptorMutation_ADD {
if used, err := ck.UsesColumn(n.tableDesc.TableDesc(), col.ID); err != nil {
return err
} else if used {
return fmt.Errorf("referencing constraint %q in the middle of being added, try again later", ck.Name)
}
}
}

found := false
for i := range n.tableDesc.Columns {
if n.tableDesc.Columns[i].ID == col.ID {
Expand Down Expand Up @@ -443,13 +468,19 @@ func (n *alterTableNode) startExec(params runParams) error {
case sqlbase.ConstraintTypeUnique:
return fmt.Errorf("UNIQUE constraint depends on index %q, use DROP INDEX with CASCADE if you really want to drop it", t.Constraint)
case sqlbase.ConstraintTypeCheck:
for i := range n.tableDesc.Checks {
if n.tableDesc.Checks[i].Name == name {
found := false
for i, ck := range n.tableDesc.Checks {
if ck.Name == name {
n.tableDesc.MaybeAddCheckDropMutation(*ck)
n.tableDesc.Checks = append(n.tableDesc.Checks[:i], n.tableDesc.Checks[i+1:]...)
descriptorChanged = true
found = true
break
}
}
if !found {
return fmt.Errorf("constraint %q in the middle of being added, try again later", t.Constraint)
}
case sqlbase.ConstraintTypeFK:
idx, err := n.tableDesc.FindIndexByID(details.Index.ID)
if err != nil {
Expand Down Expand Up @@ -488,7 +519,7 @@ func (n *alterTableNode) startExec(params runParams) error {
}
}
if !found {
panic("constraint returned by GetConstraintInfo not found")
return fmt.Errorf("constraint %q in the middle of being added, try again later", t.Constraint)
}
ck := n.tableDesc.Checks[idx]
if err := params.p.validateCheckExpr(
Expand Down Expand Up @@ -764,19 +795,6 @@ func applyColumnMutation(
return nil
}

func labeledRowValues(cols []sqlbase.ColumnDescriptor, values tree.Datums) string {
var s bytes.Buffer
for i := range cols {
if i != 0 {
s.WriteString(`, `)
}
s.WriteString(cols[i].Name)
s.WriteString(`=`)
s.WriteString(values[i].String())
}
return s.String()
}

// injectTableStats implements the INJECT STATISTICS command, which deletes any
// existing statistics on the table and replaces them with the statistics in the
// given json object (in the same format as the result of SHOW STATISTICS USING
Expand Down
78 changes: 73 additions & 5 deletions pkg/sql/backfill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,15 @@ const (
// processed per chunk during column truncate or backfill.
columnTruncateAndBackfillChunkSize = 200

// checkConstraintBackfillChunkSize is the maximum number of rows
// processed per chunk during check constraint validation. This value
// is larger than the other chunk constants because the operation involves
// only running a scan and does not write.
checkConstraintBackfillChunkSize = 1600

// indexTruncateChunkSize is the maximum number of index entries truncated
// per chunk during an index truncation. This value is larger than the
// other chunk constants because the operation involves only running a
// per chunk during an index truncation. This value is larger than other
// chunk constants because the operation involves only running a
// DeleteRange().
indexTruncateChunkSize = 600

Expand Down Expand Up @@ -132,13 +138,18 @@ func (sc *SchemaChanger) runBackfill(
tableDesc.Name, tableDesc.Version, sc.mutationID)

needColumnBackfill := false
needCheckValidation := false
for _, m := range tableDesc.Mutations {
if m.MutationID != sc.mutationID {
break
}
switch m.Direction {
case sqlbase.DescriptorMutation_ADD:
switch t := m.Descriptor_.(type) {
case *sqlbase.DescriptorMutation_Check:
if t.Check.Validity == sqlbase.ConstraintValidity_Validating {
needCheckValidation = true
}
case *sqlbase.DescriptorMutation_Column:
if sqlbase.ColumnNeedsBackfill(m.GetColumn()) {
needColumnBackfill = true
Expand All @@ -151,6 +162,8 @@ func (sc *SchemaChanger) runBackfill(

case sqlbase.DescriptorMutation_DROP:
switch t := m.Descriptor_.(type) {
case *sqlbase.DescriptorMutation_Check:
// Nothing to do.
case *sqlbase.DescriptorMutation_Column:
needColumnBackfill = true
case *sqlbase.DescriptorMutation_Index:
Expand Down Expand Up @@ -186,6 +199,13 @@ func (sc *SchemaChanger) runBackfill(
}
}

// Validate checks.
if needCheckValidation {
if err := sc.validateChecks(ctx, evalCtx, lease, version); err != nil {
return err
}
}

return nil
}

Expand All @@ -202,6 +222,18 @@ func (sc *SchemaChanger) getTableVersion(
return tableDesc, nil
}

func (sc *SchemaChanger) validateChecks(
ctx context.Context,
evalCtx *extendedEvalContext,
lease *sqlbase.TableDescriptor_SchemaChangeLease,
version sqlbase.DescriptorVersion,
) error {
return sc.distBackfill(
ctx, evalCtx,
lease, version, checkConstraintBackfill, checkConstraintBackfillChunkSize,
backfill.CheckMutationFilter)
}

func (sc *SchemaChanger) truncateIndexes(
ctx context.Context,
lease *sqlbase.TableDescriptor_SchemaChangeLease,
Expand Down Expand Up @@ -286,6 +318,7 @@ const (
_ backfillType = iota
columnBackfill
indexBackfill
checkConstraintBackfill
)

// getJobIDForMutationWithDescriptor returns a job id associated with a mutation given
Expand Down Expand Up @@ -517,9 +550,9 @@ func runSchemaChangesInTxn(
return nil
}

// Only needed because columnBackfillInTxn() backfills
// all column mutations.
doneColumnBackfill := false
// Only needed because columnBackfillInTxn() and checkValidateInTxn()
// backfills are applied to all related mutations.
doneColumnBackfill, doneCheckValidation := false, false
for _, m := range tableDesc.Mutations {
immutDesc := sqlbase.NewImmutableTableDescriptor(*tableDesc.TableDesc())
switch m.Direction {
Expand All @@ -539,6 +572,15 @@ func runSchemaChangesInTxn(
return err
}

case *sqlbase.DescriptorMutation_Check:
if doneCheckValidation || m.GetCheck().Validity != sqlbase.ConstraintValidity_Validating {
break
}
if err := checkValidateInTxn(ctx, txn, evalCtx, immutDesc, traceKV); err != nil {
return err
}
doneCheckValidation = true

default:
return errors.Errorf("unsupported mutation: %+v", m)
}
Expand All @@ -560,6 +602,9 @@ func runSchemaChangesInTxn(
return err
}

case *sqlbase.DescriptorMutation_Check:
// No-op.

default:
return errors.Errorf("unsupported mutation: %+v", m)
}
Expand All @@ -574,6 +619,29 @@ func runSchemaChangesInTxn(
return nil
}

func checkValidateInTxn(
ctx context.Context,
txn *client.Txn,
evalCtx *tree.EvalContext,
tableDesc *sqlbase.ImmutableTableDescriptor,
traceKV bool,
) error {
var backfiller backfill.CheckBackfiller
if err := backfiller.Init(evalCtx, tableDesc); err != nil {
return err
}

sp := tableDesc.PrimaryIndexSpan()
for sp.Key != nil {
var err error
sp.Key, err = backfiller.RunCheckBackfillChunk(ctx, txn, tableDesc, sp, checkConstraintBackfillChunkSize, traceKV)
if err != nil {
return err
}
}
return nil
}

// columnBackfillInTxn backfills columns for all mutation columns in
// the mutation list.
func columnBackfillInTxn(
Expand Down
Loading

0 comments on commit cb6ba5c

Please sign in to comment.