generated from TBD54566975/tbd-project-template
-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Loading status checks…
chore: split DAL into two packages, with moduleconfig queries moved t…
…o common/configuration (#1840) Fixes #1696 Approach: * Duplicate `- engine` block of `sqlc.yaml` to support multiple `out` locations * Refactor common error handling vars + funcs into separate `dalerrs` package. This now lives under the new top level `db` dir, where we can add additional common DB logic as we split the controller dal further. This change incurred a lot of NOOP lines changed. :/ * Meat of the change: split moduleconfig CRUD queries out of `backend/controller/sql` and into `common/configuration/sql`. Correspondingly, split the moduleconfig DAL functions out of `backend/controller/dal` and into `common/configuration/dal`. * Questionable choice: Copied `backend/controller/sql/conn.go` to `common/configuration/sql/conn.go` with the unnecessary logic stripped out (`Tx`...). That means that what remains (`sql.DBI` and `sql.NewDB`), though a small amount of code, is duplicated with the original `conn.go`. I could refactor this to use a shared interface instead, but that could also make this messier for readability and take things in a worse direction overall, so pending better ideas, we can rethink this when splitting the controller DAL.
- v0.434.0
- v0.433.2
- v0.433.1
- v0.433.0
- v0.432.0
- v0.431.1
- v0.431.0
- v0.430.1
- v0.430.0
- v0.429.0
- v0.428.1
- v0.428.0
- v0.427.2
- v0.427.1
- v0.427.0
- v0.426.0
- v0.425.0
- v0.424.2
- v0.424.1
- v0.424.0
- v0.423.1
- v0.423.0
- v0.422.2
- v0.422.1
- v0.422.0
- v0.421.0
- v0.420.1
- v0.420.0
- v0.419.0
- v0.418.0
- v0.417.0
- v0.416.0
- v0.415.1
- v0.415.0
- v0.414.0
- v0.413.2
- v0.413.1
- v0.413.0
- v0.412.6
- v0.412.5
- v0.412.4
- v0.412.3
- v0.412.2
- v0.412.1
- v0.412.0
- v0.411.0
- v0.410.2
- v0.410.1
- v0.410.0
- v0.409.0
- v0.408.2
- v0.408.1
- v0.408.0
- v0.407.4
- v0.407.3
- v0.407.2
- v0.407.1
- v0.407.0
- v0.406.0
- v0.405.5
- v0.405.4
- v0.405.3
- v0.405.2
- v0.405.1
- v0.405.0
- v0.404.1
- v0.404.0
- v0.403.2
- v0.403.1
- v0.403.0
- v0.402.0
- v0.401.0
- v0.400.0
- v0.399.0
- v0.398.1
- v0.398.0
- v0.397.2
- v0.397.1
- v0.397.0
- v0.396.0
- v0.395.0
- v0.394.1
- v0.394.0
- v0.393.1
- v0.393.0
- v0.392.2
- v0.392.1
- v0.392.0
- v0.391.2
- v0.391.1
- v0.391.0
- v0.390.0
- v0.389.0
- v0.388.0
- v0.387.2
- v0.387.1
- v0.387.0
- v0.386.3
- v0.386.2
- v0.386.1
- v0.386.0
- v0.385.0
- v0.384.0
- v0.383.1
- v0.383.0
- v0.382.0
- v0.381.1
- v0.381.0
- v0.380.0
- v0.379.1
- v0.379.0
- v0.378.2
- v0.378.1
- v0.378.0
- v0.377.3
- v0.377.2
- v0.377.1
- v0.377.0
- v0.376.1
- v0.376.0
- v0.375.0
- v0.374.0
- v0.373.1
- v0.373.0
- v0.372.4
- v0.372.3
- v0.372.2
- v0.372.1
- v0.372.0
- v0.371.0
- v0.370.0
- v0.369.0
- v0.368.1
- v0.368.0
- v0.367.0
- v0.366.0
- v0.365.0
- v0.364.1
- v0.364.0
- v0.363.0
- v0.362.1
- v0.362.0
- v0.361.5
- v0.361.4
- v0.361.3
- v0.361.2
- v0.361.1
- v0.361.0
- v0.360.0
- v0.359.1
- v0.359.0
- v0.358.0
- v0.357.0
- v0.356.0
- v0.355.1
- v0.355.0
- v0.354.2
- v0.354.1
- v0.354.0
- v0.353.0
- v0.352.0
- v0.351.0
- v0.350.0
- v0.349.2
- v0.349.1
- v0.349.0
- v0.348.3
- v0.348.2
- v0.348.1
- v0.348.0
- v0.347.0
- v0.346.0
- v0.345.2
- v0.345.1
- v0.345.0
- v0.344.0
- v0.343.0
- v0.342.0
- v0.341.1
- v0.341.0
- v0.340.0
- v0.339.0
- v0.338.0
- v0.337.1
- v0.337.0
- v0.336.0
- v0.335.4
- v0.335.3
- v0.335.2
- v0.335.1
- v0.335.0
- v0.334.1
- v0.334.0
- v0.333.2
- v0.333.1
- v0.333.0
- v0.332.1
- v0.332.0
- v0.331.0
- v0.330.0
- v0.329.0
- v0.328.5
- v0.328.4
- v0.328.3
- v0.328.2
- v0.328.1
- v0.328.0
- v0.327.2
- v0.327.1
- v0.327.0
- v0.326.0
- v0.325.2
- v0.325.1
- v0.325.0
- v0.324.0
- v0.323.2
- v0.323.1
- v0.323.0
- v0.322.2
- v0.322.1
- v0.322.0
- v0.321.0
- v0.320.0
- v0.319.1
- v0.319.0
- v0.318.1
- v0.318.0
- v0.317.1
- v0.317.0
- v0.316.2
- v0.316.1
- v0.316.0
- v0.315.0
- v0.314.0
- v0.313.1
- v0.313.0
- v0.312.1
- v0.312.0
- v0.311.2
- v0.311.1
- v0.311.0
- v0.310.1
- v0.310.0
- v0.309.1
- v0.309.0
- v0.308.1
- v0.308.0
- v0.307.2
- v0.307.1
- v0.307.0
- v0.306.4
- v0.306.3
- v0.306.2
- v0.306.1
- v0.306.0
- v0.305.0
- v0.304.1
- v0.304.0
- v0.303.1
- v0.303.0
- v0.302.3
- v0.302.2
- v0.302.1
- v0.302.0
- v0.301.1
- v0.301.0
- v0.300.0
- v0.299.0
- v0.298.0
- v0.297.7
- v0.297.6
- v0.297.5
- v0.297.4
- v0.297.3
- v0.297.2
- v0.297.1
- v0.297.0
- v0.296.11
- v0.296.10
- v0.296.9
- v0.296.8
- v0.296.7
- v0.296.6
- v0.296.5
- v0.296.4
- v0.296.3
- v0.296.2
- v0.296.1
- v0.296.0
- v0.295.1
- v0.295.0
- v0.294.0
- v0.293.0
- v0.292.3
- v0.292.2
- v0.292.1
- v0.292.0
- v0.291.0
- v0.290.0
- v0.289.0
- v0.288.0
- v0.287.0
- v0.286.0
- v0.285.5
- v0.285.4
- v0.285.3
- v0.285.2
- v0.285.1
- v0.285.0
- v0.284.2
- v0.284.1
- v0.284.0
- v0.283.0
- v0.282.2
- v0.282.1
- v0.282.0
- v0.281.9
- v0.281.8
- v0.281.7
- v0.281.6
- v0.281.5
- v0.281.4
- v0.281.3
- v0.281.2
- v0.281.1
- v0.281.0
- v0.280.4
- v0.280.3
- v0.280.2
- v0.280.1
- v0.280.0
- v0.279.0
- v0.278.0
- v0.277.3
- v0.277.2
- v0.277.1
- v0.277.0
- v0.276.7
- v0.276.6
- v0.276.5
- v0.276.4
- v0.276.3
- v0.276.2
- v0.276.1
- v0.276.0
- v0.275.1
- v0.275.0
- v0.274.3
- v0.274.2
- v0.274.1
- v0.274.0
- v0.273.0
- v0.272.0
- v0.271.8
- v0.271.7
- v0.271.6
- v0.271.5
- v0.271.4
- v0.271.3
- v0.271.2
- v0.271.1
- v0.271.0
- v0.270.0
- v0.269.2
- v0.269.1
- v0.269.0
- v0.268.3
- v0.268.2
- v0.268.1
- v0.268.0
- v0.267.0
- v0.266.1
- v0.266.0
- v0.265.3
- v0.265.2
- v0.265.1
- v0.265.0
- v0.264.3
- v0.264.2
- v0.264.1
Showing
32 changed files
with
1,087 additions
and
401 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
// Package dal provides a data abstraction layer for managing module configurations | ||
package dal | ||
|
||
import ( | ||
"context" | ||
|
||
"github.com/alecthomas/types/optional" | ||
"github.com/jackc/pgx/v5/pgxpool" | ||
|
||
"github.com/TBD54566975/ftl/common/configuration/sql" | ||
"github.com/TBD54566975/ftl/db/dalerrs" | ||
) | ||
|
||
type DAL struct { | ||
db sql.DBI | ||
} | ||
|
||
func New(ctx context.Context, pool *pgxpool.Pool) (*DAL, error) { | ||
dal := &DAL{db: sql.NewDB(pool)} | ||
return dal, nil | ||
} | ||
|
||
func (d *DAL) GetModuleConfiguration(ctx context.Context, module optional.Option[string], name string) ([]byte, error) { | ||
b, err := d.db.GetModuleConfiguration(ctx, module, name) | ||
if err != nil { | ||
return nil, dalerrs.TranslatePGError(err) | ||
} | ||
return b, nil | ||
} | ||
|
||
func (d *DAL) SetModuleConfiguration(ctx context.Context, module optional.Option[string], name string, value []byte) error { | ||
err := d.db.SetModuleConfiguration(ctx, module, name, value) | ||
return dalerrs.TranslatePGError(err) | ||
} | ||
|
||
func (d *DAL) UnsetModuleConfiguration(ctx context.Context, module optional.Option[string], name string) error { | ||
err := d.db.UnsetModuleConfiguration(ctx, module, name) | ||
return dalerrs.TranslatePGError(err) | ||
} | ||
|
||
func (d *DAL) ListModuleConfiguration(ctx context.Context) ([]sql.ModuleConfiguration, error) { | ||
l, err := d.db.ListModuleConfiguration(ctx) | ||
if err != nil { | ||
return nil, dalerrs.TranslatePGError(err) | ||
} | ||
return l, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
package dal | ||
|
||
import ( | ||
"context" | ||
"testing" | ||
|
||
"github.com/alecthomas/assert/v2" | ||
"github.com/alecthomas/types/optional" | ||
|
||
"github.com/TBD54566975/ftl/backend/controller/sql" | ||
"github.com/TBD54566975/ftl/backend/controller/sql/sqltest" | ||
"github.com/TBD54566975/ftl/internal/log" | ||
) | ||
|
||
func TestModuleConfiguration(t *testing.T) { | ||
ctx := log.ContextWithNewDefaultLogger(context.Background()) | ||
conn := sqltest.OpenForTesting(ctx, t) | ||
dal, err := New(ctx, conn) | ||
assert.NoError(t, err) | ||
assert.NotZero(t, dal) | ||
|
||
tests := []struct { | ||
TestName string | ||
ModuleSet optional.Option[string] | ||
ModuleGet optional.Option[string] | ||
PresetGlobal bool | ||
}{ | ||
{ | ||
"SetModuleGetModule", | ||
optional.Some("echo"), | ||
optional.Some("echo"), | ||
false, | ||
}, | ||
{ | ||
"SetGlobalGetGlobal", | ||
optional.None[string](), | ||
optional.None[string](), | ||
false, | ||
}, | ||
{ | ||
"SetGlobalGetModule", | ||
optional.None[string](), | ||
optional.Some("echo"), | ||
false, | ||
}, | ||
{ | ||
"SetModuleOverridesGlobal", | ||
optional.Some("echo"), | ||
optional.Some("echo"), | ||
true, | ||
}, | ||
} | ||
|
||
b := []byte(`"asdf"`) | ||
for _, test := range tests { | ||
t.Run(test.TestName, func(t *testing.T) { | ||
if test.PresetGlobal { | ||
err := dal.SetModuleConfiguration(ctx, optional.None[string](), "configname", []byte(`"qwerty"`)) | ||
assert.NoError(t, err) | ||
} | ||
err := dal.SetModuleConfiguration(ctx, test.ModuleSet, "configname", b) | ||
assert.NoError(t, err) | ||
gotBytes, err := dal.GetModuleConfiguration(ctx, test.ModuleGet, "configname") | ||
assert.NoError(t, err) | ||
assert.Equal(t, b, gotBytes) | ||
err = dal.UnsetModuleConfiguration(ctx, test.ModuleGet, "configname") | ||
assert.NoError(t, err) | ||
}) | ||
} | ||
|
||
t.Run("List", func(t *testing.T) { | ||
sortedList := []sql.ModuleConfiguration{ | ||
{ | ||
Module: optional.Some("echo"), | ||
Name: "a", | ||
}, | ||
{ | ||
Module: optional.Some("echo"), | ||
Name: "b", | ||
}, | ||
{ | ||
Module: optional.None[string](), | ||
Name: "a", | ||
}, | ||
} | ||
|
||
// Insert entries in a separate order from how they should be returned to | ||
// test sorting logic in the SQL query | ||
err := dal.SetModuleConfiguration(ctx, sortedList[1].Module, sortedList[1].Name, []byte(`""`)) | ||
assert.NoError(t, err) | ||
err = dal.SetModuleConfiguration(ctx, sortedList[2].Module, sortedList[2].Name, []byte(`""`)) | ||
assert.NoError(t, err) | ||
err = dal.SetModuleConfiguration(ctx, sortedList[0].Module, sortedList[0].Name, []byte(`""`)) | ||
assert.NoError(t, err) | ||
|
||
gotList, err := dal.ListModuleConfiguration(ctx) | ||
assert.NoError(t, err) | ||
for i := range sortedList { | ||
assert.Equal(t, sortedList[i].Module, gotList[i].Module) | ||
assert.Equal(t, sortedList[i].Name, gotList[i].Name) | ||
} | ||
}) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
package sql | ||
|
||
type DBI interface { | ||
Querier | ||
Conn() ConnI | ||
} | ||
|
||
type ConnI interface { | ||
DBTX | ||
} | ||
|
||
type DB struct { | ||
conn ConnI | ||
*Queries | ||
} | ||
|
||
func NewDB(conn ConnI) *DB { | ||
return &DB{conn: conn, Queries: New(conn)} | ||
} | ||
|
||
func (d *DB) Conn() ConnI { return d.conn } |
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
-- name: GetModuleConfiguration :one | ||
SELECT value | ||
FROM module_configuration | ||
WHERE | ||
(module IS NULL OR module = @module) | ||
AND name = @name | ||
ORDER BY module NULLS LAST | ||
LIMIT 1; | ||
|
||
-- name: ListModuleConfiguration :many | ||
SELECT * | ||
FROM module_configuration | ||
ORDER BY module, name; | ||
|
||
-- name: SetModuleConfiguration :exec | ||
INSERT INTO module_configuration (module, name, value) | ||
VALUES ($1, $2, $3); | ||
|
||
-- name: UnsetModuleConfiguration :exec | ||
DELETE FROM module_configuration | ||
WHERE module = @module AND name = @name; |
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
// Package dalerrs provides common error handling utilities for all domain-specific DALs, | ||
// e.g. controller DAL and configuration DAL, which all connect to the same underlying DB | ||
// and maintain the same interface guarantees | ||
package dalerrs | ||
|
||
import ( | ||
stdsql "database/sql" | ||
"errors" | ||
"fmt" | ||
"strings" | ||
|
||
"github.com/jackc/pgerrcode" | ||
"github.com/jackc/pgx/v5" | ||
"github.com/jackc/pgx/v5/pgconn" | ||
) | ||
|
||
var ( | ||
// ErrConflict is returned by select methods in the DAL when a resource already exists. | ||
// | ||
// Its use will be documented in the corresponding methods. | ||
ErrConflict = errors.New("conflict") | ||
// ErrNotFound is returned by select methods in the DAL when no results are found. | ||
ErrNotFound = errors.New("not found") | ||
// ErrConstraint is returned by select methods in the DAL when a constraint is violated. | ||
ErrConstraint = errors.New("constraint violation") | ||
) | ||
|
||
func IsNotFound(err error) bool { | ||
return errors.Is(err, stdsql.ErrNoRows) || errors.Is(err, pgx.ErrNoRows) | ||
} | ||
|
||
func TranslatePGError(err error) error { | ||
if err == nil { | ||
return nil | ||
} | ||
var pgErr *pgconn.PgError | ||
if errors.As(err, &pgErr) { | ||
switch pgErr.Code { | ||
case pgerrcode.ForeignKeyViolation: | ||
return fmt.Errorf("%s: %w", strings.TrimSuffix(strings.TrimPrefix(pgErr.ConstraintName, pgErr.TableName+"_"), "_id_fkey"), ErrNotFound) | ||
case pgerrcode.UniqueViolation: | ||
return fmt.Errorf("%s: %w", pgErr.Message, ErrConflict) | ||
case pgerrcode.IntegrityConstraintViolation, | ||
pgerrcode.RestrictViolation, | ||
pgerrcode.NotNullViolation, | ||
pgerrcode.CheckViolation, | ||
pgerrcode.ExclusionViolation: | ||
return fmt.Errorf("%s: %w", pgErr.Message, ErrConstraint) | ||
default: | ||
} | ||
} else if IsNotFound(err) { | ||
return ErrNotFound | ||
} | ||
return err | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters