go implicitNull: Add support for converting zero value fields to NULL

Summary: Adds a new tag `sql:",implicitNull"` we can use on thunder fields to mark that the "Zero" value of a field should be treated as NULL in the database.
samsarahq · Oct 4, 2018 · 62f12b4 · 62f12b4
1 parent afe9314
commit 62f12b4
Show file tree

Hide file tree

Showing 4 changed files with 264 additions and 14 deletions.
diff --git a/internal/fields/sql.go b/internal/fields/sql.go
@@ -77,6 +77,10 @@ func (f Valuer) Value() (driver.Value, error) {
 			return iface.MarshalJSON()
 		}
 		return json.Marshal(i)
+	case f.Tags.Contains("implicitNull"):
+		if IsZero(f.value) {
+			return nil, nil
+		}
 	}
 
 	// At this point we have already handled `nil` above, so we can assume that all

diff --git a/sqlgen/implicitnull_test.go b/sqlgen/implicitnull_test.go
@@ -0,0 +1,238 @@
+package sqlgen
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/samsarahq/thunder/batch"
+	"github.com/samsarahq/thunder/internal/testfixtures"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func setupImplicitNull() (*testfixtures.TestDatabase, *DB, error) {
+	testDb, err := testfixtures.NewTestDatabase()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if _, err = testDb.Exec(`
+		CREATE TABLE implicitnull (
+			id   BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+			null_str VARCHAR(255),
+			null_int BIGINT,
+			null_float DOUBLE,
+			null_bool TINYINT(1),
+			null_byte BLOB,
+			null_time DATETIME(6)
+		)
+	`); err != nil {
+		return nil, nil, err
+	}
+	schema := NewSchema()
+	schema.MustRegisterType("implicitnull", AutoIncrement, ImplicitNull{})
+
+	return testDb, NewDB(testDb.DB, schema), nil
+}
+
+type ImplicitNull struct {
+	Id        int64     `sql:",primary"`
+	NullStr   string    `sql:",implicitNull"`
+	NullInt   int64     `sql:",implicitNull"`
+	NullFloat float64   `sql:",implicitNull"`
+	NullBool  bool      `sql:",implicitNull"`
+	NullByte  []byte    `sql:",implicitNull"`
+	NullTime  time.Time `sql:",implicitNull"`
+}
+
+func TestImplicitNullIsNullInDB(t *testing.T) {
+	tdb, db, err := setupImplicitNull()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tdb.Close()
+
+	newRow := &ImplicitNull{}
+	if _, err := db.InsertRow(context.Background(), newRow); err != nil {
+		t.Error(err)
+	}
+
+	count, err := GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_str IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 1, count, "Expected null_str zero value to become null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_int IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 1, count, "Expected null_int zero value to become null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_float IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 1, count, "Expected null_float zero value to become null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_bool IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 1, count, "Expected null_bool zero value to become null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_byte IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 1, count, "Expected null_byte zero value to become null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_time IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 1, count, "Expected null_time zero value to become null")
+
+	var rows []*ImplicitNull
+	require.NoError(t, db.Query(context.Background(), &rows, nil, nil))
+	assert.Equal(t, []*ImplicitNull{
+		{
+			Id: 1,
+			// All other fields should be the "zero" value
+		},
+	}, rows)
+}
+
+func TestImplicitNullIsNotNullInDB(t *testing.T) {
+	tdb, db, err := setupImplicitNull()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tdb.Close()
+
+	timeField := time.Date(2000, 10, 3, 0, 0, 0, 0, time.UTC)
+	newRow := &ImplicitNull{
+		NullStr:   "hello",
+		NullInt:   123,
+		NullFloat: 1.23,
+		NullBool:  true,
+		NullByte:  []byte("hello there"),
+		NullTime:  timeField,
+	}
+	if _, err := db.InsertRow(context.Background(), newRow); err != nil {
+		t.Error(err)
+	}
+
+	count, err := GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_str IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 0, count, "Expected null_str to not be null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_int IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 0, count, "Expected null_int to not be null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_float IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 0, count, "Expected null_float to not be null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_bool IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 0, count, "Expected null_bool to not be null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_byte IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 0, count, "Expected null_byte to not be null")
+
+	count, err = GetCount(db.Conn, "SELECT COUNT(*) FROM implicitnull WHERE null_time IS NULL")
+	require.NoError(t, err)
+	assert.Equal(t, 0, count, "Expected null_time to not be null")
+
+	newRow.Id = 1
+
+	var rows []*ImplicitNull
+	require.NoError(t, db.Query(context.Background(), &rows, nil, nil))
+	assert.Equal(t, []*ImplicitNull{
+		newRow,
+	}, rows)
+}
+
+func GetCount(db *sql.DB, countQuery string) (int, error) {
+	dbrows, err := db.Query(countQuery)
+	if err != nil {
+		return 0, err
+	}
+
+	var c int
+	for dbrows.Next() {
+		if err := dbrows.Scan(&c); err != nil {
+			return 0, err
+		}
+	}
+
+	return c, nil
+}
+
+func TestInvalidImplicitNullField(t *testing.T) {
+	testDb, err := testfixtures.NewTestDatabase()
+	require.NoError(t, err)
+	defer testDb.Close()
+
+	type InvalidImplicitNull struct {
+		Id      int64   `sql:",primary"`
+		NullStr *string `sql:",implicitNull"` // Should not allow pointers
+	}
+
+	_, err = testDb.Exec(`
+		CREATE TABLE invalidimplicitnull (
+			id   BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+			null_str VARCHAR(255)
+		)
+	`)
+	require.NoError(t, err)
+
+	schema := NewSchema()
+	err = schema.RegisterType("invalidimplicitnull", AutoIncrement, InvalidImplicitNull{})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "column null_str cannot use `implicitNull` with pointer type")
+}
+
+func TestImplicitNullFiltersWorkForNonNull(t *testing.T) {
+	tdb, db, err := setupImplicitNull()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tdb.Close()
+
+	timeField := time.Date(2000, 10, 3, 0, 0, 0, 0, time.UTC)
+	newRow := &ImplicitNull{
+		NullStr:   "hello",
+		NullInt:   123,
+		NullFloat: 1.23,
+		NullBool:  true,
+		NullByte:  []byte("hello there"),
+		NullTime:  timeField,
+	}
+	if _, err := db.InsertRow(context.Background(), newRow); err != nil {
+		t.Error(err)
+	}
+
+	testCtx := []struct {
+		ctx context.Context
+	}{
+		{batch.WithBatching(context.Background())},
+		{context.Background()},
+	}
+
+	for _, tt := range testCtx {
+		t.Run("", func(t *testing.T) {
+			ctx := tt.ctx
+			var imp *ImplicitNull
+
+			require.NoError(t, db.QueryRow(ctx, &imp, Filter{"id": int64(1)}, nil))
+			require.NoError(t, db.QueryRow(ctx, &imp, Filter{"null_str": "hello"}, nil))
+			require.NoError(t, db.QueryRow(ctx, &imp, Filter{"null_int": int64(123)}, nil))
+			require.NoError(t, db.QueryRow(ctx, &imp, Filter{"null_float": 1.23}, nil))
+			require.NoError(t, db.QueryRow(ctx, &imp, Filter{"null_bool": true}, nil))
+			require.NoError(t, db.QueryRow(ctx, &imp, Filter{"null_byte": []byte("hello there")}, nil))
+			require.NoError(t, db.QueryRow(ctx, &imp, Filter{"null_time": timeField}, nil))
+
+			newRow.Id = 1
+			var rows []*ImplicitNull
+			require.NoError(t, db.Query(context.Background(), &rows, nil, nil))
+			assert.Equal(t, []*ImplicitNull{
+				newRow,
+			}, rows)
+		})
+	}
+}
diff --git a/sqlgen/integration_test.go b/sqlgen/integration_test.go
@@ -23,7 +23,8 @@ func setup() (*testfixtures.TestDatabase, *DB, error) {
 			id   BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
 			name VARCHAR(255),
 			uuid VARCHAR(255),
-			mood VARCHAR(255)
+			mood VARCHAR(255),
+			implicit_null VARCHAR(255)
 		)
 	`); err != nil {
 		return nil, nil, err
@@ -35,18 +36,20 @@ func setup() (*testfixtures.TestDatabase, *DB, error) {
 }
 
 type User struct {
-	Id   int64 `sql:",primary"`
-	Name string
-	Uuid testfixtures.CustomType
-	Mood *testfixtures.CustomType
+	Id           int64 `sql:",primary"`
+	Name         string
+	Uuid         testfixtures.CustomType
+	Mood         *testfixtures.CustomType
+	ImplicitNull string `sql:",implicitNull"`
 }
 
 type Complex struct {
-	Id       int64 `sql:",primary"`
-	Name     string
-	Text     []byte            `sql:",string"`
-	Blob     []byte            `sql:",binary"`
-	Mappings map[string]string `sql:",json"`
+	Id           int64 `sql:",primary"`
+	Name         string
+	Text         []byte            `sql:",string"`
+	Blob         []byte            `sql:",binary"`
+	Mappings     map[string]string `sql:",json"`
+	ImplicitNull string            `sql:",implicitNull"`
 }
 
 func TestTagOverrides(t *testing.T) {
@@ -201,10 +204,11 @@ func Benchmark(b *testing.B) {
 
 	mood := testfixtures.CustomType{'f', 'o', 'o', 'o', 'o', 'o', 'o'}
 	user := &User{
-		Id:   1,
-		Name: "Bob",
-		Uuid: testfixtures.CustomType{'1', '1', '2', '3', '8', '4', '9', '1', '2', '9', '3'},
-		Mood: &mood,
+		Id:           1,
+		Name:         "Bob",
+		Uuid:         testfixtures.CustomType{'1', '1', '2', '3', '8', '4', '9', '1', '2', '9', '3'},
+		Mood:         &mood,
+		ImplicitNull: "test",
 	}
 
 	if _, err := db.InsertRow(ctx, user); err != nil {

diff --git a/sqlgen/reflect.go b/sqlgen/reflect.go
@@ -193,6 +193,10 @@ func (s *Schema) buildDescriptor(table string, primaryKeyType PrimaryKeyType, ty
 					primary = true
 				case "binary", "json", "string":
 					// Do nothing, fields will handle these.
+				case "implicitNull":
+					if field.Type.Kind() == reflect.Ptr {
+						return nil, fmt.Errorf("bad type %s: column %s cannot use `implicitNull` with pointer type", typ, column)
+					}
 				default:
 					return nil, fmt.Errorf("bad type %s: column %s has unexpected tag %s", typ, column, tag)
 				}