Skip to content

Commit

Permalink
codec: support time.Time natively in all formats, and remove Builtin …
Browse files Browse the repository at this point in the history 
…mode.

All codec's now have native support for time.Time, either as a builtin
type, an extension or a convention (e.g. json using RFC3339).

- json: RFC3339
- cbor: timestamp extension
- msgpack: timestamp extension
- binc: built-in time type
- simple: built-in time type

Consequently, it is ok to remove the "IsBuiltin" functionality
and assume all codecs support time.Time.

We also removed builtin functionality from taking effect.
All formats say that they do not support builtins.

However, to ensure that consuming libraries that have
codecgen'ed files do not need to re-generate, we allowed
the functionality as stubs so the files compile but the
code path is never run.
  • Loading branch information
@ugorji
ugorji committed Nov 12, 2017
1 parent 5a66da2 commit debb8e2
Show file tree
Hide file tree
Showing 12 changed files with 406 additions and 74 deletions.
70 changes: 40 additions & 30 deletions codec/binc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,26 +63,31 @@ type bincEncDriver struct {
s uint16 // symbols sequencer
// encNoSeparator
encDriverNoopContainerWriter
noBuiltInTypes
}

// func (e *bincEncDriver) IsBuiltinType(rt uintptr) bool {
// return rt == timeTypId
// }

func (e *bincEncDriver) EncodeBuiltin(rt uintptr, v interface{}) {
if rt == timeTypId {
bs := encodeTime(v.(time.Time))
e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs)))
e.w.writeb(bs)
return
}
e.e.errorf("binc error encoding builtin: expect time.Time, received %T", v)
}

func (e *bincEncDriver) EncodeNil() {
e.w.writen1(bincVdSpecial<<4 | bincSpNil)
}

// func (e *bincEncDriver) EncodeBuiltin(rt uintptr, v interface{}) {
// if rt == timeTypId {
// e.EncodeTime(v.(time.Time))
// return
// }
// e.e.errorf("binc error encoding builtin: expect time.Time, received %T", v)
// }

func (e *bincEncDriver) EncodeTime(t time.Time) {
bs := encodeTime(t)
e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs)))
e.w.writeb(bs)
}

func (e *bincEncDriver) EncodeBool(b bool) {
if b {
e.w.writen1(bincVdSpecial<<4 | bincSpTrue)
Expand Down Expand Up @@ -335,6 +340,7 @@ type bincDecDriver struct {
// because we typically expect < 32 symbols in each stream.
s []bincDecSymbol
decDriverNoopContainerReader
noBuiltInTypes
}

func (d *bincDecDriver) readNextBd() {
Expand Down Expand Up @@ -387,27 +393,31 @@ func (d *bincDecDriver) TryDecodeAsNil() bool {
// return rt == timeTypId
// }

func (d *bincDecDriver) DecodeBuiltin(rt uintptr, v interface{}) {
func (d *bincDecDriver) DecodeTime() (tt time.Time) {
if !d.bdRead {
d.readNextBd()
}
if rt == timeTypId {
if d.vd != bincVdTimestamp {
d.d.errorf("Invalid d.vd. Expecting 0x%x. Received: 0x%x", bincVdTimestamp, d.vd)
return
}
tt, err := decodeTime(d.r.readx(int(d.vs)))
if err != nil {
panic(err)
}
var vt = v.(*time.Time)
*vt = tt
d.bdRead = false
if d.vd != bincVdTimestamp {
d.d.errorf("Invalid d.vd. Expecting 0x%x. Received: 0x%x", bincVdTimestamp, d.vd)
return
}
d.d.errorf("binc error decoding builtin: expect *time.Time, received %T", v)
tt, err := decodeTime(d.r.readx(int(d.vs)))
if err != nil {
panic(err)
}
d.bdRead = false
return
}

// func (d *bincDecDriver) DecodeBuiltin(rt uintptr, v interface{}) {
// if rt == timeTypId {
// var vt = v.(*time.Time)
// *vt = d.DecodeTime()
// return
// }
// d.d.errorf("binc error decoding builtin: expect *time.Time, received %T", v)
// }

func (d *bincDecDriver) decFloatPre(vs, defaultLen byte) {
if vs&0x8 == 0 {
d.r.readb(d.b[0:defaultLen])
Expand Down Expand Up @@ -860,7 +870,7 @@ func (d *bincDecDriver) DecodeNaked() {
n.v = valueTypeBytes
n.l = d.DecodeBytes(nil, false)
case bincVdTimestamp:
n.v = valueTypeTimestamp
n.v = valueTypeTime
tt, err := decodeTime(d.r.readx(int(d.vs)))
if err != nil {
panic(err)
Expand Down Expand Up @@ -924,11 +934,11 @@ func (h *BincHandle) newDecDriver(d *Decoder) decDriver {
return &bincDecDriver{d: d, h: h, r: d.r, br: d.bytes}
}

// IsBuiltinType returns true for time.Time, else false.
// only time.Time is builtin.
func (h *BincHandle) IsBuiltinType(rt uintptr) bool {
return rt == timeTypId
}
// // IsBuiltinType returns true for time.Time, else false.
// // only time.Time is builtin.
// func (h *BincHandle) IsBuiltinType(rt uintptr) bool {
// return rt == timeTypId
// }

func (e *bincEncDriver) reset() {
e.w = e.e.w
Expand Down
82 changes: 82 additions & 0 deletions codec/cbor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package codec
import (
"math"
"reflect"
"time"
)

const (
Expand Down Expand Up @@ -126,6 +127,30 @@ func (e *cborEncDriver) encLen(bd byte, length int) {
e.encUint(uint64(length), bd)
}

func (e *cborEncDriver) EncodeTime(t time.Time) {
if e.h.TimeRFC3339 {
e.encUint(0, cborBaseTag)
e.EncodeString(cUTF8, t.Format(time.RFC3339Nano))
} else {
e.encUint(1, cborBaseTag)
// fmt.Printf(">>>> - encoding time: %v\n", t)
t = t.UTC().Round(0).Round(time.Microsecond)
// fmt.Printf(">>>> - encoding time: %v\n", t)
sec, nsec := t.Unix(), uint64(t.Nanosecond())
if nsec == 0 {
e.EncodeInt(sec)
// fmt.Printf(">>>> i encoding time using: %v\n", sec)
} else {
e.EncodeFloat64(float64(sec) + float64(nsec)/1e9)
// round nsec to microseconds, so it fits into float64 without losing resolution
// e.EncodeFloat64(float64(sec) + round(float64(nsec)/1e3)/1e6)
// e.EncodeFloat64(float64(sec) + float64(nsec/1e3)/1e6)
// fmt.Printf(">>>> f encoding time using: %v + %v = %v, nsec: %v, \n",
// float64(sec), round(float64(nsec)/1e3)/1e6, float64(sec)+round(float64(nsec)/1e3)/1e6, nsec)
}
}
}

func (e *cborEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, en *Encoder) {
e.encUint(uint64(xtag), cborBaseTag)
if v := ext.ConvertExt(rv); v == nil {
Expand Down Expand Up @@ -497,6 +522,54 @@ func (d *cborDecDriver) DecodeStringAsBytes() (s []byte) {
return d.DecodeBytes(d.b[:], true)
}

func (d *cborDecDriver) DecodeTime() (t time.Time) {
if !d.bdRead {
d.readNextBd()
}
xtag := d.decUint()
d.bdRead = false
return d.decodeTime(xtag)
}

func (d *cborDecDriver) decodeTime(xtag uint64) (t time.Time) {
if !d.bdRead {
d.readNextBd()
}
switch xtag {
case 0:
var err error
if t, err = time.Parse(time.RFC3339, stringView(d.DecodeStringAsBytes())); err != nil {
d.d.error(err)
}
case 1:
// decode an int64 or a float, and infer time.Time from there.
// for floats, round to microseconds, as that is what is guaranteed to fit well.
switch {
case d.bd == cborBdFloat16, d.bd == cborBdFloat32:
f1, f2 := math.Modf(d.DecodeFloat(true))
// t = time.Unix(int64(f1), int64(round(f2*1e6))*1e3).Round(time.Microsecond).UTC()
t = time.Unix(int64(f1), int64(f2*1e9))
case d.bd == cborBdFloat64:
f1, f2 := math.Modf(d.DecodeFloat(false))
// fmt.Printf(">>>> f decoding time using: %v + %v --> %v %v\n",
// f1, f2, int64(f2*1e9), int64(round(f2*1e6))*1e3)
// t = time.Unix(int64(f1), (int64(f2*1e9)/1e3)*1e3).UTC()
// t = time.Unix(int64(f1), int64(round(f2*1e6))*1e3).Round(time.Microsecond).UTC()
t = time.Unix(int64(f1), int64(f2*1e9))
// fmt.Printf(">>>> f decoding time using: %v + %v = %v\n", t.Unix(), t.Nanosecond(), t)
case d.bd >= cborBaseUint && d.bd < cborBaseNegInt, d.bd >= cborBaseNegInt && d.bd < cborBaseBytes:
t = time.Unix(d.DecodeInt(64), 0)
// fmt.Printf(">>>> i decoding time using: %v + %v = %v\n", t.Unix(), t.Nanosecond(), t)
default:
d.d.errorf("cbor: time.Time can only be decoded from a number (or RFC3339 string)")
}
default:
d.d.errorf("cbor: invalid tag for time.Time - expecting 0 or 1, got 0x%x", xtag)
}
t = t.Round(time.Microsecond).UTC()
return
}

func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
if !d.bdRead {
d.readNextBd()
Expand Down Expand Up @@ -584,6 +657,11 @@ func (d *cborDecDriver) DecodeNaked() {
n.v = valueTypeExt
n.u = d.decUint()
n.l = nil
if n.u == 0 || n.u == 1 {
d.bdRead = false
n.v = valueTypeTime
n.t = d.decodeTime(n.u)
}
// d.bdRead = false
// d.d.decode(&re.Value) // handled by decode itself.
// decodeFurther = true
Expand Down Expand Up @@ -623,6 +701,10 @@ type CborHandle struct {

// IndefiniteLength=true, means that we encode using indefinitelength
IndefiniteLength bool

// TimeRFC3339 says to encode time.Time using RFC3339 format.
// If unset, we encode time.Time using seconds past epoch.
TimeRFC3339 bool
}

// SetInterfaceExt sets an extension
Expand Down
14 changes: 11 additions & 3 deletions codec/codec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,8 @@ func testTableVerify(f testVerifyFlag, h Handle) (av []interface{}) {
av[i] = testVerifyVal(v, f, h)
case map[interface{}]interface{}:
av[i] = testVerifyVal(v, f, h)
case time.Time:
av[i] = testVerifyVal(v, f, h)
default:
av[i] = v
}
Expand Down Expand Up @@ -447,6 +449,7 @@ func testVerifyVal(v interface{}, f testVerifyFlag, h Handle) (v2 interface{}) {
// - all positive integers are unsigned 64-bit ints
// - all floats are float64
_, isMsgp := h.(*MsgpackHandle)
_, isCbor := h.(*CborHandle)
switch iv := v.(type) {
case int8:
v2 = testVerifyValInt(int64(iv), isMsgp)
Expand Down Expand Up @@ -537,6 +540,11 @@ func testVerifyVal(v interface{}, f testVerifyFlag, h Handle) (v2 interface{}) {
} else {
v2 = int64(iv2)
}
case isMsgp:
v2 = iv.UTC()
case isCbor:
// fmt.Printf("%%%% cbor verifier\n")
v2 = iv.UTC().Round(0).Round(time.Microsecond)
default:
v2 = v
}
Expand Down Expand Up @@ -626,7 +634,7 @@ func doTestCodecTableOne(t *testing.T, testNil bool, h Handle,
}
}

logT(t, " v1 returned: %T, %#v", v1, v1)
logT(t, " v1 returned: %T, %v %#v", v1, v1, v1)
// if v1 != nil {
// logT(t, " v1 returned: %T, %#v", v1, v1)
// //we always indirect, because ptr to typed value may be passed (if not testNil)
Expand All @@ -647,8 +655,8 @@ func doTestCodecTableOne(t *testing.T, testNil bool, h Handle,
// logT(t, "-------- Before and After marshal do not match: Error: %v"+
// " ====> GOLDEN: (%T) %#v, DECODED: (%T) %#v\n", err, v0check, v0check, v1, v1)
logT(t, "-------- FAIL: Before and After marshal do not match: Error: %v", err)
logT(t, " ....... GOLDEN: (%T) %#v", v0check, v0check)
logT(t, " ....... DECODED: (%T) %#v", v1, v1)
logT(t, " ....... GOLDEN: (%T) %v %#v", v0check, v0check, v0check)
logT(t, " ....... DECODED: (%T) %v %#v", v1, v1, v1)
failT(t)
}
}
Expand Down
22 changes: 15 additions & 7 deletions codec/decode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ type decDriver interface {
// vt is one of: Bytes, String, Nil, Slice or Map. Return unSet if not known.
ContainerType() (vt valueType)
// IsBuiltinType(rt uintptr) bool

// Deprecated: left here for now so that old codecgen'ed filed will work. TODO: remove.
DecodeBuiltin(rt uintptr, v interface{})

// DecodeNaked will decode primitives (number, bool, string, []byte) and RawExt.
Expand All @@ -80,6 +82,7 @@ type decDriver interface {
DecodeUint(bitsize uint8) (ui uint64)
DecodeFloat(chkOverflow32 bool) (f float64)
DecodeBool() (b bool)
DecodeTime() (t time.Time)
// DecodeString can also decode symbols.
// It looks redundant as DecodeBytes is available.
// However, some codecs (e.g. binc) support symbols and can
Expand Down Expand Up @@ -881,9 +884,9 @@ func (z *bytesDecReader) stopTrack() (bs []byte) {

// ----------------------------------------

func (d *Decoder) builtin(f *codecFnInfo, rv reflect.Value) {
d.d.DecodeBuiltin(f.ti.rtid, rv2i(rv))
}
// func (d *Decoder) builtin(f *codecFnInfo, rv reflect.Value) {
// d.d.DecodeBuiltin(f.ti.rtid, rv2i(rv))
// }

func (d *Decoder) rawExt(f *codecFnInfo, rv reflect.Value) {
d.d.DecodeExt(rv2i(rv), 0, nil)
Expand Down Expand Up @@ -1055,7 +1058,7 @@ func (d *Decoder) kInterfaceNaked(f *codecFnInfo) (rvn reflect.Value) {
rvn = n.rr[decNakedStringIdx] // d.np.get(&n.s)
case valueTypeBytes:
rvn = n.rr[decNakedBytesIdx] // d.np.get(&n.l)
case valueTypeTimestamp:
case valueTypeTime:
rvn = n.rr[decNakedTimeIdx] // d.np.get(&n.t)
default:
panic(fmt.Errorf("kInterfaceNaked: unexpected valueType: %d", n.v))
Expand Down Expand Up @@ -2051,6 +2054,8 @@ func (d *Decoder) decode(iv interface{}) {
case *[]uint8:
*v = d.d.DecodeBytes(*v, false)

case *time.Time:
*v = d.d.DecodeTime()
case *Raw:
*v = d.rawBytes()

Expand Down Expand Up @@ -2192,12 +2197,15 @@ func (d *Decoder) error(err error) {
panic(err)
}

func (d *Decoder) errorf(format string, params ...interface{}) {
func (d *Decoder) errorvf(format string, params ...interface{}) (err error) {
params2 := make([]interface{}, len(params)+1)
params2[0] = d.r.numread()
copy(params2[1:], params)
err := fmt.Errorf("[pos %d]: "+format, params2...)
panic(err)
return fmt.Errorf("[pos %d]: "+format, params2...)
}

func (d *Decoder) errorf(format string, params ...interface{}) {
panic(d.errorvf(format, params...))
}

// Possibly get an interned version of a string
Expand Down
Loading

0 comments on commit debb8e2

Please sign in to comment.