From debb8e2d2e8bd8cf1e9b6d806cf5e58df86b970c Mon Sep 17 00:00:00 2001 From: Ugorji Nwoke Date: Sun, 12 Nov 2017 11:52:56 -0500 Subject: [PATCH] codec: support time.Time natively in all formats, and remove Builtin mode. All codec's now have native support for time.Time, either as a builtin type, an extension or a convention (e.g. json using RFC3339). - json: RFC3339 - cbor: timestamp extension - msgpack: timestamp extension - binc: built-in time type - simple: built-in time type Consequently, it is ok to remove the "IsBuiltin" functionality and assume all codecs support time.Time. We also removed builtin functionality from taking effect. All formats say that they do not support builtins. However, to ensure that consuming libraries that have codecgen'ed files do not need to re-generate, we allowed the functionality as stubs so the files compile but the code path is never run. --- codec/binc.go | 70 +++++++++++++++++------------- codec/cbor.go | 82 +++++++++++++++++++++++++++++++++++ codec/codec_test.go | 14 ++++-- codec/decode.go | 22 +++++++--- codec/encode.go | 45 ++++++++++++++++--- codec/gen.go | 23 +++++----- codec/helper.go | 33 +++++++++++--- codec/helper_not_unsafe.go | 9 ++++ codec/helper_unsafe.go | 11 +++++ codec/json.go | 20 +++++++++ codec/msgpack.go | 89 +++++++++++++++++++++++++++++++++++++- codec/simple.go | 62 +++++++++++++++++++++----- 12 files changed, 406 insertions(+), 74 deletions(-) diff --git a/codec/binc.go b/codec/binc.go index a36b6a97..3298a2b3 100644 --- a/codec/binc.go +++ b/codec/binc.go @@ -63,26 +63,31 @@ type bincEncDriver struct { s uint16 // symbols sequencer // encNoSeparator encDriverNoopContainerWriter + noBuiltInTypes } // func (e *bincEncDriver) IsBuiltinType(rt uintptr) bool { // return rt == timeTypId // } -func (e *bincEncDriver) EncodeBuiltin(rt uintptr, v interface{}) { - if rt == timeTypId { - bs := encodeTime(v.(time.Time)) - e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs))) - e.w.writeb(bs) - return - } - e.e.errorf("binc error encoding builtin: expect time.Time, received %T", v) -} - func (e *bincEncDriver) EncodeNil() { e.w.writen1(bincVdSpecial<<4 | bincSpNil) } +// func (e *bincEncDriver) EncodeBuiltin(rt uintptr, v interface{}) { +// if rt == timeTypId { +// e.EncodeTime(v.(time.Time)) +// return +// } +// e.e.errorf("binc error encoding builtin: expect time.Time, received %T", v) +// } + +func (e *bincEncDriver) EncodeTime(t time.Time) { + bs := encodeTime(t) + e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs))) + e.w.writeb(bs) +} + func (e *bincEncDriver) EncodeBool(b bool) { if b { e.w.writen1(bincVdSpecial<<4 | bincSpTrue) @@ -335,6 +340,7 @@ type bincDecDriver struct { // because we typically expect < 32 symbols in each stream. s []bincDecSymbol decDriverNoopContainerReader + noBuiltInTypes } func (d *bincDecDriver) readNextBd() { @@ -387,27 +393,31 @@ func (d *bincDecDriver) TryDecodeAsNil() bool { // return rt == timeTypId // } -func (d *bincDecDriver) DecodeBuiltin(rt uintptr, v interface{}) { +func (d *bincDecDriver) DecodeTime() (tt time.Time) { if !d.bdRead { d.readNextBd() } - if rt == timeTypId { - if d.vd != bincVdTimestamp { - d.d.errorf("Invalid d.vd. Expecting 0x%x. Received: 0x%x", bincVdTimestamp, d.vd) - return - } - tt, err := decodeTime(d.r.readx(int(d.vs))) - if err != nil { - panic(err) - } - var vt = v.(*time.Time) - *vt = tt - d.bdRead = false + if d.vd != bincVdTimestamp { + d.d.errorf("Invalid d.vd. Expecting 0x%x. Received: 0x%x", bincVdTimestamp, d.vd) return } - d.d.errorf("binc error decoding builtin: expect *time.Time, received %T", v) + tt, err := decodeTime(d.r.readx(int(d.vs))) + if err != nil { + panic(err) + } + d.bdRead = false + return } +// func (d *bincDecDriver) DecodeBuiltin(rt uintptr, v interface{}) { +// if rt == timeTypId { +// var vt = v.(*time.Time) +// *vt = d.DecodeTime() +// return +// } +// d.d.errorf("binc error decoding builtin: expect *time.Time, received %T", v) +// } + func (d *bincDecDriver) decFloatPre(vs, defaultLen byte) { if vs&0x8 == 0 { d.r.readb(d.b[0:defaultLen]) @@ -860,7 +870,7 @@ func (d *bincDecDriver) DecodeNaked() { n.v = valueTypeBytes n.l = d.DecodeBytes(nil, false) case bincVdTimestamp: - n.v = valueTypeTimestamp + n.v = valueTypeTime tt, err := decodeTime(d.r.readx(int(d.vs))) if err != nil { panic(err) @@ -924,11 +934,11 @@ func (h *BincHandle) newDecDriver(d *Decoder) decDriver { return &bincDecDriver{d: d, h: h, r: d.r, br: d.bytes} } -// IsBuiltinType returns true for time.Time, else false. -// only time.Time is builtin. -func (h *BincHandle) IsBuiltinType(rt uintptr) bool { - return rt == timeTypId -} +// // IsBuiltinType returns true for time.Time, else false. +// // only time.Time is builtin. +// func (h *BincHandle) IsBuiltinType(rt uintptr) bool { +// return rt == timeTypId +// } func (e *bincEncDriver) reset() { e.w = e.e.w diff --git a/codec/cbor.go b/codec/cbor.go index 3808a5f4..3d9c46ff 100644 --- a/codec/cbor.go +++ b/codec/cbor.go @@ -6,6 +6,7 @@ package codec import ( "math" "reflect" + "time" ) const ( @@ -126,6 +127,30 @@ func (e *cborEncDriver) encLen(bd byte, length int) { e.encUint(uint64(length), bd) } +func (e *cborEncDriver) EncodeTime(t time.Time) { + if e.h.TimeRFC3339 { + e.encUint(0, cborBaseTag) + e.EncodeString(cUTF8, t.Format(time.RFC3339Nano)) + } else { + e.encUint(1, cborBaseTag) + // fmt.Printf(">>>> - encoding time: %v\n", t) + t = t.UTC().Round(0).Round(time.Microsecond) + // fmt.Printf(">>>> - encoding time: %v\n", t) + sec, nsec := t.Unix(), uint64(t.Nanosecond()) + if nsec == 0 { + e.EncodeInt(sec) + // fmt.Printf(">>>> i encoding time using: %v\n", sec) + } else { + e.EncodeFloat64(float64(sec) + float64(nsec)/1e9) + // round nsec to microseconds, so it fits into float64 without losing resolution + // e.EncodeFloat64(float64(sec) + round(float64(nsec)/1e3)/1e6) + // e.EncodeFloat64(float64(sec) + float64(nsec/1e3)/1e6) + // fmt.Printf(">>>> f encoding time using: %v + %v = %v, nsec: %v, \n", + // float64(sec), round(float64(nsec)/1e3)/1e6, float64(sec)+round(float64(nsec)/1e3)/1e6, nsec) + } + } +} + func (e *cborEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, en *Encoder) { e.encUint(uint64(xtag), cborBaseTag) if v := ext.ConvertExt(rv); v == nil { @@ -497,6 +522,54 @@ func (d *cborDecDriver) DecodeStringAsBytes() (s []byte) { return d.DecodeBytes(d.b[:], true) } +func (d *cborDecDriver) DecodeTime() (t time.Time) { + if !d.bdRead { + d.readNextBd() + } + xtag := d.decUint() + d.bdRead = false + return d.decodeTime(xtag) +} + +func (d *cborDecDriver) decodeTime(xtag uint64) (t time.Time) { + if !d.bdRead { + d.readNextBd() + } + switch xtag { + case 0: + var err error + if t, err = time.Parse(time.RFC3339, stringView(d.DecodeStringAsBytes())); err != nil { + d.d.error(err) + } + case 1: + // decode an int64 or a float, and infer time.Time from there. + // for floats, round to microseconds, as that is what is guaranteed to fit well. + switch { + case d.bd == cborBdFloat16, d.bd == cborBdFloat32: + f1, f2 := math.Modf(d.DecodeFloat(true)) + // t = time.Unix(int64(f1), int64(round(f2*1e6))*1e3).Round(time.Microsecond).UTC() + t = time.Unix(int64(f1), int64(f2*1e9)) + case d.bd == cborBdFloat64: + f1, f2 := math.Modf(d.DecodeFloat(false)) + // fmt.Printf(">>>> f decoding time using: %v + %v --> %v %v\n", + // f1, f2, int64(f2*1e9), int64(round(f2*1e6))*1e3) + // t = time.Unix(int64(f1), (int64(f2*1e9)/1e3)*1e3).UTC() + // t = time.Unix(int64(f1), int64(round(f2*1e6))*1e3).Round(time.Microsecond).UTC() + t = time.Unix(int64(f1), int64(f2*1e9)) + // fmt.Printf(">>>> f decoding time using: %v + %v = %v\n", t.Unix(), t.Nanosecond(), t) + case d.bd >= cborBaseUint && d.bd < cborBaseNegInt, d.bd >= cborBaseNegInt && d.bd < cborBaseBytes: + t = time.Unix(d.DecodeInt(64), 0) + // fmt.Printf(">>>> i decoding time using: %v + %v = %v\n", t.Unix(), t.Nanosecond(), t) + default: + d.d.errorf("cbor: time.Time can only be decoded from a number (or RFC3339 string)") + } + default: + d.d.errorf("cbor: invalid tag for time.Time - expecting 0 or 1, got 0x%x", xtag) + } + t = t.Round(time.Microsecond).UTC() + return +} + func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) { if !d.bdRead { d.readNextBd() @@ -584,6 +657,11 @@ func (d *cborDecDriver) DecodeNaked() { n.v = valueTypeExt n.u = d.decUint() n.l = nil + if n.u == 0 || n.u == 1 { + d.bdRead = false + n.v = valueTypeTime + n.t = d.decodeTime(n.u) + } // d.bdRead = false // d.d.decode(&re.Value) // handled by decode itself. // decodeFurther = true @@ -623,6 +701,10 @@ type CborHandle struct { // IndefiniteLength=true, means that we encode using indefinitelength IndefiniteLength bool + + // TimeRFC3339 says to encode time.Time using RFC3339 format. + // If unset, we encode time.Time using seconds past epoch. + TimeRFC3339 bool } // SetInterfaceExt sets an extension diff --git a/codec/codec_test.go b/codec/codec_test.go index 4dd7d1c3..940dc943 100644 --- a/codec/codec_test.go +++ b/codec/codec_test.go @@ -419,6 +419,8 @@ func testTableVerify(f testVerifyFlag, h Handle) (av []interface{}) { av[i] = testVerifyVal(v, f, h) case map[interface{}]interface{}: av[i] = testVerifyVal(v, f, h) + case time.Time: + av[i] = testVerifyVal(v, f, h) default: av[i] = v } @@ -447,6 +449,7 @@ func testVerifyVal(v interface{}, f testVerifyFlag, h Handle) (v2 interface{}) { // - all positive integers are unsigned 64-bit ints // - all floats are float64 _, isMsgp := h.(*MsgpackHandle) + _, isCbor := h.(*CborHandle) switch iv := v.(type) { case int8: v2 = testVerifyValInt(int64(iv), isMsgp) @@ -537,6 +540,11 @@ func testVerifyVal(v interface{}, f testVerifyFlag, h Handle) (v2 interface{}) { } else { v2 = int64(iv2) } + case isMsgp: + v2 = iv.UTC() + case isCbor: + // fmt.Printf("%%%% cbor verifier\n") + v2 = iv.UTC().Round(0).Round(time.Microsecond) default: v2 = v } @@ -626,7 +634,7 @@ func doTestCodecTableOne(t *testing.T, testNil bool, h Handle, } } - logT(t, " v1 returned: %T, %#v", v1, v1) + logT(t, " v1 returned: %T, %v %#v", v1, v1, v1) // if v1 != nil { // logT(t, " v1 returned: %T, %#v", v1, v1) // //we always indirect, because ptr to typed value may be passed (if not testNil) @@ -647,8 +655,8 @@ func doTestCodecTableOne(t *testing.T, testNil bool, h Handle, // logT(t, "-------- Before and After marshal do not match: Error: %v"+ // " ====> GOLDEN: (%T) %#v, DECODED: (%T) %#v\n", err, v0check, v0check, v1, v1) logT(t, "-------- FAIL: Before and After marshal do not match: Error: %v", err) - logT(t, " ....... GOLDEN: (%T) %#v", v0check, v0check) - logT(t, " ....... DECODED: (%T) %#v", v1, v1) + logT(t, " ....... GOLDEN: (%T) %v %#v", v0check, v0check, v0check) + logT(t, " ....... DECODED: (%T) %v %#v", v1, v1, v1) failT(t) } } diff --git a/codec/decode.go b/codec/decode.go index 02fca780..a0370d89 100644 --- a/codec/decode.go +++ b/codec/decode.go @@ -62,6 +62,8 @@ type decDriver interface { // vt is one of: Bytes, String, Nil, Slice or Map. Return unSet if not known. ContainerType() (vt valueType) // IsBuiltinType(rt uintptr) bool + + // Deprecated: left here for now so that old codecgen'ed filed will work. TODO: remove. DecodeBuiltin(rt uintptr, v interface{}) // DecodeNaked will decode primitives (number, bool, string, []byte) and RawExt. @@ -80,6 +82,7 @@ type decDriver interface { DecodeUint(bitsize uint8) (ui uint64) DecodeFloat(chkOverflow32 bool) (f float64) DecodeBool() (b bool) + DecodeTime() (t time.Time) // DecodeString can also decode symbols. // It looks redundant as DecodeBytes is available. // However, some codecs (e.g. binc) support symbols and can @@ -881,9 +884,9 @@ func (z *bytesDecReader) stopTrack() (bs []byte) { // ---------------------------------------- -func (d *Decoder) builtin(f *codecFnInfo, rv reflect.Value) { - d.d.DecodeBuiltin(f.ti.rtid, rv2i(rv)) -} +// func (d *Decoder) builtin(f *codecFnInfo, rv reflect.Value) { +// d.d.DecodeBuiltin(f.ti.rtid, rv2i(rv)) +// } func (d *Decoder) rawExt(f *codecFnInfo, rv reflect.Value) { d.d.DecodeExt(rv2i(rv), 0, nil) @@ -1055,7 +1058,7 @@ func (d *Decoder) kInterfaceNaked(f *codecFnInfo) (rvn reflect.Value) { rvn = n.rr[decNakedStringIdx] // d.np.get(&n.s) case valueTypeBytes: rvn = n.rr[decNakedBytesIdx] // d.np.get(&n.l) - case valueTypeTimestamp: + case valueTypeTime: rvn = n.rr[decNakedTimeIdx] // d.np.get(&n.t) default: panic(fmt.Errorf("kInterfaceNaked: unexpected valueType: %d", n.v)) @@ -2051,6 +2054,8 @@ func (d *Decoder) decode(iv interface{}) { case *[]uint8: *v = d.d.DecodeBytes(*v, false) + case *time.Time: + *v = d.d.DecodeTime() case *Raw: *v = d.rawBytes() @@ -2192,12 +2197,15 @@ func (d *Decoder) error(err error) { panic(err) } -func (d *Decoder) errorf(format string, params ...interface{}) { +func (d *Decoder) errorvf(format string, params ...interface{}) (err error) { params2 := make([]interface{}, len(params)+1) params2[0] = d.r.numread() copy(params2[1:], params) - err := fmt.Errorf("[pos %d]: "+format, params2...) - panic(err) + return fmt.Errorf("[pos %d]: "+format, params2...) +} + +func (d *Decoder) errorf(format string, params ...interface{}) { + panic(d.errorvf(format, params...)) } // Possibly get an interned version of a string diff --git a/codec/encode.go b/codec/encode.go index cbaac039..63f6f45f 100644 --- a/codec/encode.go +++ b/codec/encode.go @@ -11,6 +11,7 @@ import ( "reflect" "sort" "sync" + "time" ) const defEncByteBufSize = 1 << 6 // 4:16, 6:64, 8:256, 10:1024 @@ -47,6 +48,8 @@ type encWriter interface { // encDriver abstracts the actual codec (binc vs msgpack, etc) type encDriver interface { // IsBuiltinType(rt uintptr) bool + + // Deprecated: left here for now so that old codecgen'ed filed will work. TODO: remove. EncodeBuiltin(rt uintptr, v interface{}) EncodeNil() EncodeInt(i int64) @@ -67,7 +70,7 @@ type encDriver interface { EncodeString(c charEncoding, v string) EncodeSymbol(v string) EncodeStringBytes(c charEncoding, v []byte) - + EncodeTime(time.Time) //TODO //encBignum(f *big.Int) //encStringRunes(c charEncoding, v []rune) @@ -338,9 +341,9 @@ func (z *bytesEncWriter) growAlloc(n int, oldcursor int) { // --------------------------------------------- -func (e *Encoder) builtin(f *codecFnInfo, rv reflect.Value) { - e.e.EncodeBuiltin(f.ti.rtid, rv2i(rv)) -} +// func (e *Encoder) builtin(f *codecFnInfo, rv reflect.Value) { +// e.e.EncodeBuiltin(f.ti.rtid, rv2i(rv)) +// } func (e *Encoder) rawExt(f *codecFnInfo, rv reflect.Value) { // rev := rv2i(rv).(RawExt) @@ -913,6 +916,28 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl } e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true) } + case reflect.Struct: + if rv.Type() == timeTyp { + mksv := make([]timeRv, len(mks)) + for i, k := range mks { + v := &mksv[i] + v.r = k + v.v = rv2i(k).(time.Time) + } + sort.Sort(timeRvSlice(mksv)) + for i := range mksv { + if elemsep { + ee.WriteMapElemKey() + } + ee.EncodeTime(mksv[i].v) + if elemsep { + ee.WriteMapElemValue() + } + e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true) + } + break + } + fallthrough default: // out-of-band // first encode each key to a []byte first, then sort them, then record @@ -1169,10 +1194,14 @@ func (e *Encoder) encode(iv interface{}) { e.e.EncodeFloat32(v) case float64: e.e.EncodeFloat64(v) - + case time.Time: + e.e.EncodeTime(v) case []uint8: e.e.EncodeStringBytes(cRAW, v) + case *Raw: + e.rawBytes(*v) + case *string: e.e.EncodeString(cUTF8, *v) case *bool: @@ -1203,6 +1232,8 @@ func (e *Encoder) encode(iv interface{}) { e.e.EncodeFloat32(*v) case *float64: e.e.EncodeFloat64(*v) + case *time.Time: + e.e.EncodeTime(*v) case *[]uint8: e.e.EncodeStringBytes(cRAW, *v) @@ -1317,3 +1348,7 @@ func (e *Encoder) errorf(format string, params ...interface{}) { err := fmt.Errorf(format, params...) panic(err) } + +func (e *Encoder) error(err error) { + panic(err) +} diff --git a/codec/gen.go b/codec/gen.go index d8c1def2..19a7b2f6 100644 --- a/codec/gen.go +++ b/codec/gen.go @@ -30,7 +30,6 @@ import ( // codecgen supports the full cycle of reflection-based codec: // - RawExt // - Raw -// - Builtins // - Extensions // - (Binary|Text|JSON)(Unm|M)arshal // - generic by-kind @@ -709,11 +708,12 @@ func (x *genRunner) enc(varname string, t reflect.Type) { // HACK: Support for Builtins. // Currently, only Binc supports builtins, and the only builtin type is time.Time. // Have a method that returns the rtid for time.Time if Handle is Binc. - if t == timeTyp { - vrtid := genTempVarPfx + "m" + x.varsfx() - x.linef("} else if %s := z.TimeRtidIfBinc(); %s != 0 { ", vrtid, vrtid) - x.linef("r.EncodeBuiltin(%s, *%s)", vrtid, varname) - } + // 2017-11-12: builtin no longer supported - comment out + // if t == timeTyp { + // vrtid := genTempVarPfx + "m" + x.varsfx() + // x.linef("} else if %s := z.TimeRtidIfBinc(); %s != 0 { ", vrtid, vrtid) + // x.linef("r.EncodeBuiltin(%s, *%s)", vrtid, varname) + // } // only check for extensions if the type is named, and has a packagePath. if !x.nx && genImportPath(t) != "" && t.Name() != "" { // first check if extensions are configued, before doing the interface conversion @@ -1162,11 +1162,12 @@ func (x *genRunner) dec(varname string, t reflect.Type) { // HACK: Support for Builtins. // Currently, only Binc supports builtins, and the only builtin type is time.Time. // Have a method that returns the rtid for time.Time if Handle is Binc. - if t == timeTyp { - vrtid := genTempVarPfx + "m" + x.varsfx() - x.linef("} else if %s := z.TimeRtidIfBinc(); %s != 0 { ", vrtid, vrtid) - x.linef("r.DecodeBuiltin(%s, %s)", vrtid, varname) - } + // 2017-11-12: builtin no longer supported - comment out + // if t == timeTyp { + // vrtid := genTempVarPfx + "m" + x.varsfx() + // x.linef("} else if %s := z.TimeRtidIfBinc(); %s != 0 { ", vrtid, vrtid) + // x.linef("r.DecodeBuiltin(%s, %s)", vrtid, varname) + // } // only check for extensions if the type is named, and has a packagePath. if !x.nx && genImportPath(t) != "" && t.Name() != "" { // first check if extensions are configued, before doing the interface conversion diff --git a/codec/helper.go b/codec/helper.go index 985e4258..116f2188 100644 --- a/codec/helper.go +++ b/codec/helper.go @@ -184,7 +184,7 @@ const ( valueTypeBytes valueTypeMap valueTypeArray - valueTypeTimestamp + valueTypeTime valueTypeExt // valueTypeInvalid = 0xff @@ -1213,6 +1213,14 @@ func implIntf(rt, iTyp reflect.Type) (base bool, indir bool) { return rt.Implements(iTyp), reflect.PtrTo(rt).Implements(iTyp) } +// func round(x float64) float64 { +// t := math.Trunc(x) +// if math.Abs(x-t) >= 0.5 { +// return t + math.Copysign(1, x) +// } +// return t +// } + func xprintf(format string, a ...interface{}) { if xDebug { fmt.Fprintf(os.Stderr, format, a...) @@ -1364,6 +1372,9 @@ func (c *codecFner) get(rt reflect.Type, checkFastpath, checkCodecSelfer bool) ( fi.addrF = true fi.addrD = ti.csp fi.addrE = ti.csp + } else if rtid == timeTypId { + fn.fe = (*Encoder).kTime + fn.fd = (*Decoder).kTime } else if rtid == rawTypId { fn.fe = (*Encoder).raw fn.fd = (*Decoder).raw @@ -1373,11 +1384,12 @@ func (c *codecFner) get(rt reflect.Type, checkFastpath, checkCodecSelfer bool) ( fi.addrF = true fi.addrD = true fi.addrE = true - } else if c.hh.IsBuiltinType(rtid) { - fn.fe = (*Encoder).builtin - fn.fd = (*Decoder).builtin - fi.addrF = true - fi.addrD = true + } else if false && c.hh.IsBuiltinType(rtid) { + // TODO: remove this whole block. currently turned off with the "false &&" + // fn.fe = (*Encoder).builtin + // fn.fd = (*Decoder).builtin + // fi.addrF = true + // fi.addrD = true } else if xfFn := c.h.getExt(rtid); xfFn != nil { fi.xfTag, fi.xfFn = xfFn.tag, xfFn.ext fn.fe = (*Encoder).ext @@ -1673,6 +1685,11 @@ type bytesRv struct { r reflect.Value } type bytesRvSlice []bytesRv +type timeRv struct { + v time.Time + r reflect.Value +} +type timeRvSlice []timeRv func (p intRvSlice) Len() int { return len(p) } func (p intRvSlice) Less(i, j int) bool { return p[i].v < p[j].v } @@ -1700,6 +1717,10 @@ func (p boolRvSlice) Len() int { return len(p) } func (p boolRvSlice) Less(i, j int) bool { return !p[i].v && p[j].v } func (p boolRvSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p timeRvSlice) Len() int { return len(p) } +func (p timeRvSlice) Less(i, j int) bool { return p[i].v.Before(p[j].v) } +func (p timeRvSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + // ----------------- type bytesI struct { diff --git a/codec/helper_not_unsafe.go b/codec/helper_not_unsafe.go index caee852b..9d229363 100644 --- a/codec/helper_not_unsafe.go +++ b/codec/helper_not_unsafe.go @@ -8,6 +8,7 @@ package codec import ( "reflect" "sync/atomic" + "time" ) const safeMode = true @@ -107,6 +108,10 @@ func (d *Decoder) kBool(f *codecFnInfo, rv reflect.Value) { rv.SetBool(d.d.DecodeBool()) } +func (d *Decoder) kTime(f *codecFnInfo, rv reflect.Value) { + rv.Set(reflect.ValueOf(d.d.DecodeTime())) +} + func (d *Decoder) kFloat32(f *codecFnInfo, rv reflect.Value) { rv.SetFloat(d.d.DecodeFloat(true)) } @@ -165,6 +170,10 @@ func (e *Encoder) kBool(f *codecFnInfo, rv reflect.Value) { e.e.EncodeBool(rv.Bool()) } +func (e *Encoder) kTime(f *codecFnInfo, rv reflect.Value) { + e.e.EncodeTime(rv2i(rv).(time.Time)) +} + func (e *Encoder) kString(f *codecFnInfo, rv reflect.Value) { e.e.EncodeString(cUTF8, rv.String()) } diff --git a/codec/helper_unsafe.go b/codec/helper_unsafe.go index 593945e0..55921c06 100644 --- a/codec/helper_unsafe.go +++ b/codec/helper_unsafe.go @@ -10,6 +10,7 @@ package codec import ( "reflect" "sync/atomic" + "time" "unsafe" ) @@ -152,6 +153,11 @@ func (d *Decoder) kBool(f *codecFnInfo, rv reflect.Value) { *(*bool)(urv.ptr) = d.d.DecodeBool() } +func (d *Decoder) kTime(f *codecFnInfo, rv reflect.Value) { + urv := (*unsafeReflectValue)(unsafe.Pointer(&rv)) + *(*time.Time)(urv.ptr) = d.d.DecodeTime() +} + func (d *Decoder) kFloat32(f *codecFnInfo, rv reflect.Value) { urv := (*unsafeReflectValue)(unsafe.Pointer(&rv)) *(*float32)(urv.ptr) = float32(d.d.DecodeFloat(true)) @@ -224,6 +230,11 @@ func (e *Encoder) kBool(f *codecFnInfo, rv reflect.Value) { e.e.EncodeBool(*(*bool)(v.ptr)) } +func (e *Encoder) kTime(f *codecFnInfo, rv reflect.Value) { + v := (*unsafeReflectValue)(unsafe.Pointer(&rv)) + e.e.EncodeTime(*(*time.Time)(v.ptr)) +} + func (e *Encoder) kString(f *codecFnInfo, rv reflect.Value) { v := (*unsafeReflectValue)(unsafe.Pointer(&rv)) e.e.EncodeString(cUTF8, *(*string)(v.ptr)) diff --git a/codec/json.go b/codec/json.go index f465a336..4c95d0e9 100644 --- a/codec/json.go +++ b/codec/json.go @@ -36,6 +36,7 @@ import ( "encoding/base64" "reflect" "strconv" + "time" "unicode" "unicode/utf16" "unicode/utf8" @@ -272,6 +273,15 @@ func (e *jsonEncDriver) EncodeNil() { // } } +func (e *jsonEncDriver) EncodeTime(t time.Time) { + v, err := t.MarshalJSON() + if err != nil { + e.e.error(err) + return + } + e.w.writeb(v) +} + func (e *jsonEncDriver) EncodeBool(b bool) { if e.h.MapKeyAsString && e.c == containerMapKey { if b { @@ -668,6 +678,16 @@ func (d *jsonDecDriver) DecodeBool() (v bool) { return } +func (d *jsonDecDriver) DecodeTime() (t time.Time) { + // read string, and pass the string into json.unmarshal + d.appendStringAsBytes() + t, err := time.Parse(time.RFC3339, stringView(d.bs)) + if err != nil { + d.d.error(err) + } + return +} + func (d *jsonDecDriver) ContainerType() (vt valueType) { // check container type by checking the first char if d.tok == 0 { diff --git a/codec/msgpack.go b/codec/msgpack.go index 4f920557..b191b285 100644 --- a/codec/msgpack.go +++ b/codec/msgpack.go @@ -25,6 +25,7 @@ import ( "math" "net/rpc" "reflect" + "time" ) const ( @@ -78,6 +79,9 @@ const ( mpNegFixNumMax = 0xff ) +var mpTimeExtTag int8 = -1 +var mpTimeExtTagU = uint8(mpTimeExtTag) + // MsgpackSpecRpcMultiArgs is a special type which signifies to the MsgpackSpecRpcCodec // that the backend RPC service takes multiple arguments, which have been arranged // in sequence in the slice. @@ -190,6 +194,35 @@ func (e *msgpackEncDriver) EncodeFloat64(f float64) { bigenHelper{e.x[:8], e.w}.writeUint64(math.Float64bits(f)) } +func (e *msgpackEncDriver) EncodeTime(t time.Time) { + t = t.UTC() + sec, nsec := t.Unix(), uint64(t.Nanosecond()) + var data64 uint64 + var l = 4 + if sec >= 0 && sec>>34 == 0 { + data64 = (nsec << 34) | uint64(sec) + if data64&0xffffffff00000000 != 0 { + l = 8 + } + } else { + l = 12 + } + if e.h.WriteExt { + e.encodeExtPreamble(mpTimeExtTagU, l) + } else { + e.writeContainerLen(msgpackContainerStr, l) + } + switch l { + case 4: + bigenHelper{e.x[:4], e.w}.writeUint32(uint32(data64)) + case 8: + bigenHelper{e.x[:8], e.w}.writeUint64(data64) + case 12: + bigenHelper{e.x[:4], e.w}.writeUint32(uint32(nsec)) + bigenHelper{e.x[:8], e.w}.writeUint64(uint64(sec)) + } +} + func (e *msgpackEncDriver) EncodeExt(v interface{}, xtag uint64, ext Ext, _ *Encoder) { bs := ext.WriteExt(v) if bs == nil { @@ -388,7 +421,12 @@ func (d *msgpackDecDriver) DecodeNaked() { n.v = valueTypeExt clen := d.readExtLen() n.u = uint64(d.r.readn1()) - n.l = d.r.readx(clen) + if n.u == uint64(mpTimeExtTagU) { + n.v = valueTypeTime + n.t = d.decodeTime(clen) + } else { + n.l = d.r.readx(clen) + } default: d.d.errorf("Nil-Deciphered DecodeValue: %s: hex: %x, dec: %d", msgBadDesc, bd, bd) } @@ -584,7 +622,7 @@ func (d *msgpackDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) d.bdRead = false return bs default: - d.d.errorf("invalid container type: expecting bin|str|array") + d.d.errorf("invalid container type: expecting bin|str|array, got: 0x%x", uint8(vt)) return } @@ -722,6 +760,53 @@ func (d *msgpackDecDriver) readExtLen() (clen int) { return } +func (d *msgpackDecDriver) DecodeTime() (t time.Time) { + // decode time from string bytes or ext + if !d.bdRead { + d.readNextBd() + } + var clen int + switch d.ContainerType() { + case valueTypeBytes, valueTypeString: + clen = d.readContainerLen(msgpackContainerStr) + default: + // expect to see mpFixExt4,-1 OR mpFixExt8,-1 OR mpExt8,12,-1 + d.bdRead = false + b2 := d.r.readn1() + if d.bd == mpFixExt4 && b2 == mpTimeExtTagU { + clen = 4 + } else if d.bd == mpFixExt8 && b2 == mpTimeExtTagU { + clen = 8 + } else if d.bd == mpExt8 && b2 == 12 && d.r.readn1() == mpTimeExtTagU { + clen = 12 + } else { + d.d.errorf("invalid sequence of bytes for decoding time as an extension: got 0x%x, 0x%x", d.bd, b2) + return + } + } + return d.decodeTime(clen) +} + +func (d *msgpackDecDriver) decodeTime(clen int) (t time.Time) { + // bs = d.r.readx(clen) + d.bdRead = false + switch clen { + case 4: + t = time.Unix(int64(bigen.Uint32(d.r.readx(4))), 0).UTC() + case 8: + tv := bigen.Uint64(d.r.readx(8)) + t = time.Unix(int64(tv&0x00000003ffffffff), int64(tv>>34)).UTC() + case 12: + nsec := bigen.Uint32(d.r.readx(4)) + sec := bigen.Uint64(d.r.readx(8)) + t = time.Unix(int64(sec), int64(nsec)).UTC() + default: + d.d.errorf("invalid length of bytes for decoding time - expecting 4 or 8 or 12, got %d", clen) + return + } + return +} + func (d *msgpackDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) { if xtag > 0xff { d.d.errorf("decodeExt: tag must be <= 0xff; got: %v", xtag) diff --git a/codec/simple.go b/codec/simple.go index 3e9195f1..a408e9a3 100644 --- a/codec/simple.go +++ b/codec/simple.go @@ -6,6 +6,7 @@ package codec import ( "math" "reflect" + "time" ) const ( @@ -20,6 +21,8 @@ const ( simpleVdPosInt = 8 simpleVdNegInt = 12 + simpleVdTime = 24 + // containers: each lasts for 4 (ie n, n+1, n+2, ... n+7) simpleVdString = 216 simpleVdByteArray = 224 @@ -194,6 +197,21 @@ func (e *simpleEncDriver) EncodeStringBytes(c charEncoding, v []byte) { e.w.writeb(v) } +func (e *simpleEncDriver) EncodeTime(t time.Time) { + if e.h.EncZeroValuesAsNil && e.c != containerMapKey && t.IsZero() { + e.EncodeNil() + return + } + v, err := t.MarshalBinary() + if err != nil { + e.e.error(err) + return + } + // time.Time marshalbinary takes about 14 bytes. + e.w.writen2(simpleVdTime, uint8(len(v))) + e.w.writeb(v) +} + //------------------------------------ type simpleDecDriver struct { @@ -226,20 +244,19 @@ func (d *simpleDecDriver) ContainerType() (vt valueType) { if !d.bdRead { d.readNextBd() } - if d.bd == simpleVdNil { + switch d.bd { + case simpleVdNil: return valueTypeNil - } else if d.bd == simpleVdByteArray || d.bd == simpleVdByteArray+1 || - d.bd == simpleVdByteArray+2 || d.bd == simpleVdByteArray+3 || d.bd == simpleVdByteArray+4 { + case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4: return valueTypeBytes - } else if d.bd == simpleVdString || d.bd == simpleVdString+1 || - d.bd == simpleVdString+2 || d.bd == simpleVdString+3 || d.bd == simpleVdString+4 { + case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4: return valueTypeString - } else if d.bd == simpleVdArray || d.bd == simpleVdArray+1 || - d.bd == simpleVdArray+2 || d.bd == simpleVdArray+3 || d.bd == simpleVdArray+4 { + case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4: return valueTypeArray - } else if d.bd == simpleVdMap || d.bd == simpleVdMap+1 || - d.bd == simpleVdMap+2 || d.bd == simpleVdMap+3 || d.bd == simpleVdMap+4 { + case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4: return valueTypeMap + // case simpleVdTime: + // return valueTypeTime } // else { // d.d.errorf("isContainerType: unsupported parameter: %v", vt) @@ -460,6 +477,27 @@ func (d *simpleDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) { return decByteSlice(d.r, clen, d.d.h.MaxInitLen, bs) } +func (d *simpleDecDriver) DecodeTime() (t time.Time) { + if !d.bdRead { + d.readNextBd() + } + if d.bd == simpleVdNil { + d.bdRead = false + return + } + if d.bd != simpleVdTime { + d.d.errorf("invalid descriptor for time.Time - expect 0x%x, received 0x%x", simpleVdTime, d.bd) + return + } + d.bdRead = false + clen := int(d.r.readn1()) + b := d.r.readx(clen) + if err := (&t).UnmarshalBinary(b); err != nil { + d.d.error(err) + } + return +} + func (d *simpleDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) { if xtag > 0xff { d.d.errorf("decodeExt: tag must be <= 0xff; got: %v", xtag) @@ -493,7 +531,7 @@ func (d *simpleDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs [ case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4: xbs = d.DecodeBytes(nil, true) default: - d.d.errorf("Invalid d.bd for extensions (Expecting extensions or byte array). Got: 0x%x", d.bd) + d.d.errorf("Invalid descriptor for extensions (Expecting extensions or byte array). Got: 0x%x", d.bd) return } d.bdRead = false @@ -534,6 +572,9 @@ func (d *simpleDecDriver) DecodeNaked() { case simpleVdFloat64: n.v = valueTypeFloat n.f = d.DecodeFloat(false) + case simpleVdTime: + n.v = valueTypeTime + n.t = d.DecodeTime() case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4: n.v = valueTypeString n.s = d.DecodeString() @@ -579,6 +620,7 @@ func (d *simpleDecDriver) DecodeNaked() { // - arrays are encoded as [bd] [length] [value]... // - extensions are encoded as [bd] [length] [tag] [byte]... // - strings/bytearrays are encoded as [bd] [length] [byte]... +// - time.Time are encoded as [bd] [length] [byte]... // // The full spec will be published soon. type SimpleHandle struct {