Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove fastpath and unsafe variants #14

Merged
merged 7 commits into from
Dec 5, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- uses: actions/setup-go@v3
with:
go-version: ${{ needs.get-go-version.outputs.go-version }}
- run: '[ "$(gofmt -l . | wc -l)" -gt 0 ] && false'
- run: 'exit $(( $(gofmt -s -l . | wc -l) != 0 ))'

test:
name: codec test
Expand Down
24 changes: 1 addition & 23 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ This repository contains the `go-codec` library.
To install:

```
go get github.com/ugorji/go/codec
go get github.com/hashicorp/go-msgpack/codec
```

# Package Documentation
Expand Down Expand Up @@ -242,28 +242,6 @@ You can run the tag 'safe' to run tests or build in safe mode. e.g.
Please see http://github.com/ugorji/go-codec-bench .


## Managing Binary Size

This package could add up to 10MB to the size of your binaries.

This is because we include some a auto-generated file:
`fast-path.generated.go` to help with performance when encoding/decoding
slices and maps of built in numeric, boolean, string and interface{} types.

You can override this by building (or running tests and benchmarks) with the
tag: `notfastpath`.

```
go install -tags notfastpath
go build -tags notfastpath
go test -tags notfastpath
```

Be aware that, at least in our representative microbenchmarks for cbor (for
example), we see up to 33% increase in decoding and 50% increase in encoding
speeds. YMMV.


## Caveats

Struct fields matching the following are ignored during encoding and
Expand Down
2 changes: 1 addition & 1 deletion codec/bench/bench.sh
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ _gen() {
#
_suite() {
local t="alltests x"
local a=( "" "safe" "notfastpath" "notfastpath safe" "codecgen" "codecgen safe")
local a=( "" "safe" "codecgen" "codecgen safe")
local b=( "generated" "generated safe")
for i in "${a[@]}"
do
Expand Down
3 changes: 1 addition & 2 deletions codec/bench/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,4 @@ Here, we have the benchmark files comparing against other encoding libraries.

See README.md for more details.
*/
package codec

package codec
2 changes: 1 addition & 1 deletion codec/bench/shared_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ import (
"testing"
)

import . "github.com/ugorji/go/codec"
import . "github.com/hashicorp/go-msgpack/v2/codec"

type testHED struct {
H Handle
Expand Down
4 changes: 2 additions & 2 deletions codec/bench/x_bench_gen_test.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// +build x
// +build generated
//go:build x && generated
// +build x,generated

// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.
Expand Down
1 change: 1 addition & 0 deletions codec/bench/x_bench_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//go:build x
// +build x

// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
Expand Down
1 change: 1 addition & 0 deletions codec/bench/z_all_bench_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.

//go:build (alltests || codecgen) && go1.7
// +build alltests codecgen
// +build go1.7

Expand Down
6 changes: 2 additions & 4 deletions codec/bench/z_all_x_bench_gen_test.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.

// +build alltests
// +build x
// +build go1.7
// +build generated
//go:build alltests && x && go1.7 && generated
// +build alltests,x,go1.7,generated

package codec

Expand Down
5 changes: 2 additions & 3 deletions codec/bench/z_all_x_bench_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.

// +build alltests
// +build x
// +build go1.7
//go:build alltests && x && go1.7
// +build alltests,x,go1.7

package codec

Expand Down
91 changes: 45 additions & 46 deletions codec/binc.go
Original file line number Diff line number Diff line change
Expand Up @@ -970,18 +970,18 @@ func (d *bincDecDriver) DecodeNaked() {

//------------------------------------

//BincHandle is a Handle for the Binc Schema-Free Encoding Format
//defined at https://github.com/ugorji/binc .
// BincHandle is a Handle for the Binc Schema-Free Encoding Format
// defined at https://github.com/ugorji/binc .
//
//BincHandle currently supports all Binc features with the following EXCEPTIONS:
// - only integers up to 64 bits of precision are supported.
// big integers are unsupported.
// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types).
// extended precision and decimal IEEE 754 floats are unsupported.
// - Only UTF-8 strings supported.
// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported.
// BincHandle currently supports all Binc features with the following EXCEPTIONS:
// - only integers up to 64 bits of precision are supported.
// big integers are unsupported.
// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types).
// extended precision and decimal IEEE 754 floats are unsupported.
// - Only UTF-8 strings supported.
// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported.
//
//Note that these EXCEPTIONS are temporary and full support is possible and may happen soon.
// Note that these EXCEPTIONS are temporary and full support is possible and may happen soon.
type BincHandle struct {
BasicHandle
binaryEncodingType
Expand Down Expand Up @@ -1046,50 +1046,49 @@ func (d *bincDecDriver) reset() {
//
// Format Description
//
// A timestamp is composed of 3 components:
// A timestamp is composed of 3 components:
//
// - secs: signed integer representing seconds since unix epoch
// - nsces: unsigned integer representing fractional seconds as a
// nanosecond offset within secs, in the range 0 <= nsecs < 1e9
// - tz: signed integer representing timezone offset in minutes east of UTC,
// and a dst (daylight savings time) flag
// - secs: signed integer representing seconds since unix epoch
// - nsces: unsigned integer representing fractional seconds as a
// nanosecond offset within secs, in the range 0 <= nsecs < 1e9
// - tz: signed integer representing timezone offset in minutes east of UTC,
// and a dst (daylight savings time) flag
//
// When encoding a timestamp, the first byte is the descriptor, which
// defines which components are encoded and how many bytes are used to
// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it
// is not encoded in the byte array explicitly*.
// When encoding a timestamp, the first byte is the descriptor, which
// defines which components are encoded and how many bytes are used to
// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it
// is not encoded in the byte array explicitly*.
//
// Descriptor 8 bits are of the form `A B C DDD EE`:
// A: Is secs component encoded? 1 = true
// B: Is nsecs component encoded? 1 = true
// C: Is tz component encoded? 1 = true
// DDD: Number of extra bytes for secs (range 0-7).
// If A = 1, secs encoded in DDD+1 bytes.
// If A = 0, secs is not encoded, and is assumed to be 0.
// If A = 1, then we need at least 1 byte to encode secs.
// DDD says the number of extra bytes beyond that 1.
// E.g. if DDD=0, then secs is represented in 1 byte.
// if DDD=2, then secs is represented in 3 bytes.
// EE: Number of extra bytes for nsecs (range 0-3).
// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above)
// Descriptor 8 bits are of the form `A B C DDD EE`:
// A: Is secs component encoded? 1 = true
// B: Is nsecs component encoded? 1 = true
// C: Is tz component encoded? 1 = true
// DDD: Number of extra bytes for secs (range 0-7).
// If A = 1, secs encoded in DDD+1 bytes.
// If A = 0, secs is not encoded, and is assumed to be 0.
// If A = 1, then we need at least 1 byte to encode secs.
// DDD says the number of extra bytes beyond that 1.
// E.g. if DDD=0, then secs is represented in 1 byte.
// if DDD=2, then secs is represented in 3 bytes.
// EE: Number of extra bytes for nsecs (range 0-3).
// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above)
//
// Following the descriptor bytes, subsequent bytes are:
// Following the descriptor bytes, subsequent bytes are:
//
// secs component encoded in `DDD + 1` bytes (if A == 1)
// nsecs component encoded in `EE + 1` bytes (if B == 1)
// tz component encoded in 2 bytes (if C == 1)
// secs component encoded in `DDD + 1` bytes (if A == 1)
// nsecs component encoded in `EE + 1` bytes (if B == 1)
// tz component encoded in 2 bytes (if C == 1)
//
// secs and nsecs components are integers encoded in a BigEndian
// 2-complement encoding format.
// secs and nsecs components are integers encoded in a BigEndian
// 2-complement encoding format.
//
// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to
// Least significant bit 0 are described below:
//
// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes).
// Bit 15 = have\_dst: set to 1 if we set the dst flag.
// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not.
// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format.
// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to
// Least significant bit 0 are described below:
//
// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes).
// Bit 15 = have\_dst: set to 1 if we set the dst flag.
// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not.
// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format.
func bincEncodeTime(t time.Time) []byte {
//t := rv.Interface().(time.Time)
tsecs, tnsecs := t.Unix(), t.Nanosecond()
Expand Down
46 changes: 21 additions & 25 deletions codec/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@

_tests() {
local gover=$( go version | cut -f 3 -d ' ' )
# note that codecgen requires fastpath, so you cannot do "codecgen notfastpath"
local a=( "" "safe" "notfastpath" "notfastpath safe" "codecgen" "codecgen safe" )
local a=( "" "codecgen" )
for i in "${a[@]}"
do
echo ">>>> TAGS: $i"
Expand All @@ -17,7 +16,7 @@ _tests() {
*) go vet -printfuncs "errorf" "$@" &&
go test ${zargs[*]} -vet off -tags "alltests $i" -run "Suite" -coverprofile "${i2// /-}.cov.out" "$@" ;;
esac
if [[ "$?" != 0 ]]; then return 1; fi
if [[ "$?" != 0 ]]; then return 1; fi
done
echo "++++++++ TEST SUITES ALL PASSED ++++++++"
}
Expand All @@ -26,10 +25,10 @@ _tests() {
# is a generation needed?
_ng() {
local a="$1"
if [[ ! -e "$a" ]]; then echo 1; return; fi
if [[ ! -e "$a" ]]; then echo 1; return; fi
for i in `ls -1 *.go.tmpl gen.go values_test.go`
do
if [[ "$a" -ot "$i" ]]; then echo 1; return; fi
if [[ "$a" -ot "$i" ]]; then echo 1; return; fi
done
}

Expand All @@ -42,19 +41,18 @@ EOF
rm -f ${1}
}

# _build generates fast-path.go and gen-helper.go.
# _build generates gen-helper.go.
_build() {
if ! [[ "${zforce}" || $(_ng "fast-path.generated.go") || $(_ng "gen-helper.generated.go") || $(_ng "gen.generated.go") ]]; then return 0; fi
if ! [[ "${zforce}" || $(_ng "gen-helper.generated.go") || $(_ng "gen.generated.go") ]]; then return 0; fi

if [ "${zbak}" ]; then
_zts=`date '+%m%d%Y_%H%M%S'`
_gg=".generated.go"
[ -e "gen-helper${_gg}" ] && mv gen-helper${_gg} gen-helper${_gg}__${_zts}.bak
[ -e "fast-path${_gg}" ] && mv fast-path${_gg} fast-path${_gg}__${_zts}.bak
[ -e "gen${_gg}" ] && mv gen${_gg} gen${_gg}__${_zts}.bak
fi
rm -f gen-helper.generated.go fast-path.generated.go gen.generated.go \
*safe.generated.go *_generated_test.go *.generated_ffjson_expose.go
fi
rm -f gen-helper.generated.go gen.generated.go \
*_generated_test.go *.generated_ffjson_expose.go

cat > gen.generated.go <<EOF
// +build codecgen.exec
Expand Down Expand Up @@ -85,7 +83,7 @@ EOF
\`
EOF
cat > gen-from-tmpl.codec.generated.go <<EOF
package codec
package codec
import "io"
func GenInternalGoFile(r io.Reader, w io.Writer) error {
return genInternalGoFile(r, w)
Expand All @@ -112,18 +110,17 @@ if err != nil { panic(err) }
}

func main() {
run("fast-path.go.tmpl", "fast-path.generated.go")
run("gen-helper.go.tmpl", "gen-helper.generated.go")
run("mammoth-test.go.tmpl", "mammoth_generated_test.go")
run("mammoth2-test.go.tmpl", "mammoth2_generated_test.go")
}
EOF

sed -e 's+// __DO_NOT_REMOVE__NEEDED_FOR_REPLACING__IMPORT_PATH__FOR_CODEC_BENCH__+import . "github.com/ugorji/go/codec"+' \
sed -e 's+// __DO_NOT_REMOVE__NEEDED_FOR_REPLACING__IMPORT_PATH__FOR_CODEC_BENCH__+import . "github.com/hashicorp/go-msgpack/v2/codec"+' \
shared_test.go > bench/shared_test.go

# explicitly return 0 if this passes, else return 1
go run -tags "notfastpath safe codecgen.exec" gen-from-tmpl.generated.go &&
go run -tags "codecgen.exec" gen-from-tmpl.generated.go &&
rm -f gen-from-tmpl.*generated.go &&
return 0
return 1
Expand All @@ -136,7 +133,7 @@ _codegenerators() {
local c9="codecgen-scratch.go"

if ! [[ $zforce || $(_ng "values_codecgen${c5}") ]]; then return 0; fi

# Note: ensure you run the codecgen for this codebase/directory i.e. ./codecgen/codecgen
true &&
echo "codecgen ... " &&
Expand All @@ -145,20 +142,20 @@ _codegenerators() {
fi &&
$c8 -rt codecgen -t 'codecgen generated' -o values_codecgen${c5} -d 19780 $zfin $zfin2 &&
cp mammoth2_generated_test.go $c9 &&
$c8 -t '!notfastpath' -o mammoth2_codecgen${c5} -d 19781 mammoth2_generated_test.go &&
$c8 -o mammoth2_codecgen${c5} -d 19781 mammoth2_generated_test.go &&
rm -f $c9 &&
echo "generators done!"
echo "generators done!"
}

_prebuild() {
echo "prebuild: zforce: $zforce"
local d="$PWD"
zfin="test_values.generated.go"
zfin2="test_values_flex.generated.go"
zpkg="github.com/ugorji/go/codec"
zpkg="github.com/hashicorp/go-msgpack/v2/codec"
# zpkg=${d##*/src/}
# zgobase=${d%%/src/*}
# rm -f *_generated_test.go
# rm -f *_generated_test.go
rm -f codecgen-*.go &&
_build &&
cp $d/values_test.go $d/$zfin &&
Expand Down Expand Up @@ -221,7 +218,7 @@ EOF

_usage() {
cat <<EOF
primary usage: $0
primary usage: $0
-[tmpfxnld] -> [tests, make, prebuild (force) (external), inlining diagnostics, mid-stack inlining, race detector]
-v -> verbose
EOF
Expand Down Expand Up @@ -263,5 +260,4 @@ _main() {
unset zforce zargs zbenchflags
}

[ "." = `dirname $0` ] && _main "$@"

[ "." = `dirname $0` ] && _main "$@"
1 change: 1 addition & 0 deletions codec/codecgen.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//go:build codecgen || generated
// +build codecgen generated

package codec
Expand Down
Loading