diff --git a/adl/rot13adl/example_test.go b/adl/rot13adl/example_test.go index 599eeffb..98393377 100644 --- a/adl/rot13adl/example_test.go +++ b/adl/rot13adl/example_test.go @@ -20,7 +20,7 @@ func ExampleUnmarshallingToADL() { nb := rot13adl.Prototype.SubstrateRoot.NewBuilder() // Unmarshal -- using the substrate's nodebuilder just like you'd unmarshal with any other nodebuilder. - err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(`"n pbby fgevat"`))) + err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(`"n pbby fgevat"`)), true) fmt.Printf("unmarshal error: %v\n", err) // Use `Reify` to get the synthetic high-level view of the ADL data. @@ -59,7 +59,7 @@ func ExampleCreatingViaADL() { // To marshal the ADL, just use marshal methods on its substrate as normal: var marshalBuffer bytes.Buffer - err := dagjson.Marshal(substrateNode, json.NewEncoder(&marshalBuffer, json.EncodeOptions{})) + err := dagjson.Marshal(substrateNode, json.NewEncoder(&marshalBuffer, json.EncodeOptions{}), true) fmt.Printf("marshalled: %v\n", marshalBuffer.String()) fmt.Printf("marshal error: %v\n", err) diff --git a/codec/dagjson/marshal.go b/codec/dagjson/marshal.go index a17bd3c9..d94da113 100644 --- a/codec/dagjson/marshal.go +++ b/codec/dagjson/marshal.go @@ -14,7 +14,7 @@ import ( // except for the `case ipld.Kind_Link` block, // which is dag-json's special sauce for schemafree links. -func Marshal(n ipld.Node, sink shared.TokenSink) error { +func Marshal(n ipld.Node, sink shared.TokenSink, allowLinks bool) error { var tk tok.Token switch n.Kind() { case ipld.Kind_Invalid: @@ -44,7 +44,7 @@ func Marshal(n ipld.Node, sink shared.TokenSink) error { if _, err := sink.Step(&tk); err != nil { return err } - if err := Marshal(v, sink); err != nil { + if err := Marshal(v, sink, allowLinks); err != nil { return err } } @@ -66,7 +66,7 @@ func Marshal(n ipld.Node, sink shared.TokenSink) error { if err != nil { return err } - if err := Marshal(v, sink); err != nil { + if err := Marshal(v, sink, allowLinks); err != nil { return err } } @@ -120,6 +120,9 @@ func Marshal(n ipld.Node, sink shared.TokenSink) error { _, err = sink.Step(&tk) return err case ipld.Kind_Link: + if !allowLinks { + return fmt.Errorf("cannot Marshal ipld links to JSON") + } v, err := n.AsLink() if err != nil { return err diff --git a/codec/dagjson/multicodec.go b/codec/dagjson/multicodec.go index f0b693ad..7a035d2e 100644 --- a/codec/dagjson/multicodec.go +++ b/codec/dagjson/multicodec.go @@ -21,9 +21,7 @@ func init() { } func Decode(na ipld.NodeAssembler, r io.Reader) error { - // Shell out directly to generic builder path. - // (There's not really any fastpaths of note for json.) - err := Unmarshal(na, json.NewDecoder(r)) + err := Unmarshal(na, json.NewDecoder(r), true) if err != nil { return err } @@ -49,7 +47,6 @@ func Decode(na ipld.NodeAssembler, r io.Reader) error { return err } } - return err } func Encode(n ipld.Node, w io.Writer) error { @@ -59,5 +56,5 @@ func Encode(n ipld.Node, w io.Writer) error { return Marshal(n, json.NewEncoder(w, json.EncodeOptions{ Line: []byte{'\n'}, Indent: []byte{'\t'}, - })) + }), true) } diff --git a/codec/dagjson/unmarshal.go b/codec/dagjson/unmarshal.go index 4fd5586a..18993658 100644 --- a/codec/dagjson/unmarshal.go +++ b/codec/dagjson/unmarshal.go @@ -19,8 +19,9 @@ import ( // several steps of handling maps, because it necessitates peeking several // tokens before deciding what kind of value to create). -func Unmarshal(na ipld.NodeAssembler, tokSrc shared.TokenSource) error { +func Unmarshal(na ipld.NodeAssembler, tokSrc shared.TokenSource, parseLinks bool) error { var st unmarshalState + st.parseLinks = parseLinks done, err := tokSrc.Step(&st.tk[0]) if err != nil { return err @@ -32,8 +33,9 @@ func Unmarshal(na ipld.NodeAssembler, tokSrc shared.TokenSource) error { } type unmarshalState struct { - tk [4]tok.Token // mostly, only 0'th is used... but [1:4] are used during lookahead for links. - shift int // how many times to slide something out of tk[1:4] instead of getting a new token. + tk [4]tok.Token // mostly, only 0'th is used... but [1:4] are used during lookahead for links. + shift int // how many times to slide something out of tk[1:4] instead of getting a new token. + parseLinks bool } // step leaves a "new" token in tk[0], @@ -120,7 +122,7 @@ func (st *unmarshalState) linkLookahead(na ipld.NodeAssembler, tokSrc shared.Tok if err != nil { return false, err } - if err := na.AssignLink(cidlink.Link{elCid}); err != nil { + if err := na.AssignLink(cidlink.Link{Cid: elCid}); err != nil { return false, err } return true, nil @@ -135,12 +137,14 @@ func (st *unmarshalState) unmarshal(na ipld.NodeAssembler, tokSrc shared.TokenSo case tok.TMapOpen: // dag-json has special needs: we pump a few tokens ahead to look for dag-json's "link" pattern. // We can't actually call BeginMap until we're sure it's not gonna turn out to be a link. - gotLink, err := st.linkLookahead(na, tokSrc) - if err != nil { // return in error if any token peeks failed or if structure looked like a link but failed to parse as CID. - return err - } - if gotLink { - return nil + if st.parseLinks { + gotLink, err := st.linkLookahead(na, tokSrc) + if err != nil { // return in error if any token peeks failed or if structure looked like a link but failed to parse as CID. + return err + } + if gotLink { + return nil + } } // Okay, now back to regularly scheduled map logic. diff --git a/codec/json/multicodec.go b/codec/json/multicodec.go new file mode 100644 index 00000000..c3c807dd --- /dev/null +++ b/codec/json/multicodec.go @@ -0,0 +1,63 @@ +package json + +import ( + "fmt" + "io" + + rfmtjson "github.com/polydawn/refmt/json" + + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagjson" + "github.com/ipld/go-ipld-prime/multicodec" +) + +var ( + _ ipld.Decoder = Decode + _ ipld.Encoder = Encode +) + +func init() { + multicodec.RegisterEncoder(0x0200, Encode) + multicodec.RegisterDecoder(0x0200, Decode) +} + +func Decode(na ipld.NodeAssembler, r io.Reader) error { + // Shell out directly to generic builder path. + // (There's not really any fastpaths of note for json.) + err := dagjson.Unmarshal(na, rfmtjson.NewDecoder(r), false) + if err != nil { + return err + } + // Slurp any remaining whitespace. + // (This is relevant if our reader is tee'ing bytes to a hasher, and + // the json contained any trailing whitespace.) + // (We can't actually support multiple objects per reader from here; + // we can't unpeek if we find a non-whitespace token, so our only + // option is to error if this reader seems to contain more content.) + var buf [1]byte + for { + _, err := r.Read(buf[:]) + switch buf[0] { + case ' ', 0x0, '\t', '\r', '\n': // continue + default: + return fmt.Errorf("unexpected content after end of json object") + } + if err == nil { + continue + } else if err == io.EOF { + return nil + } else { + return err + } + } +} + +func Encode(n ipld.Node, w io.Writer) error { + // Shell out directly to generic inspection path. + // (There's not really any fastpaths of note for json.) + // Write another function if you need to tune encoding options about whitespace. + return dagjson.Marshal(n, rfmtjson.NewEncoder(w, rfmtjson.EncodeOptions{ + Line: []byte{'\n'}, + Indent: []byte{'\t'}, + }), false) +} diff --git a/codec/jst/jst.go b/codec/jst/jst.go index ea5c09b7..48747f00 100644 --- a/codec/jst/jst.go +++ b/codec/jst/jst.go @@ -168,8 +168,8 @@ func (tab *table) Finalize() { var buf bytes.Buffer for _, cn := range cols { buf.Reset() - dagjson.Marshal(basicnode.NewString(string(cn)), json.NewEncoder(&buf, json.EncodeOptions{})) // FIXME this would be a lot less irritating if we had more plumbing access to the json encoding -- we want to encode exactly one string into a buffer, it literally can't error. - tab.keySize[cn] = buf.Len() // FIXME this is ignoring charsets, renderable glyphs, etc at present. + dagjson.Marshal(basicnode.NewString(string(cn)), json.NewEncoder(&buf, json.EncodeOptions{}), false) // FIXME this would be a lot less irritating if we had more plumbing access to the json encoding -- we want to encode exactly one string into a buffer, it literally can't error. + tab.keySize[cn] = buf.Len() // FIXME this is ignoring charsets, renderable glyphs, etc at present. } } @@ -299,7 +299,7 @@ func marshal(ctx *state, n ipld.Node, w io.Writer) error { func marshalPlain(ctx *state, n ipld.Node, w io.Writer) error { err := dagjson.Marshal(n, json.NewEncoder(w, json.EncodeOptions{ // never indent here: these values will always end up being emitted mid-line. - })) + }), true) if err != nil { return recordErrorPosition(ctx, err) } @@ -470,7 +470,7 @@ func emitKey(ctx *state, k ipld.Node, w io.Writer) error { if ctx.cfg.Color.Enabled { w.Write(ctx.cfg.Color.KeyHighlight) } - if err := dagjson.Marshal(k, json.NewEncoder(w, json.EncodeOptions{})); err != nil { + if err := dagjson.Marshal(k, json.NewEncoder(w, json.EncodeOptions{}), true); err != nil { return recordErrorPosition(ctx, err) } if ctx.cfg.Color.Enabled { diff --git a/schema/gen/go/testcase_test.go b/schema/gen/go/testcase_test.go index 5911e4c5..78bcf27e 100644 --- a/schema/gen/go/testcase_test.go +++ b/schema/gen/go/testcase_test.go @@ -190,7 +190,7 @@ func (tcase testcase) Test(t *testing.T, np, npr ipld.NodePrototype) { func testUnmarshal(t *testing.T, np ipld.NodePrototype, data string, expectFail error) ipld.Node { t.Helper() nb := np.NewBuilder() - err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(data))) + err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(data)), true) switch { case expectFail == nil && err != nil: t.Fatalf("fixture parse failed: %s", err) @@ -209,7 +209,7 @@ func testMarshal(t *testing.T, n ipld.Node, data string) { // We'll marshal with "pretty" linebreaks and indents (and re-format the fixture to the same) for better diffing. prettyprint := json.EncodeOptions{Line: []byte{'\n'}, Indent: []byte{'\t'}} var buf bytes.Buffer - err := dagjson.Marshal(n, json.NewEncoder(&buf, prettyprint)) + err := dagjson.Marshal(n, json.NewEncoder(&buf, prettyprint), true) if err != nil { t.Errorf("marshal failed: %s", err) } diff --git a/schema/schema2/typesystem_test.go b/schema/schema2/typesystem_test.go index 5546d19a..8d245ab6 100644 --- a/schema/schema2/typesystem_test.go +++ b/schema/schema2/typesystem_test.go @@ -206,7 +206,7 @@ func testParse(t *testing.T, schemajson string, expectParseErr error, expectType func parseSchema(schemajson string) (schemadmt.Schema, error) { nb := schemadmt.Type.Schema__Repr.NewBuilder() - if err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(schemajson))); err != nil { + if err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(schemajson)), true); err != nil { return nil, err } return nb.Build().(schemadmt.Schema), nil