diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6f952299 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# To prevent CRLF breakages on Windows for fragile files, like testdata. +* -text diff --git a/cmd/car/car.go b/cmd/car/car.go index 12a1f1f1..dca4c0f2 100644 --- a/cmd/car/car.go +++ b/cmd/car/car.go @@ -31,6 +31,10 @@ func main1() int { Usage: "A file to read CIDs from", TakesFile: true, }, + &cli.BoolFlag{ + Name: "append", + Usage: "Append cids to an existing output file", + }, }, }, { diff --git a/cmd/car/filter.go b/cmd/car/filter.go index a941a8b3..0a0f2893 100644 --- a/cmd/car/filter.go +++ b/cmd/car/filter.go @@ -50,10 +50,36 @@ func FilterCar(c *cli.Context) error { outRoots = append(outRoots, r) } } + + outPath := c.Args().Get(1) + if !c.Bool("append") { + if _, err := os.Stat(outPath); err == nil || !os.IsNotExist(err) { + // output to an existing file. + if err := os.Truncate(outPath, 0); err != nil { + return err + } + } + } else { + // roots will need to be whatever is in the output already. + cv2r, err := carv2.OpenReader(outPath) + if err != nil { + return err + } + if cv2r.Version != 2 { + return fmt.Errorf("can only append to version 2 car files") + } + outRoots, err = cv2r.Roots() + if err != nil { + return err + } + _ = cv2r.Close() + } + if len(outRoots) == 0 { fmt.Fprintf(os.Stderr, "warning: no roots defined after filtering\n") } - bs, err := blockstore.OpenReadWrite(c.Args().Get(1), outRoots) + + bs, err := blockstore.OpenReadWrite(outPath, outRoots) if err != nil { return err } diff --git a/cmd/car/testdata/script/filter.txt b/cmd/car/testdata/script/filter.txt new file mode 100644 index 00000000..0c0b12de --- /dev/null +++ b/cmd/car/testdata/script/filter.txt @@ -0,0 +1,30 @@ +# basic filter +stdin filteredcids.txt +car filter ${INPUTS}/sample-wrapped-v2.car out.car +stderr 'warning: no roots defined after filtering' +car list out.car +! stderr . +cmp stdout filteredcids.txt + +# filter with root CID +stdin filteredroot.txt +car filter ${INPUTS}/sample-wrapped-v2.car out.car +! stderr . +car list out.car +! stderr . +cmp stdout filteredroot.txt + +# append other cids +stdin filteredcids.txt +car filter -append ${INPUTS}/sample-wrapped-v2.car out.car +! stderr . +car list out.car +stdout -count=4 '^bafy' + + +-- filteredcids.txt -- +bafy2bzacebohz654namrgmwjjx4qmtwgxixsd7pn4tlanyrc3g3hwj75hlxrw +bafy2bzaceaqtiesyfqd2jibmofz22oolguzf5wscwh73rmeypglfu2xhkptri +bafy2bzacebct3dm7izgyauijzkaf3yd7ylni725k66rq7dfp3jr5ywhpprj3k +-- filteredroot.txt -- +bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy diff --git a/cmd/car/testdata/script/verify.txt b/cmd/car/testdata/script/verify.txt new file mode 100644 index 00000000..3ef3c49e --- /dev/null +++ b/cmd/car/testdata/script/verify.txt @@ -0,0 +1,3 @@ +# "verify" should exit with code 0 on reasonable cars. +car verify ${INPUTS}/sample-v1.car +car verify ${INPUTS}/sample-wrapped-v2.car diff --git a/cmd/car/verify.go b/cmd/car/verify.go index c9c753d7..3a43de93 100644 --- a/cmd/car/verify.go +++ b/cmd/car/verify.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/go-cid" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/index" + "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" ) @@ -44,12 +45,15 @@ func VerifyCar(c *cli.Context) error { if err != nil { return err } - lengthToIndex := carv2.PragmaSize + carv2.HeaderSize + rx.Header.DataOffset + rx.Header.DataSize + lengthToIndex := carv2.PragmaSize + carv2.HeaderSize + rx.Header.DataSize if uint64(flen.Size()) > lengthToIndex && rx.Header.IndexOffset == 0 { return fmt.Errorf("header claims no index, but extra bytes in file beyond data size") } + if rx.Header.DataOffset < carv2.PragmaSize+carv2.HeaderSize { + return fmt.Errorf("data offset places data within carv2 header") + } if rx.Header.IndexOffset < lengthToIndex { - return fmt.Errorf("index offset overlaps with data") + return fmt.Errorf("index offset overlaps with data. data ends at %d. index offset of %d", lengthToIndex, rx.Header.IndexOffset) } } @@ -87,6 +91,13 @@ func VerifyCar(c *cli.Context) error { return err } for _, c := range cidList { + cidHash, err := multihash.Decode(c.Hash()) + if err != nil { + return err + } + if cidHash.Code == multihash.IDENTITY { + continue + } if err := idx.GetAll(c, func(_ uint64) bool { return true }); err != nil { diff --git a/cmd/go.mod b/cmd/go.mod index a37fb88a..eb608dc3 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -7,10 +7,11 @@ require ( github.com/ipfs/go-cid v0.1.0 github.com/ipfs/go-ipfs-blockstore v1.0.3 github.com/ipld/go-car v0.3.1 - github.com/ipld/go-car/v2 v2.0.2 + github.com/ipld/go-car/v2 v2.0.3-0.20210914083849-7544041c42bb github.com/ipld/go-codec-dagpb v1.2.0 github.com/ipld/go-ipld-prime v0.12.3-0.20210910135350-e6597215c6d5 github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 + github.com/multiformats/go-multihash v0.0.15 github.com/multiformats/go-varint v0.0.6 github.com/rogpeppe/go-internal v1.8.0 github.com/urfave/cli/v2 v2.3.0 diff --git a/cmd/go.sum b/cmd/go.sum index 343f9fbd..4c6f8789 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -234,8 +234,8 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs= github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs= -github.com/ipld/go-car/v2 v2.0.2 h1:R1oIAPwrGp26mEFzcGf5bfTZAAHDOkaVnZTEVebaWX4= -github.com/ipld/go-car/v2 v2.0.2/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= +github.com/ipld/go-car/v2 v2.0.3-0.20210914083849-7544041c42bb h1:w86j9nGrl+kCsrWF2KAevmqnkT7gMWjr4uSZ/a5jwWY= +github.com/ipld/go-car/v2 v2.0.3-0.20210914083849-7544041c42bb/go.mod h1:Xr6GwkDhv8dtOtgHzOynAkIOg0t0YiPc5DxBPppWqZA= github.com/ipld/go-codec-dagpb v1.2.0 h1:2umV7ud8HBMkRuJgd8gXw95cLhwmcYrihS3cQEy9zpI= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= @@ -391,7 +391,6 @@ github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multicodec v0.2.1-0.20210713081508-b421db6850ae/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 h1:ZrUuMKNgJ52qHPoQ+bx0h0uBfcWmN7Px+4uKSZeesiI= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=