diff --git a/.gitignore b/.gitignore index b134141e..6515690c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ txexport.log txexport.log.gz +.idea diff --git a/extractor/.gitignore b/extractor/.gitignore new file mode 100644 index 00000000..03603429 --- /dev/null +++ b/extractor/.gitignore @@ -0,0 +1,6 @@ +.idea +logs +extracted +.env +testdir +test \ No newline at end of file diff --git a/extractor/README.md b/extractor/README.md new file mode 100644 index 00000000..dcb7bc9d --- /dev/null +++ b/extractor/README.md @@ -0,0 +1,23 @@ +# Gno Source Code Extractor + +This tool is a simple parser to extract source code (packages & realms) from logs created by the [tx-archive](https://github.com/gnolang/tx-archive) tool for Gno chains. + +## Running the extractor + +The extractor takes in three arguments: +- the filetype of the archive files, +- output directory for the extracted packages, +- the root directory where the archive files are located. + +``` +USAGE + [flags] + +The Gno source code extractor service + +FLAGS + -file-type .jsonl the file type for analysis, with a preceding period (ie .log) + -output-dir ./extracted the output directory for the extracted Gno source code + -source-dir . the root folder containing transaction data +``` + diff --git a/extractor/go.mod b/extractor/go.mod new file mode 100644 index 00000000..cddb9c5c --- /dev/null +++ b/extractor/go.mod @@ -0,0 +1,47 @@ +module extractor + +go 1.20 + +require ( + github.com/gnolang/gno v0.0.0-20231006162410-fa8eb7753dc5 + github.com/go-test/deep v1.1.0 + github.com/peterbourgon/ff/v3 v3.4.0 + github.com/stretchr/testify v1.8.4 + golang.org/x/sync v0.4.0 +) + +require ( + github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c // indirect + github.com/btcsuite/btcd/btcutil v1.0.0 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cockroachdb/apd v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/badger/v3 v3.2103.4 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gnolang/goleveldb v0.0.9 // indirect + github.com/gnolang/overflow v0.0.0-20170615021017-4d914c927216 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/jmhodges/levigo v1.0.0 // indirect + github.com/klauspost/compress v1.12.3 // indirect + github.com/linxGnu/grocksdb v1.8.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect + go.etcd.io/bbolt v1.3.7 // indirect + go.opencensus.io v0.22.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/tools v0.6.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/extractor/go.sum b/extractor/go.sum new file mode 100644 index 00000000..b5daecbd --- /dev/null +++ b/extractor/go.sum @@ -0,0 +1,226 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c h1:lnAMg3ra/Gw4AkRMxrxYs8nrprWsHowg8H9zaYsJOo4= +github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= +github.com/btcsuite/btcd/btcutil v1.0.0 h1:dB36qRTOucIh6NUe40UCieOS+axPhP6VNyRtYkTUKKk= +github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v3 v3.2103.4 h1:WE1B07YNTTJTtG9xjBcSW2wn0RJLyiV99h959RKZqM4= +github.com/dgraph-io/badger/v3 v3.2103.4/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gnolang/gno v0.0.0-20231006162410-fa8eb7753dc5 h1:807vK/Yz1dUO05Op9Sc1hwgz7rugyC+ejStBHDT9bsE= +github.com/gnolang/gno v0.0.0-20231006162410-fa8eb7753dc5/go.mod h1:naCUZBEdOgcD9y1wb0MjN8fpWicsrnxrMhIX1j0tc8E= +github.com/gnolang/goleveldb v0.0.9 h1:Q7rGko9oXMKtQA+Apeeed5a3sjba/mcDhzJGoTVLCKE= +github.com/gnolang/goleveldb v0.0.9/go.mod h1:Dz6p9bmpy/FBESTgduiThZt5mToVDipcHGzj/zUOo8E= +github.com/gnolang/overflow v0.0.0-20170615021017-4d914c927216 h1:GKvsK3oLWG9B1GL7WP/VqwM6C92j5tIvB844oggL9Lk= +github.com/gnolang/overflow v0.0.0-20170615021017-4d914c927216/go.mod h1:xJhtEL7ahjM1WJipt89gel8tHzfIl/LyMY+lCYh38d8= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jaekwon/testify v1.6.1 h1:4AtAJcR9GzXN5W4DdY7ie74iCPiJV1JJUJL90t2ZUyw= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= +github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo= +github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= +github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/extractor/main.go b/extractor/main.go new file mode 100644 index 00000000..9be6c03b --- /dev/null +++ b/extractor/main.go @@ -0,0 +1,311 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "github.com/gnolang/gno/gno.land/pkg/sdk/vm" + "github.com/gnolang/gno/tm2/pkg/amino" + "github.com/gnolang/gno/tm2/pkg/std" + "github.com/peterbourgon/ff/v3/ffcli" + "golang.org/x/sync/errgroup" + "io" + "os" + "path/filepath" + "strings" +) + +// Define constants +const ( + packageMetadataFile = "pkg_metadata.json" +) + +var ( + errInvalidFileType = errors.New("no file type specified") + errInvalidSourceDir = errors.New("invalid source directory") + errInvalidOutputDir = errors.New("invalid output directory") + errNoSourceFilesFound = errors.New("no source files found, exiting") +) + +// Define extractor config +type extractorCfg struct { + fileType string + sourceDir string + outputDir string +} + +func main() { + var ( + cfg = &extractorCfg{} + fs = flag.NewFlagSet("root", flag.ExitOnError) + ) + + // Register the flags + cfg.registerFlags(fs) + + // Create the command + cmd := &ffcli.Command{ + ShortUsage: "[flags]", + LongHelp: "The Gno / TM2 source code extractor service", + FlagSet: fs, + Exec: func(ctx context.Context, _ []string) error { + return execExtract(ctx, cfg) + }, + } + + // Run the command + if err := cmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil { + fmt.Fprintf(os.Stderr, "%+v", err) + + os.Exit(1) + } +} + +// registerFlags registers the extractor service flag set +func (c *extractorCfg) registerFlags(fs *flag.FlagSet) { + fs.StringVar( + &c.fileType, + "file-type", + ".jsonl", + "the file type for analysis, with a preceding period (ie .jsonl)", + ) + + fs.StringVar( + &c.sourceDir, + "source-dir", + ".", + "the root folder containing transaction data", + ) + + fs.StringVar( + &c.outputDir, + "output-dir", + "./extracted", + "the output directory for the extracted Gno source code", + ) +} + +// execExtract runs the extract service for Gno source code +func execExtract(ctx context.Context, cfg *extractorCfg) error { + // Check the file type is valid + if cfg.fileType == "" { + return errInvalidFileType + } + + // Check the source dir is valid + if cfg.sourceDir == "" { + return errInvalidSourceDir + } + + // Check the output dir is valid + if cfg.outputDir == "" { + return errInvalidOutputDir + } + + // Find the files that need to be analyzed + sourceFiles, findErr := findFilePaths(cfg.sourceDir, cfg.fileType) + if findErr != nil { + return fmt.Errorf("unable to find file paths, %w", findErr) + } + + if len(sourceFiles) == 0 { + return errNoSourceFilesFound + } + + // Concurrently process the source files + g, ctx := errgroup.WithContext(ctx) + + for _, sourceFile := range sourceFiles { + sourceFile := sourceFile + + g.Go(func() error { + // Extract messages + msgs, processErr := extractAddMessages(sourceFile) + if processErr != nil { + return processErr + } + + // Process messages + for _, msg := range msgs { + outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) + + // Write dir before writing files + if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil { + return fmt.Errorf("unable to write dir, %w", dirWriteErr) + } + + // Write the package source code + if writeErr := writePackageFiles(msg, outputDir); writeErr != nil { + return writeErr + } + + // Write the package metadata + if writeErr := writePackageMetadata(metadataFromMsg(msg), outputDir); writeErr != nil { + return writeErr + } + } + + return nil + }) + } + + return g.Wait() +} + +// writePackageFiles writes all files from a single package to the output directory +func writePackageFiles(msg vm.MsgAddPackage, outputDir string) error { + for _, file := range msg.Package.Files { + // Get the output path + writePath := filepath.Join(outputDir, file.Name) + + if writeErr := os.WriteFile(writePath, []byte(file.Body), 0644); writeErr != nil { + return fmt.Errorf("unable to write file %s, %w", file.Name, writeErr) + } + } + + return nil +} + +// writePackageMetadata writes the package metadata to the output directory +func writePackageMetadata(metadata Metadata, outputDir string) error { + // Get the output path + writePath := filepath.Join(outputDir, packageMetadataFile) + + // Get the JSON metadata + metadataRaw, marshalErr := json.Marshal(metadata) + if marshalErr != nil { + return fmt.Errorf("unable to JSON marshal metadata, %w", marshalErr) + } + + if writeErr := os.WriteFile(writePath, metadataRaw, 0644); writeErr != nil { + return fmt.Errorf("unable to write package metadata, %w", writeErr) + } + + return nil +} + +func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("unable to open file, %w", err) + } + + cleanup := func() error { + if closeErr := file.Close(); closeErr != nil { + return fmt.Errorf("unable to gracefully close file, %w", closeErr) + } + return nil + } + + reader := bufio.NewReader(file) + + // Used to track what was parsed in the past + touchMap := make(map[string]bool) + + // Msg array to be returned for further processing + msgArr := make([]vm.MsgAddPackage, 0) + + // Buffer to handle lines longer than 64kb + tempBuf := make([]byte, 0) + + for { + var tx std.Tx + line, isPrefix, err := reader.ReadLine() + + // Exit if no more lines in file + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("error reading lines; %w", err) + } + + // If line is too long, save it in a temporary buffer and continue reading line + if isPrefix { + tempBuf = append(tempBuf, line...) + continue + } + + // Handle long lines + if len(tempBuf) != 0 { + // Append last part of line to temporary buffer + tempBuf = append(tempBuf, line...) + + // Use line variable to pass it on to amino + line = tempBuf + } + + if err := amino.UnmarshalJSON(line, &tx); err != nil { + fmt.Errorf("Error while parsing amino JSON at line: %w\nLine:%s\n", err, line) + continue + } + + // Reset tempBuf in case it was used for a long line + if tempBuf != nil { + tempBuf = nil + } + + for _, msg := range tx.Msgs { + // Only MsgAddPkg should be parsed + if msg.Type() != "add_package" { + continue + } + + msgAddPkg, ok := msg.(vm.MsgAddPackage) + if !ok { + return nil, errors.New("could not cast into MsgAddPackage") + } + + if msgAddPkg.Package == nil { + return nil, errors.New("MsgAddPackage is nil") + } + path := msgAddPkg.Package.Path + + if _, parsed := touchMap[path]; parsed { + // Package already parsed + continue + } + + touchMap[path] = true + msgArr = append(msgArr, msgAddPkg) + } + } + + return msgArr, cleanup() +} + +// findFilePaths gathers the file paths for specific file types +func findFilePaths(startPath string, fileType string) ([]string, error) { + filePaths := make([]string, 0) + + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error accessing file: %w", err) + } + + // Check if the file is a dir + if info.IsDir() { + return nil + } + + // Check if the file type matches + if !strings.HasSuffix(info.Name(), fileType) { + return nil + } + + // File is not a directory, and is of the type + filePaths = append(filePaths, path) + + return nil + } + + // Walk the directory root recursively + if walkErr := filepath.Walk(startPath, walkFn); walkErr != nil { + return nil, fmt.Errorf("unable to walk directory, %w", walkErr) + } + + return filePaths, nil +} diff --git a/extractor/main_test.go b/extractor/main_test.go new file mode 100644 index 00000000..c0f157d6 --- /dev/null +++ b/extractor/main_test.go @@ -0,0 +1,489 @@ +package main + +import ( + "bufio" + "context" + crand "crypto/rand" + "encoding/base64" + "encoding/json" + "github.com/gnolang/gno/gno.land/pkg/sdk/vm" + "github.com/gnolang/gno/tm2/pkg/amino" + "github.com/gnolang/gno/tm2/pkg/crypto" + "github.com/gnolang/gno/tm2/pkg/sdk/bank" + "github.com/gnolang/gno/tm2/pkg/std" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "math/rand" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + "time" +) + +const ( + numSourceFiles = 20 + numTx = 100 + numMsg = 200 + msgPerTx = numMsg / numTx + txPerSourceFile = numTx / numSourceFiles + sourceFileType = ".log" +) + +func TestExtractor_Errors(t *testing.T) { + testTable := []struct { + name string + cfg *extractorCfg + expectedErr error + }{ + { + "no source files", + &extractorCfg{ + fileType: ".log", + sourceDir: "./", + outputDir: ".", + }, + errNoSourceFilesFound, + }, + { + "invalid filetype", + &extractorCfg{ + fileType: "", + sourceDir: ".", + outputDir: ".", + }, + errInvalidFileType, + }, + { + "invalid source dir", + &extractorCfg{ + fileType: ".log", + sourceDir: "", + outputDir: ".", + }, + errInvalidSourceDir, + }, + { + "invalid output dir", + &extractorCfg{ + fileType: ".log", + sourceDir: ".", + outputDir: "", + }, + errInvalidOutputDir, + }, + } + + for _, testCase := range testTable { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) + defer cancelFn() + + err := execExtract(ctx, testCase.cfg) + assert.ErrorIs(t, err, testCase.expectedErr) + }) + } +} + +func TestValidFlow(t *testing.T) { + t.Parallel() + + // Generate temporary output dir + outputDir, err := os.MkdirTemp(".", "output") + require.NoError(t, err) + t.Cleanup(removeDir(t, outputDir)) + + // Generate temporary source dir + sourceDir, err := os.MkdirTemp(".", "source") + require.NoError(t, err) + t.Cleanup(removeDir(t, sourceDir)) + + // Set correct config + var cfg = &extractorCfg{ + fileType: sourceFileType, + sourceDir: sourceDir, + outputDir: outputDir, + } + + // Generate mock messages & mock files + mockStdMsg, mockAddPkgMsg := generateMockMsgs(t) + _ = generateSourceFiles(t, sourceDir, mockStdMsg) + + // Perform extraction + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) + defer cancelFn() + + require.NoError(t, execExtract(ctx, cfg)) + + for _, msg := range mockAddPkgMsg { + basePath := filepath.Join(outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) + + // Get metadata path & open metadata file + metadataPath := filepath.Join(basePath, packageMetadataFile) + file, err := os.Open(metadataPath) + require.NoError(t, err) + + // Read Metadata + reader := bufio.NewReader(file) + retrievedMetadata, _, err := reader.ReadLine() + require.NoError(t, err) + + // Compare metadata + expectedMetadata, err := json.Marshal(metadataFromMsg(msg)) + assert.Equal(t, expectedMetadata, retrievedMetadata) + + // Close metadata file + require.NoError(t, file.Close()) + + // Check package file content + for _, f := range msg.Package.Files { + filePath := filepath.Join(basePath, f.Name) + + // Open file + file, err := os.Open(filePath) + require.NoError(t, err) + + // Read file body + reader := bufio.NewReader(file) + retrievedFileBody, _, err := reader.ReadLine() + + // Compare file bodies + assert.Equal(t, f.Body, string(retrievedFileBody)) + } + } +} + +func TestFindFilePaths(t *testing.T) { + t.Parallel() + + tempDir, err := os.MkdirTemp(".", "test") + require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) + + testFiles := make([]string, numSourceFiles) + + for i := 0; i < numSourceFiles; i++ { + testFiles[i] = "sourceFile" + strconv.Itoa(i) + sourceFileType + } + + for _, file := range testFiles { + filePath := filepath.Join(tempDir, file) + err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm) + require.NoError(t, err) + + _, err = os.Create(filePath) + require.NoError(t, err) + } + + results, err := findFilePaths(tempDir, ".log") + require.NoError(t, err) + + expectedResults := make([]string, 0, len(testFiles)) + + for _, testFile := range testFiles { + expectedResults = append(expectedResults, filepath.Join(tempDir, testFile)) + } + + sort.Slice(results, func(i, j int) bool { + return results[i] < results[j] + }) + + sort.Slice(expectedResults, func(i, j int) bool { + return expectedResults[i] < expectedResults[j] + }) + + require.Equal(t, len(results), len(expectedResults)) + + for i, result := range results { + if result != expectedResults[i] { + require.Equal(t, result, expectedResults[i]) + } + } +} + +func TestExtractAddMessages(t *testing.T) { + t.Parallel() + + tempDir, err := os.MkdirTemp(".", "test") + require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) + + mockMsgs, mockMsgsAddPackage := generateMockMsgs(t) + sourceFiles := generateSourceFiles(t, tempDir, mockMsgs) + + var results []vm.MsgAddPackage + for _, sf := range sourceFiles { + res, err := extractAddMessages(sf) + require.NoError(t, err) + results = append(results, res...) + } + + sort.Slice(results, func(i, j int) bool { + return results[i].Package.Name < results[j].Package.Name + }) + sort.Slice(mockMsgsAddPackage, func(i, j int) bool { + return mockMsgsAddPackage[i].Package.Name < mockMsgsAddPackage[j].Package.Name + }) + + require.Equal(t, results, mockMsgsAddPackage) +} + +func TestWritePackageMetadata(t *testing.T) { + t.Parallel() + + _, mockMsgsAddPackage := generateMockMsgs(t) + + // Make temp dir + tempDir, err := os.MkdirTemp(".", "test") + require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) + + for _, msg := range mockMsgsAddPackage { + md := metadataFromMsg(msg) + + // Get output dir + outputDir := filepath.Join(tempDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) + + // Write dir before writing metadata + err := os.MkdirAll(outputDir, os.ModePerm) + require.NoError(t, err) + + // Write the metadata + err = writePackageMetadata(md, outputDir) + require.NoError(t, err) + + // Read file + file, err := os.Open(filepath.Join(outputDir, packageMetadataFile)) + require.NoError(t, err) + + reader := bufio.NewReader(file) + var unmarshalledMetadata Metadata + + raw, isPrefix, err := reader.ReadLine() + require.NoError(t, err) + require.Equal(t, isPrefix, false) + + err = json.Unmarshal(raw, &unmarshalledMetadata) + require.NoError(t, err) + + require.Equal(t, md, unmarshalledMetadata) + } +} +func TestWritePackageFiles(t *testing.T) { + t.Parallel() + + _, mockMsgsAddPackage := generateMockMsgs(t) + + tempDir, err := os.MkdirTemp(".", "test") + require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) + + for _, msg := range mockMsgsAddPackage { + // Get output dir + outputDir := filepath.Join(tempDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) + + // Write dir before writing metadata + err := os.MkdirAll(outputDir, os.ModePerm) + require.NoError(t, err) + + // Write the metadata + err = writePackageFiles(msg, outputDir) + require.NoError(t, err) + + // Read & compare file + for _, f := range msg.Package.Files { + contents, err := os.ReadFile(filepath.Join(outputDir, f.Name)) + require.NoError(t, err) + require.Equal(t, f.Body, string(contents)) + } + } +} + +// Helpers +func generateSourceFiles(t *testing.T, dir string, mockMsgs []std.Msg) []string { + t.Helper() + + var ( + mockTx = make([]std.Tx, numTx) + testFiles = make([]string, numSourceFiles) + ) + + // Generate transactions to wrap messages + for i := range mockTx { + mockTx[i] = std.Tx{ + Msgs: mockMsgs[:msgPerTx], + } + mockMsgs = mockMsgs[msgPerTx:] + } + + // Generate source file names + for i := 0; i < numSourceFiles; i++ { + testFiles[i] = "sourceFile" + strconv.Itoa(i) + sourceFileType + } + + // Generate source files + for _, file := range testFiles { + filePath := filepath.Join(dir, file) + + err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm) + require.NoError(t, err) + + file, err := os.Create(filePath) + require.NoError(t, err) + + for _, tx := range mockTx[:txPerSourceFile] { + err := writeTxToFile(t, tx, file) + if err != nil { + t.Fatal(err) + } + } + mockTx = mockTx[txPerSourceFile:] + } + + for i := 0; i < numSourceFiles; i++ { + testFiles[i] = filepath.Join(dir, testFiles[i]) + } + + return testFiles +} + +func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { + t.Helper() + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + testAddresses := []string{ + "g1jg8mtutu9khhfwc4nxmuhcpftf0pajdhfvsqf5", + "g1f4v282mwyhu29afke4vq5r2xzcm6z3ftnugcnv", + "g127jydsh6cms3lrtdenydxsckh23a8d6emqcvfa", + "g1u7y667z64x2h7vc6fmpcprgey4ck233jaww9zq", + "g1rrf8s5mrmu00sx04fzfsvc399fklpeg2x0a7mz", + } + + var ret []std.Msg + var addPkgRet []vm.MsgAddPackage + + pkgID := 0 + + for i := 0; i < numMsg; i++ { + var ( + randNum = int(r.Uint32()) + msg std.Msg + randAddressIndex = randNum % len(testAddresses) + maxDepositAmount = 5000 + callerAddr = addressFromString(t, testAddresses[randAddressIndex]) + deposit = std.NewCoins(std.NewCoin("foo", int64(randNum%maxDepositAmount+1))) + path = "gno.land/" + pkgName = "package" + strconv.Itoa(pkgID) + maxArgs = 2 + maxFileBodyLength = 200 + maxFilesPerPkg = 100 + ) + + if randNum%2 == 0 { + path += "r/" + } else { + path += "p/" + } + + switch randNum % 3 { + case 0: // Making vm.MsgAddPackage msg + var files []*std.MemFile + + path += pkgName + + for j := 0; j < randNum%maxFilesPerPkg+1; j++ { + file := &std.MemFile{ + Name: "t" + strconv.Itoa(j) + ".gno", + Body: randString(t, int(r.Uint32())%maxFileBodyLength+1), + } + files = append(files, file) + } + + msg = vm.MsgAddPackage{ + Creator: callerAddr, + Package: &std.MemPackage{ + Name: pkgName, + Path: path, + Files: files, + }, + Deposit: deposit, + } + addPkgRet = append(addPkgRet, msg.(vm.MsgAddPackage)) + break + case 1: // Making vm.MsgCall msg + args := make([]string, maxArgs-randNum%2) + for i := range args { + args[i] = randString(t, 10) + } + + msg = vm.MsgCall{ + Caller: callerAddr, + Send: deposit, + PkgPath: path + pkgName, + Func: "Func" + strconv.Itoa(i), + Args: args, + } + break + case 2: // Making bank.MsgSend + // Remove already used address + ta := append(testAddresses[:randAddressIndex], testAddresses[randAddressIndex+1:]...) + + msg = bank.MsgSend{ + FromAddress: callerAddr, + ToAddress: addressFromString(t, ta[randNum%len(ta)]), + Amount: deposit, + } + } + ret = append(ret, msg) + pkgID++ + } + + return ret, addPkgRet +} + +func addressFromString(t *testing.T, addr string) crypto.Address { + t.Helper() + + ret, err := crypto.AddressFromString(addr) + require.NoError(t, err) + + return ret +} + +func randString(t *testing.T, length int) string { + t.Helper() + buf := make([]byte, length) + _, _ = crand.Read(buf) + return base64.StdEncoding.EncodeToString(buf) +} + +func writeTxToFile(t *testing.T, tx std.Tx, file *os.File) error { + t.Helper() + + data, err := amino.MarshalJSON(tx) + require.NoError(t, err) + + // Write the JSON data as a line to the file + _, err = file.Write(data) + require.NoError(t, err) + + // Write a newline character to separate JSON objects + _, err = file.Write([]byte("\n")) + require.NoError(t, err) + + return nil +} + +func removeDir(t *testing.T, dirPath string) func() { + return func() { + err := os.RemoveAll(dirPath) + require.NoError(t, err) + } +} diff --git a/extractor/types.go b/extractor/types.go new file mode 100644 index 00000000..01daccbc --- /dev/null +++ b/extractor/types.go @@ -0,0 +1,20 @@ +package main + +import ( + "github.com/gnolang/gno/gno.land/pkg/sdk/vm" +) + +// Metadata defines the metadata info that accompanies +// gno source code +type Metadata struct { + Creator string `json:"creator"` // the creator of the source code (deployer) + Deposit string `json:"deposit"` // the deposit associated with the deployment +} + +// metadataFromMsg extracts the metadata from a message +func metadataFromMsg(msg vm.MsgAddPackage) Metadata { + return Metadata{ + Creator: msg.Creator.String(), + Deposit: msg.Deposit.String(), + } +}