From da5f8d625078e30c0ff712635216944d97bf14ce Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 4 Oct 2023 18:19:52 +0200 Subject: [PATCH 01/41] add extractor --- pkg-extract/.env.example | 2 + pkg-extract/.gitignore | 5 ++ pkg-extract/README.md | 2 + pkg-extract/extractor.go | 116 +++++++++++++++++++++++++++++++++++++++ pkg-extract/go.mod | 5 ++ pkg-extract/helpers.go | 30 ++++++++++ 6 files changed, 160 insertions(+) create mode 100644 pkg-extract/.env.example create mode 100644 pkg-extract/.gitignore create mode 100644 pkg-extract/README.md create mode 100644 pkg-extract/extractor.go create mode 100644 pkg-extract/go.mod create mode 100644 pkg-extract/helpers.go diff --git a/pkg-extract/.env.example b/pkg-extract/.env.example new file mode 100644 index 00000000..5c17a998 --- /dev/null +++ b/pkg-extract/.env.example @@ -0,0 +1,2 @@ +EXTRACTION_DIR="" +LOGS_DIR="" \ No newline at end of file diff --git a/pkg-extract/.gitignore b/pkg-extract/.gitignore new file mode 100644 index 00000000..41348382 --- /dev/null +++ b/pkg-extract/.gitignore @@ -0,0 +1,5 @@ +.idea +logs +extracted +.env +go.sum \ No newline at end of file diff --git a/pkg-extract/README.md b/pkg-extract/README.md new file mode 100644 index 00000000..564a2f7f --- /dev/null +++ b/pkg-extract/README.md @@ -0,0 +1,2 @@ +# pkg-extract +Simple parser for archiving packages uploaded to GnoLand \ No newline at end of file diff --git a/pkg-extract/extractor.go b/pkg-extract/extractor.go new file mode 100644 index 00000000..fe4b5a17 --- /dev/null +++ b/pkg-extract/extractor.go @@ -0,0 +1,116 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "github.com/joho/godotenv" + "log" + "os" + "strings" + "sync" +) + +func processFile(fileName string, wg *sync.WaitGroup) { + defer wg.Done() + + file, err := os.Open(fileName) + + if err != nil { + fmt.Print("Error opening file: ", err) + return + } + + defer func(file *os.File) { + err := file.Close() + if err != nil { + log.Fatalf("Could not close file %s properly.", file.Name()) + } + }(file) + + pkgMap := make(map[string]TX) // path -> package + + scanner := bufio.NewScanner(file) + for i := 0; scanner.Scan(); i++ { + var tx TX + + line := scanner.Text() + + if err := json.Unmarshal([]byte(line), &tx); err != nil { + fmt.Printf("Error parsing JSON at line %d: %v\n", i, err) + continue + } + + if tx.Msg[0].Type == "/vm.m_addpkg" { + path := tx.Msg[0].Package.Path + + // do not add duplicates + _, ok := pkgMap[path] + if !ok { + writePkg(tx) + pkgMap[path] = tx + } + } + } +} + +func writePkg(tx TX) { + msg := tx.Msg[0] + trimmedPath := strings.TrimLeft(msg.Package.Path, "gno.land/") + + // write dirs needed to write package + writePath := extractionDir + trimmedPath + "/" + if err := os.MkdirAll(writePath, os.ModePerm); err != nil { + log.Fatal(err) + } + + metadata, err := tx.MarshalMetadata() + + if err != nil { + log.Fatal("Failed to marshal metadata: " + trimmedPath) + } + + // write metadata + err = os.WriteFile(writePath+"pkg_metadata.json", metadata, 0644) + if err != nil { + log.Fatal(err) + } + + // write files + for _, file := range msg.Package.Files { + err := os.WriteFile(writePath+file.Name, []byte(file.Body), 0644) + if err != nil { + log.Fatal(err) + } + } +} + +var ( + // dirs have trailing slashes + logDir string + extractionDir string +) + +func main() { + // load env + err := godotenv.Load() + if err != nil { + log.Fatal("Error loading .env file") + } + logDir = os.Getenv("LOGS_DIR") + extractionDir = os.Getenv("EXTRACTION_DIR") + + // read log files + entries, err := os.ReadDir(logDir) + if err != nil { + log.Fatal(err) + } + + // goroutine for each log file + var wg sync.WaitGroup + for _, e := range entries { + wg.Add(1) + go processFile(logDir+e.Name(), &wg) + } + wg.Wait() +} diff --git a/pkg-extract/go.mod b/pkg-extract/go.mod new file mode 100644 index 00000000..400caac3 --- /dev/null +++ b/pkg-extract/go.mod @@ -0,0 +1,5 @@ +module pkg-extract + +go 1.21.0 + +require github.com/joho/godotenv v1.5.1 // indirect diff --git a/pkg-extract/helpers.go b/pkg-extract/helpers.go new file mode 100644 index 00000000..21f65335 --- /dev/null +++ b/pkg-extract/helpers.go @@ -0,0 +1,30 @@ +package main + +import ( + "encoding/json" +) + +type TX struct { + Msg []struct { + Type string `json:"@type"` + Creator string `json:"creator"` + Package struct { + Name string `json:"Name"` + Path string `json:"Path"` + Files []struct { + Name string `json:"Name"` + Body string `json:"Body"` + } `json:"Files"` + } `json:"package"` + Deposit string `json:"deposit"` + } `json:"msg"` +} + +func (tx TX) MarshalMetadata() ([]byte, error) { + data := map[string]interface{}{ + "creator": tx.Msg[0].Creator, + "deposit": tx.Msg[0].Deposit, + // add what is needed + } + return json.MarshalIndent(data, "", " ") +} From 596f696f072bf0b55e94eb57b83c7810a90fe3bc Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 4 Oct 2023 18:22:42 +0200 Subject: [PATCH 02/41] file refactor --- {pkg-extract => extractor}/.env.example | 0 {pkg-extract => extractor}/.gitignore | 0 {pkg-extract => extractor}/README.md | 0 {pkg-extract => extractor}/extractor.go | 2 +- {pkg-extract => extractor}/go.mod | 2 +- {pkg-extract => extractor}/helpers.go | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) rename {pkg-extract => extractor}/.env.example (100%) rename {pkg-extract => extractor}/.gitignore (100%) rename {pkg-extract => extractor}/README.md (100%) rename {pkg-extract => extractor}/extractor.go (99%) rename {pkg-extract => extractor}/go.mod (77%) rename {pkg-extract => extractor}/helpers.go (97%) diff --git a/pkg-extract/.env.example b/extractor/.env.example similarity index 100% rename from pkg-extract/.env.example rename to extractor/.env.example diff --git a/pkg-extract/.gitignore b/extractor/.gitignore similarity index 100% rename from pkg-extract/.gitignore rename to extractor/.gitignore diff --git a/pkg-extract/README.md b/extractor/README.md similarity index 100% rename from pkg-extract/README.md rename to extractor/README.md diff --git a/pkg-extract/extractor.go b/extractor/extractor.go similarity index 99% rename from pkg-extract/extractor.go rename to extractor/extractor.go index fe4b5a17..dfd03306 100644 --- a/pkg-extract/extractor.go +++ b/extractor/extractor.go @@ -1,4 +1,4 @@ -package main +package extractor import ( "bufio" diff --git a/pkg-extract/go.mod b/extractor/go.mod similarity index 77% rename from pkg-extract/go.mod rename to extractor/go.mod index 400caac3..736a7fb1 100644 --- a/pkg-extract/go.mod +++ b/extractor/go.mod @@ -1,4 +1,4 @@ -module pkg-extract +module extractor go 1.21.0 diff --git a/pkg-extract/helpers.go b/extractor/helpers.go similarity index 97% rename from pkg-extract/helpers.go rename to extractor/helpers.go index 21f65335..e33431f3 100644 --- a/pkg-extract/helpers.go +++ b/extractor/helpers.go @@ -1,4 +1,4 @@ -package main +package extractor import ( "encoding/json" From af2906384991bddec972b22eb60b786d2a795926 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 4 Oct 2023 18:48:20 +0200 Subject: [PATCH 03/41] added walker log file discovery --- .gitignore | 3 +++ extractor/extractor.go | 30 +++++++++++++++++++++++------- extractor/helpers.go | 2 +- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index b134141e..4011e9f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ txexport.log txexport.log.gz +.idea +extractor/extracted +extractor/.env \ No newline at end of file diff --git a/extractor/extractor.go b/extractor/extractor.go index dfd03306..8cfdcbd1 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -1,4 +1,4 @@ -package extractor +package main import ( "bufio" @@ -7,6 +7,7 @@ import ( "github.com/joho/godotenv" "log" "os" + "path/filepath" "strings" "sync" ) @@ -100,17 +101,32 @@ func main() { logDir = os.Getenv("LOGS_DIR") extractionDir = os.Getenv("EXTRACTION_DIR") - // read log files - entries, err := os.ReadDir(logDir) + logFiles := []string{} + + walker := func(path string, info os.FileInfo, err error) error { + if err != nil { + fmt.Println("Error accessing file:", err) + return err + } + + if !info.IsDir() && strings.HasSuffix(info.Name(), ".log") { + logFiles = append(logFiles, path) + } + + return nil + } + + err = filepath.Walk(logDir, walker) if err != nil { - log.Fatal(err) + fmt.Println("Error walking directory:", err) } - // goroutine for each log file + //goroutine for each log file var wg sync.WaitGroup - for _, e := range entries { + for _, file := range logFiles { wg.Add(1) - go processFile(logDir+e.Name(), &wg) + fmt.Printf("opening goroutine for: %s\n", file) + go processFile(file, &wg) } wg.Wait() } diff --git a/extractor/helpers.go b/extractor/helpers.go index e33431f3..21f65335 100644 --- a/extractor/helpers.go +++ b/extractor/helpers.go @@ -1,4 +1,4 @@ -package extractor +package main import ( "encoding/json" From 2fbdfbd5d748167b3080cfec2c104f0b2c1e2ce5 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 4 Oct 2023 18:59:29 +0200 Subject: [PATCH 04/41] add whitespace --- extractor/extractor.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/extractor.go b/extractor/extractor.go index 8cfdcbd1..ae800601 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -118,14 +118,13 @@ func main() { err = filepath.Walk(logDir, walker) if err != nil { - fmt.Println("Error walking directory:", err) + fmt.Println("Error walking directory: ", err) } //goroutine for each log file var wg sync.WaitGroup for _, file := range logFiles { wg.Add(1) - fmt.Printf("opening goroutine for: %s\n", file) go processFile(file, &wg) } wg.Wait() From 48ab9ff29155e73527fc524581d531b060c2d6b2 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Thu, 12 Oct 2023 00:05:26 +0200 Subject: [PATCH 05/41] WIP full refactor in progress --- extractor/.gitignore | 2 +- extractor/extractor.go | 131 ------------------ extractor/go.mod | 42 +++++- extractor/go.sum | 219 +++++++++++++++++++++++++++++ extractor/helpers.go | 30 ---- extractor/main.go | 308 +++++++++++++++++++++++++++++++++++++++++ extractor/main_test.go | 55 ++++++++ extractor/types.go | 20 +++ 8 files changed, 643 insertions(+), 164 deletions(-) delete mode 100644 extractor/extractor.go create mode 100644 extractor/go.sum delete mode 100644 extractor/helpers.go create mode 100644 extractor/main.go create mode 100644 extractor/main_test.go create mode 100644 extractor/types.go diff --git a/extractor/.gitignore b/extractor/.gitignore index 41348382..3f5e539f 100644 --- a/extractor/.gitignore +++ b/extractor/.gitignore @@ -2,4 +2,4 @@ logs extracted .env -go.sum \ No newline at end of file +test \ No newline at end of file diff --git a/extractor/extractor.go b/extractor/extractor.go deleted file mode 100644 index ae800601..00000000 --- a/extractor/extractor.go +++ /dev/null @@ -1,131 +0,0 @@ -package main - -import ( - "bufio" - "encoding/json" - "fmt" - "github.com/joho/godotenv" - "log" - "os" - "path/filepath" - "strings" - "sync" -) - -func processFile(fileName string, wg *sync.WaitGroup) { - defer wg.Done() - - file, err := os.Open(fileName) - - if err != nil { - fmt.Print("Error opening file: ", err) - return - } - - defer func(file *os.File) { - err := file.Close() - if err != nil { - log.Fatalf("Could not close file %s properly.", file.Name()) - } - }(file) - - pkgMap := make(map[string]TX) // path -> package - - scanner := bufio.NewScanner(file) - for i := 0; scanner.Scan(); i++ { - var tx TX - - line := scanner.Text() - - if err := json.Unmarshal([]byte(line), &tx); err != nil { - fmt.Printf("Error parsing JSON at line %d: %v\n", i, err) - continue - } - - if tx.Msg[0].Type == "/vm.m_addpkg" { - path := tx.Msg[0].Package.Path - - // do not add duplicates - _, ok := pkgMap[path] - if !ok { - writePkg(tx) - pkgMap[path] = tx - } - } - } -} - -func writePkg(tx TX) { - msg := tx.Msg[0] - trimmedPath := strings.TrimLeft(msg.Package.Path, "gno.land/") - - // write dirs needed to write package - writePath := extractionDir + trimmedPath + "/" - if err := os.MkdirAll(writePath, os.ModePerm); err != nil { - log.Fatal(err) - } - - metadata, err := tx.MarshalMetadata() - - if err != nil { - log.Fatal("Failed to marshal metadata: " + trimmedPath) - } - - // write metadata - err = os.WriteFile(writePath+"pkg_metadata.json", metadata, 0644) - if err != nil { - log.Fatal(err) - } - - // write files - for _, file := range msg.Package.Files { - err := os.WriteFile(writePath+file.Name, []byte(file.Body), 0644) - if err != nil { - log.Fatal(err) - } - } -} - -var ( - // dirs have trailing slashes - logDir string - extractionDir string -) - -func main() { - // load env - err := godotenv.Load() - if err != nil { - log.Fatal("Error loading .env file") - } - logDir = os.Getenv("LOGS_DIR") - extractionDir = os.Getenv("EXTRACTION_DIR") - - logFiles := []string{} - - walker := func(path string, info os.FileInfo, err error) error { - if err != nil { - fmt.Println("Error accessing file:", err) - return err - } - - if !info.IsDir() && strings.HasSuffix(info.Name(), ".log") { - logFiles = append(logFiles, path) - } - - return nil - } - - err = filepath.Walk(logDir, walker) - if err != nil { - fmt.Println("Error walking directory: ", err) - } - - //goroutine for each log file - var wg sync.WaitGroup - for _, file := range logFiles { - wg.Add(1) - go processFile(file, &wg) - } - wg.Wait() -} diff --git a/extractor/go.mod b/extractor/go.mod index 736a7fb1..712750d3 100644 --- a/extractor/go.mod +++ b/extractor/go.mod @@ -1,5 +1,43 @@ module extractor -go 1.21.0 +go 1.20 -require github.com/joho/godotenv v1.5.1 // indirect +require ( + github.com/gnolang/gno v0.0.0-20231006162410-fa8eb7753dc5 + github.com/peterbourgon/ff/v3 v3.4.0 + golang.org/x/sync v0.4.0 +) + +require ( + github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c // indirect + github.com/btcsuite/btcd/btcutil v1.0.0 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cockroachdb/apd v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/badger/v3 v3.2103.4 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gnolang/goleveldb v0.0.9 // indirect + github.com/gnolang/overflow v0.0.0-20170615021017-4d914c927216 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/jmhodges/levigo v1.0.0 // indirect + github.com/klauspost/compress v1.12.3 // indirect + github.com/linxGnu/grocksdb v1.8.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect + go.etcd.io/bbolt v1.3.7 // indirect + go.opencensus.io v0.22.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/tools v0.6.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/extractor/go.sum b/extractor/go.sum new file mode 100644 index 00000000..b679b22c --- /dev/null +++ b/extractor/go.sum @@ -0,0 +1,219 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c h1:lnAMg3ra/Gw4AkRMxrxYs8nrprWsHowg8H9zaYsJOo4= +github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= +github.com/btcsuite/btcd/btcutil v1.0.0 h1:dB36qRTOucIh6NUe40UCieOS+axPhP6VNyRtYkTUKKk= +github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v3 v3.2103.4 h1:WE1B07YNTTJTtG9xjBcSW2wn0RJLyiV99h959RKZqM4= +github.com/dgraph-io/badger/v3 v3.2103.4/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gnolang/gno v0.0.0-20231006162410-fa8eb7753dc5 h1:807vK/Yz1dUO05Op9Sc1hwgz7rugyC+ejStBHDT9bsE= +github.com/gnolang/gno v0.0.0-20231006162410-fa8eb7753dc5/go.mod h1:naCUZBEdOgcD9y1wb0MjN8fpWicsrnxrMhIX1j0tc8E= +github.com/gnolang/goleveldb v0.0.9 h1:Q7rGko9oXMKtQA+Apeeed5a3sjba/mcDhzJGoTVLCKE= +github.com/gnolang/goleveldb v0.0.9/go.mod h1:Dz6p9bmpy/FBESTgduiThZt5mToVDipcHGzj/zUOo8E= +github.com/gnolang/overflow v0.0.0-20170615021017-4d914c927216 h1:GKvsK3oLWG9B1GL7WP/VqwM6C92j5tIvB844oggL9Lk= +github.com/gnolang/overflow v0.0.0-20170615021017-4d914c927216/go.mod h1:xJhtEL7ahjM1WJipt89gel8tHzfIl/LyMY+lCYh38d8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jaekwon/testify v1.6.1 h1:4AtAJcR9GzXN5W4DdY7ie74iCPiJV1JJUJL90t2ZUyw= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= +github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo= +github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= +github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/extractor/helpers.go b/extractor/helpers.go deleted file mode 100644 index 21f65335..00000000 --- a/extractor/helpers.go +++ /dev/null @@ -1,30 +0,0 @@ -package main - -import ( - "encoding/json" -) - -type TX struct { - Msg []struct { - Type string `json:"@type"` - Creator string `json:"creator"` - Package struct { - Name string `json:"Name"` - Path string `json:"Path"` - Files []struct { - Name string `json:"Name"` - Body string `json:"Body"` - } `json:"Files"` - } `json:"package"` - Deposit string `json:"deposit"` - } `json:"msg"` -} - -func (tx TX) MarshalMetadata() ([]byte, error) { - data := map[string]interface{}{ - "creator": tx.Msg[0].Creator, - "deposit": tx.Msg[0].Deposit, - // add what is needed - } - return json.MarshalIndent(data, "", " ") -} diff --git a/extractor/main.go b/extractor/main.go new file mode 100644 index 00000000..0563e1d0 --- /dev/null +++ b/extractor/main.go @@ -0,0 +1,308 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "github.com/gnolang/gno/gno.land/pkg/sdk/vm" + "github.com/gnolang/gno/tm2/pkg/amino" + "github.com/gnolang/gno/tm2/pkg/std" + "github.com/peterbourgon/ff/v3/ffcli" + "golang.org/x/sync/errgroup" + "io" + "os" + "path/filepath" + "strings" +) + +// Define constants +const ( + packageMetadataFile = "pkg_metadata.json" + MaxScanTokenSize = 64 * 1024 +) + +// Define errors +var ( + errInvalidFileType = errors.New("invalid file type specified") + errInvalidSourceDir = errors.New("invalid source directory") + errInvalidOutputDir = errors.New("invalid output directory") +) + +// Define extractor config +type extractorCfg struct { + fileType string + sourceDir string + outputDir string +} + +func main() { + var ( + cfg = &extractorCfg{} + fs = flag.NewFlagSet("root", flag.ExitOnError) + ) + + // Register the flags + cfg.registerFlags(fs) + + // Create the command + cmd := &ffcli.Command{ + ShortUsage: "[flags] [...]", + LongHelp: "The Gno / TM2 source code extractor service", + FlagSet: fs, + Exec: func(ctx context.Context, _ []string) error { + return execExtract(ctx, cfg) + }, + } + + // Run the command + if err := cmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%+v", err) + + os.Exit(1) + } +} + +// registerFlags registers the extractor service flag set +func (c *extractorCfg) registerFlags(fs *flag.FlagSet) { + fs.StringVar( + &c.fileType, + "file-type", + ".log", + "the file type for analysis, with a preceding period (ex .type)", + ) + + fs.StringVar( + &c.sourceDir, + "source-dir", + ".", + "the root folder containing transaction data", + ) + + fs.StringVar( + &c.outputDir, + "output-dir", + "./extracted", + "the output directory for the extracted Gno source code", + ) +} + +// execExtract runs the extract service for Gno source code +func execExtract(ctx context.Context, cfg *extractorCfg) error { + // Check the file type is valid + if cfg.fileType == "" { + return errInvalidFileType + } + + // Check the source dir is valid + if cfg.sourceDir == "" { + return errInvalidSourceDir + } + + // Check the output dir is valid + if cfg.outputDir == "" { + return errInvalidOutputDir + } + + // Find the files that need to be analyzed + sourceFiles, findErr := findFilePaths(cfg.sourceDir, cfg.fileType) + if findErr != nil { + return fmt.Errorf("unable to find file paths, %w", findErr) + } + + // Concurrently process the source files + g, ctx := errgroup.WithContext(ctx) + + for _, sourceFile := range sourceFiles { + sourceFile := sourceFile + + g.Go(func() error { + // Extract messages + msgs, processErr := processSourceFile(sourceFile) + if processErr != nil { + return processErr + } + + // Process messages + for _, msg := range msgs { + outputDir := strings.TrimLeft(msg.Package.Path, "gno.land/") + + // Write dir before writing files + if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil { + return fmt.Errorf("unable to write dir, %w", dirWriteErr) + } + + // Write the package source code + if writeErr := writePackageFiles(msg, outputDir); writeErr != nil { + return writeErr + } + + // Write the package metadata + if writeErr := writePackageMetadata(metadataFromMsg(msg), outputDir); writeErr != nil { + return writeErr + } + } + + return nil + }) + } + + return g.Wait() +} + +// writePackageFiles writes all files from a single package to the output directory +func writePackageFiles(msg vm.MsgAddPackage, outputDir string) error { + for _, file := range msg.Package.Files { + // Get the output path + writePath := filepath.Join(outputDir, file.Name) + + if writeErr := os.WriteFile(writePath, []byte(file.Body), 0644); writeErr != nil { + return fmt.Errorf("unable to write file %s, %w", file.Name, writeErr) + } + } + + return nil +} + +// writePackageMetadata writes the package metadata to the output directory +func writePackageMetadata(metadata Metadata, outputDir string) error { + + fmt.Println("writing metadata: ", outputDir) + // Get the output path + writePath := filepath.Join(outputDir, packageMetadataFile) + + // Get the JSON metadata + metadataRaw, marshalErr := json.Marshal(metadata) + if marshalErr != nil { + return fmt.Errorf("unable to JSON marshal metadata, %w", marshalErr) + } + + if writeErr := os.WriteFile(writePath, metadataRaw, 0644); writeErr != nil { + return fmt.Errorf("unable to write package metadata, %w", writeErr) + } + + return nil +} + +func processSourceFile(filePath string) ([]vm.MsgAddPackage, error) { + file, openErr := os.Open(filePath) + if openErr != nil { + return nil, fmt.Errorf("unable to open file, %w", openErr) + } + + cleanup := func() error { + if closeErr := file.Close(); closeErr != nil { + return fmt.Errorf("unable to gracefully close file, %w", closeErr) + } + return nil + } + + reader := bufio.NewReader(file) + + // Used to track what was parsed in the past + //touchMap := make(map[string]vm.MsgAddPackage) + + // Msg array to be returned for further processing + msgArr := make([]vm.MsgAddPackage, 0) + + // Buffer to handle lines longer than 64kb + //tempBuf := make([]byte, MaxScanTokenSize) + + i := 0 + for { + var tx std.Tx + line, _, err := reader.ReadLine() + + // Exit if no more lines in file + if err != nil { + if errors.Is(err, io.EOF) { + fmt.Println("Reading file finished...") + break + } + } + + //// If line is too long, save it in a temporary buffer and continue reading line + //if isPrefix { + // tempBuf = append(tempBuf, line...) + // continue + //} + // + //// Handle long lines + //if len(tempBuf) != 0 { + // // Append last part of line to temporary buffer + // tempBuf = append(tempBuf, line...) + // + // // Use line variable to pass it on to amino + // line = tempBuf + //} + + if aminoErr := amino.UnmarshalJSON(line, &tx); aminoErr != nil { + fmt.Printf("Error while parsing amino JSON at line %d: %v\n", i, aminoErr.Error()) + continue + } + + fmt.Println(tx) + + //// Reset tempBuf in case it was used for a long line + //if tempBuf != nil { + // tempBuf = nil + //} + + //for _, msg := range tx.Msgs { + // // Only MsgAddPkg should be parsed + // if msg.Type() != "/vm.m_addpkg" { + // continue + // } + // + // msgAddPkg := msg.(vm.MsgAddPackage) + // path := msgAddPkg.Package.Path + // + // if _, parsed := touchMap[path]; parsed { + // // Package already parsed + // continue + // } + // + // touchMap[path] = msgAddPkg + // msgArr = append(msgArr, msgAddPkg) + //} + i++ + } + + return msgArr, cleanup() +} + +// findFilePaths gathers the file paths for specific file types +func findFilePaths(startPath string, fileType string) ([]string, error) { + filePaths := make([]string, 0) + + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + fmt.Println("Error accessing file:", err) + return err + } + + // Check if the file is a dir + if info.IsDir() { + return nil + } + + // Check if the file type matches + if !strings.HasSuffix(info.Name(), fileType) { + return nil + } + + // File is not a directory, and is of the type + filePaths = append(filePaths, path) + + return nil + } + + // Walk the directory root recursively + if walkErr := filepath.Walk(startPath, walkFn); walkErr != nil { + return nil, fmt.Errorf("unable to walk directory, %v", walkErr) + } + + return filePaths, nil +} diff --git a/extractor/main_test.go b/extractor/main_test.go new file mode 100644 index 00000000..e888b1c1 --- /dev/null +++ b/extractor/main_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFindFilePaths(t *testing.T) { + + tempDir, err := os.MkdirTemp(".", "test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + testFiles := []string{ + "file1.txt", + "file2.log", + "file3.log", + "file4.log", + } + + for _, file := range testFiles { + filePath := filepath.Join(tempDir, file) + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + t.Fatal(err) + } + if _, err := os.Create(filePath); err != nil { + t.Fatal(err) + } + } + + results, err := findFilePaths(tempDir, ".log") + if err != nil { + t.Fatal(err) + } + + expectedResults := []string{ + filepath.Join(tempDir, "file2.log"), + filepath.Join(tempDir, "file3.log"), + filepath.Join(tempDir, "file4.log"), + } + + if len(results) != len(expectedResults) { + t.Fatalf("Expected %d results, but got %d", len(expectedResults), len(results)) + } + + for i, result := range results { + if result != expectedResults[i] { + t.Errorf("Expected %s, but got %s", expectedResults[i], result) + } + } + +} diff --git a/extractor/types.go b/extractor/types.go new file mode 100644 index 00000000..01daccbc --- /dev/null +++ b/extractor/types.go @@ -0,0 +1,20 @@ +package main + +import ( + "github.com/gnolang/gno/gno.land/pkg/sdk/vm" +) + +// Metadata defines the metadata info that accompanies +// gno source code +type Metadata struct { + Creator string `json:"creator"` // the creator of the source code (deployer) + Deposit string `json:"deposit"` // the deposit associated with the deployment +} + +// metadataFromMsg extracts the metadata from a message +func metadataFromMsg(msg vm.MsgAddPackage) Metadata { + return Metadata{ + Creator: msg.Creator.String(), + Deposit: msg.Deposit.String(), + } +} From ba70aac9fc0cf3cde93af76bcc49e668e58f0823 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Thu, 12 Oct 2023 13:58:58 +0200 Subject: [PATCH 06/41] fixed decoding --- extractor/main.go | 86 ++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 0563e1d0..da6f2bf1 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -127,7 +127,8 @@ func execExtract(ctx context.Context, cfg *extractorCfg) error { // Process messages for _, msg := range msgs { - outputDir := strings.TrimLeft(msg.Package.Path, "gno.land/") + + outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) // Write dir before writing files if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil { @@ -202,18 +203,18 @@ func processSourceFile(filePath string) ([]vm.MsgAddPackage, error) { reader := bufio.NewReader(file) // Used to track what was parsed in the past - //touchMap := make(map[string]vm.MsgAddPackage) + touchMap := make(map[string]vm.MsgAddPackage) // Msg array to be returned for further processing msgArr := make([]vm.MsgAddPackage, 0) // Buffer to handle lines longer than 64kb - //tempBuf := make([]byte, MaxScanTokenSize) + tempBuf := make([]byte, 0) i := 0 for { var tx std.Tx - line, _, err := reader.ReadLine() + line, isPrefix, err := reader.ReadLine() // Exit if no more lines in file if err != nil { @@ -223,50 +224,51 @@ func processSourceFile(filePath string) ([]vm.MsgAddPackage, error) { } } - //// If line is too long, save it in a temporary buffer and continue reading line - //if isPrefix { - // tempBuf = append(tempBuf, line...) - // continue - //} - // - //// Handle long lines - //if len(tempBuf) != 0 { - // // Append last part of line to temporary buffer - // tempBuf = append(tempBuf, line...) - // - // // Use line variable to pass it on to amino - // line = tempBuf - //} + // If line is too long, save it in a temporary buffer and continue reading line + if isPrefix { + tempBuf = append(tempBuf, line...) + continue + } + + // Handle long lines + if len(tempBuf) != 0 { + // Append last part of line to temporary buffer + tempBuf = append(tempBuf, line...) + + // Use line variable to pass it on to amino + line = tempBuf + } if aminoErr := amino.UnmarshalJSON(line, &tx); aminoErr != nil { fmt.Printf("Error while parsing amino JSON at line %d: %v\n", i, aminoErr.Error()) continue } - fmt.Println(tx) - - //// Reset tempBuf in case it was used for a long line - //if tempBuf != nil { - // tempBuf = nil - //} - - //for _, msg := range tx.Msgs { - // // Only MsgAddPkg should be parsed - // if msg.Type() != "/vm.m_addpkg" { - // continue - // } - // - // msgAddPkg := msg.(vm.MsgAddPackage) - // path := msgAddPkg.Package.Path - // - // if _, parsed := touchMap[path]; parsed { - // // Package already parsed - // continue - // } - // - // touchMap[path] = msgAddPkg - // msgArr = append(msgArr, msgAddPkg) - //} + // Reset tempBuf in case it was used for a long line + if tempBuf != nil { + tempBuf = nil + } + + for _, msg := range tx.Msgs { + // Only MsgAddPkg should be parsed + + if msg.Type() != "add_package" { + continue + } + + msgAddPkg := msg.(vm.MsgAddPackage) + + path := msgAddPkg.Package.Path + + if _, parsed := touchMap[path]; parsed { + // Package already parsed + continue + } + + touchMap[path] = msgAddPkg + msgArr = append(msgArr, msgAddPkg) + + } i++ } From 485dfb479fccd3cad6bb49a1108554f8e35abe34 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Fri, 20 Oct 2023 11:04:57 -0400 Subject: [PATCH 07/41] add tests, updated README --- extractor/.env.example | 2 - extractor/.gitignore | 1 + extractor/README.md | 25 ++- extractor/go.mod | 4 + extractor/go.sum | 7 + extractor/main.go | 7 +- extractor/main_test.go | 370 +++++++++++++++++++++++++++++++++++++++-- 7 files changed, 397 insertions(+), 19 deletions(-) delete mode 100644 extractor/.env.example diff --git a/extractor/.env.example b/extractor/.env.example deleted file mode 100644 index 5c17a998..00000000 --- a/extractor/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -EXTRACTION_DIR="" -LOGS_DIR="" \ No newline at end of file diff --git a/extractor/.gitignore b/extractor/.gitignore index 3f5e539f..03603429 100644 --- a/extractor/.gitignore +++ b/extractor/.gitignore @@ -2,4 +2,5 @@ logs extracted .env +testdir test \ No newline at end of file diff --git a/extractor/README.md b/extractor/README.md index 564a2f7f..60f1066e 100644 --- a/extractor/README.md +++ b/extractor/README.md @@ -1,2 +1,23 @@ -# pkg-extract -Simple parser for archiving packages uploaded to GnoLand \ No newline at end of file +# Gno Source Code Extractor + +This tool is a simple parser to extract source code (packages & realms) from logs created by the [tx-archive](https://github.com/gnolang/tx-archive) tool for Gno chains. + +## Running the extractor + +The extractor takes in three arguments: +- the filetype of the archive files, +- output directory for the extracted packages, +- the root directory where the archive files are located. + +``` +USAGE + [flags] [...] + +The Gno source code extractor service + +FLAGS + -file-type .jsonl the file type for analysis, with a preceding period (ie .log) + -output-dir ./extracted the output directory for the extracted Gno source code + -source-dir . the root folder containing transaction data +``` + diff --git a/extractor/go.mod b/extractor/go.mod index 712750d3..cddb9c5c 100644 --- a/extractor/go.mod +++ b/extractor/go.mod @@ -4,7 +4,9 @@ go 1.20 require ( github.com/gnolang/gno v0.0.0-20231006162410-fa8eb7753dc5 + github.com/go-test/deep v1.1.0 github.com/peterbourgon/ff/v3 v3.4.0 + github.com/stretchr/testify v1.8.4 golang.org/x/sync v0.4.0 ) @@ -30,6 +32,7 @@ require ( github.com/klauspost/compress v1.12.3 // indirect github.com/linxGnu/grocksdb v1.8.4 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect go.etcd.io/bbolt v1.3.7 // indirect go.opencensus.io v0.22.5 // indirect @@ -40,4 +43,5 @@ require ( golang.org/x/sys v0.12.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/extractor/go.sum b/extractor/go.sum index b679b22c..b5daecbd 100644 --- a/extractor/go.sum +++ b/extractor/go.sum @@ -51,6 +51,8 @@ github.com/gnolang/goleveldb v0.0.9 h1:Q7rGko9oXMKtQA+Apeeed5a3sjba/mcDhzJGoTVLC github.com/gnolang/goleveldb v0.0.9/go.mod h1:Dz6p9bmpy/FBESTgduiThZt5mToVDipcHGzj/zUOo8E= github.com/gnolang/overflow v0.0.0-20170615021017-4d914c927216 h1:GKvsK3oLWG9B1GL7WP/VqwM6C92j5tIvB844oggL9Lk= github.com/gnolang/overflow v0.0.0-20170615021017-4d914c927216/go.mod h1:xJhtEL7ahjM1WJipt89gel8tHzfIl/LyMY+lCYh38d8= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -86,9 +88,11 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo= github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= @@ -123,6 +127,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -209,6 +214,7 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -216,4 +222,5 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/extractor/main.go b/extractor/main.go index da6f2bf1..9f8707b1 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -21,7 +21,6 @@ import ( // Define constants const ( packageMetadataFile = "pkg_metadata.json" - MaxScanTokenSize = 64 * 1024 ) // Define errors @@ -70,8 +69,8 @@ func (c *extractorCfg) registerFlags(fs *flag.FlagSet) { fs.StringVar( &c.fileType, "file-type", - ".log", - "the file type for analysis, with a preceding period (ex .type)", + ".jsonl", + "the file type for analysis, with a preceding period (ie .jsonl)", ) fs.StringVar( @@ -170,7 +169,6 @@ func writePackageFiles(msg vm.MsgAddPackage, outputDir string) error { // writePackageMetadata writes the package metadata to the output directory func writePackageMetadata(metadata Metadata, outputDir string) error { - fmt.Println("writing metadata: ", outputDir) // Get the output path writePath := filepath.Join(outputDir, packageMetadataFile) @@ -219,7 +217,6 @@ func processSourceFile(filePath string) ([]vm.MsgAddPackage, error) { // Exit if no more lines in file if err != nil { if errors.Is(err, io.EOF) { - fmt.Println("Reading file finished...") break } } diff --git a/extractor/main_test.go b/extractor/main_test.go index e888b1c1..9b09da8d 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -1,24 +1,55 @@ package main import ( + "bufio" + "encoding/json" + "fmt" + "github.com/gnolang/gno/gno.land/pkg/sdk/vm" + "github.com/gnolang/gno/tm2/pkg/amino" + "github.com/gnolang/gno/tm2/pkg/crypto" + "github.com/gnolang/gno/tm2/pkg/sdk/bank" + "github.com/gnolang/gno/tm2/pkg/std" + "github.com/go-test/deep" + "github.com/stretchr/testify/require" + "math/rand" "os" "path/filepath" + "sort" + "strconv" + "strings" "testing" + "time" ) -func TestFindFilePaths(t *testing.T) { +const ( + NumSourceFiles = 20 + NumTx = 100 + NumMsg = 200 + MaxFilesPerPkg = 100 + MaxFileBodyLength = 200 + MsgPerTx = NumMsg / NumTx + TxPerSourceFile = NumTx / NumSourceFiles + MaxDepositAmount = 5000 + MaxArgs = 2 + SourceFileType = ".log" +) + +var ( + chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890\\n\\t!@#$%^&*()_+> Date: Thu, 26 Oct 2023 14:06:01 -0400 Subject: [PATCH 08/41] removed unused errors, removed args --- .gitignore | 3 +-- extractor/README.md | 2 +- extractor/main.go | 17 +++++------------ 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 4011e9f5..d65c31ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ txexport.log txexport.log.gz .idea -extractor/extracted -extractor/.env \ No newline at end of file +extractor/extracted \ No newline at end of file diff --git a/extractor/README.md b/extractor/README.md index 60f1066e..dcb7bc9d 100644 --- a/extractor/README.md +++ b/extractor/README.md @@ -11,7 +11,7 @@ The extractor takes in three arguments: ``` USAGE - [flags] [...] + [flags] The Gno source code extractor service diff --git a/extractor/main.go b/extractor/main.go index 9f8707b1..51cfab87 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -23,13 +23,6 @@ const ( packageMetadataFile = "pkg_metadata.json" ) -// Define errors -var ( - errInvalidFileType = errors.New("invalid file type specified") - errInvalidSourceDir = errors.New("invalid source directory") - errInvalidOutputDir = errors.New("invalid output directory") -) - // Define extractor config type extractorCfg struct { fileType string @@ -48,7 +41,7 @@ func main() { // Create the command cmd := &ffcli.Command{ - ShortUsage: "[flags] [...]", + ShortUsage: "[flags]", LongHelp: "The Gno / TM2 source code extractor service", FlagSet: fs, Exec: func(ctx context.Context, _ []string) error { @@ -56,7 +49,7 @@ func main() { }, } - // Run the command + // Run the command11 if err := cmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil { _, _ = fmt.Fprintf(os.Stderr, "%+v", err) @@ -92,17 +85,17 @@ func (c *extractorCfg) registerFlags(fs *flag.FlagSet) { func execExtract(ctx context.Context, cfg *extractorCfg) error { // Check the file type is valid if cfg.fileType == "" { - return errInvalidFileType + return errors.New("invalid file type specified") } // Check the source dir is valid if cfg.sourceDir == "" { - return errInvalidSourceDir + return errors.New("invalid source directory") } // Check the output dir is valid if cfg.outputDir == "" { - return errInvalidOutputDir + return errors.New("invalid output directory") } // Find the files that need to be analyzed From 7bedfa35b575785e352ed7fe3a69c900e106bb55 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Fri, 27 Oct 2023 17:38:46 -0400 Subject: [PATCH 09/41] test fixes --- extractor/main_test.go | 81 +++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index 9b09da8d..54b33357 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -22,16 +22,16 @@ import ( ) const ( - NumSourceFiles = 20 - NumTx = 100 - NumMsg = 200 - MaxFilesPerPkg = 100 - MaxFileBodyLength = 200 - MsgPerTx = NumMsg / NumTx - TxPerSourceFile = NumTx / NumSourceFiles - MaxDepositAmount = 5000 - MaxArgs = 2 - SourceFileType = ".log" + numSourceFiles = 20 + numTx = 100 + numMsg = 200 + maxFilesPerPkg = 100 + maxFileBodyLength = 200 + msgPerTx = numMsg / numTx + txPerSourceFile = numTx / numSourceFiles + maxDepositAmount = 5000 + maxArgs = 2 + sourceFileType = ".log" ) var ( @@ -40,16 +40,18 @@ var ( // Tests func TestFindFilePaths(t *testing.T) { + t.Parallel() + tempDir, err := os.MkdirTemp(".", "test") if err != nil { t.Fatal(err) } defer os.RemoveAll(tempDir) - testFiles := make([]string, NumSourceFiles) + testFiles := make([]string, numSourceFiles) - for i := 0; i < NumSourceFiles; i++ { - testFiles[i] = "sourceFile" + strconv.Itoa(i) + SourceFileType + for i := 0; i < numSourceFiles; i++ { + testFiles[i] = "sourceFile" + strconv.Itoa(i) + sourceFileType } for _, file := range testFiles { @@ -93,6 +95,8 @@ func TestFindFilePaths(t *testing.T) { } func TestProcessSourceFiles(t *testing.T) { + t.Parallel() + mockMsgs, mockMsgsAddPackage := generateMockMsgs(t) sourceFiles := generateSourceFiles(t, mockMsgs) @@ -118,6 +122,8 @@ func TestProcessSourceFiles(t *testing.T) { } func TestWritePackageMetadata(t *testing.T) { + t.Parallel() + _, mockMsgsAddPackage := generateMockMsgs(t) // Make temp dir @@ -171,6 +177,8 @@ func TestWritePackageMetadata(t *testing.T) { } func TestWritePackageFiles(t *testing.T) { + t.Parallel() + _, mockMsgsAddPackage := generateMockMsgs(t) tempDir, err := os.MkdirTemp(".", "test") @@ -217,21 +225,21 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { } var ( - mockTx = make([]std.Tx, NumTx) - testFiles = make([]string, NumSourceFiles) + mockTx = make([]std.Tx, numTx) + testFiles = make([]string, numSourceFiles) ) // Generate transactions to wrap messages for i := range mockTx { // num mockTx[i] = std.Tx{ - Msgs: mockMsgs[:MsgPerTx], + Msgs: mockMsgs[:msgPerTx], } - mockMsgs = mockMsgs[MsgPerTx:] + mockMsgs = mockMsgs[msgPerTx:] } // Generate source file names - for i := 0; i < NumSourceFiles; i++ { - testFiles[i] = "sourceFile" + strconv.Itoa(i) + SourceFileType + for i := 0; i < numSourceFiles; i++ { + testFiles[i] = "sourceFile" + strconv.Itoa(i) + sourceFileType } // Generate source files @@ -246,17 +254,17 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { t.Fatal(err) } - for _, tx := range mockTx[:TxPerSourceFile] { + for _, tx := range mockTx[:txPerSourceFile] { - err := writeTxToFile(tx, file) + err := writeTxToFile(tx, file, t) if err != nil { t.Fatal(err) } } - mockTx = mockTx[TxPerSourceFile:] + mockTx = mockTx[txPerSourceFile:] } - for i := 0; i < NumSourceFiles; i++ { + for i := 0; i < numSourceFiles; i++ { testFiles[i] = filepath.Join(tempDir, testFiles[i]) } @@ -288,14 +296,14 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { pkgID := 0 - for i := 0; i < NumMsg; i++ { + for i := 0; i < numMsg; i++ { var ( randNum = int(r.Uint32()) msg std.Msg randAddressIndex = randNum % len(testAddresses) - callerAddr = addressFromString(testAddresses[randAddressIndex]) - deposit = std.NewCoins(std.NewCoin("foo", int64(randNum%MaxDepositAmount+1))) + callerAddr = addressFromString(testAddresses[randAddressIndex], t) + deposit = std.NewCoins(std.NewCoin("foo", int64(randNum%maxDepositAmount+1))) path = "gno.land/" pkgName = "package" + strconv.Itoa(pkgID) ) @@ -314,10 +322,10 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { path += pkgName - for j := 0; j < randNum%MaxFilesPerPkg+1; j++ { + for j := 0; j < randNum%maxFilesPerPkg+1; j++ { file := &std.MemFile{ Name: "t" + strconv.Itoa(j) + ".gno", - Body: randString(int(r.Uint32()) % MaxFileBodyLength), + Body: randString(int(r.Uint32()) % maxFileBodyLength), } files = append(files, file) } @@ -335,7 +343,7 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { break case 1: // Making vm.MsgCall msg - args := make([]string, MaxArgs-randNum%2) + args := make([]string, maxArgs-randNum%2) for i := range args { args[i] = randString(10) } @@ -355,7 +363,7 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { msg = bank.MsgSend{ FromAddress: callerAddr, - ToAddress: addressFromString(ta[randNum%len(ta)]), + ToAddress: addressFromString(ta[randNum%len(ta)], t), Amount: deposit, } } @@ -366,10 +374,10 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { return ret, addPkgRet } -func addressFromString(addr string) crypto.Address { +func addressFromString(addr string, t *testing.T) crypto.Address { ret, err := crypto.AddressFromString(addr) if err != nil { - fmt.Errorf("cannot convert string to address, %v", err) + t.Errorf("cannot convert string to address, %v", err) } return ret } @@ -382,23 +390,22 @@ func randString(length int) string { return string(b) } -func writeTxToFile(tx std.Tx, file *os.File) error { - +func writeTxToFile(tx std.Tx, file *os.File, t *testing.T) error { data, err := amino.MarshalJSON(tx) if err != nil { - return fmt.Errorf("unable to marshal JSON data, %w", err) + t.Errorf("unable to marshal JSON data, %v", err) } // Write the JSON data as a line to the file _, err = file.Write(data) if err != nil { - return fmt.Errorf("unable to write to output, %w", err) + t.Errorf("unable to write to output, %v", err) } // Write a newline character to separate JSON objects _, err = file.Write([]byte("\n")) if err != nil { - return fmt.Errorf("unable to write newline output, %w", err) + t.Errorf("unable to write newline output, %v", err) } return nil From 34792b19ccf5e9656bd419ac44a6c9c6b427cbfb Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 16:27:59 +0100 Subject: [PATCH 10/41] fix typo --- extractor/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/main.go b/extractor/main.go index 51cfab87..5741ae02 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -49,7 +49,7 @@ func main() { }, } - // Run the command11 + // Run the command if err := cmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil { _, _ = fmt.Fprintf(os.Stderr, "%+v", err) From fd51c9630bb2a14fa0d34e598bdda6fc88156908 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 16:30:24 +0100 Subject: [PATCH 11/41] fix newlines --- extractor/main.go | 4 ---- extractor/main_test.go | 1 - 2 files changed, 5 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 5741ae02..8f3f82f9 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -119,7 +119,6 @@ func execExtract(ctx context.Context, cfg *extractorCfg) error { // Process messages for _, msg := range msgs { - outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) // Write dir before writing files @@ -161,7 +160,6 @@ func writePackageFiles(msg vm.MsgAddPackage, outputDir string) error { // writePackageMetadata writes the package metadata to the output directory func writePackageMetadata(metadata Metadata, outputDir string) error { - // Get the output path writePath := filepath.Join(outputDir, packageMetadataFile) @@ -241,7 +239,6 @@ func processSourceFile(filePath string) ([]vm.MsgAddPackage, error) { for _, msg := range tx.Msgs { // Only MsgAddPkg should be parsed - if msg.Type() != "add_package" { continue } @@ -257,7 +254,6 @@ func processSourceFile(filePath string) ([]vm.MsgAddPackage, error) { touchMap[path] = msgAddPkg msgArr = append(msgArr, msgAddPkg) - } i++ } diff --git a/extractor/main_test.go b/extractor/main_test.go index 54b33357..349e2373 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -297,7 +297,6 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { pkgID := 0 for i := 0; i < numMsg; i++ { - var ( randNum = int(r.Uint32()) msg std.Msg From ee2f8d4fe5ce0a92fddc03bd8976e3caa7a80b8b Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 16:34:31 +0100 Subject: [PATCH 12/41] rename function --- extractor/main.go | 4 ++-- extractor/main_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 8f3f82f9..0cf361d2 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -112,7 +112,7 @@ func execExtract(ctx context.Context, cfg *extractorCfg) error { g.Go(func() error { // Extract messages - msgs, processErr := processSourceFile(sourceFile) + msgs, processErr := extractAddMessages(sourceFile) if processErr != nil { return processErr } @@ -176,7 +176,7 @@ func writePackageMetadata(metadata Metadata, outputDir string) error { return nil } -func processSourceFile(filePath string) ([]vm.MsgAddPackage, error) { +func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { file, openErr := os.Open(filePath) if openErr != nil { return nil, fmt.Errorf("unable to open file, %w", openErr) diff --git a/extractor/main_test.go b/extractor/main_test.go index 349e2373..d09aa35f 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -102,7 +102,7 @@ func TestProcessSourceFiles(t *testing.T) { var results []vm.MsgAddPackage for _, sf := range sourceFiles { - res, err := processSourceFile(sf) + res, err := extractAddMessages(sf) if err != nil { t.Fatal(err) } From ea4e9996fed22bdda948e2945e1fc3e3fe349b5a Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 16:40:04 +0100 Subject: [PATCH 13/41] remove variable, expand err msg --- extractor/main.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 0cf361d2..b07a1d15 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -200,7 +200,6 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { // Buffer to handle lines longer than 64kb tempBuf := make([]byte, 0) - i := 0 for { var tx std.Tx line, isPrefix, err := reader.ReadLine() @@ -228,7 +227,7 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { } if aminoErr := amino.UnmarshalJSON(line, &tx); aminoErr != nil { - fmt.Printf("Error while parsing amino JSON at line %d: %v\n", i, aminoErr.Error()) + fmt.Printf("Error while parsing amino JSON at line: %v\nLine:%s\n", aminoErr.Error(), line) continue } @@ -255,7 +254,6 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { touchMap[path] = msgAddPkg msgArr = append(msgArr, msgAddPkg) } - i++ } return msgArr, cleanup() From 43941083d44684648473bff56f7282b657d98407 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 16:44:01 +0100 Subject: [PATCH 14/41] edit error message --- extractor/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index b07a1d15..7ae52b70 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -265,8 +265,7 @@ func findFilePaths(startPath string, fileType string) ([]string, error) { walkFn := func(path string, info os.FileInfo, err error) error { if err != nil { - fmt.Println("Error accessing file:", err) - return err + return fmt.Errorf("error accessing file: %v", err) } // Check if the file is a dir From 25130b37b1d98224abe382e2cb9c33ee67009afe Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 17:10:15 +0100 Subject: [PATCH 15/41] replace conditionals with require --- extractor/main_test.go | 91 ++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 60 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index d09aa35f..d7e8f682 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -43,9 +43,8 @@ func TestFindFilePaths(t *testing.T) { t.Parallel() tempDir, err := os.MkdirTemp(".", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + defer os.RemoveAll(tempDir) testFiles := make([]string, numSourceFiles) @@ -56,18 +55,15 @@ func TestFindFilePaths(t *testing.T) { for _, file := range testFiles { filePath := filepath.Join(tempDir, file) - if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { - t.Fatal(err) - } - if _, err := os.Create(filePath); err != nil { - t.Fatal(err) - } + err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm) + require.NoError(t, err) + + _, err = os.Create(filePath) + require.NoError(t, err) } results, err := findFilePaths(tempDir, ".log") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) expectedResults := make([]string, 0, len(testFiles)) @@ -89,7 +85,7 @@ func TestFindFilePaths(t *testing.T) { for i, result := range results { if result != expectedResults[i] { - t.Errorf("Expected %s, but got %s", expectedResults[i], result) + t.Fatalf("Expected %s, but got %s", expectedResults[i], result) } } } @@ -103,9 +99,7 @@ func TestProcessSourceFiles(t *testing.T) { var results []vm.MsgAddPackage for _, sf := range sourceFiles { res, err := extractAddMessages(sf) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) results = append(results, res...) } @@ -128,9 +122,7 @@ func TestWritePackageMetadata(t *testing.T) { // Make temp dir tempDir, err := os.MkdirTemp(".", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) for _, msg := range mockMsgsAddPackage { md := metadataFromMsg(msg) @@ -139,12 +131,11 @@ func TestWritePackageMetadata(t *testing.T) { outputDir := filepath.Join(tempDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) // Write dir before writing metadata - if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil { - t.Fatal(dirWriteErr) - } + err := os.MkdirAll(outputDir, os.ModePerm) + require.NoError(t, err) // Write the metadata - err := writePackageMetadata(md, outputDir) + err = writePackageMetadata(md, outputDir) require.NoError(t, err) // Read file @@ -182,21 +173,18 @@ func TestWritePackageFiles(t *testing.T) { _, mockMsgsAddPackage := generateMockMsgs(t) tempDir, err := os.MkdirTemp(".", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) for _, msg := range mockMsgsAddPackage { // Get output dir outputDir := filepath.Join(tempDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) // Write dir before writing metadata - if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil { - t.Fatal(dirWriteErr) - } + err := os.MkdirAll(outputDir, os.ModePerm) + require.NoError(t, err) // Write the metadata - err := writePackageFiles(msg, outputDir) + err = writePackageFiles(msg, outputDir) require.NoError(t, err) // Read & compare file @@ -208,10 +196,7 @@ func TestWritePackageFiles(t *testing.T) { } t.Cleanup(func() { err := os.RemoveAll(tempDir) - if err != nil { - fmt.Printf("could not clean up temp dir, %v", err) - return - } + require.NoError(t, err) }) } @@ -220,9 +205,7 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { t.Helper() tempDir, err := os.MkdirTemp(".", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) var ( mockTx = make([]std.Tx, numTx) @@ -245,14 +228,12 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { // Generate source files for _, file := range testFiles { filePath := filepath.Join(tempDir, file) - if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { - t.Fatal(err) - } - file, err := os.Create(filePath) - if err != nil { - t.Fatal(err) - } + err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm) + require.NoError(t, err) + + file, err := os.Create(filePath) + require.NoError(t, err) for _, tx := range mockTx[:txPerSourceFile] { @@ -270,10 +251,7 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { t.Cleanup(func() { err := os.RemoveAll(tempDir) - if err != nil { - fmt.Printf("could not clean up temp dir, %v", err) - return - } + require.NoError(t, err) }) return testFiles @@ -375,9 +353,8 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { func addressFromString(addr string, t *testing.T) crypto.Address { ret, err := crypto.AddressFromString(addr) - if err != nil { - t.Errorf("cannot convert string to address, %v", err) - } + require.NoError(t, err) + return ret } @@ -391,21 +368,15 @@ func randString(length int) string { func writeTxToFile(tx std.Tx, file *os.File, t *testing.T) error { data, err := amino.MarshalJSON(tx) - if err != nil { - t.Errorf("unable to marshal JSON data, %v", err) - } + require.NoError(t, err) // Write the JSON data as a line to the file _, err = file.Write(data) - if err != nil { - t.Errorf("unable to write to output, %v", err) - } + require.NoError(t, err) // Write a newline character to separate JSON objects _, err = file.Write([]byte("\n")) - if err != nil { - t.Errorf("unable to write newline output, %v", err) - } + require.NoError(t, err) return nil } From 9ee8f5603bc65b1bb64f25c7417ba94b4a31ca1e Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 17:13:47 +0100 Subject: [PATCH 16/41] replace conditional with require#2 --- extractor/main_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index d7e8f682..4882de38 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -157,10 +157,7 @@ func TestWritePackageMetadata(t *testing.T) { t.Cleanup(func() { err := os.RemoveAll(tempDir) - if err != nil { - fmt.Printf("could not clean up temp dir, %v", err) - return - } + require.NoError(t, err) }) require.Equal(t, md, unmarshalledMetadata) From 9fe69009c7e1cbb219eabb74b893656de9275dad Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 17:29:12 +0100 Subject: [PATCH 17/41] add cleanup helper, replace defer with t.cleanup --- extractor/main_test.go | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index 4882de38..f4a217b7 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -44,8 +44,7 @@ func TestFindFilePaths(t *testing.T) { tempDir, err := os.MkdirTemp(".", "test") require.NoError(t, err) - - defer os.RemoveAll(tempDir) + t.Cleanup(removeDir(t, tempDir)) testFiles := make([]string, numSourceFiles) @@ -123,6 +122,7 @@ func TestWritePackageMetadata(t *testing.T) { // Make temp dir tempDir, err := os.MkdirTemp(".", "test") require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) for _, msg := range mockMsgsAddPackage { md := metadataFromMsg(msg) @@ -155,11 +155,6 @@ func TestWritePackageMetadata(t *testing.T) { err = json.Unmarshal(raw, &unmarshalledMetadata) require.NoError(t, err) - t.Cleanup(func() { - err := os.RemoveAll(tempDir) - require.NoError(t, err) - }) - require.Equal(t, md, unmarshalledMetadata) } } @@ -171,6 +166,7 @@ func TestWritePackageFiles(t *testing.T) { tempDir, err := os.MkdirTemp(".", "test") require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) for _, msg := range mockMsgsAddPackage { // Get output dir @@ -191,10 +187,6 @@ func TestWritePackageFiles(t *testing.T) { require.Equal(t, f.Body, string(contents)) } } - t.Cleanup(func() { - err := os.RemoveAll(tempDir) - require.NoError(t, err) - }) } // Helpers @@ -203,6 +195,7 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { tempDir, err := os.MkdirTemp(".", "test") require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) var ( mockTx = make([]std.Tx, numTx) @@ -246,11 +239,6 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { testFiles[i] = filepath.Join(tempDir, testFiles[i]) } - t.Cleanup(func() { - err := os.RemoveAll(tempDir) - require.NoError(t, err) - }) - return testFiles } @@ -377,3 +365,10 @@ func writeTxToFile(tx std.Tx, file *os.File, t *testing.T) error { return nil } + +func removeDir(t *testing.T, dirPath string) func() { + return func() { + err := os.RemoveAll(dirPath) + require.NoError(t, err) + } +} From c8be5c3b060bb7cadd1c8f09cb9ad22b48f0096c Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 17:32:17 +0100 Subject: [PATCH 18/41] reorg args --- extractor/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index f4a217b7..3808e9bc 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -351,7 +351,7 @@ func randString(length int) string { return string(b) } -func writeTxToFile(tx std.Tx, file *os.File, t *testing.T) error { +func writeTxToFile(t *testing.T, tx std.Tx, file *os.File) error { data, err := amino.MarshalJSON(tx) require.NoError(t, err) From 58ea5eeb9ec464a0816d37975528693affb1f90e Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 17:33:02 +0100 Subject: [PATCH 19/41] add t.Helper() --- extractor/main_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/main_test.go b/extractor/main_test.go index 3808e9bc..6c494b7a 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -352,6 +352,8 @@ func randString(length int) string { } func writeTxToFile(t *testing.T, tx std.Tx, file *os.File) error { + t.Helper() + data, err := amino.MarshalJSON(tx) require.NoError(t, err) From 7d9669044eb23c243c1d64c0a44a73fd1326a6f4 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 17:59:26 +0100 Subject: [PATCH 20/41] replace t.fatal with require --- extractor/main_test.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index 6c494b7a..ad444c66 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -78,13 +78,11 @@ func TestFindFilePaths(t *testing.T) { return expectedResults[i] < expectedResults[j] }) - if len(results) != len(expectedResults) { - t.Fatalf("Expected %d results, but got %d", len(expectedResults), len(results)) - } + require.Equal(t, len(results), len(expectedResults)) for i, result := range results { if result != expectedResults[i] { - t.Fatalf("Expected %s, but got %s", expectedResults[i], result) + require.Equal(t, result, expectedResults[i]) } } } @@ -226,7 +224,6 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { require.NoError(t, err) for _, tx := range mockTx[:txPerSourceFile] { - err := writeTxToFile(tx, file, t) if err != nil { t.Fatal(err) @@ -277,9 +274,7 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { } switch randNum % 3 { - case 0: // Making vm.MsgAddPackage msg - var files []*std.MemFile path += pkgName @@ -304,7 +299,6 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { addPkgRet = append(addPkgRet, msg.(vm.MsgAddPackage)) break case 1: // Making vm.MsgCall msg - args := make([]string, maxArgs-randNum%2) for i := range args { args[i] = randString(10) @@ -319,7 +313,6 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { } break case 2: // Making bank.MsgSend - // Remove already used address ta := append(testAddresses[:randAddressIndex], testAddresses[randAddressIndex+1:]...) From 5819f944438fd3dc74e49223f62354cdd2062fac Mon Sep 17 00:00:00 2001 From: leohhhn Date: Tue, 12 Dec 2023 17:59:58 +0100 Subject: [PATCH 21/41] fix arg order --- extractor/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index ad444c66..10c06123 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -224,7 +224,7 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { require.NoError(t, err) for _, tx := range mockTx[:txPerSourceFile] { - err := writeTxToFile(tx, file, t) + err := writeTxToFile(t, tx, file) if err != nil { t.Fatal(err) } From fc289a9a52fb594e53145cf2bb473501fb6b1c65 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 13 Dec 2023 16:47:56 +0100 Subject: [PATCH 22/41] modify random string func in testing --- .gitignore | 1 - extractor/main_test.go | 30 ++++++++++++------------------ 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index d65c31ba..6515690c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ txexport.log txexport.log.gz .idea -extractor/extracted \ No newline at end of file diff --git a/extractor/main_test.go b/extractor/main_test.go index 10c06123..c5cf4254 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -2,14 +2,14 @@ package main import ( "bufio" + crand "crypto/rand" + "encoding/base64" "encoding/json" - "fmt" "github.com/gnolang/gno/gno.land/pkg/sdk/vm" "github.com/gnolang/gno/tm2/pkg/amino" "github.com/gnolang/gno/tm2/pkg/crypto" "github.com/gnolang/gno/tm2/pkg/sdk/bank" "github.com/gnolang/gno/tm2/pkg/std" - "github.com/go-test/deep" "github.com/stretchr/testify/require" "math/rand" "os" @@ -34,10 +34,6 @@ const ( sourceFileType = ".log" ) -var ( - chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890\\n\\t!@#$%^&*()_+> Date: Wed, 13 Dec 2023 16:52:47 +0100 Subject: [PATCH 23/41] replace t.Fatalf with require --- extractor/main_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index c5cf4254..f7eb8aca 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -139,10 +139,7 @@ func TestWritePackageMetadata(t *testing.T) { raw, isPrefix, err := reader.ReadLine() require.NoError(t, err) - - if isPrefix { - t.Fatalf("Metadata longer then buffer max size at %s\n", outputDir) - } + require.Equal(t, isPrefix, false) err = json.Unmarshal(raw, &unmarshalledMetadata) require.NoError(t, err) From 20dd04324dda8cd17481efe188e7a458e214e493 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 13 Dec 2023 17:13:31 +0100 Subject: [PATCH 24/41] move global variables to smaller scopes --- extractor/main_test.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index f7eb8aca..d656b71c 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -22,16 +22,12 @@ import ( ) const ( - numSourceFiles = 20 - numTx = 100 - numMsg = 200 - maxFilesPerPkg = 100 - maxFileBodyLength = 200 - msgPerTx = numMsg / numTx - txPerSourceFile = numTx / numSourceFiles - maxDepositAmount = 5000 - maxArgs = 2 - sourceFileType = ".log" + numSourceFiles = 20 + numTx = 100 + numMsg = 200 + msgPerTx = numMsg / numTx + txPerSourceFile = numTx / numSourceFiles + sourceFileType = ".log" ) // Tests @@ -250,13 +246,17 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { for i := 0; i < numMsg; i++ { var ( - randNum = int(r.Uint32()) - msg std.Msg - randAddressIndex = randNum % len(testAddresses) - callerAddr = addressFromString(testAddresses[randAddressIndex], t) - deposit = std.NewCoins(std.NewCoin("foo", int64(randNum%maxDepositAmount+1))) - path = "gno.land/" - pkgName = "package" + strconv.Itoa(pkgID) + randNum = int(r.Uint32()) + msg std.Msg + randAddressIndex = randNum % len(testAddresses) + maxDepositAmount = 5000 + callerAddr = addressFromString(testAddresses[randAddressIndex], t) + deposit = std.NewCoins(std.NewCoin("foo", int64(randNum%maxDepositAmount+1))) + path = "gno.land/" + pkgName = "package" + strconv.Itoa(pkgID) + maxArgs = 2 + maxFileBodyLength = 200 + maxFilesPerPkg = 100 ) if randNum%2 == 0 { From 36253a371f0934b8608fff4bda2d0de8bfe7bb83 Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:51:05 +0100 Subject: [PATCH 25/41] rename error Co-authored-by: Antonio Navarro Perez --- extractor/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 7ae52b70..05cd3ec6 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -177,9 +177,9 @@ func writePackageMetadata(metadata Metadata, outputDir string) error { } func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { - file, openErr := os.Open(filePath) - if openErr != nil { - return nil, fmt.Errorf("unable to open file, %w", openErr) + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("unable to open file, %w", err) } cleanup := func() error { From b56850ae8d3135a84fbc3833818c3993a99f9250 Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:51:42 +0100 Subject: [PATCH 26/41] remove unused anon vars Co-authored-by: Antonio Navarro Perez --- extractor/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/main.go b/extractor/main.go index 05cd3ec6..a38c0eb8 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -51,7 +51,7 @@ func main() { // Run the command if err := cmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "%+v", err) + fmt.Fprintf(os.Stderr, "%+v", err) os.Exit(1) } From 0932074fa7081d223c1db7f5a1f3bf83a0666baf Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:52:25 +0100 Subject: [PATCH 27/41] edit comment Co-authored-by: Antonio Navarro Perez --- extractor/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/main.go b/extractor/main.go index a38c0eb8..0319d92b 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -85,7 +85,7 @@ func (c *extractorCfg) registerFlags(fs *flag.FlagSet) { func execExtract(ctx context.Context, cfg *extractorCfg) error { // Check the file type is valid if cfg.fileType == "" { - return errors.New("invalid file type specified") + return errors.New("no file type specified") } // Check the source dir is valid From 55c4fa03ae2811ba5f99f2654d738c23a2ec7c38 Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:52:49 +0100 Subject: [PATCH 28/41] rename error Co-authored-by: Antonio Navarro Perez --- extractor/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 0319d92b..025cd9e5 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -226,8 +226,8 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { line = tempBuf } - if aminoErr := amino.UnmarshalJSON(line, &tx); aminoErr != nil { - fmt.Printf("Error while parsing amino JSON at line: %v\nLine:%s\n", aminoErr.Error(), line) + if err := amino.UnmarshalJSON(line, &tx); err != nil { + fmt.Printf("Error while parsing amino JSON at line: %v\nLine:%s\n", err, line) continue } From afc74870d4297cc8fc3de8520303ca59a57d8b21 Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:53:22 +0100 Subject: [PATCH 29/41] update map storage for paths Co-authored-by: Antonio Navarro Perez --- extractor/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/main.go b/extractor/main.go index 025cd9e5..2e9daad6 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -192,7 +192,7 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { reader := bufio.NewReader(file) // Used to track what was parsed in the past - touchMap := make(map[string]vm.MsgAddPackage) + touchMap := make(map[string]bool) // Msg array to be returned for further processing msgArr := make([]vm.MsgAddPackage, 0) From 3a8640738b353a000338a4e18714e0efa1cbf252 Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:53:42 +0100 Subject: [PATCH 30/41] update map storage for paths #2 Co-authored-by: Antonio Navarro Perez --- extractor/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/main.go b/extractor/main.go index 2e9daad6..b78e6263 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -251,7 +251,7 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { continue } - touchMap[path] = msgAddPkg + touchMap[path] = true msgArr = append(msgArr, msgAddPkg) } } From 01ca7fb28cf5cf4786bb166a5f320ceb2ea39186 Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:54:11 +0100 Subject: [PATCH 31/41] add conversion check Co-authored-by: Antonio Navarro Perez --- extractor/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extractor/main.go b/extractor/main.go index b78e6263..2eaa9093 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -242,7 +242,10 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { continue } - msgAddPkg := msg.(vm.MsgAddPackage) + msgAddPkg, ok := msg.(vm.MsgAddPackage) + if !ok { + //TODO: throw error + } path := msgAddPkg.Package.Path From 6b6349cff8345eb4f2240d070febf8e7cc22cad5 Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:54:35 +0100 Subject: [PATCH 32/41] add error check Co-authored-by: Antonio Navarro Perez --- extractor/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extractor/main.go b/extractor/main.go index 2eaa9093..9f6d051c 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -247,6 +247,9 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { //TODO: throw error } + if msgAddPkg.Package == nil { + // TODO: throw error + } path := msgAddPkg.Package.Path if _, parsed := touchMap[path]; parsed { From 928801c40dc6d206670a6f48d388eb1012b481ad Mon Sep 17 00:00:00 2001 From: leohhhn Date: Thu, 21 Dec 2023 09:03:31 +0100 Subject: [PATCH 33/41] format & handle errors --- extractor/main.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 9f6d051c..9da92b46 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -205,10 +205,11 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { line, isPrefix, err := reader.ReadLine() // Exit if no more lines in file + if errors.Is(err, io.EOF) { + break + } if err != nil { - if errors.Is(err, io.EOF) { - break - } + return nil, fmt.Errorf("error reading lines; %w", err) } // If line is too long, save it in a temporary buffer and continue reading line @@ -227,7 +228,7 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { } if err := amino.UnmarshalJSON(line, &tx); err != nil { - fmt.Printf("Error while parsing amino JSON at line: %v\nLine:%s\n", err, line) + fmt.Errorf("Error while parsing amino JSON at line: %w\nLine:%s\n", err, line) continue } @@ -244,11 +245,11 @@ func extractAddMessages(filePath string) ([]vm.MsgAddPackage, error) { msgAddPkg, ok := msg.(vm.MsgAddPackage) if !ok { - //TODO: throw error + return nil, errors.New("could not cast into MsgAddPackage") } if msgAddPkg.Package == nil { - // TODO: throw error + return nil, errors.New("MsgAddPackage is nil") } path := msgAddPkg.Package.Path @@ -271,7 +272,7 @@ func findFilePaths(startPath string, fileType string) ([]string, error) { walkFn := func(path string, info os.FileInfo, err error) error { if err != nil { - return fmt.Errorf("error accessing file: %v", err) + return fmt.Errorf("error accessing file: %w", err) } // Check if the file is a dir @@ -292,7 +293,7 @@ func findFilePaths(startPath string, fileType string) ([]string, error) { // Walk the directory root recursively if walkErr := filepath.Walk(startPath, walkFn); walkErr != nil { - return nil, fmt.Errorf("unable to walk directory, %v", walkErr) + return nil, fmt.Errorf("unable to walk directory, %w", walkErr) } return filePaths, nil From 872fd12c139703a4ccfe8064735c3ce5ce2c02f7 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Thu, 21 Dec 2023 09:31:59 +0100 Subject: [PATCH 34/41] move cleanup above err check --- extractor/main_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index d656b71c..5c57a832 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -35,8 +35,8 @@ func TestFindFilePaths(t *testing.T) { t.Parallel() tempDir, err := os.MkdirTemp(".", "test") - require.NoError(t, err) t.Cleanup(removeDir(t, tempDir)) + require.NoError(t, err) testFiles := make([]string, numSourceFiles) @@ -109,8 +109,8 @@ func TestWritePackageMetadata(t *testing.T) { // Make temp dir tempDir, err := os.MkdirTemp(".", "test") - require.NoError(t, err) t.Cleanup(removeDir(t, tempDir)) + require.NoError(t, err) for _, msg := range mockMsgsAddPackage { md := metadataFromMsg(msg) @@ -150,8 +150,8 @@ func TestWritePackageFiles(t *testing.T) { _, mockMsgsAddPackage := generateMockMsgs(t) tempDir, err := os.MkdirTemp(".", "test") - require.NoError(t, err) t.Cleanup(removeDir(t, tempDir)) + require.NoError(t, err) for _, msg := range mockMsgsAddPackage { // Get output dir @@ -179,8 +179,8 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { t.Helper() tempDir, err := os.MkdirTemp(".", "test") - require.NoError(t, err) t.Cleanup(removeDir(t, tempDir)) + require.NoError(t, err) var ( mockTx = make([]std.Tx, numTx) From f20d8f3191b100134d132cde39f8a8ac7ed2e7fa Mon Sep 17 00:00:00 2001 From: leohhhn Date: Thu, 21 Dec 2023 11:57:42 +0100 Subject: [PATCH 35/41] Revert "move cleanup above err check" This reverts commit 872fd12c139703a4ccfe8064735c3ce5ce2c02f7. --- extractor/main_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index 5c57a832..d656b71c 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -35,8 +35,8 @@ func TestFindFilePaths(t *testing.T) { t.Parallel() tempDir, err := os.MkdirTemp(".", "test") - t.Cleanup(removeDir(t, tempDir)) require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) testFiles := make([]string, numSourceFiles) @@ -109,8 +109,8 @@ func TestWritePackageMetadata(t *testing.T) { // Make temp dir tempDir, err := os.MkdirTemp(".", "test") - t.Cleanup(removeDir(t, tempDir)) require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) for _, msg := range mockMsgsAddPackage { md := metadataFromMsg(msg) @@ -150,8 +150,8 @@ func TestWritePackageFiles(t *testing.T) { _, mockMsgsAddPackage := generateMockMsgs(t) tempDir, err := os.MkdirTemp(".", "test") - t.Cleanup(removeDir(t, tempDir)) require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) for _, msg := range mockMsgsAddPackage { // Get output dir @@ -179,8 +179,8 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { t.Helper() tempDir, err := os.MkdirTemp(".", "test") - t.Cleanup(removeDir(t, tempDir)) require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) var ( mockTx = make([]std.Tx, numTx) From 65bb9c931ab2e8e1c2d24f1163fe3f67bafa9f0e Mon Sep 17 00:00:00 2001 From: leohhhn Date: Thu, 21 Dec 2023 11:58:55 +0100 Subject: [PATCH 36/41] swap arg order --- extractor/main_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index d656b71c..3408752b 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -310,7 +310,7 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { msg = bank.MsgSend{ FromAddress: callerAddr, - ToAddress: addressFromString(ta[randNum%len(ta)], t), + ToAddress: addressFromString(t, ta [randNum%len(ta)), Amount: deposit, } } @@ -321,7 +321,9 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { return ret, addPkgRet } -func addressFromString(addr string, t *testing.T) crypto.Address { +func addressFromString(t *testing.T, addr string) crypto.Address { + t.Helper() + ret, err := crypto.AddressFromString(addr) require.NoError(t, err) From b24b21addfefb13ec8bd74b5966429651f727c60 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 27 Dec 2023 12:34:27 +0100 Subject: [PATCH 37/41] wip tests --- extractor/main.go | 16 +++++++-- extractor/main_test.go | 76 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 9da92b46..66f37e28 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -23,6 +23,12 @@ const ( packageMetadataFile = "pkg_metadata.json" ) +var ( + errInvalidFileType = errors.New("no file type specified") + errInvalidSourceDir = errors.New("invalid source directory") + errInvalidOutputDir = errors.New("invalid output directory") +) + // Define extractor config type extractorCfg struct { fileType string @@ -85,17 +91,17 @@ func (c *extractorCfg) registerFlags(fs *flag.FlagSet) { func execExtract(ctx context.Context, cfg *extractorCfg) error { // Check the file type is valid if cfg.fileType == "" { - return errors.New("no file type specified") + return errInvalidFileType } // Check the source dir is valid if cfg.sourceDir == "" { - return errors.New("invalid source directory") + return errInvalidSourceDir } // Check the output dir is valid if cfg.outputDir == "" { - return errors.New("invalid output directory") + return errInvalidOutputDir } // Find the files that need to be analyzed @@ -104,6 +110,10 @@ func execExtract(ctx context.Context, cfg *extractorCfg) error { return fmt.Errorf("unable to find file paths, %w", findErr) } + if len(sourceFiles) == 0 { + fmt.Println("no source files found, exiting.") + } + // Concurrently process the source files g, ctx := errgroup.WithContext(ctx) diff --git a/extractor/main_test.go b/extractor/main_test.go index 3408752b..55e4da89 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "context" crand "crypto/rand" "encoding/base64" "encoding/json" @@ -31,6 +32,77 @@ const ( ) // Tests +func TestHappyFlow(t *testing.T) { + ctx := context.Background() + + // Set correct config + cfg := &extractorCfg{ + fileType: ".log", + sourceDir: ".", + outputDir: ".", + } + + // Generate mock messages & mock files + mockStdMsg, _ := generateMockMsgs(t) + _ = generateSourceFiles(t, mockStdMsg) + + // Try extraction + err := execExtract(ctx, cfg) + require.NoError(t, err) + +} + +func TestSadFlow(t *testing.T) { + ctx := context.Background() + + // Set correct config + cfg := &extractorCfg{ + fileType: ".log", + sourceDir: ".", + outputDir: ".", + } + + // Do not generate any test messages or files + + // Try extraction + err := execExtract(ctx, cfg) + require.NoError(t, err) +} + +func TestErrors(t *testing.T) { + var ctx context.Context + + // Test invalid filetype error + cfg := &extractorCfg{ + fileType: "", + sourceDir: ".", + outputDir: ".", + } + + err := execExtract(ctx, cfg) + require.Equal(t, err, errInvalidFileType) + + // Test invalid source dir error + cfg = &extractorCfg{ + fileType: ".log", + sourceDir: "", + outputDir: ".", + } + + err = execExtract(ctx, cfg) + require.Equal(t, err, errInvalidSourceDir) + + // Test invalid output dir error + cfg = &extractorCfg{ + fileType: ".log", + sourceDir: ".", + outputDir: "", + } + + err = execExtract(ctx, cfg) + require.Equal(t, err, errInvalidOutputDir) +} + func TestFindFilePaths(t *testing.T) { t.Parallel() @@ -250,7 +322,7 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { msg std.Msg randAddressIndex = randNum % len(testAddresses) maxDepositAmount = 5000 - callerAddr = addressFromString(testAddresses[randAddressIndex], t) + callerAddr = addressFromString(t, testAddresses[randAddressIndex]) deposit = std.NewCoins(std.NewCoin("foo", int64(randNum%maxDepositAmount+1))) path = "gno.land/" pkgName = "package" + strconv.Itoa(pkgID) @@ -310,7 +382,7 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { msg = bank.MsgSend{ FromAddress: callerAddr, - ToAddress: addressFromString(t, ta [randNum%len(ta)), + ToAddress: addressFromString(t, ta[randNum%len(ta)]), Amount: deposit, } } From 03fa2de3dec655a4bed81cbb4c0469b9458e5a99 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 27 Dec 2023 15:19:54 +0100 Subject: [PATCH 38/41] added more tests --- extractor/main.go | 9 +++-- extractor/main_test.go | 92 +++++++++++++++++++++++++++++++----------- 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/extractor/main.go b/extractor/main.go index 66f37e28..9be6c03b 100644 --- a/extractor/main.go +++ b/extractor/main.go @@ -24,9 +24,10 @@ const ( ) var ( - errInvalidFileType = errors.New("no file type specified") - errInvalidSourceDir = errors.New("invalid source directory") - errInvalidOutputDir = errors.New("invalid output directory") + errInvalidFileType = errors.New("no file type specified") + errInvalidSourceDir = errors.New("invalid source directory") + errInvalidOutputDir = errors.New("invalid output directory") + errNoSourceFilesFound = errors.New("no source files found, exiting") ) // Define extractor config @@ -111,7 +112,7 @@ func execExtract(ctx context.Context, cfg *extractorCfg) error { } if len(sourceFiles) == 0 { - fmt.Println("no source files found, exiting.") + return errNoSourceFilesFound } // Concurrently process the source files diff --git a/extractor/main_test.go b/extractor/main_test.go index 55e4da89..d2f4bb5a 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -32,27 +32,7 @@ const ( ) // Tests -func TestHappyFlow(t *testing.T) { - ctx := context.Background() - - // Set correct config - cfg := &extractorCfg{ - fileType: ".log", - sourceDir: ".", - outputDir: ".", - } - - // Generate mock messages & mock files - mockStdMsg, _ := generateMockMsgs(t) - _ = generateSourceFiles(t, mockStdMsg) - - // Try extraction - err := execExtract(ctx, cfg) - require.NoError(t, err) - -} - -func TestSadFlow(t *testing.T) { +func TestNoFilesFlow(t *testing.T) { ctx := context.Background() // Set correct config @@ -66,7 +46,7 @@ func TestSadFlow(t *testing.T) { // Try extraction err := execExtract(ctx, cfg) - require.NoError(t, err) + require.Equal(t, err, errNoSourceFilesFound) } func TestErrors(t *testing.T) { @@ -103,6 +83,72 @@ func TestErrors(t *testing.T) { require.Equal(t, err, errInvalidOutputDir) } +func TestValidFlow(t *testing.T) { + ctx := context.Background() + outputPath := "./" + + //Generate temporary output dir + outputDir, err := os.MkdirTemp(outputPath, "") + require.NoError(t, err) + t.Cleanup(removeDir(t, outputDir)) + + // Set correct config + cfg := &extractorCfg{ + fileType: sourceFileType, + sourceDir: ".", + outputDir: outputDir, + } + + // Generate mock messages & mock files + mockStdMsg, mockAddPkgMsg := generateMockMsgs(t) + _ = generateSourceFiles(t, mockStdMsg) + + // Perform extraction + err = execExtract(ctx, cfg) + require.NoError(t, err) + + for _, msg := range mockAddPkgMsg { + basePath := filepath.Join(outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) + + // Get metadata path & open metadata file + metadataPath := filepath.Join(basePath, packageMetadataFile) + file, err := os.Open(metadataPath) + require.NoError(t, err) + + // Read Metadata + reader := bufio.NewReader(file) + retrievedMetadata, _, err := reader.ReadLine() + require.NoError(t, err) + + // Compare metadata + expectedMetadata, err := json.Marshal(metadataFromMsg(msg)) + require.Equal(t, expectedMetadata, retrievedMetadata) + + // Close metadata file + err = file.Close() + require.NoError(t, err) + + // Check package file content + for _, f := range msg.Package.Files { + filePath := filepath.Join(basePath, f.Name) + + // Open file + file, err := os.Open(filePath) + require.NoError(t, err) + + // Read file body + retrievedFileBody := make([]byte, 0) + + reader := bufio.NewReader(file) + retrievedFileBody, _, err = reader.ReadLine() + + // Compare file bodies + require.Equal(t, f.Body, string(retrievedFileBody)) + + } + } +} + func TestFindFilePaths(t *testing.T) { t.Parallel() @@ -346,7 +392,7 @@ func generateMockMsgs(t *testing.T) ([]std.Msg, []vm.MsgAddPackage) { for j := 0; j < randNum%maxFilesPerPkg+1; j++ { file := &std.MemFile{ Name: "t" + strconv.Itoa(j) + ".gno", - Body: randString(t, int(r.Uint32())%maxFileBodyLength), + Body: randString(t, int(r.Uint32())%maxFileBodyLength+1), } files = append(files, file) } From 76384894c7404e43c9c84df8966c65c86b31bc6f Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 27 Dec 2023 15:23:19 +0100 Subject: [PATCH 39/41] modify source path --- extractor/main_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index d2f4bb5a..fa1f74c5 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -92,16 +92,18 @@ func TestValidFlow(t *testing.T) { require.NoError(t, err) t.Cleanup(removeDir(t, outputDir)) + sourcePath := "." + // Set correct config cfg := &extractorCfg{ fileType: sourceFileType, - sourceDir: ".", + sourceDir: sourcePath, outputDir: outputDir, } // Generate mock messages & mock files mockStdMsg, mockAddPkgMsg := generateMockMsgs(t) - _ = generateSourceFiles(t, mockStdMsg) + _ = generateSourceFiles(t, mockStdMsg, sourcePath) // Perform extraction err = execExtract(ctx, cfg) @@ -200,8 +202,9 @@ func TestFindFilePaths(t *testing.T) { func TestExtractAddMessages(t *testing.T) { t.Parallel() + sourcePath := "." mockMsgs, mockMsgsAddPackage := generateMockMsgs(t) - sourceFiles := generateSourceFiles(t, mockMsgs) + sourceFiles := generateSourceFiles(t, mockMsgs, sourcePath) var results []vm.MsgAddPackage for _, sf := range sourceFiles { @@ -293,10 +296,10 @@ func TestWritePackageFiles(t *testing.T) { } // Helpers -func generateSourceFiles(t *testing.T, mockMsgs []std.Msg) []string { +func generateSourceFiles(t *testing.T, mockMsgs []std.Msg, sourcePath string) []string { t.Helper() - tempDir, err := os.MkdirTemp(".", "test") + tempDir, err := os.MkdirTemp(sourcePath, "test") require.NoError(t, err) t.Cleanup(removeDir(t, tempDir)) From 1ed77fcd870fa593c2f43e98b643598a8ae48438 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Wed, 27 Dec 2023 17:07:35 +0100 Subject: [PATCH 40/41] parallel test failing, save --- extractor/main_test.go | 155 ++++++++++++++++++++++------------------- 1 file changed, 82 insertions(+), 73 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index fa1f74c5..47279f63 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -11,6 +11,7 @@ import ( "github.com/gnolang/gno/tm2/pkg/crypto" "github.com/gnolang/gno/tm2/pkg/sdk/bank" "github.com/gnolang/gno/tm2/pkg/std" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "math/rand" "os" @@ -31,83 +32,94 @@ const ( sourceFileType = ".log" ) -// Tests -func TestNoFilesFlow(t *testing.T) { - ctx := context.Background() - - // Set correct config - cfg := &extractorCfg{ - fileType: ".log", - sourceDir: ".", - outputDir: ".", - } - - // Do not generate any test messages or files - - // Try extraction - err := execExtract(ctx, cfg) - require.Equal(t, err, errNoSourceFilesFound) -} - -func TestErrors(t *testing.T) { - var ctx context.Context - - // Test invalid filetype error - cfg := &extractorCfg{ - fileType: "", - sourceDir: ".", - outputDir: ".", - } - - err := execExtract(ctx, cfg) - require.Equal(t, err, errInvalidFileType) - - // Test invalid source dir error - cfg = &extractorCfg{ - fileType: ".log", - sourceDir: "", - outputDir: ".", - } - - err = execExtract(ctx, cfg) - require.Equal(t, err, errInvalidSourceDir) - - // Test invalid output dir error - cfg = &extractorCfg{ - fileType: ".log", - sourceDir: ".", - outputDir: "", - } - - err = execExtract(ctx, cfg) - require.Equal(t, err, errInvalidOutputDir) -} +//func TestExtractor_Errors(t *testing.T) { +// testTable := []struct { +// name string +// cfg *extractorCfg +// expectedErr error +// }{ +// { +// "no source files", +// &extractorCfg{ +// fileType: ".log", +// sourceDir: "./", +// outputDir: ".", +// }, +// errNoSourceFilesFound, +// }, +// { +// "invalid filetype", +// &extractorCfg{ +// fileType: "", +// sourceDir: ".", +// outputDir: ".", +// }, +// errInvalidFileType, +// }, +// { +// "invalid source dir", +// &extractorCfg{ +// fileType: ".log", +// sourceDir: "", +// outputDir: ".", +// }, +// errInvalidSourceDir, +// }, +// { +// "invalid output dir", +// &extractorCfg{ +// fileType: ".log", +// sourceDir: ".", +// outputDir: "", +// }, +// errInvalidOutputDir, +// }, +// } +// +// for _, testCase := range testTable { +// testCase := testCase +// +// t.Run(testCase.name, func(t *testing.T) { +// t.Parallel() +// +// ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) +// defer cancelFn() +// +// err := execExtract(ctx, testCase.cfg) +// assert.ErrorIs(t, err, testCase.expectedErr) +// }) +// } +//} func TestValidFlow(t *testing.T) { - ctx := context.Background() - outputPath := "./" + t.Parallel() - //Generate temporary output dir - outputDir, err := os.MkdirTemp(outputPath, "") + // Generate temporary output dir + outputDir, err := os.MkdirTemp("./", "") require.NoError(t, err) + t.Cleanup(removeDir(t, outputDir)) - sourcePath := "." + var ( + sourcePath = "." - // Set correct config - cfg := &extractorCfg{ - fileType: sourceFileType, - sourceDir: sourcePath, - outputDir: outputDir, - } + // Set correct config + cfg = &extractorCfg{ + fileType: sourceFileType, + sourceDir: sourcePath, + outputDir: outputDir, + } + ) // Generate mock messages & mock files mockStdMsg, mockAddPkgMsg := generateMockMsgs(t) _ = generateSourceFiles(t, mockStdMsg, sourcePath) // Perform extraction - err = execExtract(ctx, cfg) - require.NoError(t, err) + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) + defer cancelFn() + + require.NoError(t, execExtract(ctx, cfg)) for _, msg := range mockAddPkgMsg { basePath := filepath.Join(outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) @@ -124,11 +136,10 @@ func TestValidFlow(t *testing.T) { // Compare metadata expectedMetadata, err := json.Marshal(metadataFromMsg(msg)) - require.Equal(t, expectedMetadata, retrievedMetadata) + assert.Equal(t, expectedMetadata, retrievedMetadata) // Close metadata file - err = file.Close() - require.NoError(t, err) + require.NoError(t, file.Close()) // Check package file content for _, f := range msg.Package.Files { @@ -139,14 +150,11 @@ func TestValidFlow(t *testing.T) { require.NoError(t, err) // Read file body - retrievedFileBody := make([]byte, 0) - reader := bufio.NewReader(file) - retrievedFileBody, _, err = reader.ReadLine() + retrievedFileBody, _, err := reader.ReadLine() // Compare file bodies require.Equal(t, f.Body, string(retrievedFileBody)) - } } } @@ -200,9 +208,9 @@ func TestFindFilePaths(t *testing.T) { } func TestExtractAddMessages(t *testing.T) { - t.Parallel() + //t.Parallel() - sourcePath := "." + sourcePath := "./source/" mockMsgs, mockMsgsAddPackage := generateMockMsgs(t) sourceFiles := generateSourceFiles(t, mockMsgs, sourcePath) @@ -301,6 +309,7 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg, sourcePath string) [] tempDir, err := os.MkdirTemp(sourcePath, "test") require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) var ( From 3965302f90c0f5773f9a0b05cadb3e66670b0028 Mon Sep 17 00:00:00 2001 From: leohhhn Date: Fri, 29 Dec 2023 16:17:26 +0100 Subject: [PATCH 41/41] fix broken tests --- extractor/main_test.go | 165 ++++++++++++++++++++--------------------- 1 file changed, 81 insertions(+), 84 deletions(-) diff --git a/extractor/main_test.go b/extractor/main_test.go index 47279f63..c0f157d6 100644 --- a/extractor/main_test.go +++ b/extractor/main_test.go @@ -32,88 +32,88 @@ const ( sourceFileType = ".log" ) -//func TestExtractor_Errors(t *testing.T) { -// testTable := []struct { -// name string -// cfg *extractorCfg -// expectedErr error -// }{ -// { -// "no source files", -// &extractorCfg{ -// fileType: ".log", -// sourceDir: "./", -// outputDir: ".", -// }, -// errNoSourceFilesFound, -// }, -// { -// "invalid filetype", -// &extractorCfg{ -// fileType: "", -// sourceDir: ".", -// outputDir: ".", -// }, -// errInvalidFileType, -// }, -// { -// "invalid source dir", -// &extractorCfg{ -// fileType: ".log", -// sourceDir: "", -// outputDir: ".", -// }, -// errInvalidSourceDir, -// }, -// { -// "invalid output dir", -// &extractorCfg{ -// fileType: ".log", -// sourceDir: ".", -// outputDir: "", -// }, -// errInvalidOutputDir, -// }, -// } -// -// for _, testCase := range testTable { -// testCase := testCase -// -// t.Run(testCase.name, func(t *testing.T) { -// t.Parallel() -// -// ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) -// defer cancelFn() -// -// err := execExtract(ctx, testCase.cfg) -// assert.ErrorIs(t, err, testCase.expectedErr) -// }) -// } -//} +func TestExtractor_Errors(t *testing.T) { + testTable := []struct { + name string + cfg *extractorCfg + expectedErr error + }{ + { + "no source files", + &extractorCfg{ + fileType: ".log", + sourceDir: "./", + outputDir: ".", + }, + errNoSourceFilesFound, + }, + { + "invalid filetype", + &extractorCfg{ + fileType: "", + sourceDir: ".", + outputDir: ".", + }, + errInvalidFileType, + }, + { + "invalid source dir", + &extractorCfg{ + fileType: ".log", + sourceDir: "", + outputDir: ".", + }, + errInvalidSourceDir, + }, + { + "invalid output dir", + &extractorCfg{ + fileType: ".log", + sourceDir: ".", + outputDir: "", + }, + errInvalidOutputDir, + }, + } + + for _, testCase := range testTable { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) + defer cancelFn() + + err := execExtract(ctx, testCase.cfg) + assert.ErrorIs(t, err, testCase.expectedErr) + }) + } +} func TestValidFlow(t *testing.T) { t.Parallel() // Generate temporary output dir - outputDir, err := os.MkdirTemp("./", "") + outputDir, err := os.MkdirTemp(".", "output") require.NoError(t, err) - t.Cleanup(removeDir(t, outputDir)) - var ( - sourcePath = "." + // Generate temporary source dir + sourceDir, err := os.MkdirTemp(".", "source") + require.NoError(t, err) + t.Cleanup(removeDir(t, sourceDir)) - // Set correct config - cfg = &extractorCfg{ - fileType: sourceFileType, - sourceDir: sourcePath, - outputDir: outputDir, - } - ) + // Set correct config + var cfg = &extractorCfg{ + fileType: sourceFileType, + sourceDir: sourceDir, + outputDir: outputDir, + } // Generate mock messages & mock files mockStdMsg, mockAddPkgMsg := generateMockMsgs(t) - _ = generateSourceFiles(t, mockStdMsg, sourcePath) + _ = generateSourceFiles(t, sourceDir, mockStdMsg) // Perform extraction ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) @@ -154,7 +154,7 @@ func TestValidFlow(t *testing.T) { retrievedFileBody, _, err := reader.ReadLine() // Compare file bodies - require.Equal(t, f.Body, string(retrievedFileBody)) + assert.Equal(t, f.Body, string(retrievedFileBody)) } } } @@ -208,11 +208,14 @@ func TestFindFilePaths(t *testing.T) { } func TestExtractAddMessages(t *testing.T) { - //t.Parallel() + t.Parallel() + + tempDir, err := os.MkdirTemp(".", "test") + require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) - sourcePath := "./source/" mockMsgs, mockMsgsAddPackage := generateMockMsgs(t) - sourceFiles := generateSourceFiles(t, mockMsgs, sourcePath) + sourceFiles := generateSourceFiles(t, tempDir, mockMsgs) var results []vm.MsgAddPackage for _, sf := range sourceFiles { @@ -272,7 +275,6 @@ func TestWritePackageMetadata(t *testing.T) { require.Equal(t, md, unmarshalledMetadata) } } - func TestWritePackageFiles(t *testing.T) { t.Parallel() @@ -304,21 +306,16 @@ func TestWritePackageFiles(t *testing.T) { } // Helpers -func generateSourceFiles(t *testing.T, mockMsgs []std.Msg, sourcePath string) []string { +func generateSourceFiles(t *testing.T, dir string, mockMsgs []std.Msg) []string { t.Helper() - tempDir, err := os.MkdirTemp(sourcePath, "test") - require.NoError(t, err) - - t.Cleanup(removeDir(t, tempDir)) - var ( mockTx = make([]std.Tx, numTx) testFiles = make([]string, numSourceFiles) ) // Generate transactions to wrap messages - for i := range mockTx { // num + for i := range mockTx { mockTx[i] = std.Tx{ Msgs: mockMsgs[:msgPerTx], } @@ -332,7 +329,7 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg, sourcePath string) [] // Generate source files for _, file := range testFiles { - filePath := filepath.Join(tempDir, file) + filePath := filepath.Join(dir, file) err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm) require.NoError(t, err) @@ -350,7 +347,7 @@ func generateSourceFiles(t *testing.T, mockMsgs []std.Msg, sourcePath string) [] } for i := 0; i < numSourceFiles; i++ { - testFiles[i] = filepath.Join(tempDir, testFiles[i]) + testFiles[i] = filepath.Join(dir, testFiles[i]) } return testFiles