Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow reding from CSV and querying AST using (Go) JQ #3051

Merged
merged 1 commit into from
Jan 26, 2024
Merged

Allow reding from CSV and querying AST using (Go) JQ #3051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/c-bata/go-prompt v0.2.6
github.com/dave/dst v0.27.2
github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c
github.com/itchyny/gojq v0.12.14
github.com/k0kubun/pp/v3 v3.2.0
github.com/kr/pretty v0.3.1
github.com/leanovate/gopter v0.2.9
Expand Down Expand Up @@ -38,12 +39,13 @@ require (
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fxamacker/circlehash v0.3.0 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
github.com/klauspost/cpuid/v2 v2.2.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-tty v0.0.3 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/pkg/term v1.2.0-beta.2 // indirect
Expand All @@ -53,7 +55,7 @@ require (
github.com/zeebo/assert v1.3.0 // indirect
github.com/zeebo/blake3 v0.2.3 // indirect
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.6.0 // indirect
gonum.org/v1/gonum v0.6.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
13 changes: 10 additions & 3 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ github.com/fxamacker/circlehash v0.3.0 h1:XKdvTtIJV9t7DDUtsf0RIpC1OcxZtPbmgIH7ek
github.com/fxamacker/circlehash v0.3.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc=
github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s=
github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
Expand Down Expand Up @@ -51,12 +55,14 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI=
github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
Expand Down Expand Up @@ -137,8 +143,9 @@ golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
154 changes: 128 additions & 26 deletions runtime/cmd/parse/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ package main

import (
"bufio"
"bytes"
"encoding/csv"
"encoding/json"
"flag"
"fmt"
Expand All @@ -32,6 +34,8 @@ import (
"testing"
"time"

"github.com/itchyny/gojq"

"github.com/onflow/cadence/runtime/ast"
"github.com/onflow/cadence/runtime/common"
"github.com/onflow/cadence/runtime/parser"
Expand All @@ -40,13 +44,15 @@ import (

var benchFlag = flag.Bool("bench", false, "benchmark the parser")
var jsonFlag = flag.Bool("json", false, "print the result formatted as JSON")
var readCSVFlag = flag.Bool("readCSV", false, "read the input file as CSV (header: location,code)")
var jqASTFlag = flag.String("jqAST", "", "query the AST using gojq")

func main() {
testing.Init()
flag.Parse()

args := flag.Args()
run(args, *benchFlag, *jsonFlag)
run(args, *benchFlag, *jsonFlag, *readCSVFlag, *jqASTFlag)
}

type benchResult struct {
Expand All @@ -63,6 +69,7 @@ type result struct {
Path string `json:"path,omitempty"`
BenchStr string `json:"-"`
Code []byte `json:"-"`
Results []any `json:"results,omitempty"`
}

type output interface {
Expand All @@ -71,10 +78,11 @@ type output interface {
}

type jsonOutput struct {
file *os.File
results []result
}

func newJSONOutput(count int) *jsonOutput {
func newJSONOutput(file *os.File, count int) *jsonOutput {
return &jsonOutput{
results: make([]result, 0, count),
}
Expand All @@ -85,67 +93,123 @@ func (j *jsonOutput) Append(r result) {
}

func (j *jsonOutput) End() {
encoder := json.NewEncoder(os.Stdout)
encoder := json.NewEncoder(j.file)
encoder.SetIndent("", " ")
err := encoder.Encode(j.results)
if err != nil {
panic(err)
}
}

type stdoutOutput struct{}
type fileOutput struct {
file *os.File
}

func (s stdoutOutput) Append(r result) {
var err error
func (s fileOutput) Append(r result) {

if len(r.Path) > 0 {
_, err = fmt.Printf("%s\n", r.Path)
if err != nil {
panic(err)
}
_, _ = fmt.Fprintf(s.file, "%s\n", r.Path)
}

if r.Error != nil {
location := common.NewStringLocation(nil, r.Path)
printErr := pretty.NewErrorPrettyPrinter(os.Stdout, true).
printErr := pretty.NewErrorPrettyPrinter(s.file, true).
PrettyPrintError(r.Error, location, map[common.Location][]byte{location: r.Code})
if printErr != nil {
panic(printErr)
}
}

if len(r.BenchStr) > 0 {
_, err = fmt.Printf("bench:\t%s\n", r.BenchStr)
if err != nil {
_, _ = fmt.Fprintf(s.file, "bench:\t%s\n", r.BenchStr)
}

if len(r.Results) > 0 {
_, _ = fmt.Fprint(s.file, "query results:\n")

for _, res := range r.Results {
_, _ = fmt.Fprintf(s.file, "- %#+v\n", res)
}
}

println()
}

func queryProgram(program *ast.Program, query *gojq.Code) []any {
// Encode to JSON
var buf bytes.Buffer
err := json.NewEncoder(&buf).Encode(program)
if err != nil {
panic(err)
}

// Decode from JSON
var decoded any
err = json.NewDecoder(&buf).Decode(&decoded)
if err != nil {
panic(err)
}

var results []any

// Run query and print results
iter := query.Run(decoded)
for {
v, ok := iter.Next()
if !ok {
break
}
if err, ok := v.(error); ok {
panic(err)
}

results = append(results, v)
}

return results
}

func (s stdoutOutput) End() {
func (s fileOutput) End() {
// no-op
}

func run(paths []string, bench bool, json bool) {
func run(paths []string, bench bool, json bool, readCSV bool, jqAST string) {
if len(paths) == 0 {
paths = []string{""}
}

var compiledQuery *gojq.Code

if jqAST != "" {
query, err := gojq.Parse(jqAST)
if err != nil {
panic(err)
}

compiledQuery, err = gojq.Compile(query)
if err != nil {
panic(err)
}

}

var out output
if json {
out = newJSONOutput(len(paths))
out = newJSONOutput(os.Stdout, len(paths))
} else {
out = stdoutOutput{}
out = fileOutput{file: os.Stdout}
}

allSucceeded := true

for _, path := range paths {
res, runSucceeded := runPath(path, bench)
if !runSucceeded {
allSucceeded = false
for _, file := range read(path, readCSV) {
res, runSucceeded := runFile(file, bench, compiledQuery)
if !runSucceeded {
allSucceeded = false
}
out.Append(res)
}
out.Append(res)
}

out.End()
Expand All @@ -155,13 +219,13 @@ func run(paths []string, bench bool, json bool) {
}
}

func runPath(path string, bench bool) (res result, succeeded bool) {
func runFile(file file, bench bool, query *gojq.Code) (res result, succeeded bool) {
res = result{
Path: path,
Path: file.path,
}
succeeded = true

code := read(path)
code := file.code
res.Code = code

var program *ast.Program
Expand All @@ -175,6 +239,8 @@ func runPath(path string, bench bool) (res result, succeeded bool) {
}
}()

_, _ = fmt.Fprintf(os.Stderr, "parsing %s\n", file.path)

program, err = parser.ParseProgram(nil, code, parser.Config{})
if !bench {
res.Program = program
Expand All @@ -199,10 +265,19 @@ func runPath(path string, bench bool) (res result, succeeded bool) {
res.BenchStr = benchRes.String()
}

if program != nil && query != nil {
res.Results = queryProgram(program, query)
}

return
}

func read(path string) []byte {
type file struct {
path string
code []byte
}

func read(path string, readCSV bool) []file {
var data []byte
var err error
if len(path) == 0 {
Expand All @@ -213,7 +288,34 @@ func read(path string) []byte {
if err != nil {
panic(err)
}
return data

if readCSV {
records, err := csv.NewReader(bytes.NewReader(data)).ReadAll()
if err != nil {
panic(err)
}

files := make([]file, 0, len(records))

// Convert all records to files, except for the header
for _, record := range records[1:] {
files = append(files,
file{
path: record[0],
code: []byte(record[1]),
},
)
}

return files
} else {
return []file{
{
path: path,
code: data,
},
}
}
}

func benchParse(parse func() (err error)) testing.BenchmarkResult {
Expand Down
Loading