Merge pull request #3446 from TrueBlocks/3423-slurp-articulate

3423 slurp articulate

docs/content/api/openapi.yaml

@@ -2345,6 +2345,14 @@ paths:
           explode: true
           schema:
             type: boolean
+        - name: articulate
+          description: articulate the retrieved data if ABIs can be found
+          required: false
+          style: form
+          in: query
+          explode: true
+          schema:
+            type: boolean
         - name: perPage
           description: the number of records to request on each page
           required: false

docs/content/chifra/other.md

@@ -85,6 +85,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

docs/readmes/other-slurp.md

@@ -27,6 +27,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

sdk/python/src/_slurp.py

@@ -9,6 +9,7 @@
 slurpOpts = {
     "types": {"hotkey": "-t", "type": "flag"},
     "appearances": {"hotkey": "-p", "type": "switch"},
+    "articulate": {"hotkey": "-a", "type": "switch"},
     "perPage": {"hotkey": "-P", "type": "flag"},
     "sleep": {"hotkey": "-s", "type": "flag"},
     "raw": {"hotkey": "-w", "type": "switch"},

sdk/typescript/src/paths/slurp.ts

@@ -12,6 +12,7 @@ export function getSlurp(
     blocks?: blknum[],
     types?: string[],
     appearances?: boolean,
+    articulate?: boolean,
     perPage?: uint64,
     sleep?: double,
     chain: string,

src/apps/chifra/cmd/slurp.go

@@ -66,6 +66,7 @@ func init() {
 	slurpCmd.Flags().StringSliceVarP(&slurpPkg.GetOptions().Types, "types", "t", nil, `which types of transactions to request
 One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]`)
 	slurpCmd.Flags().BoolVarP(&slurpPkg.GetOptions().Appearances, "appearances", "p", false, "show only the blocknumber.tx_id appearances of the exported transactions")
+	slurpCmd.Flags().BoolVarP(&slurpPkg.GetOptions().Articulate, "articulate", "a", false, "articulate the retrieved data if ABIs can be found")
 	slurpCmd.Flags().Uint64VarP(&slurpPkg.GetOptions().PerPage, "per_page", "P", 5000, "the number of records to request on each page")
 	slurpCmd.Flags().Float64VarP(&slurpPkg.GetOptions().Sleep, "sleep", "s", .25, "seconds to sleep between requests")
 	globals.InitGlobals("slurp", slurpCmd, &slurpPkg.GetOptions().Globals, capabilities)

src/apps/chifra/internal/slurp/README.md

@@ -27,6 +27,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

src/apps/chifra/internal/slurp/handle_show.go

@@ -22,6 +22,9 @@ func (opts *SlurpOptions) HandleShow() error {
 		paginator.PerPage = 100
 	}
 
+	// TODO: Turn this back on
+	// abiCache := articulate.NewAbiCache(opts.Conn, opts.Articulate)
+
 	ctx := context.Background()
 	fetchData := func(modelChan chan types.Modeler[types.RawSlurp], errorChan chan error) {
 		totalFetched := 0
@@ -49,8 +52,13 @@ func (opts *SlurpOptions) HandleShow() error {
 						if !opts.isInRange(uint(tx.BlockNumber), errorChan) {
 							continue
 						}
-						bar.Tick()
+						// if opts.Articulate {
+						// 	if err = abiCache.ArticulateSlurp(&tx); err != nil {
+						// 		errorChan <- err // continue even with an error
+						// 	}
+						// }
 						modelChan <- &tx
+						bar.Tick()
 						totalFiltered++
 					}
 
@@ -72,7 +80,11 @@ func (opts *SlurpOptions) HandleShow() error {
 		}
 	}
 
-	return output.StreamMany(ctx, fetchData, opts.Globals.OutputOpts())
+	extra := map[string]interface{}{
+		// "articulate": opts.Articulate,
+	}
+
+	return output.StreamMany(ctx, fetchData, opts.Globals.OutputOptsWithExtra(extra))
 }
 
 func (opts *SlurpOptions) isInRange(bn uint, errorChan chan error) bool {

src/apps/chifra/internal/slurp/options.go

@@ -28,6 +28,7 @@ type SlurpOptions struct {
 	BlockIds    []identifiers.Identifier `json:"blockIds,omitempty"`    // Block identifiers
 	Types       []string                 `json:"types,omitempty"`       // Which types of transactions to request
 	Appearances bool                     `json:"appearances,omitempty"` // Show only the blocknumber.tx_id appearances of the exported transactions
+	Articulate  bool                     `json:"articulate,omitempty"`  // Articulate the retrieved data if ABIs can be found
 	PerPage     uint64                   `json:"perPage,omitempty"`     // The number of records to request on each page
 	Sleep       float64                  `json:"sleep,omitempty"`       // Seconds to sleep between requests
 	Globals     globals.GlobalOptions    `json:"globals,omitempty"`     // The global options
@@ -47,6 +48,7 @@ func (opts *SlurpOptions) testLog() {
 	logger.TestLog(len(opts.Blocks) > 0, "Blocks: ", opts.Blocks)
 	logger.TestLog(len(opts.Types) > 0, "Types: ", opts.Types)
 	logger.TestLog(opts.Appearances, "Appearances: ", opts.Appearances)
+	logger.TestLog(opts.Articulate, "Articulate: ", opts.Articulate)
 	logger.TestLog(opts.PerPage != 5000, "PerPage: ", opts.PerPage)
 	logger.TestLog(opts.Sleep != float64(.25), "Sleep: ", opts.Sleep)
 	opts.Conn.TestLog(opts.getCaches())
@@ -84,6 +86,8 @@ func slurpFinishParseApi(w http.ResponseWriter, r *http.Request) *SlurpOptions {
 			}
 		case "appearances":
 			opts.Appearances = true
+		case "articulate":
+			opts.Articulate = true
 		case "perPage":
 			opts.PerPage = globals.ToUint64(value[0])
 		case "sleep":

src/apps/chifra/pkg/articulate/slurp.go

@@ -0,0 +1,19 @@
+package articulate
+
+import (
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
+)
+
+func (abiCache *AbiCache) ArticulateSlurp(slurp *types.SimpleSlurp) error {
+	tx := types.SimpleTransaction{
+		To: slurp.To,
+	}
+
+	if err := abiCache.ArticulateTransaction(&tx); err != nil {
+		return err
+	}
+
+	slurp.ArticulatedTx = tx.ArticulatedTx
+	// TODO: We could add `message` here, but we'd have to modify the `types.SimpleSlurp` struct
+	return nil
+}

src/apps/chifra/pkg/types/types_slurp.go

@@ -175,6 +175,30 @@ func (s *SimpleSlurp) Model(chain, format string, verbose bool, extraOptions map
 	}
 	model["transactionIndex"] = s.TransactionIndex
 
+	// TODO: Turn this back on
+	// var articulatedTx map[string]interface{}
+	// isArticulated := extraOptions["articulate"] == true && s.ArticulatedTx != nil
+	// if isArticulated && format != "json" {
+	// 	order = append(order, "compressedTx")
+	// }
+	// if isArticulated {
+	// 	articulatedTx = map[string]interface{}{
+	// 		"name": s.ArticulatedTx.Name,
+	// 	}
+	// 	inputModels := parametersToMap(s.ArticulatedTx.Inputs)
+	// 	if inputModels != nil {
+	// 		articulatedTx["inputs"] = inputModels
+	// 	}
+	// 	outputModels := parametersToMap(s.ArticulatedTx.Outputs)
+	// 	if outputModels != nil {
+	// 		articulatedTx["outputs"] = outputModels
+	// 	}
+	// 	sm := s.ArticulatedTx.StateMutability
+	// 	if sm != "" && sm != "nonpayable" && sm != "view" {
+	// 		articulatedTx["stateMutability"] = sm
+	// 	}
+	// }
+
 	if format == "json" {
 		a := s.ContractAddress.Hex()
 		if strings.HasPrefix(a, "0x") && len(a) == 42 {
@@ -184,6 +208,15 @@ func (s *SimpleSlurp) Model(chain, format string, verbose bool, extraOptions map
 		if len(s.Input) > 2 && s.Input != "deprecated" {
 			model["input"] = s.Input
 		}
+
+		// if isArticulated {
+		// 	model["articulatedTx"] = articulatedTx
+		// } else {
+		// 	if s.Message != "" {
+		// 		model["message"] = s.Message
+		// 	}
+		// }
+
 	} else {
 		model["hasToken"] = s.HasToken
 		model["isError"] = s.IsError
@@ -192,6 +225,20 @@ func (s *SimpleSlurp) Model(chain, format string, verbose bool, extraOptions map
 			s.Input = "0x"
 		}
 		model["input"] = s.Input
+
+		// model["compressedTx"] = ""
+		// enc := s.Input
+		// if len(s.Input) >= 10 {
+		// 	enc = s.Input[:10]
+		// }
+		// model["encoding"] = enc
+
+		// if isArticulated {
+		// 	model["compressedTx"] = makeCompressed(articulatedTx)
+		// } else if s.Message != "" {
+		// 	model["encoding"] = ""
+		// 	model["compressedTx"] = s.Message
+		// }
 	}
 
 	// EXISTING_CODE

src/cmd-line-options.csv

@@ -316,6 +316,7 @@ num,group,tags,api_route,tool,longName,hotKey,def_val,is_required,is_customizabl
 12140,tools,Other,slurp,ethslurp,blocks,,,false,false,true,true,gocmd,positional,list<blknum>,an optional range of blocks to slurp
 12160,tools,Other,slurp,ethslurp,types,t,,false,false,true,true,gocmd,flag,list<enum[ext*|int|token|nfts|1155|miner|uncles|withdrawals|all]>,which types of transactions to request
 12180,tools,Other,slurp,ethslurp,appearances,p,,false,false,true,true,gocmd,switch,<boolean>,show only the blocknumber.tx_id appearances of the exported transactions
+12185,tools,Other,slurp,ethslurp,articulate,a,,false,false,true,true,gocmd,switch,<boolean>,articulate the retrieved data if ABIs can be found
 12200,tools,Other,slurp,ethslurp,per_page,P,5000,false,false,true,true,gocmd,flag,<uint64>,the number of records to request on each page
 12210,tools,Other,slurp,ethslurp,sleep,s,.25,false,false,true,true,gocmd,flag,<double>,seconds to sleep between requests
 12220,tools,Other,slurp,ethslurp,,,,false,false,true,true,--,description,,Fetch data from Etherscan for any address.

src/dev_tools/makeClass/results/report.csv

@@ -323,6 +323,9 @@ scrape,touch,typescript
 slurp,appearances,api
 slurp,appearances,python
 slurp,appearances,typescript
+slurp,articulate,api
+slurp,articulate,python
+slurp,articulate,typescript
 slurp,cache,api
 slurp,cache,python
 slurp,perPage,api

test/gold/apps/chifra/chifra_help_slurp.txt

@@ -13,6 +13,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_cache_and_decache.txt

@@ -19,6 +19,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_caps_allowed.txt

@@ -13,6 +13,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_caps_disallowed_1.txt

@@ -13,6 +13,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_help.txt

@@ -13,6 +13,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_help_long.txt

@@ -13,6 +13,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_invalid_address.txt

@@ -15,6 +15,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_invalid_param.txt

@@ -13,6 +13,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_invalid_type_1.txt

@@ -15,6 +15,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_invalid_type_2.txt

@@ -13,6 +13,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether

test/gold/tools/ethslurp/ethslurp_no_params.txt

@@ -14,6 +14,7 @@ Flags:
   -t, --types strings   which types of transactions to request
                         One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
+  -a, --articulate      articulate the retrieved data if ABIs can be found
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)
   -H, --ether           specify value in ether