From e875a40820f3674c25e172b176faf328c19d036e Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Tue, 19 Nov 2024 12:06:36 +0100 Subject: [PATCH] fix: bulk error response --- example2.js | 20 +++++++++ internal/api/v2/controllers_bulk.go | 31 +++++++++++++- pkg/generate/generator.go | 17 ++++++-- pkg/generate/generator_test.go | 13 +++++- test/performance/example_scripts/example1.js | 10 +++-- test/rolling-upgrades/Earthfile | 44 +++++++++++--------- tools/generator/cmd/root.go | 1 + tools/generator/examples/example1.js | 10 +++-- 8 files changed, 113 insertions(+), 33 deletions(-) create mode 100644 example2.js diff --git a/example2.js b/example2.js new file mode 100644 index 000000000..80aa7bc7b --- /dev/null +++ b/example2.js @@ -0,0 +1,20 @@ +function next() { + let postings = []; + for(let i = 0; i < 500000; i++) { + postings.push({ + source: `world`, + destination: `banks`, + amount: 100, + asset: 'USD' + }) + } + return { + action: 'CREATE_TRANSACTION', + data: { + postings + } + } +} + + + diff --git a/internal/api/v2/controllers_bulk.go b/internal/api/v2/controllers_bulk.go index 7ecbecf32..9eb3ce18a 100644 --- a/internal/api/v2/controllers_bulk.go +++ b/internal/api/v2/controllers_bulk.go @@ -32,7 +32,11 @@ func bulkHandler(bulkMaxSize int) http.HandlerFunc { w.Header().Set("Content-Type", "application/json") ret, errorsInBulk, err := ProcessBulk(r.Context(), common.LedgerFromContext(r.Context()), b, api.QueryParamBool(r, "continueOnFailure")) - if err != nil || errorsInBulk { + if err != nil { + api.InternalServerError(w, r, err) + return + } + if errorsInBulk { w.WriteHeader(http.StatusBadRequest) } @@ -125,6 +129,31 @@ func ProcessBulk( continueOnFailure bool, ) ([]Result, bool, error) { + for i, element := range bulk { + switch element.Action { + case ActionCreateTransaction: + req := &TransactionRequest{} + if err := json.Unmarshal(element.Data, req); err != nil { + return nil, false, fmt.Errorf("error parsing element %d: %s", i, err) + } + case ActionAddMetadata: + req := &AddMetadataRequest{} + if err := json.Unmarshal(element.Data, req); err != nil { + return nil, false, fmt.Errorf("error parsing element %d: %s", i, err) + } + case ActionRevertTransaction: + req := &RevertTransactionRequest{} + if err := json.Unmarshal(element.Data, req); err != nil { + return nil, false, fmt.Errorf("error parsing element %d: %s", i, err) + } + case ActionDeleteMetadata: + req := &DeleteMetadataRequest{} + if err := json.Unmarshal(element.Data, req); err != nil { + return nil, false, fmt.Errorf("error parsing element %d: %s", i, err) + } + } + } + ret := make([]Result, 0, len(bulk)) errorsInBulk := false diff --git a/pkg/generate/generator.go b/pkg/generate/generator.go index 5681c6950..7f319d23b 100644 --- a/pkg/generate/generator.go +++ b/pkg/generate/generator.go @@ -6,9 +6,9 @@ import ( "errors" "fmt" "github.com/dop251/goja" + "github.com/formancehq/go-libs/v2/collectionutils" ledger "github.com/formancehq/ledger/internal" v2 "github.com/formancehq/ledger/internal/api/v2" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/formancehq/ledger/pkg/client" "github.com/formancehq/ledger/pkg/client/models/components" "github.com/formancehq/ledger/pkg/client/models/operations" @@ -45,7 +45,7 @@ func (r Action) Apply(ctx context.Context, client *client.V2, l string) (*Result var bulkElement components.V2BulkElement switch r.Action { case v2.ActionCreateTransaction: - transactionRequest := &ledgercontroller.RunScript{} + transactionRequest := &v2.TransactionRequest{} err := json.Unmarshal(r.Data, transactionRequest) if err != nil { return nil, fmt.Errorf("failed to unmarshal transaction request: %w", err) @@ -61,8 +61,18 @@ func (r Action) Apply(ctx context.Context, client *client.V2, l string) (*Result }(), Script: &components.V2PostTransactionScript{ Plain: transactionRequest.Script.Plain, - Vars: transactionRequest.Script.Vars, + Vars: collectionutils.ConvertMap(transactionRequest.Script.Vars, func(from any) string { + return fmt.Sprint(from) + }), }, + Postings: collectionutils.Map(transactionRequest.Postings, func(p ledger.Posting) components.V2Posting { + return components.V2Posting{ + Amount: p.Amount, + Asset: p.Asset, + Destination: p.Destination, + Source: p.Source, + } + }), Reference: func() *string { if transactionRequest.Reference == "" { return nil @@ -161,6 +171,7 @@ func (r Action) Apply(ctx context.Context, client *client.V2, l string) (*Result if err != nil { return nil, fmt.Errorf("creating transaction: %w", err) } + if errorResponse := response.V2BulkResponse.Data[0].V2BulkElementResultError; errorResponse != nil { if errorResponse.ErrorCode != "" { errorDescription := errorResponse.ErrorDescription diff --git a/pkg/generate/generator_test.go b/pkg/generate/generator_test.go index dd32406f4..b06977485 100644 --- a/pkg/generate/generator_test.go +++ b/pkg/generate/generator_test.go @@ -71,14 +71,23 @@ function next(iteration) { return { action: 'CREATE_TRANSACTION', data: { - plain: ` + "`" + ` + script: { + vars: { + dst: "bank" + }, + plain: ` + "`" + ` +vars { + account $dst +} + send [USD/2 100] ( source = @world - destination = @bank + destination = $dst ) set_tx_meta("globalMetadata", "${globalMetadata}") ` + "`" + ` + } } } case 1: diff --git a/test/performance/example_scripts/example1.js b/test/performance/example_scripts/example1.js index 81e7c0997..0157f6c5b 100644 --- a/test/performance/example_scripts/example1.js +++ b/test/performance/example_scripts/example1.js @@ -19,10 +19,12 @@ function next(iteration) { return { action: 'CREATE_TRANSACTION', data: { - plain, - vars: { - order: `orders:${uuid()}`, - seller: `sellers:${iteration % 5}` + script: { + plain, + vars: { + order: `orders:${uuid()}`, + seller: `sellers:${iteration % 5}` + } } } } diff --git a/test/rolling-upgrades/Earthfile b/test/rolling-upgrades/Earthfile index 5df23e983..e05900a22 100644 --- a/test/rolling-upgrades/Earthfile +++ b/test/rolling-upgrades/Earthfile @@ -58,7 +58,7 @@ cluster-create: RUN curl -L -o vcluster "https://github.com/loft-sh/vcluster/releases/download/v0.20.4/vcluster-linux-${TARGETARCH}" RUN install -c -m 0755 vcluster /usr/local/bin && rm -f vcluster ARG CLUSTER_NAME=test - RUN --no-cache vcluster create $CLUSTER_NAME --connect=false --upgrade + RUN vcluster create $CLUSTER_NAME --connect=false run: ARG CLUSTER_NAME=test @@ -86,24 +86,30 @@ run: ARG NO_CLEANUP=false ARG NO_CLEANUP_ON_FAILURE=false - RUN --secret PULUMI_ACCESS_TOKEN --secret GITHUB_TOKEN sh -c ' - echo "Connecting to VCluster..." - vcluster connect ${CLUSTER_NAME} --namespace vcluster-${CLUSTER_NAME} & - export KUBECONFIG=/root/.kube/config; - - echo "Waiting for VCluster to be ready..." - until kubectl get nodes; do sleep 1s; done; - - echo "Running test..." - go test \ - --test-image ghcr.io/formancehq/ledger:$CLUSTER_NAME-rolling-upgrade-test \ - --latest-version $CLUSTER_NAME-main \ - --actual-version $CLUSTER_NAME-current \ - --project ledger \ - --stack-prefix-name $CLUSTER_NAME- \ - --no-cleanup=$NO_CLEANUP \ - --no-cleanup-on-failure=$NO_CLEANUP_ON_FAILURE; - ' + WITH DOCKER + RUN --secret PULUMI_ACCESS_TOKEN --secret GITHUB_TOKEN sh -c ' + set -e; + + echo "Connecting to VCluster..." + vcluster connect ${CLUSTER_NAME} --namespace vcluster-${CLUSTER_NAME}; + + echo "Connected on context '$(kubectl config current-context)'"; + + echo "Waiting for VCluster to be ready..." + until kubectl get nodes; do sleep 1s; done; + + echo "Running test..." + go test \ + --test-image ghcr.io/formancehq/ledger:$CLUSTER_NAME-rolling-upgrade-test \ + --latest-version $CLUSTER_NAME-main \ + --actual-version $CLUSTER_NAME-current \ + --project ledger \ + --stack-prefix-name $CLUSTER_NAME- \ + --no-cleanup=$NO_CLEANUP \ + --no-cleanup-on-failure=$NO_CLEANUP_ON_FAILURE; + ' + END + IF [ $NO_CLEANUP = "false" ] RUN vcluster delete $CLUSTER_NAME --delete-namespace END diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go index 158d66770..84cca4463 100644 --- a/tools/generator/cmd/root.go +++ b/tools/generator/cmd/root.go @@ -62,6 +62,7 @@ func Execute() { } func run(cmd *cobra.Command, args []string) error { + ledgerUrl := args[0] scriptLocation := args[1] diff --git a/tools/generator/examples/example1.js b/tools/generator/examples/example1.js index cefe6c0b5..fc50dd1a1 100644 --- a/tools/generator/examples/example1.js +++ b/tools/generator/examples/example1.js @@ -19,10 +19,12 @@ function next(iteration) { return { action: 'CREATE_TRANSACTION', data: { - plain, - vars: { - order: `orders:${uuid()}`, - seller: `sellers:${iteration % 5}` + script: { + plain, + vars: { + order: `orders:${uuid()}`, + seller: `sellers:${iteration % 5}` + } } } }