From fa1c1b67d8a1ae03337bbe9a7a9afc1aaf5d6345 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 20 Aug 2020 18:48:23 -0400 Subject: [PATCH 1/9] rosetta/leverage-docker-caching Leverage docker caching to speed up docker-rosetta builds but use that time to build a testnet_postake_medium_curves image in addition to the original dev image --- buildkite/scripts/docker-artifact.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index 59e0159c753..e143033e96d 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -10,6 +10,12 @@ scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION} if [[ -n $CODA_BUILD_ROSETTA ]]; then echo "--- Build/Release coda-rosetta to docker hub" + # Could also cache-from opam-deps but we would need to get that automatically building nightly or at least when src/opam.export changes + # build-deps is updated by manually running scripts/build-rosetta-stages.sh which always builds + pushes each stage + scripts/release-docker.sh --service "coda-rosetta" --version "dev-${CODA_VERSION}"\ + --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/coda-rosetta-build-deps:develop" + # Also build with the standard DUNE_PROFILE, and use the dev profile as a cache. + # This means it will use the opam-deps stage from the previous step, but make a new builder stage because the DUNE_PROFILE arg changed scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}"\ - --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH}" + --extra-args "--build-arg --build-arg CODA_BRANCH=${CODA_GIT_BRANCH} --cache-from codaprotocol/coda-rosetta:dev-${CODA_VERSION}" fi From 3ee8e525662d5243e83ac9d8d89df207bfca9cf6 Mon Sep 17 00:00:00 2001 From: Brandon Kase Date: Wed, 2 Sep 2020 21:31:19 +0300 Subject: [PATCH 2/9] WIP --- src/app/rosetta/lib/dune | 1 + src/app/rosetta/ocaml-signer/dune | 25 +++++++ src/app/rosetta/ocaml-signer/signer.ml | 90 ++++++++++++++++++++++++++ src/app/rosetta/start.sh | 2 +- src/app/rosetta/test-agent/dune | 1 - src/app/rosetta/test-agent/signer.ml | 87 ------------------------- 6 files changed, 117 insertions(+), 89 deletions(-) create mode 100644 src/app/rosetta/ocaml-signer/dune create mode 100644 src/app/rosetta/ocaml-signer/signer.ml delete mode 100644 src/app/rosetta/test-agent/signer.ml diff --git a/src/app/rosetta/lib/dune b/src/app/rosetta/lib/dune index 6d1c4bc6e05..b4f59642b25 100644 --- a/src/app/rosetta/lib/dune +++ b/src/app/rosetta/lib/dune @@ -20,6 +20,7 @@ yojson archive_lib signature_lib + secrets unsigned_extended ) (preprocess (pps diff --git a/src/app/rosetta/ocaml-signer/dune b/src/app/rosetta/ocaml-signer/dune new file mode 100644 index 00000000000..f4b514537c7 --- /dev/null +++ b/src/app/rosetta/ocaml-signer/dune @@ -0,0 +1,25 @@ +(executable + (package signer) + (name signer) + (public_name signer) + (modes native) + (libraries + async + async_ssl + core_kernel + logger + rosetta_models + lib + ppx_deriving_yojson.runtime + yojson + ) + (preprocess (pps + graphql_ppx + ppx_coda + ppx_deriving.show + ppx_deriving_yojson + ppx_jane + ppx_version + ppx_deriving.eq + )) + ) diff --git a/src/app/rosetta/ocaml-signer/signer.ml b/src/app/rosetta/ocaml-signer/signer.ml new file mode 100644 index 00000000000..0ee438d1e3f --- /dev/null +++ b/src/app/rosetta/ocaml-signer/signer.ml @@ -0,0 +1,90 @@ +(** An agent that pokes at Coda and peeks at Rosetta to see if things look alright *) + +open Core_kernel +open Async +open Rosetta_lib +open Signature_lib +open Lib + +let sign_command = + let open Command.Let_syntax in + let%map_open unsigned_transaction = + flag "unsigned-transaction" + ~doc:"Unsigned transaction string returned from Rosetta" + (required string) + and private_key = + flag "private-key" ~doc:"Private key hex bytes" (required string) + in + let open Deferred.Let_syntax in + fun () -> + let keys = Signer.Keys.of_private_key_bytes private_key in + match + Signer.sign ~keys ~unsigned_transaction_string:unsigned_transaction + with + | Ok signature -> + printf "%s\n" signature ; return () + | Error e -> + eprintf "Failed to sign transaction %s" (Errors.show e) ; + exit 1 + +let verify_command = + let open Command.Let_syntax in + let%map_open signed_transaction = + flag "signed-transaction" + ~doc:"Signed transaction string returned from Rosetta" (required string) + and public_key = + flag "public-key" ~doc:"Public key hex bytes" (required string) + in + let open Deferred.Let_syntax in + fun () -> + match + Signer.verify ~public_key_hex_bytes:public_key + ~signed_transaction_string:signed_transaction + with + | Ok b when b -> + return () + | Ok _b (* when not _b *) -> + eprintf "Signature does not verify against this public key" ; + exit 1 + | Error e -> + eprintf "Failed to verify signature %s" (Errors.show e) ; + exit 1 + +let derive_command = + let open Command.Let_syntax in + let%map_open private_key = + flag "private-key" ~doc:"Private key hex bytes" (required string) + in + let open Deferred.Let_syntax in + fun () -> + printf "%s\n" + Signer.Keys.(of_private_key_bytes private_key).public_key_hex_bytes ; + return () + +let generate_command = + let open Deferred.Let_syntax in + Command.Param.return + @@ fun () -> + let keypair = Keypair.create () in + printf "%s\n" Signer.Keys.(of_keypair keypair |> to_private_key_bytes) ; + return () + +let () = + Command.run + (Command.group + ~summary:"OCaml reference signer implementation for Rosetta." + [ ( "sign" + , Command.async ~summary:"Sign an unsigned transaction" sign_command + ) + ; ( "verify" + , Command.async + ~summary: + "Verify the signature of a signed transaction. Exits 0 if the \ + signature verifies." + verify_command ) + ; ( "derive-public-key" + , Command.async ~summary:"Import a private key, returns a public-key" + derive_command ) + ; ( "generate-private-key" + , Command.async ~summary:"Generate a new private key" generate_command + ) ]) diff --git a/src/app/rosetta/start.sh b/src/app/rosetta/start.sh index bfefff5b04e..db113dbde27 100755 --- a/src/app/rosetta/start.sh +++ b/src/app/rosetta/start.sh @@ -23,7 +23,7 @@ PG_CONN=postgres://$USER:$USER@localhost:5432/archiver # rebuild pushd ../../../ -PATH=/usr/local/bin:$PATH dune b src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe src/app/cli/src/coda.exe src/app/archive/archive.exe src/app/rosetta/rosetta.exe src/app/rosetta/test-agent/agent.exe +PATH=/usr/local/bin:$PATH dune b src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe src/app/cli/src/coda.exe src/app/archive/archive.exe src/app/rosetta/rosetta.exe src/app/rosetta/test-agent/agent.exe src/app/rosetta/ocaml-signer/signer.exe popd # make genesis (synchronously) diff --git a/src/app/rosetta/test-agent/dune b/src/app/rosetta/test-agent/dune index 26846c1d5ad..8d4d3fd28ef 100644 --- a/src/app/rosetta/test-agent/dune +++ b/src/app/rosetta/test-agent/dune @@ -18,7 +18,6 @@ lib ppx_deriving_yojson.runtime yojson - secrets ) (preprocess (pps graphql_ppx diff --git a/src/app/rosetta/test-agent/signer.ml b/src/app/rosetta/test-agent/signer.ml deleted file mode 100644 index 81f0b260327..00000000000 --- a/src/app/rosetta/test-agent/signer.ml +++ /dev/null @@ -1,87 +0,0 @@ -(* A to-spec signer library that uses internal coda libs *) - -open Core_kernel -open Signature_lib -open Rosetta_lib -open Rosetta_coding -module Signature = Coda_base.Signature -module User_command = Coda_base.User_command - -module Keys = struct - type t = {keypair: Keypair.t; public_key_hex_bytes: string} - - let of_keypair keypair = - { keypair - ; public_key_hex_bytes= - Rosetta_coding.Coding.of_public_key keypair.public_key } - - let of_private_key_box secret_box_string = - let json = Yojson.Safe.from_string secret_box_string in - let which = Secrets.Keypair.T.which in - let sb : Secrets.Secret_box.t = - Secrets.Secret_box.of_yojson json |> Result.ok |> Option.value_exn - in - let output : Bytes.t = - Secrets.Secret_box.decrypt ~password:(Bytes.of_string "") ~which sb - |> Result.ok |> Option.value_exn - in - let sk = output |> Bigstring.of_bytes |> Private_key.of_bigstring_exn in - of_keypair (Keypair.of_private_key_exn sk) -end - -(* Returns signed_transaction_string *) -let sign ~(keys : Keys.t) ~unsigned_transaction_string = - let open Result.Let_syntax in - let%bind json = - try return (Yojson.Safe.from_string unsigned_transaction_string) - with _ -> Result.fail (Errors.create (`Json_parse None)) - in - let%map unsigned_transaction = - Transaction.Unsigned.Rendered.of_yojson json - |> Result.map_error ~f:(fun e -> Errors.create (`Json_parse (Some e))) - |> Result.bind ~f:Transaction.Unsigned.of_rendered - in - let user_command_payload = - User_command_info.Partial.to_user_command_payload - ~nonce:unsigned_transaction.nonce - unsigned_transaction.Transaction.Unsigned.command - |> Result.ok |> Option.value_exn - in - let signature = - Schnorr.sign keys.keypair.private_key - unsigned_transaction.random_oracle_input - in - let signature' = - User_command.sign_payload keys.keypair.private_key user_command_payload - in - [%test_eq: Signature.t] signature signature' ; - signature |> Signature.Raw.encode - -let verify ~public_key_hex_bytes ~signed_transaction_string = - let open Result.Let_syntax in - let%bind json = - try return (Yojson.Safe.from_string signed_transaction_string) - with _ -> Result.fail (Errors.create (`Json_parse None)) - in - let%bind signed_transaction = - Transaction.Signed.Rendered.of_yojson json - |> Result.map_error ~f:(fun e -> Errors.create (`Json_parse (Some e))) - |> Result.bind ~f:Transaction.Signed.of_rendered - in - let public_key : Public_key.t = Coding.to_public_key public_key_hex_bytes in - let%map signature = - Signature.Raw.decode signed_transaction.signature - |> Result.of_option - ~error: - (Errors.create ~context:"Signature malformed" (`Json_parse None)) - in - let user_command_payload = - User_command_info.Partial.to_user_command_payload - ~nonce:signed_transaction.nonce - signed_transaction.Transaction.Signed.command - |> Result.ok |> Option.value_exn - in - let message = User_command.to_input user_command_payload in - Schnorr.verify signature - (Snark_params.Tick.Inner_curve.of_affine public_key) - message From ddf56837852b30869dcd467e5aacc3818c0773c4 Mon Sep 17 00:00:00 2001 From: Brandon Kase Date: Wed, 2 Sep 2020 22:25:50 +0300 Subject: [PATCH 3/9] Forgot to add signer.ml --- src/app/rosetta/lib/signer.ml | 91 +++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/app/rosetta/lib/signer.ml diff --git a/src/app/rosetta/lib/signer.ml b/src/app/rosetta/lib/signer.ml new file mode 100644 index 00000000000..8533e7c1d41 --- /dev/null +++ b/src/app/rosetta/lib/signer.ml @@ -0,0 +1,91 @@ +(* A to-spec signer library that uses internal coda libs *) + +open Core_kernel +open Signature_lib +open Rosetta_lib +open Rosetta_coding +module Signature = Coda_base.Signature +module User_command = Coda_base.User_command + +module Keys = struct + type t = {keypair: Keypair.t; public_key_hex_bytes: string} + + let of_keypair keypair = + {keypair; public_key_hex_bytes= Coding.of_public_key keypair.public_key} + + let to_private_key_bytes t = Coding.of_scalar t.keypair.private_key + + let of_private_key_bytes s = + Coding.to_scalar s |> Keypair.of_private_key_exn |> of_keypair + + let of_private_key_box secret_box_string = + let json = Yojson.Safe.from_string secret_box_string in + let which = Secrets.Keypair.T.which in + let sb : Secrets.Secret_box.t = + Secrets.Secret_box.of_yojson json |> Result.ok |> Option.value_exn + in + let output : Bytes.t = + Secrets.Secret_box.decrypt ~password:(Bytes.of_string "") ~which sb + |> Result.ok |> Option.value_exn + in + let sk = output |> Bigstring.of_bytes |> Private_key.of_bigstring_exn in + (*printf !"Secret key hex bytes is: %s\n" (Coding.of_scalar sk) ;*) + of_keypair (Keypair.of_private_key_exn sk) +end + +(* Returns signed_transaction_string *) +let sign ~(keys : Keys.t) ~unsigned_transaction_string = + let open Result.Let_syntax in + let%bind json = + try return (Yojson.Safe.from_string unsigned_transaction_string) + with _ -> Result.fail (Errors.create (`Json_parse None)) + in + let%map unsigned_transaction = + Transaction.Unsigned.Rendered.of_yojson json + |> Result.map_error ~f:(fun e -> Errors.create (`Json_parse (Some e))) + |> Result.bind ~f:Transaction.Unsigned.of_rendered + in + let user_command_payload = + User_command_info.Partial.to_user_command_payload + ~nonce:unsigned_transaction.Transaction.Unsigned.nonce + unsigned_transaction.command + |> Result.ok |> Option.value_exn + in + let signature = + Schnorr.sign keys.keypair.private_key + unsigned_transaction.random_oracle_input + in + let signature' = + User_command.sign_payload keys.keypair.private_key user_command_payload + in + [%test_eq: Signature.t] signature signature' ; + signature |> Signature.Raw.encode + +let verify ~public_key_hex_bytes ~signed_transaction_string = + let open Result.Let_syntax in + let%bind json = + try return (Yojson.Safe.from_string signed_transaction_string) + with _ -> Result.fail (Errors.create (`Json_parse None)) + in + let%bind signed_transaction = + Transaction.Signed.Rendered.of_yojson json + |> Result.map_error ~f:(fun e -> Errors.create (`Json_parse (Some e))) + |> Result.bind ~f:Transaction.Signed.of_rendered + in + let public_key : Public_key.t = Coding.to_public_key public_key_hex_bytes in + let%map signature = + Signature.Raw.decode signed_transaction.signature + |> Result.of_option + ~error: + (Errors.create ~context:"Signature malformed" (`Json_parse None)) + in + let user_command_payload = + User_command_info.Partial.to_user_command_payload + ~nonce:signed_transaction.nonce + signed_transaction.Transaction.Signed.command + |> Result.ok |> Option.value_exn + in + let message = User_command.to_input user_command_payload in + Schnorr.verify signature + (Snark_params.Tick.Inner_curve.of_affine public_key) + message From a090577e5b59fe6bc7e209b3a3f88db2f24c0581 Mon Sep 17 00:00:00 2001 From: Brandon Kase Date: Wed, 2 Sep 2020 22:53:47 +0300 Subject: [PATCH 4/9] Adds signer.opam --- src/app/rosetta/signer.opam | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/app/rosetta/signer.opam diff --git a/src/app/rosetta/signer.opam b/src/app/rosetta/signer.opam new file mode 100644 index 00000000000..a1b56499734 --- /dev/null +++ b/src/app/rosetta/signer.opam @@ -0,0 +1,5 @@ +opam-version: "1.2" +version: "0.1" +build: [ + ["dune" "build" "--only" "src" "--root" "." "-j" jobs "@install"] +] From 2d8714e0fdde0b4c99a2bb89b21b2708db23462e Mon Sep 17 00:00:00 2001 From: Brandon Kase Date: Sat, 5 Sep 2020 00:40:47 +0300 Subject: [PATCH 5/9] Also builds rosetta with archive node --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3edfe448b02..445431d0b23 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ build: git_hooks reformat-diff libp2p_helper build_archive: git_hooks reformat-diff $(info Starting Build) - ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe --profile=$(DUNE_PROFILE) + ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/rosetta/rosetta.exe src/app/rosetta/ocaml-signer/signer.exe --profile=$(DUNE_PROFILE) $(info Build complete) build_rosetta: From b893ed94c58100a3ac9cba5a171636ef9bee4401 Mon Sep 17 00:00:00 2001 From: Brandon Kase Date: Sat, 5 Sep 2020 00:51:19 +0300 Subject: [PATCH 6/9] Changes signature scheme to match rosetta-spec name --- src/app/rosetta/lib/construction.ml | 2 +- src/app/rosetta/test-agent/offline.ml | 2 +- src/lib/rosetta_models/enums.ml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/rosetta/lib/construction.ml b/src/app/rosetta/lib/construction.ml index bce367b45f5..326e83b10bc 100644 --- a/src/app/rosetta/lib/construction.ml +++ b/src/app/rosetta/lib/construction.ml @@ -374,7 +374,7 @@ module Payloads = struct in pk) ; hex_bytes= pk - ; signature_type= Some "schnorr" } ] } + ; signature_type= Some "schnorr_poseidon" } ] } end module Real = Impl (Deferred.Result) diff --git a/src/app/rosetta/test-agent/offline.ml b/src/app/rosetta/test-agent/offline.ml index 27f565bc9bd..0e9b7734c8e 100644 --- a/src/app/rosetta/test-agent/offline.ml +++ b/src/app/rosetta/test-agent/offline.ml @@ -100,7 +100,7 @@ module Combine = struct ; public_key= { Public_key.hex_bytes= public_key_hex_bytes ; curve_type= "tweedle" } - ; signature_type= "schnorr" + ; signature_type= "schnorr_poseidon" ; hex_bytes= signature } ] } |> to_yojson) ~path:"construction/combine" diff --git a/src/lib/rosetta_models/enums.ml b/src/lib/rosetta_models/enums.ml index 5c3bb9b0ff6..72cc4a21379 100644 --- a/src/lib/rosetta_models/enums.ml +++ b/src/lib/rosetta_models/enums.ml @@ -1,5 +1,5 @@ type curvetype = (* tweedle *) string [@@deriving yojson, show] -type signaturetype = (* schnorr *) string [@@deriving yojson, show] +type signaturetype = (* schnorr_poseidon *) string [@@deriving yojson, show] type coinaction = () [@@deriving yojson, show, eq] From 73fe9523dff2a45926a2b099421fd7cf2712e948 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Fri, 4 Sep 2020 18:11:57 -0400 Subject: [PATCH 7/9] rosetta/mini-signer-exe Address nits --- Makefile | 4 ++-- dockerfiles/Dockerfile-rosetta | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 445431d0b23..ba98e39c05c 100644 --- a/Makefile +++ b/Makefile @@ -83,12 +83,12 @@ build: git_hooks reformat-diff libp2p_helper build_archive: git_hooks reformat-diff $(info Starting Build) - ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/rosetta/rosetta.exe src/app/rosetta/ocaml-signer/signer.exe --profile=$(DUNE_PROFILE) + ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe --profile=$(DUNE_PROFILE) $(info Build complete) build_rosetta: $(info Starting Build) - ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/rosetta/rosetta.exe --profile=$(DUNE_PROFILE) + ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/rosetta/rosetta.exe src/app/rosetta/ocaml-signer/signer.exe --profile=$(DUNE_PROFILE) $(info Build complete) client_sdk : diff --git a/dockerfiles/Dockerfile-rosetta b/dockerfiles/Dockerfile-rosetta index 2ce0348a510..c5a89d932c8 100644 --- a/dockerfiles/Dockerfile-rosetta +++ b/dockerfiles/Dockerfile-rosetta @@ -239,6 +239,7 @@ RUN eval $(opam config env) \ src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ src/app/generate_keypair/generate_keypair.exe \ src/app/rosetta/test-agent/agent.exe \ + src/app/rosetta/ocaml-signer/signer.exe \ && _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ --config-file src/app/rosetta/demo-config.json \ --genesis-dir ${HOME}/demo-genesis \ From 2d8db72e48ac2722bf761e39198bebe77ab0b3fa Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Fri, 4 Sep 2020 20:44:15 -0400 Subject: [PATCH 8/9] rosetta/mini-signer-exe Fix mkdir error --- scripts/release-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 4ba2b4b7983..ea346e58f89 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -56,7 +56,7 @@ coda-demo) ;; coda-rosetta) DOCKERFILE_PATH="dockerfiles/Dockerfile-rosetta" - mkdir some_empty_dir + mkdir -p some_empty_dir DOCKER_CONTEXT="./some_empty_dir" ;; leaderboard) From 2b09b5125e6dee2989d4b0a0351961f98c3e6575 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Fri, 4 Sep 2020 21:14:56 -0400 Subject: [PATCH 9/9] rosetta/mini-signer-exe Pull docker image to use as cache --- buildkite/scripts/docker-artifact.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index e143033e96d..8d1aa2f2711 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -10,6 +10,7 @@ scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION} if [[ -n $CODA_BUILD_ROSETTA ]]; then echo "--- Build/Release coda-rosetta to docker hub" + docker pull gcr.io/o1labs-192920/coda-rosetta-build-deps:develop # Could also cache-from opam-deps but we would need to get that automatically building nightly or at least when src/opam.export changes # build-deps is updated by manually running scripts/build-rosetta-stages.sh which always builds + pushes each stage scripts/release-docker.sh --service "coda-rosetta" --version "dev-${CODA_VERSION}"\