Merge branch 'develop' into fix/copy-keys

buildkite/scripts/docker-artifact.sh

@@ -17,9 +17,9 @@ if [[ -n $CODA_BUILD_ROSETTA ]]; then
   # Could also cache-from opam-deps but we would need to get that automatically building nightly or at least when src/opam.export changes
   # build-deps is updated by manually running scripts/build-rosetta-stages.sh which always builds + pushes each stage
   scripts/release-docker.sh --service "coda-rosetta" --version "dev-${CODA_VERSION}"\
-    --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:develop"
+    --extra-args "--build-arg DUNE_PROFILE=dev --build-arg MINA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:develop"
   # Also build with the standard DUNE_PROFILE, and use the dev profile as a cache.
   # This means it will use the opam-deps stage from the previous step, but make a new builder stage because the DUNE_PROFILE arg changed
   scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}"\
-    --extra-args "--build-arg CODA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:develop"
+    --extra-args "--build-arg MINA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:develop"
 fi

dockerfiles/Dockerfile-rosetta

@@ -139,24 +139,24 @@ RUN git clone \
 #################################################################################################
 # The "opam-deps" Stage
 # - Continues from the build-deps image
-# - Installs all opam dependencies and pins from coda's github
-# - Includes the entire coda codebase and submodules in "${CODA_DIR}" (must be writable by opam user)
+# - Installs all opam dependencies and pins from mina's github
+# - Includes the entire mina codebase and submodules in "${MINA_DIR}" (must be writable by opam user)
 # - Largely mirrors/replaces ./scripts/setup-opam.sh
 #################################################################################################
 FROM build-deps AS opam-deps
 
 # location of repo used for pins and external package commits
-ARG CODA_DIR=coda
+ARG MINA_DIR=mina
 # branch to checkout on first clone (this will be the only availible branch in the container)
 # can also be a tagged release
 # TODO: change this to two distinct variables, one for opam/dependency clone
-#       and a distinct one for the coda codebase in the next stage
+#       and a distinct one for the mina codebase in the next stage
 ARG OPAM_BRANCH=develop
 
 # location of external packages
-ARG EXTERNAL_PKG_DIR=$CODA_DIR/src/external
+ARG EXTERNAL_PKG_DIR=$MINA_DIR/src/external
 
-# don't keep sources, to force reinstall of pinned packages from Coda sources
+# don't keep sources, to force reinstall of pinned packages from Mina sources
 # and to keep Docker image reasonable size
 ENV OPAMKEEPBUILDDIR false
 ENV OPAMREUSEBUILDDIR false
@@ -167,9 +167,9 @@ RUN git clone \
   --depth 1 \
   --shallow-submodules \
   --recurse-submodules \
-  https://github.com/MinaProtocol/mina ${HOME}/${CODA_DIR}
+  https://github.com/MinaProtocol/mina ${HOME}/${MINA_DIR}
 
-WORKDIR $HOME/$CODA_DIR
+WORKDIR $HOME/$MINA_DIR
 
 ENV OPAMYES 1
 # TODO: handle this opam work without cloning the full repository (directly pull src/opam.export)
@@ -207,7 +207,7 @@ RUN eval $(opam config env) \
 
 #################################################################################################
 # The "builder" Stage
-# - builds coda and any other binaries required to run a node
+# - builds mina and any other binaries required to run a node
 # - should not include any data related to joining a specific network, only the node software itself
 #################################################################################################
 FROM opam-deps AS builder
@@ -218,24 +218,24 @@ ARG DUNE_PROFILE=testnet_postake_medium_curves
 # branch to checkout on first clone (this will be the only availible branch in the container)
 # can also be a tagged release
 # TODO: change this to two distinct variables, one for opam/dependency clone
-#       and a distinct one for the coda codebase in the next stage
-ARG CODA_BRANCH=develop
+#       and a distinct one for the mina codebase in the next stage
+ARG MINA_BRANCH=develop
 
 # location of repo used for pins and external package commits
-ARG CODA_DIR=coda
+ARG MINA_DIR=mina
 
 ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin"
 
 # git will clone into an empty dir, but this also helps us set the workdir in advance
-RUN cd $HOME && rm -rf $HOME/${CODA_DIR} \
+RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \
   && git clone \
-  -b "${CODA_BRANCH}" \
+  -b "${MINA_BRANCH}" \
   --depth 1 \
   --shallow-submodules \
   --recurse-submodules \
-  https://github.com/MinaProtocol/mina ${HOME}/${CODA_DIR}
+  https://github.com/MinaProtocol/mina ${HOME}/${MINA_DIR}
 
-WORKDIR $HOME/${CODA_DIR}
+WORKDIR $HOME/${MINA_DIR}
 
 # Build libp2p_helper and clear go module caches
 # Could maybe also delete go during this step / build it as part of the deps stage?
@@ -265,7 +265,7 @@ RUN eval $(opam config env) \
   && mv _build/default/src/app $HOME/app \
   && rm -rf _build
 
-# TODO: consider making a coda-rosetta package with the minimum coda binaries/configs for rosetta.
+# TODO: consider making a mina-rosetta package with the minimum mina binaries/configs for rosetta.
 # Right now make deb is overkill
 #RUN eval $(opam config env) && make deb
 
@@ -276,8 +276,8 @@ RUN eval $(opam config env) \
 FROM debian:buster-slim AS production
 
 ARG POSTGRES_DATA_DIR=/data/postgresql
-ARG CODA_DAEMON_PORT=10101
-ARG CODA_CONFIG_DIR=/root/.coda-config
+ARG MINA_DAEMON_PORT=10101
+ARG MINA_CONFIG_DIR=/root/.mina-config
 # Sample public key for use in dev profile / demo mode genesis block
 ARG PK=B62qrPN5Y5yq8kGE3FbVKbGTdTAJNdtNtB5sNVpxyRwWGcDEhpMzc8g
 
@@ -303,30 +303,30 @@ RUN apt-get -y update \
 
 # Make all directories for configuration and empty s3 cache dirs
 RUN mkdir -p /tmp/s3_cache_dir \
-    &&  mkdir -p /tmp/coda_cache_dir \
+    && mkdir -p /tmp/mina_cache_dir \
     && mkdir -p ${POSTGRES_DATA_DIR} \
     && chown postgres ${POSTGRES_DATA_DIR}
 
-COPY --from=builder /home/opam/app/* /coda-bin/
-COPY --from=builder /home/opam/coda/src/app /app
-COPY --from=builder /home/opam/coda/src/app/rosetta/config.json /data/config.json
+COPY --from=builder /home/opam/app/* /mina-bin/
+COPY --from=builder /home/opam/mina/src/app /app
+COPY --from=builder /home/opam/mina/src/app/rosetta/config.json /data/config.json
 
 # all s3-cached files will be downloaded as needed at runtime
 
 # Uncomment these to optimize start up time at the expense of image size
 # COPY --from=builder /tmp/s3_cache_dir /tmp/s3_cache_dir
-# COPY --from=builder /tmp/coda_cache_dir /tmp/coda_cache_dir
+# COPY --from=builder /tmp/mina_cache_dir /tmp/mina_cache_dir
 
 
-# Set up coda config dir with demo mode keys and genesis
-COPY --from=builder /home/opam/demo-genesis ${CODA_CONFIG_DIR}/demo-genesis
-COPY --from=builder /home/opam/coda/src/app/rosetta/demo-config.json ${CODA_CONFIG_DIR}/daemon.json
+# Set up mina config dir with demo mode keys and genesis
+COPY --from=builder /home/opam/demo-genesis ${MINA_CONFIG_DIR}/demo-genesis
+COPY --from=builder /home/opam/mina/src/app/rosetta/demo-config.json ${MINA_CONFIG_DIR}/daemon.json
 
-RUN mkdir -p --mode=700 ${CODA_CONFIG_DIR}/wallets/store/ \
-  && echo "$PK" >  ${CODA_CONFIG_DIR}/wallets/store/$PK.pub \
+RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \
+  && echo "$PK" >  ${MINA_CONFIG_DIR}/wallets/store/$PK.pub \
   && echo '{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"8jGuTAxw3zxtWasVqcD1H6rEojHLS1yJmG3aHHd","pwsalt":"AiUCrMJ6243h3TBmZ2rqt3Voim1Y","pwdiff":[134217728,6],"ciphertext":"DbAy736GqEKWe9NQWT4yaejiZUo9dJ6rsK7cpS43APuEf5AH1Qw6xb1s35z8D2akyLJBrUr6m"}' \
-  > ${CODA_CONFIG_DIR}/wallets/store/${PK} \
-  && chmod go-rwx ${CODA_CONFIG_DIR}/wallets/store/${PK}
+  > ${MINA_CONFIG_DIR}/wallets/store/${PK} \
+  && chmod go-rwx ${MINA_CONFIG_DIR}/wallets/store/${PK}
 
 USER postgres
 
@@ -348,6 +348,6 @@ USER root
 WORKDIR /app/rosetta
 
 EXPOSE 3087
-EXPOSE $CODA_DAEMON_PORT
+EXPOSE $MINA_DAEMON_PORT
 
 ENTRYPOINT ["bash", "./docker-start.sh"]

scripts/build-and-test-rosetta-from-git.sh

@@ -1,15 +1,20 @@
+#!/bin/bash
+
 GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD)
 TAG=$(echo ${GITBRANCH} | sed 's!/!-!; s!_!-!g')
 
-docker pull gcr.io/o1labs-192920/coda-rosetta-opam-deps:${TAG}
+docker pull gcr.io/o1labs-192920/mina-rosetta-opam-deps:${TAG}
 
 cat dockerfiles/Dockerfile-rosetta | docker build \
   --target production \
-  -t gcr.io/o1labs-192920/coda-rosetta:${TAG} \
-  --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:${TAG} \
+  --force-rm \
+  --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:${TAG} \
+  -t gcr.io/o1labs-192920/mina-rosetta:${TAG} \
   --build-arg "DUNE_PROFILE=dev" \
-  --build-arg "CODA_BRANCH=${GITBRANCH}" -
+  --build-arg "OPAM_BRANCH=${GITBRANCH}" \
+  --build-arg "MINA_BRANCH=${GITBRANCH}" -
+
 
-docker run -it --entrypoint=./docker-test-start.sh gcr.io/o1labs-192920/coda-rosetta:${TAG}
+docker run -it --entrypoint=./docker-test-start.sh gcr.io/o1labs-192920/mina-rosetta:${TAG}
 
-[[ $? -eq 0 ]] && docker push gcr.io/o1labs-192920/coda-rosetta:${TAG} || echo "Tests failed, not pushing"
+[[ $? -eq 0 ]] && docker push gcr.io/o1labs-192920/mina-rosetta:${TAG} || echo "Tests failed, not pushing"

src/app/replayer/replayer.ml

@@ -29,6 +29,9 @@ let create_ledger accounts =
       Ledger.create_new_account_exn ledger acct_id acct ) ;
   ledger
 
+let json_ledger_hash_of_ledger ledger =
+  Ledger_hash.to_yojson @@ Ledger.merkle_root ledger
+
 let create_output target_state_hash target_proof ledger =
   let target_ledger = Ledger.to_list ledger in
   {target_state_hash; target_proof; target_ledger}
@@ -245,13 +248,14 @@ let run_user_command ~logger ~pool ~ledger (cmd : Sql.User_command.t) =
     cmd.nonce cmd.global_slot cmd.sequence_no ;
   let%bind body = body_of_sql_user_cmd pool cmd in
   let%map fee_payer_pk = pk_of_pk_id pool cmd.fee_payer_id in
+  let memo = Signed_command_memo.of_string cmd.memo in
   let payload =
     Signed_command_payload.create
       ~fee:(Currency.Fee.of_uint64 @@ Unsigned.UInt64.of_int64 cmd.fee)
       ~fee_token:(Token_id.of_uint64 @@ Unsigned.UInt64.of_int64 cmd.fee_token)
       ~fee_payer_pk
       ~nonce:(Unsigned.UInt32.of_int64 cmd.nonce)
-      ~valid_until:None ~memo:Signed_command_memo.dummy ~body
+      ~valid_until:None ~memo ~body
   in
   (* when applying the transaction, there's a check that the fee payer and
      signer keys are the same; since this transaction was accepted, we know
@@ -314,6 +318,20 @@ let main ~input_file ~output_file ~archive_uri () =
         State_hash.to_yojson input.target_state_hash
         |> unquoted_string_of_yojson
       in
+      [%log info] "Loading global slots" ;
+      let%bind global_slots =
+        match%bind
+          Caqti_async.Pool.use
+            (fun db -> Sql.Global_slots.run db state_hash)
+            pool
+        with
+        | Ok slots ->
+            return (Int64.Set.of_list slots)
+        | Error msg ->
+            [%log error] "Error getting global slots"
+              ~metadata:[("error", `String (Caqti_error.show msg))] ;
+            exit 1
+      in
       [%log info] "Loading user command ids" ;
       let%bind user_cmd_ids =
         match%bind
@@ -364,8 +382,13 @@ let main ~input_file ~output_file ~archive_uri () =
                   id (Caqti_error.show msg) () )
       in
       let unsorted_internal_cmds = List.concat unsorted_internal_cmds_list in
+      (* filter out internal commands in blocks not along chain from target state hash *)
+      let filtered_internal_cmds =
+        List.filter unsorted_internal_cmds ~f:(fun cmd ->
+            Int64.Set.mem global_slots cmd.global_slot )
+      in
       let sorted_internal_cmds =
-        List.sort unsorted_internal_cmds ~compare:(fun ic1 ic2 ->
+        List.sort filtered_internal_cmds ~compare:(fun ic1 ic2 ->
             let tuple (ic : Sql.Internal_command.t) =
               (ic.global_slot, ic.sequence_no, ic.secondary_sequence_no)
             in
@@ -392,33 +415,47 @@ let main ~input_file ~output_file ~archive_uri () =
                   (Caqti_error.show msg) () )
       in
       let unsorted_user_cmds = List.concat unsorted_user_cmds_list in
+      (* filter out user commands in blocks not along chain from target state hash *)
+      let filtered_user_cmds =
+        List.filter unsorted_user_cmds ~f:(fun cmd ->
+            Int64.Set.mem global_slots cmd.global_slot )
+      in
       let sorted_user_cmds =
-        List.sort unsorted_user_cmds ~compare:(fun uc1 uc2 ->
+        List.sort filtered_user_cmds ~compare:(fun uc1 uc2 ->
             let tuple (uc : Sql.User_command.t) =
               (uc.global_slot, uc.sequence_no)
             in
             [%compare: int64 * int] (tuple uc1) (tuple uc2) )
       in
       (* apply commands in global slot, sequence order *)
       let rec apply_commands (internal_cmds : Sql.Internal_command.t list)
-          (user_cmds : Sql.User_command.t list) =
+          (user_cmds : Sql.User_command.t list) ~last_global_slot =
+        let log_on_slot_change curr_global_slot =
+          if Int64.( > ) curr_global_slot last_global_slot then
+            [%log info] "Applied all commands at global slot %Ld, ledger hash"
+              ~metadata:[("ledger_hash", json_ledger_hash_of_ledger ledger)]
+              last_global_slot
+        in
         let combine_or_run_internal_cmds (ic : Sql.Internal_command.t)
             (ics : Sql.Internal_command.t list) =
           match ics with
           | ic2 :: ics2
-            when Int.equal ic.sequence_no ic2.sequence_no
+            when Int64.equal ic.global_slot ic2.global_slot
+                 && Int.equal ic.sequence_no ic2.sequence_no
                  && String.equal ic.type_ "fee_transfer"
                  && String.equal ic.type_ ic2.type_ ->
               (* combining situation 2
-             two fee transfer commands with same sequence number
-          *)
+                 two fee transfer commands with same global slot, sequence number
+              *)
+              log_on_slot_change ic.global_slot ;
               let%bind () =
                 apply_combined_fee_transfer ~logger ~pool ~ledger ic ic2
               in
-              apply_commands ics2 user_cmds
+              apply_commands ics2 user_cmds ~last_global_slot:ic.global_slot
           | _ ->
+              log_on_slot_change ic.global_slot ;
               let%bind () = run_internal_command ~logger ~pool ~ledger ic in
-              apply_commands ics user_cmds
+              apply_commands ics user_cmds ~last_global_slot:ic.global_slot
         in
         (* choose command with least global slot, sequence number
            TODO: check for gaps?
@@ -432,11 +469,13 @@ let main ~input_file ~output_file ~archive_uri () =
         | [], [] ->
             Deferred.unit
         | [], uc :: ucs ->
+            log_on_slot_change uc.global_slot ;
             let%bind () = run_user_command ~logger ~pool ~ledger uc in
-            apply_commands [] ucs
+            apply_commands [] ucs ~last_global_slot:uc.global_slot
         | ic :: _, uc :: ucs when cmp_ic_uc ic uc > 0 ->
+            log_on_slot_change uc.global_slot ;
             let%bind () = run_user_command ~logger ~pool ~ledger uc in
-            apply_commands internal_cmds ucs
+            apply_commands internal_cmds ucs ~last_global_slot:uc.global_slot
         | ic :: ics, [] ->
             combine_or_run_internal_cmds ic ics
         | ic :: ics, uc :: _ when cmp_ic_uc ic uc < 0 ->
@@ -447,7 +486,14 @@ let main ~input_file ~output_file ~archive_uri () =
                slot %Ld and sequence number %d"
               ic.global_slot ic.sequence_no ()
       in
-      let%bind () = apply_commands sorted_internal_cmds sorted_user_cmds in
+      [%log info] "At genesis, ledger hash"
+        ~metadata:[("ledger_hash", json_ledger_hash_of_ledger ledger)] ;
+      let%bind () =
+        apply_commands sorted_internal_cmds sorted_user_cmds
+          ~last_global_slot:0L
+      in
+      [%log info] "After applying all commands, ledger hash"
+        ~metadata:[("ledger_hash", json_ledger_hash_of_ledger ledger)] ;
       [%log info] "Writing output to $output_file"
         ~metadata:[("output_file", `String output_file)] ;
       let output =