Merge pull request #6196 from MinaProtocol/rosetta/add-docker-build-i…

…nstructions

Rosetta/add docker build instructions

buildkite/scripts/docker-artifact.sh

@@ -17,9 +17,9 @@ if [[ -n $CODA_BUILD_ROSETTA ]]; then
   # Could also cache-from opam-deps but we would need to get that automatically building nightly or at least when src/opam.export changes
   # build-deps is updated by manually running scripts/build-rosetta-stages.sh which always builds + pushes each stage
   scripts/release-docker.sh --service "coda-rosetta" --version "dev-${CODA_VERSION}"\
-    --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:develop"
+    --extra-args "--build-arg DUNE_PROFILE=dev --build-arg MINA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:develop"
   # Also build with the standard DUNE_PROFILE, and use the dev profile as a cache.
   # This means it will use the opam-deps stage from the previous step, but make a new builder stage because the DUNE_PROFILE arg changed
   scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}"\
-    --extra-args "--build-arg CODA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:develop"
+    --extra-args "--build-arg MINA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:develop"
 fi

dockerfiles/Dockerfile-rosetta

@@ -139,24 +139,24 @@ RUN git clone \
 #################################################################################################
 # The "opam-deps" Stage
 # - Continues from the build-deps image
-# - Installs all opam dependencies and pins from coda's github
-# - Includes the entire coda codebase and submodules in "${CODA_DIR}" (must be writable by opam user)
+# - Installs all opam dependencies and pins from mina's github
+# - Includes the entire mina codebase and submodules in "${MINA_DIR}" (must be writable by opam user)
 # - Largely mirrors/replaces ./scripts/setup-opam.sh
 #################################################################################################
 FROM build-deps AS opam-deps
 
 # location of repo used for pins and external package commits
-ARG CODA_DIR=coda
+ARG MINA_DIR=mina
 # branch to checkout on first clone (this will be the only availible branch in the container)
 # can also be a tagged release
 # TODO: change this to two distinct variables, one for opam/dependency clone
-#       and a distinct one for the coda codebase in the next stage
+#       and a distinct one for the mina codebase in the next stage
 ARG OPAM_BRANCH=develop
 
 # location of external packages
-ARG EXTERNAL_PKG_DIR=$CODA_DIR/src/external
+ARG EXTERNAL_PKG_DIR=$MINA_DIR/src/external
 
-# don't keep sources, to force reinstall of pinned packages from Coda sources
+# don't keep sources, to force reinstall of pinned packages from Mina sources
 # and to keep Docker image reasonable size
 ENV OPAMKEEPBUILDDIR false
 ENV OPAMREUSEBUILDDIR false
@@ -167,9 +167,9 @@ RUN git clone \
   --depth 1 \
   --shallow-submodules \
   --recurse-submodules \
-  https://github.com/MinaProtocol/mina ${HOME}/${CODA_DIR}
+  https://github.com/MinaProtocol/mina ${HOME}/${MINA_DIR}
 
-WORKDIR $HOME/$CODA_DIR
+WORKDIR $HOME/$MINA_DIR
 
 ENV OPAMYES 1
 # TODO: handle this opam work without cloning the full repository (directly pull src/opam.export)
@@ -207,7 +207,7 @@ RUN eval $(opam config env) \
 
 #################################################################################################
 # The "builder" Stage
-# - builds coda and any other binaries required to run a node
+# - builds mina and any other binaries required to run a node
 # - should not include any data related to joining a specific network, only the node software itself
 #################################################################################################
 FROM opam-deps AS builder
@@ -218,24 +218,24 @@ ARG DUNE_PROFILE=testnet_postake_medium_curves
 # branch to checkout on first clone (this will be the only availible branch in the container)
 # can also be a tagged release
 # TODO: change this to two distinct variables, one for opam/dependency clone
-#       and a distinct one for the coda codebase in the next stage
-ARG CODA_BRANCH=develop
+#       and a distinct one for the mina codebase in the next stage
+ARG MINA_BRANCH=develop
 
 # location of repo used for pins and external package commits
-ARG CODA_DIR=coda
+ARG MINA_DIR=mina
 
 ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin"
 
 # git will clone into an empty dir, but this also helps us set the workdir in advance
-RUN cd $HOME && rm -rf $HOME/${CODA_DIR} \
+RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \
   && git clone \
-  -b "${CODA_BRANCH}" \
+  -b "${MINA_BRANCH}" \
   --depth 1 \
   --shallow-submodules \
   --recurse-submodules \
-  https://github.com/MinaProtocol/mina ${HOME}/${CODA_DIR}
+  https://github.com/MinaProtocol/mina ${HOME}/${MINA_DIR}
 
-WORKDIR $HOME/${CODA_DIR}
+WORKDIR $HOME/${MINA_DIR}
 
 # Build libp2p_helper and clear go module caches
 # Could maybe also delete go during this step / build it as part of the deps stage?
@@ -265,7 +265,7 @@ RUN eval $(opam config env) \
   && mv _build/default/src/app $HOME/app \
   && rm -rf _build
 
-# TODO: consider making a coda-rosetta package with the minimum coda binaries/configs for rosetta.
+# TODO: consider making a mina-rosetta package with the minimum mina binaries/configs for rosetta.
 # Right now make deb is overkill
 #RUN eval $(opam config env) && make deb
 
@@ -276,8 +276,8 @@ RUN eval $(opam config env) \
 FROM debian:buster-slim AS production
 
 ARG POSTGRES_DATA_DIR=/data/postgresql
-ARG CODA_DAEMON_PORT=10101
-ARG CODA_CONFIG_DIR=/root/.coda-config
+ARG MINA_DAEMON_PORT=10101
+ARG MINA_CONFIG_DIR=/root/.mina-config
 # Sample public key for use in dev profile / demo mode genesis block
 ARG PK=B62qrPN5Y5yq8kGE3FbVKbGTdTAJNdtNtB5sNVpxyRwWGcDEhpMzc8g
 
@@ -303,30 +303,30 @@ RUN apt-get -y update \
 
 # Make all directories for configuration and empty s3 cache dirs
 RUN mkdir -p /tmp/s3_cache_dir \
-    &&  mkdir -p /tmp/coda_cache_dir \
+    && mkdir -p /tmp/mina_cache_dir \
     && mkdir -p ${POSTGRES_DATA_DIR} \
     && chown postgres ${POSTGRES_DATA_DIR}
 
-COPY --from=builder /home/opam/app/* /coda-bin/
-COPY --from=builder /home/opam/coda/src/app /app
-COPY --from=builder /home/opam/coda/src/app/rosetta/config.json /data/config.json
+COPY --from=builder /home/opam/app/* /mina-bin/
+COPY --from=builder /home/opam/mina/src/app /app
+COPY --from=builder /home/opam/mina/src/app/rosetta/config.json /data/config.json
 
 # all s3-cached files will be downloaded as needed at runtime
 
 # Uncomment these to optimize start up time at the expense of image size
 # COPY --from=builder /tmp/s3_cache_dir /tmp/s3_cache_dir
-# COPY --from=builder /tmp/coda_cache_dir /tmp/coda_cache_dir
+# COPY --from=builder /tmp/mina_cache_dir /tmp/mina_cache_dir
 
 
-# Set up coda config dir with demo mode keys and genesis
-COPY --from=builder /home/opam/demo-genesis ${CODA_CONFIG_DIR}/demo-genesis
-COPY --from=builder /home/opam/coda/src/app/rosetta/demo-config.json ${CODA_CONFIG_DIR}/daemon.json
+# Set up mina config dir with demo mode keys and genesis
+COPY --from=builder /home/opam/demo-genesis ${MINA_CONFIG_DIR}/demo-genesis
+COPY --from=builder /home/opam/mina/src/app/rosetta/demo-config.json ${MINA_CONFIG_DIR}/daemon.json
 
-RUN mkdir -p --mode=700 ${CODA_CONFIG_DIR}/wallets/store/ \
-  && echo "$PK" >  ${CODA_CONFIG_DIR}/wallets/store/$PK.pub \
+RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \
+  && echo "$PK" >  ${MINA_CONFIG_DIR}/wallets/store/$PK.pub \
   && echo '{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"8jGuTAxw3zxtWasVqcD1H6rEojHLS1yJmG3aHHd","pwsalt":"AiUCrMJ6243h3TBmZ2rqt3Voim1Y","pwdiff":[134217728,6],"ciphertext":"DbAy736GqEKWe9NQWT4yaejiZUo9dJ6rsK7cpS43APuEf5AH1Qw6xb1s35z8D2akyLJBrUr6m"}' \
-  > ${CODA_CONFIG_DIR}/wallets/store/${PK} \
-  && chmod go-rwx ${CODA_CONFIG_DIR}/wallets/store/${PK}
+  > ${MINA_CONFIG_DIR}/wallets/store/${PK} \
+  && chmod go-rwx ${MINA_CONFIG_DIR}/wallets/store/${PK}
 
 USER postgres
 
@@ -348,6 +348,6 @@ USER root
 WORKDIR /app/rosetta
 
 EXPOSE 3087
-EXPOSE $CODA_DAEMON_PORT
+EXPOSE $MINA_DAEMON_PORT
 
 ENTRYPOINT ["bash", "./docker-start.sh"]

scripts/build-and-test-rosetta-from-git.sh

@@ -1,15 +1,20 @@
+#!/bin/bash
+
 GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD)
 TAG=$(echo ${GITBRANCH} | sed 's!/!-!; s!_!-!g')
 
-docker pull gcr.io/o1labs-192920/coda-rosetta-opam-deps:${TAG}
+docker pull gcr.io/o1labs-192920/mina-rosetta-opam-deps:${TAG}
 
 cat dockerfiles/Dockerfile-rosetta | docker build \
   --target production \
-  -t gcr.io/o1labs-192920/coda-rosetta:${TAG} \
-  --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:${TAG} \
+  --force-rm \
+  --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:${TAG} \
+  -t gcr.io/o1labs-192920/mina-rosetta:${TAG} \
   --build-arg "DUNE_PROFILE=dev" \
-  --build-arg "CODA_BRANCH=${GITBRANCH}" -
+  --build-arg "OPAM_BRANCH=${GITBRANCH}" \
+  --build-arg "MINA_BRANCH=${GITBRANCH}" -
+
 
-docker run -it --entrypoint=./docker-test-start.sh gcr.io/o1labs-192920/coda-rosetta:${TAG}
+docker run -it --entrypoint=./docker-test-start.sh gcr.io/o1labs-192920/mina-rosetta:${TAG}
 
-[[ $? -eq 0 ]] && docker push gcr.io/o1labs-192920/coda-rosetta:${TAG} || echo "Tests failed, not pushing"
+[[ $? -eq 0 ]] && docker push gcr.io/o1labs-192920/mina-rosetta:${TAG} || echo "Tests failed, not pushing"

src/app/rosetta/README.md

@@ -10,6 +10,15 @@ Implementation of the [Rosetta API](https://www.rosetta-api.org/) for Coda.
 - Handles case where there are multiple blocks at the same height
 - "Failed transactions" decode into operations and reconcile properly
 
+## How to build your own docker image
+
+Checkout the "rosetta-stable" branch of the mina repository and then run the following:
+
+`cat dockerfiles/Dockerfile-rosetta | docker build -t mina-rosetta:stable --build-arg "MINA_BRANCH=rosetta-stable" -`
+
+This creates an image (mina-rosetta:stable) based on the most up-to-date changes that support rosetta. This image
+can be used as a drop-in replacement for `gcr.io/o1labs-192920/coda-rosetta:debug-v1.1` in any of the below commands for testing.
+
 ## How to Run
 
 As there is not currently a live network, the best way to run Rosetta is to run it against a sandbox node. Rosetta is best run using the official docker images provided here that run the Coda daemon, an archive node, and the rosetta process for you. See [Reproduce agent and rosetta-cli Validation](#reproduce-agent-and-rosetta-cli-validation) below for details.

src/app/rosetta/docker-demo-start.sh

@@ -6,11 +6,11 @@ function cleanup
 {
   CODE=${1:-0}
   echo "Killing archive.exe"
-  kill $(ps aux | egrep '_build/default/src/app/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true
+  kill $(ps aux | egrep '/mina-bin/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true
   echo "Killing coda.exe"
-  kill $(ps aux | egrep '_build/default/src/app/.*coda.exe'    | grep -v grep | awk '{ print $2 }') || true
+  kill $(ps aux | egrep '/mina-bin/.*coda.exe'    | grep -v grep | awk '{ print $2 }') || true
   echo "Killing rosetta.exe"
-  kill $(ps aux | egrep '_build/default/src/app/rosetta'       | grep -v grep | awk '{ print $2 }') || true
+  kill $(ps aux | egrep '/mina-bin/rosetta'       | grep -v grep | awk '{ print $2 }') || true
   exit $CODE
 }
 
@@ -28,7 +28,7 @@ sleep 3
 
 # archive
 echo "========================= STARTING ARCHIVE PROCESS ==========================="
-/coda-bin/archive/archive.exe run \
+/mina-bin/archive/archive.exe run \
   -postgres-uri $PG_CONN \
   -server-port 3086 \
   -log-level fatal \
@@ -42,23 +42,25 @@ PK=${PK:-B62qrPN5Y5yq8kGE3FbVKbGTdTAJNdtNtB5sNVpxyRwWGcDEhpMzc8g}
 SNARK_PK=${SNARK_PK:-B62qiWSQiF5Q9CsAHgjMHoEEyR2kJnnCvN9fxRps2NXULU15EeXbzPf}
 genesis_time=$(date -d '2019-01-30 20:00:00.000000Z' '+%s')
 now_time=$(date +%s)
+
 export CODA_TIME_OFFSET=$(( $now_time - $genesis_time ))
 export CODA_PRIVKEY_PASS=""
-export CODA_LIBP2P_HELPER_PATH=/coda-bin/libp2p_helper
-CODA_CONFIG_DIR=/root/.coda-config
+export CODA_LIBP2P_HELPER_PATH=/mina-bin/libp2p_helper
+
+MINA_CONFIG_DIR=/root/.coda-config
 
-# CODA_CONFIG_DIR is exposed by the dockerfile and contains demo mode essentials
+# MINA_CONFIG_DIR is exposed by the dockerfile and contains demo mode essentials
 echo "========================= STARTING DAEMON ==========================="
-/coda-bin/cli/src/coda.exe daemon \
+/mina-bin/cli/src/coda.exe daemon \
   -archive-address 3086 \
   -background \
-  -block-producer-key "$CODA_CONFIG_DIR/wallets/store/$PK" \
-  -config-dir "$CODA_CONFIG_DIR" \
-  -config-file "$CODA_CONFIG_DIR/daemon.json" \
+  -block-producer-key "$MINA_CONFIG_DIR/wallets/store/$PK" \
+  -config-dir "$MINA_CONFIG_DIR" \
+  -config-file "$MINA_CONFIG_DIR/daemon.json" \
   -demo-mode \
   -disable-telemetry \
   -external-ip 127.0.0.1 \
-  -external-port "${CODA_DAEMON_PORT:-10101}" \
+  -external-port "${MINA_DAEMON_PORT:-10101}" \
   -insecure-rest-server \
   -log-level debug \
   -log-json \
@@ -78,7 +80,7 @@ sleep 3
 
 # rosetta
 echo "========================= STARTING ROSETTA API on PORT 3087 ==========================="
-/coda-bin/rosetta/rosetta.exe \
+/mina-bin/rosetta/rosetta.exe \
   -archive-uri $PG_CONN \
   -graphql-uri http://localhost:3085/graphql \
   -log-level debug \

src/app/rosetta/docker-start.sh

@@ -5,11 +5,11 @@ set -eou pipefail
 function cleanup
 {
   echo "Killing archive.exe"
-  kill $(ps aux | egrep '/coda-bin/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true
+  kill $(ps aux | egrep '/mina-bin/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true
   echo "Killing coda.exe"
-  kill $(ps aux | egrep '/coda-bin/.*coda.exe'    | grep -v grep | awk '{ print $2 }') || true
+  kill $(ps aux | egrep '/mina-bin/.*coda.exe'    | grep -v grep | awk '{ print $2 }') || true
   echo "Killing rosetta.exe"
-  kill $(ps aux | egrep '/coda-bin/rosetta'       | grep -v grep | awk '{ print $2 }') || true
+  kill $(ps aux | egrep '/mina-bin/rosetta'       | grep -v grep | awk '{ print $2 }') || true
   echo "Stopping postgres"
   pg_ctlcluster 11 main stop
   exit
@@ -27,34 +27,35 @@ pg_ctlcluster 11 main start
 sleep 3
 
 # archive
-/coda-bin/archive/archive.exe run \
+/mina-bin/archive/archive.exe run \
   -postgres-uri $PG_CONN \
   -server-port 3086 &
 
 # wait for it to settle
 sleep 3
 
 export CODA_PRIVKEY_PASS=""
-export CODA_CONFIG_FILE=${CODA_CONFIG_FILE:=/data/config.json}
+export CODA_LIBP2P_HELPER_PATH=/mina-bin/libp2p_helper
+
+export MINA_CONFIG_FILE=${MINA_CONFIG_FILE:=/data/config.json}
 export PEER_ID=${PEER_ID:=/ip4/34.74.175.158/tcp/10001/ipfs/12D3KooWAFFq2yEQFFzhU5dt64AWqawRuomG9hL8rSmm5vxhAsgr/}
-export CODA_PORT=${CODA_DAEMON_PORT:=10101}
-DEFAULT_FLAGS="-generate-genesis-proof true -peer ${PEER_ID} -archive-address 0.0.0.0:3086 -insecure-rest-server -log-level debug -external-port ${CODA_PORT}"
-export CODA_FLAGS=${CODA_FLAGS:=$DEFAULT_FLAGS}
-export CODA_LIBP2P_HELPER_PATH=/coda-bin/libp2p_helper
-PK=${CODA_PK:=ZsMSUuKL9zLAF7sMn951oakTFRCCDw9rDfJgqJ55VMtPXaPa5vPwntQRFJzsHyeh8R8}
+export MINA_PORT=${MINA_DAEMON_PORT:=10101}
+DEFAULT_FLAGS="-generate-genesis-proof true -peer ${PEER_ID} -archive-address 0.0.0.0:3086 -insecure-rest-server -log-level debug -external-port ${MINA_PORT}"
+export MINA_FLAGS=${MINA_FLAGS:=$DEFAULT_FLAGS}
+PK=${MINA_PK:=ZsMSUuKL9zLAF7sMn951oakTFRCCDw9rDfJgqJ55VMtPXaPa5vPwntQRFJzsHyeh8R8}
 
-echo "CODA Flags: $CODA_FLAGS -config-file ${CODA_CONFIG_FILE}"
+echo "MINA Flags: $MINA_FLAGS -config-file ${MINA_CONFIG_FILE}"
 
 # Daemon w/ mounted config file, initial file is phase 3 config.json
-/coda-bin/cli/src/coda.exe daemon \
-    -config-file ${CODA_CONFIG_FILE} \
-    ${CODA_FLAGS} $@ &
+/mina-bin/cli/src/coda.exe daemon \
+    -config-file ${MINA_CONFIG_FILE} \
+    ${MINA_FLAGS} $@ &
 
 # wait for it to settle
 sleep 3
 
 # rosetta
-/coda-bin/rosetta/rosetta.exe \
+/mina-bin/rosetta/rosetta.exe \
   -archive-uri $PG_CONN \
   -graphql-uri http://localhost:3085/graphql \
   -log-level debug \