diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index 884463e4504..f410b734beb 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -17,9 +17,9 @@ if [[ -n $CODA_BUILD_ROSETTA ]]; then # Could also cache-from opam-deps but we would need to get that automatically building nightly or at least when src/opam.export changes # build-deps is updated by manually running scripts/build-rosetta-stages.sh which always builds + pushes each stage scripts/release-docker.sh --service "coda-rosetta" --version "dev-${CODA_VERSION}"\ - --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:develop" + --extra-args "--build-arg DUNE_PROFILE=dev --build-arg MINA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:develop" # Also build with the standard DUNE_PROFILE, and use the dev profile as a cache. # This means it will use the opam-deps stage from the previous step, but make a new builder stage because the DUNE_PROFILE arg changed scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}"\ - --extra-args "--build-arg CODA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:develop" + --extra-args "--build-arg MINA_BRANCH=${CODA_GIT_BRANCH} --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:develop" fi diff --git a/dockerfiles/Dockerfile-rosetta b/dockerfiles/Dockerfile-rosetta index d9f5d8750dd..5a342218f10 100644 --- a/dockerfiles/Dockerfile-rosetta +++ b/dockerfiles/Dockerfile-rosetta @@ -139,24 +139,24 @@ RUN git clone \ ################################################################################################# # The "opam-deps" Stage # - Continues from the build-deps image -# - Installs all opam dependencies and pins from coda's github -# - Includes the entire coda codebase and submodules in "${CODA_DIR}" (must be writable by opam user) +# - Installs all opam dependencies and pins from mina's github +# - Includes the entire mina codebase and submodules in "${MINA_DIR}" (must be writable by opam user) # - Largely mirrors/replaces ./scripts/setup-opam.sh ################################################################################################# FROM build-deps AS opam-deps # location of repo used for pins and external package commits -ARG CODA_DIR=coda +ARG MINA_DIR=mina # branch to checkout on first clone (this will be the only availible branch in the container) # can also be a tagged release # TODO: change this to two distinct variables, one for opam/dependency clone -# and a distinct one for the coda codebase in the next stage +# and a distinct one for the mina codebase in the next stage ARG OPAM_BRANCH=develop # location of external packages -ARG EXTERNAL_PKG_DIR=$CODA_DIR/src/external +ARG EXTERNAL_PKG_DIR=$MINA_DIR/src/external -# don't keep sources, to force reinstall of pinned packages from Coda sources +# don't keep sources, to force reinstall of pinned packages from Mina sources # and to keep Docker image reasonable size ENV OPAMKEEPBUILDDIR false ENV OPAMREUSEBUILDDIR false @@ -167,9 +167,9 @@ RUN git clone \ --depth 1 \ --shallow-submodules \ --recurse-submodules \ - https://github.com/MinaProtocol/mina ${HOME}/${CODA_DIR} + https://github.com/MinaProtocol/mina ${HOME}/${MINA_DIR} -WORKDIR $HOME/$CODA_DIR +WORKDIR $HOME/$MINA_DIR ENV OPAMYES 1 # TODO: handle this opam work without cloning the full repository (directly pull src/opam.export) @@ -207,7 +207,7 @@ RUN eval $(opam config env) \ ################################################################################################# # The "builder" Stage -# - builds coda and any other binaries required to run a node +# - builds mina and any other binaries required to run a node # - should not include any data related to joining a specific network, only the node software itself ################################################################################################# FROM opam-deps AS builder @@ -218,24 +218,24 @@ ARG DUNE_PROFILE=testnet_postake_medium_curves # branch to checkout on first clone (this will be the only availible branch in the container) # can also be a tagged release # TODO: change this to two distinct variables, one for opam/dependency clone -# and a distinct one for the coda codebase in the next stage -ARG CODA_BRANCH=develop +# and a distinct one for the mina codebase in the next stage +ARG MINA_BRANCH=develop # location of repo used for pins and external package commits -ARG CODA_DIR=coda +ARG MINA_DIR=mina ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" # git will clone into an empty dir, but this also helps us set the workdir in advance -RUN cd $HOME && rm -rf $HOME/${CODA_DIR} \ +RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ && git clone \ - -b "${CODA_BRANCH}" \ + -b "${MINA_BRANCH}" \ --depth 1 \ --shallow-submodules \ --recurse-submodules \ - https://github.com/MinaProtocol/mina ${HOME}/${CODA_DIR} + https://github.com/MinaProtocol/mina ${HOME}/${MINA_DIR} -WORKDIR $HOME/${CODA_DIR} +WORKDIR $HOME/${MINA_DIR} # Build libp2p_helper and clear go module caches # Could maybe also delete go during this step / build it as part of the deps stage? @@ -265,7 +265,7 @@ RUN eval $(opam config env) \ && mv _build/default/src/app $HOME/app \ && rm -rf _build -# TODO: consider making a coda-rosetta package with the minimum coda binaries/configs for rosetta. +# TODO: consider making a mina-rosetta package with the minimum mina binaries/configs for rosetta. # Right now make deb is overkill #RUN eval $(opam config env) && make deb @@ -276,8 +276,8 @@ RUN eval $(opam config env) \ FROM debian:buster-slim AS production ARG POSTGRES_DATA_DIR=/data/postgresql -ARG CODA_DAEMON_PORT=10101 -ARG CODA_CONFIG_DIR=/root/.coda-config +ARG MINA_DAEMON_PORT=10101 +ARG MINA_CONFIG_DIR=/root/.mina-config # Sample public key for use in dev profile / demo mode genesis block ARG PK=B62qrPN5Y5yq8kGE3FbVKbGTdTAJNdtNtB5sNVpxyRwWGcDEhpMzc8g @@ -303,30 +303,30 @@ RUN apt-get -y update \ # Make all directories for configuration and empty s3 cache dirs RUN mkdir -p /tmp/s3_cache_dir \ - && mkdir -p /tmp/coda_cache_dir \ + && mkdir -p /tmp/mina_cache_dir \ && mkdir -p ${POSTGRES_DATA_DIR} \ && chown postgres ${POSTGRES_DATA_DIR} -COPY --from=builder /home/opam/app/* /coda-bin/ -COPY --from=builder /home/opam/coda/src/app /app -COPY --from=builder /home/opam/coda/src/app/rosetta/config.json /data/config.json +COPY --from=builder /home/opam/app/* /mina-bin/ +COPY --from=builder /home/opam/mina/src/app /app +COPY --from=builder /home/opam/mina/src/app/rosetta/config.json /data/config.json # all s3-cached files will be downloaded as needed at runtime # Uncomment these to optimize start up time at the expense of image size # COPY --from=builder /tmp/s3_cache_dir /tmp/s3_cache_dir -# COPY --from=builder /tmp/coda_cache_dir /tmp/coda_cache_dir +# COPY --from=builder /tmp/mina_cache_dir /tmp/mina_cache_dir -# Set up coda config dir with demo mode keys and genesis -COPY --from=builder /home/opam/demo-genesis ${CODA_CONFIG_DIR}/demo-genesis -COPY --from=builder /home/opam/coda/src/app/rosetta/demo-config.json ${CODA_CONFIG_DIR}/daemon.json +# Set up mina config dir with demo mode keys and genesis +COPY --from=builder /home/opam/demo-genesis ${MINA_CONFIG_DIR}/demo-genesis +COPY --from=builder /home/opam/mina/src/app/rosetta/demo-config.json ${MINA_CONFIG_DIR}/daemon.json -RUN mkdir -p --mode=700 ${CODA_CONFIG_DIR}/wallets/store/ \ - && echo "$PK" > ${CODA_CONFIG_DIR}/wallets/store/$PK.pub \ +RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ + && echo "$PK" > ${MINA_CONFIG_DIR}/wallets/store/$PK.pub \ && echo '{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"8jGuTAxw3zxtWasVqcD1H6rEojHLS1yJmG3aHHd","pwsalt":"AiUCrMJ6243h3TBmZ2rqt3Voim1Y","pwdiff":[134217728,6],"ciphertext":"DbAy736GqEKWe9NQWT4yaejiZUo9dJ6rsK7cpS43APuEf5AH1Qw6xb1s35z8D2akyLJBrUr6m"}' \ - > ${CODA_CONFIG_DIR}/wallets/store/${PK} \ - && chmod go-rwx ${CODA_CONFIG_DIR}/wallets/store/${PK} + > ${MINA_CONFIG_DIR}/wallets/store/${PK} \ + && chmod go-rwx ${MINA_CONFIG_DIR}/wallets/store/${PK} USER postgres @@ -348,6 +348,6 @@ USER root WORKDIR /app/rosetta EXPOSE 3087 -EXPOSE $CODA_DAEMON_PORT +EXPOSE $MINA_DAEMON_PORT ENTRYPOINT ["bash", "./docker-start.sh"] diff --git a/scripts/build-and-test-rosetta-from-git.sh b/scripts/build-and-test-rosetta-from-git.sh index fe709067e27..aec03dd1c21 100755 --- a/scripts/build-and-test-rosetta-from-git.sh +++ b/scripts/build-and-test-rosetta-from-git.sh @@ -1,15 +1,20 @@ +#!/bin/bash + GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD) TAG=$(echo ${GITBRANCH} | sed 's!/!-!; s!_!-!g') -docker pull gcr.io/o1labs-192920/coda-rosetta-opam-deps:${TAG} +docker pull gcr.io/o1labs-192920/mina-rosetta-opam-deps:${TAG} cat dockerfiles/Dockerfile-rosetta | docker build \ --target production \ - -t gcr.io/o1labs-192920/coda-rosetta:${TAG} \ - --cache-from gcr.io/o1labs-192920/coda-rosetta-opam-deps:${TAG} \ + --force-rm \ + --cache-from gcr.io/o1labs-192920/mina-rosetta-opam-deps:${TAG} \ + -t gcr.io/o1labs-192920/mina-rosetta:${TAG} \ --build-arg "DUNE_PROFILE=dev" \ - --build-arg "CODA_BRANCH=${GITBRANCH}" - + --build-arg "OPAM_BRANCH=${GITBRANCH}" \ + --build-arg "MINA_BRANCH=${GITBRANCH}" - + -docker run -it --entrypoint=./docker-test-start.sh gcr.io/o1labs-192920/coda-rosetta:${TAG} +docker run -it --entrypoint=./docker-test-start.sh gcr.io/o1labs-192920/mina-rosetta:${TAG} -[[ $? -eq 0 ]] && docker push gcr.io/o1labs-192920/coda-rosetta:${TAG} || echo "Tests failed, not pushing" +[[ $? -eq 0 ]] && docker push gcr.io/o1labs-192920/mina-rosetta:${TAG} || echo "Tests failed, not pushing" diff --git a/src/app/rosetta/README.md b/src/app/rosetta/README.md index 6a58dfd6d18..a27d1faa9a8 100644 --- a/src/app/rosetta/README.md +++ b/src/app/rosetta/README.md @@ -10,6 +10,15 @@ Implementation of the [Rosetta API](https://www.rosetta-api.org/) for Coda. - Handles case where there are multiple blocks at the same height - "Failed transactions" decode into operations and reconcile properly +## How to build your own docker image + +Checkout the "rosetta-stable" branch of the mina repository and then run the following: + +`cat dockerfiles/Dockerfile-rosetta | docker build -t mina-rosetta:stable --build-arg "MINA_BRANCH=rosetta-stable" -` + +This creates an image (mina-rosetta:stable) based on the most up-to-date changes that support rosetta. This image +can be used as a drop-in replacement for `gcr.io/o1labs-192920/coda-rosetta:debug-v1.1` in any of the below commands for testing. + ## How to Run As there is not currently a live network, the best way to run Rosetta is to run it against a sandbox node. Rosetta is best run using the official docker images provided here that run the Coda daemon, an archive node, and the rosetta process for you. See [Reproduce agent and rosetta-cli Validation](#reproduce-agent-and-rosetta-cli-validation) below for details. diff --git a/src/app/rosetta/docker-demo-start.sh b/src/app/rosetta/docker-demo-start.sh index f250fe011d7..1c6c3d824fd 100755 --- a/src/app/rosetta/docker-demo-start.sh +++ b/src/app/rosetta/docker-demo-start.sh @@ -6,11 +6,11 @@ function cleanup { CODE=${1:-0} echo "Killing archive.exe" - kill $(ps aux | egrep '_build/default/src/app/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true echo "Killing coda.exe" - kill $(ps aux | egrep '_build/default/src/app/.*coda.exe' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/.*coda.exe' | grep -v grep | awk '{ print $2 }') || true echo "Killing rosetta.exe" - kill $(ps aux | egrep '_build/default/src/app/rosetta' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/rosetta' | grep -v grep | awk '{ print $2 }') || true exit $CODE } @@ -28,7 +28,7 @@ sleep 3 # archive echo "========================= STARTING ARCHIVE PROCESS ===========================" -/coda-bin/archive/archive.exe run \ +/mina-bin/archive/archive.exe run \ -postgres-uri $PG_CONN \ -server-port 3086 \ -log-level fatal \ @@ -42,23 +42,25 @@ PK=${PK:-B62qrPN5Y5yq8kGE3FbVKbGTdTAJNdtNtB5sNVpxyRwWGcDEhpMzc8g} SNARK_PK=${SNARK_PK:-B62qiWSQiF5Q9CsAHgjMHoEEyR2kJnnCvN9fxRps2NXULU15EeXbzPf} genesis_time=$(date -d '2019-01-30 20:00:00.000000Z' '+%s') now_time=$(date +%s) + export CODA_TIME_OFFSET=$(( $now_time - $genesis_time )) export CODA_PRIVKEY_PASS="" -export CODA_LIBP2P_HELPER_PATH=/coda-bin/libp2p_helper -CODA_CONFIG_DIR=/root/.coda-config +export CODA_LIBP2P_HELPER_PATH=/mina-bin/libp2p_helper + +MINA_CONFIG_DIR=/root/.coda-config -# CODA_CONFIG_DIR is exposed by the dockerfile and contains demo mode essentials +# MINA_CONFIG_DIR is exposed by the dockerfile and contains demo mode essentials echo "========================= STARTING DAEMON ===========================" -/coda-bin/cli/src/coda.exe daemon \ +/mina-bin/cli/src/coda.exe daemon \ -archive-address 3086 \ -background \ - -block-producer-key "$CODA_CONFIG_DIR/wallets/store/$PK" \ - -config-dir "$CODA_CONFIG_DIR" \ - -config-file "$CODA_CONFIG_DIR/daemon.json" \ + -block-producer-key "$MINA_CONFIG_DIR/wallets/store/$PK" \ + -config-dir "$MINA_CONFIG_DIR" \ + -config-file "$MINA_CONFIG_DIR/daemon.json" \ -demo-mode \ -disable-telemetry \ -external-ip 127.0.0.1 \ - -external-port "${CODA_DAEMON_PORT:-10101}" \ + -external-port "${MINA_DAEMON_PORT:-10101}" \ -insecure-rest-server \ -log-level debug \ -log-json \ @@ -78,7 +80,7 @@ sleep 3 # rosetta echo "========================= STARTING ROSETTA API on PORT 3087 ===========================" -/coda-bin/rosetta/rosetta.exe \ +/mina-bin/rosetta/rosetta.exe \ -archive-uri $PG_CONN \ -graphql-uri http://localhost:3085/graphql \ -log-level debug \ diff --git a/src/app/rosetta/docker-start.sh b/src/app/rosetta/docker-start.sh index 72cdd3b338a..4921575a4f1 100755 --- a/src/app/rosetta/docker-start.sh +++ b/src/app/rosetta/docker-start.sh @@ -5,11 +5,11 @@ set -eou pipefail function cleanup { echo "Killing archive.exe" - kill $(ps aux | egrep '/coda-bin/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true echo "Killing coda.exe" - kill $(ps aux | egrep '/coda-bin/.*coda.exe' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/.*coda.exe' | grep -v grep | awk '{ print $2 }') || true echo "Killing rosetta.exe" - kill $(ps aux | egrep '/coda-bin/rosetta' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/rosetta' | grep -v grep | awk '{ print $2 }') || true echo "Stopping postgres" pg_ctlcluster 11 main stop exit @@ -27,7 +27,7 @@ pg_ctlcluster 11 main start sleep 3 # archive -/coda-bin/archive/archive.exe run \ +/mina-bin/archive/archive.exe run \ -postgres-uri $PG_CONN \ -server-port 3086 & @@ -35,26 +35,27 @@ sleep 3 sleep 3 export CODA_PRIVKEY_PASS="" -export CODA_CONFIG_FILE=${CODA_CONFIG_FILE:=/data/config.json} +export CODA_LIBP2P_HELPER_PATH=/mina-bin/libp2p_helper + +export MINA_CONFIG_FILE=${MINA_CONFIG_FILE:=/data/config.json} export PEER_ID=${PEER_ID:=/ip4/34.74.175.158/tcp/10001/ipfs/12D3KooWAFFq2yEQFFzhU5dt64AWqawRuomG9hL8rSmm5vxhAsgr/} -export CODA_PORT=${CODA_DAEMON_PORT:=10101} -DEFAULT_FLAGS="-generate-genesis-proof true -peer ${PEER_ID} -archive-address 0.0.0.0:3086 -insecure-rest-server -log-level debug -external-port ${CODA_PORT}" -export CODA_FLAGS=${CODA_FLAGS:=$DEFAULT_FLAGS} -export CODA_LIBP2P_HELPER_PATH=/coda-bin/libp2p_helper -PK=${CODA_PK:=ZsMSUuKL9zLAF7sMn951oakTFRCCDw9rDfJgqJ55VMtPXaPa5vPwntQRFJzsHyeh8R8} +export MINA_PORT=${MINA_DAEMON_PORT:=10101} +DEFAULT_FLAGS="-generate-genesis-proof true -peer ${PEER_ID} -archive-address 0.0.0.0:3086 -insecure-rest-server -log-level debug -external-port ${MINA_PORT}" +export MINA_FLAGS=${MINA_FLAGS:=$DEFAULT_FLAGS} +PK=${MINA_PK:=ZsMSUuKL9zLAF7sMn951oakTFRCCDw9rDfJgqJ55VMtPXaPa5vPwntQRFJzsHyeh8R8} -echo "CODA Flags: $CODA_FLAGS -config-file ${CODA_CONFIG_FILE}" +echo "MINA Flags: $MINA_FLAGS -config-file ${MINA_CONFIG_FILE}" # Daemon w/ mounted config file, initial file is phase 3 config.json -/coda-bin/cli/src/coda.exe daemon \ - -config-file ${CODA_CONFIG_FILE} \ - ${CODA_FLAGS} $@ & +/mina-bin/cli/src/coda.exe daemon \ + -config-file ${MINA_CONFIG_FILE} \ + ${MINA_FLAGS} $@ & # wait for it to settle sleep 3 # rosetta -/coda-bin/rosetta/rosetta.exe \ +/mina-bin/rosetta/rosetta.exe \ -archive-uri $PG_CONN \ -graphql-uri http://localhost:3085/graphql \ -log-level debug \ diff --git a/src/app/rosetta/docker-test-start.sh b/src/app/rosetta/docker-test-start.sh index 66e9c4abb0b..53e1abb5644 100755 --- a/src/app/rosetta/docker-test-start.sh +++ b/src/app/rosetta/docker-test-start.sh @@ -6,13 +6,13 @@ function cleanup { CODE=${1:-0} echo "Killing archive.exe" - kill $(ps aux | egrep '_build/default/src/app/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/.*archive.exe' | grep -v grep | awk '{ print $2 }') || true echo "Killing coda.exe" - kill $(ps aux | egrep '_build/default/src/app/.*coda.exe' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/.*coda.exe' | grep -v grep | awk '{ print $2 }') || true echo "Killing agent.exe" - kill $(ps aux | egrep '_build/default/src/app/rosetta/test-agent/agent.exe' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/rosetta/test-agent/agent.exe' | grep -v grep | awk '{ print $2 }') || true echo "Killing rosetta.exe" - kill $(ps aux | egrep '_build/default/src/app/rosetta' | grep -v grep | awk '{ print $2 }') || true + kill $(ps aux | egrep '/mina-bin/rosetta' | grep -v grep | awk '{ print $2 }') || true exit $CODE } @@ -28,7 +28,7 @@ pg_ctlcluster 11 main start sleep 3 # archive -/coda-bin/archive/archive.exe run \ +/mina-bin/archive/archive.exe run \ -postgres-uri $PG_CONN \ -log-json \ -server-port 3086 & @@ -42,17 +42,17 @@ genesis_time=$(date -d '2019-01-30 20:00:00.000000Z' '+%s') now_time=$(date +%s) export CODA_TIME_OFFSET=$(( $now_time - $genesis_time )) export CODA_PRIVKEY_PASS="" -export CODA_LIBP2P_HELPER_PATH=/coda-bin/libp2p_helper -CODA_CONFIG_DIR=/root/.coda-config +export CODA_LIBP2P_HELPER_PATH=/mina-bin/libp2p_helper +MINA_CONFIG_DIR=/root/.coda-config -# CODA_CONFIG_DIR is exposed by the dockerfile and contains demo mode essentials -/coda-bin/cli/src/coda.exe daemon \ +# MINA_CONFIG_DIR is exposed by the dockerfile and contains demo mode essentials +/mina-bin/cli/src/coda.exe daemon \ -seed \ -demo-mode \ - -block-producer-key "$CODA_CONFIG_DIR/wallets/store/$PK" \ + -block-producer-key "MINA_CONFIG_DIR/wallets/store/$PK" \ -run-snark-worker $PK \ - -config-file "$CODA_CONFIG_DIR/daemon.json" \ - -config-dir "$CODA_CONFIG_DIR" \ + -config-file "$MINA_CONFIG_DIR/daemon.json" \ + -config-dir "$MINA_CONFIG_DIR" \ -insecure-rest-server \ -external-ip 127.0.0.1 \ -archive-address 3086 \ @@ -63,7 +63,7 @@ CODA_CONFIG_DIR=/root/.coda-config sleep 3 # rosetta -/coda-bin/rosetta/rosetta.exe \ +/mina-bin/rosetta/rosetta.exe \ -archive-uri $PG_CONN \ -graphql-uri http://localhost:3085/graphql \ -log-level debug \ @@ -74,7 +74,7 @@ sleep 3 sleep 3 # test agent -/coda-bin/rosetta/test-agent/agent.exe \ +/mina-bin/rosetta/test-agent/agent.exe \ -graphql-uri http://localhost:3085/graphql \ -rosetta-uri http://localhost:3087/ \ -log-level Trace \