Lk86/resolve rosetta conflicts

dockerfiles/stages/1-build-deps-ubuntu

@@ -0,0 +1,140 @@
+#################################################################################################
+# The "build-deps" stage
+# - Installs all compilers/interpreters, tools, and OS packages on Ubuntu 20.04
+#################################################################################################
+FROM ubuntu:20.04 AS build-deps
+
+# Ocaml Version
+ARG OCAML_VERSION=4.11
+ARG OCAML_REVISION=.2
+ARG OCAML_VARIANT=
+ARG OCAML_PACKAGE=
+ARG OPAM_VERSION=2.0.7
+
+# Golang version number used to detemine tarball name
+ARG GO_VERSION=1.16.5
+
+# Go Capnp Version (for capnpc dependency)
+ARG GO_CAPNP_VERSION=v3.0.0-alpha.1
+
+# Rust Version passed into rustup-init, can also be "stable", "nightly" or similar
+ARG RUST_VERSION=1.52.1
+
+# Rocksdb commit tag/branch to clone
+ARG ROCKSDB_VERSION=v5.18.4
+
+# OS package dependencies
+# First add support for https and pkg-config for apt, then install everything else
+# TODO: make sure this is the minimum compile-time deps
+# Organized as two alphabetized lists, first libraries and then tools/other packages
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+  && apt-get install --yes \
+    libboost-dev \
+    libboost-program-options-dev \
+    libbz2-dev \
+    libcap-dev \
+    libffi-dev \
+    libgflags-dev \
+    libgmp-dev \
+    libgmp3-dev \
+    libjemalloc-dev \
+    libpq-dev \
+    libprocps-dev \
+    libsodium-dev \
+    libssl-dev \
+    zlib1g-dev \
+    build-essential \
+    capnproto \
+    cmake \
+    curl \
+    file \
+    git \
+    m4 \
+    pkg-config \
+    rsync \
+    sudo \
+    unzip
+
+# Create opam user (for later) and give sudo to make opam happy
+RUN adduser --uid 65533 --disabled-password --gecos '' opam \
+  && passwd -l opam \
+  && chown -R opam:opam /home/opam \
+  && echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam \
+  && chmod 440 /etc/sudoers.d/opam \
+  && chown root:root /etc/sudoers.d/opam \
+  && chmod 777 /tmp
+
+# Opam install of a given OPAM_VERSION from github release
+RUN curl -sL \
+  "https://github.com/ocaml/opam/releases/download/${OPAM_VERSION}/opam-${OPAM_VERSION}-x86_64-linux" \
+  -o /usr/bin/opam \
+  && chmod +x /usr/bin/opam
+
+# Golang install of a given GO_VERSION (add -v for spam output of each file from the go dist)
+RUN curl -s "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar -xz -C /usr/lib/
+
+# Rust install via rustup-init to a given RUST_VERSION
+USER opam
+RUN curl --proto '=https' --tlsv1.2 -sSf -o /tmp/rustup-init \
+  https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init \
+  && chmod +x /tmp/rustup-init \
+  && /tmp/rustup-init -y --default-toolchain "${RUST_VERSION}" --profile minimal \
+  && rm /tmp/rustup-init
+# For more about rustup-init see: https://github.com/rust-lang/rustup/blob/master/README.md
+# As opposed to introducing another shell script here (that mostly just determines the platform)
+# we just download the binary for the only platform we care about in this docker environment
+
+USER root
+ENV CXXFLAGS '-Wno-error=deprecated-copy -Wno-error=pessimizing-move -Wno-error=class-memaccess'
+# This builds and installs just the rocksdb static lib for us, and cleans up after itself
+RUN git clone https://github.com/facebook/rocksdb \
+  --depth 1 --shallow-submodules \
+  -b "${ROCKSDB_VERSION}" /rocksdb \
+  && make -C /rocksdb static_lib PORTABLE=1 -j$(nproc) \
+  && cp /rocksdb/librocksdb.a /usr/local/lib/librocksdb_coda.a \
+  && rm -rf /rocksdb \
+  && strip -S /usr/local/lib/librocksdb_coda.a
+ENV CXXFLAGS ""
+
+###########################################################################################
+# Initialize opam in a minimal fashion
+###########################################################################################
+
+# Set up environment for running as opam user
+WORKDIR /home/opam
+USER opam
+ENV HOME /home/opam
+
+# Create the following user directory configs as the Opam user:
+## Add go + rust to the path, unlimit the opam user,
+## unlimit stack for future shells that might use spacetime,
+## disable ipv6
+## disable sandboxing to allow unprivledged builds
+RUN mkdir --mode=700 ~/.gnupg \
+  && echo 'export PATH="$PATH:/usr/lib/go/bin:$HOME/.cargo/bin"' >> ~/.bashrc \
+  && echo 'ulimit -s unlimited' >> ~/.bashrc \
+  && echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf
+
+# Ocaml install of a given OCAML_VERSION via opam switch
+# additionally initializes opam with sandboxing disabled, as we did not install bubblewrap above.
+RUN git clone \
+  https://github.com/ocaml/opam-repository.git \
+  --depth 1 \
+  /home/opam/opam-repository \
+  && opam init --disable-sandboxing -k git -a ~/opam-repository --bare \
+  && opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git \
+  && opam switch create "${OCAML_VERSION}${OCAML_REVISION}" "${OCAML_PACKAGE}${OCAML_VERSION}${OCAML_REVISION}${OCAML_VARIANT}" \
+  && opam switch "${OCAML_VERSION}${OCAML_REVISION}"
+
+# --- Build and install capnproto go compiler
+# This package was particularly tricky to get to install correctly because of how the package's source code is structured.
+# To work around this, we install it as a dependency for a local go module and build the binary from the GOPATH.
+RUN mkdir go-tmp \
+    && cd go-tmp \
+    && /usr/lib/go/bin/go mod init local/build \
+    && /usr/lib/go/bin/go get capnproto.org/go/capnp/v3@$GO_CAPNP_VERSION \
+    && /usr/lib/go/bin/go build -o capnpc-go capnproto.org/go/capnp/v3/capnpc-go \
+    && sudo mv capnpc-go /usr/local/bin \
+    && cd - \
+    && sudo rm -rf go-tmp

dockerfiles/stages/3-opam-deps → dockerfiles/stages/2-opam-deps

@@ -5,7 +5,7 @@
 # - Includes the entire mina codebase and submodules in "${MINA_DIR}" (must be writable by opam user)
 # - Largely mirrors/replaces ./scripts/setup-opam.sh
 #################################################################################################
-FROM toolchain AS opam-deps
+FROM build-deps AS opam-deps
 
 # location of repo used for pins and external package commits
 ARG MINA_DIR=mina

dockerfiles/stages/2-toolchain → dockerfiles/stages/3-toolchain

@@ -2,7 +2,7 @@
 # The "toolchain" Stage
 # - installs optional tools that are required in our CI/CD but are not required to build mina
 #################################################################################################
-FROM build-deps AS toolchain
+FROM opam-deps AS toolchain
 
 ARG DOCKER_VERSION=19.03.4
 ARG TERRAFORM_VERSION=0.14.11

dockerfiles/stages/4-deb-builder

@@ -2,7 +2,7 @@
 # The "deb-builder" Stage
 # - builds mina and any other binaries required to run `make deb`
 #################################################################################################
-FROM opam-deps AS deb-builder
+FROM toolchain AS deb-builder
 
 # Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI
 ARG DUNE_PROFILE=devnet

dockerfiles/stages/5-prod-ubuntu → dockerfiles/stages/4-prod-ubuntu

@@ -2,10 +2,10 @@
 # The "production" Stage
 # - sets up the final container with built binaries and a running postgresql archive node setup
 #################################################################################################
-FROM ubuntu:18.04 AS production
+FROM ubuntu:20.04 AS production
 
 ARG MINA_DAEMON_PORT=10101
-ARG MINA_CONFIG_DIR=/root/.mina-config
+ARG MINA_CONFIG_DIR=/data/.mina-config
 # Sample public key for use in dev profile / demo mode genesis block
 ARG PK=B62qiZfzW27eavtPrnF6DeDSAKEjXuGFdkouC3T5STRa6rrYLiDUP2p
 
@@ -27,15 +27,18 @@ RUN apt-get -y update \
     postgresql \
     postgresql-contrib \
     tzdata \
-    libjemalloc1 \
-    libprocps6 \
-    libffi6 \
+    libjemalloc2 \
+    libprocps8 \
+    libffi7 \
     jq \
     sudo
 
 # Install rosetta-cli
 RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/master/scripts/install.sh | sh -s
 
+# Generate en_US.UTF-8 to allow use of O(1) Labs DB dumps
+RUN locale-gen en_US.UTF-8
+
 RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \
   && echo "$PK" >  ${MINA_CONFIG_DIR}/wallets/store/$PK.pub \
   && echo '{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"6pcvpWSLkMi393dT5VSLR6ft56AWKkCYRqJoYia","pwsalt":"ASoBkV3NsY7ZRuxztyPJdmJCiz3R","pwdiff":[134217728,6],"ciphertext":"Dmq1Qd8uNbZRT1NT7zVbn3eubpn9Myx9Je9ZQGTKDxUv4BoPNmZAGox18qVfbbEUSuhT4ZGDt"}' \
@@ -49,20 +52,18 @@ COPY --from=builder /home/opam/mina/src/app/rosetta/README.md /rosetta/
 COPY --from=builder /home/opam/mina/genesis_ledgers /genesis_ledgers/
 
 USER postgres
-
 RUN POSTGRES_VERSION=$(psql -V | cut -d " " -f 3 | sed 's/.[[:digit:]]*$//g') \
     && echo "$POSTGRES_VERSION" "$(psql -V)" \
-    && echo "data_directory = '/data/postgresql'" >> /etc/postgresql/${POSTGRES_VERSION}/main/postgresql.conf \
-    && echo "listen_addresses='*'" >> /etc/postgresql/${POSTGRES_VERSION}/main/postgresql.conf \
-    && echo "max_locks_per_transaction=1024" >> /etc/postgresql/${POSTGRES_VERSION}/main/postgresql.conf \
-    && echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/${POSTGRES_VERSION}/main/pg_hba.conf
+    && echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/${POSTGRES_VERSION}/main/pg_hba.conf \
+    && pg_dropcluster --stop ${POSTGRES_VERSION} main
 
 USER root
 
-WORKDIR /rosetta
+# Run as root so it can create /data/postgresql
+RUN POSTGRES_VERSION=$(psql -V | cut -d " " -f 3 | sed 's/.[[:digit:]]*$//g') \
+  && pg_createcluster --start -d /data/postgresql --createclusterconf /rosetta/postgresql.conf ${POSTGRES_VERSION} main
 
-# Generate en_US.UTF-8 to allow use of O(1) Labs DB dumps
-RUN locale-gen en_US.UTF-8
+WORKDIR /rosetta
 
 EXPOSE 3087
 EXPOSE $MINA_DAEMON_PORT

dockerfiles/stages/5-production → dockerfiles/stages/4-production

@@ -50,6 +50,9 @@ RUN test "$deb_codename" = "stretch" \
 # Install rosetta-cli
 RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/master/scripts/install.sh | sh -s
 
+# Generate en_US.UTF-8 to allow use of O(1) Labs DB dumps
+RUN locale-gen en_US.UTF-8
+
 RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \
   && echo "$PK" >  ${MINA_CONFIG_DIR}/wallets/store/$PK.pub \
   && echo '{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"6pcvpWSLkMi393dT5VSLR6ft56AWKkCYRqJoYia","pwsalt":"ASoBkV3NsY7ZRuxztyPJdmJCiz3R","pwdiff":[134217728,6],"ciphertext":"Dmq1Qd8uNbZRT1NT7zVbn3eubpn9Myx9Je9ZQGTKDxUv4BoPNmZAGox18qVfbbEUSuhT4ZGDt"}' \
@@ -63,20 +66,14 @@ COPY --from=builder /home/opam/mina/src/app/rosetta/README.md /rosetta/
 COPY --from=builder /home/opam/mina/genesis_ledgers /genesis_ledgers/
 
 USER postgres
-
 RUN POSTGRES_VERSION=$(psql -V | cut -d " " -f 3 | sed 's/.[[:digit:]]*$//g') \
     && echo "$POSTGRES_VERSION" "$(psql -V)" \
-    && echo "data_directory = '/data/postgresql'" >> /etc/postgresql/${POSTGRES_VERSION}/main/postgresql.conf \
-    && echo "listen_addresses='*'" >> /etc/postgresql/${POSTGRES_VERSION}/main/postgresql.conf \
     && echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/${POSTGRES_VERSION}/main/pg_hba.conf
 
 USER root
 
 WORKDIR /rosetta
 
-# Generate en_US.UTF-8 to allow use of O(1) Labs DB dumps
-RUN locale-gen en_US.UTF-8
-
 EXPOSE 3087
 EXPOSE $MINA_DAEMON_PORT
 

frontend/client_sdk/test/Test.ts

@@ -84,45 +84,27 @@ deepStrictEqual(
   "Invalid signed delegation was verified"
 );
 
-const signedRosettaTnxMock = `
-{
-    "signature": "389ac7d4077f3d485c1494782870979faa222cd906b25b2687333a92f41e40b925adb08705eddf2a7098e5ac9938498e8a0ce7c70b25ea392f4846b854086d43",
-    "payment": {
-      "to": "B62qnzbXmRNo9q32n4SNu2mpB8e7FYYLH8NmaX6oFCBYjjQ8SbD7uzV",
-      "from": "B62qnzbXmRNo9q32n4SNu2mpB8e7FYYLH8NmaX6oFCBYjjQ8SbD7uzV",
-      "fee": "10000000",
-      "token": "1",
-      "nonce": "0",
-      "memo": null,
-      "amount": "1000000000",
-      "valid_until": "4294967295"
-    },
-    "stake_delegation": null,
-    "create_token": null,
-    "create_token_account": null,
-    "mint_tokens": null
-  }
-`;
+const signedRosettaTxnMock = "{\"signature\":\"C1B98ADACCEC9BB0BC7646C43A128BF83FC31B44CFC4F28BA7874489AFA7312B251D96FE23D9195C65B77430CA0D326626009C28FDBE1AA47990C4235238A436\",\"payment\":{\"to\":\"B62qoDWfBZUxKpaoQCoFqr12wkaY84FrhxXNXzgBkMUi2Tz4K8kBDiv\",\"from\":\"B62qkUHaJUHERZuCHQhXCQ8xsGBqyYSgjQsKnKN5HhSJecakuJ4pYyk\",\"fee\":\"2000000000\",\"token\":\"1\",\"nonce\":\"2\",\"memo\":\"hello\",\"amount\":\"3000000000\",\"valid_until\":\"10000000\"},\"stake_delegation\":null,\"create_token\":null,\"create_token_account\":null,\"mint_tokens\":null}";
 
 const signedGraphQLCommand =
-  MinaSDK.signedRosettaTransactionToSignedCommand(signedRosettaTnxMock);
-const signedRosettaTnxMockJson = JSON.parse(signedRosettaTnxMock);
+  MinaSDK.signedRosettaTransactionToSignedCommand(signedRosettaTxnMock);
+const signedRosettaTxnMockJson = JSON.parse(signedRosettaTxnMock);
 const signedGraphQLCommandJson = JSON.parse(signedGraphQLCommand);
 
 deepStrictEqual(
-  signedRosettaTnxMockJson.payment.to,
+  signedRosettaTxnMockJson.payment.to,
   signedGraphQLCommandJson.data.payload.body[1].receiver_pk,
   "Rosetta to GraphQL transaction conversion has mismatched receiver public key"
 );
 
 deepStrictEqual(
-  signedRosettaTnxMockJson.payment.from,
+  signedRosettaTxnMockJson.payment.from,
   signedGraphQLCommandJson.data.payload.body[1].source_pk,
   "Rosetta to GraphQL transaction conversion has mismatched source public key"
 );
 
 deepStrictEqual(
-  signedRosettaTnxMockJson.payment.amount,
+  signedRosettaTxnMockJson.payment.amount,
   signedGraphQLCommandJson.data.payload.body[1].amount,
   "Rosetta to GraphQL transaction conversion has mismatched amount"
 );