Merge pull request #1780 from helium/adt/ubuntu-validators

Use a different Dockerfile for validator builds

.buildkite/scripts/make_image.sh

@@ -25,7 +25,7 @@ if [[ "$IMAGE_ARCH" == "arm64" ]]; then
 fi
 
 VERSION=$(echo $VERSION_TAG | sed "s/^${BUILD_NET}_//" | sed "s/^${BUILD_TYPE}//")
-DOCKER_BUILD_ARGS="--build-arg BUILDER_IMAGE=$BUILD_IMAGE --build-arg RUNNER_IMAGE=$RUN_IMAGE --build-arg VERSION=$VERSION --build-arg BUILD_NET=$BUILD_NET"
+DOCKER_BUILD_ARGS=" --build-arg VERSION=$VERSION --build-arg BUILD_NET=$BUILD_NET"
 
 if [[ ! $TEST_BUILD -eq "0" ]]; then
     REGISTRY_NAME="test-builds"
@@ -36,37 +36,37 @@ FULL_REGISTRY_NAME="$REGISTRY_HOST/$REGISTRY_ORG/$REGISTRY_NAME"
 case "${BUILD_TYPE}-$BUILD_NET" in
     "validator-mainnet")
         echo "Doing a mainnet validator image build for $IMAGE_ARCH"
-        DOCKER_BUILD_ARGS="--build-arg REBAR_BUILD_TARGET=docker_val $DOCKER_BUILD_ARGS"
+        DOCKER_BUILD_ARGS="-f Dockerfile-ubuntu --build-arg REBAR_BUILD_TARGET=docker_val $DOCKER_BUILD_ARGS"
         BASE_DOCKER_NAME="validator"
         DOCKER_NAME="${BASE_DOCKER_NAME}-${IMAGE_ARCH}_${VERSION}"
         ;;
     "validator-testnet")
         echo "Doing a testnet validator image build for ${IMAGE_ARCH}"
-        DOCKER_BUILD_ARGS="--build-arg REBAR_BUILD_TARGET=docker_testval $DOCKER_BUILD_ARGS"
+        DOCKER_BUILD_ARGS="-f Dockerfile-ubuntu --build-arg REBAR_BUILD_TARGET=docker_testval $DOCKER_BUILD_ARGS"
         BASE_DOCKER_NAME="validator"
         DOCKER_NAME="${BASE_DOCKER_NAME}-${IMAGE_ARCH}_${VERSION}"
         ;;
     "validator-devnet")
         echo "Doing a devnet validator image build for ${IMAGE_ARCH}"
-        DOCKER_BUILD_ARGS="--build-arg REBAR_BUILD_TARGET=docker_testval $DOCKER_BUILD_ARGS"
+        DOCKER_BUILD_ARGS="-f Dockerfile-ubuntu --build-arg REBAR_BUILD_TARGET=docker_testval $DOCKER_BUILD_ARGS"
         BASE_DOCKER_NAME="validator"
         DOCKER_NAME="${BASE_DOCKER_NAME}-${IMAGE_ARCH}_${VERSION}"
         ;;
     "miner-mainnet")
         echo "Doing a miner image build for ${IMAGE_ARCH}"
-        DOCKER_BUILD_ARGS="--build-arg EXTRA_BUILD_APK_PACKAGES=apk-tools --build-arg EXTRA_RUNNER_APK_PACKAGES=apk-tools --build-arg REBAR_BUILD_TARGET=docker $DOCKER_BUILD_ARGS"
+        DOCKER_BUILD_ARGS="--build-arg BUILDER_IMAGE=$BUILD_IMAGE --build-arg RUNNER_IMAGE=$RUN_IMAGE --build-arg EXTRA_BUILD_APK_PACKAGES=apk-tools --build-arg EXTRA_RUNNER_APK_PACKAGES=apk-tools --build-arg REBAR_BUILD_TARGET=docker $DOCKER_BUILD_ARGS"
         BASE_DOCKER_NAME=$(basename $(pwd))
         DOCKER_NAME="${BASE_DOCKER_NAME}-${IMAGE_ARCH}_${VERSION}"
         ;;
     "miner-testnet")
         echo "Doing a testnet miner image build for ${IMAGE_ARCH}"
-        DOCKER_BUILD_ARGS="--build-arg EXTRA_BUILD_APK_PACKAGES=apk-tools --build-arg EXTRA_RUNNER_APK_PACKAGES=apk-tools --build-arg REBAR_BUILD_TARGET=docker_testminer $DOCKER_BUILD_ARGS"
+        DOCKER_BUILD_ARGS="--build-arg BUILDER_IMAGE=$BUILD_IMAGE --build-arg RUNNER_IMAGE=$RUN_IMAGE --build-arg EXTRA_BUILD_APK_PACKAGES=apk-tools --build-arg EXTRA_RUNNER_APK_PACKAGES=apk-tools --build-arg REBAR_BUILD_TARGET=docker_testminer $DOCKER_BUILD_ARGS"
         BASE_DOCKER_NAME=$(basename $(pwd))
         DOCKER_NAME="${BASE_DOCKER_NAME}-${IMAGE_ARCH}_${VERSION}"
         ;;
     "miner-devnet")
         echo "Doing a devnet miner image build for ${IMAGE_ARCH}"
-        DOCKER_BUILD_ARGS="--build-arg EXTRA_BUILD_APK_PACKAGES=apk-tools --build-arg EXTRA_RUNNER_APK_PACKAGES=apk-tools --build-arg REBAR_BUILD_TARGET=docker_testminer $DOCKER_BUILD_ARGS"
+        DOCKER_BUILD_ARGS="--build-arg BUILDER_IMAGE=$BUILD_IMAGE --build-arg RUNNER_IMAGE=$RUN_IMAGE --build-arg EXTRA_BUILD_APK_PACKAGES=apk-tools --build-arg EXTRA_RUNNER_APK_PACKAGES=apk-tools --build-arg REBAR_BUILD_TARGET=docker_testminer $DOCKER_BUILD_ARGS"
         BASE_DOCKER_NAME=$(basename $(pwd))
         DOCKER_NAME="${BASE_DOCKER_NAME}-${IMAGE_ARCH}_${VERSION}"
         ;;

Dockerfile-ubuntu

@@ -0,0 +1,105 @@
+ARG BUILDER_IMAGE=erlang:24-slim
+ARG RUNNER_IMAGE=debian:bullseye-slim
+FROM ${BUILDER_IMAGE} as deps-compiler
+
+ARG REBAR_DIAGNOSTIC=0
+ENV DIAGNOSTIC=${REBAR_DIAGNOSTIC}
+
+ARG REBAR_BUILD_TARGET
+ARG TAR_PATH=_build/$REBAR_BUILD_TARGET/rel/*/*.tar.gz
+ARG EXTRA_BUILD_APT_PACKAGES
+
+RUN apt update \
+    && apt install -y --no-install-recommends \
+       autoconf \
+       automake \
+       bison \
+       build-essential \
+       bzip2 \
+       ca-certificates \
+       cmake \
+       curl \
+       flex \
+       git \
+       libdbus-1-dev \
+       libgmp-dev \ 
+       libprotobuf-dev \
+       libsodium-dev \
+       libssl-dev \
+       libtool \
+       lz4 \
+       libprotobuf-dev \
+       wget \
+       ${EXTRA_BUILD_APT_PACKAGES} 
+
+# Strip debug symbols from ERTS binarys to reduce size
+RUN scanelf --nobanner -E ET_DYN -BF '%F' --recursive /usr/local/lib/erlang | xargs -r strip --strip-unneeded
+
+# Install Rust toolchain
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+WORKDIR /usr/src/miner
+
+ENV CC=gcc CXX=g++ CFLAGS="-O2" CXXFLAGS="-O2" \
+    ERLANG_ROCKSDB_OPTS="-DWITH_BUNDLE_SNAPPY=ON -DWITH_BUNDLE_LZ4=ON" \
+    ERL_COMPILER_OPTIONS="[deterministic]" \
+    PATH="/root/.cargo/bin:$PATH" 
+
+# Add and compile the dependencies to cache
+COPY ./rebar* ./Makefile ./
+COPY ./config/grpc_client_gen_local.config ./config/
+COPY ./config/grpc_client_gen.config ./config/
+
+RUN ./rebar3 compile
+
+FROM deps-compiler as builder
+
+ARG VERSION
+ARG REBAR_DIAGNOSTIC=0
+
+# default to building for mainnet
+ARG BUILD_NET=mainnet
+ENV DIAGNOSTIC=${REBAR_DIAGNOSTIC}
+
+ARG REBAR_BUILD_TARGET
+ARG TAR_PATH=_build/$REBAR_BUILD_TARGET/rel/*/*.tar.gz
+
+# Now add our code
+COPY . .
+
+RUN ./rebar3 as ${REBAR_BUILD_TARGET} tar -n miner -v ${VERSION}
+
+RUN mkdir -p /opt/docker/update
+RUN tar -zxvf ${TAR_PATH} -C /opt/docker
+RUN wget -O /opt/docker/update/genesis https://snapshots.helium.wtf/genesis.${BUILD_NET}
+
+FROM ${RUNNER_IMAGE} as runner
+
+ARG VERSION
+ARG EXTRA_RUNNER_APT_PACKAGES
+
+RUN apt update \
+    && apt install -y \
+       dbus \
+       iproute2 \
+       libncurses6 \
+       libsodium23 \
+       libstdc++6 \
+       ${EXTRA_RUNNER_APT_PACKAGES} \
+    && rm -rf /var/lib/apt/lists \
+    && ln -sf /opt/miner/releases/${VERSION} /config
+
+WORKDIR /opt/miner
+
+ENV COOKIE=miner \
+    # Write files generated during startup to /tmp
+    RELX_OUT_FILE_PATH=/tmp \
+    # add miner to path, for easy interactions
+    PATH=$PATH:/opt/miner/bin
+
+COPY --from=builder /opt/docker /opt/miner
+
+VOLUME ["/opt/miner/hotfix", "/var/data"]
+
+ENTRYPOINT ["/opt/miner/bin/miner"]
+CMD ["foreground"]

src/miner_hbbft_sidecar.erl

@@ -86,9 +86,9 @@ new_round(Buf, BinTxns) ->
 prefilter_round(Buf, Txns) ->
     gen_server:call(?SERVER, {prefilter_round, Buf, Txns}, infinity).
 
-handle_txn(submit = RequestType, Txn) ->
+handle_txn(submit = _RequestType, Txn) ->
     gen_server:call(?SERVER, {submit, Txn}, infinity);
-handle_txn(update = RequestType, Txn) ->
+handle_txn(update = _RequestType, Txn) ->
     gen_server:call(?SERVER, {query_txn, Txn}, infinity).
 
 %%%===================================================================
@@ -124,11 +124,11 @@ handle_call({set_group, Group}, _From, #state{group = OldGroup} = State) ->
             ok = libp2p_swarm:add_stream_handler(blockchain_swarm:tid(), ?TX_PROTOCOL_V2,
                                                  {libp2p_framed_stream, server,
                                                   [blockchain_txn_handler, ?TX_PROTOCOL_V2, self(),
-                                                   fun(submit, T) -> ?MODULE:submit(T); (_, T) -> {error, req_not_supported} end]}),
+                                                   fun(submit, T) -> ?MODULE:submit(T); (_, _T) -> {error, req_not_supported} end]}),
             ok = libp2p_swarm:add_stream_handler(blockchain_swarm:tid(), ?TX_PROTOCOL_V1,
                                                  {libp2p_framed_stream, server,
                                                   [blockchain_txn_handler, ?TX_PROTOCOL_V1, self(),
-                                                   fun(submit, T) -> ?MODULE:submit(T); (_, T) -> {error, req_not_supported} end]});
+                                                   fun(submit, T) -> ?MODULE:submit(T); (_, _T) -> {error, req_not_supported} end]});
         {P, undefined} when is_pid(P) ->
             libp2p_swarm:remove_stream_handler(blockchain_swarm:tid(), ?TX_PROTOCOL_V3),
             libp2p_swarm:remove_stream_handler(blockchain_swarm:tid(), ?TX_PROTOCOL_V2),