diff --git a/.circleci/config.yml b/.circleci/config.yml index f7e8477ebe7..bd456b0351d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,12 +104,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Set Up Database command: | @@ -172,8 +172,8 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - save_cache: name: Save cache - opam @@ -183,7 +183,7 @@ jobs: no_output_timeout: 1h - run: name: Build Archive Process - command: bash -c 'eval `opam config env` && export PATH="$HOME/.cargo/bin:$PATH" && LIBP2P_NIXLESS=1 make build_archive' + command: bash -c 'eval `opam config env` && export PATH="$HOME/.cargo/bin:$PATH" && make build_archive' environment: DUNE_PROFILE: testnet_postake_medium_curves # NOTE: If we were using a machine executor we would be able to mount the sql file in @@ -228,18 +228,18 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Check circle CI configuration rendering command: ./scripts/test.py render --check .circleci/config.yml.jinja .mergify.yml.jinja - run: name: OCamlformat (make check-format) - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'eval `opam config env` && LIBP2P_NIXLESS=1 make check-format' + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'eval `opam config env` && make check-format' - run: name: Snarky tracks master (make check-snarky-submodule) - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make check-snarky-submodule' + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make check-snarky-submodule' - run: name: Check ppx_optcomp preprocessor_deps command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c './scripts/lint_preprocessor_deps.sh' @@ -282,8 +282,8 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Compare versioned types in PR @@ -322,8 +322,8 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Compare test signatures for consensus, nonconsensus code @@ -431,8 +431,8 @@ jobs: - opam-v8-{{ checksum "opam_ci_cache.sig" }}-{{ checksum "scripts/Brewfile" }} - run: - name: Install macos dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install macos dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - save_cache: name: Save cache - opam key: opam-v8-{{ checksum "opam_ci_cache.sig" }}-{{ checksum "scripts/Brewfile" }} @@ -440,19 +440,11 @@ jobs: paths: - "/Users/distiller/.opam" no_output_timeout: 1h - ### nix/cachix - - run: - name: Install nix and cachix - command: | - curl -L https://nixos.org/nix/install | sh - . ~/.nix-profile/etc/profile.d/nix.sh - nix-env -iA cachix -f https://cachix.org/api/v1/install ### dune - run: name: Build ocaml environment: DUNE_PROFILE: testnet_postake_medium_curves - EXTRA_NIX_ARGS: --option sandbox false command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && make build 2>&1 | tee /tmp/buildocaml.log' no_output_timeout: 20m - run: @@ -482,7 +474,6 @@ jobs: echo 'libp2p_helper' cp src/app/libp2p_helper/result/bin/libp2p_helper package/coda-libp2p_helper chmod +w package/coda-libp2p_helper - ./scripts/librewrite-macos.sh package/coda-libp2p_helper echo 'coda-logproc' cp _build/default/src/app/logproc/logproc.exe package/coda-logproc chmod +wx package/coda-logproc @@ -513,7 +504,7 @@ jobs: - run: name: Build portable binary command: | - LIBP2P_NIXLESS=1 make macos-portable + make macos-portable cp _build/coda-daemon-macos.zip package/. - run: name: Copy artifacts to cloud @@ -553,12 +544,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build client SDK for Javascript - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && LIBP2P_NIXLESS=1 make client_sdk 2>&1 | tee /tmp/artifacts/buildclientsdk.log' + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && make client_sdk 2>&1 | tee /tmp/artifacts/buildclientsdk.log' environment: DUNE_PROFILE: nonconsensus_medium_curves no_output_timeout: 10m @@ -600,20 +591,19 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: GO=/usr/lib/go/bin/go LIBP2P_NIXLESS=1 make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper # Explicitly generate PV-keys and uploading before building # See https://bkase.dev/posts/ocaml-writer#fn-3 for rationale - run: name: Generate PV keys - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && PATH="$HOME/.cargo/bin:$PATH" LIBP2P_NIXLESS=1 make build_pv_keys 2>&1 | tee /tmp/artifacts/buildocaml.log' + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && PATH="$HOME/.cargo/bin:$PATH" make build_pv_keys 2>&1 | tee /tmp/artifacts/buildocaml.log' environment: DUNE_PROFILE: testnet_postake_medium_curves - EXTRA_NIX_ARGS: --option sandbox false no_output_timeout: 20m - run: name: Upload generated PV keys @@ -625,9 +615,7 @@ jobs: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'eval `opam config env` && export PATH="$HOME/.cargo/bin:$PATH" && dune build src/app/generate_keypair/generate_keypair.exe' environment: DUNE_PROFILE: testnet_postake_medium_curves - EXTRA_NIX_ARGS: --option sandbox false GO: /usr/lib/go/bin/go - LIBP2P_NIXLESS: 1 no_output_timeout: 20m - run: @@ -647,18 +635,15 @@ jobs: command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make deb' environment: DUNE_PROFILE: testnet_postake_medium_curves - LIBP2P_NIXLESS: 1 no_output_timeout: 20m - run: name: Store genesis public/private keypairs command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make genesiskeys' environment: - LIBP2P_NIXLESS: 1 - run: name: Upload deb to repo command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make publish_deb' environment: - LIBP2P_NIXLESS: 1 no_output_timeout: 20m - run: name: Copy artifacts to cloud @@ -763,18 +748,17 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Run unit tests command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && make build && (dune runtest src/lib --profile=dev -j8 || (./scripts/link-coredumps.sh && false))' environment: DUNE_PROFILE: dev - LIBP2P_NIXLESS: 1 GO: /usr/lib/go/bin/go no_output_timeout: 30m - store_artifacts: @@ -810,19 +794,18 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Run unit tests command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && (dune runtest src/nonconsensus --profile=nonconsensus_medium_curves -j8 || (./scripts/link-coredumps.sh && false))' no_output_timeout: 30m environment: DUNE_PROFILE: nonconsensus_medium_curves - LIBP2P_NIXLESS: 1 GO: /usr/lib/go/bin/go - store_artifacts: path: core_dumps @@ -855,18 +838,17 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Run unit tests command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && export GO=/usr/lib/go/bin/go && make build && (dune runtest src/lib --profile=dev_medium_curves -j8 || (./scripts/link-coredumps.sh && false))' environment: DUNE_PROFILE: dev_medium_curves - LIBP2P_NIXLESS: 1 GO: /usr/lib/go/bin/go no_output_timeout: 1h - store_artifacts: @@ -900,12 +882,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Build OCaml command: | @@ -913,7 +895,6 @@ jobs: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'echo built > should_run_tests' environment: DUNE_PROFILE: dev - LIBP2P_NIXLESS: 1 GO: /usr/lib/go/bin/go - save_cache: name: Store config-agnostic test binaries @@ -937,7 +918,6 @@ jobs: name: Running test -- dev:coda-bootstrap-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-bootstrap-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -955,7 +935,6 @@ jobs: name: Running test -- dev:coda-shared-state-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-shared-state-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -973,7 +952,6 @@ jobs: name: Running test -- dev:coda-batch-payment-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-batch-payment-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -991,7 +969,6 @@ jobs: name: Running test -- dev:coda-peers-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-peers-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -1009,7 +986,6 @@ jobs: name: Running test -- dev:coda-transitive-peers-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-transitive-peers-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -1027,7 +1003,6 @@ jobs: name: Running test -- dev:coda-block-production-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-block-production-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -1045,7 +1020,6 @@ jobs: name: Running test -- dev:coda-shared-prefix-test -who-produces 0 command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-shared-prefix-test -who-produces 0"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -1063,7 +1037,6 @@ jobs: name: Running test -- dev:coda-shared-prefix-test -who-produces 1 command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-shared-prefix-test -who-produces 1"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -1081,7 +1054,6 @@ jobs: name: Running test -- dev:coda-change-snark-worker-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-change-snark-worker-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -1099,7 +1071,6 @@ jobs: name: Running test -- dev:coda-archive-node-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-archive-node-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -1117,7 +1088,6 @@ jobs: name: Running test -- dev:coda-delegation-test command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "dev:coda-delegation-test"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -1150,12 +1120,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_catchup:coda-restart-node-test command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_catchup:coda-restart-node-test"' @@ -1189,12 +1159,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_five_even_txns:coda-shared-prefix-multiproducer-test -num-block-producers 5 -payments command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_five_even_txns:coda-shared-prefix-multiproducer-test -num-block-producers 5 -payments"' @@ -1228,12 +1198,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_snarkless:full-test command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_snarkless:full-test"' @@ -1270,12 +1240,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_split:coda-shared-prefix-multiproducer-test -num-block-producers 2 command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_split:coda-shared-prefix-multiproducer-test -num-block-producers 2"' @@ -1309,12 +1279,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_three_producers:coda-txns-and-restart-non-producers command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_three_producers:coda-txns-and-restart-non-producers"' @@ -1348,12 +1318,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_full_epoch:full-test command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_full_epoch:full-test"' @@ -1387,12 +1357,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_medium_curves:full-test command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_medium_curves:full-test"' @@ -1431,12 +1401,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_snarkless_medium_curves:full-test command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_snarkless_medium_curves:full-test"' @@ -1473,12 +1443,12 @@ jobs: keys: - opam-linux-v1-{{ checksum "opam_ci_cache.sig" }} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Running test -- test_postake_split_medium_curves:coda-shared-prefix-multiproducer-test -num-block-producers 2 command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && ./scripts/test.py run --non-interactive --collect-artifacts --yes "test_postake_split_medium_curves:coda-shared-prefix-multiproducer-test -num-block-producers 2"' diff --git a/.circleci/config.yml.jinja b/.circleci/config.yml.jinja index 48230d3852a..9554ab372f3 100644 --- a/.circleci/config.yml.jinja +++ b/.circleci/config.yml.jinja @@ -20,8 +20,8 @@ keys: - opam-linux-v1-{{'{{'}} checksum "opam_ci_cache.sig" {{'}}'}} - run: - name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install opam dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' {%endset%} {% set checkout_no_lfs %} @@ -102,7 +102,7 @@ jobs: {{ opam_init_linux }} - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Set Up Database command: | @@ -152,7 +152,7 @@ jobs: no_output_timeout: 1h - run: name: Build Archive Process - command: bash -c 'eval `opam config env` && export PATH="$HOME/.cargo/bin:$PATH" && LIBP2P_NIXLESS=1 make build_archive' + command: bash -c 'eval `opam config env` && export PATH="$HOME/.cargo/bin:$PATH" && make build_archive' environment: DUNE_PROFILE: testnet_postake_medium_curves # NOTE: If we were using a machine executor we would be able to mount the sql file in @@ -181,10 +181,10 @@ jobs: command: ./scripts/test.py render --check .circleci/config.yml.jinja .mergify.yml.jinja - run: name: OCamlformat (make check-format) - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'eval `opam config env` && LIBP2P_NIXLESS=1 make check-format' + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'eval `opam config env` && make check-format' - run: name: Snarky tracks master (make check-snarky-submodule) - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make check-snarky-submodule' + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make check-snarky-submodule' - run: name: Check ppx_optcomp preprocessor_deps command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c './scripts/lint_preprocessor_deps.sh' @@ -307,8 +307,8 @@ jobs: - opam-v8-{{'{{'}} checksum "opam_ci_cache.sig" {{'}}'}}-{{'{{'}} checksum "scripts/Brewfile" {{'}}'}} - run: - name: Install macos dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam' + name: Install macos dependencies - opam -- make setup-opam + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam' - save_cache: name: Save cache - opam key: opam-v8-{{'{{'}} checksum "opam_ci_cache.sig" {{'}}'}}-{{'{{'}} checksum "scripts/Brewfile" {{'}}'}} @@ -316,19 +316,11 @@ jobs: paths: - "/Users/distiller/.opam" no_output_timeout: 1h - ### nix/cachix - - run: - name: Install nix and cachix - command: | - curl -L https://nixos.org/nix/install | sh - . ~/.nix-profile/etc/profile.d/nix.sh - nix-env -iA cachix -f https://cachix.org/api/v1/install ### dune - run: name: Build ocaml environment: DUNE_PROFILE: testnet_postake_medium_curves - EXTRA_NIX_ARGS: --option sandbox false command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && make build 2>&1 | tee /tmp/buildocaml.log' no_output_timeout: 20m - run: @@ -358,7 +350,6 @@ jobs: echo 'libp2p_helper' cp src/app/libp2p_helper/result/bin/libp2p_helper package/coda-libp2p_helper chmod +w package/coda-libp2p_helper - ./scripts/librewrite-macos.sh package/coda-libp2p_helper echo 'coda-logproc' cp _build/default/src/app/logproc/logproc.exe package/coda-logproc chmod +wx package/coda-logproc @@ -389,7 +380,7 @@ jobs: - run: name: Build portable binary command: | - LIBP2P_NIXLESS=1 make macos-portable + make macos-portable cp _build/coda-daemon-macos.zip package/. - run: name: Copy artifacts to cloud @@ -410,7 +401,7 @@ jobs: {{ opam_init_linux }} - run: name: Build client SDK for Javascript - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && LIBP2P_NIXLESS=1 make client_sdk 2>&1 | tee /tmp/artifacts/buildclientsdk.log' + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && make client_sdk 2>&1 | tee /tmp/artifacts/buildclientsdk.log' environment: DUNE_PROFILE: nonconsensus_medium_curves no_output_timeout: 10m @@ -435,15 +426,14 @@ jobs: {{ opam_init_linux }} - run: name: Build libp2p_helper - command: GO=/usr/lib/go/bin/go LIBP2P_NIXLESS=1 make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper # Explicitly generate PV-keys and uploading before building # See https://bkase.dev/posts/ocaml-writer#fn-3 for rationale - run: name: Generate PV keys - command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && PATH="$HOME/.cargo/bin:$PATH" LIBP2P_NIXLESS=1 make build_pv_keys 2>&1 | tee /tmp/artifacts/buildocaml.log' + command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && PATH="$HOME/.cargo/bin:$PATH" make build_pv_keys 2>&1 | tee /tmp/artifacts/buildocaml.log' environment: DUNE_PROFILE: {{profile}} - EXTRA_NIX_ARGS: --option sandbox false no_output_timeout: 20m - run: name: Upload generated PV keys @@ -455,9 +445,7 @@ jobs: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'eval `opam config env` && export PATH="$HOME/.cargo/bin:$PATH" && dune build src/app/generate_keypair/generate_keypair.exe' environment: DUNE_PROFILE: {{profile}} - EXTRA_NIX_ARGS: --option sandbox false GO: /usr/lib/go/bin/go - LIBP2P_NIXLESS: 1 no_output_timeout: 20m - run: @@ -477,18 +465,15 @@ jobs: command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make deb' environment: DUNE_PROFILE: {{profile}} - LIBP2P_NIXLESS: 1 no_output_timeout: 20m - run: name: Store genesis public/private keypairs command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make genesiskeys' environment: - LIBP2P_NIXLESS: 1 - run: name: Upload deb to repo command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make publish_deb' environment: - LIBP2P_NIXLESS: 1 no_output_timeout: 20m - run: name: Copy artifacts to cloud @@ -555,13 +540,12 @@ jobs: {{ opam_init_linux }} - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Run unit tests command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && make build && (dune runtest src/lib --profile={{profile}} -j8 || (./scripts/link-coredumps.sh && false))' environment: DUNE_PROFILE: {{profile}} - LIBP2P_NIXLESS: 1 GO: /usr/lib/go/bin/go no_output_timeout: 30m - store_artifacts: @@ -579,14 +563,13 @@ jobs: {{ opam_init_linux }} - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Run unit tests command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && (dune runtest src/nonconsensus --profile=nonconsensus_medium_curves -j8 || (./scripts/link-coredumps.sh && false))' no_output_timeout: 30m environment: DUNE_PROFILE: nonconsensus_medium_curves - LIBP2P_NIXLESS: 1 GO: /usr/lib/go/bin/go - store_artifacts: path: core_dumps @@ -602,13 +585,12 @@ jobs: {{ opam_init_linux }} - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Run unit tests command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && export GO=/usr/lib/go/bin/go && make build && (dune runtest src/lib --profile={{profile}} -j8 || (./scripts/link-coredumps.sh && false))' environment: DUNE_PROFILE: {{profile}} - LIBP2P_NIXLESS: 1 GO: /usr/lib/go/bin/go no_output_timeout: 1h - store_artifacts: @@ -626,7 +608,7 @@ jobs: {{ opam_init_linux }} - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper - run: name: Build OCaml command: | @@ -634,7 +616,6 @@ jobs: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'echo built > should_run_tests' environment: DUNE_PROFILE: {{profile}} - LIBP2P_NIXLESS: 1 GO: /usr/lib/go/bin/go - save_cache: name: Store config-agnostic test binaries @@ -660,7 +641,6 @@ jobs: name: Running test -- {{profile}}:{{test}} command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "{{profile}}:{{test}}"; fi environment: - LIBP2P_NIXLESS: 1 CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper GO: /usr/lib/go/bin/go - store_artifacts: @@ -679,7 +659,7 @@ jobs: {{ opam_init_linux }} - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper {%- for test in small_curves_tests[profile] %} - run: name: Running test -- {{profile}}:{{test}} @@ -699,7 +679,7 @@ jobs: {{ opam_init_linux }} - run: name: Build libp2p_helper - command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper + command: GO=/usr/lib/go/bin/go make libp2p_helper {%- for test in medium_curves_and_other_tests[profile] %} - run: name: Running test -- {{profile}}:{{test}} diff --git a/Makefile b/Makefile index 3fa79ff1b4d..e5e273c409e 100644 --- a/Makefile +++ b/Makefile @@ -65,14 +65,8 @@ clean: @rm -rf src/$(COVERAGE_DIR) # TEMP HACK (for circle-ci) -ifeq ($(LIBP2P_NIXLESS),1) libp2p_helper: $(WRAPAPP) bash -c "set -e && cd src/app/libp2p_helper && rm -rf result && mkdir -p result/bin && cd src && $(GO) mod download && cd .. && for f in generate_methodidx libp2p_helper; do cd src/\$$f && $(GO) build; cp \$$f ../../result/bin/\$$f; cd ../../; done" -else -libp2p_helper: - $(WRAPAPP) bash -c "set -o pipefail ; if [ -z \"$${USER}\" ]; then export USER=opam ; fi && source ~/.nix-profile/etc/profile.d/nix.sh && (if [ -z \"$${CACHIX_SIGNING_KEY+x}\" ]; then cd src/app/libp2p_helper && nix-build $${EXTRA_NIX_ARGS} default.nix; else cachix use codaprotocol && cd src/app/libp2p_helper && nix-build $${EXTRA_NIX_ARGS} default.nix | cachix push codaprotocol ; fi)" -endif - GENESIS_DIR := $(TMPDIR)/coda_cache_dir diff --git a/README-dev.md b/README-dev.md index 19b09ecc723..9ebf0bcc225 100644 --- a/README-dev.md +++ b/README-dev.md @@ -26,8 +26,8 @@ The short version: - This might fail with `git@github.com: Permission denied (publickey).`. If that happens it means you need to [set up SSH keys on your machine](https://help.github.com/en/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent). 4. Install Docker, GNU make, and bash - 5. `make LIBP2P_NIXLESS=1 USEDOCKER=TRUE dev` - 6. `make LIBP2P_NIXLESS=1 USEDOCKER=TRUE deb` + 5. `make USEDOCKER=TRUE dev` + 6. `make USEDOCKER=TRUE deb` Now you'll have a `src/_build/codaclient.deb` ready to install on Ubuntu or Debian! @@ -41,8 +41,6 @@ of the repo. ### Developer Setup (MacOS) * Invoke `make macos-setup` -* Make sure [`nix` is installed](https://nixos.org/download.html) (if you are running the latest macOS, [this workaround](https://medium.com/@robinbb/install-nix-on-macos-catalina-ca8c03a225fc) or a similar one will be necessary). -* Wait a long time... * Invoke `make build` * Jump to [customizing your editor for autocomplete](#dev-env) @@ -153,10 +151,6 @@ you need, you run `opam switch import src/opam.export`. Some of our dependencies aren't taken from `opam`, and aren't integrated with `dune`, so you need to add them manually, by running `scripts/pin-external-packages.sh`. -You will need to install [Nix](https://nixos.org/nix/download.html) and also -[cachix](https://cachix.org) with `nix-env -iA cachix -f -https://cachix.org/api/v1/install`. - There are a variety of C libraries we expect to be available in the system. These are also listed in the dockerfiles. Unlike most of the C libraries, which are installed using `apt` in the dockerfiles, the libraries for RocksDB are diff --git a/buildkite/agent.nix b/buildkite/agent.nix deleted file mode 100644 index f85ec564a40..00000000000 --- a/buildkite/agent.nix +++ /dev/null @@ -1,104 +0,0 @@ -# Minimal nix config for describing agent deps -# -# To generate a local environment to test your scripts or to run an agent (for -# example): -# -# nix-shell shell.nix # optionally add --pure to _only_ use these packages -# buildkite-agent start --tags size=small --token --build-path ~/buildkite -# -# Replace `tags` with anything relevant and token with the buildkite token for O(1) Labs -# -# -# To generate a docker image: -# -# Make sure you're in a linux environment with nix installed (not macOS), and pointing to the nixpkgs-unstable channel: -# For example: -# -# docker run -v $PWD:/workdir -it nixos/nix -# nix-channel --add https://nixos.org/channels/nixpkgs-unstable nixpkgs -# nix-channel --update -# cd /workdir -# -# Then build the docker image: -# -# nix-build docker.nix -# -# Note: The first built has to download things, but subsequent builds will cache -# hit no matter the order of the packages in the below array. -# -# Finally, copy the built image to your host OS (if necessary) and load it: -# -# export CONTAINER_ID=$(docker ps | grep nixos | awk '{ print $1 }') -# docker cp ${CONTAINER_ID}: image.tar.gz -# docker load < image.tar.gz -# -# You will now see: -# -# docker images -# # codaprotocol/ci-toolchain-base, latest, ... -# -# If you'd like you can push it up: -# -# docker push codaprotocol/ci-toolchain-base:latest -# - -{ pkgs ? import {} }: - -# Stick deps in here -let - deps = with pkgs; [ - # buildkite - buildkite-agent - - # ssl - cacert - - # dhall - dhall - dhall-json - - # shell stuff - bash - git - coreutils - gnused - gnugrep - findutils - diffutils - gnumake - ]; -in - # little nix derivation that caches dhall - # Note: `make check` runs in the buildPhase to make sure to fully - # populate the dhall cache -let - dhallCache = pkgs.stdenv.mkDerivation { - name = "dhall-ci-cache-0.0.1"; - src = ./.; - buildInputs = [ pkgs.dhall ]; - buildPhase = '' - #!${pkgs.runtimeShell} - mkdir -p $out/.cache - export XDG_CACHE_HOME=$out/.cache - make check - ''; - installPhase = "echo 'skip'"; - }; -in -{ - shell = - pkgs.mkShell { - buildInputs = deps; - shellHook = '' - mkdir -p ~/buildkite - export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt - export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - ''; - }; - docker = - pkgs.dockerTools.buildLayeredImage { - name = "codaprotocol/ci-toolchain-base"; - tag = "latest"; - contents = deps ++ [ dhallCache ]; - }; -} diff --git a/buildkite/docker.nix b/buildkite/docker.nix deleted file mode 100644 index 51a30662930..00000000000 --- a/buildkite/docker.nix +++ /dev/null @@ -1 +0,0 @@ -(import ./agent.nix {}).docker diff --git a/buildkite/shell.nix b/buildkite/shell.nix deleted file mode 100644 index 3b6ded9890c..00000000000 --- a/buildkite/shell.nix +++ /dev/null @@ -1 +0,0 @@ -(import ./agent.nix {}).shell diff --git a/buildkite/src/Command/Libp2pHelperBuild.dhall b/buildkite/src/Command/Libp2pHelperBuild.dhall index c00296cb73c..c3bafa3e2cd 100644 --- a/buildkite/src/Command/Libp2pHelperBuild.dhall +++ b/buildkite/src/Command/Libp2pHelperBuild.dhall @@ -12,7 +12,7 @@ let commands : List Cmd.Type = Cmd.runInDocker Cmd.Docker::{ image = (../Constants/ContainerImages.dhall).codaToolchain, - extraEnv = [ "LIBP2P_NIXLESS=1", "GO=/usr/lib/go/bin/go" ] + extraEnv = [ "GO=/usr/lib/go/bin/go" ] } "make libp2p_helper", Cmd.run "cd src/app/libp2p_helper/result/bin/ && buildkite-agent artifact upload libp2p_helper" diff --git a/buildkite/src/Jobs/ArchiveNode.dhall b/buildkite/src/Jobs/ArchiveNode.dhall index a9d1ad41659..672bd73996c 100644 --- a/buildkite/src/Jobs/ArchiveNode.dhall +++ b/buildkite/src/Jobs/ArchiveNode.dhall @@ -44,7 +44,7 @@ Pipeline.build , WithCargo.withCargo "eval $(opam config env) && dune runtest src/app/archive" , "PGPASSWORD=codarules psql -h localhost -p 5432 -U admin -d archiver -a -f src/app/archive/drop_tables.sql" , "PGPASSWORD=${password} psql -h localhost -p 5432 -U ${user} -d ${db} -a -f src/app/archive/create_schema.sql" - , "LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper" + , "GO=/usr/lib/go/bin/go make libp2p_helper" , "./scripts/test.py run --non-interactive --collect-artifacts --yes --out-dir 'test_output' 'test_archive_processor:coda-archive-processor-test'" ]) , label = "Archive-node unit tests" diff --git a/dockerfiles/Dockerfile-toolchain b/dockerfiles/Dockerfile-toolchain index efbda3bb6ce..8d91e385241 100644 --- a/dockerfiles/Dockerfile-toolchain +++ b/dockerfiles/Dockerfile-toolchain @@ -78,9 +78,6 @@ RUN sudo git clone https://github.com/facebook/rocksdb -b v5.17.2 /rocksdb # This builds and installs just the static lib for us RUN cd /rocksdb && sudo make static_lib PORTABLE=1 -j$(nproc) && sudo cp librocksdb.a /usr/local/lib/librocksdb_coda.a && sudo rm -rf /rocksdb && sudo strip -S /usr/local/lib/librocksdb_coda.a -## clone Coda repo for the .nix-keys.gpg.asc - -RUN sudo git clone https://github.com/CodaProtocol/coda $CODA_DIR # OPAM dependencies RUN git -C /home/opam/opam-repository pull @@ -89,19 +86,9 @@ RUN opam update -y && opam upgrade -y RUN mkdir --mode=700 /home/opam/.gnupg RUN echo "disable-ipv6" >> /home/opam/.gnupg/dirmngr.conf -RUN sudo apt-get install gnupg2 && curl -o install-nix-2.3.2 https://releases.nixos.org/nix/nix-2.3.3/install && \ - curl -o install-nix-2.3.2.asc https://releases.nixos.org/nix/nix-2.3.3/install.asc && \ - gpg2 --import < $CODA_DIR/.nix-key.gpg.asc && \ - gpg2 --keyserver keyserver.ubuntu.com --recv-keys B541D55301270E0BCF15CA5D8170B4726D7198DE && \ - gpg2 --verify ./install-nix-2.3.2.asc && \ - sh ./install-nix-2.3.2 && \ - USER=opam . ~/.nix-profile/etc/profile.d/nix.sh && \ - nix-env --option sandbox false -iA cachix -f https://cachix.org/api/v1/install - # we used to delete the repo here # we now leave it intact, in case updating OPAM in CI requires pinned package sources -# TEMP HACK (for libp2p_helper nixless builds) RUN sudo apt-get install --yes wget RUN cd /usr/lib && sudo wget https://dl.google.com/go/go1.13.10.linux-amd64.tar.gz && sudo tar -xvf go1.13.10.linux-amd64.tar.gz && sudo rm go1.13.10.linux-amd64.tar.gz RUN echo 'export PATH="$PATH:/usr/lib/go/bin"' >> /home/opam/.bashrc diff --git a/dockerfiles/Dockerfile-toolchain-libp2p b/dockerfiles/Dockerfile-toolchain-libp2p deleted file mode 100644 index c5be1591f04..00000000000 --- a/dockerfiles/Dockerfile-toolchain-libp2p +++ /dev/null @@ -1,42 +0,0 @@ -FROM nixos/nix:latest - -# Simple toolchain & build container that compiles and patches a binary for coda use -ARG PKGNAME=libp2p_helper - -# Add OS tools -RUN apk add patchelf dpkg tar - -# Update nixkgs -RUN nix-channel --add https://nixos.org/channels/nixpkgs-unstable nixpkgs \ - && nix-channel --update \ - && nix-env -iA cachix -f https://cachix.org/api/v1/install - -# Copy source tree -ADD /src/app/libp2p_helper /src -WORKDIR /src - -# Generate a unique signature for the source tree path -RUN find . -type f -print0 | xargs -0 sha1sum | sort | sha1sum | cut -f 1 -d ' ' > /tmp/sha1sig \ - && cat /tmp/sha1sig - -# Build -RUN cachix use codaprotocol \ - && nix-build default.nix \ - && ls -l /src/result/bin - -# Adjust elf headers (de-nix) -RUN patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 /src/result/bin/${PKGNAME} - -# Deb staging -RUN mkdir -p /src/build/usr/local/bin \ - && cp /src/result/bin/${PKGNAME} /src/build//usr/local/bin/coda-${PKGNAME} \ - && DATE=$(date +%Y-%m-%d) \ - && sed -i "s/DATE/${DATE}/" /src/build/DEBIAN/control \ - && HASH=$(cat /tmp/sha1sig)\ - && sed -i "s/HASH/${HASH}/" /src/build/DEBIAN/control \ - && cat /src/build/DEBIAN/control - -# Build and copy deb -RUN dpkg-deb --build /src/build \ - && cp /src/build.deb /src/coda-${PKGNAME}.deb \ - && ls -l /src/*.deb diff --git a/scripts/librewrite-macos.sh b/scripts/librewrite-macos.sh deleted file mode 100755 index 490f25d1781..00000000000 --- a/scripts/librewrite-macos.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# xcode technique to rewrite dynamic lib links -# tweaks kademlia binaries built with nix to work with os installed libs - -check_file() { - mybin="$1" - if [ ! -f ${mybin} ]; then - echo "ERROR: ${mybin} missing" - exit 1 - fi -} - -# Allow for specific target -TARGET=${1:-kademlia} - -# check for xcode tools -for mybin in /usr/bin/install_name_tool /usr/bin/otool -do - check_file $mybin -done - -BUILD_DIR_NAME=package - -# Swap nix libs for brew libs (ugly hack) -rewrite_lib() { - TARGET="$1" - LIB_NAME="$2" - NEW_PATH="$3" - # build env may not have these libs - #check_file ${NEW_PATH} - OLD_PATH=$(/usr/bin/otool -X -L ${TARGET} | grep "${LIB_NAME}" | grep '/nix' | awk '{print $1}') - echo "Updating ${TARGET} - rewriting ${OLD_PATH} to ${NEW_PATH}" - /usr/bin/install_name_tool -change "$OLD_PATH" "$NEW_PATH" "${TARGET}" -} - -# rewrite the libs -rewrite_lib $TARGET 'libgmp' "/usr/local/opt/gmp/lib/libgmp.10.dylib" -rewrite_lib $TARGET 'libffi' "/usr/local/opt/libffi/lib/libffi.6.dylib" -rewrite_lib $TARGET 'libSystem' "/usr/lib/libSystem.B.dylib" -rewrite_lib $TARGET 'libiconv' "/usr/lib/libiconv.dylib" \ No newline at end of file