Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove LIBP2P_NIXLESS #5618

Merged
merged 5 commits into from
Aug 13, 2020
Merged

Remove LIBP2P_NIXLESS #5618

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4d54b0e
Remove LIBP2P_NIXLESS
mrmr1993 Aug 11, 2020
2e59efe
Merge branch 'develop' into feature/remove-libp2p-nixless
mrmr1993 Aug 12, 2020
b58d2ab
Remove remaining references to, uses of nix
mrmr1993 Aug 12, 2020
4ef6514
Merge branch 'develop' into feature/remove-libp2p-nixless
mrmr1993 Aug 12, 2020
d48ae5d
Merge branch 'develop' into feature/remove-libp2p-nixless
mrmr1993 Aug 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 63 additions & 82 deletions .circleci/config.yml
View file
Open in desktop

Large diffs are not rendered by default.

45 changes: 18 additions & 27 deletions .circleci/config.yml.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
keys:
- opam-linux-v1-{{'{{'}} checksum "opam_ci_cache.sig" {{'}}'}}
- run:
name: Install opam dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam'
name: Install opam dependencies - opam -- make setup-opam
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam'
{%endset%}

{% set checkout_no_lfs %}
Expand Down Expand Up @@ -102,7 +102,7 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build libp2p_helper
command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper
command: GO=/usr/lib/go/bin/go make libp2p_helper
- run:
name: Set Up Database
command: |
Expand Down Expand Up @@ -152,7 +152,7 @@ jobs:
no_output_timeout: 1h
- run:
name: Build Archive Process
command: bash -c 'eval `opam config env` && export PATH="$HOME/.cargo/bin:$PATH" && LIBP2P_NIXLESS=1 make build_archive'
command: bash -c 'eval `opam config env` && export PATH="$HOME/.cargo/bin:$PATH" && make build_archive'
environment:
DUNE_PROFILE: testnet_postake_medium_curves
# NOTE: If we were using a machine executor we would be able to mount the sql file in
Expand Down Expand Up @@ -181,10 +181,10 @@ jobs:
command: ./scripts/test.py render --check .circleci/config.yml.jinja .mergify.yml.jinja
- run:
name: OCamlformat (make check-format)
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'eval `opam config env` && LIBP2P_NIXLESS=1 make check-format'
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'eval `opam config env` && make check-format'
- run:
name: Snarky tracks master (make check-snarky-submodule)
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make check-snarky-submodule'
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make check-snarky-submodule'
- run:
name: Check ppx_optcomp preprocessor_deps
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c './scripts/lint_preprocessor_deps.sh'
Expand Down Expand Up @@ -307,8 +307,8 @@ jobs:
- opam-v8-{{'{{'}} checksum "opam_ci_cache.sig" {{'}}'}}-{{'{{'}} checksum "scripts/Brewfile" {{'}}'}}

- run:
name: Install macos dependencies - opam -- LIBP2P_NIXLESS=1 make setup-opam
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'LIBP2P_NIXLESS=1 make setup-opam'
name: Install macos dependencies - opam -- make setup-opam
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make setup-opam'
- save_cache:
name: Save cache - opam
key: opam-v8-{{'{{'}} checksum "opam_ci_cache.sig" {{'}}'}}-{{'{{'}} checksum "scripts/Brewfile" {{'}}'}}
Expand Down Expand Up @@ -389,7 +389,7 @@ jobs:
- run:
name: Build portable binary
command: |
LIBP2P_NIXLESS=1 make macos-portable
make macos-portable
cp _build/coda-daemon-macos.zip package/.
- run:
name: Copy artifacts to cloud
Expand All @@ -410,7 +410,7 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build client SDK for Javascript
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && LIBP2P_NIXLESS=1 make client_sdk 2>&1 | tee /tmp/artifacts/buildclientsdk.log'
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && make client_sdk 2>&1 | tee /tmp/artifacts/buildclientsdk.log'
environment:
DUNE_PROFILE: nonconsensus_medium_curves
no_output_timeout: 10m
Expand All @@ -435,12 +435,12 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build libp2p_helper
command: GO=/usr/lib/go/bin/go LIBP2P_NIXLESS=1 make libp2p_helper
command: GO=/usr/lib/go/bin/go make libp2p_helper
# Explicitly generate PV-keys and uploading before building
# See https://bkase.dev/posts/ocaml-writer#fn-3 for rationale
- run:
name: Generate PV keys
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && PATH="$HOME/.cargo/bin:$PATH" LIBP2P_NIXLESS=1 make build_pv_keys 2>&1 | tee /tmp/artifacts/buildocaml.log'
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'set -o pipefail; eval `opam config env` && PATH="$HOME/.cargo/bin:$PATH" make build_pv_keys 2>&1 | tee /tmp/artifacts/buildocaml.log'
environment:
DUNE_PROFILE: {{profile}}
EXTRA_NIX_ARGS: --option sandbox false
Expand All @@ -457,7 +457,6 @@ jobs:
DUNE_PROFILE: {{profile}}
EXTRA_NIX_ARGS: --option sandbox false
GO: /usr/lib/go/bin/go
LIBP2P_NIXLESS: 1

no_output_timeout: 20m
- run:
Expand All @@ -477,18 +476,15 @@ jobs:
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make deb'
environment:
DUNE_PROFILE: {{profile}}
LIBP2P_NIXLESS: 1
no_output_timeout: 20m
- run:
name: Store genesis public/private keypairs
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make genesiskeys'
environment:
LIBP2P_NIXLESS: 1
- run:
name: Upload deb to repo
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'make publish_deb'
environment:
LIBP2P_NIXLESS: 1
no_output_timeout: 20m
- run:
name: Copy artifacts to cloud
Expand Down Expand Up @@ -555,13 +551,12 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build libp2p_helper
command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper
command: GO=/usr/lib/go/bin/go make libp2p_helper
- run:
name: Run unit tests
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && make build && (dune runtest src/lib --profile={{profile}} -j8 || (./scripts/link-coredumps.sh && false))'
environment:
DUNE_PROFILE: {{profile}}
LIBP2P_NIXLESS: 1
GO: /usr/lib/go/bin/go
no_output_timeout: 30m
- store_artifacts:
Expand All @@ -579,14 +574,13 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build libp2p_helper
command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper
command: GO=/usr/lib/go/bin/go make libp2p_helper
- run:
name: Run unit tests
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && (dune runtest src/nonconsensus --profile=nonconsensus_medium_curves -j8 || (./scripts/link-coredumps.sh && false))'
no_output_timeout: 30m
environment:
DUNE_PROFILE: nonconsensus_medium_curves
LIBP2P_NIXLESS: 1
GO: /usr/lib/go/bin/go
- store_artifacts:
path: core_dumps
Expand All @@ -602,13 +596,12 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build libp2p_helper
command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper
command: GO=/usr/lib/go/bin/go make libp2p_helper
- run:
name: Run unit tests
command: ./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && export GO=/usr/lib/go/bin/go && make build && (dune runtest src/lib --profile={{profile}} -j8 || (./scripts/link-coredumps.sh && false))'
environment:
DUNE_PROFILE: {{profile}}
LIBP2P_NIXLESS: 1
GO: /usr/lib/go/bin/go
no_output_timeout: 1h
- store_artifacts:
Expand All @@ -626,15 +619,14 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build libp2p_helper
command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper
command: GO=/usr/lib/go/bin/go make libp2p_helper
- run:
name: Build OCaml
command: |
./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'source ~/.profile && make build'
./scripts/skip_if_only_frontend_or_rfcs.sh bash -c 'echo built > should_run_tests'
environment:
DUNE_PROFILE: {{profile}}
LIBP2P_NIXLESS: 1
GO: /usr/lib/go/bin/go
- save_cache:
name: Store config-agnostic test binaries
Expand All @@ -660,7 +652,6 @@ jobs:
name: Running test -- {{profile}}:{{test}}
command: if [ -f should_run_tests ]; then source ~/.profile && ./scripts/test.py run --no-build --non-interactive --collect-artifacts --yes "{{profile}}:{{test}}"; fi
environment:
LIBP2P_NIXLESS: 1
CODA_LIBP2P_HELPER_PATH: /home/opam/project/src/app/libp2p_helper/result/bin/libp2p_helper
GO: /usr/lib/go/bin/go
- store_artifacts:
Expand All @@ -679,7 +670,7 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build libp2p_helper
command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper
command: GO=/usr/lib/go/bin/go make libp2p_helper
{%- for test in small_curves_tests[profile] %}
- run:
name: Running test -- {{profile}}:{{test}}
Expand All @@ -699,7 +690,7 @@ jobs:
{{ opam_init_linux }}
- run:
name: Build libp2p_helper
command: LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper
command: GO=/usr/lib/go/bin/go make libp2p_helper
{%- for test in medium_curves_and_other_tests[profile] %}
- run:
name: Running test -- {{profile}}:{{test}}
Expand Down
6 changes: 0 additions & 6 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,8 @@ clean:
@rm -rf src/$(COVERAGE_DIR)

# TEMP HACK (for circle-ci)
ifeq ($(LIBP2P_NIXLESS),1)
libp2p_helper:
$(WRAPAPP) bash -c "set -e && cd src/app/libp2p_helper && rm -rf result && mkdir -p result/bin && cd src && $(GO) mod download && cd .. && for f in generate_methodidx libp2p_helper; do cd src/\$$f && $(GO) build; cp \$$f ../../result/bin/\$$f; cd ../../; done"
else
libp2p_helper:
$(WRAPAPP) bash -c "set -o pipefail ; if [ -z \"$${USER}\" ]; then export USER=opam ; fi && source ~/.nix-profile/etc/profile.d/nix.sh && (if [ -z \"$${CACHIX_SIGNING_KEY+x}\" ]; then cd src/app/libp2p_helper && nix-build $${EXTRA_NIX_ARGS} default.nix; else cachix use codaprotocol && cd src/app/libp2p_helper && nix-build $${EXTRA_NIX_ARGS} default.nix | cachix push codaprotocol ; fi)"
endif



GENESIS_DIR := $(TMPDIR)/coda_cache_dir
Expand Down
4 changes: 2 additions & 2 deletions README-dev.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ The short version:
- This might fail with `[email protected]: Permission denied (publickey).`. If that happens it means
you need to [set up SSH keys on your machine](https://help.github.com/en/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent).
4. Install Docker, GNU make, and bash
5. `make LIBP2P_NIXLESS=1 USEDOCKER=TRUE dev`
6. `make LIBP2P_NIXLESS=1 USEDOCKER=TRUE deb`
5. `make USEDOCKER=TRUE dev`
6. `make USEDOCKER=TRUE deb`

Now you'll have a `src/_build/codaclient.deb` ready to install on Ubuntu or Debian!

Expand Down
1 change: 0 additions & 1 deletion buildkite/scripts/build-artifact.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ eval `opam config env`
export PATH=$HOME/.cargo/bin:$PATH

# TODO: Stop building lib_p2p multiple times (due to excessive dependencies in make)
export LIBP2P_NIXLESS=1

echo "--- Explicitly generate PV-keys and upload before building"
make build_pv_keys 2>&1 | tee /tmp/buildocaml.log
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Command/Libp2pHelperBuild.dhall
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ let commands : List Cmd.Type =
Cmd.runInDocker
Cmd.Docker::{
image = (../Constants/ContainerImages.dhall).codaToolchain,
extraEnv = [ "LIBP2P_NIXLESS=1", "GO=/usr/lib/go/bin/go" ]
extraEnv = [ "GO=/usr/lib/go/bin/go" ]
}
"make libp2p_helper",
Cmd.run "cd src/app/libp2p_helper/result/bin/ && buildkite-agent artifact upload libp2p_helper"
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Jobs/ArchiveNode.dhall
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ Pipeline.build
, WithCargo.withCargo "eval $(opam config env) && dune runtest src/app/archive"
, "PGPASSWORD=codarules psql -h localhost -p 5432 -U admin -d archiver -a -f src/app/archive/drop_tables.sql"
, "PGPASSWORD=${password} psql -h localhost -p 5432 -U ${user} -d ${db} -a -f src/app/archive/create_schema.sql"
, "LIBP2P_NIXLESS=1 GO=/usr/lib/go/bin/go make libp2p_helper"
, "GO=/usr/lib/go/bin/go make libp2p_helper"
, "./scripts/test.py run --non-interactive --collect-artifacts --yes --out-dir 'test_output' 'test_archive_processor:coda-archive-processor-test'"
])
, label = "Archive-node unit tests"
Expand Down