From a9decb67c7c6064f26941064cfff4aadd5060e7a Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 25 Mar 2024 15:05:43 +0000 Subject: [PATCH 1/9] fix(ci): fix ctest --- barretenberg/cpp/Earthfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 282362004d5..b21a725d3a9 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -138,8 +138,6 @@ test: BUILD +test-clang-format FROM +preset-release-assert-all COPY --dir ./srs_db/+build/. srs_db - # Fiddly bit: we need to reconstruct the paths that CMake built with - RUN mkdir -p build && mv bin CMakeCache.txt CMakeFiles build && cd build RUN GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure # Functions @@ -155,7 +153,11 @@ RUN_CMAKE: RUN --mount type=cache,id="$configure-build",target=/build/build \ (cmake $configure -Bbuild || (rm -f build/CMakeCache.txt && cmake $configure -Bbuild)) && \ cmake --build build $build && \ - cp -r build/CMakeCache.txt build/CMakeFiles build/bin . + cp -r build build-tmp + # move the temporary build folder back + # this is because the cached build goes away + RUN mv build-tmp build + WORKDIR /build/build BENCH_RELEASE: FUNCTION From 4a42402b3970faf74a649b4d02db78ed3f7fbf72 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 25 Mar 2024 15:10:40 +0000 Subject: [PATCH 2/9] fixes --- barretenberg/cpp/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index b21a725d3a9..80d4be540db 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -86,7 +86,7 @@ preset-wasm: # wasi-sdk COPY +get-wasi-sdk/wasi-sdk src/wasi-sdk DO +RUN_CMAKE --configure="--preset wasm-threads" --build="--target barretenberg.wasm" - RUN ./src/wasi-sdk/bin/llvm-strip ./bin/barretenberg.wasm + RUN ../src/wasi-sdk/bin/llvm-strip ./bin/barretenberg.wasm SAVE ARTIFACT bin SAVE IMAGE --cache-hint @@ -137,7 +137,7 @@ build: # default target test: BUILD +test-clang-format FROM +preset-release-assert-all - COPY --dir ./srs_db/+build/. srs_db + COPY --dir ./srs_db/+build/. ../srs_db RUN GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure # Functions From 51b569050931e1de16f48bb2109fb4c0c1e4fbae Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 25 Mar 2024 15:13:51 +0000 Subject: [PATCH 3/9] allow rerunning cmake --- barretenberg/cpp/Earthfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 80d4be540db..543649c19ed 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -142,6 +142,7 @@ test: # Functions RUN_CMAKE: + WORKDIR /build # Runs cmake build and leaves binary artifacts at 'bin'. Uses a mounted cache # for incremental rebuilds. FUNCTION @@ -156,7 +157,7 @@ RUN_CMAKE: cp -r build build-tmp # move the temporary build folder back # this is because the cached build goes away - RUN mv build-tmp build + RUN rm -rf build-tmp && mv build-tmp build WORKDIR /build/build BENCH_RELEASE: From 567de3f76a016a54aa97dda7afa61e78c6d41ac8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 25 Mar 2024 15:16:14 +0000 Subject: [PATCH 4/9] earthly restarts --- scripts/earthly-cloud | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/earthly-cloud b/scripts/earthly-cloud index d474dd1b533..ec559954913 100755 --- a/scripts/earthly-cloud +++ b/scripts/earthly-cloud @@ -63,12 +63,16 @@ elif ! earthly $EARTHLY_FLAGS $@ 2>&1 | tee .output.txt >&2 ; then # we got a failure, handle retries # TODO potentially handle other intermittent errors here if grep 'failed to get edge: inconsistent graph state' .output.txt >/dev/null ; then - # TODO we need to limit earthly parallelism or sometimes get + # TODO when earthly is overloaded we sometimes get # 'failed to solve: failed to get edge: inconsistent graph state' # so we use arbitrary WAIT statement groups. Pending earthly support. # for now we just try to recover from the failures. echo "Got 'inconsistent graph state'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'" earthly $EARTHLY_FLAGS $@ + elif grep 'unexpected content-type' .output.txt >/dev/null ; then + # TODO intermittent error it seems + echo "Got 'unexpected content-type'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'" + earthly $EARTHLY_FLAGS $@ else # otherwise, propagate error exit 1 From 51b093a0d95010d7a5b68fcff34deeebc074f98d Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 25 Mar 2024 15:16:40 +0000 Subject: [PATCH 5/9] fix --- barretenberg/cpp/Earthfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 543649c19ed..aece51235ef 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -142,13 +142,12 @@ test: # Functions RUN_CMAKE: - WORKDIR /build + FUNCTION # Runs cmake build and leaves binary artifacts at 'bin'. Uses a mounted cache # for incremental rebuilds. - FUNCTION ARG configure # cmake configure flags ARG build # cmake build flags - + WORKDIR /build # Use a mount for incremental builds locally. # TODO(AD): To be determined: does this get us in trouble in CI? RUN --mount type=cache,id="$configure-build",target=/build/build \ From f6fc58292b68ab6e24b3863512de87541ffd732c Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 25 Mar 2024 15:17:57 +0000 Subject: [PATCH 6/9] retries --- scripts/earthly-cloud | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/earthly-cloud b/scripts/earthly-cloud index ec559954913..df7ab46cf79 100755 --- a/scripts/earthly-cloud +++ b/scripts/earthly-cloud @@ -60,7 +60,8 @@ if $INTERACTIVE ; then # don't play nice with tee if interactive earthly $EARTHLY_FLAGS $@ elif ! earthly $EARTHLY_FLAGS $@ 2>&1 | tee .output.txt >&2 ; then - # we got a failure, handle retries + # we try earthly once, capturing output + # if we get one of our (unfortunate) known failures, handle retries # TODO potentially handle other intermittent errors here if grep 'failed to get edge: inconsistent graph state' .output.txt >/dev/null ; then # TODO when earthly is overloaded we sometimes get From b409f2a8593dd331bb0da1d2e81ee5a04d06906b Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 25 Mar 2024 15:22:06 +0000 Subject: [PATCH 7/9] woops --- barretenberg/cpp/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index aece51235ef..a04a862a106 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -156,7 +156,7 @@ RUN_CMAKE: cp -r build build-tmp # move the temporary build folder back # this is because the cached build goes away - RUN rm -rf build-tmp && mv build-tmp build + RUN rm -rf build && mv build-tmp build WORKDIR /build/build BENCH_RELEASE: From b16a8aaee30e7bbaef0f99f60f8bfcaf4cce9a2a Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 25 Mar 2024 13:56:17 -0400 Subject: [PATCH 8/9] Update earthly-cloud --- scripts/earthly-cloud | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/earthly-cloud b/scripts/earthly-cloud index df7ab46cf79..90815faa1d1 100755 --- a/scripts/earthly-cloud +++ b/scripts/earthly-cloud @@ -66,8 +66,6 @@ elif ! earthly $EARTHLY_FLAGS $@ 2>&1 | tee .output.txt >&2 ; then if grep 'failed to get edge: inconsistent graph state' .output.txt >/dev/null ; then # TODO when earthly is overloaded we sometimes get # 'failed to solve: failed to get edge: inconsistent graph state' - # so we use arbitrary WAIT statement groups. Pending earthly support. - # for now we just try to recover from the failures. echo "Got 'inconsistent graph state'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'" earthly $EARTHLY_FLAGS $@ elif grep 'unexpected content-type' .output.txt >/dev/null ; then @@ -78,4 +76,4 @@ elif ! earthly $EARTHLY_FLAGS $@ 2>&1 | tee .output.txt >&2 ; then # otherwise, propagate error exit 1 fi -fi \ No newline at end of file +fi From 5b80714887f8c078d93a42872d7aa42bc261832b Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 26 Mar 2024 12:10:05 -0400 Subject: [PATCH 9/9] Update earthly-cloud --- scripts/earthly-cloud | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/earthly-cloud b/scripts/earthly-cloud index 90815faa1d1..af39247965a 100755 --- a/scripts/earthly-cloud +++ b/scripts/earthly-cloud @@ -68,10 +68,6 @@ elif ! earthly $EARTHLY_FLAGS $@ 2>&1 | tee .output.txt >&2 ; then # 'failed to solve: failed to get edge: inconsistent graph state' echo "Got 'inconsistent graph state'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'" earthly $EARTHLY_FLAGS $@ - elif grep 'unexpected content-type' .output.txt >/dev/null ; then - # TODO intermittent error it seems - echo "Got 'unexpected content-type'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'" - earthly $EARTHLY_FLAGS $@ else # otherwise, propagate error exit 1