From 2b690b4c719ac60aa3e2aa0ec3182f384ed03f0f Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Wed, 6 Nov 2024 22:22:17 +0000 Subject: [PATCH 01/11] chore: support parallel library generation --- hermetic_build/library_generation/generate_library.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index 313b8243f8..5078f91cb1 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -115,7 +115,7 @@ if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -temp_destination_path="${output_folder}/temp_preprocessed" +temp_destination_path="${output_folder}/temp_preprocessed/$(uuidgen)" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run From 622767c18c0c872e92d8f9cf6b6b03d6ff0bdb80 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Wed, 6 Nov 2024 23:39:47 +0000 Subject: [PATCH 02/11] install tool --- .cloudbuild/library_generation/library_generation.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 0c8442cb93..376f1bbece 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -62,7 +62,7 @@ ENV HOME=/home ENV OS_ARCHITECTURE="linux-x86_64" # install OS tools -RUN apk update && apk add unzip curl rsync openjdk11 jq bash nodejs npm git +RUN apk update && apk add unzip curl rsync openjdk11 jq bash nodejs npm git uuidgen SHELL [ "/bin/bash", "-c" ] From 4883c0a9e4e9a7c4ac09af0335130e2fbf473a01 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 7 Nov 2024 15:37:00 -0500 Subject: [PATCH 03/11] use buildin func --- .cloudbuild/library_generation/library_generation.Dockerfile | 2 +- hermetic_build/library_generation/generate_library.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 376f1bbece..0c8442cb93 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -62,7 +62,7 @@ ENV HOME=/home ENV OS_ARCHITECTURE="linux-x86_64" # install OS tools -RUN apk update && apk add unzip curl rsync openjdk11 jq bash nodejs npm git uuidgen +RUN apk update && apk add unzip curl rsync openjdk11 jq bash nodejs npm git SHELL [ "/bin/bash", "-c" ] diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index 5078f91cb1..aa0f10a92c 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -114,8 +114,8 @@ fi if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi - -temp_destination_path="${output_folder}/temp_preprocessed/$(uuidgen)" +random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 10 | head -n 1) +temp_destination_path="${output_folder}/temp_preprocessed/${random_suffix}" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run From bd36a266af6725482ee0238d9c76b22cc8e4d555 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 7 Nov 2024 15:52:22 -0500 Subject: [PATCH 04/11] debug --- hermetic_build/library_generation/generate_library.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index aa0f10a92c..53dfa0c792 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eo pipefail +set -xeo pipefail # parse input parameters while [[ $# -gt 0 ]]; do From e6d3a4a8d7f77bb673ae2decc9cf8f5a3a243bd6 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 7 Nov 2024 21:02:07 +0000 Subject: [PATCH 05/11] do not generate num --- hermetic_build/library_generation/generate_library.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index 53dfa0c792..2579bc212d 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -114,7 +114,7 @@ fi if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 10 | head -n 1) +random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | fold -w 20 | head -n 1) temp_destination_path="${output_folder}/temp_preprocessed/${random_suffix}" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then From a24cc713886eea89ed76be43f6a60d54498ce30c Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 7 Nov 2024 21:14:12 +0000 Subject: [PATCH 06/11] debug --- hermetic_build/library_generation/generate_library.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index 2579bc212d..da36178d71 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -115,7 +115,9 @@ if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | fold -w 20 | head -n 1) +echo "1" temp_destination_path="${output_folder}/temp_preprocessed/${random_suffix}" +echo "2" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run From 664d976a209ec5dda993e5c498ad54dcbb090dc2 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 7 Nov 2024 21:25:53 +0000 Subject: [PATCH 07/11] change cmd --- hermetic_build/library_generation/generate_library.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index da36178d71..32779b1fe9 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -114,10 +114,8 @@ fi if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | fold -w 20 | head -n 1) -echo "1" +random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | fold -w 20 | awk 'NR==1 {print; exit}') temp_destination_path="${output_folder}/temp_preprocessed/${random_suffix}" -echo "2" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run From cd662fe50b00326d46a0d37e112c75b52645e967 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 7 Nov 2024 22:40:34 +0000 Subject: [PATCH 08/11] change cmd --- hermetic_build/library_generation/generate_library.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index 32779b1fe9..69a701c76a 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -114,7 +114,7 @@ fi if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | fold -w 20 | awk 'NR==1 {print; exit}') +random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | head -c 30 | fold -w 20 | head -n 1) temp_destination_path="${output_folder}/temp_preprocessed/${random_suffix}" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then From 561d37588ebec7b00a7f4ac75e2acb3ac5397507 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 7 Nov 2024 22:49:18 +0000 Subject: [PATCH 09/11] change cmd --- hermetic_build/library_generation/generate_library.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index 69a701c76a..a89e4f6faf 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -114,7 +114,7 @@ fi if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | head -c 30 | fold -w 20 | head -n 1) +random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | head -c 20) temp_destination_path="${output_folder}/temp_preprocessed/${random_suffix}" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then From 8ba432d057592a913a26b9795d4de4d1a193bfb0 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 7 Nov 2024 23:02:22 +0000 Subject: [PATCH 10/11] change cmd --- hermetic_build/library_generation/generate_library.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index a89e4f6faf..f03fa517e8 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -114,8 +114,8 @@ fi if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -random_suffix=$(LC_ALL=C tr -dc 'a-zA-Z' < /dev/urandom | head -c 20) -temp_destination_path="${output_folder}/temp_preprocessed/${random_suffix}" + +temp_destination_path="${output_folder}/temp_preprocessed-$RANDOM" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run From 96aaadfc5e59edb42bc698e46d1a1723401724b2 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 7 Nov 2024 23:15:56 +0000 Subject: [PATCH 11/11] remove debug --- hermetic_build/library_generation/generate_library.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index f03fa517e8..f5cae1ba5a 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -xeo pipefail +set -eo pipefail # parse input parameters while [[ $# -gt 0 ]]; do