Skip to content

Commit

Permalink
feat: Build system handles dynamic deps first class. (#2283)
Browse files Browse the repository at this point in the history 
If no deps provided, query-manifest exectutes script in project, uses 1
line of jq to get deps for ts.
  • Loading branch information
@charlielye
charlielye authored Sep 13, 2023
1 parent 262dbb9 commit f66077a
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 174 deletions.
55 changes: 42 additions & 13 deletions build-system/scripts/query_manifest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,46 @@ if [ $(jq "has(\"$REPO\")" $MANIFEST) == "false" ]; then
exit 1
fi

function addRebuildPatterns {
local TYPE=$(jq -r ".\"$1\".rebuildPatterns | type" $MANIFEST)
if [ "$TYPE" == "string" ]; then
local FILE=$(jq -r ".\"$1\".rebuildPatterns" $MANIFEST)
local BUILD_DIR=$($0 buildDir $1)
PATTERNS=(${PATTERNS[@]} $(cat $BUILD_DIR/$FILE))
elif [ "$TYPE" == "array" ]; then
PATTERNS=(${PATTERNS[@]} $(jq -r ".\"$1\".rebuildPatterns | .[]" $MANIFEST))
else
>&2 echo "Missing rebuildPatterns property. Either filename as string, or patterns as array."
function get_deps {
local TYPE=$(jq -r ".\"$1\".dependencies | type" $MANIFEST)
if [ "$TYPE" == "string" ]; then
# Execute string as command relative to buildDir to retrieve dependencies.
local CMD=$BUILD_DIR/$(jq -r ".\"$1\".dependencies")
if [ ! -f "$CMD" ]; then
>&2 echo "Dependency script not found: $CMD"
exit 1
fi
local PROJECT_DIR=$($0 projectDir $1)
DEPS=($($CMD $PROJECT_DIR))
elif [ "$TYPE" == "null" ]; then
# Execute default script relative to buildDir to retrieve dependencies.
local CMD=$BUILD_DIR/scripts/get_dependencies.sh
if [ ! -f "$CMD" ]; then
>&2 echo "Dependency script not found: $CMD"
exit 1
fi
local PROJECT_DIR=$($0 projectDir $1)
DEPS=($($CMD $PROJECT_DIR))
elif [ "$TYPE" == "array" ]; then
DEPS=($(jq -r ".\"$1\".dependencies // [] | .[]" $MANIFEST))
else
>&2 echo "dependencies must be a array, string or null."
exit 1
fi
}

function add_rebuild_patterns {
local TYPE=$(jq -r ".\"$1\".rebuildPatterns | type" $MANIFEST)
if [ "$TYPE" == "string" ]; then
local FILE=$(jq -r ".\"$1\".rebuildPatterns" $MANIFEST)
local BUILD_DIR=$($0 buildDir $1)
PATTERNS=(${PATTERNS[@]} $(cat $BUILD_DIR/$FILE))
elif [ "$TYPE" == "array" ]; then
PATTERNS=(${PATTERNS[@]} $(jq -r ".\"$1\".rebuildPatterns | .[]" $MANIFEST))
else
>&2 echo "Missing rebuildPatterns property. Either filename as string, or patterns as array."
exit 1
fi
}

case "$CMD" in
Expand All @@ -46,13 +74,14 @@ case "$CMD" in
jq -r ".\"$REPO\".projectDir // .\"$REPO\".buildDir" $MANIFEST
;;
dependencies)
BUILD_DIR=$($0 buildDir $REPO)
declare -A ALL_DEPS
add_deps() {
if [[ -v ALL_DEPS[$1] ]]; then
return
fi
ALL_DEPS["$1"]=1
DEPS=($(jq -r ".\"$1\".dependencies // [] | .[]" $MANIFEST))
get_deps $1
for DEP in "${DEPS[@]}"; do
add_deps $DEP
done
Expand All @@ -65,9 +94,9 @@ case "$CMD" in
rebuildPatterns)
DEPS=($($0 dependencies $REPO))
PATTERNS=()
addRebuildPatterns $REPO
add_rebuild_patterns $REPO
for DEP in "${DEPS[@]}"; do
addRebuildPatterns $DEP
add_rebuild_patterns $DEP
done
printf "%s\n" "${PATTERNS[@]}" | sort | uniq
;;
Expand Down
184 changes: 23 additions & 161 deletions build_manifest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,168 +113,85 @@
"buildDir": "yarn-project",
"projectDir": "yarn-project/acir-simulator",
"dockerfile": "acir-simulator/Dockerfile",
"rebuildPatterns": ["^yarn-project/acir-simulator/"],
"dependencies": [
"yarn-project-base",
"circuits.js",
"foundation",
"types",
"merkle-tree",
"noir-contracts"
]
"rebuildPatterns": ["^yarn-project/acir-simulator/"]
},
"archiver": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/archiver",
"dockerfile": "archiver/Dockerfile",
"rebuildPatterns": ["^yarn-project/archiver/"],
"dependencies": [
"yarn-project-base",
"circuits.js",
"ethereum",
"foundation",
"l1-artifacts",
"types"
]
"rebuildPatterns": ["^yarn-project/archiver/"]
},
"cli": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/cli",
"dockerfile": "cli/Dockerfile",
"rebuildPatterns": ["^yarn-project/cli/"],
"dependencies": [
"yarn-project-base",
"aztec.js",
"ethereum",
"foundation",
"noir-compiler",
"noir-contracts",
"types"
]
"rebuildPatterns": ["^yarn-project/cli/"]
},
"aztec-rpc": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/aztec-rpc",
"dockerfile": "aztec-rpc/Dockerfile",
"rebuildPatterns": ["^yarn-project/aztec-rpc/"],
"dependencies": [
"yarn-project-base",
"acir-simulator",
"circuits.js",
"foundation",
"key-store",
"types"
]
"rebuildPatterns": ["^yarn-project/aztec-rpc/"]
},
"aztec-sandbox": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/aztec-sandbox",
"dockerfile": "aztec-sandbox/Dockerfile",
"rebuildPatterns": ["^yarn-project/aztec-sandbox/"],
"dependencies": [
"yarn-project-base",
"aztec-node",
"aztec-rpc",
"aztec.js",
"circuits.js",
"ethereum",
"foundation",
"l1-artifacts",
"noir-contracts",
"types"
]
"rebuildPatterns": ["^yarn-project/aztec-sandbox/"]
},
"aztec.js": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/aztec.js",
"dockerfile": "aztec.js/Dockerfile",
"rebuildPatterns": ["^yarn-project/aztec.js/"],
"dependencies": ["yarn-project-base", "circuits.js", "foundation", "types"]
"rebuildPatterns": ["^yarn-project/aztec.js/"]
},
"canary-build": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/canary",
"dockerfile": "canary/Dockerfile.build",
"rebuildPatterns": ["^yarn-project/canary/"],
"dependencies": [
"yarn-project-base",
"aztec.js",
"l1-artifacts",
"noir-contracts",
"aztec-sandbox"
]
"rebuildPatterns": ["^yarn-project/canary/"]
},
"canary": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/canary",
"dockerfile": "canary/Dockerfile",
"rebuildPatterns": ["^yarn-project/canary/"],
"dependencies": [
"yarn-project-base",
"aztec.js",
"foundation",
"l1-artifacts",
"noir-contracts"
]
"rebuildPatterns": ["^yarn-project/canary/"]
},
"circuits.js": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/circuits.js",
"dockerfile": "circuits.js/Dockerfile",
"rebuildPatterns": ["^yarn-project/circuits.js/"],
"dependencies": ["yarn-project-base", "foundation"]
"rebuildPatterns": ["^yarn-project/circuits.js/"]
},
"end-to-end": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/end-to-end",
"dockerfile": "end-to-end/Dockerfile",
"rebuildPatterns": ["^yarn-project/end-to-end/"],
"dependencies": [
"yarn-project-base",
"archiver",
"aztec-node",
"aztec-rpc",
"aztec-sandbox",
"aztec.js",
"circuits.js",
"cli",
"ethereum",
"foundation",
"l1-artifacts",
"noir-contracts",
"p2p",
"sequencer-client",
"types",
"world-state"
]
"rebuildPatterns": ["^yarn-project/end-to-end/"]
},
"ethereum": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/ethereum",
"dockerfile": "ethereum/Dockerfile",
"rebuildPatterns": ["^yarn-project/ethereum/"],
"dependencies": ["yarn-project-base", "foundation", "l1-artifacts"]
"rebuildPatterns": ["^yarn-project/ethereum/"]
},
"foundation": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/foundation",
"dockerfile": "foundation/Dockerfile",
"rebuildPatterns": ["^yarn-project/foundation/"],
"dependencies": ["yarn-project-base"]
"rebuildPatterns": ["^yarn-project/foundation/"]
},
"key-store": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/key-store",
"dockerfile": "key-store/Dockerfile",
"rebuildPatterns": ["^yarn-project/key-store/"],
"dependencies": ["yarn-project-base", "circuits.js", "foundation", "types"]
"rebuildPatterns": ["^yarn-project/key-store/"]
},
"merkle-tree": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/merkle-tree",
"dockerfile": "merkle-tree/Dockerfile",
"rebuildPatterns": ["^yarn-project/merkle-tree/"],
"dependencies": ["yarn-project-base", "circuits.js", "foundation", "types"]
"rebuildPatterns": ["^yarn-project/merkle-tree/"]
},
"noir-contracts-build": {
"buildDir": "yarn-project",
Expand All @@ -283,12 +200,6 @@
"rebuildPatterns": [
"^yarn-project/noir-contracts/",
"^yarn-project/noir-libs/"
],
"dependencies": [
"yarn-project-base",
"aztec.js",
"foundation",
"noir-compiler"
]
},
"noir-contracts": {
Expand All @@ -298,109 +209,60 @@
"rebuildPatterns": [
"^yarn-project/noir-contracts/",
"^yarn-project/noir-libs/"
],
"dependencies": [
"yarn-project-base",
"aztec.js",
"foundation",
"noir-compiler"
]
},
"noir-compiler": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/noir-compiler",
"dockerfile": "noir-compiler/Dockerfile",
"rebuildPatterns": ["^yarn-project/noir-compiler/"],
"dependencies": ["yarn-project-base", "foundation"]
"rebuildPatterns": ["^yarn-project/noir-compiler/"]
},
"p2p": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/p2p",
"dockerfile": "p2p/Dockerfile",
"rebuildPatterns": ["^yarn-project/p2p/"],
"dependencies": ["yarn-project-base", "circuits.js", "foundation", "types"]
"rebuildPatterns": ["^yarn-project/p2p/"]
},
"p2p-bootstrap": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/p2p-bootstrap",
"dockerfile": "p2p/Dockerfile",
"rebuildPatterns": ["^yarn-project/p2p-bootstrap/"],
"dependencies": ["yarn-project-base", "foundation", "p2p"]
"rebuildPatterns": ["^yarn-project/p2p-bootstrap/"]
},
"prover-client": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/prover-client",
"dockerfile": "prover-client/Dockerfile",
"rebuildPatterns": ["^yarn-project/prover-client/"],
"dependencies": ["yarn-project-base", "foundation"]
"rebuildPatterns": ["^yarn-project/prover-client/"]
},
"rollup-provider": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/rollup-provider",
"dockerfile": "rollup-provider/Dockerfile",
"rebuildPatterns": ["^yarn-project/rollup-provider/"],
"dependencies": [
"yarn-project-base",
"aztec-node",
"circuits.js",
"foundation",
"types"
]
"rebuildPatterns": ["^yarn-project/rollup-provider/"]
},
"aztec-node": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/aztec-node",
"dockerfile": "aztec-node/Dockerfile",
"rebuildPatterns": ["^yarn-project/aztec-node/"],
"dependencies": [
"yarn-project-base",
"archiver",
"circuits.js",
"foundation",
"l1-artifacts",
"merkle-tree",
"p2p",
"sequencer-client",
"types",
"world-state"
]
"rebuildPatterns": ["^yarn-project/aztec-node/"]
},
"sequencer-client": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/sequencer-client",
"dockerfile": "sequencer-client/Dockerfile",
"rebuildPatterns": ["^yarn-project/sequencer-client/"],
"dependencies": [
"yarn-project-base",
"acir-simulator",
"circuits.js",
"ethereum",
"foundation",
"l1-artifacts",
"merkle-tree",
"p2p",
"types",
"world-state"
]
"rebuildPatterns": ["^yarn-project/sequencer-client/"]
},
"types": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/types",
"dockerfile": "types/Dockerfile",
"rebuildPatterns": ["^yarn-project/types/"],
"dependencies": ["yarn-project-base", "circuits.js", "foundation"]
"rebuildPatterns": ["^yarn-project/types/"]
},
"world-state": {
"buildDir": "yarn-project",
"projectDir": "yarn-project/world-state",
"dockerfile": "world-state/Dockerfile",
"rebuildPatterns": ["^yarn-project/world-state/"],
"dependencies": [
"yarn-project-base",
"circuits.js",
"foundation",
"merkle-tree",
"types"
]
"rebuildPatterns": ["^yarn-project/world-state/"]
}
}
Loading

0 comments on commit f66077a

Please sign in to comment.