Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[VL] ENABLE_EP_CACHE=ON still uses cached Velox build although the build arguments were changed #3926

Merged
merged 3 commits into from
Dec 6, 2023
Merged

[VL] ENABLE_EP_CACHE=ON still uses cached Velox build although the build arguments were changed #3926

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0d867ac
fixup
zhztheplayer Dec 5, 2023
24549ae
fixup
zhztheplayer Dec 5, 2023
4c14b34
fixup
zhztheplayer Dec 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 21 additions & 16 deletions ep/build-velox/src/build_velox.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ ENABLE_EP_CACHE=OFF
ENABLE_BENCHMARK=OFF
ENABLE_TESTS=OFF
RUN_SETUP_SCRIPT=ON
OTHER_ARGUMENTS=""

OS=`uname -s`
ARCH=`uname -m`
Expand Down Expand Up @@ -77,8 +78,6 @@ for arg in "$@"; do
done

function compile {
TARGET_BUILD_COMMIT=$(git rev-parse --verify HEAD)

if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then
if [ $OS == 'Linux' ]; then
setup_linux
Expand Down Expand Up @@ -144,16 +143,22 @@ function compile {
fi
}

function get_build_summary {
COMMIT_HASH=$1
# Ideally all script arguments should be put into build summary.
echo "ENABLE_S3=$ENABLE_S3,ENABLE_GCS=$ENABLE_GCS,ENABLE_HDFS=$ENABLE_HDFS,BUILD_TYPE=$BUILD_TYPE,VELOX_HOME=$VELOX_HOME,ENABLE_EP_CACHE=$ENABLE_EP_CACHE,ENABLE_BENCHMARK=$ENABLE_BENCHMARK,ENABLE_TESTS=$ENABLE_TESTS,RUN_SETUP_SCRIPT=$RUN_SETUP_SCRIPT,OTHER_ARGUMENTS=$OTHER_ARGUMENTS,COMMIT_HASH=$COMMIT_HASH"
}

function check_commit {
if [ $ENABLE_EP_CACHE == "ON" ]; then
if [ -f ${VELOX_HOME}/velox-commit.cache ]; then
CACHED_BUILT_COMMIT="$(cat ${VELOX_HOME}/velox-commit.cache)"
if [ -n "$CACHED_BUILT_COMMIT" ]; then
if [ "$TARGET_BUILD_COMMIT" = "$CACHED_BUILT_COMMIT" ]; then
echo "Velox build of commit $TARGET_BUILD_COMMIT was cached."
if [ -f ${VELOX_HOME}/velox-build.cache ]; then
CACHED_BUILD_SUMMARY="$(cat ${VELOX_HOME}/velox-build.cache)"
if [ -n "$CACHED_BUILD_SUMMARY" ]; then
if [ "$TARGET_BUILD_SUMMARY" = "$CACHED_BUILD_SUMMARY" ]; then
echo "Velox build $TARGET_BUILD_SUMMARY was cached."
exit 0
else
echo "Found cached commit $CACHED_BUILT_COMMIT for Velox which is different with target commit $TARGET_BUILD_COMMIT."
echo "Found cached build $CACHED_BUILD_SUMMARY for Velox which is different with target build $TARGET_BUILD_SUMMARY."
fi
fi
fi
Expand All @@ -162,8 +167,8 @@ function check_commit {
git clean -dffx :/
fi

if [ -f ${VELOX_HOME}/velox-commit.cache ]; then
rm -f ${VELOX_HOME}/velox-commit.cache
if [ -f ${VELOX_HOME}/velox-build.cache ]; then
rm -f ${VELOX_HOME}/velox-build.cache
fi
}

Expand Down Expand Up @@ -245,15 +250,15 @@ echo "ENABLE_HDFS=${ENABLE_HDFS}"
echo "BUILD_TYPE=${BUILD_TYPE}"

cd ${VELOX_HOME}
TARGET_BUILD_COMMIT="$(git rev-parse --verify HEAD)"
if [ -z "$TARGET_BUILD_COMMIT" ]; then
echo "Unable to parse Velox commit: $TARGET_BUILD_COMMIT."
exit 0
TARGET_BUILD_SUMMARY=$(get_build_summary "$(git rev-parse --verify HEAD)")
if [ -z "$TARGET_BUILD_SUMMARY" ]; then
echo "Unable to parse Velox build: $TARGET_BUILD_SUMMARY."
exit 1
fi
echo "Target Velox commit: $TARGET_BUILD_COMMIT"
echo "Target Velox build: $TARGET_BUILD_SUMMARY"

check_commit
compile

echo "Successfully built Velox from Source."
echo $TARGET_BUILD_COMMIT >"${VELOX_HOME}/velox-commit.cache"
echo $TARGET_BUILD_SUMMARY >"${VELOX_HOME}/velox-build.cache"
Loading