diff --git a/presto-hive-hadoop2/bin/common.sh b/presto-hive-hadoop2/bin/common.sh
index 4de6f115996a..d190a5eeccf3 100644
--- a/presto-hive-hadoop2/bin/common.sh
+++ b/presto-hive-hadoop2/bin/common.sh
@@ -86,7 +86,7 @@ SCRIPT_DIR="${BASH_SOURCE%/*}"
INTEGRATION_TESTS_ROOT="${SCRIPT_DIR}/.."
PROJECT_ROOT="${INTEGRATION_TESTS_ROOT}/.."
DOCKER_COMPOSE_LOCATION="${INTEGRATION_TESTS_ROOT}/conf/docker-compose.yml"
-source "${BASH_SOURCE%/*}/../../presto-product-tests/conf/product-tests-defaults.sh"
+source "${PROJECT_ROOT}/presto-product-tests/conf/product-tests-defaults.sh"
# check docker and docker compose installation
docker-compose version
@@ -142,3 +142,54 @@ function get_hive_major_version() {
fi
echo "${version}"
}
+
+# $1 = base URI for table names
+function create_test_tables() {
+ local table_name table_path
+ local base_path="${1:?create_test_tables requires an argument}"
+ base_path="${base_path%/}" # remove trailing slash
+
+ table_name="presto_test_external_fs"
+ table_path="$base_path/$table_name/"
+ exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
+ exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
+ exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE $table_name(t_bigint bigint) LOCATION '${table_path}'"
+
+ table_name="presto_test_external_fs_with_header"
+ table_path="$base_path/$table_name/"
+ exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
+ exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}"
+ exec_in_hadoop_master_container /usr/bin/hive -e "
+ CREATE EXTERNAL TABLE $table_name(t_bigint bigint)
+ STORED AS TEXTFILE
+ LOCATION '${table_path}'
+ TBLPROPERTIES ('skip.header.line.count'='1')"
+
+ table_name="presto_test_external_fs_with_header_and_footer"
+ table_path="$base_path/$table_name/"
+ exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
+ exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}"
+ exec_in_hadoop_master_container /usr/bin/hive -e "
+ CREATE EXTERNAL TABLE $table_name(t_bigint bigint)
+ STORED AS TEXTFILE
+ LOCATION '${table_path}'
+ TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')"
+}
+
+# $1 = basename of core-site.xml template
+# other arguments are names of variables to substitute in the file
+function deploy_core_site_xml() {
+ local template="${1:?deploy_core_site_xml expects at least one argument}"
+ shift
+ local args=()
+ local name value
+ for name; do
+ shift
+ value="${!name//\\/\\\\}" # escape \ as \\
+ value="${value//|/\\|}" # escape | as \|
+ args+=(-e "s|%$name%|$value|g")
+ done
+ exec_in_hadoop_master_container bash -c \
+ 'sed "${@:2}" "/docker/files/$1" > /etc/hadoop/conf/core-site.xml' \
+ bash "$template" "${args[@]}"
+}
diff --git a/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh b/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh
index b98495a7b2e8..f81c00db8cba 100755
--- a/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh
+++ b/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh
@@ -12,43 +12,16 @@ cleanup_hadoop_docker_containers
start_hadoop_docker_containers
test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)"
-test_root="abfs://${ABFS_CONTAINER}@${ABFS_ACCOUNT}.dfs.core.windows.net/${test_directory}"
# insert Azure credentials
-# TODO replace core-site.xml.abfs-template with apply-site-xml-override.sh
-exec_in_hadoop_master_container cp /docker/files/core-site.xml.abfs-template /etc/hadoop/conf/core-site.xml
-exec_in_hadoop_master_container sed -i \
- -e "s|%ABFS_ACCESS_KEY%|${ABFS_ACCESS_KEY}|g" \
- -e "s|%ABFS_ACCOUNT%|${ABFS_ACCOUNT}|g" \
- /etc/hadoop/conf/core-site.xml
+deploy_core_site_xml core-site.xml.abfs-template \
+ ABFS_ACCESS_KEY ABFS_ACCOUNT
# restart hive-server2 to apply changes in core-site.xml
-docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2
+exec_in_hadoop_master_container supervisorctl restart hive-server2
retry check_hadoop
-# create test table
-table_path="${test_root}/presto_test_external_fs/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'"
-
-table_path="${test_root}/presto_test_external_fs_with_header/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
- CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint)
- STORED AS TEXTFILE
- LOCATION '${table_path}'
- TBLPROPERTIES ('skip.header.line.count'='1')"
-
-table_path="${test_root}/presto_test_external_fs_with_header_and_footer/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
- CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint)
- STORED AS TEXTFILE
- LOCATION '${table_path}'
- TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')"
+create_test_tables "abfs://${ABFS_CONTAINER}@${ABFS_ACCOUNT}.dfs.core.windows.net/${test_directory}"
stop_unnecessary_hadoop_services
diff --git a/presto-hive-hadoop2/bin/run_hive_adl_tests.sh b/presto-hive-hadoop2/bin/run_hive_adl_tests.sh
index 2f12d4480a31..dc61f8848867 100755
--- a/presto-hive-hadoop2/bin/run_hive_adl_tests.sh
+++ b/presto-hive-hadoop2/bin/run_hive_adl_tests.sh
@@ -2,7 +2,7 @@
set -euo pipefail -x
-. ${BASH_SOURCE%/*}/common.sh
+. "${BASH_SOURCE%/*}/common.sh"
test -v ADL_NAME
test -v ADL_CLIENT_ID
@@ -13,44 +13,16 @@ cleanup_hadoop_docker_containers
start_hadoop_docker_containers
test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)"
-test_root="adl://${ADL_NAME}.azuredatalakestore.net/${test_directory}"
# insert Azure credentials
-# TODO replace core-site.xml.adl-template with apply-site-xml-override.sh
-exec_in_hadoop_master_container cp /docker/files/core-site.xml.adl-template /etc/hadoop/conf/core-site.xml
-exec_in_hadoop_master_container sed -i \
- -e "s|%ADL_CLIENT_ID%|${ADL_CLIENT_ID}|g" \
- -e "s|%ADL_CREDENTIAL%|${ADL_CREDENTIAL}|g" \
- -e "s|%ADL_REFRESH_URL%|${ADL_REFRESH_URL}|g" \
- /etc/hadoop/conf/core-site.xml
+deploy_core_site_xml core-site.xml.adl-template \
+ ADL_CLIENT_ID ADL_CREDENTIAL ADL_REFRESH_URL
# restart hive-server2 to apply changes in core-site.xml
-docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2
+exec_in_hadoop_master_container supervisorctl restart hive-server2
retry check_hadoop
-# create test table
-table_path="${test_root}/presto_test_external_fs/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'"
-
-table_path="${test_root}/presto_test_external_fs_with_header/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
- CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint)
- STORED AS TEXTFILE
- LOCATION '${table_path}'
- TBLPROPERTIES ('skip.header.line.count'='1')"
-
-table_path="${test_root}/presto_test_external_fs_with_header_and_footer/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
- CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint)
- STORED AS TEXTFILE
- LOCATION '${table_path}'
- TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')"
+create_test_tables "adl://${ADL_NAME}.azuredatalakestore.net/${test_directory}"
stop_unnecessary_hadoop_services
diff --git a/presto-hive-hadoop2/bin/run_hive_s3_tests.sh b/presto-hive-hadoop2/bin/run_hive_s3_tests.sh
index b4a8241b9d8f..c4aa9430fcee 100755
--- a/presto-hive-hadoop2/bin/run_hive_s3_tests.sh
+++ b/presto-hive-hadoop2/bin/run_hive_s3_tests.sh
@@ -10,15 +10,11 @@ start_hadoop_docker_containers
test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)"
# insert AWS credentials
-# TODO replace core-site.xml.s3-template with apply-site-xml-override.sh
-exec_in_hadoop_master_container cp /docker/files/core-site.xml.s3-template /etc/hadoop/conf/core-site.xml
-exec_in_hadoop_master_container sed -i \
- -e "s|%AWS_ACCESS_KEY%|${AWS_ACCESS_KEY_ID}|g" \
- -e "s|%AWS_SECRET_KEY%|${AWS_SECRET_ACCESS_KEY}|g" \
- -e "s|%S3_BUCKET_ENDPOINT%|${S3_BUCKET_ENDPOINT}|g" \
- /etc/hadoop/conf/core-site.xml
+deploy_core_site_xml core-site.xml.s3-template \
+ AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY S3_BUCKET_ENDPOINT
-# create test table
+# create test tables
+# can't use create_test_tables because the first table is created with different commands
table_path="s3a://${S3_BUCKET}/${test_directory}/presto_test_external_fs/"
exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
exec_in_hadoop_master_container /docker/files/hadoop-put.sh /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
diff --git a/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh b/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh
index e2d002aa8d97..460b4040bbce 100755
--- a/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh
+++ b/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh
@@ -12,43 +12,16 @@ cleanup_hadoop_docker_containers
start_hadoop_docker_containers
test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)"
-test_root="wasb://${WASB_CONTAINER}@${WASB_ACCOUNT}.blob.core.windows.net/${test_directory}"
# insert Azure credentials
-# TODO replace core-site.xml.wasb-template with apply-site-xml-override.sh
-exec_in_hadoop_master_container cp /docker/files/core-site.xml.wasb-template /etc/hadoop/conf/core-site.xml
-exec_in_hadoop_master_container sed -i \
- -e "s|%WASB_ACCESS_KEY%|${WASB_ACCESS_KEY}|g" \
- -e "s|%WASB_ACCOUNT%|${WASB_ACCOUNT}|g" \
- /etc/hadoop/conf/core-site.xml
+deploy_core_site_xml core-site.xml.wasb-template \
+ WASB_ACCESS_KEY WASB_ACCOUNT
# restart hive-server2 to apply changes in core-site.xml
-docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2
+exec_in_hadoop_master_container supervisorctl restart hive-server2
retry check_hadoop
-# create test table
-table_path="${test_root}/presto_test_external_fs/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'"
-
-table_path="${test_root}/presto_test_external_fs_with_header/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
- CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint)
- STORED AS TEXTFILE
- LOCATION '${table_path}'
- TBLPROPERTIES ('skip.header.line.count'='1')"
-
-table_path="${test_root}/presto_test_external_fs_with_header_and_footer/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
- CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint)
- STORED AS TEXTFILE
- LOCATION '${table_path}'
- TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')"
+create_test_tables "wasb://${WASB_CONTAINER}@${WASB_ACCOUNT}.blob.core.windows.net/${test_directory}"
stop_unnecessary_hadoop_services
diff --git a/presto-hive-hadoop2/conf/files/core-site.xml.s3-template b/presto-hive-hadoop2/conf/files/core-site.xml.s3-template
index 3fb63f0bdb60..984026e39e38 100644
--- a/presto-hive-hadoop2/conf/files/core-site.xml.s3-template
+++ b/presto-hive-hadoop2/conf/files/core-site.xml.s3-template
@@ -35,22 +35,22 @@
fs.s3.awsAccessKeyId
- %AWS_ACCESS_KEY%
+ %AWS_ACCESS_KEY_ID%
fs.s3.awsSecretAccessKey
- %AWS_SECRET_KEY%
+ %AWS_SECRET_ACCESS_KEY%
fs.s3a.access.key
- %AWS_ACCESS_KEY%
+ %AWS_ACCESS_KEY_ID%
fs.s3a.secret.key
- %AWS_SECRET_KEY%
+ %AWS_SECRET_ACCESS_KEY%