diff --git a/presto-hive-hadoop2/bin/common.sh b/presto-hive-hadoop2/bin/common.sh index 4de6f115996a..d190a5eeccf3 100644 --- a/presto-hive-hadoop2/bin/common.sh +++ b/presto-hive-hadoop2/bin/common.sh @@ -86,7 +86,7 @@ SCRIPT_DIR="${BASH_SOURCE%/*}" INTEGRATION_TESTS_ROOT="${SCRIPT_DIR}/.." PROJECT_ROOT="${INTEGRATION_TESTS_ROOT}/.." DOCKER_COMPOSE_LOCATION="${INTEGRATION_TESTS_ROOT}/conf/docker-compose.yml" -source "${BASH_SOURCE%/*}/../../presto-product-tests/conf/product-tests-defaults.sh" +source "${PROJECT_ROOT}/presto-product-tests/conf/product-tests-defaults.sh" # check docker and docker compose installation docker-compose version @@ -142,3 +142,54 @@ function get_hive_major_version() { fi echo "${version}" } + +# $1 = base URI for table names +function create_test_tables() { + local table_name table_path + local base_path="${1:?create_test_tables requires an argument}" + base_path="${base_path%/}" # remove trailing slash + + table_name="presto_test_external_fs" + table_path="$base_path/$table_name/" + exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" + exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}" + exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE $table_name(t_bigint bigint) LOCATION '${table_path}'" + + table_name="presto_test_external_fs_with_header" + table_path="$base_path/$table_name/" + exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" + exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}" + exec_in_hadoop_master_container /usr/bin/hive -e " + CREATE EXTERNAL TABLE $table_name(t_bigint bigint) + STORED AS TEXTFILE + LOCATION '${table_path}' + TBLPROPERTIES ('skip.header.line.count'='1')" + + table_name="presto_test_external_fs_with_header_and_footer" + table_path="$base_path/$table_name/" + exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" + exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}" + exec_in_hadoop_master_container /usr/bin/hive -e " + CREATE EXTERNAL TABLE $table_name(t_bigint bigint) + STORED AS TEXTFILE + LOCATION '${table_path}' + TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')" +} + +# $1 = basename of core-site.xml template +# other arguments are names of variables to substitute in the file +function deploy_core_site_xml() { + local template="${1:?deploy_core_site_xml expects at least one argument}" + shift + local args=() + local name value + for name; do + shift + value="${!name//\\/\\\\}" # escape \ as \\ + value="${value//|/\\|}" # escape | as \| + args+=(-e "s|%$name%|$value|g") + done + exec_in_hadoop_master_container bash -c \ + 'sed "${@:2}" "/docker/files/$1" > /etc/hadoop/conf/core-site.xml' \ + bash "$template" "${args[@]}" +} diff --git a/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh b/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh index b98495a7b2e8..f81c00db8cba 100755 --- a/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh +++ b/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh @@ -12,43 +12,16 @@ cleanup_hadoop_docker_containers start_hadoop_docker_containers test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)" -test_root="abfs://${ABFS_CONTAINER}@${ABFS_ACCOUNT}.dfs.core.windows.net/${test_directory}" # insert Azure credentials -# TODO replace core-site.xml.abfs-template with apply-site-xml-override.sh -exec_in_hadoop_master_container cp /docker/files/core-site.xml.abfs-template /etc/hadoop/conf/core-site.xml -exec_in_hadoop_master_container sed -i \ - -e "s|%ABFS_ACCESS_KEY%|${ABFS_ACCESS_KEY}|g" \ - -e "s|%ABFS_ACCOUNT%|${ABFS_ACCOUNT}|g" \ - /etc/hadoop/conf/core-site.xml +deploy_core_site_xml core-site.xml.abfs-template \ + ABFS_ACCESS_KEY ABFS_ACCOUNT # restart hive-server2 to apply changes in core-site.xml -docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2 +exec_in_hadoop_master_container supervisorctl restart hive-server2 retry check_hadoop -# create test table -table_path="${test_root}/presto_test_external_fs/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'" - -table_path="${test_root}/presto_test_external_fs_with_header/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e " - CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint) - STORED AS TEXTFILE - LOCATION '${table_path}' - TBLPROPERTIES ('skip.header.line.count'='1')" - -table_path="${test_root}/presto_test_external_fs_with_header_and_footer/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e " - CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint) - STORED AS TEXTFILE - LOCATION '${table_path}' - TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')" +create_test_tables "abfs://${ABFS_CONTAINER}@${ABFS_ACCOUNT}.dfs.core.windows.net/${test_directory}" stop_unnecessary_hadoop_services diff --git a/presto-hive-hadoop2/bin/run_hive_adl_tests.sh b/presto-hive-hadoop2/bin/run_hive_adl_tests.sh index 2f12d4480a31..dc61f8848867 100755 --- a/presto-hive-hadoop2/bin/run_hive_adl_tests.sh +++ b/presto-hive-hadoop2/bin/run_hive_adl_tests.sh @@ -2,7 +2,7 @@ set -euo pipefail -x -. ${BASH_SOURCE%/*}/common.sh +. "${BASH_SOURCE%/*}/common.sh" test -v ADL_NAME test -v ADL_CLIENT_ID @@ -13,44 +13,16 @@ cleanup_hadoop_docker_containers start_hadoop_docker_containers test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)" -test_root="adl://${ADL_NAME}.azuredatalakestore.net/${test_directory}" # insert Azure credentials -# TODO replace core-site.xml.adl-template with apply-site-xml-override.sh -exec_in_hadoop_master_container cp /docker/files/core-site.xml.adl-template /etc/hadoop/conf/core-site.xml -exec_in_hadoop_master_container sed -i \ - -e "s|%ADL_CLIENT_ID%|${ADL_CLIENT_ID}|g" \ - -e "s|%ADL_CREDENTIAL%|${ADL_CREDENTIAL}|g" \ - -e "s|%ADL_REFRESH_URL%|${ADL_REFRESH_URL}|g" \ - /etc/hadoop/conf/core-site.xml +deploy_core_site_xml core-site.xml.adl-template \ + ADL_CLIENT_ID ADL_CREDENTIAL ADL_REFRESH_URL # restart hive-server2 to apply changes in core-site.xml -docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2 +exec_in_hadoop_master_container supervisorctl restart hive-server2 retry check_hadoop -# create test table -table_path="${test_root}/presto_test_external_fs/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'" - -table_path="${test_root}/presto_test_external_fs_with_header/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e " - CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint) - STORED AS TEXTFILE - LOCATION '${table_path}' - TBLPROPERTIES ('skip.header.line.count'='1')" - -table_path="${test_root}/presto_test_external_fs_with_header_and_footer/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e " - CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint) - STORED AS TEXTFILE - LOCATION '${table_path}' - TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')" +create_test_tables "adl://${ADL_NAME}.azuredatalakestore.net/${test_directory}" stop_unnecessary_hadoop_services diff --git a/presto-hive-hadoop2/bin/run_hive_s3_tests.sh b/presto-hive-hadoop2/bin/run_hive_s3_tests.sh index b4a8241b9d8f..c4aa9430fcee 100755 --- a/presto-hive-hadoop2/bin/run_hive_s3_tests.sh +++ b/presto-hive-hadoop2/bin/run_hive_s3_tests.sh @@ -10,15 +10,11 @@ start_hadoop_docker_containers test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)" # insert AWS credentials -# TODO replace core-site.xml.s3-template with apply-site-xml-override.sh -exec_in_hadoop_master_container cp /docker/files/core-site.xml.s3-template /etc/hadoop/conf/core-site.xml -exec_in_hadoop_master_container sed -i \ - -e "s|%AWS_ACCESS_KEY%|${AWS_ACCESS_KEY_ID}|g" \ - -e "s|%AWS_SECRET_KEY%|${AWS_SECRET_ACCESS_KEY}|g" \ - -e "s|%S3_BUCKET_ENDPOINT%|${S3_BUCKET_ENDPOINT}|g" \ - /etc/hadoop/conf/core-site.xml +deploy_core_site_xml core-site.xml.s3-template \ + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY S3_BUCKET_ENDPOINT -# create test table +# create test tables +# can't use create_test_tables because the first table is created with different commands table_path="s3a://${S3_BUCKET}/${test_directory}/presto_test_external_fs/" exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" exec_in_hadoop_master_container /docker/files/hadoop-put.sh /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}" diff --git a/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh b/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh index e2d002aa8d97..460b4040bbce 100755 --- a/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh +++ b/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh @@ -12,43 +12,16 @@ cleanup_hadoop_docker_containers start_hadoop_docker_containers test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)" -test_root="wasb://${WASB_CONTAINER}@${WASB_ACCOUNT}.blob.core.windows.net/${test_directory}" # insert Azure credentials -# TODO replace core-site.xml.wasb-template with apply-site-xml-override.sh -exec_in_hadoop_master_container cp /docker/files/core-site.xml.wasb-template /etc/hadoop/conf/core-site.xml -exec_in_hadoop_master_container sed -i \ - -e "s|%WASB_ACCESS_KEY%|${WASB_ACCESS_KEY}|g" \ - -e "s|%WASB_ACCOUNT%|${WASB_ACCOUNT}|g" \ - /etc/hadoop/conf/core-site.xml +deploy_core_site_xml core-site.xml.wasb-template \ + WASB_ACCESS_KEY WASB_ACCOUNT # restart hive-server2 to apply changes in core-site.xml -docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2 +exec_in_hadoop_master_container supervisorctl restart hive-server2 retry check_hadoop -# create test table -table_path="${test_root}/presto_test_external_fs/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'" - -table_path="${test_root}/presto_test_external_fs_with_header/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e " - CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint) - STORED AS TEXTFILE - LOCATION '${table_path}' - TBLPROPERTIES ('skip.header.line.count'='1')" - -table_path="${test_root}/presto_test_external_fs_with_header_and_footer/" -exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}" -exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}" -exec_in_hadoop_master_container /usr/bin/hive -e " - CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint) - STORED AS TEXTFILE - LOCATION '${table_path}' - TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')" +create_test_tables "wasb://${WASB_CONTAINER}@${WASB_ACCOUNT}.blob.core.windows.net/${test_directory}" stop_unnecessary_hadoop_services diff --git a/presto-hive-hadoop2/conf/files/core-site.xml.s3-template b/presto-hive-hadoop2/conf/files/core-site.xml.s3-template index 3fb63f0bdb60..984026e39e38 100644 --- a/presto-hive-hadoop2/conf/files/core-site.xml.s3-template +++ b/presto-hive-hadoop2/conf/files/core-site.xml.s3-template @@ -35,22 +35,22 @@ fs.s3.awsAccessKeyId - %AWS_ACCESS_KEY% + %AWS_ACCESS_KEY_ID% fs.s3.awsSecretAccessKey - %AWS_SECRET_KEY% + %AWS_SECRET_ACCESS_KEY% fs.s3a.access.key - %AWS_ACCESS_KEY% + %AWS_ACCESS_KEY_ID% fs.s3a.secret.key - %AWS_SECRET_KEY% + %AWS_SECRET_ACCESS_KEY%