Share more setup code between hive test scripts

trinodb · Sep 17, 2020 · 655a19d · 655a19d
1 parent a381973
commit 655a19d
Show file tree

Hide file tree

Showing 6 changed files with 73 additions and 108 deletions.
diff --git a/presto-hive-hadoop2/bin/common.sh b/presto-hive-hadoop2/bin/common.sh
@@ -86,7 +86,7 @@ SCRIPT_DIR="${BASH_SOURCE%/*}"
 INTEGRATION_TESTS_ROOT="${SCRIPT_DIR}/.."
 PROJECT_ROOT="${INTEGRATION_TESTS_ROOT}/.."
 DOCKER_COMPOSE_LOCATION="${INTEGRATION_TESTS_ROOT}/conf/docker-compose.yml"
-source "${BASH_SOURCE%/*}/../../presto-product-tests/conf/product-tests-defaults.sh"
+source "${PROJECT_ROOT}/presto-product-tests/conf/product-tests-defaults.sh"
 
 # check docker and docker compose installation
 docker-compose version
@@ -142,3 +142,54 @@ function get_hive_major_version() {
     fi
     echo "${version}"
 }
+
+# $1 = base URI for table names
+function create_test_tables() {
+    local table_name table_path
+    local base_path="${1:?create_test_tables requires an argument}"
+    base_path="${base_path%/}" # remove trailing slash
+
+    table_name="presto_test_external_fs"
+    table_path="$base_path/$table_name/"
+    exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
+    exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
+    exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE $table_name(t_bigint bigint) LOCATION '${table_path}'"
+
+    table_name="presto_test_external_fs_with_header"
+    table_path="$base_path/$table_name/"
+    exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
+    exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}"
+    exec_in_hadoop_master_container /usr/bin/hive -e "
+        CREATE EXTERNAL TABLE $table_name(t_bigint bigint)
+        STORED AS TEXTFILE
+        LOCATION '${table_path}'
+        TBLPROPERTIES ('skip.header.line.count'='1')"
+
+    table_name="presto_test_external_fs_with_header_and_footer"
+    table_path="$base_path/$table_name/"
+    exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
+    exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}"
+    exec_in_hadoop_master_container /usr/bin/hive -e "
+        CREATE EXTERNAL TABLE $table_name(t_bigint bigint)
+        STORED AS TEXTFILE
+        LOCATION '${table_path}'
+        TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')"
+}
+
+# $1 = basename of core-site.xml template
+# other arguments are names of variables to substitute in the file
+function deploy_core_site_xml() {
+    local template="${1:?deploy_core_site_xml expects at least one argument}"
+    shift
+    local args=()
+    local name value
+    for name; do
+        shift
+        value="${!name//\\/\\\\}" # escape \ as \\
+        value="${value//|/\\|}" # escape | as \|
+        args+=(-e "s|%$name%|$value|g")
+    done
+    exec_in_hadoop_master_container bash -c \
+        'sed "${@:2}" "/docker/files/$1" > /etc/hadoop/conf/core-site.xml' \
+        bash "$template" "${args[@]}"
+}
diff --git a/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh b/presto-hive-hadoop2/bin/run_hive_abfs_tests.sh
@@ -12,43 +12,16 @@ cleanup_hadoop_docker_containers
 start_hadoop_docker_containers
 
 test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)"
-test_root="abfs://${ABFS_CONTAINER}@${ABFS_ACCOUNT}.dfs.core.windows.net/${test_directory}"
 
 # insert Azure credentials
-# TODO replace core-site.xml.abfs-template with apply-site-xml-override.sh
-exec_in_hadoop_master_container cp /docker/files/core-site.xml.abfs-template /etc/hadoop/conf/core-site.xml
-exec_in_hadoop_master_container sed -i \
-    -e "s|%ABFS_ACCESS_KEY%|${ABFS_ACCESS_KEY}|g" \
-    -e "s|%ABFS_ACCOUNT%|${ABFS_ACCOUNT}|g" \
-    /etc/hadoop/conf/core-site.xml
+deploy_core_site_xml core-site.xml.abfs-template \
+    ABFS_ACCESS_KEY ABFS_ACCOUNT
 
 # restart hive-server2 to apply changes in core-site.xml
-docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2
+exec_in_hadoop_master_container supervisorctl restart hive-server2
 retry check_hadoop
 
-# create test table
-table_path="${test_root}/presto_test_external_fs/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'"
-
-table_path="${test_root}/presto_test_external_fs_with_header/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
-    CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint)
-    STORED AS TEXTFILE
-    LOCATION '${table_path}'
-    TBLPROPERTIES ('skip.header.line.count'='1')"
-
-table_path="${test_root}/presto_test_external_fs_with_header_and_footer/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
-    CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint)
-    STORED AS TEXTFILE
-    LOCATION '${table_path}'
-    TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')"
+create_test_tables "abfs://${ABFS_CONTAINER}@${ABFS_ACCOUNT}.dfs.core.windows.net/${test_directory}"
 
 stop_unnecessary_hadoop_services
 

diff --git a/presto-hive-hadoop2/bin/run_hive_adl_tests.sh b/presto-hive-hadoop2/bin/run_hive_adl_tests.sh
@@ -2,7 +2,7 @@
 
 set -euo pipefail -x
 
-. ${BASH_SOURCE%/*}/common.sh
+. "${BASH_SOURCE%/*}/common.sh"
 
 test -v ADL_NAME
 test -v ADL_CLIENT_ID
@@ -13,44 +13,16 @@ cleanup_hadoop_docker_containers
 start_hadoop_docker_containers
 
 test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)"
-test_root="adl://${ADL_NAME}.azuredatalakestore.net/${test_directory}"
 
 # insert Azure credentials
-# TODO replace core-site.xml.adl-template with apply-site-xml-override.sh
-exec_in_hadoop_master_container cp /docker/files/core-site.xml.adl-template /etc/hadoop/conf/core-site.xml
-exec_in_hadoop_master_container sed -i \
-    -e "s|%ADL_CLIENT_ID%|${ADL_CLIENT_ID}|g" \
-    -e "s|%ADL_CREDENTIAL%|${ADL_CREDENTIAL}|g" \
-    -e "s|%ADL_REFRESH_URL%|${ADL_REFRESH_URL}|g" \
-    /etc/hadoop/conf/core-site.xml
+deploy_core_site_xml core-site.xml.adl-template \
+    ADL_CLIENT_ID ADL_CREDENTIAL ADL_REFRESH_URL
 
 # restart hive-server2 to apply changes in core-site.xml
-docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2
+exec_in_hadoop_master_container supervisorctl restart hive-server2
 retry check_hadoop
 
-# create test table
-table_path="${test_root}/presto_test_external_fs/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'"
-
-table_path="${test_root}/presto_test_external_fs_with_header/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
-    CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint)
-    STORED AS TEXTFILE
-    LOCATION '${table_path}'
-    TBLPROPERTIES ('skip.header.line.count'='1')"
-
-table_path="${test_root}/presto_test_external_fs_with_header_and_footer/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
-    CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint)
-    STORED AS TEXTFILE
-    LOCATION '${table_path}'
-    TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')"
+create_test_tables "adl://${ADL_NAME}.azuredatalakestore.net/${test_directory}"
 
 stop_unnecessary_hadoop_services
 

diff --git a/presto-hive-hadoop2/bin/run_hive_s3_tests.sh b/presto-hive-hadoop2/bin/run_hive_s3_tests.sh
@@ -10,15 +10,11 @@ start_hadoop_docker_containers
 test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)"
 
 # insert AWS credentials
-# TODO replace core-site.xml.s3-template with apply-site-xml-override.sh
-exec_in_hadoop_master_container cp /docker/files/core-site.xml.s3-template /etc/hadoop/conf/core-site.xml
-exec_in_hadoop_master_container sed -i \
-    -e "s|%AWS_ACCESS_KEY%|${AWS_ACCESS_KEY_ID}|g" \
-    -e "s|%AWS_SECRET_KEY%|${AWS_SECRET_ACCESS_KEY}|g" \
-    -e "s|%S3_BUCKET_ENDPOINT%|${S3_BUCKET_ENDPOINT}|g" \
-    /etc/hadoop/conf/core-site.xml
+deploy_core_site_xml core-site.xml.s3-template \
+    AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY S3_BUCKET_ENDPOINT
 
-# create test table
+# create test tables
+# can't use create_test_tables because the first table is created with different commands
 table_path="s3a://${S3_BUCKET}/${test_directory}/presto_test_external_fs/"
 exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
 exec_in_hadoop_master_container /docker/files/hadoop-put.sh /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"

diff --git a/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh b/presto-hive-hadoop2/bin/run_hive_wasb_tests.sh
@@ -12,43 +12,16 @@ cleanup_hadoop_docker_containers
 start_hadoop_docker_containers
 
 test_directory="$(date '+%Y%m%d-%H%M%S')-$(uuidgen | sha1sum | cut -b 1-6)"
-test_root="wasb://${WASB_CONTAINER}@${WASB_ACCOUNT}.blob.core.windows.net/${test_directory}"
 
 # insert Azure credentials
-# TODO replace core-site.xml.wasb-template with apply-site-xml-override.sh
-exec_in_hadoop_master_container cp /docker/files/core-site.xml.wasb-template /etc/hadoop/conf/core-site.xml
-exec_in_hadoop_master_container sed -i \
-    -e "s|%WASB_ACCESS_KEY%|${WASB_ACCESS_KEY}|g" \
-    -e "s|%WASB_ACCOUNT%|${WASB_ACCOUNT}|g" \
-    /etc/hadoop/conf/core-site.xml
+deploy_core_site_xml core-site.xml.wasb-template \
+    WASB_ACCESS_KEY WASB_ACCOUNT
 
 # restart hive-server2 to apply changes in core-site.xml
-docker exec "$(hadoop_master_container)" supervisorctl restart hive-server2
+exec_in_hadoop_master_container supervisorctl restart hive-server2
 retry check_hadoop
 
-# create test table
-table_path="${test_root}/presto_test_external_fs/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "CREATE EXTERNAL TABLE presto_test_external_fs(t_bigint bigint) LOCATION '${table_path}'"
-
-table_path="${test_root}/presto_test_external_fs_with_header/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
-    CREATE EXTERNAL TABLE presto_test_external_fs_with_header(t_bigint bigint)
-    STORED AS TEXTFILE
-    LOCATION '${table_path}'
-    TBLPROPERTIES ('skip.header.line.count'='1')"
-
-table_path="${test_root}/presto_test_external_fs_with_header_and_footer/"
-exec_in_hadoop_master_container hadoop fs -mkdir -p "${table_path}"
-exec_in_hadoop_master_container hadoop fs -copyFromLocal -f /docker/files/test_table_with_header_and_footer.csv{,.gz,.bz2,.lz4} "${table_path}"
-exec_in_hadoop_master_container /usr/bin/hive -e "
-    CREATE EXTERNAL TABLE presto_test_external_fs_with_header_and_footer(t_bigint bigint)
-    STORED AS TEXTFILE
-    LOCATION '${table_path}'
-    TBLPROPERTIES ('skip.header.line.count'='2', 'skip.footer.line.count'='2')"
+create_test_tables "wasb://${WASB_CONTAINER}@${WASB_ACCOUNT}.blob.core.windows.net/${test_directory}"
 
 stop_unnecessary_hadoop_services
 

diff --git a/presto-hive-hadoop2/conf/files/core-site.xml.s3-template b/presto-hive-hadoop2/conf/files/core-site.xml.s3-template
@@ -35,22 +35,22 @@
 
     <property>
         <name>fs.s3.awsAccessKeyId</name>
-        <value>%AWS_ACCESS_KEY%</value>
+        <value>%AWS_ACCESS_KEY_ID%</value>
     </property>
 
     <property>
         <name>fs.s3.awsSecretAccessKey</name>
-        <value>%AWS_SECRET_KEY%</value>
+        <value>%AWS_SECRET_ACCESS_KEY%</value>
     </property>
 
     <property>
         <name>fs.s3a.access.key</name>
-        <value>%AWS_ACCESS_KEY%</value>
+        <value>%AWS_ACCESS_KEY_ID%</value>
     </property>
 
     <property>
         <name>fs.s3a.secret.key</name>
-        <value>%AWS_SECRET_KEY%</value>
+        <value>%AWS_SECRET_ACCESS_KEY%</value>
     </property>
 
     <property>