Use hdp3 on centos7

trinodb · Jun 27, 2019 · 5c90577 · 5c90577
1 parent bdd8eef
commit 5c90577
Show file tree

Hide file tree

Showing 9 changed files with 267 additions and 10 deletions.
diff --git a/prestodev/centos7-oj8/Dockerfile b/prestodev/centos7-oj8/Dockerfile
@@ -0,0 +1,37 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM library/centos:7
+MAINTAINER Presto community <https://prestosql.io/community.html>
+
+# Install Java and presto-admin dependences (already has python 2.6)
+RUN \
+    set -xeu && \
+    yum install -y \
+        java-1.8.0-openjdk-devel \
+        wget \
+        && \
+    \
+    # install supervisor
+    yum --enablerepo=extras install -y setuptools epel-release && \
+    yum install -y python-pip && \
+    pip install supervisor==3.4.0 && \
+    \
+    # install commonly needed packages
+    yum install -y \
+        sudo \
+        vim `# helpful when troubleshooting product tests` \
+        && \
+    # cleanup
+    yum -y clean all && rm -rf /tmp/* /var/tmp/*
+
+ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
diff --git a/prestodev/centos7-oj8/README.md b/prestodev/centos7-oj8/README.md
@@ -0,0 +1,13 @@
+# centos7-oj8 [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
+
+[layers-badge]: https://images.microbadger.com/badges/image/prestodev/centos7-oj8.svg
+[layers-link]: https://microbadger.com/images/prestodev/centos7-oj8
+[version-badge]: https://images.microbadger.com/badges/version/prestodev/centos7-oj8.svg
+[dockerhub-link]: https://hub.docker.com/r/prestodev/centos7-oj8
+
+Docker image of CentOS 7 with Oracle JDK 8 installed.
+
+## Oracle license
+
+By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
+[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
diff --git a/prestodev/hdp3-hive3/Dockerfile b/prestodev/hdp3-hive3/Dockerfile
@@ -10,17 +10,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM prestodev/centos6-oj8:unlabelled
+FROM prestodev/centos7-oj8:unlabelled
 MAINTAINER Presto community <https://prestosql.io/community.html>
 
 # Change default timezone
 RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone
 
 # Install HDP repo
-RUN wget -nv http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.5.0/hdp.repo -P /etc/yum.repos.d
+RUN wget -nv http://public-repo-1.hortonworks.com/HDP/centos7/3.x/updates/3.1.0.0/hdp.repo -P /etc/yum.repos.d
+
+# mysql
+RUN wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm && \
+  rpm -ivh mysql-community-release-el7-5.noarch.rpm && \
+  yum update -y
 
 # Install Hadoop, Hive (w/ MySQL)
 RUN yum install -y \
+    yum-plugin-ovl \
     hadoop-hdfs-namenode \
     hadoop-hdfs-secondarynamenode \
     hadoop-hdfs-datanode \
@@ -31,6 +37,7 @@ RUN yum install -y \
     hive \
     hive-metastore \
     hive-server2 \
+    tez \
 
     mysql-server mysql-connector-java \
 
@@ -59,7 +66,7 @@ RUN chmod 755 /root && chmod 700 /root/.ssh
 RUN passwd --unlock root
 
 # HDFS ports
-EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470
+EXPOSE 1004 1006 8020 9866 9867 9870 9864 50470
 
 # YARN ports
 EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888

diff --git a/prestodev/hdp3-hive3/README.md b/prestodev/hdp3-hive3/README.md
@@ -1,9 +1,9 @@
-# hdp2.6-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
-           
-[layers-badge]: https://images.microbadger.com/badges/image/prestodev/hdp2.6-hive.svg
+# hdp3-hive3 [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
+
+[layers-badge]: https://images.microbadger.com/badges/image/prestodev/hdp3-hive3.svg
 [layers-link]: https://microbadger.com/images/prestodev/hdp2.6-hive
-[version-badge]: https://images.microbadger.com/badges/version/prestodev/hdp2.6-hive.svg
-[dockerhub-link]: https://hub.docker.com/r/prestodev/hdp2.6-hive
+[version-badge]: https://images.microbadger.com/badges/version/prestodev/hdp3-hive3.svg
+[dockerhub-link]: https://hub.docker.com/r/prestodev/hdp3-hive3
 
 A pseudo-distributed Hadoop image for testing Presto, based on the HDP distro.
 

diff --git a/prestodev/hdp3-hive3/files/etc/hadoop/conf/yarn-site.xml b/prestodev/hdp3-hive3/files/etc/hadoop/conf/yarn-site.xml
@@ -87,4 +87,15 @@
         <value>http://hadoop-master:19888/jobhistory/logs</value>
     </property>
 
+    <property>
+        <name>yarn.nodemanager.vmem-check-enabled</name>
+        <value>false</value>
+        <description>Whether virtual memory limits will be enforced for containers.</description>
+    </property>
+
+    <property>
+        <name>yarn.scheduler.minimum-allocation-mb </name>
+        <value>256</value>
+    </property>
+
 </configuration>
diff --git a/prestodev/hdp3-hive3/files/etc/hive/conf/hive-site.xml b/prestodev/hdp3-hive3/files/etc/hive/conf/hive-site.xml
@@ -64,4 +64,34 @@
         <value>hdfs,hive</value>
     </property>
 
+    <!--  https://stackoverflow.com/questions/47287359/hive-reach-max-worker-and-cannot-connect-to-hiveserver2 -->
+    <property>
+      <name>hive.server2.authentication</name>
+      <value>NOSASL</value>
+    </property>
+
+     <property>
+        <name>hive.server2.enable.doAs</name>
+        <value>false</value>
+    </property>
+
+    <property>
+      <name>tez.lib.uris</name>
+      <value>file:///usr/hdp/3.1.0.0-78/tez</value>
+    </property>
+
+    <property>
+      <name>hive.support.concurrency</name>
+      <value>true</value>
+    </property>
+
+    <property>
+      <name>hive.txn.manager</name>
+      <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
+    </property>
+
+    <property>
+      <name>tez.lib.uris</name>
+      <value>file:///usr/hdp/3.1.0.0-78/tez/lib/tez.tar.gz</value>
+    </property>
 </configuration>
diff --git a/prestodev/hdp3-hive3/files/etc/supervisord.d/mysql-metastore.conf b/prestodev/hdp3-hive3/files/etc/supervisord.d/mysql-metastore.conf
@@ -1,5 +1,5 @@
 [program:mysql-metastore]
-command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
+command=/bin/bash -c "(chown -R mysql:mysql /var/lib/mysql) && (/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe)"
 startsecs=2
 stopwaitsecs=10
 user=mysql

diff --git a/prestodev/hdp3-hive3/files/etc/tez/conf/tez-site.xml b/prestodev/hdp3-hive3/files/etc/tez/conf/tez-site.xml
@@ -0,0 +1,147 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+<property>
+<name>tez.lib.uris.ignore</name>
+<value>false</value>
+</property>
+
+<property>
+<name>tez.lib.uris</name>
+<value>file:///usr/hdp/3.1.0.0-78/tez/lib/tez.tar.gz</value>
+</property>
+<property>
+  <name>tez.am.mode.session</name>
+  <value>false</value>
+</property>
+<property>
+  <name>tez.am.acl.enabled</name>
+  <value>false</value>
+</property>
+
+<property>
+  <name>tez.am.log.level</name>
+  <value>WARN</value>
+</property>
+<property>
+  <name>tez.task.log.level</name>
+  <value>WARN</value>
+</property>
+
+<property>
+  <name>tez.runtime.io.sort.mb</name>
+  <value>512</value>
+</property>
+
+<property>
+  <name>tez.shuffle-vertex-manager.enable.auto-parallel</name>
+  <value>true</value>
+</property>
+<property>
+  <name>tez.am.max.app.attempts</name>
+  <value>1</value>
+</property>
+
+<property>
+  <name>tez.am.task.max.failed.attempts</name>
+  <value>1</value>
+</property>
+
+<!--
+<property>
+  <name>tez.shuffle-vertex-manager.desired-task-input-size</name>
+  <value>true</value>
+</property>
+-->
+
+<property>
+  <name>tez.shuffle-vertex-manager.min-src-fraction</name>
+  <value>0.10</value>
+</property>
+<property>
+  <name>tez.shuffle-vertex-manager.max-src-fraction</name>
+  <value>1.00</value>
+</property>
+
+<property>
+  <name>tez.am.launch.cmd-opts</name>
+  <value>-server -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC -Dhadoop.metrics.log.level=WARN</value>
+</property>
+<property>
+  <name>tez.am.resource.memory.mb</name>
+  <value>1024</value>
+</property>
+
+<property>
+  <name>tez.task.launch.cmd-opts</name>
+  <value>-server -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC -Dhadoop.metrics.log.level=WARN</value>
+</property>
+<property>
+  <name>tez.task.resource.memory.mb</name>
+  <value>1024</value>
+</property>
+<property>
+  <name>tez.task.resource.cpu.vcores</name>
+  <value>1</value>
+</property>
+<!--
+<property>
+  <name>tez.runtime.sort.spill.percent</name>
+  <value>0.5</value>
+</property>
+<property>
+  <name>tez.runtime.shuffle.merge.percent</name>
+  <value>0.5</value>
+</property>
+<property>
+  <name>tez.runtime.shuffle.buffersize</name>
+  <value>8192</value>
+</property>
+-->
+<property>
+  <name>tez.runtime.sort.threads</name>
+  <value>1</value>
+</property>
+<property>
+  <name>tez.runtime.io.sort.factor</name>
+  <value>100</value>
+</property>
+<!--
+<property>
+  <name>tez.runtime.task.input.buffer.percent</name>
+  <value>0.2</value>
+</property>
+<property>
+  <name>shuffle.memory.limit.percent</name>
+  <value>0.5</value>
+</property>
+-->
+<property>
+  <name>tez.runtime.shuffle.memory-to-memory.enable</name>
+  <value>false</value>
+</property>
+<property>
+  <name>tez.runtime.optimize.local.fetch</name>
+  <value>true</value>
+</property>
+
+<property>
+  <name>hive.tez.container.size</name>
+  <value>1024</value>
+</property>
+
+</configuration>
diff --git a/prestodev/hdp3-hive3/files/root/setup.sh b/prestodev/hdp3-hive3/files/root/setup.sh
@@ -6,6 +6,12 @@ echo "127.0.0.1 hadoop-master" >> /etc/hosts
 # 1 format namenode
 chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/
 
+mkdir -p /usr/hdp/3.1.0.0-78/hadoop/logs
+mkdir -p /var/log/hadoop-hdfs
+mkdir -p  /var/log/hadoop-yarn
+sudo chgrp -R hadoop /usr/hdp/3.1.0.0-78/hadoop/logs /var/log/hadoop-hdfs /var/log/hadoop-yarn
+chmod -R 770 /usr/hdp/3.1.0.0-78/hadoop/logs /var/log/hadoop-hdfs /var/log/hadoop-yarn
+
 # workaround for 'could not open session' bug as suggested here:
 # https://github.com/docker/docker/issues/7056#issuecomment-49371610
 rm -f /etc/security/limits.d/hdfs.conf
@@ -39,12 +45,18 @@ killall java
 # 7 setup metastore
 mysql_install_db
 
+# Without this getting perm error https://dba.stackexchange.com/questions/56441/cant-find-file-mysql-plugin-frm-when-starting-up-mysql
+chown -R mysql:mysql /var/lib/mysql
+chgrp -R mysql /var/lib/mysql
+rm -f  /var/lib/mysql/ib_logfile0
+rm -f  /var/lib/mysql/ib_logfile1
+
 /usr/bin/mysqld_safe &
 sleep 10s
 
 cd /usr/hdp/current/hive-metastore/scripts/metastore/upgrade/mysql/
 echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
-echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.2.1000.mysql.sql;" | mysql
+echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-3.1.0.mysql.sql;" | mysql
 /usr/bin/mysqladmin -u root password 'root'
 
 killall mysqld