Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ignore this PR #6

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM openjdk:8u242-jre
FROM openjdk:18-slim-bullseye

WORKDIR /opt

Expand All @@ -8,7 +8,9 @@ ENV METASTORE_VERSION=3.0.0
ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION}
ENV HIVE_HOME=/opt/apache-hive-metastore-${METASTORE_VERSION}-bin

RUN curl -L https://www-us.apache.org/dist/hive/hive-standalone-metastore-${METASTORE_VERSION}/hive-standalone-metastore-${METASTORE_VERSION}-bin.tar.gz | tar zxf - && \
RUN apt-get update -y && apt-get install -y net-tools netcat curl vim && \

curl -L https://downloads.apache.org/hive/hive-standalone-metastore-3.0.0/hive-standalone-metastore-3.0.0-bin.tar.gz | tar zxf - && \
curl -L https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | tar zxf - && \
curl -L https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.19.tar.gz | tar zxf - && \
cp mysql-connector-java-8.0.19/mysql-connector-java-8.0.19.jar ${HIVE_HOME}/lib/ && \
Expand All @@ -25,4 +27,5 @@ RUN groupadd -r hive --gid=1000 && \
USER hive
EXPOSE 9083

ENTRYPOINT ["sh", "-c", "/entrypoint.sh"]
ENTRYPOINT ["sh", "-c", "/entrypoint.sh"]

5 changes: 2 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@

### About

Example of running standalone Hive Metastore. Minio is used as S3 storage for external
tables.
Example of running standalone Hive Metastore. Note that this particular fork omits Minio, which means you may want
to deploy other S3 storage for external tables. Check the original github repo ( https://github.com/andypern/hive-metastore-docker ) for an example which includes Minio.

It contains following containers:
- mariadb as dependency
- minio to test S3 access (make sure that you specify correct volume to be mounted)
- hive metastore 3.x

### How to run
Expand Down
6 changes: 3 additions & 3 deletions conf/metastore-site.xml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
</property>
<property>
<name>metastore.warehouse.dir</name>
<value>s3a://spark/warehouse/</value>
<value>s3a://bucket/warehouse/</value>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
Expand All @@ -23,7 +23,7 @@

<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://mariadb:3306/metastore_db</value>
<value>jdbc:mysql://localhost:3306/metastore_db</value>
</property>

<property>
Expand All @@ -46,7 +46,7 @@
</property>
<property>
<name>fs.s3a.endpoint</name>
<value>http://minio:9000</value>
<value>http://my.s3.endpoint:80</value>
</property>
<property>
<name>fs.s3a.path.style.access</name>
Expand Down
15 changes: 3 additions & 12 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ version: "2"

services:
mariadb:
image: mariadb:latest
network_mode: host
image: mariadb:10.7
ports:
- 3306:3306
environment:
Expand All @@ -11,19 +12,9 @@ services:
MYSQL_PASSWORD: admin
MYSQL_DATABASE: metastore_db

# make sure that you specify correct volume to be mounted
minio:
image: minio/minio
environment:
- MINIO_ACCESS_KEY=accesskey
- MINIO_SECRET_KEY=secretkey
volumes:
- /data1/minio:/data
ports:
- 9000:9000
command: server /data

hive-metastore:
network_mode: host
build: .
image: hive-metastore:latest
ports:
Expand Down
32 changes: 0 additions & 32 deletions kubernetes/minio.yaml

This file was deleted.

27 changes: 0 additions & 27 deletions kubernetes/minio_srv.yml

This file was deleted.

18 changes: 17 additions & 1 deletion scripts/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,23 @@

export HADOOP_HOME=/opt/hadoop-3.2.0
export HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar:${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-aws-3.2.0.jar
export JAVA_HOME=/usr/local/openjdk-8
export JAVA_HOME=/usr/local/openjdk-18

#this ensures that the mariadb instance is running before we try to start hive-meta

echo "sleeping for 60s to let mariadb spin up"
sleep 10
echo "sleeping for 50s.."
sleep 10
echo "sleeping for 40s"
sleep 10
echo "sleeping for 30s"
sleep 10
echo "sleeping for 20s"
sleep 10
echo "sleeping for 10s"
sleep 10
echo "tired of sleeping..time to work"

/opt/apache-hive-metastore-3.0.0-bin/bin/schematool -initSchema -dbType mysql
/opt/apache-hive-metastore-3.0.0-bin/bin/start-metastore