-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbootstrap.sh
76 lines (71 loc) · 2.95 KB
/
bootstrap.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
#!/usr/bin/env bash
apt-get update
apt-get install -y python3 default-jre python3-pip
# a simple two-panel file commander
apt-get install -y mc
# uncomment one of the following for graphical desktops
# NOTE: the graphical desktop is accessible through
# the main VirtualBox window (Show button)
#
# - minimal: wm & graphical server
# apt-get install -y icewm xinit xterm python3-tk
#
# - minimal desktop env: lxqt
# apt-get install -y xinit lxqt
#
# cd to the shared directory
cd /vagrant
# python packages
pip3 install matplotlib pandas seaborn
# jupyter
pip3 install jupyter
# uncomment and modify to remove a previously installed Spark version
# rm -rf /usr/local/spark-3.0.0-preview2-bin-hadoop2.7
# since the download was broken we do not remove the
# compressed folder from the root directory
if ! [ -d /usr/local/spark-3.3.1-bin-hadoop3 ]; then
# current link as of 2022-11-28:
wget https://dlcdn.apache.org/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz
tar -C /usr/local -xvzf spark-3.3.1-bin-hadoop3.tgz
fi
if ! [ -d /usr/local/hadoop-3.3.4 ]; then
wget https://downloads.apache.org/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz
tar -C /usr/local -xvzf hadoop-3.3.4.tar.gz
chown --recursive ubuntu:ubuntu /usr/local/hadoop-3.3.4
rm hadoop-3.3.4.tar.gz
fi
if ! grep "export HADOOP_INSTALL=/usr/local/hadoop-3.3.4" /home/vagrant/.bashrc; then
echo "export HADOOP_INSTALL=/usr/local/hadoop-3.3.4" >> /home/vagrant/.bashrc
fi
if ! grep "export HADOOP_HOME=/usr/local/hadoop-3.3.4" /home/vagrant/.bashrc; then
echo "export HADOOP_HOME=/usr/local/hadoop-3.3.4" >> /home/vagrant/.bashrc
fi
if ! grep "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" /home/vagrant/.bashrc; then
echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" >> /home/vagrant/.bashrc
fi
if ! grep "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" /usr/local/hadoop-3.3.4/etc/hadoop/hadoop-env.sh; then
echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" >> /usr/local/hadoop-3.3.4/etc/hadoop/hadoop-env.sh
fi
if ! grep "export HADOOP_MAPRED_HOME=/usr/local/hadoop-3.3.4" /home/vagrant/.bashrc; then
echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop-3.3.4" >> /home/vagrant/.bashrc
fi
if ! grep "export PYSPARK_PYTHON=/usr/bin/python3" /home/vagrant/.bashrc; then
echo "export PYSPARK_PYTHON=/usr/bin/python3" >> /home/vagrant/.bashrc
fi
if ! grep "export PYSPARK_DRIVER_PYTHON=jupyter" /home/vagrant/.bashrc; then
echo "export PYSPARK_DRIVER_PYTHON=jupyter" >> /home/vagrant/.bashrc
fi
if ! grep "export PYSPARK_DRIVER_PYTHON_OPTS=notebook" /home/vagrant/.bashrc; then
echo "export PYSPARK_DRIVER_PYTHON_OPTS=notebook" >> /home/vagrant/.bashrc
fi
# switch to user ubuntu
sudo -i -u ubuntu bash << EOF
echo "Switched to user ubuntu"
if ! ( echo exit | ssh localhost ) ; then
echo "Creating keys and authorizing"
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys
fi
EOF
echo "Exited user ubuntu"