-
Notifications
You must be signed in to change notification settings - Fork 0
/
Dockerfile
executable file
·148 lines (118 loc) · 4.91 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
FROM amd64/openjdk:8
MAINTAINER CHOJUAN
RUN apt-get update -y && \
apt-get -y install sudo && \
apt-get install curl -y && \
apt-get install vim -y && \
apt-get install wget -y && \
apt-get install ssh -y && \
apt-get install rsync -y && \
#apt-get install mysql-server -y && \
apt install maven -y && apt-get install zsh -y && \
apt-get install git-core -y && sudo apt install netcat -y
## ZSH
RUN wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | zsh
RUN chsh -s `which zsh` && zsh
#RUN git clone https://github.com/zsh-users/zsh-autosuggestions $HOME/.zsh/zsh-autosuggestions
#RUN echo "/bin/bash -c 'source $HOME/.zsh/zsh-autosuggestions/zsh-autosuggestions.zsh'" >> $HOME/.zshrc && ls -la $HOME
#RUN /bin/bash -c "source $HOME/.zshrc"
## END ZSH
ENV DEBIAN_FRONTEND noninteractive
# SSH KEYS
RUN mkdir ~/.ssh && ssh-keygen -q -t rsa -P '' -f ~/.ssh/id_rsa
RUN cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
#END SSH
# DOWNLOAD HADOOP, PIG, HIVE, SQOOP
RUN wget https://archive.apache.org/dist/hadoop/core/hadoop-2.7.2/hadoop-2.7.2.tar.gz
RUN tar xfz hadoop-2.7.2.tar.gz
RUN mkdir -p /home/bigdata/hadoop
RUN mv hadoop-2.7.2 /home/bigdata/hadoop
RUN wget https://dlcdn.apache.org/pig/pig-0.16.0/pig-0.16.0.tar.gz
RUN tar xfz pig-0.16.0.tar.gz
RUN mkdir /home/bigdata/pig
RUN mv pig-0.16.0 /home/bigdata/pig
RUN wget https://archive.apache.org/dist/hive/hive-2.1.0/apache-hive-2.1.0-bin.tar.gz
RUN tar xfz apache-hive-2.1.0-bin.tar.gz
RUN mkdir /home/bigdata/hive
RUN mv apache-hive-2.1.0-bin/* /home/bigdata/hive/
RUN wget https://archive.apache.org/dist/sqoop/1.4.7/sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz
RUN tar xfz sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz
RUN mkdir /home/bigdata/sqoop
RUN mv sqoop-1.4.7.bin__hadoop-2.6.0/* /home/bigdata/sqoop/
RUN wget https://dlcdn.apache.org/spark/spark-3.3.1/spark-3.3.1-bin-hadoop2.tgz
RUN tar -xvf spark-3.3.1-bin-hadoop2.tgz
RUN mkdir /home/bigdata/spark
RUN mv spark-3.3.1-bin-hadoop2/* /home/bigdata/spark/
RUN wget https://dlcdn.apache.org/flume/1.11.0/apache-flume-1.11.0-bin.tar.gz
RUN tar -xvf apache-flume-1.11.0-bin.tar.gz
RUN mkdir /home/bigdata/flume
RUN mv apache-flume-1.11.0-bin/* /home/bigdata/flume/
RUN wget https://dlcdn.apache.org/flink/flink-1.16.0/flink-1.16.0-bin-scala_2.12.tgz
RUN tar -xvf flink-1.16.0-bin-scala_2.12.tgz
RUN mkdir /home/bigdata/flink
RUN mv flink-1.16.0/* /home/bigdata/flink/
#END DOWNLOAD
# CONFIG FLINK
ENV FLINK_HOME=/home/bigdata/flink
ENV PATH=$PATH:$FLINK_HOME/bin
COPY configs/flink-conf.yaml $FLINK_HOME/conf/
# END FLINK CONFIG
# CONFIG HADOOP
ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
ENV HADOOP_HOME=/home/bigdata/hadoop/hadoop-2.7.2
ENV PATH=$PATH:$HADOOP_HOME/bin
ENV PATH=$PATH:$HADOOP_HOME/sbin
ENV HADOOP_MAPRED_HOME=$HADOOP_HOME
ENV HADOOP_COMMON_HOME=$HADOOP_HOME
ENV HADOOP_HDFS_HOME=$HADOOP_HOME
ENV HADOOP_YARN_HOME=$HADOOP_HOME
ENV HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
COPY configs/core-site.xml $HADOOP_HOME/etc/hadoop/
RUN echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh
RUN echo "export HADOOP_SSH_OPTS='-p 22' " >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh
RUN mkdir -p /home/bigdata/hadoop_store/hdfs/namenode
RUN mkdir -p /home/bigdata/hadoop_store/hdfs/datanode
COPY configs/hdfs-site.xml $HADOOP_HOME/etc/hadoop/
RUN cp $HADOOP_HOME/etc/hadoop/mapred-site.xml.template $HADOOP_HOME/etc/hadoop/mapred-site.xml
COPY configs/mapred-site.xml $HADOOP_HOME/etc/hadoop/
COPY configs/yarn-site.xml $HADOOP_HOME/etc/hadoop/
#END CONFIG
# CONFIG PIG
ENV PIG_HOME=/home/bigdata/pig/pig-0.16.0
ENV PATH=$PATH:$PIG_HOME/bin
COPY configs/.pigbootup $PIG_HOME/
RUN echo "pig.load.default.statements = /home/bigdata/pig/pig-0.16.0/.pigbootup" >> $PIG_HOME/conf/pig.properties
RUN echo "log4j.rootLogger=fatal" >> $PIG_HOME/conf/nolog.conf
# END COONFIG
#CONFIG HIVE
ENV HIVE_HOME=/home/bigdata/hive
ENV PATH=$PATH:$HIVE_HOME/bin
#problemas al iniciar hive, ejecutar esto y volver a lanzarlo. $HOME.
#schematool -initSchema -dbType derby
#mv metastore_db metastore_db.tmp
#schematool -initSchema -dbType derby
#END CONFIG
#Sqoop START
ENV SQOOP_HOME=/home/bigdata/sqoop
ENV PATH=$PATH:$SQOOP_HOME/bin
COPY configs/sqoop-env.sh $SQOOP_HOME/conf/
ENV HCAT_HOME=$HIVE_HOME/hcatalog/
ENV PATH=$HCAT_HOME/bin:$PATH
#REVISAR ESTÓ !!!
#RUN wget http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar
#RUN mv mysql-connector-java-5.1.38.jar $SQOOP_HOME/lib/
#Sqoop END
#SPARK CONFIG
ENV SPARK_HOME=/home/bigdata/spark
ENV PATH=$PATH:$SPARK_HOME/bin
COPY configs/spark-env.sh $SPARK_HOME/conf/
###
#FLUME CONFIG
ENV FLUME_HOME=/home/bigdata/flume
ENV PATH=$PATH:$FLUME_HOME/bin
###
COPY run.sh $HOME
EXPOSE 50070 8088 19888
ENTRYPOINT ["/bin/bash"]
#RUN ssh-copy-id -i $HOME/.ssh/id_rsa.pub root@$SERVICE_HOST_NAME