cloud-hw3

installation

git clone https://github.com/big-data-europe/docker-hadoop.git
cp Dockerfile docker-hadoop/namenode/Dockerfile

run hadoop

cd docker-hadoop
docker-compose up

find hadoop streaming jar

find / -name *streaming*.jar
stream="path to hadoop streaming jar"

run task

docker cp ./mapper.py namenode:/home/
docker cp ./reducer.py namenode:/home/
docker cp ./access_log.txt namenode:/home/
docker exec -it namenode /bin/bash
hadoop fs -mkdir /poodah
hadoop fs -mkdir /poodah/in
hdfs dfs -put access_log.txt /poodah/in
hadoop jar $stream -file mapper.py -mapper mapper.py --file reducer.py -reducer reducer.py -input /poodah/in -output /poodah/out
hdfs dfs -get /poodah/out/ ./