Showing posts with label setup. Show all posts
Showing posts with label setup. Show all posts

Monday, February 1, 2010

Setup Hadoop

# setup hadoop

#sudo adduser --ingroup hadoop hadoop

sudo useradd -d /home/hadoop -m hadoop
sudo usermod -s /bin/bash hadoop

sudo passwd hadoop

su - hadoop
ssh-keygen -t rsa -P ""

cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys

sudo cp hadoop-0.20.1.tar.gz /usr/local
cd /usr/local
sudo tar xzf hadoop-0.20.1.tar.gz
sudo chown -R hadoop:hadoop hadoop-0.20.1/
sudo ln -s /usr/local/hadoop-0.20.1/ /usr/local/hadoop

nano /usr/local/hadoop/conf/hadoop-
env.sh
set:
export JAVA_HOME=/usr/lib/jvm/java-6-sun
# allows hadoop to bind to 0.0.0.0 without binding to IP6 interface
export HADOOP_OPTS=-Djava.net.preferIPv4Stack=true

edit:
core-site.xml
hdfs-site.xml
mapred-site.xml

# format name node
./bin/hadoop namenode -format

# start cluster
./bin/start-all.sh

# command should list hadoop processes
jps

#----------------
# Test

./bin/hadoop dfs -copyFromLocal /tmp/davinci/ davinci
./bin/hadoop dfs -ls
./bin/hadoop dfs -ls davinci

Install MogileFS on Ubuntu

add repository

add to /etc/apt/sources.list
deb http://ppa.launchpad.net/awmcclain/ppa/ubuntu jaunty main

install mogile

sudo apt-get update
sudo apt-get install mogilefsd mogstored

install mysql

sudo apt-get install mysql-server
mysql -u root -p
CREATE DATABASE mogilefs;
CREATE USER 'mogilefs'@'localhost' IDENTIFIED BY 'mogilefs';
GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,ALTER ON mogilefs.* TO 'mogilefs'@'localhost';
flush privileges;
commit;

mogdbsetup --dbhost=localhost --dbname=mogilefs --dbuser=mogilefs --dbpassword=mogilefs

add mogadmin

sudo apt-get install build-essential
sudo perl -MCPAN -e shell;
install MogileFS::Utils

edit configs

sudo nano /etc/mogilefs/mogilefsd.conf

* change 'db_user' to 'mogilefs'
* change 'db_pass' to 'mogilefs'
* change 'listen' to '0.0.0.0:7001' (bind on all local IPs)

start mogile

sudo /etc/init.d/mogilefsd start

configure storage hosts/devices

mogadm --trackers=10.31.248.128:7001 host add 10.31.248.128 --status=alive
mogadm --trackers=10.31.248.128:7001 device add 10.31.248.128 1
sudo mkdir -p /usr/mogdata/dev1
sudo chown mogstored /usr/mogdata/dev1
mogadm --trackers=10.31.248.128:7001 domain add companyRawRecords
mogadm --trackers=10.31.248.128:7001 class add companyRawRecords companyRawRecords --mindevcount=1

for additional storage nodes on tracker node
mogadm --trackers=10.31.248.128:7001 host add 10.31.248.127 --status=alive
mogadm --trackers=10.31.248.128:7001 device add 10.31.248.127 2
on additional storage nodes
sudo mkdir -p /usr/mogdata/dev2
sudo chown mogstored /usr/mogdata/dev2

run Apache for storage
stop mogstored

sudo /etc/init.d/mogstored stop

edit /etc/init.d/mogstored

add -- --server=none to the following line (need the double -- --)

start-stop-daemon --start --quiet --exec $DAEMON --pidfile $PIDFILE -b -m --name $NAME --chuid $MOGSTORED_RUNASUSER -- --server=none

install/configure apache

sudo apt-get install apache2

edit /etc/apache2/sites-available/mogilefs

Listen 7500

DocumentRoot /usr/mogdata
CustomLog logs/mogilefs_log common


Options +Indexes +FollowSymLinks



DAV On
Order Deny,Allow
Deny from all
Allow from 127.0.0.1


run:

sudo a2enmod dav
sudo a2enmod dav_fs

sudo a2ensite mogilefs

sudo groupadd mogstored
sudo chown -R mogstored:mogstored /usr/mogdata/
sudo usermod -G mogstored www-data
sudo usermod -G mogstored mogstored
sudo chmod -R g+w /usr/mogdata

sudo /etc/init.d/apache2 force-reload
sudo /etc/init.d/apache2 stop
sudo /etc/init.d/apache2 start

patch mogilefs

download attached patch

sudo patch /usr/share/perl5/MogileFS/Device.pm <>
install configure lighthttpd (do this OR apache)


sudo apt-get install lighttpd lighttpd-mod-webdav apache2-utils
sudo /etc/init.d/lighttpd stop



sudo nano /etc/lighttpd/lighttpd.conf
uncomment "mod_webdav"
set document root to the mogile storage location
set port to 7500


browse MySql database remotely


on tracker machine edit /etc/mysql/my.cnf

change 'bind-address' to '0.0.0.0'

run:


mysql -u root -p
mysql> GRANT ALL PRIVILEGES ON *.* TO root@'%' IDENTIFIED BY 'password';



restart mysql:


sudo /etc/init.d/mysql restart

on a remote machine, install query browser tools - http://dev.mysql.com/downloads/gui-tools/5.0.html

run query browser and connect, setting the default schema to 'mogilefs'