Hadoop on OSX

setting up hadoop on OSX

create a hadoop user in settings

su to hadoop


su - hadoop
ssh-keygen -t rsa -P ""

cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys

download hadoop

http://mirror.metrocast.net/apache/hadoop/common/hadoop-1.0.4/hadoop-1.0.4.tar.gz

in bash profile

nano .bash_profile


export PATH=$PATH:~/bin
export JAVA_HOME=/Library/Java/Home
export HADOOP_HOME=/Users/hadoop/data/workspace/hadoop-1.0.4



link to hadoop binary

ln -s /Users/hadoop/data/workspace/hadoop-1.0.4/bin/hadoop hadoop

set config

nano conf/hadoop-env.sh

# The java implementation to use.  Required.
export JAVA_HOME=/Library/Java/Home

# The maximum amount of heap to use, in MB. Default is 1000.
export HADOOP_HEAPSIZE=2000

nano conf/core-site.xml



   
        hadoop.tmp.dir
        /Users/${user.name}/data/hadoop-store
        A base for other temporary directories.
   
   
        fs.default.name
        hdfs://localhost:8020
   


nano conf/mapred-site.xml




   
      mapred.job.tracker
      localhost:9001
   

   
        mapred.tasktracker.map.tasks.maximum
        2
   

   
        mapred.tasktracker.reduce.tasks.maximum
        2
   


nano conf/hdfs-site.xml

<configuration>

    <property>

      <name>dfs.replication</name>

      <value>1</value>

    </property>

</configuration>




# format name node

./bin/hadoop namenode -format