Tuesday, April 2, 2013

Hadoop on OSX


setting up hadoop on OSX

create a hadoop user in settings

su to hadoop


su - hadoop
ssh-keygen -t rsa -P ""

cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys


download hadoop

http://mirror.metrocast.net/apache/hadoop/common/hadoop-1.0.4/hadoop-1.0.4.tar.gz

in bash profile

nano .bash_profile


export PATH=$PATH:~/bin
export JAVA_HOME=/Library/Java/Home
export HADOOP_HOME=/Users/hadoop/data/workspace/hadoop-1.0.4



link to hadoop binary

ln -s /Users/hadoop/data/workspace/hadoop-1.0.4/bin/hadoop hadoop

set config

nano conf/hadoop-env.sh

# The java implementation to use.  Required.
export JAVA_HOME=/Library/Java/Home

# The maximum amount of heap to use, in MB. Default is 1000.
export HADOOP_HEAPSIZE=2000

nano conf/core-site.xml



   
        hadoop.tmp.dir
        /Users/${user.name}/data/hadoop-store
        A base for other temporary directories.
   
   
        fs.default.name
        hdfs://localhost:8020
   


nano conf/mapred-site.xml




   
      mapred.job.tracker
      localhost:9001
   

   
        mapred.tasktracker.map.tasks.maximum
        2
   

   
        mapred.tasktracker.reduce.tasks.maximum
        2
   


nano conf/hdfs-site.xml


<configuration>
    <property>
      <name>dfs.replication</name>
      <value>1</value>
    </property>
</configuration>

# format name node
./bin/hadoop namenode -format





Wednesday, March 3, 2010

git ommands

git commit -m "added a check for nil"
105 git remote
106 git push origin master

119 git push origin master
120 git pull origin master

Tuesday, March 2, 2010

ubuntu resolution

cvt 1900 1200

creates modline

xrandr --newmode 1900x1200 63.50 1900 1072 1176 1328 1200 771 775 798 -hsync +vsync

xrandr --addmode VBOX1 1900x1200

Since above command is not permanent sollution to add new resolution, you need to edit /etc/gdm/PreSession/Default
Enter followings at the end of file /etc/gdm/PreSession/Default

  1. xrandr --newmode "1240x1024_60.00" 105.50 1240 1320 1448 1656 1024 1027 1037 1063 -hsync +vsync
  2. xrandr --addmode VGA1 1240x1024_60.00

setting up rails on ubuntu

install mysql-server
install mysql-admin mysql-query-browser

sudo apt-get install git-gui
sudo apt-get install ruby-full build-essential
sudo gem install rails
sudo apt-get install libmysql-ruby libmysqlclient-dev
sudo gem install mysql

database.yml

adapter: mysql
username: hospital
password: hospital
database: hospital
encoding: UTF8
timeout: 5000
database: mynewapp_development
pool: 5
socket: /var/run/mysqld/mysqld.sock


IMPORT data

mysql --verbose --user=XXXXXXXX --password=XXXXXXXX DB_NAME < /PATH/TO/DUMPFILE.SQL gem install mongrel sudo apt-get install apache2
sudo apt-get install apache2
sudo apt-get install libapache2-mod-proxy-html
sudo a2enmod proxy
sudo ln -s /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled/proxy_http.load

nano /etc/apache2/sites-available/hospitalcompare

Listen 8000
<VirtualHost *:8000>
# ServerName otherserver.me

ProxyPass /hospital_compare http://localhost:3000/hospital_compare
ProxyPassReverse /hospital_compare http://localhost:3000/hospital_compare
ProxyPreserveHost on

<Proxy *>
Order allow,deny
Allow from all
</Proxy>
</VirtualHost>

copy and untar drupal stuff to /var/www/drupal
http://mugdemo.info/drupal.tar


sudo a2ensite hospitalcompare



Tuesday, February 9, 2010

xorg.conf

# xorg.conf (X.Org X Window System server configuration file)
#
# This file was generated by dexconf, the Debian X Configuration tool, using
# values from the debconf database.
#
# Edit this file with caution, and see the xorg.conf manual page.
# (Type "man xorg.conf" at the shell prompt.)
#
# This file is automatically updated on xserver-xorg package upgrades *only*
# if it has not been modified since the last upgrade of the xserver-xorg
# package.
#
# Note that some configuration settings that could be done previously
# in this file, now are automatically configured by the server and settings
# here are ignored.
#
# If you have edited this file but would like it to be automatically updated
# again, run the following command:
# sudo dpkg-reconfigure -phigh xserver-xorg

Section "Monitor"
Identifier "Configured Monitor"
DisplaySize 1680 1050
EndSection

Section "Screen"
Identifier "Default Screen"
Monitor "Configured Monitor"
Device "Configured Video Device"
SubSection "Display"
Virtual 2720 900
EndSubSection
EndSection

Section "Device"
Identifier "Configured Video Device"
EndSection

Monday, February 1, 2010

jboss startup script

#!/bin/sh
#
# $Id: jboss_init_redhat.sh 71252 2008-03-25 17:52:00Z dbhole $
#
# JBoss Control Script
#
# To use this script run it as root - it will switch to the specified user
#
# Here is a little (and extremely primitive) startup/shutdown script
# for RedHat systems. It assumes that JBoss lives in /usr/local/jboss,
# it's run by user 'jboss' and JDK binaries are in /usr/local/jdk/bin.
# All this can be changed in the script itself.
#
# Either modify this script for your requirements or just ensure that
# the following variables are set correctly before calling the script.

#define where jboss is - this is the directory containing directories log, bin, conf etc
JBOSS_HOME=${JBOSS_HOME:-"/home/jboss/jboss-4.2.3.GA"}

#define the user under which jboss will run, or use 'RUNASIS' to run as the current user
JBOSS_USER=jboss

#make sure java is in your path
JAVAPTH=${JAVAPTH:-"/usr/local/jdk/bin"}

#configuration to use, usually one of 'minimal', 'default', 'all'
#JBOSS_CONF=${JBOSS_CONF:-"default"}
JBOSS_CONF=${JBOSS_CONF:-"all"}

#if JBOSS_HOST specified, use -b to bind jboss services to that address
JBOSS_BIND_ADDR=${JBOSS_HOST:-"-b 10.31.140.79"}

#define the script to use to start jboss
JBOSSSH=${JBOSSSH:-"$JBOSS_HOME/bin/run.sh -c $JBOSS_CONF $JBOSS_BIND_ADDR"}

JBOSS_CONSOLE=/dev/null

if [ "$JBOSS_USER" = "RUNASIS" ]; then
SUBIT=""
else
SUBIT="su - $JBOSS_USER -c "
fi

if [ -n "$JBOSS_CONSOLE" -a ! -d "$JBOSS_CONSOLE" ]; then
# ensure the file exists
touch $JBOSS_CONSOLE
if [ ! -z "$SUBIT" ]; then
chown $JBOSS_USER $JBOSS_CONSOLE
fi
fi

if [ -n "$JBOSS_CONSOLE" -a ! -f "$JBOSS_CONSOLE" ]; then
echo "WARNING: location for saving console log invalid: $JBOSS_CONSOLE"
echo "WARNING: ignoring it and using /dev/null"
JBOSS_CONSOLE="/dev/null"
fi

#define what will be done with the console log
JBOSS_CONSOLE=${JBOSS_CONSOLE:-"/dev/null"}

JBOSS_CMD_START="cd $JBOSS_HOME/bin; $JBOSSSH"

if [ -z "`echo $PATH | grep $JAVAPTH`" ]; then
export PATH=$PATH:$JAVAPTH
fi

if [ ! -d "$JBOSS_HOME" ]; then
echo JBOSS_HOME does not exist as a valid directory : $JBOSS_HOME
exit 1
fi

echo JBOSS_CMD_START = $JBOSS_CMD_START

procrunning() {
procid=0
# JBOSSSCRIPT=$(echo $JBOSSSH | awk '{print $1}' | sed 's/\//\\\//g')
JBOSSSCRIPT=run.sh
for procid in `/bin/pidof -x "$JBOSSSCRIPT"`; do
ps -fp $procid | grep "${JBOSSSH% *}" > /dev/null && pid=$procid
done
}


stop() {
pid=0
procrunning
if [ "x$pid" = "x0" ]; then
echo -n -e "\nNo JBossas is currently running\n"
exit 1
fi

RETVAL=1

# If process is still running

# First, try to kill it nicely
for id in `ps --ppid $pid | awk '{print $1}' | grep -v "^PID$"`; do
if [ -z "$SUBIT" ]; then
kill -15 $id
else
$SUBIT "kill -15 $id"
fi
done

sleep=0
while [ $sleep -lt 120 -a $RETVAL -eq 1 ]; do
echo -n "\nwaiting for processes to stop";
sleep 10
sleep=`expr $sleep + 10`
pid=0
procrunning
if [ "x$pid" != "x0" ]; then
echo "duh" > /dev/null
else
RETVAL=0
fi
done

# Still not dead... kill it

count=0
pid=0
procrunning

if [ "x$RETVAL" != "x0" ] ; then
echo -e "\nTimeout: Shutdown command was sent, but process is still running with

PID $pid"
exit 1
fi

echo
exit 0
}

case "$1" in
start)

echo "user=$USER jboss_user=$JBOSS_USER jboss_home=$JBOSS_HOME" > /home/jboss/blah.txt
echo "command=$JBOSS_CMD_START console=$JBOSS_CONSOLE" >> /home/jboss/blah.txt
cd $JBOSS_HOME/bin
if [ -z "$SUBIT" ]; then
eval $JBOSS_CMD_START >${JBOSS_CONSOLE} 2>&1 &
else
$SUBIT "$JBOSS_CMD_START >${JBOSS_CONSOLE} 2>&1 &"
fi
;;
stop)
stop
;;
restart)
$0 stop
$0 start
;;
*)
echo "usage: $0 (start|stop|restart|help)"
esac

solr startup script

#!/bin/sh

# Starts, stops, and restarts solr

SOLR_DIR="/usr/solr-root"
JAVA_OPTIONS="-Xms2g -Xmx6g -DSTOP.PORT=8079 -DSTOP.KEY=zoomsolrstop -jar start.jar"
LOG_FILE="/var/log/solr/solr.log"
JAVA="/usr/bin/java"
SOLR_USER="solr"

case $1 in
start)
echo "Starting Solr"
cd $SOLR_DIR
sudo -u $SOLR_USER $JAVA $JAVA_OPTIONS 2> $LOG_FILE &
;;
stop)
echo "Stopping Solr"
cd $SOLR_DIR
sudo -u $SOLR_USER $JAVA $JAVA_OPTIONS --stop
;;
restart)
$0 stop
sleep 1
$0 start
;;
*)
echo "Usage: $0 {start|stop|restart}" >&2
exit 1
;;
esac