xorg.conf

# xorg.conf (X.Org X Window System server configuration file)
#
# This file was generated by dexconf, the Debian X Configuration tool, using
# values from the debconf database.
#
# Edit this file with caution, and see the xorg.conf manual page.
# (Type "man xorg.conf" at the shell prompt.)
#
# This file is automatically updated on xserver-xorg package upgrades *only*
# if it has not been modified since the last upgrade of the xserver-xorg
# package.
#
# Note that some configuration settings that could be done previously
# in this file, now are automatically configured by the server and settings
# here are ignored.
#
# If you have edited this file but would like it to be automatically updated
# again, run the following command:
# sudo dpkg-reconfigure -phigh xserver-xorg

Section "Monitor"
Identifier "Configured Monitor"
DisplaySize 1680 1050
EndSection

Section "Screen"
Identifier "Default Screen"
Monitor "Configured Monitor"
Device "Configured Video Device"
SubSection "Display"
Virtual 2720 900
EndSubSection
EndSection

Section "Device"
Identifier "Configured Video Device"
EndSection

jboss startup script

#!/bin/sh
#
# $Id: jboss_init_redhat.sh 71252 2008-03-25 17:52:00Z dbhole $
#
# JBoss Control Script
#
# To use this script run it as root - it will switch to the specified user
#
# Here is a little (and extremely primitive) startup/shutdown script
# for RedHat systems. It assumes that JBoss lives in /usr/local/jboss,
# it's run by user 'jboss' and JDK binaries are in /usr/local/jdk/bin.
# All this can be changed in the script itself.
#
# Either modify this script for your requirements or just ensure that
# the following variables are set correctly before calling the script.

#define where jboss is - this is the directory containing directories log, bin, conf etc
JBOSS_HOME=${JBOSS_HOME:-"/home/jboss/jboss-4.2.3.GA"}

#define the user under which jboss will run, or use 'RUNASIS' to run as the current user
JBOSS_USER=jboss

#make sure java is in your path
JAVAPTH=${JAVAPTH:-"/usr/local/jdk/bin"}

#configuration to use, usually one of 'minimal', 'default', 'all'
#JBOSS_CONF=${JBOSS_CONF:-"default"}
JBOSS_CONF=${JBOSS_CONF:-"all"}

#if JBOSS_HOST specified, use -b to bind jboss services to that address
JBOSS_BIND_ADDR=${JBOSS_HOST:-"-b 10.31.140.79"}

#define the script to use to start jboss
JBOSSSH=${JBOSSSH:-"$JBOSS_HOME/bin/run.sh -c $JBOSS_CONF $JBOSS_BIND_ADDR"}

JBOSS_CONSOLE=/dev/null

if [ "$JBOSS_USER" = "RUNASIS" ]; then
SUBIT=""
else
SUBIT="su - $JBOSS_USER -c "
fi

if [ -n "$JBOSS_CONSOLE" -a ! -d "$JBOSS_CONSOLE" ]; then
# ensure the file exists
touch $JBOSS_CONSOLE
if [ ! -z "$SUBIT" ]; then
chown $JBOSS_USER $JBOSS_CONSOLE
fi
fi

if [ -n "$JBOSS_CONSOLE" -a ! -f "$JBOSS_CONSOLE" ]; then
echo "WARNING: location for saving console log invalid: $JBOSS_CONSOLE"
echo "WARNING: ignoring it and using /dev/null"
JBOSS_CONSOLE="/dev/null"
fi

#define what will be done with the console log
JBOSS_CONSOLE=${JBOSS_CONSOLE:-"/dev/null"}

JBOSS_CMD_START="cd $JBOSS_HOME/bin; $JBOSSSH"

if [ -z "`echo $PATH | grep $JAVAPTH`" ]; then
export PATH=$PATH:$JAVAPTH
fi

if [ ! -d "$JBOSS_HOME" ]; then
echo JBOSS_HOME does not exist as a valid directory : $JBOSS_HOME
exit 1
fi

echo JBOSS_CMD_START = $JBOSS_CMD_START

procrunning() {
procid=0
# JBOSSSCRIPT=$(echo $JBOSSSH | awk '{print $1}' | sed 's/\//\\\//g')
JBOSSSCRIPT=run.sh
for procid in `/bin/pidof -x "$JBOSSSCRIPT"`; do
ps -fp $procid | grep "${JBOSSSH% *}" > /dev/null && pid=$procid
done
}


stop() {
pid=0
procrunning
if [ "x$pid" = "x0" ]; then
echo -n -e "\nNo JBossas is currently running\n"
exit 1
fi

RETVAL=1

# If process is still running

# First, try to kill it nicely
for id in `ps --ppid $pid | awk '{print $1}' | grep -v "^PID$"`; do
if [ -z "$SUBIT" ]; then
kill -15 $id
else
$SUBIT "kill -15 $id"
fi
done

sleep=0
while [ $sleep -lt 120 -a $RETVAL -eq 1 ]; do
echo -n "\nwaiting for processes to stop";
sleep 10
sleep=`expr $sleep + 10`
pid=0
procrunning
if [ "x$pid" != "x0" ]; then
echo "duh" > /dev/null
else
RETVAL=0
fi
done

# Still not dead... kill it

count=0
pid=0
procrunning

if [ "x$RETVAL" != "x0" ] ; then
echo -e "\nTimeout: Shutdown command was sent, but process is still running with

PID $pid"
exit 1
fi

echo
exit 0
}

case "$1" in
start)

echo "user=$USER jboss_user=$JBOSS_USER jboss_home=$JBOSS_HOME" > /home/jboss/blah.txt
echo "command=$JBOSS_CMD_START console=$JBOSS_CONSOLE" >> /home/jboss/blah.txt
cd $JBOSS_HOME/bin
if [ -z "$SUBIT" ]; then
eval $JBOSS_CMD_START >${JBOSS_CONSOLE} 2>&1 &
else
$SUBIT "$JBOSS_CMD_START >${JBOSS_CONSOLE} 2>&1 &"
fi
;;
stop)
stop
;;
restart)
$0 stop
$0 start
;;
*)
echo "usage: $0 (start|stop|restart|help)"
esac

solr startup script

#!/bin/sh

# Starts, stops, and restarts solr

SOLR_DIR="/usr/solr-root"
JAVA_OPTIONS="-Xms2g -Xmx6g -DSTOP.PORT=8079 -DSTOP.KEY=zoomsolrstop -jar start.jar"
LOG_FILE="/var/log/solr/solr.log"
JAVA="/usr/bin/java"
SOLR_USER="solr"

case $1 in
start)
echo "Starting Solr"
cd $SOLR_DIR
sudo -u $SOLR_USER $JAVA $JAVA_OPTIONS 2> $LOG_FILE &
;;
stop)
echo "Stopping Solr"
cd $SOLR_DIR
sudo -u $SOLR_USER $JAVA $JAVA_OPTIONS --stop
;;
restart)
$0 stop
sleep 1
$0 start
;;
*)
echo "Usage: $0 {start|stop|restart}" >&2
exit 1
;;
esac

gzip files recursively

find . -type f -exec sh -c 'cat {} | gzip > {}.tmp; mv {}.tmp {}' \;

git basics

git config --global user.name "Andre Quina"
git config --global user.email andre.quina@gmail.com
mkdir testgit
cd testgit/
git init
git add readme
git add README
git commit -m 'first commit'
git remote add origin git@github.com:andrequina/testgit.git

git push origin master

Linux paste and sed -- join and remove blanks

following commands 1) join the three files interleaving the lines of the files 2) remove any blank lines that may have been created if the files are not the same length

paste one.txt two.txt three.txt -d'\n'
sed '/^$/d' preQuery.txt > no_blanks

Linux command - remove lines

following command finds the lines in file1 that are missing from file2

fgrep -vxf file1_sorted file2_sorted > removed_ids

SSH without password

the following commands allow an ssh connection without a password

ssh-keygen -t rsa -P ""
cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
ssh-copy-id hadoop@slave01

Setup Hadoop

# setup hadoop

#sudo adduser --ingroup hadoop hadoop

sudo useradd -d /home/hadoop -m hadoop
sudo usermod -s /bin/bash hadoop

sudo passwd hadoop

su - hadoop
ssh-keygen -t rsa -P ""

cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys

sudo cp hadoop-0.20.1.tar.gz /usr/local
cd /usr/local
sudo tar xzf hadoop-0.20.1.tar.gz
sudo chown -R hadoop:hadoop hadoop-0.20.1/
sudo ln -s /usr/local/hadoop-0.20.1/ /usr/local/hadoop

nano /usr/local/hadoop/conf/hadoop-

env.sh
set:
export JAVA_HOME=/usr/lib/jvm/java-6-sun
# allows hadoop to bind to 0.0.0.0 without binding to IP6 interface
export HADOOP_OPTS=-Djava.net.preferIPv4Stack=true

edit:
core-site.xml
hdfs-site.xml
mapred-site.xml

# format name node
./bin/hadoop namenode -format

# start cluster
./bin/start-all.sh

# command should list hadoop processes
jps

#----------------
# Test

./bin/hadoop dfs -copyFromLocal /tmp/davinci/ davinci
./bin/hadoop dfs -ls
./bin/hadoop dfs -ls davinci

Patch Hadoop

patch is from http://issues.apache.org/jira/browse/MAPREDUCE-375
originally targeted to 0.21.0, but modified to work with 0.20.1 (mostly resolving directory changes made in 0.21.0)

svn co http://svn.apache.org/repos/asf/hadoop/common/tags/release-0.20.1/ hadoop-0.20.1-src
cd hadoop-0.20.1-src
patch -p0 -i /zoom/libraries/apache/hadoop/patch-375-2.txt

there should be one rejection on NLineInputFormat in mapred

take a look at the rejection file, it should be fairly strait forward applying the change.
the rejection is caused by a small inconsistency between the NLineInputFormat in the two versions
it looks like a small bug fix in 0.21.0 not not in 0.20.1... should be fine as the patch just
dumps the code in the method and delegates to the mapreduce NLineInputFormat.

Install MogileFS on Ubuntu

add repository

add to /etc/apt/sources.list
deb http://ppa.launchpad.net/awmcclain/ppa/ubuntu jaunty main

install mogile

sudo apt-get update
sudo apt-get install mogilefsd mogstored

install mysql

sudo apt-get install mysql-server
mysql -u root -p
CREATE DATABASE mogilefs;
CREATE USER 'mogilefs'@'localhost' IDENTIFIED BY 'mogilefs';
GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,ALTER ON mogilefs.* TO 'mogilefs'@'localhost';
flush privileges;
commit;

mogdbsetup --dbhost=localhost --dbname=mogilefs --dbuser=mogilefs --dbpassword=mogilefs

add mogadmin

sudo apt-get install build-essential
sudo perl -MCPAN -e shell;
install MogileFS::Utils

edit configs

sudo nano /etc/mogilefs/mogilefsd.conf

* change 'db_user' to 'mogilefs'
* change 'db_pass' to 'mogilefs'
* change 'listen' to '0.0.0.0:7001' (bind on all local IPs)

start mogile

sudo /etc/init.d/mogilefsd start

configure storage hosts/devices

mogadm --trackers=10.31.248.128:7001 host add 10.31.248.128 --status=alive
mogadm --trackers=10.31.248.128:7001 device add 10.31.248.128 1
sudo mkdir -p /usr/mogdata/dev1
sudo chown mogstored /usr/mogdata/dev1
mogadm --trackers=10.31.248.128:7001 domain add companyRawRecords
mogadm --trackers=10.31.248.128:7001 class add companyRawRecords companyRawRecords --mindevcount=1

for additional storage nodes on tracker node
mogadm --trackers=10.31.248.128:7001 host add 10.31.248.127 --status=alive
mogadm --trackers=10.31.248.128:7001 device add 10.31.248.127 2
on additional storage nodes
sudo mkdir -p /usr/mogdata/dev2
sudo chown mogstored /usr/mogdata/dev2

run Apache for storage
stop mogstored

sudo /etc/init.d/mogstored stop

edit /etc/init.d/mogstored

add -- --server=none to the following line (need the double -- --)

start-stop-daemon --start --quiet --exec $DAEMON --pidfile $PIDFILE -b -m --name $NAME --chuid $MOGSTORED_RUNASUSER -- --server=none

install/configure apache

sudo apt-get install apache2

edit /etc/apache2/sites-available/mogilefs

Listen 7500

DocumentRoot /usr/mogdata
CustomLog logs/mogilefs_log common


Options +Indexes +FollowSymLinks



DAV On
Order Deny,Allow
Deny from all
Allow from 127.0.0.1


run:

sudo a2enmod dav
sudo a2enmod dav_fs

sudo a2ensite mogilefs

sudo groupadd mogstored
sudo chown -R mogstored:mogstored /usr/mogdata/
sudo usermod -G mogstored www-data
sudo usermod -G mogstored mogstored
sudo chmod -R g+w /usr/mogdata

sudo /etc/init.d/apache2 force-reload
sudo /etc/init.d/apache2 stop
sudo /etc/init.d/apache2 start

patch mogilefs

download attached patch

sudo patch /usr/share/perl5/MogileFS/Device.pm <>
install configure lighthttpd (do this OR apache)


sudo apt-get install lighttpd lighttpd-mod-webdav apache2-utils
sudo /etc/init.d/lighttpd stop



sudo nano /etc/lighttpd/lighttpd.conf
uncomment "mod_webdav"
set document root to the mogile storage location
set port to 7500


browse MySql database remotely


on tracker machine edit /etc/mysql/my.cnf

change 'bind-address' to '0.0.0.0'

run:


mysql -u root -p
mysql> GRANT ALL PRIVILEGES ON *.* TO root@'%' IDENTIFIED BY 'password';



restart mysql:


sudo /etc/init.d/mysql restart

on a remote machine, install query browser tools - http://dev.mysql.com/downloads/gui-tools/5.0.html

run query browser and connect, setting the default schema to 'mogilefs'