Notes

#Install Git to copy codes and start.sh from the bitbucket -
sudo yum install git
mkdir Downloads
cd Downloads
git clone https://[email protected]/agapoorv/old_cluster1.git
git clone https://[email protected]/agapoorv/fairfax-codes.git
cd fairfax-codes
chmod +x start.sh
cp start.sh ~/
cd ~
vim .bash_profile
export AWS_ACCESS_KEY_ID=AKIAIQOF4DEJUVLTPYSQ
export AWS_SECRET_ACCESS_KEY=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

ps -ef | fgrep yum | more
kill -9 parent_pid

cd ~
vim start.sh
change the secret key and access key
find -name "core-site.xml"
vim spark/conf/core-site.xml
vim persistent-hdfs/conf/core-site.xml
vim ephemeral-hdfs/conf/core-site.xml
<property>
<name>fs.s3n.awsAccessKeyId</name>
<value>AKIAIQOF4DEJUVLTPYSQ</value>
</property>

<property?
<name>fs.s3n.awsSecretAccessKey</name>
<value>pV4ji+qObbYT6kxTKcPtzV2A5UkjpxDmcvSp10nX</value>
</property>

cd ~
./start.sh

#Download S3FS - no need for Spark and all. But is required if trying to copy-paste files to/from S3. works as a shared drive.
cd /etc/yum.repos.d/
wget http://s3tools.org/repo/RHEL_6/s3tools.repo
yum install s3cmd
s3cmd --configure
update the access and secret keys here. Give yes to the https protocol thingy
#Mounting S3 bucket
cd ~/Downloads/
wget http://s3fs.googlecode.com/files/s3fs-1.73.tar.gz
tar xvzf s3fs-1.73.tar.gz
cd s3fs-1.73
yum install gcc libstdc++-devel gcc-c++ fuse fuse-devel curl-devel libxml2-devel openssl-devel mail
./configure --prefix=/usr
make install
cd /mnt
mkdir ffx-analytics
echo AKIAIQOF4DEJUVLTPYSQ:pV4ji+qObbYT6kxTKcPtzV2A5UkjpxDmcvSp10nX > ~/.passwd-s3fs
chmod 600 ~/.passwd-s3fs
/usr/bin/s3fs ffx-analytics /mnt/ffx-analytics/



#Install ua-parser and SBT
cd ~/Downloads
git clone https://github.com/tobie/ua-parser.git
wget http://mirrors.gigenet.com/apache/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz
su -c "tar -zxvf apache-maven-3.0.5-bin.tar.gz -C /opt/"
cd /opt/
mv apache-maven-3.0.5 /usr/local/apache-maven
su -c "vi /etc/profile.d/maven.sh"

# Add the following lines to maven.sh
export M2_HOME=/usr/local/apache-maven
#Change the PAth to where the Maven folder is
export M2=$M2_HOME/bin
PATH=$M2:$PATH

chmod +x /etc/profile.d/maven.sh
ln -s /usr/local/apache-maven/bin/mvn /usr/bin/mvn
mvn -version


cd ~/Downloads/ua-parser/java
mvn install
cd ~/spark/bin
wget https://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.7/sbt-launch.jar
vim sbt
SBT_OPTS="-Xms512M -Xmx1536M -Xss1M -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=256M"
java $SBT_OPTS -jar `dirname $0`/sbt-launch.jar "$@"
chmod u+x ~/spark/bin/sbt

mkdir src && cd src && mkdir main && cd main && mkdir scala && cd scala
Create Projects and have fun

Load a file to hdfs prior to using it -
./ephemeral-hdfs/bin/hadoop -put fs <localfile> <hdfs location>
path to access files in S3 -
s3n://ffx-analytics/all-interactions/filename

in case a window is hung and we re-login from another tab, need to stop and start all clusters.
./ephemeral-hdfs/bin/stop-all.sh
./persistent-hdfs/bin/stop-all.sh
./spark/sbin/stop-all.sh
./ephemeral-hdfs/bin/start-all.sh
./persistent-hdfs/bin/start-all.sh
./spark/sbin/start-all.sh


export classpath="$(for j in /root/jars/jar* ;do echo -n $j: ; done)"
alias 'spark-shell'="/root/spark/bin/spark-shell --driver-class-path $classpath"


~/spark/bin/sbt package -mem 600
~/spark/bin/spark-submit --class "test1" ~/spark/prog1/target/scala-2.10/test1_2.10-1.0.jar

~/spark/bin/spark-submit --class "Analysis" ~/spark/Projects/prog1/target/scala-2.10/prog1_2.10-0.1.0.jar




}