Notes

In this task, we will download datasets from https://github.com/rashidaligee/HDPCD-Certification.

1- Go to https://github.com/rashidaligee/HDPCD-Certification
2- Click on Download ZIP button on right of the screen
3- Download and Unzip the file in your favourite directory

/* This is for loading flight clean */

A = LOAD '/user/nany/flightdelays/flight*' using PigStorage(',') AS Year, Month, DayofMonth, DayOfWeek, DepTime, CRSDepTime, ArrTime, CRSArrTime, UniqueCarrier, FlightNum, TailNum, ActualElapsedTime, CRSElapsedTime, AirTime, ArrDelay, DepDelay, Origin, Dest, Distance, TaxiIn, TaxiOut, Cancelled, CancellationCode, Diverted, CarrierDelay, WeatherDelay, NASDelay, SecurityDelay, LateAircraftDelay ;
B = filter A BY DepTime <> "NA";
C = FOREACH B GENERATE Year, Month, DayofMonth, DepTime, UniqueCarrier, FlightNum, ArrDelay, Origin and Dest;
D = STORE C INTO '/user/nany/flightdelays/flight-clean' using PigStorage();


https://stackoverflow.com/questions/14204275/how-can-i-add-a-header-row-to-files-created-from-pig-hadoop

https://cwiki.apache.org/confluence/display/PIG/Pig+Training

Oracle:

create table weather (
station varchar(100) ,
"year" NUMBER(11) ,
"month" NUMBER(11) ,
dayofmonth NUMBER(11) ,
precipitation NUMBER(11) ,
maxtemp NUMBER(11) ,
mNUMBERemp NUMBER(11)
);

MySQL:
create table weather (
station varchar(100) ,
`year` int(11) ,
`month` int(11) ,
dayofmonth int(11) ,
precipitation int(11) ,
maxtemp int(11) ,
mintemp int(11)
);


$ bin/flume-ng agent -n agent_fagent -c conf -f conf/fagent.conf
# flume ingest from local to HDFS:
# fagent.conf list the source, sink, and channels for the agent
agent_fagent.sources = src
agent_fagent.sinks = hdfs-sink
agent_fagent.channels = mem-channel

# properties of local file system
agent_fagent.sources.src.type = spooldir
agent_fagent.sources.src.spoolDir = /home/nany/flumeData

# properties of mem-channel
agent_fagent.channels.mem-channel.type = memory
agent_fagent.channels.mem-channel.capacity = 1000
agent_fagent.channels.mem-channel.transactionCapacity = 100

# properties of hdfs-sink
agent_fagent.sinks.hdfs-sink.type = hdfs
agent_fagent.sinks.hdfs-sink.hdfs.path = /user/nany/flumeData
agent_fagent.sinks.hdfs-sink.hdfs.filePrefix = logs-

# set channel for source
agent_fagent.sources.src.channels = mem-channel

# set channel for sink
agent_fagent.sinks.hdfs-sink.channel = mem-channel


$sqoop export --connect jdbc:mysql://namenode:3306/flightinfo --username root --password hadoop --table weather --export-dir /user/nany/flumeData --input-fields-terminated-by ','
$sqoop import --connect jdbc:mysql://namenode:3306/flightinfo --username root --password hadoop --query 'SELECT * FROM weather WHERE $CONDITIONS' --split-by year --target-dir /user/nany/flumeData --as-textfile --direct