<food>
<name>Belgian Waffles</name>
<price>$5.95</price>
<description>
Two of our famous Belgian Waffles with plenty of real maple syrup
</description>
<calories>650</calories>
</food>
<food>
<name>Strawberry Belgian Waffles</name>
<price>$7.95</price>
<description>
Light Belgian waffles covered with strawberries and whipped cream
</description>
<calories>900</calories>
</food>
<food>
<name>Berry-Berry Belgian Waffles</name>
<price>$8.95</price>
<description>
Light Belgian waffles covered with an assortment of fresh berries and whipped cream
</description>
<calories>900</calories>
</food>
<food>
<name>French Toast</name>
<price>$4.50</price>
<description>
Thick slices made from our homemade sourdough bread
</description>
<calories>600</calories>
</food>
<food>
<name>Homestyle Breakfast</name>
<price>$6.95</price>
<description>
Two eggs, bacon or sausage, toast, and our ever-popular hash browns
</description>
<calories>950</calories>
</food>
Mapper Class
------------------
package mr;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class XMLMap extends Mapper<LongWritable,Text,Text,NullWritable>{
String Line="";
public void map(LongWritable k,Text v,Context con)throws IOException,InterruptedException{
String data=v.toString().trim();
Line=Line+data;
if(data.matches("</food>")){
con.write(new Text(Line),NullWritable.get());
Line="";
}
}
}
Driver Class:
------------------
package mr;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class DriveXML {
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
Configuration c=new Configuration();
Job j=new Job(c,"xmld");
j.setJarByClass(DriveXML.class);
j.setMapperClass(XMLMap.class);
j.setNumReduceTasks(0);
Path ip=new Path(args[0]);
Path op=new Path(args[2]);
FileInputFormat.addInputPath(j,ip);
FileOutputFormat.setOutputPath(j, op);
System.exit(j.waitForCompletion(true)?0:1);
}
}
OP
----------
<food><name>Belgian Waffles</name><price>$5.95</price><description>Two of our famous Belgian Waffles with plenty of real maple syrup</description><calories>650</calories></food>
<food><name>Strawberry Belgian Waffles</name><price>$7.95</price><description>Light Belgian waffles covered with strawberries and whipped cream</description><calories>900</calories></food>
<food><name>Berry-Berry Belgian Waffles</name><price>$8.95</price><description>Light Belgian waffles covered with an assortment of fresh berries and whipped cream</description><calories>900</calories></food>
<food><name>French Toast</name><price>$4.50</price><description>Thick slices made from our homemade sourdough bread</description><calories>600</calories></food>
<food><name>Homestyle Breakfast</name><price>$6.95</price><description>Two eggs, bacon or sausage, toast, and our ever-popular hash browns</description><calories>950</calories></food>
Spark
=================================
scala> val data = sc.textFile("/user/cloudera/hivegnr/xmlv")
scala> val data1 = data.map(x=>x.trim());
scala> val data2 = sc.parallelize(Array(data1.collect.mkString("")));
scala> data1.count
res9: Long = 40
scala> data2.count
res10: Long = 1
Note
---------------
Ex:Array(abcd1gnr1abce1ghd1gnr)---one element--strinng--1
split--1
map---Array(Array(abcd,gnr,abce,ghd,gnr))---one element--array--1
fltMap---Array(abcd,gnr,abce,ghd,gnr)---5 elemnts--5
mapp-add(1)---Array(abcd1,gnr1,abce1,ghd1,gnr1)
-------------------------
scala> val data3 = data2.flatMap(x=>x.split("</food>"))
scala> data3.take(1)
res11: Array[Array[String]] = Array(<food><name>Belgian Waffles</name><price>$5.95</price><description>Two of our famous Belgian Waffles with plenty of real maple syrup</description><calories>650</calories>, <food><name>Strawberry Belgian Waffles</name><price>$7.95</price><description>Light Belgian waffles covered with strawberries and whipped cream</description><calories>900</calories>, <food><name>Berry-Berry Belgian Waffles</name><price>$8.95</price><description>Light Belgian waffles covered with an assortment of fresh berries and whipped cream</description><calories>900</calories>, <food><name>French Toast</name><price>$4.50</price><description>Thick slices made from our homemade sourdough bread</description><calories>600</calories>, <food><name>Homestyle Breakfast</name><pric...
scala> val res=data3.map(x=>x+"</food>")
res: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[20] at map at <console>:35
scala> res.take(1)
res17: Array[String] = Array(<food><name>Belgian Waffles</name><price>$5.95</price><description>Two of our famous Belgian Waffles with plenty of real maple syrup</description><calories>650</calories></food>)
scala> res.collect.foreach(println)
<food><name>Belgian Waffles</name><price>$5.95</price><description>Two of our famous Belgian Waffles with plenty of real maple syrup</description><calories>650</calories></food>
<food><name>Strawberry Belgian Waffles</name><price>$7.95</price><description>Light Belgian waffles covered with strawberries and whipped cream</description><calories>900</calories></food>
<food><name>Berry-Berry Belgian Waffles</name><price>$8.95</price><description>Light Belgian waffles covered with an assortment of fresh berries and whipped cream</description><calories>900</calories></food>
<food><name>French Toast</name><price>$4.50</price><description>Thick slices made from our homemade sourdough bread</description><calories>600</calories></food>
<food><name>Homestyle Breakfast</name><price>$6.95</price><description>Two eggs, bacon or sausage, toast, and our ever-popular hash browns</description><calories>950</calories></food>
nice blog
ReplyDelete