Spark_GraphX_2

Sample Data
________________________

/user/cloudera/sparkgraphx/routes //(id-name)
1,ameerpeta
2,srnagar
3,esi
4,mosapet
5,kukatpally
6,kphb
7,jntu

/user/cloudera/sparkgraphx/routesDistance //(sid-did-attr)
1,2,1
1,3,3
1,4,5
1,5,7
1,6,9
1,7,11
2,3,2
2,4,4
2,5,6
2,6,8
2,7,10
3,4,2
3,5,4
3,6,6
3,7,8
4,5,2
4,6,4
4,7,6
5,6,2
5,7,4
6,7,2


Working
---------------------
//---------------importing packages

import org.apache.spark.graphx._
import org.apache.spark.graphx.lib._
import org.apache.spark.rdd.RDD



Readind data from File
------------------------------------------------------
scala>val routes=sc.textFile("/user/cloudera/sparkgraphx/routes")
//routes: org.apache.spark.rdd.RDD[String] = /user/cloudera/sparkgraphx/routes MapPartitionsRDD[1] at textFile at <console>:27

 routes.collect.foreach(println)
1,ameerpeta                                                                   
2,srnagar
3,esi
4,mosapet
5.kukatpally
6,kphb
7,jntu


scala> val routesdistance=sc.textFile("/user/cloudera/sparkgraphx/routesDistance")
//routesdistance: org.apache.spark.rdd.RDD[String] = /user/cloudera/sparkgraphx/routesDistance MapPartitionsRDD[12] at textFile at <console>:34

scala> routesdistance.collect.foreach(println)
1,2,1
1,3,3
1,4,5
1,5,7
1,6,9
1,7,11
2,3,2
2,4,4
2,5,6
2,6,8
2,7,10
3,4,2
3,5,4
3,6,6
3,7,8
4,5,2
4,6,4
4,7,6
5,6,2
5,7,4
6,7,2


Vertices Preparation by using routes RDD
---------------------------------------------------

scala> val route:RDD[(VertexId,String)]=routes.map{x=>
     | val w=x.split(",")
     | (w(0).toLong,w(1))
     | }
//route: org.apache.spark.rdd.RDD[(org.apache.spark.graphx.VertexId, String)] = MapPartitionsRDD[8] at map at <console>:36



scala> route.collect.foreach(println)
(1,ameerpeta)                                                                 
(2,srnagar)
(3,esi)
(4,mosapet)
(5,kukatpally)
(6,kphb)
(7,jntu)


Edges preparation by using routesdistance RDD
----------------------------------------------------------------------
val fdist=routesdistance.map{x=>
      val w=x.trim().split(",")
      Edge(w(0).toLong,w(1).toLong,w(2).toLong)
      }


scala> fdist.collect.foreach(println)
Edge(1,2,1)
Edge(1,3,3)
Edge(1,4,5)
Edge(1,5,7)
Edge(1,6,9)
Edge(1,7,11)
Edge(2,3,2)
Edge(2,4,4)
Edge(2,5,6)
Edge(2,6,8)
Edge(2,7,10)
Edge(3,4,2)
Edge(3,5,4)
Edge(3,6,6)
Edge(3,7,8)
Edge(4,5,2)
Edge(4,6,4)
Edge(4,7,6)
Edge(5,6,2)
Edge(5,7,4)
Edge(6,7,2)

scala>val bdist=routesdistance.map{x=>
       val w=x.trim().split(",")
      Edge(w(1).toLong,w(0).toLong,w(2).toLong)
      }

scala> bdist.collect.foreach(println)
Edge(2,1,1)                                                                   
Edge(3,1,3)
Edge(4,1,5)
Edge(5,1,7)
Edge(6,1,9)
Edge(7,1,11)
Edge(3,2,2)
Edge(4,2,4)
Edge(5,2,6)
Edge(6,2,8)
Edge(7,2,10)
Edge(4,3,2)
Edge(5,3,4)
Edge(6,3,6)
Edge(7,3,8)
Edge(5,4,2)
Edge(6,4,4)
Edge(7,4,6)
Edge(6,5,2)
Edge(7,5,4)
Edge(7,6,2)

scala> val dist=fdist++bdist
scala> val dist=fdist++bdist
dist: org.apache.spark.rdd.RDD[org.apache.spark.graphx.Edge[Long]] = UnionRDD[52] at $plus$plus at <console>:40

scala> dist.collect.foreach(println)
Edge(1,2,1)
Edge(1,3,3)
Edge(1,4,5)
Edge(1,5,7)
Edge(1,6,9)
Edge(1,7,11)
Edge(2,3,2)
Edge(2,4,4)
Edge(2,5,6)
Edge(2,6,8)
Edge(2,7,10)
Edge(3,4,2)
Edge(3,5,4)
Edge(3,6,6)
Edge(3,7,8)
Edge(4,5,2)
Edge(4,6,4)
Edge(4,7,6)
Edge(5,6,2)
Edge(5,7,4)
Edge(6,7,2)
Edge(2,1,1)
Edge(3,1,3)
Edge(4,1,5)
Edge(5,1,7)
Edge(6,1,9)
Edge(7,1,11)
Edge(3,2,2)
Edge(4,2,4)
Edge(5,2,6)
Edge(6,2,8)
Edge(7,2,10)
Edge(4,3,2)
Edge(5,3,4)
Edge(6,3,6)
Edge(7,3,8)
Edge(5,4,2)
Edge(6,4,4)
Edge(7,4,6)
Edge(6,5,2)
Edge(7,5,4)
Edge(7,6,2)



Preparing Graph object req (vertices,edges,user) //user is optional
----------------------------------------------------------------
scala> val graph=Graph(route,dist)
graph: org.apache.spark.graphx.Graph[String,Long] = org.apache.spark.graphx.impl.GraphImpl@4152141d

scala> graph.vertices.collect.foreach(println)
(4,mosapet)                                                                   
(6,kphb)
(2,srnagar)
(1,ameerpeta)
(3,esi)
(7,jntu)
(5,kukatpally)

scala> graph.edges.collect.foreach(println)
Edge(1,2,1)
Edge(1,3,3)
Edge(1,4,5)
Edge(1,5,7)
Edge(1,6,9)
Edge(1,7,11)
Edge(2,3,2)
Edge(2,4,4)
Edge(2,5,6)
Edge(2,6,8)
Edge(2,7,10)
Edge(3,4,2)
Edge(3,5,4)
Edge(3,6,6)
Edge(3,7,8)
Edge(4,5,2)
Edge(4,6,4)
Edge(4,7,6)
Edge(5,6,2)
Edge(5,7,4)
Edge(6,7,2)
Edge(2,1,1)
Edge(3,1,3)
Edge(3,2,2)
Edge(4,1,5)
Edge(4,2,4)
Edge(5,1,7)
Edge(5,2,6)
Edge(6,1,9)
Edge(6,2,8)
Edge(7,1,11)
Edge(7,2,10)
Edge(4,3,2)
Edge(5,3,4)
Edge(5,4,2)
Edge(6,3,6)
Edge(6,4,4)
Edge(6,5,2)
Edge(7,3,8)
Edge(7,4,6)
Edge(7,5,4)
Edge(7,6,2)



Filetring
----------------------------------
scala> val ampt=graph.triplets.filter(_.srcAttr=="ameerpeta").map(x=> x.srcAttr+" to "+x.dstAttr+" Cost :"+x.attr)

scala> ampt.collect.foreach(println)

o/p
--------------
ameerpeta to srnagar Cost :1                                                   
ameerpeta to esi Cost :3
ameerpeta to mosapet Cost :5
ameerpeta to kukatpally Cost :7
ameerpeta to kphb Cost :9
ameerpeta to jntu Cost :11

scala> val jntu=graph.triplets.filter(_.srcAttr=="jntu").map(x=> x.srcAttr+" to "+x.dstAttr+" Cost :"+x.attr)
jntu: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[96] at map at <console>:48

scala> jntu.collect.foreach(println)

jntu to ameerpeta Cost :11                                                     
jntu to srnagar Cost :10
jntu to esi Cost :8
jntu to mosapet Cost :6
jntu to kukatpally Cost :4
jntu to kphb Cost :2

Comments

Post a Comment