---------------------------------------------KAFKA------------------------------------
Download the Kafka Directory---> www-eu.apache.org/dist/kafka/0.10.1.0/kafka_2.10-0.10.1.0.tgz
(After Extract to Desktop)
-->Open super user
$: su root
pwd: cloudera
Change Directory to kafka Directory
1) Start Zookeeper Sever
bin/zookeeper-server-start.sh config/zookeeper.properties
config/zookeeper.properties---->port Number:2185 (and also change)
2) StartKafka Server (Broker)
bin/kafka-server-start.sh config/server.properties
To start Multiple brokers then
copy the config/server.properties to config/server1.properties(in thant file some change required)
(-----------------Zookeperid:2185(same for all brokers)
broker id:1
portnumber:9093
log-dir:/user/tmp/log-2------------------)
like ---------( port,id,log,zoolkeeper )
broker1 : ( 9091,1,log-1,2185 )
broker2: ( 9092,2,log-1,2185 )
broker3: ( 9093,3,log-1,2185 )
3) Create Topic
bin/kafka-topics.sh --zookeeper localhost:2185 --create --topic test6 --replication-factor 1 --partitions 5
Created topic "test6".
Note:No of Replication factor should be lessthan or equal to No of brokers
4) Start Console Producer
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test6
--------Type Msgs------
5) Strat Console Conumer
bin/kafka-console-consumer.sh --zookeeper localhost:2185 --topic test6 --from-beginning
----------Print Producer Msgs------------------
To show list of topic
------------------------
bin/kafka-topics.sh --zookeeper localhost:2185 --list
========================================================================================================================
import org.apache.spark._
import org.apache.spark.streaming._
import org.apache.spark.streaming.StreamingContext._
val ssc=new StreamingContext(sc,Seconds(5))
val lines=ssc.socketTextStream("localhost",9999)
val data=lines.map(x=>x.split(","))
val rec=data.map(x=>x(0) +" name is is "+x(1)+" and age is "+x(2))
rec.print
ssc.start
==========================================================================================================================
import org.apache.spark.SparkConf
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.kafka.KafkaUtils
val ssc=new StreamingContext(sc,Seconds(5))
val ks=KafkaUtils.createStream(ssc,"localhost:2185","spark-streaming-consumer-group",Map("spark-topic"->5))
ks.print()
===========================================================================================================================
1.Data Sending to Kafka
=============================================
package com.kafka.app;
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
public class Producer1 {
public static void main(String[] args) {
Properties p=new Properties();
p.put("bootstrap.servers","localhost:9092");
p.put("ack","all");
p.put("recall",0);
p.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
p.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
Producer<String,String> prod=new KafkaProducer<String,String>(p);
for(int i=1;i<=25;i++)
{
String key=Integer.toString(i);
String value=Integer.toString(i);
ProducerRecord rec=new ProducerRecord<String,String>("mytopic1",key,value);
prod.send(rec);
System.out.println(i+" Record sent Sucessfully to Kafka ");
}
prod.close();
System.out.println("------Done---------");
}
}
2.File To Kafka
===============================================================
package com.kafka.app;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
public class FileToKafka {
public static void main(String[] args)throws IOException {
Properties p=new Properties();
p.put("bootstrap.servers","localhost:9092");
p.put("ack","all");
p.put("recall",0);
p.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
p.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
Producer<String,String> prod=new KafkaProducer<String,String>(p);
FileInputStream fis=new FileInputStream("src/Emp");
InputStreamReader ips=new InputStreamReader(fis);
BufferedReader br=new BufferedReader(ips);
String rec="";
String key;
String val;
while((rec=br.readLine())!=null)
{
String[] w=rec.split(",");
key=w[0];
val=rec;
ProducerRecord record=new ProducerRecord<String,String>("mytopic1",key,val);
prod.send(record);
System.out.println(key+" Data Sent to Kafka ");
}
br.close();
prod.close();
System.out.println("------Done---------");
}
}
Output
---------
100 Data Sent to Kafka
101 Data Sent to Kafka
102 Data Sent to Kafka
103 Data Sent to Kafka
------Done---------
3.Mysql to Kafka
===============================================
package KafkaAPI;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
public class Producer {
public static void main(String args[])throws Exception
{
Properties p=new Properties();
p.put("bootstrap.servers","localhost:9092");
p.put("buffer.memory",33524585);
p.put("ack","all");
p.put("recall",0);
p.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
p.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
String topic ="mytopic";
KafkaProducer<String, String> prod=new KafkaProducer<String,String>(p);
try{
Class.forName("com.mysql.jdbc.Driver");
Connection con=DriverManager.getConnection("jdbc:mysql://localhost/gnr","root","cloudera");
Statement stmt=con.createStatement();
ResultSet rs=stmt.executeQuery("select deptno,dname from dept");
System.out.println("-----------Started------------");
while(rs.next())
{
int deptno=rs.getInt("deptno");
String dname=rs.getString("dname");
String key=Integer.toString(deptno);
String value=deptno+" and "+dname;
prod.send(new ProducerRecord<String,String>("dept",key,value));
}
System.out.println("-----------Sucess------------");
}catch(Exception e){
System.out.println(e);
}
/* for(int i=100;i<2000;i++)
{
System.out.println(i);
prod.send(new ProducerRecord<String,String>(topic,Integer.toString(i),Integer.toString(i)));
} */
prod.close();
}
}
4.hdfs to kafka
=============================================================
package kafkaf;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
public class Kafkafile1 {
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
Properties p = new Properties();
p.put("bootstrap.servers", "localhost:9092");
p.put("key.serializer",
"org.apache.kafka.common.serialization.StringSerializer");
p.put("value.serializer",
"org.apache.kafka.common.serialization.StringSerializer");
p.put("ack", "all");
p.put("recall", "0");
Producer<String, String> prod = new KafkaProducer<String, String>(p);
System.out.println("Preparing");
// -----------------Java
Configuration c = new Configuration();
String ip = "hdfs://quickstart.cloudera:8020/user/cloudera/batch800/emp";
Path pa = new Path(ip);
FileSystem fs = FileSystem.get(URI.create(ip), c);
// FileInputStream f=new FileInputStream("/user/cloudera/batch800/emp");
InputStreamReader ir = new InputStreamReader(fs.open(pa));
BufferedReader b = new BufferedReader(ir);
String line = "";
int i = 1;
while ((line = b.readLine()) != null) {
String key = Integer.toString(i);
String val = line;
ProducerRecord rec = new ProducerRecord<String, String>("test5",
key, val);
System.out.println(val);
prod.send(rec);
}
prod.close();
System.out.println("done");
}
}
5.mongoDb to kafka
-----------------------
package mongoAPI;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.MongoClient;
public class Retrive {
public static void main(String[] args)throws Exception{
MongoClient mc=new MongoClient("localhost",27017);
System.out.println("=====Connection Established==========");
DB db=mc.getDB("gnr");
System.out.println("=====Acess DB ==========");
DBCollection dbc=db.getCollection("dept");
System.out.println("=====Acess Collection==========");
DBCursor cur=dbc.find();
System.out.println("=====Collecting Data From Cursor==========");
while(cur.hasNext())
{
String str=cur.next().toString();
System.out.println("output is :"+str);
}
System.out.println("------Completed------");
}
}
------------------------------------------------
Dependencies
------------------------------------------
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka_2.10 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>0.10.2.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.32</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mongodb/mongo-java-driver -->
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>2.12.3</version>
</dependency>
</dependencies>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.10 -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.1.0</version>
</dependency>
Comments
Post a Comment