Kafka





---------------------------------------------KAFKA------------------------------------

Download the Kafka Directory--->     www-eu.apache.org/dist/kafka/0.10.1.0/kafka_2.10-0.10.1.0.tgz
 (After Extract   to Desktop)

-->Open super user
$: su   root
pwd:  cloudera
Change Directory to kafka Directory



1)  Start Zookeeper Sever
  bin/zookeeper-server-start.sh config/zookeeper.properties

config/zookeeper.properties---->port  Number:2185   (and also change)

2)  StartKafka Server (Broker)
bin/kafka-server-start.sh config/server.properties

To start Multiple brokers then
copy the config/server.properties to config/server1.properties(in thant file some change required)
            (-----------------Zookeperid:2185(same for all brokers)
                        broker id:1
portnumber:9093
log-dir:/user/tmp/log-2------------------)

      like ---------(  port,id,log,zoolkeeper   )
       broker1 :  (      9091,1,log-1,2185          ) 
       broker2:   (     9092,2,log-1,2185          ) 
       broker3:   (     9093,3,log-1,2185          ) 

3) Create Topic
bin/kafka-topics.sh --zookeeper localhost:2185 --create --topic test6 --replication-factor 1 --partitions 5
    Created topic "test6".


  Note:No of Replication factor should be lessthan or equal to No of brokers



4)  Start Console Producer
bin/kafka-console-producer.sh  --broker-list localhost:9092 --topic test6

                     --------Type Msgs------

5) Strat Console Conumer
bin/kafka-console-consumer.sh  --zookeeper localhost:2185  --topic test6  --from-beginning

                  ----------Print Producer Msgs------------------


To show list of topic
------------------------
bin/kafka-topics.sh  --zookeeper localhost:2185  --list



========================================================================================================================
import org.apache.spark._
import org.apache.spark.streaming._
import org.apache.spark.streaming.StreamingContext._
val ssc=new StreamingContext(sc,Seconds(5))
val lines=ssc.socketTextStream("localhost",9999)
val data=lines.map(x=>x.split(","))
val rec=data.map(x=>x(0) +" name is is "+x(1)+" and age is "+x(2))
rec.print
ssc.start


==========================================================================================================================
 import org.apache.spark.SparkConf
 import org.apache.spark.streaming.StreamingContext
 import org.apache.spark.streaming.Seconds
 import org.apache.spark.streaming.kafka.KafkaUtils
 val ssc=new StreamingContext(sc,Seconds(5))
 val ks=KafkaUtils.createStream(ssc,"localhost:2185","spark-streaming-consumer-group",Map("spark-topic"->5))
 ks.print()



===========================================================================================================================


1.Data Sending to Kafka
=============================================
package com.kafka.app;

import java.util.Properties;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;

public class Producer1 {
public static void main(String[] args) {
Properties p=new Properties();
p.put("bootstrap.servers","localhost:9092");
p.put("ack","all");
p.put("recall",0);
p.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
p.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
Producer<String,String> prod=new KafkaProducer<String,String>(p);
for(int i=1;i<=25;i++)
{
String key=Integer.toString(i);
String value=Integer.toString(i);
    ProducerRecord rec=new ProducerRecord<String,String>("mytopic1",key,value);
    prod.send(rec);
    System.out.println(i+" Record sent Sucessfully to Kafka ");
}
prod.close();
System.out.println("------Done---------");
}
}



2.File To Kafka
===============================================================
package com.kafka.app;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Properties;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;

public class FileToKafka {

public static void main(String[] args)throws IOException {
Properties p=new Properties();
p.put("bootstrap.servers","localhost:9092");
p.put("ack","all");
p.put("recall",0);
p.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
p.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
Producer<String,String> prod=new KafkaProducer<String,String>(p);
FileInputStream fis=new FileInputStream("src/Emp");
InputStreamReader ips=new InputStreamReader(fis);
BufferedReader br=new BufferedReader(ips);
String rec="";
String key;
String val;

         while((rec=br.readLine())!=null)
{
        String[] w=rec.split(",");
        key=w[0];
        val=rec;
ProducerRecord record=new ProducerRecord<String,String>("mytopic1",key,val);
    prod.send(record);
System.out.println(key+" Data Sent to Kafka ");
    }
br.close();
prod.close();
System.out.println("------Done---------");

}

}


Output
---------
100 Data Sent to Kafka
101 Data Sent to Kafka
102 Data Sent to Kafka
103 Data Sent to Kafka
------Done---------


3.Mysql to Kafka
===============================================

package KafkaAPI;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Properties;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;



public class Producer {
public static void main(String args[])throws Exception
{
Properties p=new Properties();
p.put("bootstrap.servers","localhost:9092");
p.put("buffer.memory",33524585);
p.put("ack","all");
p.put("recall",0);
p.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
p.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
    String topic ="mytopic";
KafkaProducer<String, String> prod=new KafkaProducer<String,String>(p);
try{
    Class.forName("com.mysql.jdbc.Driver");
        Connection con=DriverManager.getConnection("jdbc:mysql://localhost/gnr","root","cloudera");
    Statement stmt=con.createStatement();
    ResultSet rs=stmt.executeQuery("select deptno,dname from dept");
    System.out.println("-----------Started------------");
    while(rs.next())
    {
    int deptno=rs.getInt("deptno");
    String dname=rs.getString("dname");
    String key=Integer.toString(deptno);
    String value=deptno+" and "+dname;
   
    prod.send(new ProducerRecord<String,String>("dept",key,value));
   
   
    }
    System.out.println("-----------Sucess------------");
 
 
}catch(Exception e){
System.out.println(e);
}
 

/* for(int i=100;i<2000;i++)
{
System.out.println(i);
prod.send(new ProducerRecord<String,String>(topic,Integer.toString(i),Integer.toString(i)));
}     */


prod.close();


}

}





4.hdfs to kafka
=============================================================
package kafkaf;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.Properties;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;

public class Kafkafile1 {

public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub

Properties p = new Properties();
p.put("bootstrap.servers", "localhost:9092");
p.put("key.serializer",
"org.apache.kafka.common.serialization.StringSerializer");
p.put("value.serializer",
"org.apache.kafka.common.serialization.StringSerializer");
p.put("ack", "all");
p.put("recall", "0");
Producer<String, String> prod = new KafkaProducer<String, String>(p);
System.out.println("Preparing");
// -----------------Java
Configuration c = new Configuration();
String ip = "hdfs://quickstart.cloudera:8020/user/cloudera/batch800/emp";
Path pa = new Path(ip);
FileSystem fs = FileSystem.get(URI.create(ip), c);
// FileInputStream f=new FileInputStream("/user/cloudera/batch800/emp");
InputStreamReader ir = new InputStreamReader(fs.open(pa));
BufferedReader b = new BufferedReader(ir);
String line = "";
int i = 1;
while ((line = b.readLine()) != null) {
String key = Integer.toString(i);
String val = line;

ProducerRecord rec = new ProducerRecord<String, String>("test5",
key, val);
System.out.println(val);
prod.send(rec);
}
prod.close();

System.out.println("done");
}

}

5.mongoDb to kafka
-----------------------
package mongoAPI;

import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.MongoClient;

public class Retrive {
public static void main(String[] args)throws Exception{
MongoClient mc=new MongoClient("localhost",27017);
System.out.println("=====Connection Established==========");
DB db=mc.getDB("gnr");
System.out.println("=====Acess DB ==========");
DBCollection dbc=db.getCollection("dept");
System.out.println("=====Acess Collection==========");
DBCursor cur=dbc.find();
System.out.println("=====Collecting Data From Cursor==========");
while(cur.hasNext())
{
String str=cur.next().toString();
System.out.println("output is :"+str);
}
System.out.println("------Completed------");

}

}


------------------------------------------------
Dependencies
------------------------------------------
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka_2.10 -->
<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka_2.10</artifactId>
    <version>0.10.2.0</version>
</dependency>




<!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
<dependency>
    <groupId>mysql</groupId>
    <artifactId>mysql-connector-java</artifactId>
    <version>5.1.32</version>
</dependency>




<!-- https://mvnrepository.com/artifact/org.mongodb/mongo-java-driver -->
<dependency>
    <groupId>org.mongodb</groupId>
    <artifactId>mongo-java-driver</artifactId>
    <version>2.6</version>
</dependency>


<dependency>
        <groupId>org.mongodb</groupId>
        <artifactId>mongo-java-driver</artifactId>
        <version>2.12.3</version>
    </dependency>
</dependencies>


<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.10 -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_2.11</artifactId>
    <version>2.1.0</version>
</dependency>

Comments