Kafka setup

Step 1 : Download Kafka kafka_2.11-2.4.1.tgz and uncompress it and            

             rename kafka_2.11-2.4.1 >> kafka

Step 2 : Install Java and setup Kafka

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export HADOOP_HOME=/home/dheerendra/hadoop/hadoop-3.2.1
export KAFKA_HOME=/home/dheerendra/hadoop/kafka
export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$HIVE_HOME/bin:$JAVA_HOME/bin:$SQOOP_HOME/bin:$KAFKA_HOME/bin
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"

Step 3 : Create 2 folder at any location 

            (a) /home/dheerendra/kafka_files/kafka/kafka-logs

            (b) /home/dheerendra/kafka_files/kafka/zookeeper-data

Step 4: Modified properties file in ./config folder of kafka

            (a) zookeeper.properties

                (1) dataDir=/home/dheerendra/kafka_files/kafka/zookeeper-data

                (2) admin.enableServer=true

            (b) server.properties

                (1) log.dirs=/home/dheerendra/kafka_files/kafka/kafka-logs

Step 4: Below services

          (1) start zookeeper
                    $kafka/./bin/zookeeper-server-start.sh ./config/zookeeper.properties

           (2) start kafka server
                    $kafka/./bin/kafka-server-start.sh ./config/server.properties

           (3) create testtopic
                   $kafka/./bin/kafka-topics.sh --create --zookeeper localhost:2181                                     --replication-factor 1 --partitions 1 --topic testtopic
           

          (4) start producer
                $kafka/bin/kafka-console-producer.sh --broker-list localhost:9092                                 --topic     testtopic
 

         (5)  start consumer

                $kafka/./bin/kafka-console-consumer.sh --bootstrap-server                                         localhost:9092 --topic testtopic --from-beginning