mindol1004/clean-hexagonal-architecture-kafka-saga-outbox
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Kafka Config And Kafka Init Docker-Compose files.

Browse Source
This commit is contained in:
Ali CANLI
2022-07-10 22:08:14 +03:00
parent 1238b3ae6b
commit a3918a8cef
17 changed files with 393 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
.idea/sonarlint/issuestore/b/d/bd99e8fd918c1d01925143dbc2e08a8064258f70 generated
View File

0
.idea/sonarlint/issuestore/d/3/d3bf53ecc442740843266a635cbfe9807b992566 generated
View File

3
infrastructure/docker-compose/.env Normal file
View File

@@ -0,0 +1,3 @@
KAFKA_VERSION=7.0.1
GLOBAL_NETWORK=food-order-system
GROUP_ID=com.food-order-system

5
infrastructure/docker-compose/common.yml Normal file
View File

@@ -0,0 +1,5 @@
version: '3.7'
networks:
food-order-system:
driver: bridge

28
infrastructure/docker-compose/init_kafka.yml Normal file
View File

@@ -0,0 +1,28 @@
version: '3.7'
services:
init-kafka:
image: confluentinc/cp-kafka:${KAFKA_VERSION}
entrypoint: [ '/bin/sh', '-c' ]
command: |
"
# block until kafka is reachable
kafka-topics --bootstrap-server kafka-broker-1:9092 --list
echo -e 'Deleting kafka topics'
kafka-topics --bootstrap-server kafka-broker-1:9092 --topic payment-request --delete --if-exists
kafka-topics --bootstrap-server kafka-broker-1:9092 --topic payment-response --delete --if-exists
kafka-topics --bootstrap-server kafka-broker-1:9092 --topic restaurant-approval-request --delete --if-exists
kafka-topics --bootstrap-server kafka-broker-1:9092 --topic restaurant-approval-response --delete --if-exists
echo -e 'Creating kafka topics'
kafka-topics --bootstrap-server kafka-broker-1:9092 --create --if-not-exists --topic payment-request --replication-factor 3 --partitions 3
kafka-topics --bootstrap-server kafka-broker-1:9092 --create --if-not-exists --topic payment-response --replication-factor 3 --partitions 3
kafka-topics --bootstrap-server kafka-broker-1:9092 --create --if-not-exists --topic restaurant-approval-request --replication-factor 3 --partitions 3
kafka-topics --bootstrap-server kafka-broker-1:9092 --create --if-not-exists --topic restaurant-approval-response --replication-factor 3 --partitions 3
echo -e 'Successfully created the following topics:'
kafka-topics --bootstrap-server kafka-broker-1:9092 --list
"
networks:
- ${GLOBAL_NETWORK:-kafka}

79
infrastructure/docker-compose/kafka_cluster.yml Normal file
View File

@@ -0,0 +1,79 @@
version: '3.7'
services:
schema-registry:
image: confluentinc/cp-schema-registry:${KAFKA_VERSION}
hostname: schema-registry
depends_on:
- kafka-broker-1
- kafka-broker-2
- kafka-broker-3
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-broker-2:9092,LISTENER_LOCAL://localhost:29092
SCHEMA_REGISTRY_DEBUG: 'true'
networks:
- ${GLOBAL_NETWORK:-kafka}
kafka-broker-1:
image: confluentinc/cp-kafka:${KAFKA_VERSION}
hostname: kafka-broker-1
ports:
- "19092:19092"
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker-1:9092,LISTENER_LOCAL://localhost:19092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_COMPRESSION_TYPE: producer
volumes:
- "./volumes/kafka/broker-1:/var/lib/kafka/data"
networks:
- ${GLOBAL_NETWORK:-kafka}
kafka-broker-2:
image: confluentinc/cp-kafka:${KAFKA_VERSION}
hostname: kafka-broker-2
ports:
- "29092:29092"
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker-2:9092,LISTENER_LOCAL://localhost:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_COMPRESSION_TYPE: producer
volumes:
- "./volumes/kafka/broker-2:/var/lib/kafka/data"
networks:
- ${GLOBAL_NETWORK:-kafka}
kafka-broker-3:
image: confluentinc/cp-kafka:${KAFKA_VERSION}
hostname: kafka-broker-3
ports:
- "39092:39092"
environment:
KAFKA_BROKER_ID: 3
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker-3:9092,LISTENER_LOCAL://localhost:39092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_COMPRESSION_TYPE: producer
volumes:
- "./volumes/kafka/broker-3:/var/lib/kafka/data"
networks:
- ${GLOBAL_NETWORK:-kafka}
kafka-manager:
image: hlebalbau/kafka-manager:stable
restart: always
ports:
- "9000:9000"
environment:
ZK_HOSTS: "zookeeper:2181"
networks:
- ${GLOBAL_NETWORK:-kafka}

20
infrastructure/docker-compose/zookeeper.yml Normal file
View File

@@ -0,0 +1,20 @@
version: '3.7'
services:
zookeeper:
image: confluentinc/cp-zookeeper:${KAFKA_VERSION:-latest}
hostname: zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_INIT_LIMIT: 5
ZOOKEEPER_SYNC_LIMIT: 2
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_SERVERS: zookeeper:2181:3888
KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok"
volumes:
- "./volumes/zookeeper/data:/var/lib/zookeeper/data"
- "./volumes/zookeeper/transactions:/var/lib/zookeeper/log"
networks:
- ${GLOBAL_NETWORK:-kafka}

21
infrastructure/kafka/kafka-config/pom.xml Normal file
View File

@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kafka</artifactId>
<groupId>com.food.order</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kafka-config</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
</dependencies>
</project>

16
infrastructure/kafka/kafka-config/src/main/java/com/food/order/kafka/config/data/KafkaConfigData.java Normal file
View File

@@ -0,0 +1,16 @@
package com.food.order.kafka.config.data;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Data
@Configuration
@ConfigurationProperties(prefix = "kafka-config")
public class KafkaConfigData {
private String bootstrapServers;
private String schemaRegistryUrlKey;
private String schemaRegistryUrl;
private Integer numOfPartitions;
private Short replicationFactor;
}

27
infrastructure/kafka/kafka-config/src/main/java/com/food/order/kafka/config/data/KafkaConsumerConfigData.java Normal file
View File

@@ -0,0 +1,27 @@
package com.food.order.kafka.config.data;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Data
@Configuration
@ConfigurationProperties(prefix = "kafka-consumer-config")
public class KafkaConsumerConfigData {
private String keyDeserializer;
private String valueDeserializer;
private String autoOffsetReset;
private String specificAvroReaderKey;
private String specificAvroReader;
private Boolean batchListener;
private Boolean autoStartup;
private Integer concurrencyLevel;
private Integer sessionTimeoutMs;
private Integer heartbeatIntervalMs;
private Integer maxPollIntervalMs;
private Long pollTimeoutMs;
private Integer maxPollRecords;
private Integer maxPartitionFetchBytesDefault;
private Integer maxPartitionFetchBytesBoostFactor;
}

20
infrastructure/kafka/kafka-config/src/main/java/com/food/order/kafka/config/data/KafkaProducerConfigData.java Normal file
View File

@@ -0,0 +1,20 @@
package com.food.order.kafka.config.data;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Data
@Configuration
@ConfigurationProperties(prefix = "kafka-producer-config")
public class KafkaProducerConfigData {
private String keySerializerClass;
private String valueSerializerClass;
private String compressionType;
private String acks;
private Integer batchSize;
private Integer batchSizeBoostFactor;
private Integer lingerMs;
private Integer requestTimeoutMs;
private Integer retryCount;
}

19
infrastructure/kafka/kafka-consumer/pom.xml Normal file
View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kafka</artifactId>
<groupId>com.food.order</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kafka-consumer</artifactId>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
</project>

19
infrastructure/kafka/kafka-model/pom.xml Normal file
View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kafka</artifactId>
<groupId>com.food.order</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kafka-model</artifactId>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
</project>

19
infrastructure/kafka/kafka-producer/pom.xml Normal file
View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kafka</artifactId>
<groupId>com.food.order</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kafka-producer</artifactId>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
</project>

72
infrastructure/kafka/pom.xml Normal file
View File

@@ -0,0 +1,72 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>food-ordering-system</artifactId>
<groupId>com.food.order</groupId>
<version>1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kafka</artifactId>
<packaging>pom</packaging>
<modules>
<module>kafka-producer</module>
<module>kafka-consumer</module>
<module>kafka-model</module>
<module>kafka-config</module>
</modules>
<repositories>
<repository>
<id>confluent</id>
<url>https://packages.confluent.io/maven/</url>
</repository>
</repositories>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>${spring-kafka.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-avro-serializer</artifactId>
<version>${kavka-avro.serializer.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>io.swagger</groupId>
<artifactId>swagger-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

15
infrastructure/pom.xml Normal file
View File

@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>food-ordering-system</artifactId>
<groupId>com.food.order</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>infrastructure</artifactId>
</project>

30
pom.xml
View File

@@ -12,6 +12,8 @@
<modules>
<module>order-service</module>
<module>common</module>
<module>infrastructure</module>
<module>infrastructure/kafka</module>
</modules>
@@ -24,6 +26,9 @@
<properties>
<maven-compiler-plugin.version>3.9.0</maven-compiler-plugin.version>
<mockito.version>4.5.1</mockito.version>
<spring-kafka.version>2.8.2</spring-kafka.version>
<kavka-avro.serializer.version>7.0.1</kavka-avro.serializer.version>
<avro.version>1.11.0</avro.version>
</properties>
<dependencyManagement>
@@ -53,6 +58,31 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.food.order</groupId>
<artifactId>kafka-producer</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.food.order</groupId>
<artifactId>kafka-consumer</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.food.order</groupId>
<artifactId>kafka-model</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.food.order</groupId>
<artifactId>kafka-config</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.food.order</groupId>
<artifactId>order-data-access</artifactId>