JAVA-6390: Move kafka articles from libraries-data to new module
apache-kafka
This commit is contained in:
sampadawagde
@@ -1,20 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name=local-file-sink
|
||||
connector.class=FileStreamSink
|
||||
tasks.max=1
|
||||
file=test.sink.txt
|
||||
topics=connect-test
|
||||
@@ -1,20 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name=local-file-source
|
||||
connector.class=FileStreamSource
|
||||
tasks.max=1
|
||||
file=test.txt
|
||||
topic=connect-test
|
||||
@@ -1,44 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# These are defaults. This file just demonstrates how to override some settings.
|
||||
bootstrap.servers=localhost:9092
|
||||
|
||||
# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
|
||||
# need to configure these based on the format they want their data in when loaded from or stored into Kafka
|
||||
key.converter=org.apache.kafka.connect.json.JsonConverter
|
||||
value.converter=org.apache.kafka.connect.json.JsonConverter
|
||||
# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply
|
||||
# it to
|
||||
key.converter.schemas.enable=false
|
||||
value.converter.schemas.enable=false
|
||||
|
||||
offset.storage.file.filename=/tmp/connect.offsets
|
||||
# Flush much faster than normal, which is useful for testing/debugging
|
||||
offset.flush.interval.ms=10000
|
||||
|
||||
# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins
|
||||
# (connectors, converters, transformations). The list should consist of top level directories that include
|
||||
# any combination of:
|
||||
# a) directories immediately containing jars with plugins and their dependencies
|
||||
# b) uber-jars with plugins and their dependencies
|
||||
# c) directories immediately containing the package directory structure of classes of plugins and their dependencies
|
||||
# Note: symlinks will be followed to discover dependencies or plugins.
|
||||
# Examples:
|
||||
# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
|
||||
# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a
|
||||
# directory other than the home directory of Confluent Platform.
|
||||
plugin.path=C:\Software\confluent-5.0.0\share\java
|
||||
#plugin.path=./share/java
|
||||
@@ -1,88 +0,0 @@
|
||||
##
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
##
|
||||
|
||||
# This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended
|
||||
# to be used with the examples, and some settings may differ from those used in a production system, especially
|
||||
# the `bootstrap.servers` and those specifying replication factors.
|
||||
|
||||
# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster.
|
||||
bootstrap.servers=localhost:9092
|
||||
|
||||
# unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs
|
||||
group.id=connect-cluster
|
||||
|
||||
# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
|
||||
# need to configure these based on the format they want their data in when loaded from or stored into Kafka
|
||||
key.converter=org.apache.kafka.connect.json.JsonConverter
|
||||
value.converter=org.apache.kafka.connect.json.JsonConverter
|
||||
# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply
|
||||
# it to
|
||||
key.converter.schemas.enable=true
|
||||
value.converter.schemas.enable=true
|
||||
|
||||
# Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted.
|
||||
# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
offset.storage.topic=connect-offsets
|
||||
offset.storage.replication.factor=1
|
||||
#offset.storage.partitions=25
|
||||
|
||||
# Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated,
|
||||
# and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
config.storage.topic=connect-configs
|
||||
config.storage.replication.factor=1
|
||||
|
||||
# Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted.
|
||||
# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
status.storage.topic=connect-status
|
||||
status.storage.replication.factor=1
|
||||
#status.storage.partitions=5
|
||||
|
||||
# Flush much faster than normal, which is useful for testing/debugging
|
||||
offset.flush.interval.ms=10000
|
||||
|
||||
# These are provided to inform the user about the presence of the REST host and port configs
|
||||
# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests.
|
||||
#rest.host.name=
|
||||
#rest.port=8083
|
||||
|
||||
# The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers.
|
||||
#rest.advertised.host.name=
|
||||
#rest.advertised.port=
|
||||
|
||||
# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins
|
||||
# (connectors, converters, transformations). The list should consist of top level directories that include
|
||||
# any combination of:
|
||||
# a) directories immediately containing jars with plugins and their dependencies
|
||||
# b) uber-jars with plugins and their dependencies
|
||||
# c) directories immediately containing the package directory structure of classes of plugins and their dependencies
|
||||
# Examples:
|
||||
# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
|
||||
# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a
|
||||
# directory other than the home directory of Confluent Platform.
|
||||
plugin.path=./share/java
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "local-file-sink",
|
||||
"config": {
|
||||
"connector.class": "FileStreamSink",
|
||||
"tasks.max": 1,
|
||||
"file": "test-distributed.sink.txt",
|
||||
"topics": "connect-distributed"
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "local-file-source",
|
||||
"config": {
|
||||
"connector.class": "FileStreamSource",
|
||||
"tasks.max": 1,
|
||||
"file": "test-distributed.txt",
|
||||
"topic": "connect-distributed"
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
##
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
##
|
||||
|
||||
# This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended
|
||||
# to be used with the examples, and some settings may differ from those used in a production system, especially
|
||||
# the `bootstrap.servers` and those specifying replication factors.
|
||||
|
||||
# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster.
|
||||
bootstrap.servers=localhost:9092
|
||||
|
||||
# unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs
|
||||
group.id=connect-cluster
|
||||
|
||||
# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
|
||||
# need to configure these based on the format they want their data in when loaded from or stored into Kafka
|
||||
key.converter=org.apache.kafka.connect.json.JsonConverter
|
||||
value.converter=org.apache.kafka.connect.json.JsonConverter
|
||||
# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply
|
||||
# it to
|
||||
key.converter.schemas.enable=false
|
||||
value.converter.schemas.enable=false
|
||||
|
||||
# Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted.
|
||||
# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
offset.storage.topic=connect-offsets
|
||||
offset.storage.replication.factor=1
|
||||
#offset.storage.partitions=25
|
||||
|
||||
# Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated,
|
||||
# and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
config.storage.topic=connect-configs
|
||||
config.storage.replication.factor=1
|
||||
|
||||
# Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted.
|
||||
# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
status.storage.topic=connect-status
|
||||
status.storage.replication.factor=1
|
||||
#status.storage.partitions=5
|
||||
|
||||
# Flush much faster than normal, which is useful for testing/debugging
|
||||
offset.flush.interval.ms=10000
|
||||
|
||||
# These are provided to inform the user about the presence of the REST host and port configs
|
||||
# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests.
|
||||
#rest.host.name=
|
||||
#rest.port=8083
|
||||
|
||||
# The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers.
|
||||
#rest.advertised.host.name=
|
||||
#rest.advertised.port=
|
||||
|
||||
# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins
|
||||
# (connectors, converters, transformations). The list should consist of top level directories that include
|
||||
# any combination of:
|
||||
# a) directories immediately containing jars with plugins and their dependencies
|
||||
# b) uber-jars with plugins and their dependencies
|
||||
# c) directories immediately containing the package directory structure of classes of plugins and their dependencies
|
||||
# Examples:
|
||||
# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
|
||||
# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a
|
||||
# directory other than the home directory of Confluent Platform.
|
||||
plugin.path=./share/java
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"name": "local-file-source",
|
||||
"config": {
|
||||
"connector.class": "FileStreamSource",
|
||||
"tasks.max": 1,
|
||||
"file": "transformation.txt",
|
||||
"topic": "connect-transformation",
|
||||
"transforms": "MakeMap,InsertSource",
|
||||
"transforms.MakeMap.type": "org.apache.kafka.connect.transforms.HoistField$Value",
|
||||
"transforms.MakeMap.field": "line",
|
||||
"transforms.InsertSource.type": "org.apache.kafka.connect.transforms.InsertField$Value",
|
||||
"transforms.InsertSource.static.field": "data_source",
|
||||
"transforms.InsertSource.static.value": "test-file-source"
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"name": "mongodb-sink",
|
||||
"config": {
|
||||
"connector.class": "at.grahsl.kafka.connect.mongodb.MongoDbSinkConnector",
|
||||
"tasks.max": 1,
|
||||
"topics": "connect-custom",
|
||||
"mongodb.connection.uri": "mongodb://mongo-db/test?retryWrites=true",
|
||||
"mongodb.collection": "MyCollection",
|
||||
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
|
||||
"key.converter.schemas.enable": false,
|
||||
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
|
||||
"value.converter.schemas.enable": false
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"name": "mqtt-source",
|
||||
"config": {
|
||||
"connector.class": "io.confluent.connect.mqtt.MqttSourceConnector",
|
||||
"tasks.max": 1,
|
||||
"mqtt.server.uri": "tcp://mosquitto:1883",
|
||||
"mqtt.topics": "baeldung",
|
||||
"kafka.topic": "connect-custom",
|
||||
"value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
|
||||
"confluent.topic.bootstrap.servers": "kafka:9092",
|
||||
"confluent.topic.replication.factor": 1
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
version: '3.3'
|
||||
|
||||
services:
|
||||
mosquitto:
|
||||
image: eclipse-mosquitto:1.5.5
|
||||
hostname: mosquitto
|
||||
container_name: mosquitto
|
||||
expose:
|
||||
- "1883"
|
||||
ports:
|
||||
- "1883:1883"
|
||||
zookeeper:
|
||||
image: zookeeper:3.4.9
|
||||
restart: unless-stopped
|
||||
hostname: zookeeper
|
||||
container_name: zookeeper
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
ZOO_MY_ID: 1
|
||||
ZOO_PORT: 2181
|
||||
ZOO_SERVERS: server.1=zookeeper:2888:3888
|
||||
volumes:
|
||||
- ./zookeeper/data:/data
|
||||
- ./zookeeper/datalog:/datalog
|
||||
kafka:
|
||||
image: confluentinc/cp-kafka:5.1.0
|
||||
hostname: kafka
|
||||
container_name: kafka
|
||||
ports:
|
||||
- "9092:9092"
|
||||
environment:
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
||||
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
|
||||
KAFKA_BROKER_ID: 1
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
volumes:
|
||||
- ./kafka/data:/var/lib/kafka/data
|
||||
depends_on:
|
||||
- zookeeper
|
||||
kafka-connect:
|
||||
image: confluentinc/cp-kafka-connect:5.1.0
|
||||
hostname: kafka-connect
|
||||
container_name: kafka-connect
|
||||
ports:
|
||||
- "8083:8083"
|
||||
environment:
|
||||
CONNECT_BOOTSTRAP_SERVERS: "kafka:9092"
|
||||
CONNECT_REST_ADVERTISED_HOST_NAME: connect
|
||||
CONNECT_REST_PORT: 8083
|
||||
CONNECT_GROUP_ID: compose-connect-group
|
||||
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
|
||||
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
|
||||
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
|
||||
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||
CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars'
|
||||
CONNECT_CONFLUENT_TOPIC_REPLICATION_FACTOR: 1
|
||||
volumes:
|
||||
- /tmp/custom/jars:/etc/kafka-connect/jars
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- kafka
|
||||
- mosquitto
|
||||
mongo-db:
|
||||
image: mongo:4.0.5
|
||||
hostname: mongo-db
|
||||
container_name: mongo-db
|
||||
expose:
|
||||
- "27017"
|
||||
ports:
|
||||
- "27017:27017"
|
||||
command: --bind_ip_all --smallfiles
|
||||
volumes:
|
||||
- ./mongo-db:/data
|
||||
mongoclient:
|
||||
image: mongoclient/mongoclient:2.2.0
|
||||
container_name: mongoclient
|
||||
hostname: mongoclient
|
||||
depends_on:
|
||||
- mongo-db
|
||||
ports:
|
||||
- 3000:3000
|
||||
environment:
|
||||
MONGO_URL: "mongodb://mongo-db:27017"
|
||||
PORT: 3000
|
||||
expose:
|
||||
- "3000"
|
||||
@@ -1,62 +0,0 @@
|
||||
package com.baeldung.kafkastreams;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.apache.kafka.common.serialization.Serde;
|
||||
import org.apache.kafka.common.serialization.Serdes;
|
||||
import org.apache.kafka.streams.KafkaStreams;
|
||||
import org.apache.kafka.streams.StreamsConfig;
|
||||
import org.apache.kafka.streams.kstream.KStream;
|
||||
import org.apache.kafka.streams.kstream.KStreamBuilder;
|
||||
import org.apache.kafka.streams.kstream.KTable;
|
||||
import org.apache.kafka.test.TestUtils;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Properties;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class KafkaStreamsLiveTest {
|
||||
private String bootstrapServers = "localhost:9092";
|
||||
|
||||
@Test
|
||||
@Ignore("it needs to have kafka broker running on local")
|
||||
public void shouldTestKafkaStreams() throws InterruptedException {
|
||||
//given
|
||||
String inputTopic = "inputTopic";
|
||||
|
||||
Properties streamsConfiguration = new Properties();
|
||||
streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "wordcount-live-test");
|
||||
streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
|
||||
streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
|
||||
streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
|
||||
streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
|
||||
streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
|
||||
// Use a temporary directory for storing state, which will be automatically removed after the test.
|
||||
streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
|
||||
|
||||
//when
|
||||
KStreamBuilder builder = new KStreamBuilder();
|
||||
KStream<String, String> textLines = builder.stream(inputTopic);
|
||||
Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS);
|
||||
|
||||
KTable<String, Long> wordCounts = textLines
|
||||
.flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase())))
|
||||
.groupBy((key, word) -> word)
|
||||
.count();
|
||||
|
||||
wordCounts.foreach((word, count) -> System.out.println("word: " + word + " -> " + count));
|
||||
|
||||
String outputTopic = "outputTopic";
|
||||
final Serde<String> stringSerde = Serdes.String();
|
||||
final Serde<Long> longSerde = Serdes.Long();
|
||||
wordCounts.to(stringSerde, longSerde, outputTopic);
|
||||
|
||||
KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
|
||||
streams.start();
|
||||
|
||||
//then
|
||||
Thread.sleep(30000);
|
||||
streams.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user