From 8037776d9d73e54a3ee0254ffeb20689b56a5a55 Mon Sep 17 00:00:00 2001 From: Markus Gulden Date: Sun, 28 Oct 2018 20:53:10 +0100 Subject: [PATCH] BAEL-1658 --- .../connect-file-sink.properties | 20 +++++ .../connect-file-source.properties | 20 +++++ .../connect-standalone.properties | 44 ++++++++++ .../connect-distributed.properties | 88 +++++++++++++++++++ .../02_Distributed/connect-file-sink.json | 9 ++ .../02_Distributed/connect-file-source.json | 9 ++ .../connect-distributed.properties | 88 +++++++++++++++++++ .../connect-file-source-transform.json | 15 ++++ .../04_Custom/connect-distributed.properties | 88 +++++++++++++++++++ .../04_Custom/connect-mongodb-sink.json | 22 +++++ .../04_Custom/connect-mqtt-source.json | 11 +++ 11 files changed, 414 insertions(+) create mode 100644 libraries-data/src/main/kafka-connect/01_Quick_Start/connect-file-sink.properties create mode 100644 libraries-data/src/main/kafka-connect/01_Quick_Start/connect-file-source.properties create mode 100644 libraries-data/src/main/kafka-connect/01_Quick_Start/connect-standalone.properties create mode 100644 libraries-data/src/main/kafka-connect/02_Distributed/connect-distributed.properties create mode 100644 libraries-data/src/main/kafka-connect/02_Distributed/connect-file-sink.json create mode 100644 libraries-data/src/main/kafka-connect/02_Distributed/connect-file-source.json create mode 100644 libraries-data/src/main/kafka-connect/03_Transform/connect-distributed.properties create mode 100644 libraries-data/src/main/kafka-connect/03_Transform/connect-file-source-transform.json create mode 100644 libraries-data/src/main/kafka-connect/04_Custom/connect-distributed.properties create mode 100644 libraries-data/src/main/kafka-connect/04_Custom/connect-mongodb-sink.json create mode 100644 libraries-data/src/main/kafka-connect/04_Custom/connect-mqtt-source.json diff --git a/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-file-sink.properties b/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-file-sink.properties new file mode 100644 index 0000000000..594ccc6e95 --- /dev/null +++ b/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-file-sink.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name=local-file-sink +connector.class=FileStreamSink +tasks.max=1 +file=test.sink.txt +topics=connect-test \ No newline at end of file diff --git a/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-file-source.properties b/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-file-source.properties new file mode 100644 index 0000000000..599cf4cb2a --- /dev/null +++ b/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-file-source.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name=local-file-source +connector.class=FileStreamSource +tasks.max=1 +file=test.txt +topic=connect-test \ No newline at end of file diff --git a/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-standalone.properties b/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-standalone.properties new file mode 100644 index 0000000000..a2369fa144 --- /dev/null +++ b/libraries-data/src/main/kafka-connect/01_Quick_Start/connect-standalone.properties @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These are defaults. This file just demonstrates how to override some settings. +bootstrap.servers=localhost:9092 + +# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will +# need to configure these based on the format they want their data in when loaded from or stored into Kafka +key.converter=org.apache.kafka.connect.json.JsonConverter +value.converter=org.apache.kafka.connect.json.JsonConverter +# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply +# it to +key.converter.schemas.enable=false +value.converter.schemas.enable=false + +offset.storage.file.filename=/tmp/connect.offsets +# Flush much faster than normal, which is useful for testing/debugging +offset.flush.interval.ms=10000 + +# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins +# (connectors, converters, transformations). The list should consist of top level directories that include +# any combination of: +# a) directories immediately containing jars with plugins and their dependencies +# b) uber-jars with plugins and their dependencies +# c) directories immediately containing the package directory structure of classes of plugins and their dependencies +# Note: symlinks will be followed to discover dependencies or plugins. +# Examples: +# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors, +# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a +# directory other than the home directory of Confluent Platform. +plugin.path=C:\Software\confluent-5.0.0\share\java +#plugin.path=./share/java diff --git a/libraries-data/src/main/kafka-connect/02_Distributed/connect-distributed.properties b/libraries-data/src/main/kafka-connect/02_Distributed/connect-distributed.properties new file mode 100644 index 0000000000..5b91baddbd --- /dev/null +++ b/libraries-data/src/main/kafka-connect/02_Distributed/connect-distributed.properties @@ -0,0 +1,88 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +# This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended +# to be used with the examples, and some settings may differ from those used in a production system, especially +# the `bootstrap.servers` and those specifying replication factors. + +# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. +bootstrap.servers=localhost:9092 + +# unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs +group.id=connect-cluster + +# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will +# need to configure these based on the format they want their data in when loaded from or stored into Kafka +key.converter=org.apache.kafka.connect.json.JsonConverter +value.converter=org.apache.kafka.connect.json.JsonConverter +# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply +# it to +key.converter.schemas.enable=true +value.converter.schemas.enable=true + +# Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted. +# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +offset.storage.topic=connect-offsets +offset.storage.replication.factor=1 +#offset.storage.partitions=25 + +# Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated, +# and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +config.storage.topic=connect-configs +config.storage.replication.factor=1 + +# Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted. +# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +status.storage.topic=connect-status +status.storage.replication.factor=1 +#status.storage.partitions=5 + +# Flush much faster than normal, which is useful for testing/debugging +offset.flush.interval.ms=10000 + +# These are provided to inform the user about the presence of the REST host and port configs +# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests. +#rest.host.name= +#rest.port=8083 + +# The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers. +#rest.advertised.host.name= +#rest.advertised.port= + +# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins +# (connectors, converters, transformations). The list should consist of top level directories that include +# any combination of: +# a) directories immediately containing jars with plugins and their dependencies +# b) uber-jars with plugins and their dependencies +# c) directories immediately containing the package directory structure of classes of plugins and their dependencies +# Examples: +# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors, +# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a +# directory other than the home directory of Confluent Platform. +plugin.path=./share/java diff --git a/libraries-data/src/main/kafka-connect/02_Distributed/connect-file-sink.json b/libraries-data/src/main/kafka-connect/02_Distributed/connect-file-sink.json new file mode 100644 index 0000000000..8902ecce52 --- /dev/null +++ b/libraries-data/src/main/kafka-connect/02_Distributed/connect-file-sink.json @@ -0,0 +1,9 @@ +{ + "name": "local-file-sink", + "config": { + "connector.class": "FileStreamSink", + "tasks.max": 1, + "file": "test-distributed.sink.txt", + "topics": "connect-distributed" + } +} diff --git a/libraries-data/src/main/kafka-connect/02_Distributed/connect-file-source.json b/libraries-data/src/main/kafka-connect/02_Distributed/connect-file-source.json new file mode 100644 index 0000000000..77e949c91b --- /dev/null +++ b/libraries-data/src/main/kafka-connect/02_Distributed/connect-file-source.json @@ -0,0 +1,9 @@ +{ + "name": "local-file-source", + "config": { + "connector.class": "FileStreamSource", + "tasks.max": 1, + "file": "test-distributed.txt", + "topic": "connect-distributed" + } +} diff --git a/libraries-data/src/main/kafka-connect/03_Transform/connect-distributed.properties b/libraries-data/src/main/kafka-connect/03_Transform/connect-distributed.properties new file mode 100644 index 0000000000..fa63be24b8 --- /dev/null +++ b/libraries-data/src/main/kafka-connect/03_Transform/connect-distributed.properties @@ -0,0 +1,88 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +# This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended +# to be used with the examples, and some settings may differ from those used in a production system, especially +# the `bootstrap.servers` and those specifying replication factors. + +# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. +bootstrap.servers=localhost:9092 + +# unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs +group.id=connect-cluster + +# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will +# need to configure these based on the format they want their data in when loaded from or stored into Kafka +key.converter=org.apache.kafka.connect.json.JsonConverter +value.converter=org.apache.kafka.connect.json.JsonConverter +# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply +# it to +key.converter.schemas.enable=false +value.converter.schemas.enable=false + +# Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted. +# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +offset.storage.topic=connect-offsets +offset.storage.replication.factor=1 +#offset.storage.partitions=25 + +# Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated, +# and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +config.storage.topic=connect-configs +config.storage.replication.factor=1 + +# Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted. +# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +status.storage.topic=connect-status +status.storage.replication.factor=1 +#status.storage.partitions=5 + +# Flush much faster than normal, which is useful for testing/debugging +offset.flush.interval.ms=10000 + +# These are provided to inform the user about the presence of the REST host and port configs +# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests. +#rest.host.name= +#rest.port=8083 + +# The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers. +#rest.advertised.host.name= +#rest.advertised.port= + +# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins +# (connectors, converters, transformations). The list should consist of top level directories that include +# any combination of: +# a) directories immediately containing jars with plugins and their dependencies +# b) uber-jars with plugins and their dependencies +# c) directories immediately containing the package directory structure of classes of plugins and their dependencies +# Examples: +# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors, +# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a +# directory other than the home directory of Confluent Platform. +plugin.path=./share/java diff --git a/libraries-data/src/main/kafka-connect/03_Transform/connect-file-source-transform.json b/libraries-data/src/main/kafka-connect/03_Transform/connect-file-source-transform.json new file mode 100644 index 0000000000..e5e21a0608 --- /dev/null +++ b/libraries-data/src/main/kafka-connect/03_Transform/connect-file-source-transform.json @@ -0,0 +1,15 @@ +{ + "name": "local-file-source", + "config": { + "connector.class": "FileStreamSource", + "tasks.max": 1, + "file": "transformation.txt", + "topic": "connect-transformation", + "transforms": "MakeMap,InsertSource", + "transforms.MakeMap.type": "org.apache.kafka.connect.transforms.HoistField$Value", + "transforms.MakeMap.field": "line", + "transforms.InsertSource.type": "org.apache.kafka.connect.transforms.InsertField$Value", + "transforms.InsertSource.static.field": "data_source", + "transforms.InsertSource.static.value": "test-file-source" + } +} diff --git a/libraries-data/src/main/kafka-connect/04_Custom/connect-distributed.properties b/libraries-data/src/main/kafka-connect/04_Custom/connect-distributed.properties new file mode 100644 index 0000000000..5b91baddbd --- /dev/null +++ b/libraries-data/src/main/kafka-connect/04_Custom/connect-distributed.properties @@ -0,0 +1,88 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +# This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended +# to be used with the examples, and some settings may differ from those used in a production system, especially +# the `bootstrap.servers` and those specifying replication factors. + +# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. +bootstrap.servers=localhost:9092 + +# unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs +group.id=connect-cluster + +# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will +# need to configure these based on the format they want their data in when loaded from or stored into Kafka +key.converter=org.apache.kafka.connect.json.JsonConverter +value.converter=org.apache.kafka.connect.json.JsonConverter +# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply +# it to +key.converter.schemas.enable=true +value.converter.schemas.enable=true + +# Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted. +# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +offset.storage.topic=connect-offsets +offset.storage.replication.factor=1 +#offset.storage.partitions=25 + +# Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated, +# and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +config.storage.topic=connect-configs +config.storage.replication.factor=1 + +# Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted. +# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create +# the topic before starting Kafka Connect if a specific topic configuration is needed. +# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. +# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able +# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. +status.storage.topic=connect-status +status.storage.replication.factor=1 +#status.storage.partitions=5 + +# Flush much faster than normal, which is useful for testing/debugging +offset.flush.interval.ms=10000 + +# These are provided to inform the user about the presence of the REST host and port configs +# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests. +#rest.host.name= +#rest.port=8083 + +# The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers. +#rest.advertised.host.name= +#rest.advertised.port= + +# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins +# (connectors, converters, transformations). The list should consist of top level directories that include +# any combination of: +# a) directories immediately containing jars with plugins and their dependencies +# b) uber-jars with plugins and their dependencies +# c) directories immediately containing the package directory structure of classes of plugins and their dependencies +# Examples: +# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors, +# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a +# directory other than the home directory of Confluent Platform. +plugin.path=./share/java diff --git a/libraries-data/src/main/kafka-connect/04_Custom/connect-mongodb-sink.json b/libraries-data/src/main/kafka-connect/04_Custom/connect-mongodb-sink.json new file mode 100644 index 0000000000..333768e4b7 --- /dev/null +++ b/libraries-data/src/main/kafka-connect/04_Custom/connect-mongodb-sink.json @@ -0,0 +1,22 @@ +{ + "firstName": "John", + "lastName": "Smith", + "age": 25, + "address": { + "streetAddress": "21 2nd Street", + "city": "New York", + "state": "NY", + "postalCode": "10021" + }, + "phoneNumber": [{ + "type": "home", + "number": "212 555-1234" + }, { + "type": "fax", + "number": "646 555-4567" + } + ], + "gender": { + "type": "male" + } +} diff --git a/libraries-data/src/main/kafka-connect/04_Custom/connect-mqtt-source.json b/libraries-data/src/main/kafka-connect/04_Custom/connect-mqtt-source.json new file mode 100644 index 0000000000..02d87c5ad7 --- /dev/null +++ b/libraries-data/src/main/kafka-connect/04_Custom/connect-mqtt-source.json @@ -0,0 +1,11 @@ +{ + "name": "mqtt-source", + "config": { + "connector.class": "io.confluent.connect.mqtt.MqttSourceConnector", + "tasks.max": 1, + "mqtt.server.uri": "ws://broker.hivemq.com:8000/mqtt", + "mqtt.topics": "baeldung", + "kafka.topic": "connect-custom", + "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter" + } +}