Update for 2.0.0.M4 Release

Introduce a top level doc that aggregates various sub docs

pom.xml

@@ -2,28 +2,25 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
-	<version>1.3.1.RELEASE</version>
+	<version>2.0.0.M4</version>
 	<packaging>pom</packaging>
 	<parent>
 		<groupId>org.springframework.cloud</groupId>
 		<artifactId>spring-cloud-build</artifactId>
-		<version>1.3.7.RELEASE</version>
+		<version>2.0.0.M7</version>
 		<relativePath />
 	</parent>
 	<properties>
-		<java.version>1.7</java.version>
-		<kafka.version>0.10.1.1</kafka.version>
-		<spring-kafka.version>1.1.6.RELEASE</spring-kafka.version>
-		<spring-integration-kafka.version>2.1.2.RELEASE</spring-integration-kafka.version>
-		<spring-cloud-stream.version>1.3.1.RELEASE</spring-cloud-stream.version>
-		<spring-cloud-build.version>1.3.7.RELEASE</spring-cloud-build.version>
+		<java.version>1.8</java.version>
+		<spring-kafka.version>2.1.2.RELEASE</spring-kafka.version>
+		<spring-integration-kafka.version>3.0.1.RELEASE</spring-integration-kafka.version>
+		<kafka.version>1.0.0</kafka.version>
+		<spring-cloud-stream.version>2.0.0.M4</spring-cloud-stream.version>
 	</properties>
 	<modules>
 		<module>spring-cloud-stream-binder-kafka</module>
 		<module>spring-cloud-starter-stream-kafka</module>
 		<module>spring-cloud-stream-binder-kafka-docs</module>
-		<module>spring-cloud-stream-binder-kafka-0.10.1-test</module>
-		<module>spring-cloud-stream-binder-kafka-0.10.2-test</module>
 		<module>spring-cloud-stream-binder-kafka-core</module>
 		<module>spring-cloud-stream-binder-kstream</module>
     </modules>
@@ -45,30 +42,6 @@
 				<artifactId>spring-cloud-stream</artifactId>
 				<version>${spring-cloud-stream.version}</version>
 			</dependency>
-			<dependency>
-				<groupId>org.springframework.cloud</groupId>
-				<artifactId>spring-cloud-stream-codec</artifactId>
-				<version>${spring-cloud-stream.version}</version>
-			</dependency>
-			<dependency>
-				<groupId>org.apache.kafka</groupId>
-				<artifactId>kafka_2.11</artifactId>
-				<version>${kafka.version}</version>
-				<exclusions>
-					<exclusion>
-						<groupId>jline</groupId>
-						<artifactId>jline</artifactId>
-					</exclusion>
-					<exclusion>
-						<groupId>org.slf4j</groupId>
-						<artifactId>slf4j-log4j12</artifactId>
-					</exclusion>
-					<exclusion>
-						<groupId>log4j</groupId>
-						<artifactId>log4j</artifactId>
-					</exclusion>
-				</exclusions>
-			</dependency>
 			<dependency>
 				<groupId>org.apache.kafka</groupId>
 				<artifactId>kafka-clients</artifactId>
@@ -96,12 +69,6 @@
 				<scope>test</scope>
 				<version>${spring-kafka.version}</version>
 			</dependency>
-			<dependency>
-				<groupId>org.apache.kafka</groupId>
-				<artifactId>kafka_2.11</artifactId>
-				<classifier>test</classifier>
-				<version>${kafka.version}</version>
-			</dependency>
 			<dependency>
 				<groupId>org.apache.kafka</groupId>
 				<artifactId>kafka-streams</artifactId>
@@ -113,6 +80,27 @@
 					</exclusion>
 				</exclusions>
 			</dependency>
+			<dependency>
+				<groupId>org.apache.kafka</groupId>
+				<artifactId>kafka_2.11</artifactId>
+				<classifier>test</classifier>
+				<scope>test</scope>
+				<version>${kafka.version}</version>
+				<exclusions>
+					<exclusion>
+						<groupId>jline</groupId>
+						<artifactId>jline</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.slf4j</groupId>
+						<artifactId>slf4j-log4j12</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>log4j</groupId>
+						<artifactId>log4j</artifactId>
+					</exclusion>
+				</exclusions>
+			</dependency>
 		</dependencies>
 	</dependencyManagement>
 
@@ -124,18 +112,6 @@
 					<artifactId>maven-antrun-plugin</artifactId>
 					<version>1.7</version>
 				</plugin>
-				<plugin>
-					<groupId>org.apache.maven.plugins</groupId>
-					<artifactId>maven-checkstyle-plugin</artifactId>
-					<version>2.17</version>
-					<dependencies>
-						<dependency>
-							<groupId>com.puppycrawl.tools</groupId>
-							<artifactId>checkstyle</artifactId>
-							<version>7.1</version>
-						</dependency>
-					</dependencies>
-				</plugin>
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-javadoc-plugin</artifactId>
@@ -159,26 +135,15 @@
 				<dependencies>
 					<dependency>
 						<groupId>org.springframework.cloud</groupId>
-						<artifactId>spring-cloud-build-tools</artifactId>
-						<version>${spring-cloud-build.version}</version>
+						<artifactId>spring-cloud-stream-tools</artifactId>
+						<version>${spring-cloud-stream.version}</version>
 					</dependency>
 				</dependencies>
-				<executions>
-					<execution>
-						<id>checkstyle-validation</id>
-						<phase>validate</phase>
-						<configuration>
-							<configLocation>checkstyle.xml</configLocation>
-							<encoding>UTF-8</encoding>
-							<consoleOutput>true</consoleOutput>
-							<failsOnError>true</failsOnError>
-							<includeTestSourceDirectory>true</includeTestSourceDirectory>
-						</configuration>
-						<goals>
-							<goal>check</goal>
-						</goals>
-					</execution>
-				</executions>
+				<configuration>
+					<configLocation>checkstyle.xml</configLocation>
+					<headerLocation>checkstyle-header.txt</headerLocation>
+					<includeTestSourceDirectory>true</includeTestSourceDirectory>
+				</configuration>
 			</plugin>
 		</plugins>
 	</build>

spring-cloud-starter-stream-kafka/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>org.springframework.cloud</groupId>
 		<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
-		<version>1.3.1.RELEASE</version>
+		<version>2.0.0.M4</version>
 	</parent>
 	<artifactId>spring-cloud-starter-stream-kafka</artifactId>
 	<description>Spring Cloud Starter Stream Kafka</description>

spring-cloud-stream-binder-kafka-0.10.1-test/pom.xml

@@ -1,132 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <parent>
-        <groupId>org.springframework.cloud</groupId>
-        <artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
-        <version>1.3.1.RELEASE</version>
-    </parent>
-    <artifactId>spring-cloud-stream-binder-kafka-0.10.1-test</artifactId>
-    <description>Spring Cloud Stream Kafka Binder 0.10.1 Tests</description>
-    <url>http://projects.spring.io/spring-cloud</url>
-    <organization>
-        <name>Pivotal Software, Inc.</name>
-        <url>http://www.spring.io</url>
-    </organization>
-    <properties>
-        <main.basedir>${basedir}/../..</main.basedir>
-        <kafka.version>0.10.1.1</kafka.version>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-kafka-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-kafka</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.kafka</groupId>
-            <artifactId>spring-kafka</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka-clients</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.kafka</groupId>
-            <artifactId>spring-kafka-test</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.integration</groupId>
-            <artifactId>spring-integration-kafka</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.integration</groupId>
-            <artifactId>spring-integration-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.integration</groupId>
-            <artifactId>spring-integration-jmx</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-kafka</artifactId>
-            <version>${project.version}</version>
-            <type>test-jar</type>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-kafka-0.10.2-test</artifactId>
-            <version>${project.version}</version>
-            <type>test-jar</type>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-test</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-schema</artifactId>
-            <version>${spring-cloud-stream.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>io.confluent</groupId>
-            <artifactId>kafka-avro-serializer</artifactId>
-            <version>3.1.2</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>io.confluent</groupId>
-            <artifactId>kafka-schema-registry</artifactId>
-            <version>3.1.2</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <repositories>
-        <repository>
-            <id>confluent</id>
-            <url>http://packages.confluent.io/maven/</url>
-        </repository>
-    </repositories>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-jar-plugin</artifactId>
-                <version>3.0.2</version>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>test-jar</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>

spring-cloud-stream-binder-kafka-0.10.1-test/src/test/java/org/springframework/cloud/stream/binder/kafka/Kafka_0_10_1_BinderTests.java

@@ -1,242 +0,0 @@
-/*
- * Copyright 2014-2016 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.springframework.cloud.stream.binder.kafka;
-
-import io.confluent.kafka.schemaregistry.rest.SchemaRegistryConfig;
-import io.confluent.kafka.schemaregistry.rest.SchemaRegistryRestApplication;
-import kafka.utils.ZKStringSerializer$;
-import kafka.utils.ZkUtils;
-import org.I0Itec.zkclient.ZkClient;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.assertj.core.api.Assertions;
-import org.eclipse.jetty.server.Server;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.springframework.cloud.stream.binder.Binder;
-import org.springframework.cloud.stream.binder.Binding;
-import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
-import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
-import org.springframework.cloud.stream.binder.Spy;
-import org.springframework.cloud.stream.binder.kafka.admin.Kafka10AdminUtilsOperation;
-import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
-import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
-import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
-import org.springframework.integration.channel.DirectChannel;
-import org.springframework.integration.channel.QueueChannel;
-import org.springframework.kafka.core.ConsumerFactory;
-import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
-import org.springframework.kafka.support.KafkaHeaders;
-import org.springframework.kafka.test.core.BrokerAddress;
-import org.springframework.kafka.test.rule.KafkaEmbedded;
-import org.springframework.messaging.Message;
-import org.springframework.messaging.MessageChannel;
-import org.springframework.messaging.SubscribableChannel;
-import org.springframework.messaging.support.MessageBuilder;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Integration tests for the {@link KafkaMessageChannelBinder}.
- *
- * This test specifically tests for the 0.10.1.x version of Kafka.
- *
- * @author Eric Bottard
- * @author Marius Bogoevici
- * @author Mark Fisher
- * @author Ilayaperumal Gopinathan
- */
-public class Kafka_0_10_1_BinderTests extends Kafka_0_10_2_BinderTests {
-
-	private final String CLASS_UNDER_TEST_NAME = KafkaMessageChannelBinder.class.getSimpleName();
-
-	@ClassRule
-	public static KafkaEmbedded embeddedKafka = new KafkaEmbedded(1, true, 10);
-
-	private Kafka10TestBinder binder;
-
-	private final Kafka10AdminUtilsOperation adminUtilsOperation = new Kafka10AdminUtilsOperation();
-
-	@Override
-	protected void binderBindUnbindLatency() throws InterruptedException {
-		Thread.sleep(500);
-	}
-
-	@Override
-	protected Kafka10TestBinder getBinder() {
-		if (binder == null) {
-			KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
-			binderConfiguration.setHeaders("dlqTestHeader");
-			binder = new Kafka10TestBinder(binderConfiguration);
-		}
-		return binder;
-	}
-
-	@Override
-	protected KafkaBinderConfigurationProperties createConfigurationProperties() {
-		KafkaBinderConfigurationProperties binderConfiguration = new KafkaBinderConfigurationProperties();
-		BrokerAddress[] brokerAddresses = embeddedKafka.getBrokerAddresses();
-		List<String> bAddresses = new ArrayList<>();
-		for (BrokerAddress bAddress : brokerAddresses) {
-			bAddresses.add(bAddress.toString());
-		}
-		String[] foo = new String[bAddresses.size()];
-		binderConfiguration.setBrokers(bAddresses.toArray(foo));
-		binderConfiguration.setZkNodes(embeddedKafka.getZookeeperConnectionString());
-		return binderConfiguration;
-	}
-
-	@Override
-	protected int partitionSize(String topic) {
-		return consumerFactory().createConsumer().partitionsFor(topic).size();
-	}
-
-	@Override
-	protected ZkUtils getZkUtils(KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties) {
-		final ZkClient zkClient = new ZkClient(kafkaBinderConfigurationProperties.getZkConnectionString(),
-				kafkaBinderConfigurationProperties.getZkSessionTimeout(), kafkaBinderConfigurationProperties.getZkConnectionTimeout(),
-				ZKStringSerializer$.MODULE$);
-
-		return new ZkUtils(zkClient, null, false);
-	}
-
-	@Override
-	protected void invokeCreateTopic(ZkUtils zkUtils, String topic, int partitions, int replicationFactor, Properties topicConfig) {
-		adminUtilsOperation.invokeCreateTopic(zkUtils, topic, partitions, replicationFactor, new Properties());
-	}
-
-	@Override
-	protected int invokePartitionSize(String topic, ZkUtils zkUtils) {
-		return adminUtilsOperation.partitionSize(topic, zkUtils);
-	}
-
-	@Override
-	public String getKafkaOffsetHeaderKey() {
-		return KafkaHeaders.OFFSET;
-	}
-
-	@Override
-	protected Binder getBinder(KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties) {
-		return new Kafka10TestBinder(kafkaBinderConfigurationProperties);
-	}
-
-	@Before
-	public void init() {
-		String multiplier = System.getenv("KAFKA_TIMEOUT_MULTIPLIER");
-		if (multiplier != null) {
-			timeoutMultiplier = Double.parseDouble(multiplier);
-		}
-	}
-
-	@Override
-	protected boolean usesExplicitRouting() {
-		return false;
-	}
-
-	@Override
-	protected String getClassUnderTestName() {
-		return CLASS_UNDER_TEST_NAME;
-	}
-
-	@Override
-	public Spy spyOn(final String name) {
-		throw new UnsupportedOperationException("'spyOn' is not used by Kafka tests");
-	}
-
-
-	private ConsumerFactory<byte[], byte[]> consumerFactory() {
-		Map<String, Object> props = new HashMap<>();
-		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
-		props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, configurationProperties.getKafkaConnectionString());
-		props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
-		props.put(ConsumerConfig.GROUP_ID_CONFIG, "TEST-CONSUMER-GROUP");
-		Deserializer<byte[]> valueDecoder = new ByteArrayDeserializer();
-		Deserializer<byte[]> keyDecoder = new ByteArrayDeserializer();
-
-		return new DefaultKafkaConsumerFactory<>(props, keyDecoder, valueDecoder);
-	}
-
-	@Test
-	@SuppressWarnings("unchecked")
-	public void testCustomAvroSerialization() throws Exception {
-		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
-		final ZkClient zkClient = new ZkClient(configurationProperties.getZkConnectionString(),
-				configurationProperties.getZkSessionTimeout(), configurationProperties.getZkConnectionTimeout(),
-				ZKStringSerializer$.MODULE$);
-		final ZkUtils zkUtils = new ZkUtils(zkClient, null, false);
-		Map<String, Object> schemaRegistryProps = new HashMap<>();
-		schemaRegistryProps.put("kafkastore.connection.url", configurationProperties.getZkConnectionString());
-		schemaRegistryProps.put("listeners", "http://0.0.0.0:8082");
-		schemaRegistryProps.put("port", "8082");
-		schemaRegistryProps.put("kafkastore.topic", "_schemas");
-		SchemaRegistryConfig config = new SchemaRegistryConfig(schemaRegistryProps);
-		SchemaRegistryRestApplication app = new SchemaRegistryRestApplication(config);
-		Server server = app.createServer();
-		server.start();
-		long endTime = System.currentTimeMillis() + 5000;
-		while(true) {
-			if (server.isRunning()) {
-				break;
-			}
-			else if (System.currentTimeMillis() > endTime) {
-				Assertions.fail("Kafka Schema Registry Server failed to start");
-			}
-		}
-		User1 firstOutboundFoo = new User1();
-		String userName1 = "foo-name" + UUID.randomUUID().toString();
-		String favColor1 = "foo-color" + UUID.randomUUID().toString();
-		firstOutboundFoo.setName(userName1);
-		firstOutboundFoo.setFavoriteColor(favColor1);
-		Message<?> message = MessageBuilder.withPayload(firstOutboundFoo).build();
-		SubscribableChannel moduleOutputChannel = new DirectChannel();
-		String testTopicName = "existing" + System.currentTimeMillis();
-		invokeCreateTopic(zkUtils, testTopicName, 6, 1, new Properties());
-		configurationProperties.setAutoAddPartitions(true);
-		Binder binder = getBinder(configurationProperties);
-		QueueChannel moduleInputChannel = new QueueChannel();
-		ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
-		producerProperties.getExtension().getConfiguration().put("value.serializer", "io.confluent.kafka.serializers.KafkaAvroSerializer");
-		producerProperties.getExtension().getConfiguration().put("schema.registry.url", "http://localhost:8082");
-		producerProperties.setUseNativeEncoding(true);
-		Binding<MessageChannel> producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel, producerProperties);
-		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
-		consumerProperties.getExtension().setAutoRebalanceEnabled(false);
-		consumerProperties.getExtension().getConfiguration().put("value.deserializer", "io.confluent.kafka.serializers.KafkaAvroDeserializer");
-		consumerProperties.getExtension().getConfiguration().put("schema.registry.url", "http://localhost:8082");
-		Binding<MessageChannel> consumerBinding = binder.bindConsumer(testTopicName, "test", moduleInputChannel, consumerProperties);
-		// Let the consumer actually bind to the producer before sending a msg
-		binderBindUnbindLatency();
-		moduleOutputChannel.send(message);
-		Message<?> inbound = receive(moduleInputChannel);
-		Assertions.assertThat(inbound).isNotNull();
-		assertTrue(message.getPayload() instanceof User1);
-		User1 receivedUser = (User1) message.getPayload();
-		Assertions.assertThat(receivedUser.getName()).isEqualTo(userName1);
-		Assertions.assertThat(receivedUser.getFavoriteColor()).isEqualTo(favColor1);
-		producerBinding.unbind();
-		consumerBinding.unbind();
-	}
-}

spring-cloud-stream-binder-kafka-0.10.2-test/pom.xml

@@ -1,126 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <parent>
-        <groupId>org.springframework.cloud</groupId>
-        <artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
-        <version>1.3.1.RELEASE</version>
-    </parent>
-    <artifactId>spring-cloud-stream-binder-kafka-0.10.2-test</artifactId>
-    <description>Spring Cloud Stream Kafka Binder 0.10.2 Tests</description>
-    <url>http://projects.spring.io/spring-cloud</url>
-    <organization>
-        <name>Pivotal Software, Inc.</name>
-        <url>http://www.spring.io</url>
-    </organization>
-    <properties>
-        <main.basedir>${basedir}/../..</main.basedir>
-        <kafka.version>0.10.2.1</kafka.version>
-        <spring-kafka.version>1.2.2.RELEASE</spring-kafka.version>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-kafka-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-kafka</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.kafka</groupId>
-            <artifactId>spring-kafka</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka-clients</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.kafka</groupId>
-            <artifactId>spring-kafka-test</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.integration</groupId>
-            <artifactId>spring-integration-kafka</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.integration</groupId>
-            <artifactId>spring-integration-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.integration</groupId>
-            <artifactId>spring-integration-jmx</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-kafka</artifactId>
-            <version>${project.version}</version>
-            <type>test-jar</type>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-binder-test</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-stream-schema</artifactId>
-            <version>${spring-cloud-stream.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>io.confluent</groupId>
-            <artifactId>kafka-avro-serializer</artifactId>
-            <version>3.2.2</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>io.confluent</groupId>
-            <artifactId>kafka-schema-registry</artifactId>
-            <version>3.2.2</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <repositories>
-        <repository>
-            <id>confluent</id>
-            <url>http://packages.confluent.io/maven/</url>
-        </repository>
-    </repositories>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-jar-plugin</artifactId>
-                <version>3.0.2</version>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>test-jar</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>

spring-cloud-stream-binder-kafka-0.10.2-test/src/test/java/org/springframework/cloud/stream/binder/kafka/Kafka_0_10_2_BinderTests.java

@@ -1,246 +0,0 @@
-/*
- * Copyright 2014-2017 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.springframework.cloud.stream.binder.kafka;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import io.confluent.kafka.schemaregistry.rest.SchemaRegistryConfig;
-import io.confluent.kafka.schemaregistry.rest.SchemaRegistryRestApplication;
-
-import kafka.utils.ZKStringSerializer$;
-import kafka.utils.ZkUtils;
-
-import org.I0Itec.zkclient.ZkClient;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.assertj.core.api.Assertions;
-import org.eclipse.jetty.server.Server;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Test;
-
-import org.springframework.cloud.stream.binder.Binder;
-import org.springframework.cloud.stream.binder.Binding;
-import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
-import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
-import org.springframework.cloud.stream.binder.Spy;
-import org.springframework.cloud.stream.binder.kafka.admin.Kafka10AdminUtilsOperation;
-import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
-import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
-import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
-import org.springframework.integration.channel.DirectChannel;
-import org.springframework.integration.channel.QueueChannel;
-import org.springframework.kafka.core.ConsumerFactory;
-import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
-import org.springframework.kafka.support.KafkaHeaders;
-import org.springframework.kafka.test.core.BrokerAddress;
-import org.springframework.kafka.test.rule.KafkaEmbedded;
-import org.springframework.messaging.Message;
-import org.springframework.messaging.MessageChannel;
-import org.springframework.messaging.SubscribableChannel;
-import org.springframework.messaging.support.MessageBuilder;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Integration tests for the {@link KafkaMessageChannelBinder}.
- *
- * This test specifically tests for the 0.10.2.x version of Kafka.
- *
- * @author Eric Bottard
- * @author Marius Bogoevici
- * @author Mark Fisher
- * @author Ilayaperumal Gopinathan
- * @author Gary Russell
- */
-public class Kafka_0_10_2_BinderTests extends KafkaBinderTests {
-
-	private final String CLASS_UNDER_TEST_NAME = KafkaMessageChannelBinder.class.getSimpleName();
-
-	@ClassRule
-	public static KafkaEmbedded embeddedKafka = new KafkaEmbedded(1, true, 10);
-
-	private Kafka10TestBinder binder;
-
-	private final Kafka10AdminUtilsOperation adminUtilsOperation = new Kafka10AdminUtilsOperation();
-
-	@Override
-	protected void binderBindUnbindLatency() throws InterruptedException {
-		Thread.sleep(500);
-	}
-
-	@Override
-	protected Kafka10TestBinder getBinder() {
-		if (binder == null) {
-			KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
-			binderConfiguration.setHeaders("dlqTestHeader");
-			binder = new Kafka10TestBinder(binderConfiguration);
-		}
-		return binder;
-	}
-
-	@Override
-	protected KafkaBinderConfigurationProperties createConfigurationProperties() {
-		KafkaBinderConfigurationProperties binderConfiguration = new KafkaBinderConfigurationProperties();
-		BrokerAddress[] brokerAddresses = embeddedKafka.getBrokerAddresses();
-		List<String> bAddresses = new ArrayList<>();
-		for (BrokerAddress bAddress : brokerAddresses) {
-			bAddresses.add(bAddress.toString());
-		}
-		String[] foo = new String[bAddresses.size()];
-		binderConfiguration.setBrokers(bAddresses.toArray(foo));
-		binderConfiguration.setZkNodes(embeddedKafka.getZookeeperConnectionString());
-		return binderConfiguration;
-	}
-
-	@Override
-	protected int partitionSize(String topic) {
-		return consumerFactory().createConsumer().partitionsFor(topic).size();
-	}
-
-	@Override
-	protected ZkUtils getZkUtils(KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties) {
-		final ZkClient zkClient = new ZkClient(kafkaBinderConfigurationProperties.getZkConnectionString(),
-				kafkaBinderConfigurationProperties.getZkSessionTimeout(), kafkaBinderConfigurationProperties.getZkConnectionTimeout(),
-				ZKStringSerializer$.MODULE$);
-
-		return new ZkUtils(zkClient, null, false);
-	}
-
-	@Override
-	protected void invokeCreateTopic(ZkUtils zkUtils, String topic, int partitions, int replicationFactor, Properties topicConfig) {
-		adminUtilsOperation.invokeCreateTopic(zkUtils, topic, partitions, replicationFactor, new Properties());
-	}
-
-	@Override
-	protected int invokePartitionSize(String topic, ZkUtils zkUtils) {
-		return adminUtilsOperation.partitionSize(topic, zkUtils);
-	}
-
-	@Override
-	public String getKafkaOffsetHeaderKey() {
-		return KafkaHeaders.OFFSET;
-	}
-
-	@Override
-	protected Binder getBinder(KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties) {
-		return new Kafka10TestBinder(kafkaBinderConfigurationProperties);
-	}
-
-	@Before
-	public void init() {
-		String multiplier = System.getenv("KAFKA_TIMEOUT_MULTIPLIER");
-		if (multiplier != null) {
-			timeoutMultiplier = Double.parseDouble(multiplier);
-		}
-	}
-
-	@Override
-	protected boolean usesExplicitRouting() {
-		return false;
-	}
-
-	@Override
-	protected String getClassUnderTestName() {
-		return CLASS_UNDER_TEST_NAME;
-	}
-
-	@Override
-	public Spy spyOn(final String name) {
-		throw new UnsupportedOperationException("'spyOn' is not used by Kafka tests");
-	}
-
-
-	private ConsumerFactory<byte[], byte[]> consumerFactory() {
-		Map<String, Object> props = new HashMap<>();
-		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
-		props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, configurationProperties.getKafkaConnectionString());
-		props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
-		props.put(ConsumerConfig.GROUP_ID_CONFIG, "TEST-CONSUMER-GROUP");
-		Deserializer<byte[]> valueDecoder = new ByteArrayDeserializer();
-		Deserializer<byte[]> keyDecoder = new ByteArrayDeserializer();
-
-		return new DefaultKafkaConsumerFactory<>(props, keyDecoder, valueDecoder);
-	}
-
-	@Test
-	@SuppressWarnings("unchecked")
-	public void testCustomAvroSerialization() throws Exception {
-		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
-		final ZkClient zkClient = new ZkClient(configurationProperties.getZkConnectionString(),
-				configurationProperties.getZkSessionTimeout(), configurationProperties.getZkConnectionTimeout(),
-				ZKStringSerializer$.MODULE$);
-		final ZkUtils zkUtils = new ZkUtils(zkClient, null, false);
-		Map<String, Object> schemaRegistryProps = new HashMap<>();
-		schemaRegistryProps.put("kafkastore.connection.url", configurationProperties.getZkConnectionString());
-		schemaRegistryProps.put("listeners", "http://0.0.0.0:8082");
-		schemaRegistryProps.put("port", "8082");
-		schemaRegistryProps.put("kafkastore.topic", "_schemas");
-		SchemaRegistryConfig config = new SchemaRegistryConfig(schemaRegistryProps);
-		SchemaRegistryRestApplication app = new SchemaRegistryRestApplication(config);
-		Server server = app.createServer();
-		server.start();
-		long endTime = System.currentTimeMillis() + 5000;
-		while(true) {
-			if (server.isRunning()) {
-				break;
-			}
-			else if (System.currentTimeMillis() > endTime) {
-				Assertions.fail("Kafka Schema Registry Server failed to start");
-			}
-		}
-		User1 firstOutboundFoo = new User1();
-		String userName1 = "foo-name" + UUID.randomUUID().toString();
-		String favColor1 = "foo-color" + UUID.randomUUID().toString();
-		firstOutboundFoo.setName(userName1);
-		firstOutboundFoo.setFavoriteColor(favColor1);
-		Message<?> message = MessageBuilder.withPayload(firstOutboundFoo).build();
-		SubscribableChannel moduleOutputChannel = new DirectChannel();
-		String testTopicName = "existing" + System.currentTimeMillis();
-		invokeCreateTopic(zkUtils, testTopicName, 6, 1, new Properties());
-		configurationProperties.setAutoAddPartitions(true);
-		Binder binder = getBinder(configurationProperties);
-		QueueChannel moduleInputChannel = new QueueChannel();
-		ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
-		producerProperties.getExtension().getConfiguration().put("value.serializer", "io.confluent.kafka.serializers.KafkaAvroSerializer");
-		producerProperties.getExtension().getConfiguration().put("schema.registry.url", "http://localhost:8082");
-		producerProperties.setUseNativeEncoding(true);
-		Binding<MessageChannel> producerBinding = binder.bindProducer(testTopicName, moduleOutputChannel, producerProperties);
-		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
-		consumerProperties.getExtension().setAutoRebalanceEnabled(false);
-		consumerProperties.getExtension().getConfiguration().put("value.deserializer", "io.confluent.kafka.serializers.KafkaAvroDeserializer");
-		consumerProperties.getExtension().getConfiguration().put("schema.registry.url", "http://localhost:8082");
-		Binding<MessageChannel> consumerBinding = binder.bindConsumer(testTopicName, "test", moduleInputChannel, consumerProperties);
-		// Let the consumer actually bind to the producer before sending a msg
-		binderBindUnbindLatency();
-		moduleOutputChannel.send(message);
-		Message<?> inbound = receive(moduleInputChannel);
-		Assertions.assertThat(inbound).isNotNull();
-		assertTrue(message.getPayload() instanceof User1);
-		User1 receivedUser = (User1) message.getPayload();
-		Assertions.assertThat(receivedUser.getName()).isEqualTo(userName1);
-		Assertions.assertThat(receivedUser.getFavoriteColor()).isEqualTo(favColor1);
-		producerBinding.unbind();
-		consumerBinding.unbind();
-	}
-}

spring-cloud-stream-binder-kafka-0.10.2-test/src/test/java/org/springframework/cloud/stream/binder/kafka/User1.java

@@ -1,85 +0,0 @@
-/*
- * Copyright 2016 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.springframework.cloud.stream.binder.kafka;
-
-import java.io.IOException;
-
-import org.apache.avro.Schema;
-import org.apache.avro.reflect.Nullable;
-import org.apache.avro.specific.SpecificRecordBase;
-
-import org.springframework.core.io.ClassPathResource;
-
-/**
- * @author Marius Bogoevici
- * @author Ilayaperumal Gopinathan
- */
-public class User1 extends SpecificRecordBase {
-
-	@Nullable
-	private String name;
-
-	@Nullable
-	private String favoriteColor;
-
-	public String getName() {
-		return this.name;
-	}
-
-	public void setName(String name) {
-		this.name = name;
-	}
-
-	public String getFavoriteColor() {
-		return this.favoriteColor;
-	}
-
-	public void setFavoriteColor(String favoriteColor) {
-		this.favoriteColor = favoriteColor;
-	}
-
-	@Override
-	public Schema getSchema() {
-		try {
-			return new Schema.Parser().parse(new ClassPathResource("schemas/users_v1.schema").getInputStream());
-		}
-		catch (IOException e) {
-			throw new IllegalStateException(e);
-		}
-	}
-
-	@Override
-	public Object get(int i) {
-		if (i == 0) {
-			return getName().toString();
-		}
-		if (i == 1) {
-			return getFavoriteColor().toString();
-		}
-		return null;
-	}
-
-	@Override
-	public void put(int i, Object o) {
-		if (i == 0) {
-			setName((String) o);
-		}
-		if (i == 1) {
-			setFavoriteColor((String) o);
-		}
-	}
-}

spring-cloud-stream-binder-kafka-core/.settings/org.eclipse.jdt.ui.prefs

@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.ui.ignorelowercasenames=true
+org.eclipse.jdt.ui.importorder=java;javax;com;org;org.springframework;ch.qos;\#;
+org.eclipse.jdt.ui.ondemandthreshold=99
+org.eclipse.jdt.ui.staticondemandthreshold=99

spring-cloud-stream-binder-kafka-core/pom.xml

@@ -1,10 +1,22 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
     <parent>
         <groupId>org.springframework.cloud</groupId>
         <artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
-        <version>1.3.1.RELEASE</version>
+        <version>2.0.0.M4</version>
     </parent>
     <artifactId>spring-cloud-stream-binder-kafka-core</artifactId>
     <description>Spring Cloud Stream Kafka Binder Core</description>
@@ -26,10 +38,6 @@
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.springframework.integration</groupId>
             <artifactId>spring-integration-kafka</artifactId>
@@ -41,6 +49,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+        </dependency>
     </dependencies>
 
 </project>

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/admin/AdminUtilsOperation.java

@@ -1,75 +0,0 @@
-/*
- * Copyright 2002-2016 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.springframework.cloud.stream.binder.kafka.admin;
-
-import java.util.Properties;
-
-import kafka.utils.ZkUtils;
-
-/**
- * API around {@link kafka.admin.AdminUtils} to support
- * various versions of Kafka brokers.
- *
- * Note: Implementations that support Kafka brokers other than 0.10, need to use
- * a possible strategy that involves reflection around {@link kafka.admin.AdminUtils}.
- *
- * @author Soby Chacko
- */
-public interface AdminUtilsOperation {
-
-	/**
-	 * Invoke {@link kafka.admin.AdminUtils#addPartitions}
-	 *
-	 * @param zkUtils Zookeeper utils
-	 * @param topic name of the topic
-	 * @param numPartitions
-	 * @param replicaAssignmentStr
-	 * @param checkBrokerAvailable
-	 */
-	void invokeAddPartitions(ZkUtils zkUtils, String topic, int numPartitions,
-							String replicaAssignmentStr, boolean checkBrokerAvailable);
-
-	/**
-	 * Invoke {@link kafka.admin.AdminUtils#fetchTopicMetadataFromZk}
-	 *
-	 * @param topic name
-	 * @param zkUtils zookeeper utils
-	 * @return error code
-	 */
-	short errorCodeFromTopicMetadata(String topic, ZkUtils zkUtils);
-
-	/**
-	 * Find partition size from Kafka broker using {@link kafka.admin.AdminUtils}
-	 *
-	 * @param topic name
-	 * @param zkUtils zookeeper utils
-	 * @return partition size
-	 */
-	int partitionSize(String topic, ZkUtils zkUtils);
-
-	/**
-	 * Inovke {@link kafka.admin.AdminUtils#createTopic}
-	 *
-	 * @param zkUtils zookeeper utils
-	 * @param topic name
-	 * @param partitions
-	 * @param replicationFactor
-	 * @param topicConfig
-	 */
-	void invokeCreateTopic(ZkUtils zkUtils, String topic, int partitions,
-					int replicationFactor, Properties topicConfig);
-}

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/admin/Kafka09AdminUtilsOperation.java

@@ -1,145 +0,0 @@
-/*
- * Copyright 2002-2016 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.springframework.cloud.stream.binder.kafka.admin;
-
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.Properties;
-
-import kafka.utils.ZkUtils;
-
-import org.springframework.util.ClassUtils;
-import org.springframework.util.ReflectionUtils;
-
-/**
- * @author Soby Chacko
- */
-public class Kafka09AdminUtilsOperation implements AdminUtilsOperation {
-
-	private static Class<?> ADMIN_UTIL_CLASS;
-
-	static {
-		try {
-			ADMIN_UTIL_CLASS = ClassUtils.forName("kafka.admin.AdminUtils", null);
-		}
-		catch (ClassNotFoundException e) {
-			throw new IllegalStateException("AdminUtils class not found", e);
-		}
-	}
-
-	public void invokeAddPartitions(ZkUtils zkUtils, String topic, int numPartitions,
-									String replicaAssignmentStr, boolean checkBrokerAvailable) {
-		try {
-			Method[] declaredMethods = ADMIN_UTIL_CLASS.getDeclaredMethods();
-			Method addPartitions = null;
-			for (Method m : declaredMethods) {
-				if (m.getName().equals("addPartitions")) {
-					addPartitions = m;
-				}
-			}
-			if (addPartitions != null) {
-				addPartitions.invoke(null, zkUtils, topic, numPartitions,
-						replicaAssignmentStr, checkBrokerAvailable);
-			}
-			else {
-				throw new InvocationTargetException(
-						new RuntimeException("method not found"));
-			}
-		}
-		catch (InvocationTargetException e) {
-			ReflectionUtils.handleInvocationTargetException(e);
-		}
-		catch (IllegalAccessException e) {
-			ReflectionUtils.handleReflectionException(e);
-		}
-
-	}
-
-	public short errorCodeFromTopicMetadata(String topic, ZkUtils zkUtils) {
-		try {
-			Method fetchTopicMetadataFromZk = ReflectionUtils.findMethod(ADMIN_UTIL_CLASS, "fetchTopicMetadataFromZk", String.class, ZkUtils.class);
-			Object result = fetchTopicMetadataFromZk.invoke(null, topic, zkUtils);
-			Class<?> topicMetadataClass = ClassUtils.forName("kafka.api.TopicMetadata", null);
-			Method errorCodeMethod = ReflectionUtils.findMethod(topicMetadataClass, "errorCode");
-			return (short) errorCodeMethod.invoke(result);
-		}
-		catch (ClassNotFoundException e) {
-			throw new IllegalStateException("AdminUtils class not found", e);
-		}
-		catch (InvocationTargetException e) {
-			ReflectionUtils.handleInvocationTargetException(e);
-		}
-		catch (IllegalAccessException e) {
-			ReflectionUtils.handleReflectionException(e);
-		}
-		return 0;
-	}
-
-	@SuppressWarnings("unchecked")
-	public int partitionSize(String topic, ZkUtils zkUtils) {
-		try {
-			Method fetchTopicMetadataFromZk = ReflectionUtils.findMethod(ADMIN_UTIL_CLASS, "fetchTopicMetadataFromZk", String.class, ZkUtils.class);
-			Object result = fetchTopicMetadataFromZk.invoke(null, topic, zkUtils);
-			Class<?> topicMetadataClass = ClassUtils.forName("kafka.api.TopicMetadata", null);
-
-			Method partitionsMetadata = ReflectionUtils.findMethod(topicMetadataClass, "partitionsMetadata");
-			scala.collection.Seq<kafka.api.PartitionMetadata> partitionSize =
-					(scala.collection.Seq<kafka.api.PartitionMetadata>)partitionsMetadata.invoke(result);
-
-			return partitionSize.size();
-		}
-		catch (ClassNotFoundException e) {
-			throw new IllegalStateException("AdminUtils class not found", e);
-		}
-		catch (InvocationTargetException e) {
-			ReflectionUtils.handleInvocationTargetException(e);
-		}
-		catch (IllegalAccessException e) {
-			ReflectionUtils.handleReflectionException(e);
-		}
-		return 0;
-
-	}
-
-	public void invokeCreateTopic(ZkUtils zkUtils, String topic, int partitions,
-								int replicationFactor, Properties topicConfig) {
-		try {
-			Method[] declaredMethods = ADMIN_UTIL_CLASS.getDeclaredMethods();
-			Method createTopic = null;
-			for (Method m : declaredMethods) {
-				if (m.getName().equals("createTopic")) {
-					createTopic = m;
-					break;
-				}
-			}
-			if (createTopic != null) {
-				createTopic.invoke(null, zkUtils, topic, partitions,
-						replicationFactor, topicConfig);
-			}
-			else {
-				throw new InvocationTargetException(
-						new RuntimeException("method not found"));
-			}
-		}
-		catch (InvocationTargetException e) {
-			ReflectionUtils.handleInvocationTargetException(e);
-		}
-		catch (IllegalAccessException e) {
-			ReflectionUtils.handleReflectionException(e);
-		}
-	}
-}

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/admin/Kafka10AdminUtilsOperation.java

@@ -1,54 +0,0 @@
-/*
- * Copyright 2002-2016 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.springframework.cloud.stream.binder.kafka.admin;
-
-import java.util.Properties;
-
-import kafka.admin.AdminUtils;
-import kafka.utils.ZkUtils;
-import org.apache.kafka.common.requests.MetadataResponse;
-
-/**
- * @author Soby Chacko
- */
-public class Kafka10AdminUtilsOperation implements AdminUtilsOperation {
-
-	public void invokeAddPartitions(ZkUtils zkUtils, String topic, int numPartitions,
-									String replicaAssignmentStr, boolean checkBrokerAvailable) {
-		AdminUtils.addPartitions(zkUtils, topic, numPartitions, replicaAssignmentStr, checkBrokerAvailable, null);
-	}
-
-	public short errorCodeFromTopicMetadata(String topic, ZkUtils zkUtils) {
-
-		MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
-		return topicMetadata.error().code();
-	}
-
-	@SuppressWarnings("unchecked")
-	public int partitionSize(String topic, ZkUtils zkUtils) {
-
-		MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
-		return topicMetadata.partitionMetadata().size();
-	}
-
-	public void invokeCreateTopic(ZkUtils zkUtils, String topic, int partitions,
-									int replicationFactor, Properties topicConfig) {
-
-		AdminUtils.createTopic(zkUtils, topic, partitions, replicationFactor,
-				topicConfig, null);
-	}
-}

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/JaasLoginModuleConfiguration.java

@@ -18,6 +18,7 @@ package org.springframework.cloud.stream.binder.kafka.properties;
 
 import java.util.HashMap;
 import java.util.Map;
+
 import javax.security.auth.login.AppConfigurationEntry;
 
 import org.springframework.util.Assert;

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBinderConfigurationProperties.java

@@ -40,6 +40,10 @@ import org.springframework.util.StringUtils;
 @ConfigurationProperties(prefix = "spring.cloud.stream.kafka.binder")
 public class KafkaBinderConfigurationProperties {
 
+	private static final String DEFAULT_KAFKA_CONNECTION_STRING = "localhost:9092";
+
+	private final Transaction transaction = new Transaction();
+
 	@Autowired(required = false)
 	private KafkaProperties kafkaProperties;
 
@@ -96,6 +100,15 @@ public class KafkaBinderConfigurationProperties {
 
 	private JaasLoginModuleConfiguration jaas;
 
+	/**
+	 * The bean name of a custom header mapper to use instead of a {@link org.springframework.kafka.support.DefaultKafkaHeaderMapper}.
+	 */
+	private String headerMapperBeanName;
+
+	public Transaction getTransaction() {
+		return this.transaction;
+	}
+
 	public String getZkConnectionString() {
 		return toConnectionString(this.zkNodes, this.defaultZkPort);
 	}
@@ -104,6 +117,10 @@ public class KafkaBinderConfigurationProperties {
 		return toConnectionString(this.brokers, this.defaultBrokerPort);
 	}
 
+	public String getDefaultKafkaConnectionString() {
+		return DEFAULT_KAFKA_CONNECTION_STRING;
+	}
+
 	public String[] getHeaders() {
 		return this.headers;
 	}
@@ -352,4 +369,32 @@ public class KafkaBinderConfigurationProperties {
 		this.jaas = jaas;
 	}
 
+	public String getHeaderMapperBeanName() {
+		return this.headerMapperBeanName;
+	}
+
+	public void setHeaderMapperBeanName(String headerMapperBeanName) {
+		this.headerMapperBeanName = headerMapperBeanName;
+	}
+
+	public static class Transaction {
+
+		private final KafkaProducerProperties producer = new KafkaProducerProperties();
+
+		private String transactionIdPrefix;
+
+		public String getTransactionIdPrefix() {
+			return this.transactionIdPrefix;
+		}
+
+		public void setTransactionIdPrefix(String transactionIdPrefix) {
+			this.transactionIdPrefix = transactionIdPrefix;
+		}
+
+		public KafkaProducerProperties getProducer() {
+			return this.producer;
+		}
+
+	}
+
 }

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaConsumerProperties.java

@@ -22,6 +22,8 @@ import java.util.Map;
 /**
  * @author Marius Bogoevici
  * @author Ilayaperumal Gopinathan
+ * @author Soby Chacko
+ * @author Gary Russell
  *
  * <p>
  * Thanks to Laszlo Szabo for providing the initial patch for generic property support.
@@ -29,6 +31,28 @@ import java.util.Map;
  */
 public class KafkaConsumerProperties {
 
+	public enum StartOffset {
+		earliest(-2L),
+		latest(-1L);
+
+		private final long referencePoint;
+
+		StartOffset(long referencePoint) {
+			this.referencePoint = referencePoint;
+		}
+
+		public long getReferencePoint() {
+			return this.referencePoint;
+		}
+	}
+
+	public enum StandardHeaders {
+		none,
+		id,
+		timestamp,
+		both
+	}
+
 	private boolean autoRebalanceEnabled = true;
 
 	private boolean autoCommitOffset = true;
@@ -41,8 +65,18 @@ public class KafkaConsumerProperties {
 
 	private String dlqName;
 
+	private KafkaProducerProperties dlqProducerProperties = new KafkaProducerProperties();
+
 	private int recoveryInterval = 5000;
 
+	private String[] trustedPackages;
+
+	private StandardHeaders standardHeaders = StandardHeaders.none;
+
+	private String converterBeanName;
+
+	private long idleEventInterval = 30_000;
+
 	private Map<String, String> configuration = new HashMap<>();
 
 	public boolean isAutoCommitOffset() {
@@ -93,21 +127,6 @@ public class KafkaConsumerProperties {
 		this.autoRebalanceEnabled = autoRebalanceEnabled;
 	}
 
-	public enum StartOffset {
-		earliest(-2L),
-		latest(-1L);
-
-		private final long referencePoint;
-
-		StartOffset(long referencePoint) {
-			this.referencePoint = referencePoint;
-		}
-
-		public long getReferencePoint() {
-			return this.referencePoint;
-		}
-	}
-
 	public Map<String, String> getConfiguration() {
 		return this.configuration;
 	}
@@ -123,4 +142,44 @@ public class KafkaConsumerProperties {
 	public void setDlqName(String dlqName) {
 		this.dlqName = dlqName;
 	}
+
+	public String[] getTrustedPackages() {
+		return trustedPackages;
+	}
+
+	public void setTrustedPackages(String[] trustedPackages) {
+		this.trustedPackages = trustedPackages;
+	}
+
+	public KafkaProducerProperties getDlqProducerProperties() {
+		return dlqProducerProperties;
+	}
+
+	public void setDlqProducerProperties(KafkaProducerProperties dlqProducerProperties) {
+		this.dlqProducerProperties = dlqProducerProperties;
+	}
+	public StandardHeaders getStandardHeaders() {
+		return this.standardHeaders;
+	}
+
+	public void setStandardHeaders(StandardHeaders standardHeaders) {
+		this.standardHeaders = standardHeaders;
+	}
+
+	public String getConverterBeanName() {
+		return this.converterBeanName;
+	}
+
+	public void setConverterBeanName(String converterBeanName) {
+		this.converterBeanName = converterBeanName;
+	}
+
+	public long getIdleEventInterval() {
+		return this.idleEventInterval;
+	}
+
+	public void setIdleEventInterval(long idleEventInterval) {
+		this.idleEventInterval = idleEventInterval;
+	}
+
 }

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaExtendedBindingProperties.java

@@ -24,6 +24,7 @@ import org.springframework.cloud.stream.binder.ExtendedBindingProperties;
 
 /**
  * @author Marius Bogoevici
+ * @author Gary Russell
  */
 @ConfigurationProperties("spring.cloud.stream.kafka")
 public class KafkaExtendedBindingProperties
@@ -40,22 +41,45 @@ public class KafkaExtendedBindingProperties
 	}
 
 	@Override
-	public KafkaConsumerProperties getExtendedConsumerProperties(String channelName) {
-		if (this.bindings.containsKey(channelName) && this.bindings.get(channelName).getConsumer() != null) {
-			return this.bindings.get(channelName).getConsumer();
+	public synchronized KafkaConsumerProperties getExtendedConsumerProperties(String channelName) {
+		if (bindings.containsKey(channelName)) {
+			if (bindings.get(channelName).getConsumer() != null) {
+				return bindings.get(channelName).getConsumer();
+			}
+			else {
+				KafkaConsumerProperties properties = new KafkaConsumerProperties();
+				this.bindings.get(channelName).setConsumer(properties);
+				return properties;
+			}
 		}
 		else {
-			return new KafkaConsumerProperties();
+			KafkaConsumerProperties properties = new KafkaConsumerProperties();
+			KafkaBindingProperties rbp = new KafkaBindingProperties();
+			rbp.setConsumer(properties);
+			bindings.put(channelName, rbp);
+			return properties;
 		}
 	}
 
 	@Override
-	public KafkaProducerProperties getExtendedProducerProperties(String channelName) {
-		if (this.bindings.containsKey(channelName) && this.bindings.get(channelName).getProducer() != null) {
-			return this.bindings.get(channelName).getProducer();
+	public synchronized KafkaProducerProperties getExtendedProducerProperties(String channelName) {
+		if (bindings.containsKey(channelName)) {
+			if (bindings.get(channelName).getProducer() != null) {
+				return bindings.get(channelName).getProducer();
+			}
+			else {
+				KafkaProducerProperties properties = new KafkaProducerProperties();
+				this.bindings.get(channelName).setProducer(properties);
+				return properties;
+			}
 		}
 		else {
-			return new KafkaProducerProperties();
+			KafkaProducerProperties properties = new KafkaProducerProperties();
+			KafkaBindingProperties rbp = new KafkaBindingProperties();
+			rbp.setProducer(properties);
+			bindings.put(channelName, rbp);
+			return properties;
 		}
 	}
+
 }

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaProducerProperties.java

@@ -16,16 +16,17 @@
 
 package org.springframework.cloud.stream.binder.kafka.properties;
 
-import org.springframework.expression.Expression;
-
 import java.util.HashMap;
 import java.util.Map;
 
 import javax.validation.constraints.NotNull;
 
+import org.springframework.expression.Expression;
+
 /**
  * @author Marius Bogoevici
  * @author Henryk Konsek
+ * @author Gary Russell
  */
 public class KafkaProducerProperties {
 
@@ -39,6 +40,8 @@ public class KafkaProducerProperties {
 
 	private Expression messageKeyExpression;
 
+	private String[] headerPatterns;
+
 	private Map<String, String> configuration = new HashMap<>();
 
 	public int getBufferSize() {
@@ -82,6 +85,14 @@ public class KafkaProducerProperties {
 		this.messageKeyExpression = messageKeyExpression;
 	}
 
+	public String[] getHeaderPatterns() {
+		return this.headerPatterns;
+	}
+
+	public void setHeaderPatterns(String[] headerPatterns) {
+		this.headerPatterns = headerPatterns;
+	}
+
 	public Map<String, String> getConfiguration() {
 		return this.configuration;
 	}

spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/provisioning/KafkaTopicProvisioner.java

@@ -17,21 +17,32 @@
 package org.springframework.cloud.stream.binder.kafka.provisioning;
 
 import java.util.Collection;
-import java.util.Properties;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 
-import kafka.common.ErrorMapping;
-import kafka.utils.ZkUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.CreatePartitionsResult;
+import org.apache.kafka.clients.admin.CreateTopicsResult;
+import org.apache.kafka.clients.admin.DescribeTopicsResult;
+import org.apache.kafka.clients.admin.ListTopicsResult;
+import org.apache.kafka.clients.admin.NewPartitions;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.common.KafkaFuture;
 import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.security.JaasUtils;
 
 import org.springframework.beans.factory.InitializingBean;
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
 import org.springframework.cloud.stream.binder.BinderException;
 import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
 import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
-import org.springframework.cloud.stream.binder.kafka.admin.AdminUtilsOperation;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
@@ -40,13 +51,12 @@ import org.springframework.cloud.stream.provisioning.ConsumerDestination;
 import org.springframework.cloud.stream.provisioning.ProducerDestination;
 import org.springframework.cloud.stream.provisioning.ProvisioningException;
 import org.springframework.cloud.stream.provisioning.ProvisioningProvider;
-import org.springframework.retry.RetryCallback;
-import org.springframework.retry.RetryContext;
 import org.springframework.retry.RetryOperations;
 import org.springframework.retry.backoff.ExponentialBackOffPolicy;
 import org.springframework.retry.policy.SimpleRetryPolicy;
 import org.springframework.retry.support.RetryTemplate;
 import org.springframework.util.Assert;
+import org.springframework.util.ObjectUtils;
 import org.springframework.util.StringUtils;
 
 /**
@@ -60,18 +70,30 @@ import org.springframework.util.StringUtils;
 public class KafkaTopicProvisioner implements ProvisioningProvider<ExtendedConsumerProperties<KafkaConsumerProperties>,
 		ExtendedProducerProperties<KafkaProducerProperties>>, InitializingBean {
 
+	private static final int DEFAULT_OPERATION_TIMEOUT = 30;
+
 	private final Log logger = LogFactory.getLog(getClass());
 
-	private final KafkaBinderConfigurationProperties configurationProperties;
+	private KafkaBinderConfigurationProperties configurationProperties;
 
-	private final AdminUtilsOperation adminUtilsOperation;
+	private final AdminClient adminClient;
 
 	private RetryOperations metadataRetryOperations;
 
+	private int operationTimeout = DEFAULT_OPERATION_TIMEOUT;
+
 	public KafkaTopicProvisioner(KafkaBinderConfigurationProperties kafkaBinderConfigurationProperties,
-								AdminUtilsOperation adminUtilsOperation) {
+								KafkaProperties kafkaProperties) {
+		Assert.isTrue(kafkaProperties != null, "KafkaProperties cannot be null");
+		Map<String, Object> adminClientProperties = kafkaProperties.buildAdminProperties();
+		String kafkaConnectionString = kafkaBinderConfigurationProperties.getKafkaConnectionString();
+
+		if (ObjectUtils.isEmpty(adminClientProperties.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG))
+				|| !kafkaConnectionString.equals(kafkaBinderConfigurationProperties.getDefaultKafkaConnectionString())) {
+			adminClientProperties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaConnectionString);
+		}
 		this.configurationProperties = kafkaBinderConfigurationProperties;
-		this.adminUtilsOperation = adminUtilsOperation;
+		this.adminClient = AdminClient.create(adminClientProperties);
 	}
 
 	/**
@@ -105,14 +127,20 @@ public class KafkaTopicProvisioner implements ProvisioningProvider<ExtendedConsu
 			this.logger.info("Using kafka topic for outbound: " + name);
 		}
 		KafkaTopicUtils.validateTopicName(name);
-		createTopicsIfAutoCreateEnabledAndAdminUtilsPresent(name, properties.getPartitionCount(), false);
-		if (this.configurationProperties.isAutoCreateTopics() && adminUtilsOperation != null) {
-			final ZkUtils zkUtils = ZkUtils.apply(this.configurationProperties.getZkConnectionString(),
-					this.configurationProperties.getZkSessionTimeout(),
-					this.configurationProperties.getZkConnectionTimeout(),
-					JaasUtils.isZkSecurityEnabled());
-			int partitions = adminUtilsOperation.partitionSize(name, zkUtils);
-			return new KafkaProducerDestination(name, partitions);
+		createTopic(name, properties.getPartitionCount(), false);
+		if (this.configurationProperties.isAutoCreateTopics() && adminClient != null) {
+			DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(Collections.singletonList(name));
+			KafkaFuture<Map<String, TopicDescription>> all = describeTopicsResult.all();
+
+			try {
+				Map<String, TopicDescription> topicDescriptions = all.get(operationTimeout, TimeUnit.SECONDS);
+				TopicDescription topicDescription = topicDescriptions.get(name);
+				int partitions = topicDescription.partitions().size();
+				return new KafkaProducerDestination(name, partitions);
+			}
+			catch (Exception e) {
+				throw new ProvisioningException("Problems encountered with partitions finding", e);
+			}
 		}
 		else {
 			return new KafkaProducerDestination(name);
@@ -129,30 +157,67 @@ public class KafkaTopicProvisioner implements ProvisioningProvider<ExtendedConsu
 			throw new IllegalArgumentException("Instance count cannot be zero");
 		}
 		int partitionCount = properties.getInstanceCount() * properties.getConcurrency();
-		createTopicsIfAutoCreateEnabledAndAdminUtilsPresent(name, partitionCount, properties.getExtension().isAutoRebalanceEnabled());
-		if (this.configurationProperties.isAutoCreateTopics() && adminUtilsOperation != null) {
-			final ZkUtils zkUtils = ZkUtils.apply(this.configurationProperties.getZkConnectionString(),
-					this.configurationProperties.getZkSessionTimeout(),
-					this.configurationProperties.getZkConnectionTimeout(),
-					JaasUtils.isZkSecurityEnabled());
-			int partitions = adminUtilsOperation.partitionSize(name, zkUtils);
-			if (properties.getExtension().isEnableDlq() && !anonymous) {
-				String dlqTopic = StringUtils.hasText(properties.getExtension().getDlqName()) ?
-						properties.getExtension().getDlqName() : "error." + name + "." + group;
-				createTopicAndPartitions(dlqTopic, partitions, properties.getExtension().isAutoRebalanceEnabled());
-				return new KafkaConsumerDestination(name, partitions, dlqTopic);
+		createTopic(name, partitionCount, properties.getExtension().isAutoRebalanceEnabled());
+		if (this.configurationProperties.isAutoCreateTopics() && adminClient != null) {
+			DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(Collections.singletonList(name));
+			KafkaFuture<Map<String, TopicDescription>> all = describeTopicsResult.all();
+			try {
+				Map<String, TopicDescription> topicDescriptions = all.get(operationTimeout, TimeUnit.SECONDS);
+				TopicDescription topicDescription = topicDescriptions.get(name);
+				int partitions = topicDescription.partitions().size();
+				ConsumerDestination dlqTopic = createDlqIfNeedBe(name, group, properties, anonymous, partitions);
+				if (dlqTopic != null) return dlqTopic;
+				return new KafkaConsumerDestination(name, partitions);
+			}
+			catch (Exception e) {
+				throw new ProvisioningException("provisioning exception", e);
 			}
-			return new KafkaConsumerDestination(name, partitions);
 		}
 		return new KafkaConsumerDestination(name);
 	}
 
-	private void createTopicsIfAutoCreateEnabledAndAdminUtilsPresent(final String topicName, final int partitionCount,
-																	boolean tolerateLowerPartitionsOnBroker) {
-		if (this.configurationProperties.isAutoCreateTopics() && adminUtilsOperation != null) {
+	private ConsumerDestination createDlqIfNeedBe(String name, String group,
+												ExtendedConsumerProperties<KafkaConsumerProperties> properties,
+												boolean anonymous, int partitions) {
+		if (properties.getExtension().isEnableDlq() && !anonymous) {
+			String dlqTopic = StringUtils.hasText(properties.getExtension().getDlqName()) ?
+					properties.getExtension().getDlqName() : "error." + name + "." + group;
+			try {
+				createTopicAndPartitions(dlqTopic, partitions, properties.getExtension().isAutoRebalanceEnabled());
+			}
+			catch (Throwable throwable) {
+				if (throwable instanceof Error) {
+					throw (Error) throwable;
+				}
+				else {
+					throw new ProvisioningException("provisioning exception", throwable);
+				}
+			}
+			return new KafkaConsumerDestination(name, partitions, dlqTopic);
+		}
+		return null;
+	}
+
+	private void createTopic(String name, int partitionCount, boolean tolerateLowerPartitionsOnBroker) {
+		try {
+			createTopicIfNecessary(name, partitionCount, tolerateLowerPartitionsOnBroker);
+		}
+		catch (Throwable throwable) {
+			if (throwable instanceof Error) {
+				throw (Error) throwable;
+			}
+			else {
+				throw new ProvisioningException("provisioning exception", throwable);
+			}
+		}
+	}
+
+	private void createTopicIfNecessary(final String topicName, final int partitionCount,
+										boolean tolerateLowerPartitionsOnBroker) throws Throwable {
+		if (this.configurationProperties.isAutoCreateTopics() && adminClient != null) {
 			createTopicAndPartitions(topicName, partitionCount, tolerateLowerPartitionsOnBroker);
 		}
-		else if (this.configurationProperties.isAutoCreateTopics() && adminUtilsOperation == null) {
+		else if (this.configurationProperties.isAutoCreateTopics() && adminClient == null) {
 			this.logger.warn("Auto creation of topics is enabled, but Kafka AdminUtils class is not present on the classpath. " +
 					"No topic will be created by the binder");
 		}
@@ -166,75 +231,68 @@ public class KafkaTopicProvisioner implements ProvisioningProvider<ExtendedConsu
 	 * desired number.
 	 */
 	private void createTopicAndPartitions(final String topicName, final int partitionCount,
-										boolean tolerateLowerPartitionsOnBroker) {
+										boolean tolerateLowerPartitionsOnBroker) throws Throwable {
+		ListTopicsResult listTopicsResult = adminClient.listTopics();
+		KafkaFuture<Set<String>> namesFutures = listTopicsResult.names();
 
-		final ZkUtils zkUtils = ZkUtils.apply(this.configurationProperties.getZkConnectionString(),
-				this.configurationProperties.getZkSessionTimeout(),
-				this.configurationProperties.getZkConnectionTimeout(),
-				JaasUtils.isZkSecurityEnabled());
-		try {
-			short errorCode = adminUtilsOperation.errorCodeFromTopicMetadata(topicName, zkUtils);
-			if (errorCode == ErrorMapping.NoError()) {
-				// only consider minPartitionCount for resizing if autoAddPartitions is true
-				int effectivePartitionCount = this.configurationProperties.isAutoAddPartitions()
-						? Math.max(this.configurationProperties.getMinPartitionCount(), partitionCount)
-						: partitionCount;
-				int partitionSize = adminUtilsOperation.partitionSize(topicName, zkUtils);
-
-				if (partitionSize < effectivePartitionCount) {
-					if (this.configurationProperties.isAutoAddPartitions()) {
-						adminUtilsOperation.invokeAddPartitions(zkUtils, topicName, effectivePartitionCount, null, false);
-					}
-					else if (tolerateLowerPartitionsOnBroker) {
-						logger.warn("The number of expected partitions was: " + partitionCount + ", but "
-								+ partitionSize + (partitionSize > 1 ? " have " : " has ") + "been found instead."
-								+ "There will be " + (effectivePartitionCount - partitionSize) + " idle consumers");
-					}
-					else {
-						throw new ProvisioningException("The number of expected partitions was: " + partitionCount + ", but "
-								+ partitionSize + (partitionSize > 1 ? " have " : " has ") + "been found instead."
-								+ "Consider either increasing the partition count of the topic or enabling " +
-								"`autoAddPartitions`");
-					}
+		Set<String> names = namesFutures.get(operationTimeout, TimeUnit.SECONDS);
+		if (names.contains(topicName)) {
+			// only consider minPartitionCount for resizing if autoAddPartitions is true
+			int effectivePartitionCount = this.configurationProperties.isAutoAddPartitions()
+					? Math.max(this.configurationProperties.getMinPartitionCount(), partitionCount)
+					: partitionCount;
+			DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(Collections.singletonList(topicName));
+			KafkaFuture<Map<String, TopicDescription>> topicDescriptionsFuture = describeTopicsResult.all();
+			Map<String, TopicDescription> topicDescriptions = topicDescriptionsFuture.get(operationTimeout, TimeUnit.SECONDS);
+			TopicDescription topicDescription = topicDescriptions.get(topicName);
+			int partitionSize = topicDescription.partitions().size();
+			if (partitionSize < effectivePartitionCount) {
+				if (this.configurationProperties.isAutoAddPartitions()) {
+					CreatePartitionsResult partitions = adminClient.createPartitions(
+							Collections.singletonMap(topicName, NewPartitions.increaseTo(effectivePartitionCount)));
+					partitions.all().get(operationTimeout, TimeUnit.SECONDS);
+				}
+				else if (tolerateLowerPartitionsOnBroker) {
+					logger.warn("The number of expected partitions was: " + partitionCount + ", but "
+							+ partitionSize + (partitionSize > 1 ? " have " : " has ") + "been found instead."
+							+ "There will be " + (effectivePartitionCount - partitionSize) + " idle consumers");
+				}
+				else {
+					throw new ProvisioningException("The number of expected partitions was: " + partitionCount + ", but "
+							+ partitionSize + (partitionSize > 1 ? " have " : " has ") + "been found instead."
+							+ "Consider either increasing the partition count of the topic or enabling " +
+							"`autoAddPartitions`");
 				}
 			}
-			else if (errorCode == ErrorMapping.UnknownTopicOrPartitionCode()) {
-				// always consider minPartitionCount for topic creation
-				final int effectivePartitionCount = Math.max(this.configurationProperties.getMinPartitionCount(),
-						partitionCount);
-
-				this.metadataRetryOperations.execute(new RetryCallback<Object, RuntimeException>() {
-
-					@Override
-					public Object doWithRetry(RetryContext context) throws RuntimeException {
-
-						try {
-							adminUtilsOperation.invokeCreateTopic(zkUtils, topicName, effectivePartitionCount,
-									configurationProperties.getReplicationFactor(), new Properties());
-						}
-						catch (Exception e) {
-							String exceptionClass = e.getClass().getName();
-							if (exceptionClass.equals("kafka.common.TopicExistsException")
-									|| exceptionClass.equals("org.apache.kafka.common.errors.TopicExistsException")) {
-								if (logger.isWarnEnabled()) {
-									logger.warn("Attempt to create topic: " + topicName + ". Topic already exists.");
-								}
-							}
-							else {
-								throw e;
-							}
-						}
-						return null;
-					}
-				});
-			}
-			else {
-				throw new ProvisioningException("Error fetching Kafka topic metadata: ",
-						ErrorMapping.exceptionFor(errorCode));
-			}
 		}
-		finally {
-			zkUtils.close();
+		else if (!names.contains(topicName)) {
+			// always consider minPartitionCount for topic creation
+			final int effectivePartitionCount = Math.max(this.configurationProperties.getMinPartitionCount(),
+					partitionCount);
+			this.metadataRetryOperations.execute(context -> {
+
+				NewTopic newTopic = new NewTopic(topicName, effectivePartitionCount,
+						(short) configurationProperties.getReplicationFactor());
+				CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singletonList(newTopic));
+				try {
+					createTopicsResult.all().get(operationTimeout, TimeUnit.SECONDS);
+				}
+				catch (Exception e) {
+					if (e instanceof ExecutionException) {
+						String exceptionMessage = e.getMessage();
+						if (exceptionMessage.contains("org.apache.kafka.common.errors.TopicExistsException")) {
+							if (logger.isWarnEnabled()) {
+								logger.warn("Attempt to create topic: " + topicName + ". Topic already exists.");
+							}
+						}
+					}
+					else {
+						logger.error("Failed to create topics", e.getCause());
+						throw e.getCause();
+					}
+				}
+				return null;
+			});
 		}
 	}
 
@@ -243,27 +301,23 @@ public class KafkaTopicProvisioner implements ProvisioningProvider<ExtendedConsu
 														final Callable<Collection<PartitionInfo>> callable) {
 		try {
 			return this.metadataRetryOperations
-					.execute(new RetryCallback<Collection<PartitionInfo>, Exception>() {
-
-						@Override
-						public Collection<PartitionInfo> doWithRetry(RetryContext context) throws Exception {
-							Collection<PartitionInfo> partitions = callable.call();
-							// do a sanity check on the partition set
-							int partitionSize = partitions.size();
-							if (partitionSize < partitionCount) {
-								if (tolerateLowerPartitionsOnBroker) {
-									logger.warn("The number of expected partitions was: " + partitionCount + ", but "
-											+ partitionSize + (partitionSize > 1 ? " have " : " has ") + "been found instead."
-											+ "There will be " + (partitionCount - partitionSize) + " idle consumers");
-								}
-								else {
-									throw new IllegalStateException("The number of expected partitions was: "
-											+ partitionCount + ", but " + partitionSize
-											+ (partitionSize > 1 ? " have " : " has ") + "been found instead");
-								}
+					.execute(context -> {
+						Collection<PartitionInfo> partitions = callable.call();
+						// do a sanity check on the partition set
+						int partitionSize = partitions.size();
+						if (partitionSize < partitionCount) {
+							if (tolerateLowerPartitionsOnBroker) {
+								logger.warn("The number of expected partitions was: " + partitionCount + ", but "
+										+ partitionSize + (partitionSize > 1 ? " have " : " has ") + "been found instead."
+										+ "There will be " + (partitionCount - partitionSize) + " idle consumers");
+							}
+							else {
+								throw new IllegalStateException("The number of expected partitions was: "
+										+ partitionCount + ", but " + partitionSize
+										+ (partitionSize > 1 ? " have " : " has ") + "been found instead");
 							}
-							return partitions;
 						}
+						return partitions;
 					});
 		}
 		catch (Exception e) {

spring-cloud-stream-binder-kafka-docs/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.springframework.cloud</groupId>
         <artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
-        <version>1.3.1.RELEASE</version>
+        <version>2.0.0.M4</version>
     </parent>
 
     <artifactId>spring-cloud-stream-binder-kafka-docs</artifactId>

spring-cloud-stream-binder-kafka-docs/src/main/asciidoc/index.adoc

@@ -1,6 +1,6 @@
 [[spring-cloud-stream-binder-kafka-reference]]
 = Spring Cloud Stream Kafka Binder Reference Guide
-Sabby Anandan, Marius Bogoevici, Eric Bottard, Mark Fisher, Ilayaperumal Gopinathan, Gunnar Hillert, Mark Pollack, Patrick Peralta, Glenn Renfro, Thomas Risberg, Dave Syer, David Turanski, Janne Valkealahti, Benjamin Klein, Henryk Konsek
+Sabby Anandan, Marius Bogoevici, Eric Bottard, Mark Fisher, Ilayaperumal Gopinathan, Gunnar Hillert, Mark Pollack, Patrick Peralta, Glenn Renfro, Thomas Risberg, Dave Syer, David Turanski, Janne Valkealahti, Benjamin Klein, Henryk Konsek, Gary Russell
 :doctype: book
 :toc:
 :toclevels: 4
@@ -26,6 +26,8 @@ include::overview.adoc[]
 
 include::dlq.adoc[]
 
+include::partitions.adoc[]
+
 = Appendices
 [appendix]
 include::building.adoc[]

spring-cloud-stream-binder-kafka-docs/src/main/asciidoc/overview.adoc

@@ -120,7 +120,18 @@ spring.cloud.stream.kafka.binder.socketBufferSize::
   Size (in bytes) of the socket buffer to be used by the Kafka consumers.
 +
 Default: `2097152`.
+spring.cloud.stream.kafka.binder.transaction.transactionIdPrefix::
+  Enable transactions in the binder; see `transaction.id` in the Kafka documentation and https://docs.spring.io/spring-kafka/reference/html/_reference.html#transactions[Transactions] in the `spring-kafka` documentation.
+  When transactions are enabled, individual `producer` properties are ignored and all producers use the `spring.cloud.stream.kafka.binder.transaction.producer.*` properties.
++
+Default `null` (no transactions)
+spring.cloud.stream.kafka.binder.transaction.producer.*::
+  Global producer properties for producers in a transactional binder.
+  See `spring.cloud.stream.kafka.binder.transaction.transactionIdPrefix` and <<kafka-producer-properties>> and the general producer properties supported by all binders.
++
+Default: See individual producer properties.
 
+[[kafka-consumer-properties]]
 === Kafka Consumer Properties
 
 The following properties are available for Kafka consumers only and
@@ -163,6 +174,8 @@ enableDlq::
   By default, messages that result in errors will be forwarded to a topic named `error.<destination>.<group>`.
   The DLQ topic name can be configurable via the property `dlqName`.
   This provides an alternative option to the more common Kafka replay scenario for the case when the number of errors is relatively small and replaying the entire original topic may be too cumbersome.
+  See <<kafka-dlq-processing>> processing for more information.
+  Starting with _version 2.0_, messages sent to the DLQ topic are enhanced with the following headers: `x-original-topic`, `x-exception-message` and `x-exception-stacktrace` as `byte[]`.
 +
 Default: `false`.
 configuration::
@@ -173,7 +186,29 @@ dlqName::
   The name of the DLQ topic to receive the error messages.
 +
 Default: null (If not specified, messages that result in errors will be forwarded to a topic named `error.<destination>.<group>`).
+dlqProducerProperties::
+  Using this, dlq specific producer properties can be set.
+  All the properties available through kafka producer properties can be set through this property.
++
+Default: Default Kafka producer properties.
+standardHeaders::
+  Indicates which standard headers are populated by the inbound channel adapter.
+  `none`, `id`, `timestamp` or `both`.
+  Useful if using native deserialization and the first component to receive a message needs an `id` (such as an aggregator that is configured to use a JDBC message store).
++
+Default: `none`
+converterBeanName::
+  The name of a bean that implements `RecordMessageConverter`; used in the inbound channel adapter to replace the default `MessagingMessageConverter`.
++
+Default: `null`
+idleEventInterval::
+  The interval, in milliseconds between events indicating that no messages have recently been received.
+  Use an `ApplicationListener<ListenerContainerIdleEvent>` to receive these events.
+  See <<pause-resume>> for a usage example.
++
+Default: `30000`
 
+[[kafka-producer-properties]]
 === Kafka Producer Properties
 
 The following properties are available for Kafka producers only and
@@ -197,6 +232,14 @@ messageKeyExpression::
 For example `headers.key` or `payload.myKey`.
 +
 Default: `none`.
+headerPatterns::
+  A comma-delimited list of simple patterns to match spring-messaging headers to be mapped to the kafka `Headers` in the `ProducerRecord`.
+  Patterns can begin or end with the wildcard character (asterisk).
+  Patterns can be negated by prefixing with `!`; matching stops after the first match (positive or negative).
+  For example `!foo,fo*` will pass `fox` but not `foo`.
+  `id` and `timestamp` are never mapped.
++
+Default: `*` (all headers - except the `id` and `timestamp`)
 configuration::
   Map with a key/value pair containing generic Kafka producer properties.
 +
@@ -339,6 +382,46 @@ Usually applications may use principals that do not have administrative rights i
 In secure environments, we strongly recommend creating topics and managing ACLs administratively using Kafka tooling.
 ====
 
+[[pause-resume]]
+==== Example: Pausing and Resuming the Consumer
+
+If you wish to suspend consumption, but not cause a partition rebalance, you can pause and resume the consumer.
+This is facilitated by adding the `Consumer` as a parameter to your `@StreamListener`.
+To resume, you need an `ApplicationListener` for `ListenerContainerIdleEvent` s; the frequency at which events are published is controlled by the `idleEventInterval` property.
+Since the consumer is not thread-safe, you must call these methods on the calling thread.
+
+The following simple application shows how to pause and resume.
+
+[source, java]
+----
+@SpringBootApplication
+@EnableBinding(Sink.class)
+public class Application {
+
+	public static void main(String[] args) {
+		SpringApplication.run(Application.class, args);
+	}
+
+	@StreamListener(Sink.INPUT)
+	public void in(String in, @Header(KafkaHeaders.CONSUMER) Consumer<?, ?> consumer) {
+		System.out.println(in);
+		consumer.pause(Collections.singleton(new TopicPartition("myTopic", 0)));
+	}
+
+	@Bean
+	public ApplicationListener<ListenerContainerIdleEvent> idleListener() {
+		return event -> {
+			System.out.println(event);
+			if (event.getConsumer().paused().size() > 0) {
+				event.getConsumer().resume(event.getConsumer().paused());
+			}
+		};
+	}
+
+}
+----
+
+
 ==== Using the binder with Apache Kafka 0.10
 
 The default Kafka support in Spring Cloud Stream Kafka binder is for Kafka version 0.10.1.1. The binder also supports connecting to other 0.10 based versions and 0.9 clients.
@@ -441,7 +524,8 @@ Spring Cloud Stream Kafka support also includes a binder specifically designed f
 Using this binder, applications can be written that leverage the Kafka Streams API.
 For more information on Kafka Streams, see https://kafka.apache.org/documentation/streams/developer-guide[Kafka Streams API Developer Manual]
 
-Kafka Streams support in Spring Cloud Stream is based on the foundations provided by the Spring Kafka project. For details on that support, see http://docs.spring.io/spring-kafka/reference/html/_reference.html#kafka-streams[Kafaka Streams Support in Spring Kafka].
+Kafka Streams support in Spring Cloud Stream is based on the foundations provided by the Spring Kafka project.
+For details on that support, see http://docs.spring.io/spring-kafka/reference/html/_reference.html#kafka-streams[Kafaka Streams Support in Spring Kafka].
 
 Here are the maven coordinates for the Spring Cloud Stream KStream binder artifact.
 
@@ -453,8 +537,8 @@ Here are the maven coordinates for the Spring Cloud Stream KStream binder artifa
 </dependency>
 ----
 
-In addition to leveraging the Spring Cloud Stream programming model which is based on Spring Boot, one of the main other benefits that the KStream binder provides is the fact that it avoids the boilerplate configuration that one needs to write when using the Kafka Streams API directly.
-High level streams DSL provided through the Kafka Streams API can be used through Spring Cloud Stream in the current support.
+High level streams DSL provided through the Kafka Streams API can be used through Spring Cloud Stream support.
+Kafka Streams applications using the Spring Cloud Stream support can only be written using the processor model, i.e. messages read from an inbound topic and messages written to an outbound topic.
 
 === Usage example of high level streams DSL
 
@@ -470,36 +554,145 @@ public class WordCountProcessorApplication {
 	@SendTo("output")
 	public KStream<?, String> process(KStream<?, String> input) {
 		return input
-				.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
-				.map((key, word) -> new KeyValue<>(word, word))
-				.groupByKey(Serdes.String(), Serdes.String())
-				.count(TimeWindows.of(5000), "store-name")
-				.toStream()
-				.map((w, c) -> new KeyValue<>(null, "Count for " + w.key() + ": " + c));
-	}
+                .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
+                .groupBy((key, value) -> value)
+                .windowedBy(TimeWindows.of(5000))
+                .count(Materialized.as("WordCounts-multi"))
+                .toStream()
+                .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))));
+    }
 
 	public static void main(String[] args) {
 		SpringApplication.run(WordCountProcessorApplication.class, args);
 	}
 ----
 
-If you build it as Spring Boot runnable fat jar, you can run the above example in the following way:
+If you build it as a Spring Boot uber jar, you can run the above example in the following way:
 
 [source]
 ----
 java -jar uber.jar  --spring.cloud.stream.bindings.input.destination=words --spring.cloud.stream.bindings.output.destination=counts
 ----
 
-This means that the application will listen from the incoming Kafka topic words and write to the output topic counts.
+This means that the application will listen from the incoming Kafka topic `words` and write to the output topic `counts`.
 
 Spring Cloud Stream will ensure that the messages from both the incoming and outgoing topics are bound as KStream objects.
-As one may observe, the developer can exclusively focus on the business aspects of the code, i.e. writing the logic required in the processor rather than setting up the streams specific configuration required by the Kafka Streams infrastructure.
-All those boilerplate is handled by Spring Cloud Stream behind the scenes.
+Applications can exclusively focus on the business aspects of the code, i.e. writing the logic required in the processor rather than setting up the streams specific configuration required by the Kafka Streams infrastructure.
+All such interactions are handled by the framework.
+
+=== Message conversion in Spring Cloud Stream Kafka Streams applications
+
+If the following property is set (default is false), the framework skips all message conversions on the outbound (producer) side and it is then done by Kafka itself.
+
+`spring.cloud.stream.bindings.output.producer.useNativeEncoding`.
+
+Similarly, if the following property is set (default is false), any message conversion is skipped on the inbound (consumer) side and natively done by Kafka.
+
+`spring.cloud.stream.bindings.input.consumer.useNativeDecoding`.
+
+When native encoding is disabled, then the messages on the outbound are converted by Spring Cloud Stream using the provided `contentType` header.
+If no contentType is set by the application, it defaults to `application/json`.
+
+By default, all the out of the box message converters, serialize the data as `byte[]` encoding the proper contentType.
+In most situations, this is what you want to do, but if other formats than `byte[]` are desired, then ann appropriate message converter needs to be registered in the context and corresponding contentType specified as a property.
+When doing this way, Serdes should be overridden on the producer using the following property.
+
+`spring.cloud.stream.kstream.bindings.output.producer.valueSerde`.
+
+Keys will not get converted, but if the Serdes are different for keys from what is given as the common Serde, you can override that using the following property.
+
+`spring.cloud.stream.kstream.bindings.output.producer.keySerde`.
+
+=== Support for branching in Kafka Streams API
+
+Kafka Streams allow outbound data to be split into multiple topics based on some predicates.
+Spring Cloud Stream Kafka Streams binder provides support for this feature without losing the overall programming model exposed through `StreamListener` in the end user application.
+You write the application in the usual way as demonstrated above in the word count example.
+When using the branching feature, you are required to do a few things.
+First, you need to make sure that your return type is `KStream[]` instead of a regular `KStream`.
+Then you need to use the `SendTo` annotation containing the output bindings in the order (example below).
+For each of these output bindings, you need to configure destination, content-type etc. as required by any other standard Spring Cloud Stream application
+
+Here is an example:
+
+[source]
+----
+@EnableBinding(KStreamProcessorWithBranches.class)
+@EnableAutoConfiguration
+public static class WordCountProcessorApplication {
+
+    @Autowired
+    private TimeWindows timeWindows;
+
+    @StreamListener("input")
+    @SendTo({"output1","output2","output3})
+    public KStream<?, WordCount>[] process(KStream<Object, String> input) {
+
+			Predicate<Object, WordCount> isEnglish = (k, v) -> v.word.equals("english");
+			Predicate<Object, WordCount> isFrench =  (k, v) -> v.word.equals("french");
+			Predicate<Object, WordCount> isSpanish = (k, v) -> v.word.equals("spanish");
+
+			return input
+					.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
+					.groupBy((key, value) -> value)
+					.windowedBy(timeWindows)
+					.count(Materialized.as("WordCounts-1"))
+					.toStream()
+					.map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))))
+					.branch(isEnglish, isFrench, isSpanish);
+    }
+
+    interface KStreamProcessorWithBranches {
+
+    		@Input("input")
+    		KStream<?, ?> input();
+
+    		@Output("output1")
+    		KStream<?, ?> output1();
+
+    		@Output("output2")
+    		KStream<?, ?> output2();
+
+    		@Output("output3")
+    		KStream<?, ?> output3();
+    	}
+}
+----
+
+Then in the properties:
+
+[source]
+----
+spring.cloud.stream.bindings.output1.contentType: application/json
+spring.cloud.stream.bindings.output2.contentType: application/json
+spring.cloud.stream.bindings.output3.contentType: application/json
+spring.cloud.stream.kstream.binder.configuration.commit.interval.ms: 1000
+spring.cloud.stream.kstream.binder.configuration:
+  key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde
+  value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde
+spring.cloud.stream.bindings.output1:
+  destination: foo
+  producer:
+    headerMode: raw
+spring.cloud.stream.bindings.output2:
+  destination: bar
+  producer:
+    headerMode: raw
+spring.cloud.stream.bindings.output3:
+  destination: fox
+  producer:
+    headerMode: raw
+spring.cloud.stream.bindings.input:
+  destination: words
+  consumer:
+    headerMode: raw
+----
 
 === Support for interactive queries
 
 If access to the `KafkaStreams` is needed for interactive queries, the internal `KafkaStreams` instance can be accessed via `KStreamBuilderFactoryBean.getKafkaStreams()`.
-You can autowire the `KStreamBuilderFactoryBean` instance provided by the KStream binder. Then you can get `KafkaStreams` instance from it and retrieve the underlying store, execute queries on it, etc.
+You can autowire the `KStreamBuilderFactoryBean` instance provided by the KStream binder.
+Then you get `KafkaStreams` instance from it and retrieve the underlying store, execute queries on it, etc.
 
 === Kafka Streams properties
 
@@ -516,7 +709,7 @@ spring.cloud.stream.kstream.binder.configuration.value.serde=org.apache.kafka.co
 spring.cloud.stream.kstream.binder.configuration.commit.interval.ms=1000
 ----
 
-  For more information about all the properties that may go into streams configuration, see StreamsConfig JavaDocs.
+For more information about all the properties that may go into streams configuration, see StreamsConfig JavaDocs.
 
 There can also be binding specific properties.
 
@@ -528,34 +721,17 @@ spring.cloud.stream.kstream.bindings.output.producer.keySerde=org.apache.kafka.c
 spring.cloud.stream.kstream.bindings.output.producer.valueSerde=org.apache.kafka.common.serialization.Serdes$LongSerde
 ----
 
-timewindow.length::
-  Many streaming applications written using Kafka Streams involve windowning operations.
-  If you specify this property, there is a `org.apache.kafka.streams.kstream.TimeWindows` bean automatically provided that can be autowired in applications.
-  This property must be prefixed with `spring.cloud.stream.kstream.`.
-  A bean of type `org.apache.kafka.streams.kstream.TimeWindows` is created only if this property is provided.
-
- Following is an example of using this property.
- Values are provided in milliseconds.
+TimeWindow properties:
 
 [source]
 ----
-spring.cloud.stream.kstream.timeWindow.length=5000
+spring.cloud.stream.kstream.timeWindow.length (milliseconds)
+
+When this property is given, you can autowire a `TimeWindows` bean into the application.
+
+spring.cloud.stream.kstream.timeWindow.advanceBy (milliseconds)
 ----
 
-timewindow.advanceBy::
-  This property goes hand in hand with `timewindow.length` and has no effect on its own.
-  If you provide this property, the generated `org.apache.kafka.streams.kstream.TimeWindows` bean will automatically conatin this information.
-  This property must be prefixed with `spring.cloud.stream.kstream.`.
-
- Following is an example of using this property.
- Values are provided in milliseconds.
-
-[source]
-----
-spring.cloud.stream.kstream.timeWindow.advanceBy=1000
-----
-
-
 [[kafka-error-channels]]
 == Error Channels
 
@@ -567,8 +743,7 @@ The payload of the `ErrorMessage` for a send failure is a `KafkaSendFailureExcep
 * `failedMessage` - the spring-messaging `Message<?>` that failed to be sent.
 * `record` - the raw `ProducerRecord` that was created from the `failedMessage`
 
-There is no automatic handling of these exceptions (such as sending to a <<kafka-dlq-processing, Dead-Letter queue>>); you can consume these exceptions with your own Spring Integration flow.
-
+There is no automatic handling of producer exceptions (such as sending to a <<kafka-dlq-processing, Dead-Letter queue>>); you can consume these exceptions with your own Spring Integration flow.
 
 [[kafka-metrics]]
 == Kafka Metrics

spring-cloud-stream-binder-kafka-docs/src/main/asciidoc/partitions.adoc

@@ -0,0 +1,100 @@
+== Partitioning with the Kafka Binder
+
+Apache Kafka supports topic partitioning natively.
+
+Sometimes it is advantageous to send data to specific partitions, for example when you want to strictly order message processing - all messages for a particular customer should go to the same partition.
+
+The following illustrates how to configure the producer and consumer side:
+
+[source, java]
+----
+@SpringBootApplication
+@EnableBinding(Source.class)
+public class KafkaPartitionProducerApplication {
+
+    private static final Random RANDOM = new Random(System.currentTimeMillis());
+
+    private static final String[] data = new String[] {
+            "foo1", "bar1", "qux1",
+            "foo2", "bar2", "qux2",
+            "foo3", "bar3", "qux3",
+            "foo4", "bar4", "qux4",
+            };
+
+    public static void main(String[] args) {
+        new SpringApplicationBuilder(KafkaPartitionProducerApplication.class)
+            .web(false)
+            .run(args);
+    }
+
+    @InboundChannelAdapter(channel = Source.OUTPUT, poller = @Poller(fixedRate = "5000"))
+    public Message<?> generate() {
+        String value = data[RANDOM.nextInt(data.length)];
+        System.out.println("Sending: " + value);
+        return MessageBuilder.withPayload(value)
+                .setHeader("partitionKey", value)
+                .build();
+    }
+
+}
+----
+
+.application.yml
+[source, yaml]
+----
+spring:
+  cloud:
+    stream:
+      bindings:
+        output:
+          destination: partitioned.topic
+          producer:
+            partitioned: true
+            partition-key-expression: headers['partitionKey']
+            partition-count: 12
+----
+
+IMPORTANT: The topic must be provisioned to have enough partitions to achieve the desired concurrency for all consumer groups.
+The above configuration will support up to 12 consumer instances (or 6 if their `concurrency` is 2, etc.).
+It is generally best to "over provision" the partitions to allow for future increases in consumers and/or concurrency.
+
+NOTE: The above configuration uses the default partitioning (`key.hashCode() % partitionCount`).
+This may or may not provide a suitably balanced algorithm, depending on the key values; you can override this default by using the `partitionSelectorExpression` or `partitionSelectorClass` properties.
+
+Since partitions are natively handled by Kafka, no special configuration is needed on the consumer side.
+Kafka will allocate partitions across the instances.
+
+[source, java]
+----
+@SpringBootApplication
+@EnableBinding(Sink.class)
+public class KafkaPartitionConsumerApplication {
+
+    public static void main(String[] args) {
+        new SpringApplicationBuilder(KafkaPartitionConsumerApplication.class)
+            .web(false)
+            .run(args);
+    }
+
+    @StreamListener(Sink.INPUT)
+    public void listen(@Payload String in, @Header(KafkaHeaders.RECEIVED_PARTITION_ID) int partition) {
+        System.out.println(in + " received from partition " + partition);
+    }
+
+}
+----
+
+.application.yml
+[source, yaml]
+----
+spring:
+  cloud:
+    stream:
+      bindings:
+        input:
+          destination: partitioned.topic
+          group: myGroup
+----
+
+You can add instances as needed; Kafka will rebalance the partition allocations.
+If the instance count (or `instance count * concurrency`) exceeds the number of partitions, some consumers will be idle.

spring-cloud-stream-binder-kafka-docs/src/main/asciidoc/spring-cloud-stream-binder-kafka-aggregate.adoc

@@ -0,0 +1,3 @@
+include::overview.adoc[leveloffset=+1]
+include::dlq.adoc[leveloffset=+1]
+include::partitions.adoc[leveloffset=+1]

spring-cloud-stream-binder-kafka/.settings/org.eclipse.jdt.ui.prefs

@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.ui.ignorelowercasenames=true
+org.eclipse.jdt.ui.importorder=java;javax;com;org;org.springframework;ch.qos;\#;
+org.eclipse.jdt.ui.ondemandthreshold=99
+org.eclipse.jdt.ui.staticondemandthreshold=99

spring-cloud-stream-binder-kafka/pom.xml

@@ -10,7 +10,7 @@
 	<parent>
 		<groupId>org.springframework.cloud</groupId>
 		<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
-		<version>1.3.1.RELEASE</version>
+		<version>2.0.0.M4</version>
 	</parent>
 
 	<dependencies>
@@ -27,10 +27,6 @@
 			<groupId>org.springframework.cloud</groupId>
 			<artifactId>spring-cloud-stream</artifactId>
 		</dependency>
-		<dependency>
-			<groupId>org.springframework.cloud</groupId>
-			<artifactId>spring-cloud-stream-codec</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-autoconfigure</artifactId>
@@ -41,10 +37,6 @@
 			<artifactId>spring-cloud-stream-binder-test</artifactId>
 			<scope>test</scope>
 		</dependency>
-		<dependency>
-			<groupId>org.apache.kafka</groupId>
-			<artifactId>kafka_2.11</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>org.apache.kafka</groupId>
 			<artifactId>kafka-clients</artifactId>
@@ -63,12 +55,6 @@
 			<artifactId>spring-kafka-test</artifactId>
 			<scope>test</scope>
 		</dependency>
-		<dependency>
-			<groupId>org.apache.kafka</groupId>
-			<artifactId>kafka_2.11</artifactId>
-			<classifier>test</classifier>
-			<scope>test</scope>
-		</dependency>
 		<dependency>
 			<groupId>org.springframework.cloud</groupId>
 			<artifactId>spring-cloud-stream-binder-test</artifactId>
@@ -76,20 +62,4 @@
 		</dependency>
 	</dependencies>
 
-	<build>
-		<plugins>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-jar-plugin</artifactId>
-				<version>3.0.2</version>
-				<executions>
-					<execution>
-						<goals>
-							<goal>test-jar</goal>
-						</goals>
-					</execution>
-				</executions>
-			</plugin>
-		</plugins>
-	</build>
 </project>

spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java

@@ -41,6 +41,7 @@ import org.springframework.kafka.core.ConsumerFactory;
  * @author Marius Bogoevici
  * @author Henryk Konsek
  * @author Gary Russell
+ * @author Laur Aliste
  */
 public class KafkaBinderHealthIndicator implements HealthIndicator {
 
@@ -52,15 +53,16 @@ public class KafkaBinderHealthIndicator implements HealthIndicator {
 
 	private int timeout = DEFAULT_TIMEOUT;
 
-	public KafkaBinderHealthIndicator(KafkaMessageChannelBinder binder,
-			ConsumerFactory<?, ?> consumerFactory) {
+	private Consumer<?, ?> metadataConsumer;
+
+	public KafkaBinderHealthIndicator(KafkaMessageChannelBinder binder, ConsumerFactory<?, ?> consumerFactory) {
 		this.binder = binder;
 		this.consumerFactory = consumerFactory;
-
 	}
 
 	/**
 	 * Set the timeout in seconds to retrieve health information.
+	 *
 	 * @param timeout the timeout - default 60.
 	 */
 	public void setTimeout(int timeout) {
@@ -74,7 +76,10 @@ public class KafkaBinderHealthIndicator implements HealthIndicator {
 
 			@Override
 			public Health call() {
-				try (Consumer<?, ?> metadataConsumer = consumerFactory.createConsumer()) {
+				try {
+					if (metadataConsumer == null) {
+						metadataConsumer = consumerFactory.createConsumer();
+					}
 					Set<String> downMessages = new HashSet<>();
 					for (String topic : KafkaBinderHealthIndicator.this.binder.getTopicsInUse().keySet()) {
 						List<PartitionInfo> partitionInfos = metadataConsumer.partitionsFor(topic);

spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderJaasInitializerListener.java

@@ -1,118 +0,0 @@
-/*
- * Copyright 2016 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.springframework.cloud.stream.binder.kafka;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import javax.security.auth.login.AppConfigurationEntry;
-import javax.security.auth.login.Configuration;
-
-import org.apache.kafka.common.security.JaasUtils;
-
-import org.springframework.beans.BeansException;
-import org.springframework.beans.factory.DisposableBean;
-import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
-import org.springframework.context.ApplicationListener;
-import org.springframework.context.event.ContextRefreshedEvent;
-import org.springframework.util.Assert;
-
-/**
- * @author Marius Bogoevici
- */
-public class KafkaBinderJaasInitializerListener implements ApplicationListener<ContextRefreshedEvent>,
-		ApplicationContextAware, DisposableBean {
-
-	public static final String DEFAULT_ZK_LOGIN_CONTEXT_NAME = "Client";
-
-	private ApplicationContext applicationContext;
-
-	private final boolean ignoreJavaLoginConfigParamSystemProperty;
-
-	private final File placeholderJaasConfiguration;
-
-	public KafkaBinderJaasInitializerListener() throws IOException {
-		// we ignore the system property if it wasn't originally set at launch
-		this.ignoreJavaLoginConfigParamSystemProperty =
-				(System.getProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM) == null);
-		this.placeholderJaasConfiguration = File.createTempFile("kafka-client-jaas-config-placeholder", "conf");
-		this.placeholderJaasConfiguration.deleteOnExit();
-	}
-
-	@Override
-	public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
-		this.applicationContext = applicationContext;
-	}
-
-	@Override
-	public void destroy() throws Exception {
-		if (this.ignoreJavaLoginConfigParamSystemProperty) {
-			System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM);
-		}
-	}
-
-	@Override
-	public void onApplicationEvent(ContextRefreshedEvent event) {
-		if (event.getSource() == this.applicationContext) {
-			KafkaBinderConfigurationProperties binderConfigurationProperties =
-					applicationContext.getBean(KafkaBinderConfigurationProperties.class);
-			// only use programmatic support if a file is not set via system property
-			if (ignoreJavaLoginConfigParamSystemProperty
-					&& binderConfigurationProperties.getJaas() != null) {
-				Map<String, AppConfigurationEntry[]> configurationEntries = new HashMap<>();
-				AppConfigurationEntry kafkaClientConfigurationEntry = new AppConfigurationEntry
-						(binderConfigurationProperties.getJaas().getLoginModule(),
-								binderConfigurationProperties.getJaas().getControlFlagValue(),
-								binderConfigurationProperties.getJaas().getOptions() != null ?
-										binderConfigurationProperties.getJaas().getOptions() :
-										Collections.<String, Object>emptyMap());
-				configurationEntries.put(JaasUtils.LOGIN_CONTEXT_CLIENT,
-						new AppConfigurationEntry[]{ kafkaClientConfigurationEntry });
-				Configuration.setConfiguration(new InternalConfiguration(configurationEntries));
-				// Workaround for a 0.9 client issue where even if the Configuration is set
-				// a system property check is performed.
-				// Since the Configuration already exists, this will be ignored.
-				if (this.placeholderJaasConfiguration != null) {
-					System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, this.placeholderJaasConfiguration.getAbsolutePath());
-				}
-			}
-		}
-	}
-
-	/**
-	 * A {@link Configuration} set up programmatically by the Kafka binder
-	 */
-	public static class InternalConfiguration extends Configuration {
-
-		private final Map<String, AppConfigurationEntry[]> configurationEntries;
-
-		public InternalConfiguration(Map<String, AppConfigurationEntry[]> configurationEntries) {
-			Assert.notNull(configurationEntries, " cannot be null");
-			Assert.notEmpty(configurationEntries, " cannot be empty");
-			this.configurationEntries = configurationEntries;
-		}
-
-		@Override
-		public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
-			return configurationEntries.get(name);
-		}
-	}
-}

spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetrics.java

@@ -1,11 +1,11 @@
 /*
- * Copyright 2017 the original author or authors.
+ * Copyright 2016-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *	  http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,25 +13,25 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.springframework.cloud.stream.binder.kafka;
 
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.binder.MeterBinder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import org.springframework.boot.actuate.endpoint.PublicMetrics;
-import org.springframework.boot.actuate.metrics.Metric;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.kafka.core.ConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
@@ -42,9 +42,9 @@ import org.springframework.util.ObjectUtils;
  *
  * @author Henryk Konsek
  */
-public class KafkaBinderMetrics implements PublicMetrics {
+public class KafkaBinderMetrics implements MeterBinder {
 
-	private final static Logger LOG = LoggerFactory.getLogger(KafkaBinderMetrics.class);
+	private final static Log LOG = LogFactory.getLog(KafkaBinderMetrics.class);
 
 	static final String METRIC_PREFIX = "spring.cloud.stream.binder.kafka";
 
@@ -68,8 +68,7 @@ public class KafkaBinderMetrics implements PublicMetrics {
 	}
 
 	@Override
-	public Collection<Metric<?>> metrics() {
-		List<Metric<?>> metrics = new LinkedList<>();
+	public void bindTo(MeterRegistry registry) {
 		for (Map.Entry<String, KafkaMessageChannelBinder.TopicInformation> topicInfo : this.binder.getTopicsInUse()
 				.entrySet()) {
 			if (!topicInfo.getValue().isConsumerTopic()) {
@@ -96,13 +95,12 @@ public class KafkaBinderMetrics implements PublicMetrics {
 						lag += endOffset.getValue();
 					}
 				}
-				metrics.add(new Metric<>(String.format("%s.%s.%s.lag", METRIC_PREFIX, group, topic), lag));
+				registry.gauge(String.format("%s.%s.%s.lag", METRIC_PREFIX, group, topic), lag);
 			}
 			catch (Exception e) {
 				LOG.debug("Cannot generate metric for topic: " + topic, e);
 			}
 		}
-		return metrics;
 	}
 
 	private ConsumerFactory<?, ?> createConsumerFactory(String group) {
@@ -123,4 +121,4 @@ public class KafkaBinderMetrics implements PublicMetrics {
 		return new DefaultKafkaConsumerFactory<>(props);
 	}
 
-}
+}

spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2017 the original author or authors.
+ * Copyright 2014-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,36 +18,45 @@ package org.springframework.cloud.stream.binder.kafka;
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
-import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Predicate;
 
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.serialization.ByteArrayDeserializer;
 import org.apache.kafka.common.serialization.ByteArraySerializer;
-import org.apache.kafka.common.utils.Utils;
 
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
 import org.springframework.cloud.stream.binder.AbstractMessageChannelBinder;
-import org.springframework.cloud.stream.binder.Binder;
 import org.springframework.cloud.stream.binder.BinderHeaders;
-import org.springframework.cloud.stream.binder.EmbeddedHeaderUtils;
+import org.springframework.cloud.stream.binder.DefaultPollableMessageSource;
 import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
 import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
 import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder;
-import org.springframework.cloud.stream.binder.MessageValues;
+import org.springframework.cloud.stream.binder.HeaderMode;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
+import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties.StandardHeaders;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaExtendedBindingProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
 import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
@@ -58,24 +67,38 @@ import org.springframework.expression.common.LiteralExpression;
 import org.springframework.expression.spel.standard.SpelExpressionParser;
 import org.springframework.integration.core.MessageProducer;
 import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter;
+import org.springframework.integration.kafka.inbound.KafkaMessageSource;
 import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler;
 import org.springframework.integration.kafka.support.RawRecordHeaderErrorMessageStrategy;
+import org.springframework.integration.support.AcknowledgmentCallback;
+import org.springframework.integration.support.AcknowledgmentCallback.Status;
 import org.springframework.integration.support.ErrorMessageStrategy;
+import org.springframework.integration.support.StaticMessageHeaderAccessor;
+import org.springframework.kafka.KafkaException;
 import org.springframework.kafka.core.ConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaProducerFactory;
 import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.core.ProducerFactory;
 import org.springframework.kafka.listener.AbstractMessageListenerContainer;
 import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
 import org.springframework.kafka.listener.config.ContainerProperties;
+import org.springframework.kafka.support.DefaultKafkaHeaderMapper;
+import org.springframework.kafka.support.KafkaHeaderMapper;
+import org.springframework.kafka.support.KafkaHeaders;
 import org.springframework.kafka.support.ProducerListener;
 import org.springframework.kafka.support.SendResult;
 import org.springframework.kafka.support.TopicPartitionInitialOffset;
+import org.springframework.kafka.support.converter.MessagingMessageConverter;
+import org.springframework.kafka.transaction.KafkaTransactionManager;
 import org.springframework.messaging.Message;
 import org.springframework.messaging.MessageChannel;
 import org.springframework.messaging.MessageHandler;
+import org.springframework.messaging.MessageHeaders;
 import org.springframework.messaging.MessagingException;
-import org.springframework.messaging.support.MessageBuilder;
+import org.springframework.messaging.support.ErrorMessage;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+import org.springframework.transaction.support.TransactionTemplate;
 import org.springframework.util.Assert;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.ObjectUtils;
@@ -84,7 +107,7 @@ import org.springframework.util.concurrent.ListenableFuture;
 import org.springframework.util.concurrent.ListenableFutureCallback;
 
 /**
- * A {@link Binder} that uses Kafka as the underlying middleware.
+ * A {@link org.springframework.cloud.stream.binder.Binder} that uses Kafka as the underlying middleware.
  *
  * @author Eric Bottard
  * @author Marius Bogoevici
@@ -95,22 +118,25 @@ import org.springframework.util.concurrent.ListenableFutureCallback;
  * @author Soby Chacko
  * @author Henryk Konsek
  * @author Doug Saus
- * @author Aldo Sinanaj
  */
 public class KafkaMessageChannelBinder extends
-		AbstractMessageChannelBinder<ExtendedConsumerProperties<KafkaConsumerProperties>,
-			ExtendedProducerProperties<KafkaProducerProperties>, KafkaTopicProvisioner>
+		AbstractMessageChannelBinder<ExtendedConsumerProperties<KafkaConsumerProperties>, ExtendedProducerProperties<KafkaProducerProperties>, KafkaTopicProvisioner>
 		implements ExtendedPropertiesBinder<MessageChannel, KafkaConsumerProperties, KafkaProducerProperties> {
 
-	public static final String X_ORIGINAL_TOPIC = "x-original-topic";
-
-	public static final String X_EXCEPTION_MESSAGE = "x-exception-message";
-
 	public static final String X_EXCEPTION_STACKTRACE = "x-exception-stacktrace";
 
+	public static final String X_EXCEPTION_MESSAGE = "x-exception-message";
+
+	public static final String X_ORIGINAL_TOPIC = "x-original-topic";
+
+
 	private final KafkaBinderConfigurationProperties configurationProperties;
 
-	private final Map<String, TopicInformation> topicsInUse = new HashMap<>();
+	private final Map<String, TopicInformation> topicsInUse = new ConcurrentHashMap<>();
+
+	private final KafkaTransactionManager<byte[], byte[]> transactionManager;
+
+	private final TransactionTemplate transactionTemplate;
 
 	private ProducerListener<byte[], byte[]> producerListener;
 
@@ -118,8 +144,18 @@ public class KafkaMessageChannelBinder extends
 
 	public KafkaMessageChannelBinder(KafkaBinderConfigurationProperties configurationProperties,
 			KafkaTopicProvisioner provisioningProvider) {
-		super(false, headersToMap(configurationProperties), provisioningProvider);
+		super(headersToMap(configurationProperties), provisioningProvider);
 		this.configurationProperties = configurationProperties;
+		if (StringUtils.hasText(configurationProperties.getTransaction().getTransactionIdPrefix())) {
+			this.transactionManager = new KafkaTransactionManager<>(
+					getProducerFactory(configurationProperties.getTransaction().getTransactionIdPrefix(),
+							new ExtendedProducerProperties<>(configurationProperties.getTransaction().getProducer())));
+			this.transactionTemplate = new TransactionTemplate(this.transactionManager);
+		}
+		else {
+			this.transactionManager = null;
+			this.transactionTemplate = null;
+		}
 	}
 
 	private static String[] headersToMap(KafkaBinderConfigurationProperties configurationProperties) {
@@ -163,22 +199,24 @@ public class KafkaMessageChannelBinder extends
 	@Override
 	protected MessageHandler createProducerMessageHandler(final ProducerDestination destination,
 			ExtendedProducerProperties<KafkaProducerProperties> producerProperties, MessageChannel errorChannel)
-					throws Exception {
-		final DefaultKafkaProducerFactory<byte[], byte[]> producerFB = getProducerFactory(producerProperties);
+			throws Exception {
+		/*
+		 * IMPORTANT: With a transactional binder, individual producer properties for Kafka are
+		 * ignored; the global binder (spring.cloud.stream.kafka.binder.transaction.producer.*)
+		 * properties are used instead, for all producers. A binder is transactional when
+		 * 'spring.cloud.stream.kafka.binder.transaction.transaction-id-prefix' has text.
+		 */
+		final ProducerFactory<byte[], byte[]> producerFB = this.transactionManager != null
+				? this.transactionManager.getProducerFactory()
+				: getProducerFactory(null, producerProperties);
 		Collection<PartitionInfo> partitions = provisioningProvider.getPartitionsForTopic(
-				producerProperties.getPartitionCount(),
-				false,
-				new Callable<Collection<PartitionInfo>>() {
-
-					@Override
-					public Collection<PartitionInfo> call() throws Exception {
-						Producer<byte[], byte[]> producer = producerFB.createProducer();
-						List<PartitionInfo> partitionsFor = producer.partitionsFor(destination.getName());
-						producer.close();
-						producerFB.destroy();
-						return partitionsFor;
-					}
-
+				producerProperties.getPartitionCount(), false,
+				() -> {
+					Producer<byte[], byte[]> producer = producerFB.createProducer();
+					List<PartitionInfo> partitionsFor = producer.partitionsFor(destination.getName());
+					producer.close();
+					((DisposableBean) producerFB).destroy();
+					return partitionsFor;
 				});
 		this.topicsInUse.put(destination.getName(), new TopicInformation(null, partitions));
 		if (producerProperties.getPartitionCount() < partitions.size()) {
@@ -188,12 +226,11 @@ public class KafkaMessageChannelBinder extends
 						+ partitions.size() + " of the topic. The larger number will be used instead.");
 			}
 			/*
-			 * This is dirty; it relies on the fact that we, and the partition
-			 * interceptor, share a hard reference to the producer properties instance.
-			 * But I don't see another way to fix it since the interceptor has already
-			 * been added to the channel, and we don't have access to the channel here; if
-			 * we did, we could inject the proper partition count there.
-			 * TODO: Consider this when doing the 2.0 binder restructuring.
+			 * This is dirty; it relies on the fact that we, and the partition interceptor, share a
+			 * hard reference to the producer properties instance. But I don't see another way to fix
+			 * it since the interceptor has already been added to the channel, and we don't have
+			 * access to the channel here; if we did, we could inject the proper partition count
+			 * there. TODO: Consider this when doing the 2.0 binder restructuring.
 			 */
 			producerProperties.setPartitionCount(partitions.size());
 		}
@@ -207,10 +244,41 @@ public class KafkaMessageChannelBinder extends
 		if (errorChannel != null) {
 			handler.setSendFailureChannel(errorChannel);
 		}
+		KafkaHeaderMapper mapper = null;
+		if (this.configurationProperties.getHeaderMapperBeanName() != null) {
+			mapper = getApplicationContext().getBean(this.configurationProperties.getHeaderMapperBeanName(),
+					KafkaHeaderMapper.class);
+		}
+		/*
+		 *  Even if the user configures a bean, we must not use it if the header
+		 *  mode is not the default (headers); setting the mapper to null
+		 *  disables populating headers in the message handler.
+		 */
+		if (producerProperties.getHeaderMode() != null
+				&& !HeaderMode.headers.equals(producerProperties.getHeaderMode())) {
+			mapper = null;
+		}
+		else if (mapper == null) {
+			String[] headerPatterns = producerProperties.getExtension().getHeaderPatterns();
+			if (headerPatterns != null && headerPatterns.length > 0) {
+				List<String> patterns = new LinkedList<>(Arrays.asList(headerPatterns));
+				if (!patterns.contains("!" + MessageHeaders.TIMESTAMP)) {
+					patterns.add(0, "!" + MessageHeaders.TIMESTAMP);
+				}
+				if (!patterns.contains("!" + MessageHeaders.ID)) {
+					patterns.add(0, "!" + MessageHeaders.ID);
+				}
+				mapper = new DefaultKafkaHeaderMapper(patterns.toArray(new String[patterns.size()]));
+			}
+			else {
+				mapper = new DefaultKafkaHeaderMapper();
+			}
+		}
+		handler.setHeaderMapper(mapper);
 		return handler;
 	}
 
-	private DefaultKafkaProducerFactory<byte[], byte[]> getProducerFactory(
+	protected DefaultKafkaProducerFactory<byte[], byte[]> getProducerFactory(String transactionIdPrefix,
 			ExtendedProducerProperties<KafkaProducerProperties> producerProperties) {
 		Map<String, Object> props = new HashMap<>();
 		props.put(ProducerConfig.RETRIES_CONFIG, 0);
@@ -239,7 +307,11 @@ public class KafkaMessageChannelBinder extends
 		if (!ObjectUtils.isEmpty(producerProperties.getExtension().getConfiguration())) {
 			props.putAll(producerProperties.getExtension().getConfiguration());
 		}
-		return new DefaultKafkaProducerFactory<>(props);
+		DefaultKafkaProducerFactory<byte[], byte[]> producerFactory = new DefaultKafkaProducerFactory<>(props);
+		if (transactionIdPrefix != null) {
+			producerFactory.setTransactionIdPrefix(transactionIdPrefix);
+		}
+		return producerFactory;
 	}
 
 	@Override
@@ -256,19 +328,8 @@ public class KafkaMessageChannelBinder extends
 		int partitionCount = extendedConsumerProperties.getInstanceCount()
 				* extendedConsumerProperties.getConcurrency();
 
-		Collection<PartitionInfo> allPartitions = provisioningProvider.getPartitionsForTopic(partitionCount,
-				extendedConsumerProperties.getExtension().isAutoRebalanceEnabled(),
-				new Callable<Collection<PartitionInfo>>() {
-
-					@Override
-					public Collection<PartitionInfo> call() throws Exception {
-						Consumer<?, ?> consumer = consumerFactory.createConsumer();
-						List<PartitionInfo> partitionsFor = consumer.partitionsFor(destination.getName());
-						consumer.close();
-						return partitionsFor;
-					}
-
-				});
+		Collection<PartitionInfo> allPartitions = getPartitionInfo(destination, extendedConsumerProperties,
+				consumerFactory, partitionCount);
 
 		Collection<PartitionInfo> listenedPartitions;
 
@@ -296,11 +357,14 @@ public class KafkaMessageChannelBinder extends
 				|| extendedConsumerProperties.getExtension().isAutoRebalanceEnabled()
 						? new ContainerProperties(destination.getName())
 						: new ContainerProperties(topicPartitionInitialOffsets);
+		if (this.transactionManager != null) {
+			containerProperties.setTransactionManager(this.transactionManager);
+		}
+		containerProperties.setIdleEventInterval(extendedConsumerProperties.getExtension().getIdleEventInterval());
 		int concurrency = Math.min(extendedConsumerProperties.getConcurrency(), listenedPartitions.size());
 		@SuppressWarnings("rawtypes")
 		final ConcurrentMessageListenerContainer<?, ?> messageListenerContainer =
-				new ConcurrentMessageListenerContainer(
-						consumerFactory, containerProperties) {
+				new ConcurrentMessageListenerContainer(consumerFactory, containerProperties) {
 
 			@Override
 			public void stop(Runnable callback) {
@@ -309,6 +373,9 @@ public class KafkaMessageChannelBinder extends
 
 		};
 		messageListenerContainer.setConcurrency(concurrency);
+		// these won't be needed if the container is made a bean
+		messageListenerContainer.setApplicationEventPublisher(getApplicationContext());
+		messageListenerContainer.setBeanName(destination.getName() + ".container");
 		if (!extendedConsumerProperties.getExtension().isAutoCommitOffset()) {
 			messageListenerContainer.getContainerProperties()
 					.setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL);
@@ -322,8 +389,9 @@ public class KafkaMessageChannelBinder extends
 			this.logger.debug(
 					"Listened partitions: " + StringUtils.collectionToCommaDelimitedString(listenedPartitions));
 		}
-		final KafkaMessageDrivenChannelAdapter<?, ?> kafkaMessageDrivenChannelAdapter = new KafkaMessageDrivenChannelAdapter<>(
-				messageListenerContainer);
+		final KafkaMessageDrivenChannelAdapter<?, ?> kafkaMessageDrivenChannelAdapter =
+				new KafkaMessageDrivenChannelAdapter<>(messageListenerContainer);
+		kafkaMessageDrivenChannelAdapter.setMessageConverter(getMessageConverter(extendedConsumerProperties));
 		kafkaMessageDrivenChannelAdapter.setBeanFactory(this.getBeanFactory());
 		ErrorInfrastructure errorInfrastructure = registerErrorInfrastructure(destination, consumerGroup,
 				extendedConsumerProperties);
@@ -337,6 +405,119 @@ public class KafkaMessageChannelBinder extends
 		return kafkaMessageDrivenChannelAdapter;
 	}
 
+	@Override
+	protected PolledConsumerResources createPolledConsumerResources(String name, String group,
+			ConsumerDestination destination, ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties) {
+		boolean anonymous = !StringUtils.hasText(group);
+		Assert.isTrue(!anonymous || !consumerProperties.getExtension().isEnableDlq(),
+				"DLQ support is not available for anonymous subscriptions");
+		String consumerGroup = anonymous ? "anonymous." + UUID.randomUUID().toString() : group;
+		final ConsumerFactory<?, ?> consumerFactory = createKafkaConsumerFactory(anonymous, consumerGroup,
+				consumerProperties);
+		KafkaMessageSource<?, ?> source = new KafkaMessageSource<>(consumerFactory, destination.getName());
+		source.setMessageConverter(getMessageConverter(consumerProperties));
+		source.setRawMessageHeader(consumerProperties.getExtension().isEnableDlq());
+
+		// I copied this from the regular consumer - it looks bogus to me - includes all partitions
+		// not just the ones this binding is listening to; doesn't seem right for a health check.
+		Collection<PartitionInfo> partitionInfos = getPartitionInfo(destination, consumerProperties, consumerFactory,
+				-1);
+		this.topicsInUse.put(destination.getName(), new TopicInformation(group, partitionInfos));
+
+		source.setRebalanceListener(new ConsumerRebalanceListener() {
+
+			@Override
+			public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
+				KafkaMessageChannelBinder.this.logger.info("Revoked: " + partitions);
+			}
+
+			@Override
+			public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
+				KafkaMessageChannelBinder.this.logger.info("Assigned: " + partitions);
+			}
+
+		});
+		return new PolledConsumerResources(source,
+				registerErrorInfrastructure(destination, group, consumerProperties, true));
+	}
+
+	@Override
+	protected void postProcessPollableSource(DefaultPollableMessageSource bindingTarget) {
+		bindingTarget.setAttributesProvider((accessor, message) -> {
+			Object rawMessage = message.getHeaders().get(KafkaHeaders.RAW_DATA);
+			if (rawMessage != null) {
+				accessor.setAttribute(KafkaHeaders.RAW_DATA, rawMessage);
+			}
+		});
+	}
+
+	private MessagingMessageConverter getMessageConverter(
+			final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties) {
+		MessagingMessageConverter messageConverter;
+		if (extendedConsumerProperties.getExtension().getConverterBeanName() == null) {
+			messageConverter = new MessagingMessageConverter();
+			StandardHeaders standardHeaders = extendedConsumerProperties.getExtension().getStandardHeaders();
+			messageConverter.setGenerateMessageId(StandardHeaders.id.equals(standardHeaders)
+					|| StandardHeaders.both.equals(standardHeaders));
+			messageConverter.setGenerateTimestamp(StandardHeaders.timestamp.equals(standardHeaders)
+					|| StandardHeaders.both.equals(standardHeaders));
+		}
+		else {
+			try {
+				messageConverter = getApplicationContext().getBean(
+					extendedConsumerProperties.getExtension().getConverterBeanName(),
+						MessagingMessageConverter.class);
+			}
+			catch (NoSuchBeanDefinitionException e) {
+				throw new IllegalStateException("Converter bean not present in application context", e);
+			}
+		}
+		messageConverter.setHeaderMapper(getHeaderMapper(extendedConsumerProperties));
+		return messageConverter;
+	}
+
+	private KafkaHeaderMapper getHeaderMapper(
+			final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties) {
+		KafkaHeaderMapper mapper = null;
+		if (this.configurationProperties.getHeaderMapperBeanName() != null) {
+			mapper = getApplicationContext().getBean(this.configurationProperties.getHeaderMapperBeanName(),
+					KafkaHeaderMapper.class);
+		}
+		if (mapper == null) {
+			DefaultKafkaHeaderMapper headerMapper = new DefaultKafkaHeaderMapper() {
+
+				@Override
+				public void toHeaders(Headers source, Map<String, Object> headers) {
+					super.toHeaders(source, headers);
+					if (headers.size() > 0) {
+						headers.put(BinderHeaders.NATIVE_HEADERS_PRESENT, Boolean.TRUE);
+					}
+				}
+
+			};
+			String[] trustedPackages = extendedConsumerProperties.getExtension().getTrustedPackages();
+			if (!StringUtils.isEmpty(trustedPackages)) {
+				headerMapper.addTrustedPackages(trustedPackages);
+			}
+			mapper = headerMapper;
+		}
+		return mapper;
+	}
+
+	private Collection<PartitionInfo> getPartitionInfo(final ConsumerDestination destination,
+			final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties,
+			final ConsumerFactory<?, ?> consumerFactory, int partitionCount) {
+		Collection<PartitionInfo> allPartitions = provisioningProvider.getPartitionsForTopic(partitionCount,
+				extendedConsumerProperties.getExtension().isAutoRebalanceEnabled(),
+				() -> {
+					Consumer<?, ?> consumer = consumerFactory.createConsumer();
+					List<PartitionInfo> partitionsFor = consumer.partitionsFor(destination.getName());
+					consumer.close();
+					return partitionsFor;
+				});
+		return allPartitions;
+	}
+
 	@Override
 	protected ErrorMessageStrategy getErrorMessageStrategy() {
 		return new RawRecordHeaderErrorMessageStrategy();
@@ -344,80 +525,106 @@ public class KafkaMessageChannelBinder extends
 
 	@Override
 	protected MessageHandler getErrorMessageHandler(final ConsumerDestination destination, final String group,
-			final ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties) {
-		if (extendedConsumerProperties.getExtension().isEnableDlq()) {
-			DefaultKafkaProducerFactory<byte[], byte[]> producerFactory = getProducerFactory(
-					new ExtendedProducerProperties<>(new KafkaProducerProperties()));
-			final KafkaTemplate<byte[], byte[]> kafkaTemplate = new KafkaTemplate<>(producerFactory);
-			return new MessageHandler() {
+			final ExtendedConsumerProperties<KafkaConsumerProperties> properties) {
+		KafkaConsumerProperties kafkaConsumerProperties = properties.getExtension();
+		if (kafkaConsumerProperties.isEnableDlq()) {
+			KafkaProducerProperties dlqProducerProperties = kafkaConsumerProperties.getDlqProducerProperties();
+			ProducerFactory<?,?> producerFactory = this.transactionManager != null
+					? this.transactionManager.getProducerFactory()
+					: getProducerFactory(null,
+						new ExtendedProducerProperties<>(dlqProducerProperties));
+			final KafkaTemplate<?,?> kafkaTemplate = new KafkaTemplate<>(producerFactory);
+			String dlqName = StringUtils.hasText(kafkaConsumerProperties.getDlqName())
+					? kafkaConsumerProperties.getDlqName()
+					: "error." + destination.getName() + "." + group;
 
-				@Override
-				public void handleMessage(Message<?> message) throws MessagingException {
-					final ConsumerRecord<?, ?> record = message.getHeaders()
-							.get(KafkaMessageDrivenChannelAdapter.KAFKA_RAW_DATA, ConsumerRecord.class);
-					final byte[] key = record.key() != null
-							? Utils.toArray(ByteBuffer.wrap((byte[]) record.key()))
-							: null;
-					final byte[] payload;
-					if (message.getPayload() instanceof Throwable) {
-						final Throwable throwable = (Throwable) message.getPayload();
-						final String failureMessage = throwable.getMessage();
+			@SuppressWarnings({"unchecked", "rawtypes"})
+			DlqSender<?,?> dlqSender = new DlqSender(kafkaTemplate, dlqName);
 
-						try {
-							MessageValues messageValues = EmbeddedHeaderUtils
-									.extractHeaders(MessageBuilder.withPayload((byte[]) record.value()).build(),
-											false);
-							messageValues.put(X_ORIGINAL_TOPIC, record.topic());
-							messageValues.put(X_EXCEPTION_MESSAGE, failureMessage);
-							messageValues.put(X_EXCEPTION_STACKTRACE, getStackTraceAsString(throwable));
+			return message -> {
+				final ConsumerRecord<?, ?> record = message.getHeaders()
+						.get(KafkaHeaders.RAW_DATA, ConsumerRecord.class);
 
-							final String[] headersToEmbed = new ArrayList<>(messageValues.keySet()).toArray(
-									new String[messageValues.keySet().size()]);
-							payload = EmbeddedHeaderUtils.embedHeaders(messageValues,
-									EmbeddedHeaderUtils.headersToEmbed(headersToEmbed));
+				if (properties.isUseNativeDecoding()) {
+					if (record != null) {
+						Map<String, String> configuration = this.transactionManager == null ? dlqProducerProperties.getConfiguration()
+								: this.configurationProperties.getTransaction().getProducer().getConfiguration();
+						if (record.key() != null && !record.key().getClass().isInstance(byte[].class)) {
+							ensureDlqMessageCanBeProperlySerialized(
+									configuration,
+									(Map<String, String> config) -> !config.containsKey("key.serializer"), "Key");
 						}
-						catch (Exception e) {
-							throw new RuntimeException(e);
+						if (record.value() != null && !record.value().getClass().isInstance(byte[].class)) {
+							ensureDlqMessageCanBeProperlySerialized(configuration,
+									(Map<String, String> config) -> !config.containsKey("value.serializer"), "Payload");
 						}
 					}
-					else {
-						payload = record.value() != null
-								? Utils.toArray(ByteBuffer.wrap((byte[]) record.value())) : null;
-					}
-					String dlqName = StringUtils.hasText(extendedConsumerProperties.getExtension().getDlqName())
-							? extendedConsumerProperties.getExtension().getDlqName()
-							: "error." + destination.getName() + "." + group;
-					ListenableFuture<SendResult<byte[], byte[]>> sentDlq = kafkaTemplate.send(dlqName,
-							record.partition(), key, payload);
-					sentDlq.addCallback(new ListenableFutureCallback<SendResult<byte[], byte[]>>() {
-						StringBuilder sb = new StringBuilder().append(" a message with key='")
-								.append(toDisplayString(ObjectUtils.nullSafeToString(key), 50)).append("'")
-								.append(" and payload='")
-								.append(toDisplayString(ObjectUtils.nullSafeToString(payload), 50))
-								.append("'").append(" received from ")
-								.append(record.partition());
-
-						@Override
-						public void onFailure(Throwable ex) {
-							KafkaMessageChannelBinder.this.logger.error(
-									"Error sending to DLQ " + sb.toString(), ex);
-						}
-
-						@Override
-						public void onSuccess(SendResult<byte[], byte[]> result) {
-							if (KafkaMessageChannelBinder.this.logger.isDebugEnabled()) {
-								KafkaMessageChannelBinder.this.logger.debug(
-										"Sent to DLQ " + sb.toString());
-							}
-						}
-
-					});
 				}
+
+				if (record == null) {
+					this.logger.error("No raw record; cannot send to DLQ: " + message);
+					return;
+				}
+				Headers kafkaHeaders = new RecordHeaders(record.headers().toArray());
+				kafkaHeaders.add(new RecordHeader(X_ORIGINAL_TOPIC,
+						record.topic().getBytes(StandardCharsets.UTF_8)));
+				if (message.getPayload() instanceof Throwable) {
+					Throwable throwable = (Throwable) message.getPayload();
+					kafkaHeaders.add(new RecordHeader(X_EXCEPTION_MESSAGE,
+							throwable.getMessage().getBytes(StandardCharsets.UTF_8)));
+					kafkaHeaders.add(new RecordHeader(X_EXCEPTION_STACKTRACE,
+							getStackTraceAsString(throwable).getBytes(StandardCharsets.UTF_8)));
+				}
+				dlqSender.sendToDlq(record, kafkaHeaders);
 			};
 		}
 		return null;
 	}
 
+	@Override
+	protected MessageHandler getPolledConsumerErrorMessageHandler(ConsumerDestination destination, String group,
+			ExtendedConsumerProperties<KafkaConsumerProperties> properties) {
+		if (properties.getExtension().isEnableDlq()) {
+			return getErrorMessageHandler(destination, group, properties);
+		}
+		final MessageHandler superHandler = super.getErrorMessageHandler(destination, group, properties);
+		return message -> {
+			ConsumerRecord<?, ?> record = (ConsumerRecord<?, ?>) message.getHeaders().get(KafkaHeaders.RAW_DATA);
+			if (!(message instanceof ErrorMessage)) {
+				logger.error("Expected an ErrorMessage, not a " + message.getClass().toString() + " for: "
+						+ message);
+			}
+			else if (record == null) {
+				if (superHandler != null) {
+					superHandler.handleMessage(message);
+				}
+			}
+			else {
+				if (message.getPayload() instanceof MessagingException) {
+					AcknowledgmentCallback ack = StaticMessageHeaderAccessor.getAcknowledgmentCallback(
+							((MessagingException) message.getPayload()).getFailedMessage());
+					if (ack != null) {
+						if (isAutoCommitOnError(properties)) {
+							ack.acknowledge(Status.REJECT);
+						}
+						else {
+							ack.acknowledge(Status.REQUEUE);
+						}
+					}
+				}
+			}
+		};
+	}
+
+	private static void ensureDlqMessageCanBeProperlySerialized(Map<String, String> configuration,
+																Predicate<Map<String, String>> configPredicate,
+																String dataType) {
+			if (CollectionUtils.isEmpty(configuration) || configPredicate.test(configuration)) {
+				throw new IllegalArgumentException("Native decoding is used on the consumer. " +
+						dataType + " is not byte[] and no serializer is set on the DLQ producer.");
+			}
+	}
+
 	private ConsumerFactory<?, ?> createKafkaConsumerFactory(boolean anonymous, String consumerGroup,
 			ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties) {
 		Map<String, Object> props = new HashMap<>();
@@ -483,18 +690,18 @@ public class KafkaMessageChannelBinder extends
 
 		private boolean running = true;
 
-		private final DefaultKafkaProducerFactory<byte[], byte[]> producerFactory;
+		private final ProducerFactory<byte[], byte[]> producerFactory;
 
 		ProducerConfigurationMessageHandler(KafkaTemplate<byte[], byte[]> kafkaTemplate, String topic,
 				ExtendedProducerProperties<KafkaProducerProperties> producerProperties,
-				DefaultKafkaProducerFactory<byte[], byte[]> producerFactory) {
+				ProducerFactory<byte[], byte[]> producerFactory) {
 			super(kafkaTemplate);
 			setTopicExpression(new LiteralExpression(topic));
 			setMessageKeyExpression(producerProperties.getExtension().getMessageKeyExpression());
 			setBeanFactory(KafkaMessageChannelBinder.this.getBeanFactory());
 			if (producerProperties.isPartitioned()) {
 				SpelExpressionParser parser = new SpelExpressionParser();
-				setPartitionIdExpression(parser.parseExpression("headers." + BinderHeaders.PARTITION_HEADER));
+				setPartitionIdExpression(parser.parseExpression("headers['" + BinderHeaders.PARTITION_HEADER + "']"));
 			}
 			if (producerProperties.getExtension().isSync()) {
 				setSync(true);
@@ -515,7 +722,9 @@ public class KafkaMessageChannelBinder extends
 
 		@Override
 		public void stop() {
-			producerFactory.stop();
+			if (this.producerFactory instanceof Lifecycle) {
+				((Lifecycle) producerFactory).stop();
+			}
 			this.running = false;
 		}
 
@@ -523,31 +732,103 @@ public class KafkaMessageChannelBinder extends
 		public boolean isRunning() {
 			return this.running;
 		}
+
+		@Override
+		protected void handleMessageInternal(Message<?> message) throws Exception {
+			if (KafkaMessageChannelBinder.this.transactionTemplate != null
+					&& !TransactionSynchronizationManager.isActualTransactionActive()) {
+				KafkaMessageChannelBinder.this.transactionTemplate.execute(s -> {
+					try {
+						super.handleMessageInternal(message);
+					}
+					catch (Exception e) {
+						throw new KafkaException("Exception on transactional send", e);
+					}
+					return null;
+				});
+			}
+			else {
+				super.handleMessageInternal(message);
+			}
+		}
+
 	}
 
-	public static class TopicInformation {
+	static class TopicInformation {
 
 		private final String consumerGroup;
 
 		private final Collection<PartitionInfo> partitionInfos;
 
-		public TopicInformation(String consumerGroup, Collection<PartitionInfo> partitionInfos) {
+		TopicInformation(String consumerGroup, Collection<PartitionInfo> partitionInfos) {
 			this.consumerGroup = consumerGroup;
 			this.partitionInfos = partitionInfos;
 		}
 
-		public String getConsumerGroup() {
+		String getConsumerGroup() {
 			return consumerGroup;
 		}
 
-		public boolean isConsumerTopic() {
+		boolean isConsumerTopic() {
 			return consumerGroup != null;
 		}
 
-		public Collection<PartitionInfo> getPartitionInfos() {
+		Collection<PartitionInfo> getPartitionInfos() {
 			return partitionInfos;
 		}
 
 	}
 
+	private final class DlqSender<K,V> {
+
+		private final KafkaTemplate<K,V> kafkaTemplate;
+		private final String dlqName;
+
+		DlqSender(KafkaTemplate<K, V> kafkaTemplate, String dlqName) {
+			this.kafkaTemplate = kafkaTemplate;
+			this.dlqName = dlqName;
+		}
+
+		@SuppressWarnings("unchecked")
+		void sendToDlq(ConsumerRecord<?, ?> consumerRecord, Headers headers) {
+			K key = (K)consumerRecord.key();
+			V value = (V)consumerRecord.value();
+			ProducerRecord<K,V> producerRecord = new ProducerRecord<>(this.dlqName, consumerRecord.partition(),
+					key, value, headers);
+
+			StringBuilder sb = new StringBuilder().append(" a message with key='")
+					.append(toDisplayString(ObjectUtils.nullSafeToString(key), 50)).append("'")
+					.append(" and payload='")
+					.append(toDisplayString(ObjectUtils.nullSafeToString(value), 50))
+					.append("'").append(" received from ")
+					.append(consumerRecord.partition());
+			ListenableFuture<SendResult<K, V>> sentDlq = null;
+			try {
+				sentDlq = this.kafkaTemplate.send(producerRecord);
+				sentDlq.addCallback(new ListenableFutureCallback<SendResult<K, V>>() {
+
+					@Override
+					public void onFailure(Throwable ex) {
+						KafkaMessageChannelBinder.this.logger.error(
+								"Error sending to DLQ " + sb.toString(), ex);
+					}
+
+					@Override
+					public void onSuccess(SendResult<K, V> result) {
+						if (KafkaMessageChannelBinder.this.logger.isDebugEnabled()) {
+							KafkaMessageChannelBinder.this.logger.debug(
+									"Sent to DLQ " + sb.toString());
+						}
+					}
+				});
+			}
+			catch (Exception ex) {
+				if (sentDlq == null) {
+					KafkaMessageChannelBinder.this.logger.error(
+							"Error sending to DLQ " + sb.toString(), ex);
+				}
+			}
+
+		}
+	}
 }

spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java

@@ -20,43 +20,32 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import io.micrometer.core.instrument.binder.MeterBinder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.apache.kafka.common.utils.AppInfoParser;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.actuate.endpoint.PublicMetrics;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration;
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.stream.binder.Binder;
 import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealthIndicator;
-import org.springframework.cloud.stream.binder.kafka.KafkaBinderJaasInitializerListener;
 import org.springframework.cloud.stream.binder.kafka.KafkaBinderMetrics;
 import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder;
-import org.springframework.cloud.stream.binder.kafka.admin.AdminUtilsOperation;
-import org.springframework.cloud.stream.binder.kafka.admin.Kafka09AdminUtilsOperation;
-import org.springframework.cloud.stream.binder.kafka.admin.Kafka10AdminUtilsOperation;
 import org.springframework.cloud.stream.binder.kafka.properties.JaasLoginModuleConfiguration;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaExtendedBindingProperties;
 import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
-import org.springframework.cloud.stream.config.codec.kryo.KryoCodecAutoConfiguration;
 import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationListener;
 import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Condition;
-import org.springframework.context.annotation.ConditionContext;
-import org.springframework.context.annotation.Conditional;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
-import org.springframework.core.type.AnnotatedTypeMetadata;
-import org.springframework.integration.codec.Codec;
 import org.springframework.kafka.core.ConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
+import org.springframework.kafka.security.jaas.KafkaJaasLoginModuleInitializer;
 import org.springframework.kafka.support.LoggingProducerListener;
 import org.springframework.kafka.support.ProducerListener;
 import org.springframework.util.ObjectUtils;
@@ -72,15 +61,12 @@ import org.springframework.util.ObjectUtils;
  */
 @Configuration
 @ConditionalOnMissingBean(Binder.class)
-@Import({ KryoCodecAutoConfiguration.class, PropertyPlaceholderAutoConfiguration.class})
+@Import({ PropertyPlaceholderAutoConfiguration.class})
 @EnableConfigurationProperties({ KafkaBinderConfigurationProperties.class, KafkaExtendedBindingProperties.class })
 public class KafkaBinderConfiguration {
 
 	protected static final Log logger = LogFactory.getLog(KafkaBinderConfiguration.class);
 
-	@Autowired
-	private Codec codec;
-
 	@Autowired
 	private KafkaBinderConfigurationProperties configurationProperties;
 
@@ -93,19 +79,18 @@ public class KafkaBinderConfiguration {
 	@Autowired
 	private ApplicationContext context;
 
-	@Autowired(required = false)
-	private AdminUtilsOperation adminUtilsOperation;
+	@Autowired
+	private KafkaProperties kafkaProperties;
 
 	@Bean
 	KafkaTopicProvisioner provisioningProvider() {
-		return new KafkaTopicProvisioner(this.configurationProperties, this.adminUtilsOperation);
+		return new KafkaTopicProvisioner(this.configurationProperties, this.kafkaProperties);
 	}
 
 	@Bean
 	KafkaMessageChannelBinder kafkaMessageChannelBinder() {
 		KafkaMessageChannelBinder kafkaMessageChannelBinder = new KafkaMessageChannelBinder(
 				this.configurationProperties, provisioningProvider());
-		kafkaMessageChannelBinder.setCodec(this.codec);
 		kafkaMessageChannelBinder.setProducerListener(producerListener);
 		kafkaMessageChannelBinder.setExtendedBindingProperties(this.kafkaExtendedBindingProperties);
 		return kafkaMessageChannelBinder;
@@ -136,45 +121,13 @@ public class KafkaBinderConfiguration {
 	}
 
 	@Bean
-	public PublicMetrics kafkaBinderMetrics(KafkaMessageChannelBinder kafkaMessageChannelBinder) {
+	public MeterBinder kafkaBinderMetrics(KafkaMessageChannelBinder kafkaMessageChannelBinder) {
 		return new KafkaBinderMetrics(kafkaMessageChannelBinder, configurationProperties);
 	}
 
-	@Bean(name = "adminUtilsOperation")
-	@Conditional(Kafka09Present.class)
-	@ConditionalOnClass(name = "kafka.admin.AdminUtils")
-	public AdminUtilsOperation kafka09AdminUtilsOperation() {
-		logger.info("AdminUtils selected: Kafka 0.9 AdminUtils");
-		return new Kafka09AdminUtilsOperation();
-	}
-
-	@Bean(name = "adminUtilsOperation")
-	@Conditional(Kafka10Present.class)
-	@ConditionalOnClass(name = "kafka.admin.AdminUtils")
-	public AdminUtilsOperation kafka10AdminUtilsOperation() {
-		logger.info("AdminUtils selected: Kafka 0.10 AdminUtils");
-		return new Kafka10AdminUtilsOperation();
-	}
-
 	@Bean
-	public ApplicationListener<?> jaasInitializer() throws IOException {
-		return new KafkaBinderJaasInitializerListener();
-	}
-
-	static class Kafka10Present implements Condition {
-
-		@Override
-		public boolean matches(ConditionContext conditionContext, AnnotatedTypeMetadata annotatedTypeMetadata) {
-			return AppInfoParser.getVersion().startsWith("0.10");
-		}
-	}
-
-	static class Kafka09Present implements Condition {
-
-		@Override
-		public boolean matches(ConditionContext conditionContext, AnnotatedTypeMetadata annotatedTypeMetadata) {
-			return AppInfoParser.getVersion().startsWith("0.9");
-		}
+	public KafkaJaasLoginModuleInitializer jaasInitializer() throws IOException {
+		return new KafkaJaasLoginModuleInitializer();
 	}
 
 	public static class JaasConfigurationProperties {

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AbstractKafkaTestBinder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2017 the original author or authors.
+ * Copyright 2014-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,24 +13,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.springframework.cloud.stream.binder.kafka;
 
-import org.springframework.cloud.stream.binder.AbstractTestBinder;
+import org.springframework.cloud.stream.binder.AbstractPollableConsumerTestBinder;
 import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
 import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
 import org.springframework.context.ApplicationContext;
-import org.springframework.integration.codec.Codec;
-import org.springframework.integration.codec.kryo.PojoCodec;
-import org.springframework.integration.tuple.TupleKryoRegistrar;
 
 /**
  * @author Soby Chacko
  * @author Gary Russell
  */
 public abstract class AbstractKafkaTestBinder extends
-		AbstractTestBinder<KafkaMessageChannelBinder, ExtendedConsumerProperties<KafkaConsumerProperties>, ExtendedProducerProperties<KafkaProducerProperties>> {
+		AbstractPollableConsumerTestBinder<KafkaMessageChannelBinder,
+			ExtendedConsumerProperties<KafkaConsumerProperties>, ExtendedProducerProperties<KafkaProducerProperties>> {
 
 	private ApplicationContext applicationContext;
 
@@ -47,9 +46,4 @@ public abstract class AbstractKafkaTestBinder extends
 		return this.applicationContext;
 	}
 
-	protected static Codec getCodec() {
-		return new PojoCodec(new TupleKryoRegistrar());
-	}
-
 }
-

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderAutoConfigurationPropertiesTest.java

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.springframework.cloud.stream.binder.kafka;
 
 import java.lang.reflect.Field;
@@ -66,10 +67,10 @@ public class KafkaBinderAutoConfigurationPropertiesTest {
 		ExtendedProducerProperties<KafkaProducerProperties> producerProperties = new ExtendedProducerProperties<>(
 				new KafkaProducerProperties());
 		Method getProducerFactoryMethod = KafkaMessageChannelBinder.class.getDeclaredMethod("getProducerFactory",
-				ExtendedProducerProperties.class);
+				String.class, ExtendedProducerProperties.class);
 		getProducerFactoryMethod.setAccessible(true);
 		DefaultKafkaProducerFactory producerFactory = (DefaultKafkaProducerFactory) getProducerFactoryMethod
-				.invoke(this.kafkaMessageChannelBinder, producerProperties);
+				.invoke(this.kafkaMessageChannelBinder, "foo", producerProperties);
 		Field producerFactoryConfigField = ReflectionUtils.findField(DefaultKafkaProducerFactory.class, "configs",
 				Map.class);
 		ReflectionUtils.makeAccessible(producerFactoryConfigField);

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationPropertiesTest.java

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.springframework.cloud.stream.binder.kafka;
 
 import java.lang.reflect.Field;
@@ -27,6 +28,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
 import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
@@ -46,7 +48,8 @@ import static org.junit.Assert.assertTrue;
  * @author Ilayaperumal Gopinathan
  */
 @RunWith(SpringJUnit4ClassRunner.class)
-@SpringBootTest(classes = { KafkaBinderConfiguration.class, KafkaBinderConfigurationPropertiesTest.class })
+@SpringBootTest(classes = { KafkaBinderConfiguration.class, KafkaAutoConfiguration.class,
+		KafkaBinderConfigurationPropertiesTest.class })
 @TestPropertySource(locations = "classpath:binder-config.properties")
 public class KafkaBinderConfigurationPropertiesTest {
 
@@ -63,10 +66,10 @@ public class KafkaBinderConfigurationPropertiesTest {
 		ExtendedProducerProperties<KafkaProducerProperties> producerProperties = new ExtendedProducerProperties<>(
 				kafkaProducerProperties);
 		Method getProducerFactoryMethod = KafkaMessageChannelBinder.class.getDeclaredMethod("getProducerFactory",
-				ExtendedProducerProperties.class);
+				String.class, ExtendedProducerProperties.class);
 		getProducerFactoryMethod.setAccessible(true);
 		DefaultKafkaProducerFactory producerFactory = (DefaultKafkaProducerFactory) getProducerFactoryMethod
-				.invoke(this.kafkaMessageChannelBinder, producerProperties);
+				.invoke(this.kafkaMessageChannelBinder, "bar", producerProperties);
 		Field producerFactoryConfigField = ReflectionUtils.findField(DefaultKafkaProducerFactory.class, "configs",
 				Map.class);
 		ReflectionUtils.makeAccessible(producerFactoryConfigField);

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationTest.java

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.springframework.cloud.stream.binder.kafka;
 
 import java.lang.reflect.Field;
@@ -21,6 +22,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration;
 import org.springframework.kafka.support.ProducerListener;
@@ -33,7 +35,9 @@ import static org.junit.Assert.assertNotNull;
  * @author Ilayaperumal Gopinathan
  */
 @RunWith(SpringJUnit4ClassRunner.class)
-@SpringBootTest(classes = { KafkaBinderConfiguration.class, KafkaBinderConfigurationTest.class })
+@SpringBootTest(classes = { KafkaBinderConfiguration.class,
+		KafkaAutoConfiguration.class,
+		KafkaBinderConfigurationTest.class })
 public class KafkaBinderConfigurationTest {
 
 	@Autowired

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicatorTest.java

@@ -13,11 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.springframework.cloud.stream.binder.kafka;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.BDDMockito.given;
-import static org.mockito.Mockito.verify;
+package org.springframework.cloud.stream.binder.kafka;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -25,11 +22,13 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.PartitionInfo;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
@@ -38,9 +37,12 @@ import org.springframework.boot.actuate.health.Health;
 import org.springframework.boot.actuate.health.Status;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 /**
  * @author Barry Commins
  * @author Gary Russell
+ * @author Laur Aliste
  */
 public class KafkaBinderHealthIndicatorTest {
 
@@ -62,8 +64,8 @@ public class KafkaBinderHealthIndicatorTest {
 	@Before
 	public void setup() {
 		MockitoAnnotations.initMocks(this);
-		given(consumerFactory.createConsumer()).willReturn(consumer);
-		given(binder.getTopicsInUse()).willReturn(topicsInUse);
+		org.mockito.BDDMockito.given(consumerFactory.createConsumer()).willReturn(consumer);
+		org.mockito.BDDMockito.given(binder.getTopicsInUse()).willReturn(topicsInUse);
 		this.indicator = new KafkaBinderHealthIndicator(binder, consumerFactory);
 		this.indicator.setTimeout(10);
 	}
@@ -72,17 +74,16 @@ public class KafkaBinderHealthIndicatorTest {
 	public void kafkaBinderIsUp() {
 		final List<PartitionInfo> partitions = partitions(new Node(0, null, 0));
 		topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation("group", partitions));
-		given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
+		org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
 		Health health = indicator.health();
 		assertThat(health.getStatus()).isEqualTo(Status.UP);
-		verify(this.consumer).close();
 	}
 
 	@Test
 	public void kafkaBinderIsDown() {
 		final List<PartitionInfo> partitions = partitions(new Node(-1, null, 0));
 		topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation("group", partitions));
-		given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
+		org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
 		Health health = indicator.health();
 		assertThat(health.getStatus()).isEqualTo(Status.DOWN);
 	}
@@ -91,7 +92,7 @@ public class KafkaBinderHealthIndicatorTest {
 	public void kafkaBinderDoesNotAnswer() {
 		final List<PartitionInfo> partitions = partitions(new Node(-1, null, 0));
 		topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation("group", partitions));
-		given(consumer.partitionsFor(TEST_TOPIC)).willAnswer(new Answer<Object>() {
+		org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)).willAnswer(new Answer<Object>() {
 
 			@Override
 			public Object answer(InvocationOnMock invocation) throws Throwable {
@@ -106,6 +107,33 @@ public class KafkaBinderHealthIndicatorTest {
 		assertThat(health.getStatus()).isEqualTo(Status.DOWN);
 	}
 
+	@Test
+	public void createsConsumerOnceWhenInvokedMultipleTimes() {
+		final List<PartitionInfo> partitions = partitions(new Node(0, null, 0));
+		topicsInUse.put(TEST_TOPIC, new KafkaMessageChannelBinder.TopicInformation("group", partitions));
+		org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
+
+		indicator.health();
+		Health health = indicator.health();
+
+		assertThat(health.getStatus()).isEqualTo(Status.UP);
+		org.mockito.Mockito.verify(this.consumerFactory).createConsumer();
+	}
+
+	@Test
+	public void consumerCreationFailsFirstTime() {
+		org.mockito.BDDMockito.given(consumerFactory.createConsumer()).willThrow(KafkaException.class)
+				.willReturn(consumer);
+
+		Health health = indicator.health();
+		assertThat(health.getStatus()).isEqualTo(Status.DOWN);
+
+		health = indicator.health();
+		assertThat(health.getStatus()).isEqualTo(Status.UP);
+
+		org.mockito.Mockito.verify(this.consumerFactory, Mockito.times(2)).createConsumer();
+	}
+
 	private List<PartitionInfo> partitions(Node leader) {
 		List<PartitionInfo> partitions = new ArrayList<>();
 		partitions.add(new PartitionInfo(TEST_TOPIC, 0, leader, null, null));

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderJaasInitializerListenerTest.java

@@ -1,61 +0,0 @@
-/*
- * Copyright 2016 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.springframework.cloud.stream.binder.kafka;
-
-import javax.security.auth.login.AppConfigurationEntry;
-
-import com.sun.security.auth.login.ConfigFile;
-
-import org.apache.kafka.common.security.JaasUtils;
-import org.junit.Test;
-
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.context.ConfigurableApplicationContext;
-import org.springframework.core.io.ClassPathResource;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-/**
- * @author Marius Bogoevici
- */
-public class KafkaBinderJaasInitializerListenerTest {
-
-	@Test
-	public void testConfigurationParsedCorrectlyWithKafkaClient() throws Exception {
-		ConfigFile configFile = new ConfigFile(new ClassPathResource("jaas-sample-kafka-only.conf").getURI());
-		final AppConfigurationEntry[] kafkaConfigurationArray = configFile.getAppConfigurationEntry(JaasUtils.LOGIN_CONTEXT_CLIENT);
-
-		final ConfigurableApplicationContext context =
-				SpringApplication.run(SimpleApplication.class,
-						"--spring.cloud.stream.kafka.binder.jaas.options.useKeyTab=true",
-						"--spring.cloud.stream.kafka.binder.jaas.options.storeKey=true",
-						"--spring.cloud.stream.kafka.binder.jaas.options.keyTab=/etc/security/keytabs/kafka_client.keytab",
-						"--spring.cloud.stream.kafka.binder.jaas.options.principal=kafka-client-1@EXAMPLE.COM");
-		javax.security.auth.login.Configuration configuration = javax.security.auth.login.Configuration.getConfiguration();
-
-		final AppConfigurationEntry[] kafkaConfiguration = configuration.getAppConfigurationEntry(JaasUtils.LOGIN_CONTEXT_CLIENT);
-		assertThat(kafkaConfiguration).hasSize(1);
-		assertThat(kafkaConfiguration[0].getOptions()).isEqualTo(kafkaConfigurationArray[0].getOptions());
-		context.close();
-	}
-
-	@SpringBootApplication
-	public static class SimpleApplication {
-
-	}
-}

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderMetricsTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 the original author or authors.
+ * Copyright 2016-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,14 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.springframework.cloud.stream.binder.kafka;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.search.Search;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.common.Node;
@@ -31,17 +34,11 @@ import org.junit.Test;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
-import org.springframework.boot.actuate.metrics.Metric;
 import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder.TopicInformation;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 
-import static java.util.Collections.singletonMap;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.BDDMockito.given;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyCollectionOf;
-import static org.springframework.cloud.stream.binder.kafka.KafkaBinderMetrics.METRIC_PREFIX;
 
 /**
  * @author Henryk Konsek
@@ -61,6 +58,8 @@ public class KafkaBinderMetricsTest {
 	@Mock
 	private KafkaMessageChannelBinder binder;
 
+	private MeterRegistry meterRegistry = new SimpleMeterRegistry();
+
 	private Map<String, TopicInformation> topicsInUse = new HashMap<>();
 
 	@Mock
@@ -69,24 +68,23 @@ public class KafkaBinderMetricsTest {
 	@Before
 	public void setup() {
 		MockitoAnnotations.initMocks(this);
-		given(consumerFactory.createConsumer()).willReturn(consumer);
-		given(binder.getTopicsInUse()).willReturn(topicsInUse);
+		org.mockito.BDDMockito.given(consumerFactory.createConsumer()).willReturn(consumer);
+		org.mockito.BDDMockito.given(binder.getTopicsInUse()).willReturn(topicsInUse);
 		metrics = new KafkaBinderMetrics(binder, kafkaBinderConfigurationProperties, consumerFactory);
-		given(consumer.endOffsets(anyCollectionOf(TopicPartition.class)))
-				.willReturn(singletonMap(new TopicPartition(TEST_TOPIC, 0), 1000L));
+		org.mockito.BDDMockito.given(consumer.endOffsets(org.mockito.Matchers.anyCollectionOf(TopicPartition.class)))
+				.willReturn(java.util.Collections.singletonMap(new TopicPartition(TEST_TOPIC, 0), 1000L));
 	}
 
 	@Test
 	public void shouldIndicateLag() {
-		given(consumer.committed(any(TopicPartition.class))).willReturn(new OffsetAndMetadata(500));
+		org.mockito.BDDMockito.given(consumer.committed(org.mockito.Matchers.any(TopicPartition.class))).willReturn(new OffsetAndMetadata(500));
 		List<PartitionInfo> partitions = partitions(new Node(0, null, 0));
 		topicsInUse.put(TEST_TOPIC, new TopicInformation("group", partitions));
-		given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
-		Collection<Metric<?>> collectedMetrics = metrics.metrics();
-		assertThat(collectedMetrics).hasSize(1);
-		assertThat(collectedMetrics.iterator().next().getName())
-				.isEqualTo(String.format("%s.%s.%s.lag", METRIC_PREFIX, "group", TEST_TOPIC));
-		assertThat(collectedMetrics.iterator().next().getValue()).isEqualTo(500L);
+		org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
+		metrics.bindTo(meterRegistry);
+		assertThat(meterRegistry.getMeters()).hasSize(1);
+		Search group = meterRegistry.find(String.format("%s.%s.%s.lag", KafkaBinderMetrics.METRIC_PREFIX, "group", TEST_TOPIC));
+		assertThat(group.gauge().value()).isEqualTo(500.0);
 	}
 
 	@Test
@@ -94,36 +92,34 @@ public class KafkaBinderMetricsTest {
 		Map<TopicPartition, Long> endOffsets = new HashMap<>();
 		endOffsets.put(new TopicPartition(TEST_TOPIC, 0), 1000L);
 		endOffsets.put(new TopicPartition(TEST_TOPIC, 1), 1000L);
-		given(consumer.endOffsets(anyCollectionOf(TopicPartition.class))).willReturn(endOffsets);
-		given(consumer.committed(any(TopicPartition.class))).willReturn(new OffsetAndMetadata(500));
+		org.mockito.BDDMockito.given(consumer.endOffsets(org.mockito.Matchers.anyCollectionOf(TopicPartition.class))).willReturn(endOffsets);
+		org.mockito.BDDMockito.given(consumer.committed(org.mockito.Matchers.any(TopicPartition.class))).willReturn(new OffsetAndMetadata(500));
 		List<PartitionInfo> partitions = partitions(new Node(0, null, 0), new Node(0, null, 0));
 		topicsInUse.put(TEST_TOPIC, new TopicInformation("group", partitions));
-		given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
-		Collection<Metric<?>> collectedMetrics = metrics.metrics();
-		assertThat(collectedMetrics).hasSize(1);
-		assertThat(collectedMetrics.iterator().next().getName())
-				.isEqualTo(String.format("%s.%s.%s.lag", METRIC_PREFIX, "group", TEST_TOPIC));
-		assertThat(collectedMetrics.iterator().next().getValue()).isEqualTo(1000L);
+		org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
+		metrics.bindTo(meterRegistry);
+		assertThat(meterRegistry.getMeters()).hasSize(1);
+		Search group = meterRegistry.find(String.format("%s.%s.%s.lag", KafkaBinderMetrics.METRIC_PREFIX, "group", TEST_TOPIC));
+		assertThat(group.gauge().value()).isEqualTo(1000.0);
 	}
 
 	@Test
 	public void shouldIndicateFullLagForNotCommittedGroups() {
 		List<PartitionInfo> partitions = partitions(new Node(0, null, 0));
 		topicsInUse.put(TEST_TOPIC, new TopicInformation("group", partitions));
-		given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
-		Collection<Metric<?>> collectedMetrics = metrics.metrics();
-		assertThat(collectedMetrics).hasSize(1);
-		assertThat(collectedMetrics.iterator().next().getName())
-				.isEqualTo(String.format("%s.%s.%s.lag", METRIC_PREFIX, "group", TEST_TOPIC));
-		assertThat(collectedMetrics.iterator().next().getValue()).isEqualTo(1000L);
+		org.mockito.BDDMockito.given(consumer.partitionsFor(TEST_TOPIC)).willReturn(partitions);
+		metrics.bindTo(meterRegistry);
+		assertThat(meterRegistry.getMeters()).hasSize(1);
+		Search group = meterRegistry.find(String.format("%s.%s.%s.lag", KafkaBinderMetrics.METRIC_PREFIX, "group", TEST_TOPIC));
+		assertThat(group.gauge().value()).isEqualTo(1000.0);
 	}
 
 	@Test
 	public void shouldNotCalculateLagForProducerTopics() {
 		List<PartitionInfo> partitions = partitions(new Node(0, null, 0));
 		topicsInUse.put(TEST_TOPIC, new TopicInformation(null, partitions));
-		Collection<Metric<?>> collectedMetrics = metrics.metrics();
-		assertThat(collectedMetrics).isEmpty();
+		metrics.bindTo(meterRegistry);
+		assertThat(meterRegistry.getMeters()).isEmpty();
 	}
 
 	private List<PartitionInfo> partitions(Node... nodes) {

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderUnitTests.java

@@ -23,15 +23,14 @@ import java.util.Map;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.junit.Test;
 
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
 import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
-import org.springframework.cloud.stream.binder.kafka.admin.AdminUtilsOperation;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
 import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
 import org.springframework.integration.test.util.TestUtils;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
 
 /**
  * @author Gary Russell
@@ -43,9 +42,7 @@ public class KafkaBinderUnitTests {
 	@Test
 	public void testPropertyOverrides() throws Exception {
 		KafkaBinderConfigurationProperties binderConfigurationProperties = new KafkaBinderConfigurationProperties();
-		AdminUtilsOperation adminUtilsOperation = mock(AdminUtilsOperation.class);
-		KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner(binderConfigurationProperties,
-				adminUtilsOperation);
+		KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner(binderConfigurationProperties, new KafkaProperties());
 		KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(binderConfigurationProperties,
 				provisioningProvider);
 		KafkaConsumerProperties consumerProps = new KafkaConsumerProperties();

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2015-2017 the original author or authors.
+ * Copyright 2015-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,8 +17,6 @@
 package org.springframework.cloud.stream.binder.kafka;
 
 import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
-import org.springframework.cloud.stream.binder.kafka.admin.AdminUtilsOperation;
-import org.springframework.cloud.stream.binder.kafka.admin.Kafka10AdminUtilsOperation;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
 import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
@@ -31,24 +29,20 @@ import org.springframework.kafka.support.ProducerListener;
 
 /**
  * Test support class for {@link KafkaMessageChannelBinder}.
+ *
  * @author Eric Bottard
  * @author Marius Bogoevici
  * @author David Turanski
  * @author Gary Russell
  * @author Soby Chacko
  */
-public class Kafka10TestBinder extends AbstractKafkaTestBinder {
+public class KafkaTestBinder extends AbstractKafkaTestBinder {
 
-	@SuppressWarnings({ "rawtypes", "unchecked" })
-	public Kafka10TestBinder(KafkaBinderConfigurationProperties binderConfiguration) {
+	@SuppressWarnings({"rawtypes", "unchecked"})
+	KafkaTestBinder(KafkaBinderConfigurationProperties binderConfiguration, KafkaTopicProvisioner kafkaTopicProvisioner) {
 		try {
-			AdminUtilsOperation adminUtilsOperation = new Kafka10AdminUtilsOperation();
-			KafkaTopicProvisioner provisioningProvider =
-					new KafkaTopicProvisioner(binderConfiguration, adminUtilsOperation);
-			provisioningProvider.afterPropertiesSet();
-
 			KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(binderConfiguration,
-					provisioningProvider) {
+					kafkaTopicProvisioner) {
 
 				/*
 				 * Some tests use multiple instance indexes for the same topic; we need to make
@@ -56,21 +50,20 @@ public class Kafka10TestBinder extends AbstractKafkaTestBinder {
 				 */
 				@Override
 				protected String errorsBaseName(ConsumerDestination destination, String group,
-						ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties) {
+												ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties) {
 					return super.errorsBaseName(destination, group, consumerProperties) + "-"
 							+ consumerProperties.getInstanceIndex();
 				}
 
 			};
 
-			binder.setCodec(AbstractKafkaTestBinder.getCodec());
 			ProducerListener producerListener = new LoggingProducerListener();
 			binder.setProducerListener(producerListener);
 			AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(Config.class);
 			setApplicationContext(context);
 			binder.setApplicationContext(context);
 			binder.afterPropertiesSet();
-			this.setBinder(binder);
+			this.setPollableConsumerBinder(binder);
 		}
 		catch (Exception e) {
 			throw new RuntimeException(e);

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTransactionTests.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.binder.kafka;
+
+import java.util.Collections;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.TopicPartition;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.mockito.InOrder;
+
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
+import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
+import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
+import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
+import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
+import org.springframework.context.support.GenericApplicationContext;
+import org.springframework.integration.channel.DirectChannel;
+import org.springframework.kafka.core.DefaultKafkaProducerFactory;
+import org.springframework.kafka.test.rule.KafkaEmbedded;
+import org.springframework.messaging.support.GenericMessage;
+import org.springframework.retry.support.RetryTemplate;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.BDDMockito.willReturn;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
+/**
+ * @author Gary Russell
+ * @since 2.0
+ *
+ */
+public class KafkaTransactionTests {
+
+	@ClassRule
+	public static final KafkaEmbedded embeddedKafka = new KafkaEmbedded(1);
+
+	@SuppressWarnings({ "rawtypes", "unchecked" })
+	@Test
+	public void testProducerRunsInTx() {
+		KafkaProperties kafkaProperties = new KafkaProperties();
+		kafkaProperties.setBootstrapServers(Collections.singletonList(embeddedKafka.getBrokersAsString()));
+		KafkaBinderConfigurationProperties configurationProperties = new KafkaBinderConfigurationProperties();
+		configurationProperties.getTransaction().setTransactionIdPrefix("foo-");
+		KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner(configurationProperties, kafkaProperties);
+		provisioningProvider.setMetadataRetryOperations(new RetryTemplate());
+		final Producer mockProducer = mock(Producer.class);
+		willReturn(Collections.singletonList(new TopicPartition("foo", 0))).given(mockProducer).partitionsFor(anyString());
+		KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties, provisioningProvider) {
+
+			@Override
+			protected DefaultKafkaProducerFactory<byte[], byte[]> getProducerFactory(String transactionIdPrefix,
+					ExtendedProducerProperties<KafkaProducerProperties> producerProperties) {
+				DefaultKafkaProducerFactory<byte[], byte[]> producerFactory =
+						spy(super.getProducerFactory(transactionIdPrefix, producerProperties));
+				willReturn(mockProducer).given(producerFactory).createProducer();
+				return producerFactory;
+			}
+
+		};
+		GenericApplicationContext applicationContext = new GenericApplicationContext();
+		applicationContext.refresh();
+		binder.setApplicationContext(applicationContext);
+		DirectChannel channel = new DirectChannel();
+		KafkaProducerProperties extension = new KafkaProducerProperties();
+		ExtendedProducerProperties<KafkaProducerProperties> properties = new ExtendedProducerProperties<>(extension);
+		binder.bindProducer("foo", channel, properties);
+		channel.send(new GenericMessage<>("foo".getBytes()));
+		InOrder inOrder = inOrder(mockProducer);
+		inOrder.verify(mockProducer).beginTransaction();
+		inOrder.verify(mockProducer).send(any(ProducerRecord.class), any(Callback.class));
+		inOrder.verify(mockProducer).commitTransaction();
+		inOrder.verify(mockProducer).close();
+		inOrder.verifyNoMoreInteractions();
+	}
+
+}

spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderBootstrapTest.java

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.springframework.cloud.stream.binder.kafka.bootstrap;
 
 import org.junit.ClassRule;

spring-cloud-stream-binder-kafka/src/test/resources/logback.xml

@@ -4,6 +4,7 @@
             <pattern>%d{ISO8601} %5p %t %c{2}:%L - %m%n</pattern>
         </encoder>
     </appender>
+    <logger name="org.apache.kafka" level="DEBUG"/>
     <logger name="org.springframework.integration.kafka" level="INFO"/>
     <logger name="org.springframework.kafka" level="INFO"/>
     <logger name="org.springframework.cloud.stream" level="INFO" />

spring-cloud-stream-binder-kafka/src/test/resources/schemas/users_v1.schema

@@ -1,8 +0,0 @@
-{"namespace": "org.springframework.cloud.stream.binder.kafka",
-  "type": "record",
-  "name": "User1",
-  "fields": [
-    {"name": "name", "type": "string"},
-    {"name": "favoriteColor", "type": "string"}
-  ]
-}

spring-cloud-stream-binder-kstream/.settings/org.eclipse.jdt.ui.prefs

@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.ui.ignorelowercasenames=true
+org.eclipse.jdt.ui.importorder=java;javax;com;org;org.springframework;ch.qos;\#;
+org.eclipse.jdt.ui.ondemandthreshold=99
+org.eclipse.jdt.ui.staticondemandthreshold=99

spring-cloud-stream-binder-kstream/pom.xml

@@ -10,7 +10,7 @@
 	<parent>
 		<groupId>org.springframework.cloud</groupId>
 		<artifactId>spring-cloud-stream-binder-kafka-parent</artifactId>
-		<version>1.3.1.RELEASE</version>
+		<version>2.0.0.M4</version>
 	</parent>
 
 	<dependencies>
@@ -23,19 +23,11 @@
 			<artifactId>spring-boot-configuration-processor</artifactId>
 			<optional>true</optional>
 		</dependency>
-		<dependency>
-			<groupId>org.springframework.cloud</groupId>
-			<artifactId>spring-cloud-stream-codec</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-autoconfigure</artifactId>
 			<optional>true</optional>
 		</dependency>
-		<dependency>
-			<groupId>org.apache.kafka</groupId>
-			<artifactId>kafka_2.11</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>org.apache.kafka</groupId>
 			<artifactId>kafka-streams</artifactId>
@@ -52,12 +44,17 @@
 		<dependency>
 			<groupId>org.springframework.kafka</groupId>
 			<artifactId>spring-kafka-test</artifactId>
-			<scope>test</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.kafka</groupId>
 			<artifactId>kafka_2.11</artifactId>
 			<classifier>test</classifier>
+		</dependency>
+		<!-- Added back since Kafka still depends on it, but it has been removed by Boot due to EOL -->
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>1.2.17</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/KStreamBinder.java

@@ -24,17 +24,14 @@ import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
+import org.apache.kafka.streams.kstream.Produced;
 
 import org.springframework.cloud.stream.binder.AbstractBinder;
-import org.springframework.cloud.stream.binder.BinderHeaders;
 import org.springframework.cloud.stream.binder.Binding;
 import org.springframework.cloud.stream.binder.DefaultBinding;
-import org.springframework.cloud.stream.binder.EmbeddedHeaderUtils;
 import org.springframework.cloud.stream.binder.ExtendedConsumerProperties;
 import org.springframework.cloud.stream.binder.ExtendedProducerProperties;
 import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder;
-import org.springframework.cloud.stream.binder.HeaderMode;
-import org.springframework.cloud.stream.binder.MessageValues;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties;
@@ -42,20 +39,16 @@ import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProv
 import org.springframework.cloud.stream.binder.kstream.config.KStreamConsumerProperties;
 import org.springframework.cloud.stream.binder.kstream.config.KStreamExtendedBindingProperties;
 import org.springframework.cloud.stream.binder.kstream.config.KStreamProducerProperties;
-import org.springframework.messaging.Message;
-import org.springframework.messaging.MessageHeaders;
-import org.springframework.util.MimeType;
 import org.springframework.util.StringUtils;
 
 /**
  * @author Marius Bogoevici
+ * @author Soby Chacko
  */
 public class KStreamBinder extends
 		AbstractBinder<KStream<Object, Object>, ExtendedConsumerProperties<KStreamConsumerProperties>, ExtendedProducerProperties<KStreamProducerProperties>>
 		implements ExtendedPropertiesBinder<KStream<Object, Object>, KStreamConsumerProperties, KStreamProducerProperties> {
 
-	private String[] headers;
-
 	private final KafkaTopicProvisioner kafkaTopicProvisioner;
 
 	private final KStreamExtendedBindingProperties kStreamExtendedBindingProperties;
@@ -64,20 +57,25 @@ public class KStreamBinder extends
 
 	private final KafkaBinderConfigurationProperties binderConfigurationProperties;
 
-	public KStreamBinder(KafkaBinderConfigurationProperties binderConfigurationProperties, KafkaTopicProvisioner kafkaTopicProvisioner,
-						KStreamExtendedBindingProperties kStreamExtendedBindingProperties, StreamsConfig streamsConfig) {
+	private final MessageConversionDelegate messageConversionDelegate;
+
+	public KStreamBinder(KafkaBinderConfigurationProperties binderConfigurationProperties,
+						KafkaTopicProvisioner kafkaTopicProvisioner,
+						KStreamExtendedBindingProperties kStreamExtendedBindingProperties, StreamsConfig streamsConfig,
+						MessageConversionDelegate messageConversionDelegate) {
 		this.binderConfigurationProperties = binderConfigurationProperties;
-		this.headers = EmbeddedHeaderUtils.headersToEmbed(binderConfigurationProperties.getHeaders());
 		this.kafkaTopicProvisioner = kafkaTopicProvisioner;
 		this.kStreamExtendedBindingProperties = kStreamExtendedBindingProperties;
 		this.streamsConfig = streamsConfig;
+		this.messageConversionDelegate = messageConversionDelegate;
 	}
 
 	@Override
 	protected Binding<KStream<Object, Object>> doBindConsumer(String name, String group,
-															KStream<Object, Object> inputTarget, ExtendedConsumerProperties<KStreamConsumerProperties> properties) {
+															KStream<Object, Object> inputTarget,
+															ExtendedConsumerProperties<KStreamConsumerProperties> properties) {
 
-		ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties = new ExtendedConsumerProperties<KafkaConsumerProperties>(
+		ExtendedConsumerProperties<KafkaConsumerProperties> extendedConsumerProperties = new ExtendedConsumerProperties<>(
 				new KafkaConsumerProperties());
 		this.kafkaTopicProvisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);
 		return new DefaultBinding<>(name, group, inputTarget, null);
@@ -87,63 +85,43 @@ public class KStreamBinder extends
 	@SuppressWarnings("unchecked")
 	protected Binding<KStream<Object, Object>> doBindProducer(String name, KStream<Object, Object> outboundBindTarget,
 															ExtendedProducerProperties<KStreamProducerProperties> properties) {
-		ExtendedProducerProperties<KafkaProducerProperties> extendedProducerProperties = new ExtendedProducerProperties<KafkaProducerProperties>(
+		ExtendedProducerProperties<KafkaProducerProperties> extendedProducerProperties = new ExtendedProducerProperties<>(
 				new KafkaProducerProperties());
 		this.kafkaTopicProvisioner.provisionProducerDestination(name, extendedProducerProperties);
-		if (HeaderMode.embeddedHeaders.equals(properties.getHeaderMode())) {
-			outboundBindTarget = outboundBindTarget.map(new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {
-				@Override
-				public KeyValue<Object, Object> apply(Object k, Object v) {
-					if (v instanceof Message) {
-						try {
-							return new KeyValue<>(k, (Object) KStreamBinder.this.serializeAndEmbedHeadersIfApplicable((Message<?>) v));
-						}
-						catch (Exception e) {
-							throw new IllegalArgumentException(e);
-						}
-					}
-					else {
-						throw new IllegalArgumentException("Wrong type of message " + v);
-					}
+
+		Serde<?> keySerde = getKeySerde(properties);
+		Serde<?> valueSerde = getValueSerde(properties);
+
+		to(properties.isUseNativeEncoding(), name, outboundBindTarget, (Serde<Object>) keySerde, (Serde<Object>) valueSerde);
+
+		return new DefaultBinding<>(name, null, outboundBindTarget, null);
+	}
+
+	private Serde<?> getKeySerde(ExtendedProducerProperties<KStreamProducerProperties> properties) {
+		Serde<?> keySerde;
+		try {
+			if (StringUtils.hasText(properties.getExtension().getKeySerde())) {
+				keySerde = Utils.newInstance(properties.getExtension().getKeySerde(), Serde.class);
+				if (keySerde instanceof Configurable) {
+					((Configurable) keySerde).configure(streamsConfig.originals());
 				}
-			});
-		}
-		else {
-			if (!properties.isUseNativeEncoding()) {
-				outboundBindTarget = outboundBindTarget
-						.map(new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {
-							@Override
-							public KeyValue<Object, Object> apply(Object k, Object v) {
-								return KeyValue.pair(k, (Object) KStreamBinder.this.serializePayloadIfNecessary((Message<?>) v));
-							}
-						});
 			}
 			else {
-				outboundBindTarget = outboundBindTarget
-						.map(new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {
-							@Override
-							public KeyValue<Object, Object> apply(Object k, Object v) {
-								return KeyValue.pair(k, ((Message<?>) v).getPayload());
-							}
-						});
+				keySerde = this.binderConfigurationProperties.getConfiguration().containsKey("key.serde") ?
+						Utils.newInstance(this.binderConfigurationProperties.getConfiguration().get("key.serde"), Serde.class) : Serdes.ByteArray();
 			}
+
 		}
-		if (!properties.isUseNativeEncoding() || StringUtils.hasText(properties.getExtension().getKeySerde()) || StringUtils.hasText(properties.getExtension().getValueSerde())) {
-			try {
-				Serde<?> keySerde;
-				Serde<?> valueSerde;
-
-				if (StringUtils.hasText(properties.getExtension().getKeySerde())) {
-					keySerde = Utils.newInstance(properties.getExtension().getKeySerde(), Serde.class);
-					if (keySerde instanceof Configurable) {
-						((Configurable) keySerde).configure(streamsConfig.originals());
-					}
-				}
-				else {
-					keySerde = this.binderConfigurationProperties.getConfiguration().containsKey("key.serde") ?
-							Utils.newInstance(this.binderConfigurationProperties.getConfiguration().get("key.serde"), Serde.class) : Serdes.ByteArray();
-				}
+		catch (ClassNotFoundException e) {
+			throw new IllegalStateException("Serde class not found: ", e);
+		}
+		return keySerde;
+	}
 
+	private Serde<?> getValueSerde(ExtendedProducerProperties<KStreamProducerProperties> properties) {
+		Serde<?> valueSerde;
+		try {
+			if (properties.isUseNativeEncoding()) {
 				if (StringUtils.hasText(properties.getExtension().getValueSerde())) {
 					valueSerde = Utils.newInstance(properties.getExtension().getValueSerde(), Serde.class);
 					if (valueSerde instanceof Configurable) {
@@ -154,35 +132,30 @@ public class KStreamBinder extends
 					valueSerde = this.binderConfigurationProperties.getConfiguration().containsKey("value.serde") ?
 							Utils.newInstance(this.binderConfigurationProperties.getConfiguration().get("value.serde"), Serde.class) : Serdes.ByteArray();
 				}
-				outboundBindTarget.to((Serde<Object>) keySerde, (Serde<Object>) valueSerde, name);
 			}
-			catch (ClassNotFoundException e) {
-				throw new IllegalStateException("Serde class not found: ", e);
+			else {
+				valueSerde = Serdes.ByteArray();
 			}
-
 		}
-		else {
-			outboundBindTarget.to(name);
+		catch (ClassNotFoundException e) {
+			throw new IllegalStateException("Serde class not found: ", e);
 		}
-		return new DefaultBinding<>(name, null, outboundBindTarget, null);
+		return valueSerde;
 	}
 
-	private byte[] serializeAndEmbedHeadersIfApplicable(Message<?> message) throws Exception {
-		MessageValues transformed = serializePayloadIfNecessary(message);
-		byte[] payload;
-
-		Object contentType = transformed.get(MessageHeaders.CONTENT_TYPE);
-		// transform content type headers to String, so that they can be properly embedded
-		// in JSON
-		if (contentType instanceof MimeType) {
-			transformed.put(MessageHeaders.CONTENT_TYPE, contentType.toString());
+	@SuppressWarnings("unchecked")
+	private void to(boolean isNativeEncoding, String name, KStream<Object, Object> outboundBindTarget,
+				Serde<Object> keySerde, Serde<Object> valueSerde) {
+		KeyValueMapper<Object, Object, KeyValue<Object, Object>> keyValueMapper = null;
+		if (!isNativeEncoding) {
+			keyValueMapper = messageConversionDelegate.outboundKeyValueMapper(name);
 		}
-		Object originalContentType = transformed.get(BinderHeaders.BINDER_ORIGINAL_CONTENT_TYPE);
-		if (originalContentType instanceof MimeType) {
-			transformed.put(BinderHeaders.BINDER_ORIGINAL_CONTENT_TYPE, originalContentType.toString());
+		if (!isNativeEncoding) {
+				outboundBindTarget.map(keyValueMapper).to(name, Produced.with(keySerde, valueSerde));
+		}
+		else {
+			outboundBindTarget.to(name, Produced.with(keySerde, valueSerde));
 		}
-		payload = EmbeddedHeaderUtils.embedHeaders(transformed, headers);
-		return payload;
 	}
 
 	@Override
@@ -194,5 +167,4 @@ public class KStreamBinder extends
 	public KStreamProducerProperties getExtendedProducerProperties(String channelName) {
 		return this.kStreamExtendedBindingProperties.getExtendedProducerProperties(channelName);
 	}
-
 }

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/KStreamBoundElementFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 the original author or authors.
+ * Copyright 2017-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,136 +16,79 @@
 
 package org.springframework.cloud.stream.binder.kstream;
 
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
 import org.aopalliance.intercept.MethodInterceptor;
 import org.aopalliance.intercept.MethodInvocation;
 import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.KStreamBuilder;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
 
 import org.springframework.aop.framework.ProxyFactory;
-import org.springframework.cloud.stream.binder.ConsumerProperties;
-import org.springframework.cloud.stream.binder.EmbeddedHeaderUtils;
-import org.springframework.cloud.stream.binder.HeaderMode;
-import org.springframework.cloud.stream.binder.MessageSerializationUtils;
-import org.springframework.cloud.stream.binder.MessageValues;
-import org.springframework.cloud.stream.binder.StringConvertingContentTypeResolver;
 import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory;
 import org.springframework.cloud.stream.config.BindingProperties;
 import org.springframework.cloud.stream.config.BindingServiceProperties;
-import org.springframework.cloud.stream.converter.CompositeMessageConverterFactory;
-import org.springframework.integration.codec.Codec;
-import org.springframework.integration.support.MutableMessageHeaders;
 import org.springframework.messaging.Message;
-import org.springframework.messaging.converter.MessageConverter;
+import org.springframework.messaging.MessageHeaders;
 import org.springframework.messaging.support.MessageBuilder;
 import org.springframework.util.Assert;
-import org.springframework.util.MimeType;
 import org.springframework.util.StringUtils;
 
 /**
  * @author Marius Bogoevici
+ * @author Soby Chacko
  */
 public class KStreamBoundElementFactory extends AbstractBindingTargetFactory<KStream> {
 
-	private final KStreamBuilder kStreamBuilder;
+	private final StreamsBuilder kStreamBuilder;
 
 	private final BindingServiceProperties bindingServiceProperties;
 
-	private volatile Codec codec;
-
-	private final StringConvertingContentTypeResolver contentTypeResolver = new StringConvertingContentTypeResolver();
-
-	private volatile Map<String, Class<?>> payloadTypeCache = new ConcurrentHashMap<>();
-
-	private CompositeMessageConverterFactory compositeMessageConverterFactory;
-
-	public KStreamBoundElementFactory(KStreamBuilder streamBuilder, BindingServiceProperties bindingServiceProperties,
-			Codec codec, CompositeMessageConverterFactory compositeMessageConverterFactory) {
+	public KStreamBoundElementFactory(StreamsBuilder kStreamBuilder, BindingServiceProperties bindingServiceProperties) {
 		super(KStream.class);
 		this.bindingServiceProperties = bindingServiceProperties;
-		this.kStreamBuilder = streamBuilder;
-		this.codec = codec;
-		this.compositeMessageConverterFactory = compositeMessageConverterFactory;
+		this.kStreamBuilder = kStreamBuilder;
 	}
 
 	@Override
 	public KStream createInput(String name) {
 		KStream<Object, Object> stream = kStreamBuilder.stream(bindingServiceProperties.getBindingDestination(name));
-		ConsumerProperties properties = bindingServiceProperties.getConsumerProperties(name);
-		if (HeaderMode.embeddedHeaders.equals(properties.getHeaderMode())) {
-
-			stream = stream.map(new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {
-				@Override
-				public KeyValue<Object, Object> apply(Object key, Object value) {
-					if (!(value instanceof byte[])) {
-						return new KeyValue<>(key, value);
-					}
-					try {
-						MessageValues messageValues = EmbeddedHeaderUtils
-								.extractHeaders(MessageBuilder.withPayload((byte[]) value).build(), true);
-						messageValues = deserializePayloadIfNecessary(messageValues);
-						return new KeyValue<Object, Object>(null, messageValues.toMessage());
-					}
-					catch (Exception e) {
-						throw new IllegalArgumentException(e);
-					}
-				}
-			});
-		}
+		stream = stream.map((key, value) -> {
+			KeyValue<Object, Object> keyValue;
+			BindingProperties bindingProperties = bindingServiceProperties.getBindingProperties(name);
+			String contentType = bindingProperties.getContentType();
+			if (!StringUtils.isEmpty(contentType) && !bindingProperties.getConsumer().isUseNativeDecoding()) {
+				Message<Object> message = MessageBuilder.withPayload(value)
+						.setHeader(MessageHeaders.CONTENT_TYPE, contentType).build();
+				keyValue = new KeyValue<>(key, message);
+			}
+			else {
+				keyValue = new KeyValue<>(key, value);
+			}
+			return keyValue;
+		});
 		return stream;
 	}
 
 	@Override
 	@SuppressWarnings("unchecked")
 	public KStream createOutput(final String name) {
-		BindingProperties bindingProperties = bindingServiceProperties.getBindingProperties(name);
-		String contentType = bindingProperties.getContentType();
-		MessageConverter messageConverter = StringUtils.hasText(contentType) ? compositeMessageConverterFactory
-				.getMessageConverterForType(MimeType.valueOf(contentType)) : null;
-		KStreamWrapperHandler handler = new KStreamWrapperHandler(messageConverter);
+		KStreamWrapperHandler wrapper= new KStreamWrapperHandler();
 		ProxyFactory proxyFactory = new ProxyFactory(KStreamWrapper.class, KStream.class);
-		proxyFactory.addAdvice(handler);
+		proxyFactory.addAdvice(wrapper);
 		return (KStream) proxyFactory.getProxy();
 	}
 
-	private MessageValues deserializePayloadIfNecessary(MessageValues messageValues) {
-		return MessageSerializationUtils.deserializePayload(messageValues, this.contentTypeResolver, this.codec);
-	}
-
-	interface KStreamWrapper {
+	public interface KStreamWrapper {
 
 		void wrap(KStream<Object, Object> delegate);
 	}
 
-	static class KStreamWrapperHandler implements KStreamWrapper, MethodInterceptor {
+	private static class KStreamWrapperHandler implements KStreamWrapper, MethodInterceptor {
 
 		private KStream<Object, Object> delegate;
 
-		private final MessageConverter messageConverter;
-
-		public KStreamWrapperHandler(MessageConverter messageConverter) {
-			this.messageConverter = messageConverter;
-		}
-
 		public void wrap(KStream<Object, Object> delegate) {
 			Assert.notNull(delegate, "delegate cannot be null");
 			Assert.isNull(this.delegate, "delegate already set to " + this.delegate);
-			if (messageConverter != null) {
-				KeyValueMapper<Object, Object, KeyValue<Object, Object>> keyValueMapper = new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {
-					@Override
-					public KeyValue<Object, Object> apply(Object k, Object v) {
-						Message<?> message = (Message<?>) v;
-						return new KeyValue<Object, Object>(k,
-								messageConverter.toMessage(message.getPayload(),
-										new MutableMessageHeaders(((Message<?>) v).getHeaders())));
-					}
-				};
-				delegate = delegate.map(keyValueMapper);
-			}
 			this.delegate = delegate;
 		}
 

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/KStreamListenerParameterAdapter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 the original author or authors.
+ * Copyright 2017-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,16 +16,11 @@
 
 package org.springframework.cloud.stream.binder.kstream;
 
-import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
 
 import org.springframework.cloud.stream.binding.StreamListenerParameterAdapter;
 import org.springframework.core.MethodParameter;
 import org.springframework.core.ResolvableType;
-import org.springframework.messaging.Message;
-import org.springframework.messaging.converter.MessageConverter;
-import org.springframework.messaging.support.MessageBuilder;
 
 /**
  * @author Marius Bogoevici
@@ -33,10 +28,10 @@ import org.springframework.messaging.support.MessageBuilder;
  */
 public class KStreamListenerParameterAdapter implements StreamListenerParameterAdapter<KStream<?,?>, KStream<?, ?>> {
 
-	private final MessageConverter messageConverter;
+	private final MessageConversionDelegate messageConversionDelegate;
 
-	public KStreamListenerParameterAdapter(MessageConverter messageConverter) {
-		this.messageConverter = messageConverter;
+	public KStreamListenerParameterAdapter(MessageConversionDelegate messageConversionDelegate) {
+		this.messageConversionDelegate = messageConversionDelegate;
 	}
 
 	@Override
@@ -52,24 +47,7 @@ public class KStreamListenerParameterAdapter implements StreamListenerParameterA
 		final Class<?> valueClass = (resolvableType.getGeneric(1).getRawClass() != null)
 				? (resolvableType.getGeneric(1).getRawClass()) : Object.class;
 
-		return bindingTarget.map(new KeyValueMapper() {
-			@Override
-			public Object apply(Object o, Object o2) {
-				if (valueClass.isAssignableFrom(o2.getClass())) {
-					return new KeyValue<>(o, o2);
-				}
-				else if (o2 instanceof Message) {
-					return new KeyValue<>(o, messageConverter.fromMessage((Message) o2, valueClass));
-				}
-				else if(o2 instanceof String || o2 instanceof byte[]) {
-					Message<Object> message = MessageBuilder.withPayload(o2).build();
-					return new KeyValue<>(o, messageConverter.fromMessage(message, valueClass));
-				}
-				else {
-					return new KeyValue<>(o, o2);
-				}
-			}
-		});
+		return bindingTarget.map(messageConversionDelegate.inboundKeyValueMapper(valueClass));
 	}
 
 }

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/KStreamListenerSetupMethodOrchestrator.java

@@ -0,0 +1,183 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.binder.kstream;
+
+import java.lang.reflect.Method;
+import java.util.Collection;
+
+import org.apache.kafka.streams.kstream.KStream;
+
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.BeanInitializationException;
+import org.springframework.cloud.stream.annotation.StreamListener;
+import org.springframework.cloud.stream.binding.StreamListenerErrorMessages;
+import org.springframework.cloud.stream.binding.StreamListenerParameterAdapter;
+import org.springframework.cloud.stream.binding.StreamListenerResultAdapter;
+import org.springframework.cloud.stream.binding.StreamListenerSetupMethodOrchestrator;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.core.MethodParameter;
+import org.springframework.core.annotation.AnnotationUtils;
+import org.springframework.messaging.handler.annotation.SendTo;
+import org.springframework.util.Assert;
+import org.springframework.util.ObjectUtils;
+import org.springframework.util.StringUtils;
+
+/**
+ * Kafka Streams specific implementation for {@link StreamListenerSetupMethodOrchestrator}
+ * that overrides the default mechanisms for invoking StreamListener adapters.
+ *
+ * @author Soby Chacko
+ */
+public class KStreamListenerSetupMethodOrchestrator implements StreamListenerSetupMethodOrchestrator, ApplicationContextAware {
+
+	private ConfigurableApplicationContext applicationContext;
+
+	private StreamListenerParameterAdapter streamListenerParameterAdapter;
+	private Collection<StreamListenerResultAdapter> streamListenerResultAdapters;
+
+	public KStreamListenerSetupMethodOrchestrator(StreamListenerParameterAdapter streamListenerParameterAdapter,
+												Collection<StreamListenerResultAdapter> streamListenerResultAdapters) {
+		this.streamListenerParameterAdapter = streamListenerParameterAdapter;
+		this.streamListenerResultAdapters = streamListenerResultAdapters;
+	}
+
+	@Override
+	public boolean supports(Method method) {
+		return methodParameterSuppports(method) && methodReturnTypeSuppports(method);
+	}
+
+	private boolean methodReturnTypeSuppports(Method method) {
+		Class<?> returnType = method.getReturnType();
+		if (returnType.equals(KStream.class) ||
+				(returnType.isArray() && returnType.getComponentType().equals(KStream.class))) {
+			return true;
+		}
+		return false;
+	}
+
+	private boolean methodParameterSuppports(Method method) {
+		MethodParameter methodParameter = MethodParameter.forExecutable(method, 0);
+		Class<?> parameterType = methodParameter.getParameterType();
+		return parameterType.equals(KStream.class);
+	}
+
+	@Override
+	@SuppressWarnings({"rawtypes", "unchecked"})
+	public void orchestrateStreamListenerSetupMethod(StreamListener streamListener, Method method, Object bean) {
+		String[] methodAnnotatedOutboundNames = getOutboundBindingTargetNames(method);
+		validateStreamListenerMethod(streamListener, method, methodAnnotatedOutboundNames);
+
+		String methodAnnotatedInboundName = streamListener.value();
+		Object[] adaptedInboundArguments = adaptAndRetrieveInboundArguments(method, methodAnnotatedInboundName,
+				this.applicationContext,
+				this.streamListenerParameterAdapter);
+
+		try {
+			Object result = method.invoke(bean, adaptedInboundArguments);
+
+			if (result.getClass().isArray()) {
+				Assert.isTrue(methodAnnotatedOutboundNames.length == ((Object[]) result).length, "Big error");
+			} else {
+				Assert.isTrue(methodAnnotatedOutboundNames.length == 1, "Big error");
+			}
+			if (result.getClass().isArray()) {
+				Object[] outboundKStreams = (Object[]) result;
+				int i = 0;
+				for (Object outboundKStream : outboundKStreams) {
+					Object targetBean = this.applicationContext.getBean(methodAnnotatedOutboundNames[i++]);
+					for (StreamListenerResultAdapter streamListenerResultAdapter : streamListenerResultAdapters) {
+						if (streamListenerResultAdapter.supports(outboundKStream.getClass(), targetBean.getClass())) {
+							streamListenerResultAdapter.adapt(outboundKStream, targetBean);
+							break;
+						}
+					}
+				}
+			}
+			else {
+				Object targetBean = this.applicationContext.getBean(methodAnnotatedOutboundNames[0]);
+				for (StreamListenerResultAdapter streamListenerResultAdapter : streamListenerResultAdapters) {
+					if (streamListenerResultAdapter.supports(result.getClass(), targetBean.getClass())) {
+						streamListenerResultAdapter.adapt(result, targetBean);
+						break;
+					}
+				}
+			}
+		}
+		catch (Exception e) {
+			throw new BeanInitializationException("Cannot setup StreamListener for " + method, e);
+		}
+	}
+
+	@Override
+	public final void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+		this.applicationContext = (ConfigurableApplicationContext) applicationContext;
+	}
+
+	private void validateStreamListenerMethod(StreamListener streamListener, Method method, String[] methodAnnotatedOutboundNames) {
+		String methodAnnotatedInboundName = streamListener.value();
+		for (String s : methodAnnotatedOutboundNames) {
+			if (StringUtils.hasText(s)) {
+				Assert.isTrue(isDeclarativeOutput(method, s), "Method must be declarative");
+			}
+		}
+		if (StringUtils.hasText(methodAnnotatedInboundName)) {
+			int methodArgumentsLength = method.getParameterTypes().length;
+
+			for (int parameterIndex = 0; parameterIndex < methodArgumentsLength; parameterIndex++) {
+				MethodParameter methodParameter = MethodParameter.forExecutable(method, parameterIndex);
+				Assert.isTrue(isDeclarativeInput(methodAnnotatedInboundName, methodParameter), "Method must be declarative");
+			}
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	private boolean isDeclarativeOutput(Method m, String targetBeanName) {
+		boolean declarative;
+		Class<?> returnType = m.getReturnType();
+		if (returnType.isArray()){
+			Class<?> targetBeanClass = this.applicationContext.getType(targetBeanName);
+			declarative = this.streamListenerResultAdapters.stream()
+					.anyMatch(slpa -> slpa.supports(returnType.getComponentType(), targetBeanClass));
+			return declarative;
+		}
+		Class<?> targetBeanClass = this.applicationContext.getType(targetBeanName);
+		declarative = this.streamListenerResultAdapters.stream()
+				.anyMatch(slpa -> slpa.supports(returnType, targetBeanClass));
+		return declarative;
+	}
+
+	@SuppressWarnings("unchecked")
+	private boolean isDeclarativeInput(String targetBeanName, MethodParameter methodParameter) {
+		if (!methodParameter.getParameterType().isAssignableFrom(Object.class) && this.applicationContext.containsBean(targetBeanName)) {
+			Class<?> targetBeanClass = this.applicationContext.getType(targetBeanName);
+			return this.streamListenerParameterAdapter.supports(targetBeanClass, methodParameter);
+		}
+		return false;
+	}
+
+	private static String[] getOutboundBindingTargetNames(Method method) {
+		SendTo sendTo = AnnotationUtils.findAnnotation(method, SendTo.class);
+		if (sendTo != null) {
+			Assert.isTrue(!ObjectUtils.isEmpty(sendTo.value()), StreamListenerErrorMessages.ATLEAST_ONE_OUTPUT);
+			Assert.isTrue(sendTo.value().length >= 1, "At least one outbound destination need to be provided.");
+			return sendTo.value();
+		}
+		return null;
+	}
+}

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/KStreamStreamListenerResultAdapter.java

@@ -21,11 +21,8 @@ import java.io.IOException;
 
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
 
 import org.springframework.cloud.stream.binding.StreamListenerResultAdapter;
-import org.springframework.messaging.Message;
-import org.springframework.messaging.support.MessageBuilder;
 
 /**
  * @author Marius Bogoevici
@@ -40,17 +37,7 @@ public class KStreamStreamListenerResultAdapter implements StreamListenerResultA
 	@Override
 	@SuppressWarnings("unchecked")
 	public Closeable adapt(KStream streamListenerResult, KStreamBoundElementFactory.KStreamWrapper boundElement) {
-		boundElement.wrap(streamListenerResult.map(new KeyValueMapper() {
-			@Override
-			public Object apply(Object k, Object v) {
-				if (v instanceof Message<?>) {
-					return new KeyValue<>(k, v);
-				}
-				else {
-					return new KeyValue<>(k, MessageBuilder.withPayload(v).build());
-				}
-			}
-		}));
+		boundElement.wrap(streamListenerResult.map(KeyValue::new));
 		return new NoOpCloseable();
 	}
 

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/MessageConversionDelegate.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.binder.kstream;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.KeyValueMapper;
+
+import org.springframework.cloud.stream.config.BindingProperties;
+import org.springframework.cloud.stream.config.BindingServiceProperties;
+import org.springframework.cloud.stream.converter.CompositeMessageConverterFactory;
+import org.springframework.messaging.Message;
+import org.springframework.messaging.MessageHeaders;
+import org.springframework.messaging.converter.MessageConverter;
+import org.springframework.messaging.support.MessageBuilder;
+import org.springframework.util.MimeType;
+import org.springframework.util.StringUtils;
+
+/**
+ * @author Soby Chacko
+ */
+public class MessageConversionDelegate {
+
+	private final BindingServiceProperties bindingServiceProperties;
+	private final CompositeMessageConverterFactory compositeMessageConverterFactory;
+
+	public MessageConversionDelegate(BindingServiceProperties bindingServiceProperties,
+									CompositeMessageConverterFactory compositeMessageConverterFactory) {
+		this.bindingServiceProperties = bindingServiceProperties;
+		this.compositeMessageConverterFactory = compositeMessageConverterFactory;
+	}
+
+	public KeyValueMapper<Object, Object, KeyValue<Object, Object>> outboundKeyValueMapper(String name) {
+		BindingProperties bindingProperties = bindingServiceProperties.getBindingProperties(name);
+		String contentType = bindingProperties.getContentType();
+		MessageConverter messageConverter = StringUtils.hasText(contentType) ? compositeMessageConverterFactory
+				.getMessageConverterForType(MimeType.valueOf(contentType))
+				: null;
+
+		return (k, v) -> {
+			Message<?> message = v instanceof Message<?> ? (Message<?>)v :
+					MessageBuilder.withPayload(v).build();
+			Map<String, Object> headers = new HashMap<>(message.getHeaders());
+			if (!StringUtils.isEmpty(contentType)) {
+				headers.put(MessageHeaders.CONTENT_TYPE, contentType);
+			}
+			MessageHeaders messageHeaders = new MessageHeaders(headers);
+			return new KeyValue<>(k,
+					messageConverter.toMessage(message.getPayload(),
+							messageHeaders).getPayload());
+		};
+	}
+
+	@SuppressWarnings("unchecked")
+	public KeyValueMapper<Object, Object, KeyValue<Object, Object>> inboundKeyValueMapper(Class<?> valueClass) {
+		MessageConverter messageConverter = compositeMessageConverterFactory.getMessageConverterForAllRegistered();
+		return (KeyValueMapper) (o, o2) -> {
+			KeyValue<Object, Object> keyValue;
+			if (valueClass.isAssignableFrom(o2.getClass())) {
+				keyValue =  new KeyValue<>(o, o2);
+			}
+			else if (o2 instanceof Message) {
+				if (valueClass.isAssignableFrom(((Message) o2).getPayload().getClass())) {
+					keyValue = new KeyValue<>(o, ((Message) o2).getPayload());
+				}
+				else {
+					keyValue = new KeyValue<>(o, messageConverter.fromMessage((Message) o2, valueClass));
+				}
+			}
+			else if(o2 instanceof String || o2 instanceof byte[]) {
+				Message<Object> message = MessageBuilder.withPayload(o2).build();
+				keyValue =  new KeyValue<>(o, messageConverter.fromMessage(message, valueClass));
+			}
+			else {
+				keyValue =  new KeyValue<>(o, o2);
+			}
+			return keyValue;
+		};
+	}
+}

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/config/KStreamBinderConfiguration.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 the original author or authors.
+ * Copyright 2017-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,79 +18,45 @@ package org.springframework.cloud.stream.binder.kstream.config;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.kafka.common.utils.AppInfoParser;
 import org.apache.kafka.streams.StreamsConfig;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.cloud.stream.binder.kafka.admin.AdminUtilsOperation;
-import org.springframework.cloud.stream.binder.kafka.admin.Kafka09AdminUtilsOperation;
-import org.springframework.cloud.stream.binder.kafka.admin.Kafka10AdminUtilsOperation;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner;
 import org.springframework.cloud.stream.binder.kstream.KStreamBinder;
+import org.springframework.cloud.stream.binder.kstream.MessageConversionDelegate;
 import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Condition;
-import org.springframework.context.annotation.ConditionContext;
-import org.springframework.context.annotation.Conditional;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.core.type.AnnotatedTypeMetadata;
 
 /**
  * @author Marius Bogoevici
+ * @author Gary Russell
+ * @author Soby Chacko
  */
 @Configuration
 @EnableConfigurationProperties(KStreamExtendedBindingProperties.class)
 public class KStreamBinderConfiguration {
 
-	@Autowired(required = false)
-	private AdminUtilsOperation adminUtilsOperation;
-
 	private static final Log logger = LogFactory.getLog(KStreamBinderConfiguration.class);
 
+	@Autowired
+	private KafkaProperties kafkaProperties;
+
 	@Bean
 	public KafkaTopicProvisioner provisioningProvider(KafkaBinderConfigurationProperties binderConfigurationProperties) {
-		return new KafkaTopicProvisioner(binderConfigurationProperties, adminUtilsOperation);
+		return new KafkaTopicProvisioner(binderConfigurationProperties, kafkaProperties);
 	}
 
 	@Bean
 	public KStreamBinder kStreamBinder(KafkaBinderConfigurationProperties binderConfigurationProperties,
-			KafkaTopicProvisioner kafkaTopicProvisioner,
-			KStreamExtendedBindingProperties kStreamExtendedBindingProperties, StreamsConfig streamsConfig) {
-		return new KStreamBinder(binderConfigurationProperties, kafkaTopicProvisioner, kStreamExtendedBindingProperties,
-				streamsConfig);
+									KafkaTopicProvisioner kafkaTopicProvisioner,
+									KStreamExtendedBindingProperties kStreamExtendedBindingProperties, StreamsConfig streamsConfig,
+									MessageConversionDelegate messageConversionDelegate) {
+		KStreamBinder kStreamBinder = new KStreamBinder(binderConfigurationProperties, kafkaTopicProvisioner, kStreamExtendedBindingProperties,
+				streamsConfig, messageConversionDelegate);
+		return kStreamBinder;
 	}
 
-	@Bean(name = "adminUtilsOperation")
-	@Conditional(Kafka09Present.class)
-	@ConditionalOnClass(name = "kafka.admin.AdminUtils")
-	public AdminUtilsOperation kafka09AdminUtilsOperation() {
-		logger.info("AdminUtils selected: Kafka 0.9 AdminUtils");
-		return new Kafka09AdminUtilsOperation();
-	}
-
-	@Bean(name = "adminUtilsOperation")
-	@Conditional(Kafka10Present.class)
-	@ConditionalOnClass(name = "kafka.admin.AdminUtils")
-	public AdminUtilsOperation kafka10AdminUtilsOperation() {
-		logger.info("AdminUtils selected: Kafka 0.10 AdminUtils");
-		return new Kafka10AdminUtilsOperation();
-	}
-
-	static class Kafka10Present implements Condition {
-
-		@Override
-		public boolean matches(ConditionContext conditionContext, AnnotatedTypeMetadata annotatedTypeMetadata) {
-			return AppInfoParser.getVersion().startsWith("0.10");
-		}
-	}
-
-	static class Kafka09Present implements Condition {
-
-		@Override
-		public boolean matches(ConditionContext conditionContext, AnnotatedTypeMetadata annotatedTypeMetadata) {
-			return AppInfoParser.getVersion().startsWith("0.9");
-		}
-	}
 }

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/config/KStreamBinderSupportAutoConfiguration.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 the original author or authors.
+ * Copyright 2017-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,11 +16,12 @@
 
 package org.springframework.cloud.stream.binder.kstream.config;
 
+import java.util.Collection;
 import java.util.Properties;
 
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
-import org.apache.kafka.streams.kstream.KStreamBuilder;
 
 import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.beans.factory.UnsatisfiedDependencyException;
@@ -29,17 +30,20 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties;
 import org.springframework.cloud.stream.binder.kstream.KStreamBoundElementFactory;
 import org.springframework.cloud.stream.binder.kstream.KStreamListenerParameterAdapter;
+import org.springframework.cloud.stream.binder.kstream.KStreamListenerSetupMethodOrchestrator;
 import org.springframework.cloud.stream.binder.kstream.KStreamStreamListenerResultAdapter;
+import org.springframework.cloud.stream.binder.kstream.MessageConversionDelegate;
+import org.springframework.cloud.stream.binding.StreamListenerResultAdapter;
 import org.springframework.cloud.stream.config.BindingServiceProperties;
 import org.springframework.cloud.stream.converter.CompositeMessageConverterFactory;
 import org.springframework.context.annotation.Bean;
-import org.springframework.integration.codec.Codec;
 import org.springframework.kafka.annotation.KafkaStreamsDefaultConfiguration;
-import org.springframework.kafka.core.KStreamBuilderFactoryBean;
+import org.springframework.kafka.core.StreamsBuilderFactoryBean;
 import org.springframework.util.ObjectUtils;
 
 /**
  * @author Marius Bogoevici
+ * @author Soby Chacko
  */
 public class KStreamBinderSupportAutoConfiguration {
 
@@ -49,18 +53,17 @@ public class KStreamBinderSupportAutoConfiguration {
 		return new KafkaBinderConfigurationProperties();
 	}
 
-	@Bean(name = KafkaStreamsDefaultConfiguration.DEFAULT_KSTREAM_BUILDER_BEAN_NAME)
-	public KStreamBuilderFactoryBean defaultKStreamBuilder(
+	@Bean(name = KafkaStreamsDefaultConfiguration.DEFAULT_STREAMS_BUILDER_BEAN_NAME)
+	public StreamsBuilderFactoryBean defaultKafkaStreamBuilder(
 			@Qualifier(KafkaStreamsDefaultConfiguration.DEFAULT_STREAMS_CONFIG_BEAN_NAME) ObjectProvider<StreamsConfig> streamsConfigProvider) {
 		StreamsConfig streamsConfig = streamsConfigProvider.getIfAvailable();
 		if (streamsConfig != null) {
-			KStreamBuilderFactoryBean kStreamBuilderFactoryBean = new KStreamBuilderFactoryBean(streamsConfig);
+			StreamsBuilderFactoryBean kStreamBuilderFactoryBean = new StreamsBuilderFactoryBean(streamsConfig);
 			kStreamBuilderFactoryBean.setPhase(Integer.MAX_VALUE - 500);
 			return kStreamBuilderFactoryBean;
-		}
-		else {
+		} else {
 			throw new UnsatisfiedDependencyException(KafkaStreamsDefaultConfiguration.class.getName(),
-					KafkaStreamsDefaultConfiguration.DEFAULT_KSTREAM_BUILDER_BEAN_NAME, "streamsConfig",
+					KafkaStreamsDefaultConfiguration.DEFAULT_STREAMS_BUILDER_BEAN_NAME, "streamsConfig",
 					"There is no '" + KafkaStreamsDefaultConfiguration.DEFAULT_STREAMS_CONFIG_BEAN_NAME
 							+ "' StreamsConfig bean in the application context.\n");
 		}
@@ -70,10 +73,9 @@ public class KStreamBinderSupportAutoConfiguration {
 	public StreamsConfig streamsConfig(KafkaBinderConfigurationProperties binderConfigurationProperties) {
 		Properties props = new Properties();
 		props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, binderConfigurationProperties.getKafkaConnectionString());
-		props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.ByteArraySerde.class.getName());
-		props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArraySerde.class.getName());
+		props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.ByteArraySerde.class.getName());
+		props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArraySerde.class.getName());
 		props.put(StreamsConfig.APPLICATION_ID_CONFIG, "default");
-		props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, binderConfigurationProperties.getZkConnectionString());
 		if (!ObjectUtils.isEmpty(binderConfigurationProperties.getConfiguration())) {
 			props.putAll(binderConfigurationProperties.getConfiguration());
 		}
@@ -81,23 +83,33 @@ public class KStreamBinderSupportAutoConfiguration {
 	}
 
 	@Bean
-	public KStreamStreamListenerResultAdapter kStreamStreamListenerResultAdapter() {
+	public KStreamStreamListenerResultAdapter kafkaStreamStreamListenerResultAdapter() {
 		return new KStreamStreamListenerResultAdapter();
 	}
 
 	@Bean
-	public KStreamListenerParameterAdapter kStreamListenerParameterAdapter(
-			CompositeMessageConverterFactory compositeMessageConverterFactory) {
-		return new KStreamListenerParameterAdapter(
-				compositeMessageConverterFactory.getMessageConverterForAllRegistered());
+	public KStreamListenerParameterAdapter kafkaStreamListenerParameterAdapter(
+			MessageConversionDelegate messageConversionDelegate) {
+		return new KStreamListenerParameterAdapter(messageConversionDelegate);
 	}
 
 	@Bean
-	public KStreamBoundElementFactory kStreamBindableTargetFactory(KStreamBuilder kStreamBuilder,
-			BindingServiceProperties bindingServiceProperties, Codec codec,
-			CompositeMessageConverterFactory compositeMessageConverterFactory) {
-		return new KStreamBoundElementFactory(kStreamBuilder, bindingServiceProperties, codec,
-				compositeMessageConverterFactory);
+	public KStreamListenerSetupMethodOrchestrator kStreamListenerSetupMethodOrchestrator(
+			KStreamListenerParameterAdapter kafkaStreamListenerParameterAdapter,
+			Collection<StreamListenerResultAdapter> streamListenerResultAdapters){
+		return new KStreamListenerSetupMethodOrchestrator(kafkaStreamListenerParameterAdapter, streamListenerResultAdapters);
+	}
+
+	@Bean
+	public MessageConversionDelegate messageConversionDelegate(BindingServiceProperties bindingServiceProperties,
+															CompositeMessageConverterFactory compositeMessageConverterFactory) {
+		return new MessageConversionDelegate(bindingServiceProperties, compositeMessageConverterFactory);
+	}
+
+	@Bean
+	public KStreamBoundElementFactory kafkaStreamBindableTargetFactory(StreamsBuilder kStreamBuilder,
+																	BindingServiceProperties bindingServiceProperties) {
+		return new KStreamBoundElementFactory(kStreamBuilder, bindingServiceProperties);
 	}
 
 }

spring-cloud-stream-binder-kstream/src/main/java/org/springframework/cloud/stream/binder/kstream/config/KStreamProducerProperties.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 the original author or authors.
+ * Copyright 2017-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package org.springframework.cloud.stream.binder.kstream.config;
 
 /**
  * @author Marius Bogoevici
+ * @author Soby Chacko
  */
 public class KStreamProducerProperties extends KStreamCommonProperties {
 

spring-cloud-stream-binder-kstream/src/test/java/org/springframework/cloud/stream/binder/kstream/KStreamBinderPojoInputAndPrimitiveTypeOutputTests.java

@@ -18,16 +18,13 @@ package org.springframework.cloud.stream.binder.kstream;
 
 import java.util.Map;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.serialization.LongDeserializer;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.streams.kstream.Predicate;
 import org.apache.kafka.streams.kstream.TimeWindows;
-import org.apache.kafka.streams.kstream.Windowed;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -50,22 +47,22 @@ import org.springframework.messaging.handler.annotation.SendTo;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
- *
  * @author Soby Chacko
+ * @author Gary Russell
  */
 public class KStreamBinderPojoInputAndPrimitiveTypeOutputTests {
 
 	@ClassRule
 	public static KafkaEmbedded embeddedKafka = new KafkaEmbedded(1, true, "counts-id");
 
-	private static Consumer<Integer, Long> consumer;
+	private static Consumer<Integer, String> consumer;
 
 	@BeforeClass
 	public static void setUp() throws Exception {
 		Map<String, Object> consumerProps = KafkaTestUtils.consumerProps("group-id", "false", embeddedKafka);
-		consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, LongDeserializer.class.getName());
+		//consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Deserializer.class.getName());
 		consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
-		DefaultKafkaConsumerFactory<Integer, Long> cf = new DefaultKafkaConsumerFactory<>(consumerProps);
+		DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(consumerProps);
 		consumer = cf.createConsumer();
 		embeddedKafka.consumeFromAnEmbeddedTopic(consumer, "counts-id");
 	}
@@ -86,26 +83,29 @@ public class KStreamBinderPojoInputAndPrimitiveTypeOutputTests {
 				"--spring.cloud.stream.kstream.binder.configuration.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
 				"--spring.cloud.stream.kstream.binder.configuration.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
 				"--spring.cloud.stream.bindings.output.producer.headerMode=raw",
-				"--spring.cloud.stream.bindings.output.producer.useNativeEncoding=true",
 				"--spring.cloud.stream.kstream.bindings.output.producer.keySerde=org.apache.kafka.common.serialization.Serdes$IntegerSerde",
-				"--spring.cloud.stream.kstream.bindings.output.producer.valueSerde=org.apache.kafka.common.serialization.Serdes$LongSerde",
 				"--spring.cloud.stream.bindings.input.consumer.headerMode=raw",
 				"--spring.cloud.stream.kstream.binder.brokers=" + embeddedKafka.getBrokersAsString(),
 				"--spring.cloud.stream.kstream.binder.zkNodes=" + embeddedKafka.getZookeeperConnectionString());
-		receiveAndValidateFoo(context);
-		context.close();
+		try {
+			receiveAndValidateFoo(context);
+		} finally {
+			context.close();
+		}
 	}
 
-	private void receiveAndValidateFoo(ConfigurableApplicationContext context) throws Exception{
+	private void receiveAndValidateFoo(ConfigurableApplicationContext context) throws Exception {
 		Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
 		DefaultKafkaProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
 		KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf, true);
 		template.setDefaultTopic("foos");
 		template.sendDefault("{\"id\":\"123\"}");
-		ConsumerRecord<Integer, Long> cr = KafkaTestUtils.getSingleRecord(consumer, "counts-id");
+		ConsumerRecord<Integer, String> cr = KafkaTestUtils.getSingleRecord(consumer, "counts-id");
 
 		assertThat(cr.key().equals(123));
-		assertThat(cr.value().equals(1L));
+		ObjectMapper om = new ObjectMapper();
+		Long aLong = om.readValue(cr.value(), Long.class);
+		assertThat(aLong.equals(1L));
 	}
 
 	@EnableBinding(KStreamProcessor.class)
@@ -116,30 +116,12 @@ public class KStreamBinderPojoInputAndPrimitiveTypeOutputTests {
 		@SendTo("output")
 		public KStream<Integer, Long> process(KStream<Object, Product> input) {
 			return input
-					.filter(new Predicate<Object, Product>() {
-
-						@Override
-						public boolean test(Object key, Product product) {
-							return product.getId() == 123;
-						}
-					})
-					.map(new KeyValueMapper<Object, Product, KeyValue<Product, Product>>() {
-
-						@Override
-						public KeyValue<Product, Product> apply(Object key, Product value) {
-							return new KeyValue<>(value, value);
-						}
-					})
+					.filter((key, product) -> product.getId() == 123)
+					.map((key, value) -> new KeyValue<>(value, value))
 					.groupByKey(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))
 					.count(TimeWindows.of(5000), "id-count-store")
 					.toStream()
-					.map(new KeyValueMapper<Windowed<Product>, Long, KeyValue<Integer, Long>>() {
-
-						@Override
-						public KeyValue<Integer, Long> apply(Windowed<Product> key, Long value) {
-							return new KeyValue<>(key.key().id, value);
-						}
-					});
+					.map((key, value) -> new KeyValue<>(key.key().id, value));
 		}
 	}
 

spring-cloud-stream-binder-kstream/src/test/java/org/springframework/cloud/stream/binder/kstream/KStreamBinderWordCountIntegrationTests.java

@@ -18,7 +18,6 @@ package org.springframework.cloud.stream.binder.kstream;
 
 import java.util.Arrays;
 import java.util.Date;
-import java.util.List;
 import java.util.Map;
 
 import org.apache.kafka.clients.consumer.Consumer;
@@ -27,10 +26,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.kstream.TimeWindows;
-import org.apache.kafka.streams.kstream.ValueMapper;
-import org.apache.kafka.streams.kstream.Windowed;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -39,9 +35,11 @@ import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.stream.annotation.EnableBinding;
 import org.springframework.cloud.stream.annotation.StreamListener;
 import org.springframework.cloud.stream.binder.kstream.annotations.KStreamProcessor;
+import org.springframework.cloud.stream.binder.kstream.config.KStreamApplicationSupportProperties;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaProducerFactory;
@@ -53,9 +51,9 @@ import org.springframework.messaging.handler.annotation.SendTo;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
- *
  * @author Marius Bogoevici
  * @author Soby Chacko
+ * @author Gary Russell
  */
 public class KStreamBinderWordCountIntegrationTests {
 
@@ -91,17 +89,19 @@ public class KStreamBinderWordCountIntegrationTests {
 				"--spring.cloud.stream.kstream.binder.configuration.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
 				"--spring.cloud.stream.kstream.binder.configuration.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
 				"--spring.cloud.stream.bindings.output.producer.headerMode=raw",
-				"--spring.cloud.stream.bindings.output.producer.useNativeEncoding=true",
 				"--spring.cloud.stream.bindings.input.consumer.headerMode=raw",
 				"--spring.cloud.stream.kstream.timeWindow.length=5000",
 				"--spring.cloud.stream.kstream.timeWindow.advanceBy=0",
 				"--spring.cloud.stream.kstream.binder.brokers=" + embeddedKafka.getBrokersAsString(),
 				"--spring.cloud.stream.kstream.binder.zkNodes=" + embeddedKafka.getZookeeperConnectionString());
-		receiveAndValidate(context);
-		context.close();
+		try {
+			receiveAndValidate(context);
+		} finally {
+			context.close();
+		}
 	}
 
-	private void receiveAndValidate(ConfigurableApplicationContext context) throws Exception{
+	private void receiveAndValidate(ConfigurableApplicationContext context) throws Exception {
 		Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
 		DefaultKafkaProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
 		KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf, true);
@@ -113,6 +113,7 @@ public class KStreamBinderWordCountIntegrationTests {
 
 	@EnableBinding(KStreamProcessor.class)
 	@EnableAutoConfiguration
+	@EnableConfigurationProperties(KStreamApplicationSupportProperties.class)
 	public static class WordCountProcessorApplication {
 
 		@Autowired
@@ -123,30 +124,12 @@ public class KStreamBinderWordCountIntegrationTests {
 		public KStream<?, WordCount> process(KStream<Object, String> input) {
 
 			return input
-					.flatMapValues(new ValueMapper<String, Iterable<String>>() {
-
-						@Override
-						public List<String> apply(String value) {
-							return Arrays.asList(value.toLowerCase().split("\\W+"));
-						}
-					})
-					.map(new KeyValueMapper<Object, String, KeyValue<String, String>>() {
-
-						@Override
-						public KeyValue<String, String> apply(Object key, String value) {
-							return new KeyValue<>(value, value);
-						}
-					})
+					.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
+					.map((key, value) -> new KeyValue<>(value, value))
 					.groupByKey(Serdes.String(), Serdes.String())
-					.count(timeWindows, "WordCounts")
+					.count(timeWindows, "foo-WordCounts")
 					.toStream()
-					.map(new KeyValueMapper<Windowed<String>, Long, KeyValue<Object, WordCount>>() {
-
-						@Override
-						public KeyValue<Object, WordCount> apply(Windowed<String> key, Long value) {
-							return new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())));
-						}
-					});
+					.map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))));
 		}
 
 	}

spring-cloud-stream-binder-kstream/src/test/java/org/springframework/cloud/stream/binder/kstream/KStreamInteractiveQueryIntegrationTests.java

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.springframework.cloud.stream.binder.kstream;
 
 import java.util.Map;
@@ -24,8 +25,6 @@ import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.streams.kstream.Predicate;
 import org.apache.kafka.streams.state.QueryableStoreTypes;
 import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
 import org.junit.AfterClass;
@@ -43,8 +42,8 @@ import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaProducerFactory;
-import org.springframework.kafka.core.KStreamBuilderFactoryBean;
 import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.core.StreamsBuilderFactoryBean;
 import org.springframework.kafka.support.serializer.JsonSerde;
 import org.springframework.kafka.test.rule.KafkaEmbedded;
 import org.springframework.kafka.test.utils.KafkaTestUtils;
@@ -54,6 +53,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 /**
  * @author Soby Chacko
+ * @author Gary Russell
  */
 public class KStreamInteractiveQueryIntegrationTests {
 
@@ -87,15 +87,17 @@ public class KStreamInteractiveQueryIntegrationTests {
 				"--spring.cloud.stream.kstream.binder.configuration.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
 				"--spring.cloud.stream.kstream.binder.configuration.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
 				"--spring.cloud.stream.bindings.output.producer.headerMode=raw",
-				"--spring.cloud.stream.bindings.output.producer.useNativeEncoding=true",
 				"--spring.cloud.stream.bindings.input.consumer.headerMode=raw",
 				"--spring.cloud.stream.kstream.binder.brokers=" + embeddedKafka.getBrokersAsString(),
 				"--spring.cloud.stream.kstream.binder.zkNodes=" + embeddedKafka.getZookeeperConnectionString());
-		receiveAndValidateFoo(context);
-		context.close();
+		try {
+			receiveAndValidateFoo(context);
+		} finally {
+			context.close();
+		}
 	}
 
-	private void receiveAndValidateFoo(ConfigurableApplicationContext context) throws Exception{
+	private void receiveAndValidateFoo(ConfigurableApplicationContext context) throws Exception {
 		Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
 		DefaultKafkaProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
 		KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf, true);
@@ -113,49 +115,30 @@ public class KStreamInteractiveQueryIntegrationTests {
 	public static class ProductCountApplication {
 
 		@Autowired
-		private KStreamBuilderFactoryBean kStreamBuilderFactoryBean;
+		private StreamsBuilderFactoryBean kStreamBuilderFactoryBean;
 
 		@StreamListener("input")
 		@SendTo("output")
 		public KStream<?, String> process(KStream<Object, Product> input) {
 
 			return input
-					.filter(new Predicate<Object, Product>() {
-
-						@Override
-						public boolean test(Object key, Product product) {
-							return product.getId() == 123;
-						}
-					})
-					.map(new KeyValueMapper<Object, Product, KeyValue<Integer, Product>>() {
-
-						@Override
-						public KeyValue<Integer, Product> apply(Object key, Product value) {
-							return new KeyValue<>(value.id, value);
-						}
-					})
+					.filter((key, product) -> product.getId() == 123)
+					.map((key, value) -> new KeyValue<>(value.id, value))
 					.groupByKey(new Serdes.IntegerSerde(), new JsonSerde<>(Product.class))
 					.count("prod-id-count-store")
 					.toStream()
-					.map(new KeyValueMapper<Integer, Long, KeyValue<Object, String>>() {
-
-						@Override
-						public KeyValue<Object, String> apply(Integer key, Long value) {
-							return new KeyValue<>(null, "Count for product with ID 123: " + value);
-						}
-					});
+					.map((key, value) -> new KeyValue<>(null, "Count for product with ID 123: " + value));
 		}
 
 		@Bean
-		public Foo foo(KStreamBuilderFactoryBean kStreamBuilderFactoryBean) {
+		public Foo foo(StreamsBuilderFactoryBean kStreamBuilderFactoryBean) {
 			return new Foo(kStreamBuilderFactoryBean);
 		}
 
-
 		static class Foo {
-			KStreamBuilderFactoryBean kStreamBuilderFactoryBean;
+			StreamsBuilderFactoryBean kStreamBuilderFactoryBean;
 
-			Foo(KStreamBuilderFactoryBean kStreamBuilderFactoryBean) {
+			Foo(StreamsBuilderFactoryBean kStreamBuilderFactoryBean) {
 				this.kStreamBuilderFactoryBean = kStreamBuilderFactoryBean;
 			}
 
@@ -163,7 +146,7 @@ public class KStreamInteractiveQueryIntegrationTests {
 				KafkaStreams streams = kStreamBuilderFactoryBean.getKafkaStreams();
 				ReadOnlyKeyValueStore<Object, Object> keyValueStore =
 						streams.store("prod-id-count-store", QueryableStoreTypes.keyValueStore());
-				return (Long)keyValueStore.get(id);
+				return (Long) keyValueStore.get(id);
 			}
 		}
 

spring-cloud-stream-binder-kstream/src/test/java/org/springframework/cloud/stream/binder/kstream/KstreamBinderPojoInputStringOutputIntegrationTests.java

@@ -23,10 +23,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.streams.kstream.Predicate;
 import org.apache.kafka.streams.kstream.TimeWindows;
-import org.apache.kafka.streams.kstream.Windowed;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -51,6 +48,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 /**
  * @author Marius Bogoevici
  * @author Soby Chacko
+ * @author Gary Russell
  */
 public class KstreamBinderPojoInputStringOutputIntegrationTests {
 
@@ -84,13 +82,15 @@ public class KstreamBinderPojoInputStringOutputIntegrationTests {
 				"--spring.cloud.stream.kstream.binder.configuration.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
 				"--spring.cloud.stream.kstream.binder.configuration.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
 				"--spring.cloud.stream.bindings.output.producer.headerMode=raw",
-				"--spring.cloud.stream.bindings.output.producer.useNativeEncoding=true",
 				"--spring.cloud.stream.kstream.bindings.output.producer.keySerde=org.apache.kafka.common.serialization.Serdes$IntegerSerde",
 				"--spring.cloud.stream.bindings.input.consumer.headerMode=raw",
 				"--spring.cloud.stream.kstream.binder.brokers=" + embeddedKafka.getBrokersAsString(),
 				"--spring.cloud.stream.kstream.binder.zkNodes=" + embeddedKafka.getZookeeperConnectionString());
-		receiveAndValidateFoo(context);
-		context.close();
+		try {
+			receiveAndValidateFoo(context);
+		} finally {
+			context.close();
+		}
 	}
 
 	private void receiveAndValidateFoo(ConfigurableApplicationContext context) throws Exception {
@@ -112,30 +112,12 @@ public class KstreamBinderPojoInputStringOutputIntegrationTests {
 		public KStream<Integer, String> process(KStream<Object, Product> input) {
 
 			return input
-					.filter(new Predicate<Object, Product>() {
-
-						@Override
-						public boolean test(Object key, Product product) {
-							return product.getId() == 123;
-						}
-					})
-					.map(new KeyValueMapper<Object, Product, KeyValue<Product, Product>>() {
-
-						@Override
-						public KeyValue<Product, Product> apply(Object key, Product value) {
-							return new KeyValue<>(value, value);
-						}
-					})
+					.filter((key, product) -> product.getId() == 123)
+					.map((key, value) -> new KeyValue<>(value, value))
 					.groupByKey(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))
 					.count(TimeWindows.of(5000), "id-count-store")
 					.toStream()
-					.map(new KeyValueMapper<Windowed<Product>, Long, KeyValue<Integer, String>>() {
-
-						@Override
-						public KeyValue<Integer, String> apply(Windowed<Product> key, Long value) {
-							return new KeyValue<>(key.key().id, "Count for product with ID 123: " + value);
-						}
-					});
+					.map((key, value) -> new KeyValue<>(key.key().id, "Count for product with ID 123: " + value));
 		}
 	}
 

spring-cloud-stream-binder-kstream/src/test/java/org/springframework/cloud/stream/binder/kstream/WordCountMultipleBranchesIntegrationTests.java

@@ -0,0 +1,224 @@
+/*
+ * Copyright 2017 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.binder.kstream;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Predicate;
+import org.apache.kafka.streams.kstream.TimeWindows;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.cloud.stream.annotation.EnableBinding;
+import org.springframework.cloud.stream.annotation.Input;
+import org.springframework.cloud.stream.annotation.Output;
+import org.springframework.cloud.stream.annotation.StreamListener;
+import org.springframework.cloud.stream.binder.kstream.config.KStreamApplicationSupportProperties;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
+import org.springframework.kafka.core.DefaultKafkaProducerFactory;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.test.rule.KafkaEmbedded;
+import org.springframework.kafka.test.utils.KafkaTestUtils;
+import org.springframework.messaging.handler.annotation.SendTo;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Marius Bogoevici
+ * @author Soby Chacko
+ * @author Gary Russell
+ */
+public class WordCountMultipleBranchesIntegrationTests {
+
+	@ClassRule
+	public static KafkaEmbedded embeddedKafka = new KafkaEmbedded(1, true, "counts","foo","bar");
+
+	private static Consumer<String, String> consumer;
+
+	@BeforeClass
+	public static void setUp() throws Exception {
+		Map<String, Object> consumerProps = KafkaTestUtils.consumerProps("group", "false", embeddedKafka);
+		consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+		DefaultKafkaConsumerFactory<String, String> cf = new DefaultKafkaConsumerFactory<>(consumerProps);
+		consumer = cf.createConsumer();
+		embeddedKafka.consumeFromEmbeddedTopics(consumer, "counts", "foo", "bar");
+	}
+
+	@AfterClass
+	public static void tearDown() {
+		consumer.close();
+	}
+
+	@Test
+	public void testKstreamWordCountWithStringInputAndPojoOuput() throws Exception {
+		SpringApplication app = new SpringApplication(WordCountProcessorApplication.class);
+		app.setWebEnvironment(false);
+
+		ConfigurableApplicationContext context = app.run("--server.port=0",
+				"--spring.cloud.stream.bindings.input.destination=words",
+				"--spring.cloud.stream.bindings.output1.destination=counts",
+				"--spring.cloud.stream.bindings.output1.contentType=application/json",
+				"--spring.cloud.stream.bindings.output2.destination=foo",
+				"--spring.cloud.stream.bindings.output2.contentType=application/json",
+				"--spring.cloud.stream.bindings.output3.destination=bar",
+				"--spring.cloud.stream.bindings.output3.contentType=application/json",
+				"--spring.cloud.stream.kstream.binder.configuration.commit.interval.ms=1000",
+				"--spring.cloud.stream.kstream.binder.configuration.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
+				"--spring.cloud.stream.kstream.binder.configuration.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde",
+				"--spring.cloud.stream.bindings.output.producer.headerMode=raw",
+				"--spring.cloud.stream.bindings.input.consumer.headerMode=raw",
+				"--spring.cloud.stream.kstream.timeWindow.length=5000",
+				"--spring.cloud.stream.kstream.timeWindow.advanceBy=0",
+				"--spring.cloud.stream.kstream.binder.brokers=" + embeddedKafka.getBrokersAsString(),
+				"--spring.cloud.stream.kstream.binder.zkNodes=" + embeddedKafka.getZookeeperConnectionString());
+		try {
+			receiveAndValidate(context);
+		} finally {
+			context.close();
+		}
+	}
+
+	private void receiveAndValidate(ConfigurableApplicationContext context) throws Exception {
+		Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
+		DefaultKafkaProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
+		KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf, true);
+		template.setDefaultTopic("words");
+		template.sendDefault("english");
+		ConsumerRecord<String, String> cr = KafkaTestUtils.getSingleRecord(consumer, "counts");
+		assertThat(cr.value().contains("\"word\":\"english\",\"count\":1")).isTrue();
+
+		template.sendDefault("french");
+		template.sendDefault("french");
+		cr = KafkaTestUtils.getSingleRecord(consumer, "foo");
+		assertThat(cr.value().contains("\"word\":\"french\",\"count\":2")).isTrue();
+
+		template.sendDefault("spanish");
+		template.sendDefault("spanish");
+		template.sendDefault("spanish");
+		cr = KafkaTestUtils.getSingleRecord(consumer, "bar");
+		assertThat(cr.value().contains("\"word\":\"spanish\",\"count\":3")).isTrue();
+	}
+
+	@EnableBinding(KStreamProcessorX.class)
+	@EnableAutoConfiguration
+	@EnableConfigurationProperties(KStreamApplicationSupportProperties.class)
+	public static class WordCountProcessorApplication {
+
+		@Autowired
+		private TimeWindows timeWindows;
+
+		@StreamListener("input")
+		@SendTo({"output1","output2","output3"})
+		@SuppressWarnings("unchecked")
+		public KStream<?, WordCount>[] process(KStream<Object, String> input) {
+
+			Predicate<Object, WordCount> isEnglish = (k, v) -> v.word.equals("english");
+			Predicate<Object, WordCount> isFrench =  (k, v) -> v.word.equals("french");
+			Predicate<Object, WordCount> isSpanish = (k, v) -> v.word.equals("spanish");
+
+			return input
+					.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
+					.groupBy((key, value) -> value)
+					.windowedBy(timeWindows)
+					.count(Materialized.as("WordCounts-multi"))
+					.toStream()
+					.map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))))
+					.branch(isEnglish, isFrench, isSpanish);
+		}
+	}
+
+	interface KStreamProcessorX {
+
+		@Input("input")
+		KStream<?, ?> input();
+
+		@Output("output1")
+		KStream<?, ?> output1();
+
+		@Output("output2")
+		KStream<?, ?> output2();
+
+		@Output("output3")
+		KStream<?, ?> output3();
+	}
+
+	static class WordCount {
+
+		private String word;
+
+		private long count;
+
+		private Date start;
+
+		private Date end;
+
+		WordCount(String word, long count, Date start, Date end) {
+			this.word = word;
+			this.count = count;
+			this.start = start;
+			this.end = end;
+		}
+
+		public String getWord() {
+			return word;
+		}
+
+		public void setWord(String word) {
+			this.word = word;
+		}
+
+		public long getCount() {
+			return count;
+		}
+
+		public void setCount(long count) {
+			this.count = count;
+		}
+
+		public Date getStart() {
+			return start;
+		}
+
+		public void setStart(Date start) {
+			this.start = start;
+		}
+
+		public Date getEnd() {
+			return end;
+		}
+
+		public void setEnd(Date end) {
+			this.end = end;
+		}
+	}
+
+}

update-version.sh

@@ -7,7 +7,7 @@
 
 
 
-lines=$(find . -name 'pom.xml' | xargs egrep "SNAPSHOT" | grep -v regex | wc -l)
+lines=$(find . -name 'pom.xml' | xargs egrep "SNAPSHOT|M[0-9]|RC[0-9]" | grep -v regex | wc -l)
 if [ $lines -eq 0 ]; then
 	echo "No snapshots found"
 else