Include & Exclude paths in project aggregation stage.

Introduce limit(Limit) method to limit query results applying the Limit domain type.

See #4397
Original pull request: #4398

.mvn/wrapper/maven-wrapper.properties

@@ -1,2 +1,2 @@
-#Tue Jun 13 08:54:58 CEST 2023
-distributionUrl=https\://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.2/apache-maven-3.9.2-bin.zip
+#Mon Jul 03 09:49:43 CEST 2023
+distributionUrl=https\://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.3/apache-maven-3.9.3-bin.zip

ci/pipeline.properties

@@ -1,5 +1,5 @@
 # Java versions
-java.main.tag=17.0.6_10-jdk-focal
+java.main.tag=17.0.7_7-jdk-focal
 java.next.tag=20-jdk-jammy
 
 # Docker container images - standard
@@ -7,15 +7,15 @@ docker.java.main.image=harbor-repo.vmware.com/dockerhub-proxy-cache/library/ecli
 docker.java.next.image=harbor-repo.vmware.com/dockerhub-proxy-cache/library/eclipse-temurin:${java.next.tag}
 
 # Supported versions of MongoDB
-docker.mongodb.4.4.version=4.4.18
-docker.mongodb.5.0.version=5.0.14
-docker.mongodb.6.0.version=6.0.4
+docker.mongodb.4.4.version=4.4.22
+docker.mongodb.5.0.version=5.0.18
+docker.mongodb.6.0.version=6.0.7
 
 # Supported versions of Redis
-docker.redis.6.version=6.2.10
+docker.redis.6.version=6.2.12
 
 # Supported versions of Cassandra
-docker.cassandra.3.version=3.11.14
+docker.cassandra.3.version=3.11.15
 
 # Docker environment settings
 docker.java.inside.basic=-v $HOME:/tmp/jenkins-home

pom.xml

@@ -5,7 +5,7 @@
 
 	<groupId>org.springframework.data</groupId>
 	<artifactId>spring-data-mongodb-parent</artifactId>
-	<version>4.2.x-4426-SNAPSHOT</version>
+	<version>4.2.x-4428-SNAPSHOT</version>
 	<packaging>pom</packaging>
 
 	<name>Spring Data MongoDB</name>

spring-data-mongodb-benchmarks/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<groupId>org.springframework.data</groupId>
 		<artifactId>spring-data-mongodb-parent</artifactId>
-		<version>4.2.x-4426-SNAPSHOT</version>
+		<version>4.2.x-4428-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

spring-data-mongodb-distribution/pom.xml

@@ -15,7 +15,7 @@
 	<parent>
 		<groupId>org.springframework.data</groupId>
 		<artifactId>spring-data-mongodb-parent</artifactId>
-		<version>4.2.x-4426-SNAPSHOT</version>
+		<version>4.2.x-4428-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

spring-data-mongodb/pom.xml

@@ -13,7 +13,7 @@
 	<parent>
 		<groupId>org.springframework.data</groupId>
 		<artifactId>spring-data-mongodb-parent</artifactId>
-		<version>4.2.x-4426-SNAPSHOT</version>
+		<version>4.2.x-4428-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/aggregation/ProjectionOperation.java

@@ -167,6 +167,46 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 		return new ProjectionOperation(this.projections, projections);
 	}
 
+	/**
+	 * @param path
+	 * @return
+	 * @since 4.2
+	 */
+	public ProjectionOperation andIncludePath(String path) {
+		return andIncludePaths(new String[] { path });
+	}
+
+	/**
+	 * @param paths
+	 * @return
+	 * @since 4.2
+	 */
+	public ProjectionOperation andIncludePaths(String... paths) {
+
+		List<FieldProjection> projections = FieldProjection.from(Fields.fields(paths), 1);
+		return new ProjectionOperation(this.projections, projections);
+	}
+
+	/**
+	 * @param path
+	 * @return
+	 * @since 4.2
+	 */
+	public ProjectionOperation andExcludePath(String path) {
+		return andExcludePaths(new String[] { path });
+	}
+
+	/**
+	 * @param paths
+	 * @return
+	 * @since 4.2
+	 */
+	public ProjectionOperation andExcludePaths(String... paths) {
+
+		List<FieldProjection> projections = FieldProjection.from(Fields.fields(paths), 0);
+		return new ProjectionOperation(this.projections, projections);
+	}
+
 	/**
 	 * Includes the given fields into the projection.
 	 *
@@ -495,8 +535,8 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 			if (value instanceof AggregationExpression) {
 				return this.operation.and(new ExpressionProjection(Fields.field(alias, alias), (AggregationExpression) value));
 			}
-
-			return this.operation.and(new FieldProjection(Fields.field(alias, getRequiredName()), null));
+			Field field = Fields.field(alias, getRequiredName());
+			return this.operation.and(new FieldProjection(field, null));
 		}
 
 		@Override
@@ -1329,6 +1369,11 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 
 			private final Field field;
 			private final @Nullable Object value;
+			private final ProjectOn projectOn;
+
+			enum ProjectOn {
+				PATH, NAME
+			}
 
 			/**
 			 * Creates a new {@link FieldProjection} for the field of the given name, assigning the given value.
@@ -1341,11 +1386,16 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 			}
 
 			private FieldProjection(Field field, @Nullable Object value) {
+				this(field, value, value instanceof Integer ? ProjectOn.PATH : ProjectOn.NAME);
+			}
+
+			private FieldProjection(Field field, @Nullable Object value, ProjectOn project) {
 
 				super(new ExposedField(field.getName(), true));
 
 				this.field = field;
 				this.value = value;
+				this.projectOn = project;
 			}
 
 			/**
@@ -1372,7 +1422,8 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 				List<FieldProjection> projections = new ArrayList<FieldProjection>();
 
 				for (Field field : fields) {
-					projections.add(new FieldProjection(field, value));
+					projections
+							.add(new FieldProjection(field, value, value instanceof Integer ? ProjectOn.PATH : ProjectOn.NAME));
 				}
 
 				return projections;
@@ -1382,12 +1433,13 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 			 * @return {@literal true} if this field is excluded.
 			 */
 			public boolean isExcluded() {
-				return Boolean.FALSE.equals(value);
+				return Boolean.FALSE.equals(value) || value instanceof Number number && number.intValue() == 0;
 			}
 
 			@Override
 			public Document toDocument(AggregationOperationContext context) {
-				return new Document(field.getName(), renderFieldValue(context));
+				return new Document(ProjectOn.NAME.equals(projectOn) ? field.getName() : context.getReference(field.getTarget()).getRaw(),
+						renderFieldValue(context));
 			}
 
 			private Object renderFieldValue(AggregationOperationContext context) {

spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/convert/QueryMapper.java

@@ -1089,6 +1089,7 @@ public class QueryMapper {
 	protected static class MetadataBackedField extends Field {
 
 		private static final Pattern POSITIONAL_PARAMETER_PATTERN = Pattern.compile("\\.\\$(\\[.*?\\])?");
+		private static final Pattern NUMERIC_SEGMENT = Pattern.compile("\\d+");
 		private static final String INVALID_ASSOCIATION_REFERENCE = "Invalid path reference %s; Associations can only be pointed to directly or via their id property";
 
 		private final MongoPersistentEntity<?> entity;
@@ -1338,22 +1339,24 @@ public class QueryMapper {
 				return source;
 			}
 
-			List<String> path = new ArrayList<>();
+			List<String> path = new ArrayList<>(segments.length);
 
 			/* always start from a property, so we can skip the first segment.
 			   from there remove any position placeholder */
-			for (String segment : Arrays.copyOfRange(segments, 1, segments.length)) {
+			for(int i=1; i < segments.length; i++) {
+				String segment = segments[i];
 				if (segment.startsWith("[") && segment.endsWith("]")) {
 					continue;
 				}
-				if (segment.matches("\\d+")) {
+				if (NUMERIC_SEGMENT.matcher(segment).matches()) {
 					continue;
 				}
 				path.add(segment);
 			}
 
 			// when property is followed only by placeholders eg. 'values.0.3.90'
-			if (path.isEmpty()) {
+			// or when there is no difference in the number of segments
+			if (path.isEmpty() || segments.length == path.size() + 1) {
 				return source;
 			}
 

spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java

@@ -31,6 +31,7 @@ import java.util.Set;
 
 import org.bson.Document;
 import org.springframework.data.domain.KeysetScrollPosition;
+import org.springframework.data.domain.Limit;
 import org.springframework.data.domain.OffsetScrollPosition;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.ScrollPosition;
@@ -66,7 +67,7 @@ public class Query implements ReadConcernAware, ReadPreferenceAware {
 	private @Nullable Field fieldSpec = null;
 	private Sort sort = Sort.unsorted();
 	private long skip;
-	private int limit;
+	private Limit limit = Limit.unlimited();
 
 	private KeysetScrollPosition keysetScrollPosition;
 	private @Nullable ReadConcern readConcern;
@@ -155,10 +156,30 @@ public class Query implements ReadConcernAware, ReadPreferenceAware {
 	 * @return this.
 	 */
 	public Query limit(int limit) {
-		this.limit = limit;
+		this.limit = limit > 0 ? Limit.of(limit) : Limit.unlimited();
 		return this;
 	}
 
+	/**
+	 * Limit the number of returned documents to {@link Limit}.
+	 *
+	 * @param limit number of documents to return.
+	 * @return this.
+	 * @since 4.2
+	 */
+	public Query limit(Limit limit) {
+
+		Assert.notNull(limit, "Limit must not be null");
+
+		if (limit.isUnlimited()) {
+			this.limit = limit;
+			return this;
+		}
+
+		// retain zero/negative semantics for unlimited.
+		return limit(limit.max());
+	}
+
 	/**
 	 * Configures the query to use the given hint when being executed. The {@code hint} can either be an index name or a
 	 * json {@link Document} representation.
@@ -254,7 +275,7 @@ public class Query implements ReadConcernAware, ReadPreferenceAware {
 			return this;
 		}
 
-		this.limit = pageable.getPageSize();
+		this.limit = pageable.toLimit();
 		this.skip = pageable.getOffset();
 
 		return with(pageable.getSort());
@@ -457,7 +478,7 @@ public class Query implements ReadConcernAware, ReadPreferenceAware {
 	 * @since 4.1
 	 */
 	public boolean isLimited() {
-		return this.limit > 0;
+		return this.limit.isLimited();
 	}
 
 	/**
@@ -468,7 +489,7 @@ public class Query implements ReadConcernAware, ReadPreferenceAware {
 	 * @see #isLimited()
 	 */
 	public int getLimit() {
-		return this.limit;
+		return limit.isUnlimited() ? 0 : this.limit.max();
 	}
 
 	/**
@@ -683,7 +704,8 @@ public class Query implements ReadConcernAware, ReadPreferenceAware {
 		};
 
 		target.skip = source.getSkip();
-		target.limit = source.getLimit();
+
+		target.limit = source.isLimited() ? Limit.of(source.getLimit()) : Limit.unlimited();
 		target.hint = source.getHint();
 		target.collation = source.getCollation();
 		target.restrictedTypes = new HashSet<>(source.getRestrictedTypes());
@@ -746,7 +768,7 @@ public class Query implements ReadConcernAware, ReadPreferenceAware {
 		result += 31 * nullSafeHashCode(sort);
 		result += 31 * nullSafeHashCode(hint);
 		result += 31 * skip;
-		result += 31 * limit;
+		result += 31 * limit.hashCode();
 		result += 31 * nullSafeHashCode(meta);
 		result += 31 * nullSafeHashCode(collation.orElse(null));
 

spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java

@@ -21,6 +21,7 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
+import org.springframework.data.domain.Limit;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Range;
 import org.springframework.data.domain.ScrollPosition;
@@ -117,6 +118,11 @@ public class ConvertingParameterAccessor implements MongoParameterAccessor {
 		return delegate.getUpdate();
 	}
 
+	@Override
+	public Limit getLimit() {
+		return delegate.getLimit();
+	}
+
 	/**
 	 * Converts the given value with the underlying {@link MongoWriter}.
 	 *

spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/aggregation/AggregationUnitTests.java

@@ -36,6 +36,7 @@ import org.springframework.data.mongodb.core.aggregation.ProjectionOperationUnit
 import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
 import org.springframework.data.mongodb.core.convert.NoOpDbRefResolver;
 import org.springframework.data.mongodb.core.convert.QueryMapper;
+import org.springframework.data.mongodb.core.mapping.Field;
 import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
 import org.springframework.data.mongodb.core.query.Criteria;
 
@@ -653,6 +654,58 @@ public class AggregationUnitTests {
 		assertThat(documents.get(2)).isEqualTo("{ $sort : { 'serial_number' : -1, 'label_name' : -1 } }");
 	}
 
+	@Test // GH-4428
+	void projectIncludePath() {
+
+		MongoMappingContext mappingContext = new MongoMappingContext();
+		RelaxedTypeBasedAggregationOperationContext context = new RelaxedTypeBasedAggregationOperationContext(
+				Root.class, mappingContext,
+				new QueryMapper(new MappingMongoConverter(NoOpDbRefResolver.INSTANCE, mappingContext)));
+
+		assertThat(project("flat").andIncludePath("list.element").toDocument(context)).isEqualTo(
+				Document.parse("""
+						{
+						  "$project": {
+						    "flat": 1,
+						    "list.elE_m_enT": 1
+						  }
+						}""")
+		);
+	}
+
+	@Test // GH-4428
+	void projectExcludePath() {
+
+		MongoMappingContext mappingContext = new MongoMappingContext();
+		RelaxedTypeBasedAggregationOperationContext context = new RelaxedTypeBasedAggregationOperationContext(
+				Root.class, mappingContext,
+				new QueryMapper(new MappingMongoConverter(NoOpDbRefResolver.INSTANCE, mappingContext)));
+
+		assertThat(project("flat").andExcludePath("list.element").toDocument(context)).isEqualTo(
+				Document.parse("""
+						{
+						  "$project": {
+						    "flat": 1,
+						    "list.elE_m_enT": 0
+						  }
+						}""")
+		);
+	}
+
+
+	static class Root {
+		String flat;
+		List<Nested> list;
+	}
+
+	static class Nested {
+
+		@Field("elE_m_enT")
+		int element;
+		String description;
+
+	}
+
 	private Document extractPipelineElement(Document agg, int index, String operation) {
 
 		List<Document> pipeline = (List<Document>) agg.get("pipeline");

spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/query/QueryTests.java

@@ -22,6 +22,7 @@ import static org.springframework.data.mongodb.core.query.Query.*;
 import org.bson.Document;
 import org.junit.jupiter.api.Test;
 import org.springframework.aop.framework.ProxyFactory;
+import org.springframework.data.domain.Limit;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.domain.Sort.Direction;
 import org.springframework.data.domain.Sort.Order;
@@ -97,6 +98,18 @@ class QueryTests {
 		assertThat(q.getQueryObject()).isEqualTo(Document
 				.parse("{ \"name\" : { \"$gte\" : \"M\" , \"$lte\" : \"T\"} , \"age\" : { \"$not\" : { \"$gt\" : 22}}}"));
 		assertThat(q.getLimit()).isEqualTo(50);
+
+		q.limit(Limit.unlimited());
+		assertThat(q.getLimit()).isZero();
+		assertThat(q.isLimited()).isFalse();
+
+		q.limit(Limit.of(10));
+		assertThat(q.getLimit()).isEqualTo(10);
+		assertThat(q.isLimited()).isTrue();
+
+		q.limit(Limit.of(-1));
+		assertThat(q.getLimit()).isZero();
+		assertThat(q.isLimited()).isFalse();
 	}
 
 	@Test

spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java

@@ -213,6 +213,17 @@ public abstract class AbstractPersonRepositoryIntegrationTests implements Dirtie
 		assertThat(page).contains(carter);
 	}
 
+	@Test // GH-4397
+	void appliesLimitToScrollingCorrectly() {
+
+		Window<Person> page = repository.findByLastnameLikeOrderByLastnameAscFirstnameAsc("*a*",
+				ScrollPosition.keyset(), Limit.of(2));
+
+		assertThat(page.isLast()).isFalse();
+		assertThat(page.size()).isEqualTo(2);
+		assertThat(page).contains(carter);
+	}
+
 	@Test // GH-4308
 	void appliesScrollPositionWithProjectionCorrectly() {
 
@@ -236,6 +247,14 @@ public abstract class AbstractPersonRepositoryIntegrationTests implements Dirtie
 		assertThat(page).contains(carter, stefan);
 	}
 
+	@Test // GH-4397
+	void executesFinderCorrectlyWithSortAndLimit() {
+
+		List<Person> page = repository.findByLastnameLike("*a*", Sort.by(Direction.ASC, "lastname", "firstname"), Limit.of(2));
+
+		assertThat(page).containsExactly(carter, stefan);
+	}
+
 	@Test
 	void executesPagedFinderWithAnnotatedQueryCorrectly() {
 

spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java

@@ -23,6 +23,7 @@ import java.util.UUID;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
+import org.springframework.data.domain.Limit;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Range;
@@ -126,6 +127,9 @@ public interface PersonRepository extends MongoRepository<Person, String>, Query
 	Window<Person> findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc(String lastname,
 			ScrollPosition scrollPosition);
 
+	Window<Person> findByLastnameLikeOrderByLastnameAscFirstnameAsc(String lastname,
+			ScrollPosition scrollPosition, Limit limit);
+
 	/**
 	 * Returns a scroll of {@link Person}s applying projections with a lastname matching the given one (*-wildcards
 	 * supported).
@@ -145,6 +149,8 @@ public interface PersonRepository extends MongoRepository<Person, String>, Query
 	 */
 	Page<Person> findByLastnameLike(String lastname, Pageable pageable);
 
+	List<Person> findByLastnameLike(String lastname, Sort sort, Limit limit);
+
 	@Query("{ 'lastname' : { '$regex' : '?0', '$options' : 'i'}}")
 	Page<Person> findByLastnameLikeWithPageable(String lastname, Pageable pageable);
 

spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/AbstractMongoQueryUnitTests.java

@@ -36,6 +36,7 @@ import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
+import org.springframework.data.domain.Limit;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
@@ -493,6 +494,30 @@ class AbstractMongoQueryUnitTests {
 		assertThat(captor.getValue().getHint()).isEqualTo("idx-ln");
 	}
 
+	@Test // GH-4397
+	void limitShouldBeAppliedToQuery() {
+
+		createQueryForMethod("findWithLimit", String.class, Limit.class).execute(new Object[] { "dalinar", Limit.of(42) });
+
+		ArgumentCaptor<Query> captor = ArgumentCaptor.forClass(Query.class);
+		verify(withQueryMock).matching(captor.capture());
+
+		assertThat(captor.getValue().getLimit()).isEqualTo(42);
+	}
+
+	@Test // GH-4397
+	void sortAndLimitShouldBeAppliedToQuery() {
+
+		createQueryForMethod("findWithSortAndLimit", String.class, Sort.class, Limit.class)
+				.execute(new Object[] { "dalinar", Sort.by("fn"), Limit.of(42) });
+
+		ArgumentCaptor<Query> captor = ArgumentCaptor.forClass(Query.class);
+		verify(withQueryMock).matching(captor.capture());
+
+		assertThat(captor.getValue().getLimit()).isEqualTo(42);
+		assertThat(captor.getValue().getSortObject()).isEqualTo(new Document("fn", 1));
+	}
+
 	private MongoQueryFake createQueryForMethod(String methodName, Class<?>... paramTypes) {
 		return createQueryForMethod(Repo.class, methodName, paramTypes);
 	}
@@ -614,6 +639,10 @@ class AbstractMongoQueryUnitTests {
 
 		@Hint("idx-fn")
 		void findWithHintByFirstname(String firstname);
+
+		List<Person> findWithLimit(String firstname, Limit limit);
+
+		List<Person> findWithSortAndLimit(String firstname, Sort sort, Limit limit);
 	}
 
 	// DATAMONGO-1872