diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java index bc95f0cfe..a740fcc53 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java @@ -66,7 +66,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DeleteContext; import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext; import org.springframework.data.mongodb.core.QueryOperations.QueryContext; import org.springframework.data.mongodb.core.QueryOperations.UpdateContext; -import org.springframework.data.mongodb.core.ScrollUtils.KeySetScrollQuery; +import org.springframework.data.mongodb.core.ScrollUtils.KeysetScrollQuery; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; @@ -876,14 +876,14 @@ public class MongoTemplate if (query.hasKeyset()) { - KeySetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, + KeysetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, operations.getIdPropertyName(sourceClass)); List result = doFind(collectionName, createDelegate(query), keysetPaginationQuery.query(), keysetPaginationQuery.fields(), sourceClass, new QueryCursorPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback); - return ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), result, sourceClass, operations); + return ScrollUtils.createWindow(query, result, sourceClass, operations); } List result = doFind(collectionName, createDelegate(query), query.getQueryObject(), query.getFieldsObject(), diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java index 21869b6d6..594ca3541 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java @@ -80,7 +80,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DeleteContext; import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext; import org.springframework.data.mongodb.core.QueryOperations.QueryContext; import org.springframework.data.mongodb.core.QueryOperations.UpdateContext; -import org.springframework.data.mongodb.core.ScrollUtils.KeySetScrollQuery; +import org.springframework.data.mongodb.core.ScrollUtils.KeysetScrollQuery; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; @@ -855,14 +855,14 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati if (query.hasKeyset()) { - KeySetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, + KeysetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, operations.getIdPropertyName(sourceClass)); Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), keysetPaginationQuery.query(), keysetPaginationQuery.fields(), sourceClass, new QueryFindPublisherPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback).collectList(); - return result.map(it -> ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), it, sourceClass, operations)); + return result.map(it -> ScrollUtils.createWindow(query, it, sourceClass, operations)); } Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), query.getQueryObject(), diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java index 112f95270..ecd7aaf81 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java @@ -16,6 +16,7 @@ package org.springframework.data.mongodb.core; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.IntFunction; @@ -45,84 +46,24 @@ class ScrollUtils { * @param idPropertyName * @return */ - static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPropertyName) { + static KeysetScrollQuery createKeysetPaginationQuery(Query query, String idPropertyName) { KeysetScrollPosition keyset = query.getKeyset(); - Map keysetValues = keyset.getKeys(); - Document queryObject = query.getQueryObject(); + KeysetScrollDirector director = KeysetScrollDirector.of(keyset.getDirection()); + Document sortObject = director.getSortObject(idPropertyName, query); + Document fieldsObject = director.getFieldsObject(query.getFieldsObject(), sortObject); + Document queryObject = director.createQuery(keyset, query.getQueryObject(), sortObject); - Document sortObject = query.isSorted() ? query.getSortObject() : new Document(); - sortObject.put(idPropertyName, 1); - - // make sure we can extract the keyset - Document fieldsObject = query.getFieldsObject(); - if (!fieldsObject.isEmpty()) { - for (String field : sortObject.keySet()) { - fieldsObject.put(field, 1); - } - } - - List or = (List) queryObject.getOrDefault("$or", new ArrayList<>()); - List sortKeys = new ArrayList<>(sortObject.keySet()); - - if (!keysetValues.isEmpty() && !keysetValues.keySet().containsAll(sortKeys)) { - throw new IllegalStateException("KeysetScrollPosition does not contain all keyset values"); - } - - // first query doesn't come with a keyset - if (!keysetValues.isEmpty()) { - - // build matrix query for keyset paging that contains sort^2 queries - // reflecting a query that follows sort order semantics starting from the last returned keyset - for (int i = 0; i < sortKeys.size(); i++) { - - Document sortConstraint = new Document(); - - for (int j = 0; j < sortKeys.size(); j++) { - - String sortSegment = sortKeys.get(j); - int sortOrder = sortObject.getInteger(sortSegment); - Object o = keysetValues.get(sortSegment); - - if (j >= i) { // tail segment - if (o instanceof BsonNull) { - throw new IllegalStateException( - "Cannot resume from KeysetScrollPosition. Offending key: '%s' is 'null'".formatted(sortSegment)); - } - sortConstraint.put(sortSegment, new Document(getComparator(sortOrder, keyset.getDirection()), o)); - break; - } - - sortConstraint.put(sortSegment, o); - } - - if (!sortConstraint.isEmpty()) { - or.add(sortConstraint); - } - } - } - - if (!or.isEmpty()) { - queryObject.put("$or", or); - } - - return new KeySetScrollQuery(queryObject, fieldsObject, sortObject); + return new KeysetScrollQuery(queryObject, fieldsObject, sortObject); } - private static String getComparator(int sortOrder, Direction direction) { + static Window createWindow(Query query, List result, Class sourceType, EntityOperations operations) { - // use gte/lte to include the object at the cursor/keyset so that - // we can include it in the result to check whether there is a next object. - // It needs to be filtered out later on. - if (direction == Direction.Backward) { - return sortOrder == 0 ? "$gte" : "$lte"; - } + Document sortObject = query.getSortObject(); + KeysetScrollPosition keyset = query.getKeyset(); + KeysetScrollDirector director = KeysetScrollDirector.of(keyset.getDirection()); - return sortOrder == 1 ? "$gt" : "$lt"; - } - - static Window createWindow(Document sortObject, int limit, List result, Class sourceType, - EntityOperations operations) { + director.postPostProcessResults(result); IntFunction positionFunction = value -> { @@ -133,7 +74,7 @@ class ScrollUtils { return KeysetScrollPosition.of(keys); }; - return createWindow(result, limit, positionFunction); + return createWindow(result, query.getLimit(), positionFunction); } static Window createWindow(List result, int limit, IntFunction positionFunction) { @@ -153,8 +94,145 @@ class ScrollUtils { return result; } - record KeySetScrollQuery(Document query, Document fields, Document sort) { + record KeysetScrollQuery(Document query, Document fields, Document sort) { } + /** + * Director for keyset scrolling. + */ + static class KeysetScrollDirector { + + private static final KeysetScrollDirector forward = new KeysetScrollDirector(); + private static final KeysetScrollDirector reverse = new ReverseKeysetScrollDirector(); + + /** + * Factory method to obtain the right {@link KeysetScrollDirector}. + * + * @param direction + * @return + */ + public static KeysetScrollDirector of(KeysetScrollPosition.Direction direction) { + return direction == Direction.Forward ? forward : reverse; + } + + public Document getSortObject(String idPropertyName, Query query) { + + Document sortObject = query.isSorted() ? query.getSortObject() : new Document(); + sortObject.put(idPropertyName, 1); + + return sortObject; + } + + public Document getFieldsObject(Document fieldsObject, Document sortObject) { + + // make sure we can extract the keyset + if (!fieldsObject.isEmpty()) { + for (String field : sortObject.keySet()) { + fieldsObject.put(field, 1); + } + } + + return fieldsObject; + } + + public Document createQuery(KeysetScrollPosition keyset, Document queryObject, Document sortObject) { + + Map keysetValues = keyset.getKeys(); + List or = (List) queryObject.getOrDefault("$or", new ArrayList<>()); + List sortKeys = new ArrayList<>(sortObject.keySet()); + + // first query doesn't come with a keyset + if (keysetValues.isEmpty()) { + return queryObject; + } + + if (!keysetValues.keySet().containsAll(sortKeys)) { + throw new IllegalStateException("KeysetScrollPosition does not contain all keyset values"); + } + + // build matrix query for keyset paging that contains sort^2 queries + // reflecting a query that follows sort order semantics starting from the last returned keyset + for (int i = 0; i < sortKeys.size(); i++) { + + Document sortConstraint = new Document(); + + for (int j = 0; j < sortKeys.size(); j++) { + + String sortSegment = sortKeys.get(j); + int sortOrder = sortObject.getInteger(sortSegment); + Object o = keysetValues.get(sortSegment); + + if (j >= i) { // tail segment + if (o instanceof BsonNull) { + throw new IllegalStateException( + "Cannot resume from KeysetScrollPosition. Offending key: '%s' is 'null'".formatted(sortSegment)); + } + sortConstraint.put(sortSegment, new Document(getComparator(sortOrder), o)); + break; + } + + sortConstraint.put(sortSegment, o); + } + + if (!sortConstraint.isEmpty()) { + or.add(sortConstraint); + } + } + + if (!or.isEmpty()) { + queryObject.put("$or", or); + } + + return queryObject; + } + + public void postPostProcessResults(List result) { + + } + + protected String getComparator(int sortOrder) { + return sortOrder == 1 ? "$gt" : "$lt"; + } + } + + /** + * Reverse scrolling director variant applying {@link KeysetScrollPosition.Direction#Backward}. In reverse scrolling, + * we need to flip directions for the actual query so that we do not get everything from the top position and apply + * the limit but rather flip the sort direction, apply the limit and then reverse the result to restore the actual + * sort order. + */ + private static class ReverseKeysetScrollDirector extends KeysetScrollDirector { + + @Override + public Document getSortObject(String idPropertyName, Query query) { + + Document sortObject = super.getSortObject(idPropertyName, query); + + // flip sort direction for backward scrolling + + for (String field : sortObject.keySet()) { + sortObject.put(field, sortObject.getInteger(field) == 1 ? -1 : 1); + } + + return sortObject; + } + + @Override + protected String getComparator(int sortOrder) { + + // use gte/lte to include the object at the cursor/keyset so that + // we can include it in the result to check whether there is a next object. + // It needs to be filtered out later on. + return sortOrder == 1 ? "$gte" : "$lte"; + } + + @Override + public void postPostProcessResults(List result) { + // flip direction of the result list as we need to accomodate for the flipped sort order for proper offset + // querying. + Collections.reverse(result); + } + } + } diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java index 88c03e915..87e75864b 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java @@ -167,42 +167,32 @@ class MongoTemplateScrollTests { @Test // GH-4308 void shouldAllowReverseSort() { - WithNestedDocument john20 = new WithNestedDocument(null, "John", 120, new WithNestedDocument("John", 20), - new Document("name", "bar")); - WithNestedDocument john40 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 40), - new Document("name", "baz")); - WithNestedDocument john41 = new WithNestedDocument(null, "John", 141, new WithNestedDocument("John", 41), - new Document("name", "foo")); + Person jane_20 = new Person("Jane", 20); + Person jane_40 = new Person("Jane", 40); + Person jane_42 = new Person("Jane", 42); + Person john20 = new Person("John", 20); + Person john40_1 = new Person("John", 40); + Person john40_2 = new Person("John", 40); - template.insertAll(Arrays.asList(john20, john40, john41)); + template.insertAll(Arrays.asList(john20, john40_1, john40_2, jane_20, jane_40, jane_42)); + Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")); + q.with(KeysetScrollPosition.initial()).limit(6); - Query q = new Query(where("name").regex("J.*")).with(Sort.by("nested.name", "nested.age", "document.name")) - .limit(2); - q.with(KeysetScrollPosition.initial()); - - Window window = template.scroll(q, WithNestedDocument.class); - - assertThat(window.hasNext()).isTrue(); - assertThat(window.isLast()).isFalse(); - assertThat(window).hasSize(2); - assertThat(window).containsOnly(john20, john40); - - window = template.scroll(q.with(window.positionAt(window.size() - 1)), WithNestedDocument.class); + Window window = template.scroll(q, Person.class); assertThat(window.hasNext()).isFalse(); assertThat(window.isLast()).isTrue(); - assertThat(window).hasSize(1); - assertThat(window).containsOnly(john41); + assertThat(window).hasSize(6); - KeysetScrollPosition scrollPosition = (KeysetScrollPosition) window.positionAt(0); + KeysetScrollPosition scrollPosition = (KeysetScrollPosition) window.positionAt(window.size() - 1); KeysetScrollPosition reversePosition = KeysetScrollPosition.of(scrollPosition.getKeys(), Direction.Backward); - window = template.scroll(q.with(reversePosition), WithNestedDocument.class); + window = template.scroll(q.with(reversePosition).limit(2), Person.class); + assertThat(window).hasSize(2); + assertThat(window).containsOnly(john20, john40_1); assertThat(window.hasNext()).isTrue(); assertThat(window.isLast()).isFalse(); - assertThat(window).hasSize(2); - assertThat(window).containsOnly(john20, john40); } @ParameterizedTest // GH-4308