mindol1004/spring-data-mongodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix keyset backwards scrolling.

Browse Source 
We now correctly scroll backwards by reversing sort order to apply the correct limit and reverse the results again to restore the actual sort order.

Closes #4332
This commit is contained in:
Mark Paluch
2023-03-20 08:54:50 +01:00
parent d8c04f0ec9
commit 25f610cc8a
4 changed files with 172 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java
View File

@@ -66,7 +66,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DeleteContext;
import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext;
import org.springframework.data.mongodb.core.QueryOperations.QueryContext;
import org.springframework.data.mongodb.core.QueryOperations.UpdateContext;
import org.springframework.data.mongodb.core.ScrollUtils.KeySetScrollQuery;
import org.springframework.data.mongodb.core.ScrollUtils.KeysetScrollQuery;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext;
import org.springframework.data.mongodb.core.aggregation.AggregationOptions;
@@ -876,14 +876,14 @@ public class MongoTemplate
if (query.hasKeyset()) {
KeySetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query,
KeysetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query,
operations.getIdPropertyName(sourceClass));
List<T> result = doFind(collectionName, createDelegate(query), keysetPaginationQuery.query(),
keysetPaginationQuery.fields(), sourceClass,
new QueryCursorPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback);
return ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), result, sourceClass, operations);
return ScrollUtils.createWindow(query, result, sourceClass, operations);
}
List<T> result = doFind(collectionName, createDelegate(query), query.getQueryObject(), query.getFieldsObject(),

6
spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java
View File

@@ -80,7 +80,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DeleteContext;
import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext;
import org.springframework.data.mongodb.core.QueryOperations.QueryContext;
import org.springframework.data.mongodb.core.QueryOperations.UpdateContext;
import org.springframework.data.mongodb.core.ScrollUtils.KeySetScrollQuery;
import org.springframework.data.mongodb.core.ScrollUtils.KeysetScrollQuery;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext;
import org.springframework.data.mongodb.core.aggregation.AggregationOptions;
@@ -855,14 +855,14 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati
if (query.hasKeyset()) {
KeySetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query,
KeysetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query,
operations.getIdPropertyName(sourceClass));
Mono<List<T>> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query),
keysetPaginationQuery.query(), keysetPaginationQuery.fields(), sourceClass,
new QueryFindPublisherPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback).collectList();
return result.map(it -> ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), it, sourceClass, operations));
return result.map(it -> ScrollUtils.createWindow(query, it, sourceClass, operations));
}
Mono<List<T>> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), query.getQueryObject(),

224
spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java
View File

@@ -16,6 +16,7 @@
package org.springframework.data.mongodb.core;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.IntFunction;
@@ -45,84 +46,24 @@ class ScrollUtils {
* @param idPropertyName
* @return
*/
static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPropertyName) {
static KeysetScrollQuery createKeysetPaginationQuery(Query query, String idPropertyName) {
KeysetScrollPosition keyset = query.getKeyset();
Map<String, Object> keysetValues = keyset.getKeys();
Document queryObject = query.getQueryObject();
KeysetScrollDirector director = KeysetScrollDirector.of(keyset.getDirection());
Document sortObject = director.getSortObject(idPropertyName, query);
Document fieldsObject = director.getFieldsObject(query.getFieldsObject(), sortObject);
Document queryObject = director.createQuery(keyset, query.getQueryObject(), sortObject);
Document sortObject = query.isSorted() ? query.getSortObject() : new Document();
sortObject.put(idPropertyName, 1);
// make sure we can extract the keyset
Document fieldsObject = query.getFieldsObject();
if (!fieldsObject.isEmpty()) {
for (String field : sortObject.keySet()) {
fieldsObject.put(field, 1);
}
}
List<Document> or = (List<Document>) queryObject.getOrDefault("$or", new ArrayList<>());
List<String> sortKeys = new ArrayList<>(sortObject.keySet());
if (!keysetValues.isEmpty() && !keysetValues.keySet().containsAll(sortKeys)) {
throw new IllegalStateException("KeysetScrollPosition does not contain all keyset values");
}
// first query doesn't come with a keyset
if (!keysetValues.isEmpty()) {
// build matrix query for keyset paging that contains sort^2 queries
// reflecting a query that follows sort order semantics starting from the last returned keyset
for (int i = 0; i < sortKeys.size(); i++) {
Document sortConstraint = new Document();
for (int j = 0; j < sortKeys.size(); j++) {
String sortSegment = sortKeys.get(j);
int sortOrder = sortObject.getInteger(sortSegment);
Object o = keysetValues.get(sortSegment);
if (j >= i) { // tail segment
if (o instanceof BsonNull) {
throw new IllegalStateException(
"Cannot resume from KeysetScrollPosition. Offending key: '%s' is 'null'".formatted(sortSegment));
}
sortConstraint.put(sortSegment, new Document(getComparator(sortOrder, keyset.getDirection()), o));
break;
}
sortConstraint.put(sortSegment, o);
}
if (!sortConstraint.isEmpty()) {
or.add(sortConstraint);
}
}
}
if (!or.isEmpty()) {
queryObject.put("$or", or);
}
return new KeySetScrollQuery(queryObject, fieldsObject, sortObject);
return new KeysetScrollQuery(queryObject, fieldsObject, sortObject);
}
private static String getComparator(int sortOrder, Direction direction) {
static <T> Window<T> createWindow(Query query, List<T> result, Class<?> sourceType, EntityOperations operations) {
// use gte/lte to include the object at the cursor/keyset so that
// we can include it in the result to check whether there is a next object.
// It needs to be filtered out later on.
if (direction == Direction.Backward) {
return sortOrder == 0 ? "$gte" : "$lte";
}
Document sortObject = query.getSortObject();
KeysetScrollPosition keyset = query.getKeyset();
KeysetScrollDirector director = KeysetScrollDirector.of(keyset.getDirection());
return sortOrder == 1 ? "$gt" : "$lt";
}
static <T> Window<T> createWindow(Document sortObject, int limit, List<T> result, Class<?> sourceType,
EntityOperations operations) {
director.postPostProcessResults(result);
IntFunction<KeysetScrollPosition> positionFunction = value -> {
@@ -133,7 +74,7 @@ class ScrollUtils {
return KeysetScrollPosition.of(keys);
};
return createWindow(result, limit, positionFunction);
return createWindow(result, query.getLimit(), positionFunction);
}
static <T> Window<T> createWindow(List<T> result, int limit, IntFunction<? extends ScrollPosition> positionFunction) {
@@ -153,8 +94,145 @@ class ScrollUtils {
return result;
}
record KeySetScrollQuery(Document query, Document fields, Document sort) {
record KeysetScrollQuery(Document query, Document fields, Document sort) {
}
/**
* Director for keyset scrolling.
*/
static class KeysetScrollDirector {
private static final KeysetScrollDirector forward = new KeysetScrollDirector();
private static final KeysetScrollDirector reverse = new ReverseKeysetScrollDirector();
/**
* Factory method to obtain the right {@link KeysetScrollDirector}.
*
* @param direction
* @return
*/
public static KeysetScrollDirector of(KeysetScrollPosition.Direction direction) {
return direction == Direction.Forward ? forward : reverse;
}
public Document getSortObject(String idPropertyName, Query query) {
Document sortObject = query.isSorted() ? query.getSortObject() : new Document();
sortObject.put(idPropertyName, 1);
return sortObject;
}
public Document getFieldsObject(Document fieldsObject, Document sortObject) {
// make sure we can extract the keyset
if (!fieldsObject.isEmpty()) {
for (String field : sortObject.keySet()) {
fieldsObject.put(field, 1);
}
}
return fieldsObject;
}
public Document createQuery(KeysetScrollPosition keyset, Document queryObject, Document sortObject) {
Map<String, Object> keysetValues = keyset.getKeys();
List<Document> or = (List<Document>) queryObject.getOrDefault("$or", new ArrayList<>());
List<String> sortKeys = new ArrayList<>(sortObject.keySet());
// first query doesn't come with a keyset
if (keysetValues.isEmpty()) {
return queryObject;
}
if (!keysetValues.keySet().containsAll(sortKeys)) {
throw new IllegalStateException("KeysetScrollPosition does not contain all keyset values");
}
// build matrix query for keyset paging that contains sort^2 queries
// reflecting a query that follows sort order semantics starting from the last returned keyset
for (int i = 0; i < sortKeys.size(); i++) {
Document sortConstraint = new Document();
for (int j = 0; j < sortKeys.size(); j++) {
String sortSegment = sortKeys.get(j);
int sortOrder = sortObject.getInteger(sortSegment);
Object o = keysetValues.get(sortSegment);
if (j >= i) { // tail segment
if (o instanceof BsonNull) {
throw new IllegalStateException(
"Cannot resume from KeysetScrollPosition. Offending key: '%s' is 'null'".formatted(sortSegment));
}
sortConstraint.put(sortSegment, new Document(getComparator(sortOrder), o));
break;
}
sortConstraint.put(sortSegment, o);
}
if (!sortConstraint.isEmpty()) {
or.add(sortConstraint);
}
}
if (!or.isEmpty()) {
queryObject.put("$or", or);
}
return queryObject;
}
public <T> void postPostProcessResults(List<T> result) {
}
protected String getComparator(int sortOrder) {
return sortOrder == 1 ? "$gt" : "$lt";
}
}
/**
* Reverse scrolling director variant applying {@link KeysetScrollPosition.Direction#Backward}. In reverse scrolling,
* we need to flip directions for the actual query so that we do not get everything from the top position and apply
* the limit but rather flip the sort direction, apply the limit and then reverse the result to restore the actual
* sort order.
*/
private static class ReverseKeysetScrollDirector extends KeysetScrollDirector {
@Override
public Document getSortObject(String idPropertyName, Query query) {
Document sortObject = super.getSortObject(idPropertyName, query);
// flip sort direction for backward scrolling
for (String field : sortObject.keySet()) {
sortObject.put(field, sortObject.getInteger(field) == 1 ? -1 : 1);
}
return sortObject;
}
@Override
protected String getComparator(int sortOrder) {
// use gte/lte to include the object at the cursor/keyset so that
// we can include it in the result to check whether there is a next object.
// It needs to be filtered out later on.
return sortOrder == 1 ? "$gte" : "$lte";
}
@Override
public <T> void postPostProcessResults(List<T> result) {
// flip direction of the result list as we need to accomodate for the flipped sort order for proper offset
// querying.
Collections.reverse(result);
}
}
}

40
spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java
View File

@@ -167,42 +167,32 @@ class MongoTemplateScrollTests {
@Test // GH-4308
void shouldAllowReverseSort() {
WithNestedDocument john20 = new WithNestedDocument(null, "John", 120, new WithNestedDocument("John", 20),
new Document("name", "bar"));
WithNestedDocument john40 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 40),
new Document("name", "baz"));
WithNestedDocument john41 = new WithNestedDocument(null, "John", 141, new WithNestedDocument("John", 41),
new Document("name", "foo"));
Person jane_20 = new Person("Jane", 20);
Person jane_40 = new Person("Jane", 40);
Person jane_42 = new Person("Jane", 42);
Person john20 = new Person("John", 20);
Person john40_1 = new Person("John", 40);
Person john40_2 = new Person("John", 40);
template.insertAll(Arrays.asList(john20, john40, john41));
template.insertAll(Arrays.asList(john20, john40_1, john40_2, jane_20, jane_40, jane_42));
Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age"));
q.with(KeysetScrollPosition.initial()).limit(6);
Query q = new Query(where("name").regex("J.*")).with(Sort.by("nested.name", "nested.age", "document.name"))
.limit(2);
q.with(KeysetScrollPosition.initial());
Window<WithNestedDocument> window = template.scroll(q, WithNestedDocument.class);
assertThat(window.hasNext()).isTrue();
assertThat(window.isLast()).isFalse();
assertThat(window).hasSize(2);
assertThat(window).containsOnly(john20, john40);
window = template.scroll(q.with(window.positionAt(window.size() - 1)), WithNestedDocument.class);
Window<Person> window = template.scroll(q, Person.class);
assertThat(window.hasNext()).isFalse();
assertThat(window.isLast()).isTrue();
assertThat(window).hasSize(1);
assertThat(window).containsOnly(john41);
assertThat(window).hasSize(6);
KeysetScrollPosition scrollPosition = (KeysetScrollPosition) window.positionAt(0);
KeysetScrollPosition scrollPosition = (KeysetScrollPosition) window.positionAt(window.size() - 1);
KeysetScrollPosition reversePosition = KeysetScrollPosition.of(scrollPosition.getKeys(), Direction.Backward);
window = template.scroll(q.with(reversePosition), WithNestedDocument.class);
window = template.scroll(q.with(reversePosition).limit(2), Person.class);
assertThat(window).hasSize(2);
assertThat(window).containsOnly(john20, john40_1);
assertThat(window.hasNext()).isTrue();
assertThat(window.isLast()).isFalse();
assertThat(window).hasSize(2);
assertThat(window).containsOnly(john20, john40);
}
@ParameterizedTest // GH-4308