Skip to content

Commit

Permalink
Merge pull request #237 from europeana/EA-3419_fix_sorting
Browse files Browse the repository at this point in the history 
the provided find options at search time were overwriding the sort
  • Loading branch information
@gsergiu
gsergiu authored Aug 16, 2023
2 parents 4a0a2d0 + ec8b438 commit 14cf6cb
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 86 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
<version.java>11</version.java>
<version.javac.release>11</version.javac.release>
<version.log4j2>2.17.2</version.log4j2>
<version.commonsApi>0.3.21-SNAPSHOT</version.commonsApi>
<version.commonsApi>0.3.22-SNAPSHOT</version.commonsApi>
<version.springBoot>2.5.7</version.springBoot>
<version.springBootMongoStarter>2.0.3.RELEASE</version.springBootMongoStarter>
<version.swagger>3.0.0</version.swagger>
Expand Down
3 changes: 3 additions & 0 deletions ...nitions/src/main/java/eu/europeana/set/definitions/model/vocabulary/WebUserSetFields.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ public class WebUserSetFields extends WebUserSetModelFields {
*/
@Deprecated(since = "")
public static final String PARAM_SORT_ORDER = "sortOrder";
public static final String SORT_ORDER_DESC = "desc";
public static final String SORT_ORDER_ASC = "asc";

public static final String TEXT_SCORE_SORT = "score";

// JsonLd Constants
Expand Down
169 changes: 86 additions & 83 deletions set-mongo/src/main/java/eu/europeana/set/mongo/service/PersistentUserSetServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
import org.mongodb.morphia.query.Criteria;
import org.mongodb.morphia.query.FindOptions;
import org.mongodb.morphia.query.Meta;
import org.mongodb.morphia.query.Query;
import org.mongodb.morphia.query.QueryResults;
Expand Down Expand Up @@ -44,7 +43,7 @@
* @author GrafR
*
*/
//@Component(value = UserSetConfiguration.BEAN_SET_PERSITENCE_SERVICE)
// @Component(value = UserSetConfiguration.BEAN_SET_PERSITENCE_SERVICE)
public class PersistentUserSetServiceImpl extends
AbstractNoSqlServiceImpl<PersistentUserSet, String> implements PersistentUserSetService {

Expand Down Expand Up @@ -116,14 +115,14 @@ public PersistentUserSet getByIdentifier(String identifier) {
}

@Override
public long getDistinct(String field, boolean fieldIsArray, String collectionType) throws UserSetServiceException {
public long getDistinct(String field, boolean fieldIsArray, String collectionType)
throws UserSetServiceException {
long count = 0;
// Cursor is needed in aggregate command
AggregationOptions aggregationOptions =
AggregationOptions.builder().outputMode(AggregationOptions.OutputMode.CURSOR).build();
Cursor cursor = getDao().getCollection().aggregate(
getDistinctCountPipeline(field, fieldIsArray, collectionType),
aggregationOptions);
getDistinctCountPipeline(field, fieldIsArray, collectionType), aggregationOptions);
if (cursor != null && cursor.hasNext()) {
// ideally there should be only one value present.
count = Long.parseLong(cursor.next().get(UserSetMongoConstants.MONGO_FIELD_COUNT).toString());
Expand All @@ -137,15 +136,16 @@ public long getDistinct(String field, boolean fieldIsArray, String collectionTyp

return count;
}

@Override
public long countItemsInEntitySets() {
// Cursor is needed in aggregate command
AggregationOptions aggregationOptions =
AggregationOptions.builder().outputMode(AggregationOptions.OutputMode.CURSOR).build();
long totalItems = 0;
Map<String, DBObject> groupFieldsAdditional = new ConcurrentHashMap<>();
groupFieldsAdditional.put(UserSetMongoConstants.MONGO_FIELD_COUNT, new BasicDBObject(UserSetMongoConstants.MONGO_SUM, UserSetMongoConstants.MONGO_TOTAL));
groupFieldsAdditional.put(UserSetMongoConstants.MONGO_FIELD_COUNT,
new BasicDBObject(UserSetMongoConstants.MONGO_SUM, UserSetMongoConstants.MONGO_TOTAL));
Cursor cursor = getDao().getCollection().aggregate(
getAggregatePipeline(UserSetTypes.ENTITYBESTITEMSSET.getJsonValue(), groupFieldsAdditional),
aggregationOptions);
Expand All @@ -155,7 +155,7 @@ public long countItemsInEntitySets() {
}
return totalItems;
}

/*
* (non-Javadoc)
*
Expand Down Expand Up @@ -225,36 +225,33 @@ public PersistentUserSet findByID(String id) {
}

/**
* Creates a aggregation pipeline to count distinct values of
* the field provided. If the field is of array type, we first need to unwind the array values.
* 'type' of user-set value is optional. If passed match filter is added.
* query :
* [ { $match: { type: <type> }},
* { $unwind : "$items"},
* { $group: { _id: <groupField> }},
* { $count: <Field Name for the count> } ]
* Creates a aggregation pipeline to count distinct values of the field provided. If the field is
* of array type, we first need to unwind the array values. 'type' of user-set value is optional.
* If passed match filter is added. query : [ { $match: { type: <type> }}, { $unwind : "$items"},
* { $group: { _id: <groupField> }}, { $count: <Field Name for the count> } ]
*
* @param type : Optional. type of user set - Collection or BookmarkFolder.
* @param groupField : the field for which distinct count is calculated
* @return
*/
private List<DBObject> getDistinctCountPipeline(String groupField, boolean groupFieldIsArray, String type) {
List<DBObject> distinctCountPipeline = new ArrayList<>();
// add match filter if present
if (StringUtils.isNotEmpty(type)) {
distinctCountPipeline.add(getMatchFilter(WebUserSetFields.TYPE, type));
}
if(groupFieldIsArray) {
distinctCountPipeline.add(new BasicDBObject(UserSetMongoConstants.MONGO_UNWIND, groupField));
}
// add group field
distinctCountPipeline.add(new BasicDBObject(UserSetMongoConstants.MONGO_GROUP,
new BasicDBObject(UserSetMongoConstants.MONGO_ID, groupField)));
// add count
distinctCountPipeline.add(new BasicDBObject(UserSetMongoConstants.MONGO_COUNT,
UserSetMongoConstants.MONGO_FIELD_COUNT));

return distinctCountPipeline;
private List<DBObject> getDistinctCountPipeline(String groupField, boolean groupFieldIsArray,
String type) {
List<DBObject> distinctCountPipeline = new ArrayList<>();
// add match filter if present
if (StringUtils.isNotEmpty(type)) {
distinctCountPipeline.add(getMatchFilter(WebUserSetFields.TYPE, type));
}
if (groupFieldIsArray) {
distinctCountPipeline.add(new BasicDBObject(UserSetMongoConstants.MONGO_UNWIND, groupField));
}
// add group field
distinctCountPipeline.add(new BasicDBObject(UserSetMongoConstants.MONGO_GROUP,
new BasicDBObject(UserSetMongoConstants.MONGO_ID, groupField)));
// add count
distinctCountPipeline.add(new BasicDBObject(UserSetMongoConstants.MONGO_COUNT,
UserSetMongoConstants.MONGO_FIELD_COUNT));

return distinctCountPipeline;
}

@Override
Expand Down Expand Up @@ -331,40 +328,46 @@ private DBObject getOutputFieldAndStages(UserSetFacetQuery facetQuery) {
}

/**
* creates a mongo query to count the total item present in BookmarkFolder
* Mongo Query : db.getCollection('userset').aggregate([
* {$match:{"type":"BookmarkFolder"}},{$group: {_id:null, totalLikes: {$sum: "$total"}}}
* ])
* creates a mongo query to count the total item present in BookmarkFolder Mongo Query :
* db.getCollection('userset').aggregate([ {$match:{"type":"BookmarkFolder"}},{$group: {_id:null,
* totalLikes: {$sum: "$total"}}} ])
*
* @return
*/
@Override
public long countTotalLikes() {
// Cursor is needed in aggregate command
AggregationOptions aggregationOptions = AggregationOptions.builder().outputMode(AggregationOptions.OutputMode.CURSOR).build();

long totalLikes =0;
Map<String,DBObject> groupFieldsAdditional = new ConcurrentHashMap<>();
groupFieldsAdditional.put(UserSetMongoConstants.MONGO_TOTAL_LIKES, new BasicDBObject(UserSetMongoConstants.MONGO_SUM, UserSetMongoConstants.MONGO_TOTAL));
Cursor cursor =getDao().getCollection().aggregate(getAggregatePipeline(UserSetTypes.BOOKMARKSFOLDER.getJsonValue(), groupFieldsAdditional), aggregationOptions);
if (cursor != null) {
while(cursor.hasNext()) {
DBObject object = cursor.next();
totalLikes += Long.parseLong(String.valueOf(object.get(UserSetMongoConstants.MONGO_TOTAL_LIKES)));
}
// Cursor is needed in aggregate command
AggregationOptions aggregationOptions =
AggregationOptions.builder().outputMode(AggregationOptions.OutputMode.CURSOR).build();

long totalLikes = 0;
Map<String, DBObject> groupFieldsAdditional = new ConcurrentHashMap<>();
groupFieldsAdditional.put(UserSetMongoConstants.MONGO_TOTAL_LIKES,
new BasicDBObject(UserSetMongoConstants.MONGO_SUM, UserSetMongoConstants.MONGO_TOTAL));
Cursor cursor = getDao().getCollection().aggregate(
getAggregatePipeline(UserSetTypes.BOOKMARKSFOLDER.getJsonValue(), groupFieldsAdditional),
aggregationOptions);
if (cursor != null) {
while (cursor.hasNext()) {
DBObject object = cursor.next();
totalLikes +=
Long.parseLong(String.valueOf(object.get(UserSetMongoConstants.MONGO_TOTAL_LIKES)));
}
return totalLikes;
}
return totalLikes;
}

// create $match and $group for mongo query
private List<DBObject> getAggregatePipeline(String collectionType, Map<String,DBObject> groupFieldsAdditional) {
DBObject match = getMatchFilter(WebUserSetFields.TYPE, collectionType);
private List<DBObject> getAggregatePipeline(String collectionType,
Map<String, DBObject> groupFieldsAdditional) {
DBObject match = getMatchFilter(WebUserSetFields.TYPE, collectionType);

DBObject groupFields = new BasicDBObject(UserSetMongoConstants.MONGO_ID, null);
for (Map.Entry<String,DBObject> field : groupFieldsAdditional.entrySet()) {
groupFields.put(field.getKey(), field.getValue());
}
DBObject group = new BasicDBObject(UserSetMongoConstants.MONGO_GROUP, groupFields);
return Arrays.asList(match, group);
DBObject groupFields = new BasicDBObject(UserSetMongoConstants.MONGO_ID, null);
for (Map.Entry<String, DBObject> field : groupFieldsAdditional.entrySet()) {
groupFields.put(field.getKey(), field.getValue());
}
DBObject group = new BasicDBObject(UserSetMongoConstants.MONGO_GROUP, groupFields);
return Arrays.asList(match, group);
}

/**
Expand All @@ -390,16 +393,16 @@ public List<PersistentUserSet> getEntitySetsItemAndSubject(UserSetQuery userSetQ
.project(WebUserSetFields.TYPE, true).project(WebUserSetModelFields.SUBJECT, true).asList();
}

@SuppressWarnings("deprecation")
@Override
public ResultSet<PersistentUserSet> find(UserSetQuery query) {
Query<PersistentUserSet> mongoQuery = buildMongoQuery(query);
long totalInCollection = mongoQuery.count();

FindOptions options = buildMongoPaginationOptions(query);
List<PersistentUserSet> userSets = new ArrayList<PersistentUserSet>();
// workaround as limit=0 still returns all results
if (options.getLimit() > 0) {
userSets = mongoQuery.asList(options);
if (mongoQuery.getLimit() > 0) {
userSets = mongoQuery.asList();
}
ResultSet<PersistentUserSet> res = new ResultSet<>();

Expand All @@ -408,16 +411,19 @@ public ResultSet<PersistentUserSet> find(UserSetQuery query) {
return res;
}

private FindOptions buildMongoPaginationOptions(UserSetQuery query) {
FindOptions options = new FindOptions();
options.skip(query.getPageNr() * query.getPageSize());
options.limit(query.getPageSize());
return options;
@SuppressWarnings("deprecation")
private void setPaginationOptions(Query<PersistentUserSet> mongoQuery, UserSetQuery query) {
mongoQuery.offset(query.getPageNr() * query.getPageSize());
mongoQuery.limit(query.getPageSize());
}

private Query<PersistentUserSet> buildMongoQuery(UserSetQuery query) {

Query<PersistentUserSet> searchQuery = buildUserConditionsQuery(query);

setPaginationOptions(searchQuery, query);
setOrder(searchQuery, query);

if (query.isAdmin()) {
// admin can see all
return searchQuery;
Expand All @@ -436,7 +442,7 @@ private Query<PersistentUserSet> buildMongoQuery(UserSetQuery query) {
// private only, user can see only his private sets
searchQuery.filter(FIELD_CREATOR, query.getUser());
}

return searchQuery;
}

Expand Down Expand Up @@ -484,33 +490,30 @@ private Query<PersistentUserSet> buildUserConditionsQuery(UserSetQuery query) {
mongoQuery.filter(WebUserSetModelFields.TITLE + "." + query.getTitleLang() + " exists", 1);
}

if (query.getSortCriteria() == null) {
// default ordering if none is defined by the user
mongoQuery.order(Sort.descending(WebUserSetModelFields.MODIFIED));
} else {
buildSortCriteria(query, mongoQuery);
}

return mongoQuery;
}

private void buildSortCriteria(UserSetQuery query, Query<PersistentUserSet> mongoQuery) {
private void setOrder(Query<PersistentUserSet> mongoQuery, UserSetQuery query) {
// default ordering if none is defined by the user
if (query.getSortCriteria() == null) {
mongoQuery.order(Sort.descending(WebUserSetModelFields.MODIFIED));
return;
}

for (String sortField : query.getSortCriteria()) {
//check the score sort first (it can only be in desc order)
if(sortField.contains(WebUserSetFields.TEXT_SCORE_SORT)) {
// check the score sort first (it can only be in desc order)
if (sortField.contains(WebUserSetFields.TEXT_SCORE_SORT)) {
mongoQuery.order(Meta.textScore());
}
else {
} else {
if (!sortField.contains(" ")) {
mongoQuery.order(Sort.ascending(sortField));
} else {
String[] sortParts = sortField.split(" ", 2);
if (!"desc".contentEquals(sortParts[1])) {
mongoQuery.order(Sort.ascending(sortParts[0]));
} else {
if (WebUserSetFields.SORT_ORDER_DESC.equals(sortParts[1])) {
mongoQuery.order(Sort.descending(sortParts[0]));
} else {
mongoQuery.order(Sort.ascending(sortParts[0]));
}

}
}
}
Expand Down
2 changes: 1 addition & 1 deletion set-web/src/main/java/eu/europeana/set/web/search/UserSetQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ private void addSortCriterion(Map<String, String> searchCriteria, UserSetQuery s
String[] sortCriteria = toArray(sort);
if(sortCriteria!=null) {
for(String sortCriterion : sortCriteria) {
if(sortCriterion.contains(WebUserSetFields.TEXT_SCORE_SORT) && sortCriterion.contains("asc")) {
if(sortCriterion.contains(WebUserSetFields.TEXT_SCORE_SORT) && sortCriterion.contains(WebUserSetFields.SORT_ORDER_ASC)) {
throw new ParamValidationException(I18nConstants.INVALID_PARAM_VALUE, I18nConstants.INVALID_PARAM_VALUE,
new String[] { "invalid value for the sort field, it cannot contain 'score asc' since only the descending order is supported", sort });
}
Expand Down
5 changes: 4 additions & 1 deletion set-web/src/main/resources/log4j2.xml.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@
</logger>

<!-- To suppress Mongo opened/closed connection messages -->
<Logger name="org.mongodb.driver.connection" level="WARN"/>
<logger name="org.mongodb.driver.connection" level="WARN"/>

<!-- To enable logging of Mongo queries, set log level to debug -->
<logger name="org.mongodb.morphia" level="WARN"/>
</Loggers>
</Configuration>

0 comments on commit 14cf6cb

Please sign in to comment.