Skip to content

Commit

Permalink
fixing issue with too high results in some /users resources
Browse files Browse the repository at this point in the history 
the map step on the userId was missing
implementing method in ExecutionUtils that fills the JSON results
  • Loading branch information
kowatsch committed Jun 7, 2018
1 parent f559b9b commit ec5626d
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 55 deletions.
24 changes: 24 additions & 0 deletions ...n/java/org/heigit/bigspatialdata/ohsome/ohsomeApi/controller/executor/ExecutionUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.heigit.bigspatialdata.ohsome.ohsomeApi.output.dataAggregationResponse.groupByResponse.RatioGroupByResult;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.output.dataAggregationResponse.groupByResponse.ShareGroupByBoundaryResponse;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.output.dataAggregationResponse.groupByResponse.ShareGroupByResult;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.output.dataAggregationResponse.users.UsersResult;
import org.heigit.bigspatialdata.oshdb.api.generic.OSHDBTimestampAndIndex;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
Expand Down Expand Up @@ -322,6 +323,29 @@ public ElementsResult[] fillElementsResult(SortedMap<OSHDBTimestamp, ? extends N
}
return results;
}

/** Fills the UsersResult array with respective UsersResult objects. */
public UsersResult[] fillUsersResult(SortedMap<OSHDBTimestamp, ? extends Number> entryVal,
boolean isDensity, String[] toTimestamps, DecimalFormat df, Geometry geom) {

UsersResult[] results = new UsersResult[entryVal.entrySet().size()];
int count = 0;
for (Entry<OSHDBTimestamp, ? extends Number> entry : entryVal.entrySet()) {
if (isDensity) {
results[count] = new UsersResult(
TimestampFormatter.getInstance().isoDateTime(entry.getKey()),
toTimestamps[count + 1], Double.parseDouble(
df.format((entry.getValue().doubleValue() / (Geo.areaOf(geom) / 1000000)))));
} else {
results[count] =
new UsersResult(TimestampFormatter.getInstance().isoDateTime(entry.getKey()),
toTimestamps[count + 1],
Double.parseDouble(df.format(entry.getValue().doubleValue())));
}
count++;
}
return results;
}

/** Creates either a RatioResponse or a ShareResponse depending on the request. */
public RatioShareResponse createRatioShareResponse(boolean isShare, String[] timeArray,
Expand Down
135 changes: 80 additions & 55 deletions .../org/heigit/bigspatialdata/ohsome/ohsomeApi/controller/executor/UsersRequestExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.heigit.bigspatialdata.ohsome.ohsomeApi.exception.BadRequestException;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.inputProcessing.GeometryBuilder;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.inputProcessing.InputProcessor;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.inputProcessing.Utils;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.interceptor.RequestInterceptor;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.oshdb.DbConnData;
import org.heigit.bigspatialdata.ohsome.ohsomeApi.oshdb.ExtractMetadata;
Expand All @@ -26,9 +27,8 @@
import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
import org.heigit.bigspatialdata.oshdb.osm.OSMType;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
import com.vividsolutions.jts.geom.Geometry;

/** Includes the execute methods for requests mapped to /users. */
Expand Down Expand Up @@ -66,24 +66,11 @@ public static DefaultAggregationResponse executeCount(RequestParameters rPs)
result = mapRed.aggregateByTimestamp().map(contrib -> {
return contrib.getContributorUserId();
}).countUniq();
UsersResult[] resultSet = new UsersResult[result.size()];
String[] toTimestamps = iP.getUtils().getToTimestamps();
GeometryBuilder geomBuilder = iP.getGeomBuilder();
Geometry geom = exeUtils.getGeometry(iP.getBoundaryType(), geomBuilder);
int count = 0;
for (Entry<OSHDBTimestamp, Integer> entry : result.entrySet()) {
if (rPs.isDensity()) {
resultSet[count] =
new UsersResult(TimestampFormatter.getInstance().isoDateTime(entry.getKey()),
toTimestamps[count + 1], Double.parseDouble(
df.format((entry.getValue().doubleValue() / (Geo.areaOf(geom) * 0.000001)))));
} else {
resultSet[count] =
new UsersResult(TimestampFormatter.getInstance().isoDateTime(entry.getKey()),
toTimestamps[count + 1], entry.getValue().intValue());
}
count++;
}
UsersResult[] results =
exeUtils.fillUsersResult(result, rPs.isDensity(), toTimestamps, df, geom);
if (rPs.isDensity()) {
description =
"Density of distinct users per time interval (number of users per square-kilometer).";
Expand All @@ -96,7 +83,7 @@ public static DefaultAggregationResponse executeCount(RequestParameters rPs)
metadata = new Metadata(duration, description, requestURL);
}
DefaultAggregationResponse response = new DefaultAggregationResponse(new Attribution(url, text),
Application.apiVersion, metadata, resultSet);
Application.apiVersion, metadata, results);
return response;
}

Expand Down Expand Up @@ -130,30 +117,18 @@ public static GroupByResponse executeCountGroupByType(RequestParameters rPs)
result = mapRed.aggregateByTimestamp()
.aggregateBy((SerializableFunction<OSMContribution, OSMType>) f -> {
return f.getEntityAfter().getType();
}).zerofillIndices(iP.getOsmTypes()).count();
}).zerofillIndices(iP.getOsmTypes()).map(contrib -> {
return contrib.getContributorUserId();
}).countUniq();
groupByResult = MapAggregatorByTimestampAndIndex.nest_IndexThenTime(result);
GroupByResult[] resultSet = new GroupByResult[groupByResult.size()];
GeometryBuilder geomBuilder = iP.getGeomBuilder();
Geometry geom = exeUtils.getGeometry(iP.getBoundaryType(), geomBuilder);
String[] toTimestamps = iP.getUtils().getToTimestamps();
int count = 0;
int innerCount = 0;
for (Entry<OSMType, SortedMap<OSHDBTimestamp, Integer>> entry : groupByResult.entrySet()) {
UsersResult[] results = new UsersResult[entry.getValue().entrySet().size()];
innerCount = 0;
for (Entry<OSHDBTimestamp, ? extends Number> innerEntry : entry.getValue().entrySet()) {
if (rPs.isDensity())
results[innerCount] = new UsersResult(
TimestampFormatter.getInstance().isoDateTime(innerEntry.getKey()),
toTimestamps[innerCount + 1], Double.parseDouble(
df.format((innerEntry.getValue().doubleValue() / (Geo.areaOf(geom) / 1000000)))));
else
results[innerCount] =
new UsersResult(TimestampFormatter.getInstance().isoDateTime(innerEntry.getKey()),
toTimestamps[innerCount + 1],
Double.parseDouble(df.format(innerEntry.getValue().doubleValue())));
innerCount++;
}
UsersResult[] results =
exeUtils.fillUsersResult(entry.getValue(), rPs.isDensity(), toTimestamps, df, geom);
resultSet[count] = new GroupByResult(entry.getKey().toString(), results);
count++;
}
Expand Down Expand Up @@ -207,21 +182,27 @@ public static GroupByResponse executeCountGroupByTag(RequestParameters rPs, Stri
TagTranslator tt = DbConnData.tagTranslator;
Integer[] valuesInt = new Integer[groupByValues.length];
ArrayList<Pair<Integer, Integer>> zeroFill = new ArrayList<Pair<Integer, Integer>>();
mapRed = iP.processParameters(mapRed, rPs);
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt();
if (groupByValues.length != 0) {
for (int j = 0; j < groupByValues.length; j++) {
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue();
zeroFill.add(new ImmutablePair<Integer, Integer>(keysInt, valuesInt[j]));
}
}
mapRed = iP.processParameters(mapRed, rPs);
result = mapRed.map(f -> {

int[] tags;
if (!f.getEntityAfter().isVisible())
if (f.getContributionTypes().contains(ContributionType.DELETION)) {
tags = f.getEntityBefore().getRawTags();
else
} else if (f.getContributionTypes().contains(ContributionType.CREATION)) {
tags = f.getEntityAfter().getRawTags();
} else {
int[] tagsBefore = f.getEntityBefore().getRawTags();
int[] tagsAfter = f.getEntityAfter().getRawTags();
tags = new int[tagsBefore.length + tagsAfter.length];
System.arraycopy(tagsBefore, 0, tags, 0, tagsBefore.length);
System.arraycopy(tagsAfter, 0, tags, tagsBefore.length, tagsAfter.length);
}
for (int i = 0; i < tags.length; i += 2) {
int tagKeyId = tags[i];
int tagValueId = tags[i + 1];
Expand All @@ -239,7 +220,9 @@ public static GroupByResponse executeCountGroupByTag(RequestParameters rPs, Stri
}
return new ImmutablePair<>(new ImmutablePair<Integer, Integer>(-1, -1), f);
}).aggregateByTimestamp().aggregateBy(Pair::getKey).zerofillIndices(zeroFill)
.map(Pair::getValue).countUniq();
.map(Pair::getValue).map(contrib -> {
return contrib.getContributorUserId();
}).countUniq();
groupByResult = MapAggregatorByTimestampAndIndex.nest_IndexThenTime(result);
GroupByResult[] resultSet = new GroupByResult[groupByResult.size()];
String groupByName = "";
Expand All @@ -249,27 +232,14 @@ public static GroupByResponse executeCountGroupByTag(RequestParameters rPs, Stri
int count = 0;
for (Entry<Pair<Integer, Integer>, SortedMap<OSHDBTimestamp, Integer>> entry : groupByResult
.entrySet()) {
UsersResult[] results = new UsersResult[entry.getValue().entrySet().size()];
int innerCount = 0;
UsersResult[] results =
exeUtils.fillUsersResult(entry.getValue(), rPs.isDensity(), toTimestamps, df, geom);
// check for non-remainder objects (which do have the defined key and value)
if (entry.getKey().getKey() != -1 && entry.getKey().getValue() != -1) {
groupByName = tt.getOSMTagOf(keysInt, entry.getKey().getValue()).toString();
} else {
groupByName = "remainder";
}
for (Entry<OSHDBTimestamp, ? extends Number> innerEntry : entry.getValue().entrySet()) {
if (rPs.isDensity())
results[innerCount] = new UsersResult(
TimestampFormatter.getInstance().isoDateTime(innerEntry.getKey()),
toTimestamps[innerCount + 1], Double.parseDouble(
df.format((innerEntry.getValue().doubleValue() / (Geo.areaOf(geom) / 1000000)))));
else
results[innerCount] =
new UsersResult(TimestampFormatter.getInstance().isoDateTime(innerEntry.getKey()),
toTimestamps[innerCount + 1],
Double.parseDouble(df.format(innerEntry.getValue().doubleValue())));
innerCount++;
}
resultSet[count] = new GroupByResult(groupByName, results);
count++;
}
Expand All @@ -289,4 +259,59 @@ public static GroupByResponse executeCountGroupByTag(RequestParameters rPs, Stri
return response;
}

/**
* NOT IN USE YET Performs a count calculation grouped by the boundary.
* <p>
* The other parameters are described in the
* {@link org.heigit.bigspatialdata.ohsome.ohsomeApi.controller.dataAggregation.CountController#getCount(String, String, String, String[], String[], String[], String[], String[], String)
* getCount} method.
*
* @param rPs <code>RequestParameters</code> object, which holds those parameters that are used in
* every request.
* @return {@link org.heigit.bigspatialdata.ohsome.ohsomeApi.output.dataAggregationResponse.groupByResponse.GroupByResponse
* GroupByResponse Content}
*/
public static GroupByResponse executeCountGroupByBoundary(RequestParameters rPs)
throws UnsupportedOperationException, Exception {

long startTime = System.currentTimeMillis();
ExecutionUtils exeUtils = new ExecutionUtils();
SortedMap<OSHDBTimestampAndIndex<Integer>, ? extends Number> result = null;
SortedMap<Integer, ? extends SortedMap<OSHDBTimestamp, ? extends Number>> groupByResult;
MapReducer<OSMContribution> mapRed = null;
InputProcessor iP = new InputProcessor();
String description = null;
String requestURL = null;
DecimalFormat df = exeUtils.defineDecimalFormat("#.##");
if (!rPs.isPost())
requestURL = RequestInterceptor.requestUrl;
mapRed = iP.processParameters(mapRed, rPs);
result = exeUtils.computeCountLengthPerimeterAreaGBB(RequestResource.COUNT,
iP.getBoundaryType(), mapRed, iP.getGeomBuilder(), rPs.isSnapshot());
groupByResult = MapAggregatorByTimestampAndIndex.nest_IndexThenTime(result);
GroupByResult[] resultSet = new GroupByResult[groupByResult.size()];
String groupByName = "";
String[] toTimestamps = iP.getUtils().getToTimestamps();
Utils utils = iP.getUtils();
String[] boundaryIds = utils.getBoundaryIds();
int count = 0;
for (Entry<Integer, ? extends SortedMap<OSHDBTimestamp, ? extends Number>> entry : groupByResult
.entrySet()) {
UsersResult[] results =
exeUtils.fillUsersResult(entry.getValue(), rPs.isDensity(), toTimestamps, df, null);
groupByName = boundaryIds[count];
resultSet[count] = new GroupByResult(groupByName, results);
count++;
}
description = "Total count of items in absolute values aggregated on the boundary.";
Metadata metadata = null;
if (iP.getShowMetadata()) {
long duration = System.currentTimeMillis() - startTime;
metadata = new Metadata(duration, description, requestURL);
}
GroupByResponse response = new GroupByResponse(new Attribution(url, text),
Application.apiVersion, metadata, resultSet);
return response;
}

}

0 comments on commit ec5626d

Please sign in to comment.