Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallel stream #1

Open
wants to merge 7 commits into
base: PullRequest
Choose a base branch
from
Open

Parallel stream #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
39e4094
Merge pull request #16 from tenzap/PullRequest
reubot Aug 2, 2023
6f02124
use parralellStream() for comparsion operation
reubot Nov 4, 2019
8ec270d
re-enable stop progress output
reubot Mar 21, 2020
f64161c
Merge remote-tracking branch 'github/parallelStream' into parallelStream
Aug 2, 2023
55fc2bb
thread safety
reubot Mar 23, 2020
f2e7da1
more thread safety
reubot Aug 9, 2023
c3efce3
TODO handle overnight values properly
reubot Jul 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions GO_Sync/src/main/java/edu/usf/cutr/go_sync/object/RouteVariant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,19 @@ private Duration getDurationAsDuration() {
String tripDepartureTime = stops.get(firstStopIndex).getDeparture_time();
Integer lastStopIndex = Collections.max(stops.keySet());
String tripArrivalTime = stops.get(lastStopIndex).getArrival_time();
try {
LocalTime tripDepartureTimeLT = LocalTime.parse(tripDepartureTime, DateTimeFormatter.ofPattern("HH:mm:ss"));
LocalTime tripArrivalTimeLT = LocalTime.parse(tripArrivalTime, DateTimeFormatter.ofPattern("HH:mm:ss"));
return Duration.between(tripDepartureTimeLT, tripArrivalTimeLT);
}catch(Exception e)
{
//TODO handle overnight values properly
System.out.println(e);
System.out.println(e.getStackTrace());
e.printStackTrace();
return Duration.ZERO;
}

LocalTime tripDepartureTimeLT = LocalTime.parse(tripDepartureTime, DateTimeFormatter.ofPattern("HH:mm:ss"));
LocalTime tripArrivalTimeLT = LocalTime.parse(tripArrivalTime, DateTimeFormatter.ofPattern("HH:mm:ss"));
return Duration.between(tripDepartureTimeLT, tripArrivalTimeLT);
}

public String getDuration() {
Expand Down
55 changes: 38 additions & 17 deletions GO_Sync/src/main/java/edu/usf/cutr/go_sync/task/CompareData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;

import javax.swing.JTextArea;
import javax.swing.ProgressMonitor;
Expand Down Expand Up @@ -93,10 +95,16 @@ public class CompareData extends OsmTask{
// key is gtfs, value is container of potential osm matches, sorted by distance from gtfs stop
private Hashtable<Stop, TreeSet<Stop>> report =
new Hashtable<Stop, TreeSet<Stop>>();
private HashSet<Stop> noUpload = new HashSet<Stop>();
private HashSet<Stop> upload = new HashSet<Stop>();
private HashSet<Stop> modify = new HashSet<Stop>();
private HashSet<Stop> delete = new HashSet<Stop>();


private ConcurrentHashMap.KeySetView<Stop, Boolean> noUpload = ConcurrentHashMap.newKeySet();
private ConcurrentHashMap.KeySetView<Stop, Boolean> upload = ConcurrentHashMap.newKeySet();
private ConcurrentHashMap.KeySetView<Stop, Boolean> modify = ConcurrentHashMap.newKeySet();
private ConcurrentHashMap.KeySetView<Stop, Boolean> delete = ConcurrentHashMap.newKeySet();
// private HashSet<Stop> noUpload = new HashSet<Stop>();
// private HashSet<Stop> upload = new HashSet<Stop>();
// private HashSet<Stop> modify = new HashSet<Stop>();
// private HashSet<Stop> delete = new HashSet<Stop>();
private Hashtable<String, Route> routes = new Hashtable<String, Route>();
private Hashtable<String, Route> agencyRoutes = new Hashtable<String, Route>();
private Hashtable<String, Route> existingRoutes = new Hashtable<String, Route>();
Expand All @@ -107,8 +115,8 @@ public class CompareData extends OsmTask{
private double minLat=0, minLon=0, maxLat=0, maxLon=0;
private HttpRequest osmRequest;
private HashSet<String> osmActiveUsers = new HashSet<String>();
private Hashtable<String,String> osmIdToGtfsId = new Hashtable<String,String>();
private HashMap<String, Integer> osmIdToOSMNodexIndex = new HashMap<>();
private ConcurrentHashMap<String,String> osmIdToGtfsId = new ConcurrentHashMap<String,String>();
private ConcurrentHashMap<String, Integer> osmIdToOSMNodexIndex = new ConcurrentHashMap<>();

private HashMap <String, TreeMap> osmRouteVariantScoreByGtfsRoute = new HashMap<>();;
// OsmIndex - List of GtfsMatches
Expand Down Expand Up @@ -603,7 +611,7 @@ public void compareRouteData() throws InterruptedException{
osmStopTypeByGtfsStopId.put(gtfs_id.toString(), OSMNodes.get(i).geOsmPrimitiveType());
}
}

System.out.println(" for (HashMap.Entry<String, RouteVariant> rv_entry : allRouteVariants.entrySet())");
for (HashMap.Entry<String, RouteVariant> rv_entry : allRouteVariants.entrySet()) {
RouteVariant rv = rv_entry.getValue();
//Route r = new Route(rv.getOsmValue("gtfs:route_id"), rv.getOsmValue("ref"), OperatorInfo.getFullName());
Expand Down Expand Up @@ -708,7 +716,8 @@ public void compareRouteData() throws InterruptedException{
routesByShortName.put(ri.getRouteRef(),ri);}
catch (Exception e)
{
System.out.println(e);
System.out.println("exception\t" + e);
System.out.println(e.getStackTrace());
e.printStackTrace();
}
// routesByShortName.put(ri.getTag("route_short_name"),ri);
Expand Down Expand Up @@ -755,7 +764,7 @@ public void compareRouteData() throws InterruptedException{
if (network != null)
ostring = network;
else ostring = operator;

Route er;
er = new Route(tripId, routeShortName, ostring);

Expand Down Expand Up @@ -1002,16 +1011,19 @@ public void compareBusStopData() throws InterruptedException {
progressToUpdate = 50/totalOsmNode;
}
int currentTotalProgress=0;
for (int osmindex=0; osmindex<totalOsmNode; osmindex++){
// for (int osmindex=0; osmindex<totalOsmNode; osmindex++){
AtomicInteger osmprog= new AtomicInteger();
OSMTags.parallelStream().forEach(osmtag -> {
int osmindex = OSMTags.indexOf(osmtag);
if(this.flagIsDone)
return;
if((osmindex%timeToUpdate)==0) {
currentTotalProgress += progressToUpdate;
osmprog.getAndIncrement();
if((osmprog.get()%timeToUpdate)==0) {
// currentTotalProgress += progressToUpdate;
updateProgress(progressToUpdate);
this.setMessage("Comparing "+osmindex+"/"+totalOsmNode+" ...");
this.setMessage("Comparing "+osmprog.get()+"/"+totalOsmNode+" ...");
}
Hashtable<String,String> osmtag = new Hashtable<String,String>();
osmtag.putAll(OSMTags.get(osmindex));
// Hashtable<String, String> osmtag = new Hashtable<String, String>(OSMTags.get(osmindex));
String osmOperator = (String)osmtag.get(tag_defs.OSM_NETWORK_KEY);
String osmStopID = (String)osmtag.get("gtfs_id");
String osmPlatformType = (String)osmtag.get(tag_defs.OSM_STOP_TYPE_KEY);
Expand Down Expand Up @@ -1235,7 +1247,7 @@ public void compareBusStopData() throws InterruptedException {
}
}
}
}
});
// set OSM value to all stops in modify category
setStopWithOsmDataDefault();
//make sure is 50% overall
Expand Down Expand Up @@ -1335,6 +1347,9 @@ public void startCompare() throws InterruptedException{
ArrayList<AttributesImpl> tempOSMRelations = osmRequest.getExistingBusRelations(Double.toString(minLon), Double.toString(minLat),
Double.toString(maxLon), Double.toString(maxLat));
if(this.flagIsDone) return;
long tDelta = System.currentTimeMillis() - tStart;
System.out.println("OSM Downloads Completed in "+ tDelta /1000.0 + "seconds");
tStart = System.currentTimeMillis();
progressMonitor.setNote("");
if (tempOSMNodes!=null) {
OSMNodes.addAll(tempOSMNodes);
Expand Down Expand Up @@ -1460,7 +1475,13 @@ public void generateReport(){
arr.addAll(entry.getValue());
reportArrays.put(key, arr);
}
ReportViewer rv = new ReportViewer(GTFSstops, reportArrays, upload, modify, delete, reportRoute, routes, agencyRoutes, existingRoutes, taskOutput);

ReportViewer rv = new ReportViewer(GTFSstops, reportArrays,
new HashSet<Stop>(upload),
new HashSet<Stop>(modify),
new HashSet<Stop>(delete),
reportRoute, routes, agencyRoutes, existingRoutes, taskOutput);
// ReportViewer rv = new ReportViewer(GTFSstops, reportArrays, upload, modify, delete, routes, agencyRoutes, existingRoutes, taskOutput);
String info = "Active OSM bus stop mappers:\n"+osmActiveUsers.toString()+"\n\n";
info += "There are currently "+OSMNodes.size()+" OSM stops in the region\n\n";
info += "Transit agency GTFS dataset has "+GTFSstops.size()+" stops";
Expand Down