segments link correctly. add batch EDWP

Edit Distance with Projection/edwp/Program.cs

@@ -92,11 +92,18 @@ static void testRandomTraj(int n1, int n2)
             Console.WriteLine(stopwatch.Elapsed);
         }
 
-        static void outputMatrix(string filein, string fileout)
+        // read filenames and indexes from manifest
+        static void outputMatrix(string manifest, string dirin, string fileout)
         {
             var stopwatch = new System.Diagnostics.Stopwatch();
             stopwatch.Start();
-            var trajs = LoadFromFile(filein);
+            var manifestfile = File.OpenText(manifest);
+            var trajinfo = JsonConvert.DeserializeObject<List<string>>(manifestfile.ReadToEnd());
+            var trajs = new List<Trajectory>();
+            foreach (var filename in trajinfo)
+            {
+                trajs.AddRange(LoadFromFile(Path.Combine(dirin, filename)));
+            }
             var AABBs = new List<AABB>();
             int N = trajs.Count;
             var output = new float[N,N];
@@ -134,7 +141,7 @@ static void outputMatrix(string filein, string fileout)
         static void Main(string[] args)
         {
             // outputMatrix("A976C7.json", "A976C7.out.json");
-            outputMatrix(args[1], args[2]);
+            outputMatrix(args[0], args[1], args[2]);
             // testRandomTraj(40, 40);
             Console.ReadKey();
         }

dbscan.py

@@ -3,9 +3,26 @@
 import numpy as np
 import sys
 
-with open("A0C42C.out.json") as f:
+root = "data_simple-segments/"
+
+with open("edwpmanifest.json") as f:
+    filenames = json.load(f)
+
+with open("distmatrix.json") as f:
     matrix = json.load(f)
 
+# load all segments
+segments = [] # list of list of segments
+fileindex = {} # mapping from index to mother filename
+c = 0
+for filename in filenames:
+    with open(root + filename) as f:
+        data = json.load(f)
+    for traj in data:
+        segments.append(traj)
+        fileindex[c] = filename
+        c += 1
+
 for row in matrix:
     for i, val in enumerate(row):
         if val == "NaN":
@@ -14,18 +31,18 @@
 db = DBSCAN(eps=0.005, min_samples=2, metric="precomputed")
 db.fit(matrix)
 labels = db.labels_
-print(labels)
 # repackage data
-with open("data_segmented-paths/A0C42C.json") as segfile:
-    segments = json.load(segfile)
-
 # for each label, find the representative.
 maxlabel = max(labels)
 labeldict = {}
 for i, seg in enumerate(segments):
     L = labeldict.setdefault(labels[i], [])
     L.append(i)
 
+# print histogram, basically
+for label in labeldict:
+    print("{:<2}\t{:>3}".format(int(label), len(labeldict[label])))
+
 def metric_nodelength(seg):
     return len(seg)
 
@@ -54,10 +71,11 @@ def metric_pathlength(seg):
     label = labels[i]
     obj = {
         "path": segment,
+        "icao": fileindex[i],
         "label": int(label),
         "rep": True if repdict[label] == i else False 
     }
     out.append(obj)
 
-with open("A0C42C.annotated.json", "w") as outfile:
+with open("dbscanned.json", "w") as outfile:
     json.dump(out, outfile)

makemanifest.py

@@ -0,0 +1,22 @@
+import json
+import os
+
+"""
+Creates a manifest file for C#EDwP to consume, giving each segment an index
+Format:
+[
+    ["filename.json", index]
+    ...
+]
+
+The index is for the first segment of that file, with each segment incrementing by 1.
+"""
+
+root = "data_simple-segments/"
+outfile = "edwpmanifest.json"
+out = []
+for file in os.listdir(root):
+    out.append(file)
+
+with open(outfile, "w") as f:
+    json.dump(out, f)

segmentpaths.py

@@ -12,7 +12,7 @@ def getTime(ping):
     return ping[2]
 
 # only segments of len 10 and up are accepted.
-def tryAppend(output, segment, limit=50):
+def tryAppend(output, segment, limit=15):
     global pathlensum, pathsum
     if len(segment) > limit:
         output.append(segment)
@@ -43,7 +43,13 @@ def tryAppend(output, segment, limit=50):
             current_segment.append(obj[:3])
         t1 = getTime(obj)
         t2 = getTime(path[i+1])
-        if t2 - t1 > 2000 * 60: # over N minutes
+        # also break up if time length of path is over one hour
+        if len(current_segment) > 1:
+            if t2 - current_segment[0][2] > 30 * 60 * 1000:
+                segmentsAccepted += tryAppend(output, current_segment)
+                pathsegments.append(current_segment)
+                current_segment = []
+        elif t2 - t1 > 60 * 1000: # over N seconds
             segmentsAccepted += tryAppend(output, current_segment)
             pathsegments.append(current_segment)
             current_segment = []

viewpath.js

@@ -3,7 +3,7 @@ document.addEventListener("DOMContentLoaded", init)
 function init() {
 	var projection = d3.geoWinkel3(); //geoBaker
 	var path = d3.geoPath(projection);
-	var color10 = d3.schemeCategory10;
+	var color10 = d3.schemeCategory20;
 	d3.json("countries.geo.json", (e, d)=>{
 		d3.select(".countries")
 		  .append("path")
@@ -74,74 +74,74 @@ function init() {
 		   .attr("r", 1);
 	});*/
 
-	// draw segments!
-	d3.json("simplesegments.json", (e, d)=>{
-		// Structure of json file:
-		/*
-			[
-				{
-					[
-						[subpath1]
-						[subpath2]
-					]
-				},
-			]
-		*/
-		console.log(d)
-
-		d3.select("#paths")
-		  .selectAll(".pathgroup")
-		  .data(d.map(el=>el)) // for each icao
-		  .enter()
-		  .append("g")
-		  .classed("pathgroup", true)
-		  .attr("id", d=>d.icao)
-		  .style("stroke", (d,i)=>color10[i % 10])
-		  .style("stroke-width", 0.5)
-		  .selectAll("g")
-		  .data(d=>d.path) // for each subtrajectory
-		  .enter()
-		  .append("path")
-		  .attr("opacity", 0.7)
-		  .attr("d", d=>{
-		  	// repackage data to be GeoJSON
-		  	let geoobj = {type: "LineString", coordinates:[]}
-		  	geoobj.coordinates = d.map(el=>[el[0], el[1]])
-		  	return path(geoobj);
-		  });
-	})
-
-	// draw annotated segments
-	// d3.json("A0C42C.annotated.json", (e, d)=>{
+	// // draw segments!
+	// d3.json("segmentedpaths.json", (e, d)=>{
 	// 	// Structure of json file:
 	// 	/*
 	// 		[
 	// 			{
-	// 				"icao": "ICAO",
-	// 				"path": [
+	// 				[
 	// 					[subpath1]
 	// 					[subpath2]
 	// 				]
 	// 			},
 	// 		]
 	// 	*/
-	// 	console.log(d);
+	// 	console.log(d)
+
 	// 	d3.select("#paths")
-	// 	  .selectAll("path")
-	// 	  .data(d)
+	// 	  .selectAll(".pathgroup")
+	// 	  .data(d.map(el=>el)) // for each icao
+	// 	  .enter()
+	// 	  .append("g")
+	// 	  .classed("pathgroup", true)
+	// 	  .attr("id", d=>d.icao)
+	// 	  .style("stroke", (d,i)=>color10[i % 10])
+	// 	  .style("stroke-width", 0.5)
+	// 	  .selectAll("g")
+	// 	  .data(d=>d.path) // for each subtrajectory
 	// 	  .enter()
 	// 	  .append("path")
-	// 	  .attr("stroke", (d,i)=>d.label == -1 ? "#000" : color10[d.label])
-	// 	  .attr('stroke-width', d=>d.rep ? 0.3 : 0.1)
 	// 	  .attr("opacity", 0.7)
 	// 	  .attr("d", d=>{
 	// 	  	// repackage data to be GeoJSON
 	// 	  	let geoobj = {type: "LineString", coordinates:[]}
-	// 	  	geoobj.coordinates = d.path.map(el=>[el[0], el[1]])
+	// 	  	geoobj.coordinates = d.map(el=>[el[0], el[1]])
 	// 	  	return path(geoobj);
 	// 	  });
 	// })
 
+	//draw annotated segments
+	d3.json("dbscanned.json", (e, d)=>{
+		// Structure of json file:
+		/*
+			[
+				{
+					"icao": "ICAO",
+					"path": [
+						[subpath1]
+						[subpath2]
+					]
+				},
+			]
+		*/
+		console.log(d);
+		d3.select("#paths")
+		  .selectAll("path")
+		  .data(d)
+		  .enter()
+		  .append("path")
+		  .attr("stroke", (d,i)=>d.label == -1 ? "#000" : color10[d.label])
+		  .attr('stroke-width', d=>d.rep ? 0.3 : 0.3)
+		  .attr("opacity", 0.7)
+		  .attr("d", d=>{
+		  	// repackage data to be GeoJSON
+		  	let geoobj = {type: "LineString", coordinates:[]}
+		  	geoobj.coordinates = d.path.map(el=>[el[0], el[1]])
+		  	return path(geoobj);
+		  });
+	})
+
 	// draw annotated segments, in a grid
 	// d3.json("A976C7.annotated.json", (e, d)=>{
 	// 	// Structure of json file:

vw_simplify.py

@@ -111,6 +111,7 @@ def simplify(data, ratio=0.5, threshold=None, number=None):
 if __name__=="__main__":
     import os
     import json
+    import math
 
     root = "data_segmented-paths/"
     outdir = "data_simple-segments/"
@@ -119,9 +120,13 @@ def simplify(data, ratio=0.5, threshold=None, number=None):
             data = json.load(f)
         output = []
         for arr in data:
-            if len(arr) > 10:
-                arr = simplify(arr, ratio=0.2)
-                arr = [x[:3] for x in arr] # strip out area factor
+            L = len(arr)
+            if L > 50:
+                L = math.floor(L * .1 + 1)
+            else:
+                L = len(arr)
+            arr = simplify(arr, number=L)
+            arr = [x[:3] for x in arr] # strip out area factor
             output.append(arr)
         with open(outdir + file, "w") as f:
             json.dump(output, f)