fix: dump processed

scripts/dump_processed.py

@@ -28,16 +28,16 @@ def dump_dataset(output, fname, alljson):
     jsonlist = glob.glob(alljson) if "*" in alljson else alljson.split(",")
     print("Original jsons:", jsonlist)
     oldf, newf = {}, {}
-    for j in jsonlist:
-        old = json.load(open(j))
-        for o in old.keys():
-            oldf[o] = old[o]
+    oldf = json.load(open(j))
     for m in output.keys():
         for f in output[m].keys():
-            newf[f] = list(output[m][f]["fname"])
+            newf[f] = list(set(output[m][f]["fname"]))
     failed = {}
     for t in oldf.keys():
-        failed[t] = list(set(oldf[t]) - set(newf[t]))
+        for f in oldf[t]:
+            if f in newf[t]:
+                oldf[t].remove(f)
+        failed[t] = oldf[t]
 
     with open(f"{fname}_failed_dataset.json", "w") as outfile:
         json.dump(failed, outfile, indent=4)