Unify JPL and UMD virtual computing environments (#44)

- Rely on data id instead of row index to match results to labels file, addressing an issue where os.listdir provides inconsistent ordering across machines - Use data id from labels.csv instead of row number for comparison plots, allowing experiments to be run on a subset of the test data - Ensure all data ids are strings Co-authored-by: Kiri Wagstaff <[email protected]>
nasaharvest · Oct 17, 2021 · 3c57c93 · 3c57c93
1 parent 420b59f
commit 3c57c93
Show file tree

Hide file tree

Showing 7 changed files with 4,129 additions and 4,125 deletions.
diff --git a/dora_exp_pipeline/combined_plot_script.py b/dora_exp_pipeline/combined_plot_script.py
@@ -18,7 +18,8 @@ def alg_indexes(filename):
     with open(filename, 'r') as f:
         text = f.read().split("\n")[:-1]
         scores = sorted(text, key=lambda x: float(x.split(", ")[0]))
-        scores = [int(i.split(", ")[1]) for i in scores]
+        # Field 2 is the sample id
+        scores = [i.split(", ")[2] for i in scores]
 
     return scores
 
@@ -47,7 +48,7 @@ def validation_labels(filename):
     labels = {}
     for i in text:
         line = i.split(",")
-        labels[int(line[0])] = int(line[1])
+        labels[line[0]] = int(line[1])
 
     return labels
 
@@ -171,7 +172,7 @@ def combine_plots(resultsdir, label_path, precision_at_n):
         plt.savefig(f'{resultsdir}/comparison_plot_combined_p-at-n.png')
         plt.savefig(f'{resultsdir}/comparison_plot_combined_p-at-n.pdf')
     else:
-        axes.set_ylim(1, num_outliers)
+        axes.set_ylim(0, num_outliers)
         plt.savefig(f'{resultsdir}/comparison_plot_combined.png')
         plt.savefig(f'{resultsdir}/comparison_plot_combined.pdf')
 

diff --git a/dora_exp_pipeline/dora_data_loader.py b/dora_exp_pipeline/dora_data_loader.py
@@ -177,7 +177,7 @@ def _load(self, dir_path: str) -> dict:
                 img = np.reshape(img, [img.shape[0]*img.shape[1],
                                        img.shape[2]])
                 # set the ID to the index of the pixel
-                data_dict['id'] = range(img.shape[0])
+                data_dict['id'] = [str(i) for i in range(img.shape[0])]
                 data_dict['data'] = list(img)
         else:
             raise RuntimeError(f'File extension not supported. '
@@ -256,7 +256,7 @@ def _load(self, dir_path: str) -> dict:
         if dir_path.endswith('.h5'):
             # Load the .h5
             df = pd.read_hdf(dir_path)
-            data_dict['id'] = df.index
+            data_dict['id'] = df.index.astype(str)
             data_dict['data'] = df.values
 
         else:
@@ -295,7 +295,7 @@ def _load(self, dir_path: str) -> dict:
                 for row in csv_reader:
                     # Assumes the first column is the ID
                     # and all other columns are time steps
-                    data_dict['id'].append(row[0])
+                    data_dict['id'].append(str(int(row[0])))
                     data_dict['data'].append(np.array(row[1:]))
         else:
             raise RuntimeError(f'File extension not supported. '

diff --git a/dora_exp_pipeline/dora_results_organization.py b/dora_exp_pipeline/dora_results_organization.py
@@ -100,14 +100,17 @@ def _run(self, data_ids, dts_scores, dts_sels, data_to_score,
         # Outliers will be 1s and inliers will be 0s.
         labels = self._get_validation_labels(validation_dir)
 
-        x = list(range(1, len(dts_sels)+1))
+        x = list(range(1, len(labels)+1))
         y = []
         numOutliers = 0
 
-        for i in range(len(dts_sels)):
-            if(labels[dts_sels[i]] == 1):
-                numOutliers += 1
-            y.append(numOutliers)
+        for i in range(len(data_ids)):
+            label = labels.get(data_ids[i])
+            # Only include labelled values in experiment
+            if label is not None:
+                if label == 1:
+                    numOutliers += 1
+                y.append(numOutliers)
 
         fig, axes = plt.subplots()
         area = sum(y)/sum([i for i in range(len(labels))])
@@ -129,7 +132,7 @@ def _get_validation_labels(self, validation_dir):
         labels = {}
         for i in text:
             line = i.split(",")
-            labels[int(line[0])] = int(line[1])
+            labels[line[0]] = int(line[1])
 
         return labels
 

diff --git a/exp/faw_time_series/kenya/results/pca-k=3/comparison_plot_pca.png b/exp/faw_time_series/kenya/results/pca-k=3/comparison_plot_pca.png
diff --git a/exp/faw_time_series/kenya/results/pca-k=3/selections-pca.csv b/exp/faw_time_series/kenya/results/pca-k=3/selections-pca.csv
@@ -1,113 +1,113 @@
-0, 24, 24.0, 0.7807377264534382
-1, 21, 21.0, 0.6869387718751655
-2, 42, 42.0, 0.6240366827206979
-3, 56, 56.0, 0.4851048072584936
-4, 52, 52.0, 0.4295119630493599
-5, 72, 72.0, 0.40840465015931404
-6, 39, 39.0, 0.40772269409160433
-7, 58, 58.0, 0.40705787176744573
-8, 92, 92.0, 0.401037307241449
-9, 29, 29.0, 0.37823431221371456
-10, 11, 11.0, 0.37697260519430525
-11, 107, 107.0, 0.3749299841055659
-12, 98, 98.0, 0.36582938816296096
-13, 61, 61.0, 0.3537452034842545
-14, 89, 89.0, 0.3531018709397943
-15, 25, 25.0, 0.3406946724147537
-16, 8, 8.0, 0.34018612659902503
-17, 66, 66.0, 0.33754805916274744
-18, 105, 105.0, 0.3374377877826993
-19, 31, 31.0, 0.32827904004494396
-20, 110, 110.0, 0.3269612247557343
-21, 88, 88.0, 0.32617336890586457
-22, 70, 70.0, 0.32486621599131277
-23, 68, 68.0, 0.3171146568444272
-24, 41, 41.0, 0.31426111950832314
-25, 112, 112.0, 0.3043503529116
-26, 106, 106.0, 0.3031127832930907
-27, 1, 1.0, 0.30248965912824244
-28, 40, 40.0, 0.3013883195493278
-29, 12, 12.0, 0.30104949707137457
-30, 17, 17.0, 0.2927850414978757
-31, 38, 38.0, 0.29161255955080145
-32, 85, 85.0, 0.28993623595832585
-33, 99, 99.0, 0.2864543211346755
-34, 78, 78.0, 0.28561479420339364
-35, 3, 3.0, 0.2809773212701314
-36, 32, 32.0, 0.2787150391716507
-37, 76, 76.0, 0.2773539032937627
-38, 93, 93.0, 0.27635953208113795
-39, 73, 73.0, 0.27416647166330166
-40, 94, 94.0, 0.27393824745192596
-41, 46, 46.0, 0.2731691344741308
-42, 43, 43.0, 0.26325122401883716
-43, 49, 49.0, 0.26278640371044826
-44, 74, 74.0, 0.25935195128391336
-45, 67, 67.0, 0.2582587709032858
-46, 101, 101.0, 0.25818368302589007
-47, 63, 63.0, 0.25755869759046546
-48, 18, 18.0, 0.25708317213474385
-49, 81, 81.0, 0.25667665658239286
-50, 111, 111.0, 0.25485818114333486
-51, 103, 103.0, 0.253978433087608
-52, 64, 64.0, 0.25362589913134265
-53, 55, 55.0, 0.2526425099316353
-54, 23, 23.0, 0.25150128510287384
-55, 2, 2.0, 0.24833844360172297
-56, 13, 13.0, 0.24413271480276932
-57, 33, 33.0, 0.24341028250728533
-58, 0, 0.0, 0.24149488913331815
-59, 69, 69.0, 0.23990626054892247
-60, 95, 95.0, 0.2393751240372812
-61, 97, 97.0, 0.23846934467589834
-62, 57, 57.0, 0.23812199734926826
-63, 50, 50.0, 0.2363624675419848
-64, 36, 36.0, 0.23446193846785937
-65, 60, 60.0, 0.2331012335002715
-66, 100, 100.0, 0.23033497893194763
-67, 54, 54.0, 0.2284279863227031
-68, 19, 19.0, 0.2279687909470126
-69, 83, 83.0, 0.22765302855989766
-70, 79, 79.0, 0.2265115644193557
-71, 75, 75.0, 0.22647416052688366
-72, 20, 20.0, 0.22534373144617
-73, 71, 71.0, 0.22526769564390267
-74, 109, 109.0, 0.22522043929297914
-75, 51, 51.0, 0.21558794761615868
-76, 9, 9.0, 0.21500010041899478
-77, 37, 37.0, 0.21476135217162265
-78, 62, 62.0, 0.21243301684135932
-79, 48, 48.0, 0.21123566463097468
-80, 14, 14.0, 0.21103527432141825
-81, 104, 104.0, 0.20886486904512622
-82, 28, 28.0, 0.20115443797788385
-83, 108, 108.0, 0.1997568240747799
-84, 34, 34.0, 0.19776121124783938
-85, 82, 82.0, 0.19659587979567017
-86, 22, 22.0, 0.19465284903566052
-87, 16, 16.0, 0.19420631388532142
-88, 86, 86.0, 0.19390643000839589
-89, 26, 26.0, 0.18929505621919376
-90, 44, 44.0, 0.1883975234552146
-91, 96, 96.0, 0.18749234094077802
-92, 45, 45.0, 0.18561420336616946
-93, 7, 7.0, 0.1795673360686368
-94, 87, 87.0, 0.1789386033614834
-95, 59, 59.0, 0.17668960767789296
-96, 80, 80.0, 0.16992644549331354
-97, 35, 35.0, 0.16870373894675017
-98, 53, 53.0, 0.16680796024302497
-99, 77, 77.0, 0.16434060813269705
-100, 90, 90.0, 0.16398686979188393
-101, 65, 65.0, 0.16331175090854474
-102, 91, 91.0, 0.16099246523320965
-103, 4, 4.0, 0.16045855004526394
-104, 47, 47.0, 0.15708509107487473
-105, 15, 15.0, 0.15572153417782414
-106, 30, 30.0, 0.15178029890034464
-107, 84, 84.0, 0.15050943043559298
-108, 5, 5.0, 0.1482379999749222
-109, 102, 102.0, 0.14379147665625694
-110, 10, 10.0, 0.142376554702949
-111, 6, 6.0, 0.13212456021194735
-112, 27, 27.0, 0.09759000711518517
+0, 24, 24, 6.236105299485569
+1, 21, 21, 5.395500129422813
+2, 42, 42, 4.926724701388575
+3, 56, 56, 3.868843977999179
+4, 52, 52, 3.4054716625728263
+5, 72, 72, 3.238354489070928
+6, 58, 58, 3.231554261691758
+7, 39, 39, 3.192718402468726
+8, 92, 92, 3.170690970995071
+9, 11, 11, 2.997112339895486
+10, 107, 107, 2.9527577734744974
+11, 29, 29, 2.9471360175053505
+12, 98, 98, 2.8785025448889985
+13, 89, 89, 2.798295420863441
+14, 61, 61, 2.795399660389081
+15, 25, 25, 2.688683861375638
+16, 8, 8, 2.67324133450823
+17, 105, 105, 2.666176822488592
+18, 66, 66, 2.655166387175775
+19, 31, 31, 2.641223980032841
+20, 110, 110, 2.6195691001827943
+21, 88, 88, 2.612097157563989
+22, 70, 70, 2.557319971444191
+23, 41, 41, 2.5074013460493223
+24, 68, 68, 2.4985672230195672
+25, 112, 112, 2.4109620371748766
+26, 40, 40, 2.4007699922318166
+27, 12, 12, 2.397021550697726
+28, 106, 106, 2.3940302790341943
+29, 1, 1, 2.357271857621068
+30, 78, 78, 2.289971614309116
+31, 38, 38, 2.2882177732085274
+32, 85, 85, 2.2694934290429525
+33, 17, 17, 2.268929216672444
+34, 99, 99, 2.238327972569797
+35, 76, 76, 2.215837202200498
+36, 32, 32, 2.2125948934158166
+37, 93, 93, 2.184628681356776
+38, 3, 3, 2.1653591227297304
+39, 73, 73, 2.1628052428479494
+40, 46, 46, 2.1439207748144256
+41, 94, 94, 2.1370021353761914
+42, 43, 43, 2.118943238967627
+43, 49, 49, 2.1039310013592756
+44, 101, 101, 2.0618011247234254
+45, 67, 67, 2.0452740626756887
+46, 74, 74, 2.040889670136844
+47, 81, 81, 2.035702808923738
+48, 18, 18, 2.0166012543428584
+49, 111, 111, 2.0086766793436506
+50, 64, 64, 2.00297283378845
+51, 55, 55, 2.000006061834745
+52, 63, 63, 1.9943769210850977
+53, 23, 23, 1.9886488203280377
+54, 2, 2, 1.9799141776967069
+55, 103, 103, 1.9741861870354296
+56, 97, 97, 1.9306791983476506
+57, 33, 33, 1.9205407038076108
+58, 95, 95, 1.8956703886997628
+59, 57, 57, 1.8863251981662135
+60, 69, 69, 1.8834248738885389
+61, 0, 0, 1.877998432042497
+62, 13, 13, 1.867573198499903
+63, 60, 60, 1.8619859346739527
+64, 50, 50, 1.8498246412205324
+65, 36, 36, 1.8444983636114252
+66, 79, 79, 1.8024253686583291
+67, 109, 109, 1.80111204221453
+68, 19, 19, 1.8009081165406877
+69, 83, 83, 1.7979230890311402
+70, 54, 54, 1.7897631195357762
+71, 20, 20, 1.7811185695496203
+72, 75, 75, 1.780162395721135
+73, 71, 71, 1.7760514341337503
+74, 100, 100, 1.7749011283646006
+75, 51, 51, 1.6916801815359623
+76, 37, 37, 1.6869628531701386
+77, 62, 62, 1.683939376742503
+78, 9, 9, 1.681769230076405
+79, 14, 14, 1.6716462100727247
+80, 48, 48, 1.6555347848553785
+81, 104, 104, 1.6221929185844255
+82, 108, 108, 1.5940985982862603
+83, 28, 28, 1.5817011302386825
+84, 34, 34, 1.5541430810765011
+85, 22, 22, 1.5443546993296238
+86, 82, 82, 1.5376231166710235
+87, 86, 86, 1.5329187200610697
+88, 16, 16, 1.5303637042087879
+89, 26, 26, 1.5045847573911983
+90, 44, 44, 1.4891795865015862
+91, 96, 96, 1.4851875490384314
+92, 45, 45, 1.4826534674510228
+93, 87, 87, 1.435178967630551
+94, 7, 7, 1.4346760810996455
+95, 59, 59, 1.4059312066809886
+96, 53, 53, 1.3276402474346227
+97, 80, 80, 1.3215721946822583
+98, 35, 35, 1.3131395788906384
+99, 90, 90, 1.2956385735294325
+100, 77, 77, 1.2947086745984742
+101, 65, 65, 1.2865755861868062
+102, 91, 91, 1.2734346549726119
+103, 4, 4, 1.26740538351287
+104, 47, 47, 1.2421515572193944
+105, 15, 15, 1.238944042762712
+106, 30, 30, 1.2016497117015825
+107, 84, 84, 1.1830612493374424
+108, 5, 5, 1.176802684475937
+109, 10, 10, 1.1318457613219968
+110, 102, 102, 1.1288413479843604
+111, 6, 6, 1.0411599435766659
+112, 27, 27, 0.7649367119266843