-
Notifications
You must be signed in to change notification settings - Fork 1
/
downsample_bitmaps.py
executable file
·129 lines (98 loc) · 6.35 KB
/
downsample_bitmaps.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#!/usr/bin/env python
# Downsampling is done on shifted anomaly and maps
import pathlib
import numpy as np
import glob
import parse
from tqdm import tqdm # for progress bars
converted_data_dir = pathlib.Path('input_data')
master_mask_2D_shifted = np.load(converted_data_dir / "master_mask_2D_shifted-v2.npy")
evaluation_mask_3D_shifted = np.load(converted_data_dir / "evaluation_mask_3D_shifted-v2.npy")
(duration, height, width) = evaluation_mask_3D_shifted.shape
# Integer factor for downsampling
factor = 3 # change this and rerun, file names are uniquely generated
assert(width % factor == 0)
new_width = width // factor
assert(height % factor == 0)
new_height = height // factor
#Downsampling master mask
master_mask_2D_shifted_ds = np.zeros((new_height, new_width), dtype=np.float32)
for row in range(new_height):
for col in range(new_width):
master_mask_2D_shifted_ds.itemset((row, col),
master_mask_2D_shifted[row*factor:(row+1)*factor,col*factor:(col+1)*factor].astype(np.float32).sum() / (factor**2))
fname = "master_mask_2D_shifted-v2_ds_" + str(factor) + "x" + str(factor)
np.save(converted_data_dir / fname, master_mask_2D_shifted_ds)
nan_mask_3D_shifted = np.load(converted_data_dir / "nan_mask_3D_shifted-v2.npy")
imputed_anom_3D_shifted = np.load(converted_data_dir / "imputed_anom_3D_shifted-v2.npy")
#Downsampling evaluation mask
evaluation_mask_3D_shifted_ds = np.zeros((duration, new_height, new_width), dtype=np.float32)
imputed_anom_3D_shifted_ds_on_em = np.zeros((duration, new_height, new_width), dtype=np.float32)
print("Downsampling evaluation data...")
for t, eval_mask_shifted in tqdm(enumerate(evaluation_mask_3D_shifted), total=duration):
assert((eval_mask_shifted & ~master_mask_2D_shifted).all()==False)
for row in range(new_height):
for col in range(new_width):
num_of_cells = eval_mask_shifted[row*factor:(row+1)*factor, col*factor:(col+1)*factor].sum()
evaluation_mask_3D_shifted_ds.itemset((t, row, col), num_of_cells / (factor ** 2))
if(num_of_cells > 0):
imputed_anom_3D_shifted_ds_on_em.itemset((t, row, col),
imputed_anom_3D_shifted[t, row*factor:(row+1)*factor, col*factor:(col+1)*factor].sum() /
num_of_cells)
fname = converted_data_dir / ("evaluation_mask_3D_shifted-v2_ds_" + str(factor) + "x" + str(factor))
np.save(fname, evaluation_mask_3D_shifted_ds)
fname = converted_data_dir / ("imputed_anom_3D_shifted-v2_ds_" + str(factor) + "x" + str(factor))
np.save(fname, imputed_anom_3D_shifted_ds_on_em)
#Downsampling nan mask
nan_mask_3D_shifted_ds = np.zeros((duration, new_height, new_width), dtype=np.float32)
print("Downsampling nan mask...")
for t, nan_mask_shifted in tqdm(enumerate(nan_mask_3D_shifted), total=duration):
assert((nan_mask_shifted & ~master_mask_2D_shifted).all()==False)
for row in range(new_height):
for col in range(new_width):
num_of_cells = nan_mask_shifted[row*factor:(row+1)*factor, col*factor:(col+1)*factor].sum()
nan_mask_3D_shifted_ds.itemset((t, row, col), num_of_cells / (factor ** 2))
fname = "nan_mask_3D_shifted-v2_ds_" + str(factor) + "x" + str(factor)
np.save(converted_data_dir / fname, nan_mask_3D_shifted_ds)
#Downsampling multiple training and validation masks
print("Downsampling multiple training and validation masks...")
base_training_mask_file_name = "training_mask_3D_shifted-v2"
for training_mask_file_name in glob.glob(str(converted_data_dir / (base_training_mask_file_name + "*"))):
training_mask_pathlib_name = pathlib.Path(training_mask_file_name).stem
# Skips training masks that are allready downsampled
if parse.parse(base_training_mask_file_name + "-{N1:g}-{N2:g}", training_mask_pathlib_name) == None :
continue
print(" Working on ", training_mask_pathlib_name, "...")
training_mask_3D_shifted = np.load(training_mask_file_name)
# Downsampling training mask
training_mask_3D_shifted_ds = np.zeros((duration, new_height, new_width), dtype=np.float32)
imputed_anom_3D_shifted_ds_on_tm = np.zeros((duration, new_height, new_width), dtype=np.float32)
for t, train_mask_shifted in tqdm(enumerate(training_mask_3D_shifted), total=duration):
assert((train_mask_shifted & ~master_mask_2D_shifted).all()==False)
for row in range(new_height):
for col in range(new_width):
num_of_cells = train_mask_shifted[row*factor:(row+1)*factor, col*factor:(col+1)*factor].sum()
training_mask_3D_shifted_ds.itemset((t, row, col), num_of_cells / (factor ** 2))
if(num_of_cells > 0):
imputed_anom_3D_shifted_ds_on_tm.itemset((t, row, col),
imputed_anom_3D_shifted[t, row*factor:(row+1)*factor, col*factor:(col+1)*factor].sum() /
num_of_cells)
assert not np.isnan(imputed_anom_3D_shifted_ds_on_tm).any()
fname = converted_data_dir / (str(training_mask_pathlib_name) + "_ds_" + str(factor) + "x" + str(factor))
np.save(fname, training_mask_3D_shifted_ds)
# Downsampling validation mask
validation_mask_3D_shifted_ds = np.zeros((duration, new_height, new_width), dtype=np.float32)
imputed_anom_3D_shifted_ds_on_vm = np.zeros((duration, new_height, new_width), dtype=np.float32)
for t, valid_mask_shifted in tqdm(enumerate(evaluation_mask_3D_shifted & ~training_mask_3D_shifted), total=duration):
assert((valid_mask_shifted & ~master_mask_2D_shifted).all()==False)
for row in range(new_height):
for col in range(new_width):
num_of_cells = valid_mask_shifted[row*factor:(row+1)*factor, col*factor:(col+1)*factor].sum()
validation_mask_3D_shifted_ds.itemset((t, row, col), num_of_cells / (factor ** 2))
if(num_of_cells > 0):
imputed_anom_3D_shifted_ds_on_vm.itemset((t, row, col),
imputed_anom_3D_shifted[t, row*factor:(row+1)*factor, col*factor:(col+1)*factor].sum() /
num_of_cells)
assert not np.isnan(imputed_anom_3D_shifted_ds_on_vm).any()
fname = converted_data_dir / ("validation_mask_3D_shifted-v2_ds_" + str(factor) + "x" + str(factor) + "_from_" + str(training_mask_pathlib_name))
np.save(fname, validation_mask_3D_shifted_ds)